diff --git a/.gitignore b/.gitignore index 5baf00eea..54c488e68 100644 --- a/.gitignore +++ b/.gitignore @@ -200,3 +200,5 @@ docs/node_modules/ # Re-include docs content section named "build" (not a build output) !docs/docs/build/ !docs/docs/build/** +!docs/i18n/**/build/ +!docs/i18n/**/build/** diff --git a/DashAI/back/dataloaders/classes/csv_dataloader.py b/DashAI/back/dataloaders/classes/csv_dataloader.py index 2c67bb145..6d48df790 100644 --- a/DashAI/back/dataloaders/classes/csv_dataloader.py +++ b/DashAI/back/dataloaders/classes/csv_dataloader.py @@ -30,21 +30,6 @@ class CSVDataloaderSchema(BaseSchema): ``pandas.read_csv``. """ - name: schema_field( - string_field(), - "", - description=MultilingualString( - en=( - "Custom name to register your dataset. If no name is specified, " - "the name of the uploaded file will be used." - ), - es=( - "Nombre personalizado para registrar su dataset. Si no se especifica " - "un nombre, se usará el nombre del archivo subido." - ), - ), - alias=MultilingualString(en="Name", es="Nombre"), - ) # type: ignore separator: schema_field( enum_field([",", ";", "blank space", "tab"]), ",", diff --git a/DashAI/back/dataloaders/classes/dataloader.py b/DashAI/back/dataloaders/classes/dataloader.py index 1e7abb5ee..467c04e99 100644 --- a/DashAI/back/dataloaders/classes/dataloader.py +++ b/DashAI/back/dataloaders/classes/dataloader.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Dict, Final from DashAI.back.config_object import ConfigObject +from DashAI.back.core.utils import MultilingualString logger = logging.getLogger(__name__) @@ -18,6 +19,16 @@ class BaseDataLoader(ConfigObject): """Abstract class with base methods for DashAI dataloaders.""" TYPE: Final[str] = "DataLoader" + CATEGORY: Final = MultilingualString( + en="File Uploading", + es="Carga de Archivos", + ) + + @classmethod + def get_metadata(cls) -> Dict[str, Any]: + return { + "category": cls.CATEGORY if cls.CATEGORY else "File Uploading", + } @abstractmethod def load_data( diff --git a/DashAI/back/dataloaders/classes/excel_dataloader.py b/DashAI/back/dataloaders/classes/excel_dataloader.py index 6fa352937..183ea1a29 100644 --- a/DashAI/back/dataloaders/classes/excel_dataloader.py +++ b/DashAI/back/dataloaders/classes/excel_dataloader.py @@ -27,21 +27,6 @@ class ExcelDataloaderSchema(BaseSchema): leaving the sheet field empty selects the first sheet. """ - name: schema_field( - string_field(), - "", - description=MultilingualString( - en=( - "Custom name to register your dataset. If no name is specified, " - "the name of the uploaded file will be used." - ), - es=( - "Nombre personalizado para registrar su dataset. Si no se especifica " - "un nombre, se usará el nombre del archivo subido." - ), - ), - alias=MultilingualString(en="Name", es="Nombre"), - ) # type: ignore sheet: schema_field( union_type(int_field(ge=0), string_field()), placeholder=0, diff --git a/DashAI/back/dataloaders/classes/json_dataloader.py b/DashAI/back/dataloaders/classes/json_dataloader.py index ac1bb22ff..897b43eae 100644 --- a/DashAI/back/dataloaders/classes/json_dataloader.py +++ b/DashAI/back/dataloaders/classes/json_dataloader.py @@ -19,21 +19,6 @@ class JSONDataloaderSchema(BaseSchema): ``None``, the entire JSON value is interpreted as the record list. """ - name: schema_field( - string_field(), - "", - description=MultilingualString( - en=( - "Custom name to register your dataset. If no name is specified, " - "the name of the uploaded file will be used." - ), - es=( - "Nombre personalizado para registrar su dataset. Si no se especifica " - "un nombre, se usará el nombre del archivo subido." - ), - ), - alias=MultilingualString(en="Name", es="Nombre"), - ) # type: ignore data_key: schema_field( none_type(string_field()), placeholder="data", diff --git a/DashAI/back/initial_components.py b/DashAI/back/initial_components.py index 0a01fb093..3ab31b82f 100644 --- a/DashAI/back/initial_components.py +++ b/DashAI/back/initial_components.py @@ -123,21 +123,44 @@ from DashAI.back.metrics.translation.ter import Ter # Models +from DashAI.back.models.hugging_face.albert_transformer import AlbertTransformer +from DashAI.back.models.hugging_face.bert_transformer import BertTransformer +from DashAI.back.models.hugging_face.bertin_transformer import BertinTransformer +from DashAI.back.models.hugging_face.beto_transformer import BetoTransformer from DashAI.back.models.hugging_face.deberta_v3_transformer import DebertaV3Transformer from DashAI.back.models.hugging_face.distilbert_transformer import DistilBertTransformer +from DashAI.back.models.hugging_face.electra_transformer import ElectraTransformer from DashAI.back.models.hugging_face.llama_model import LlamaModel +from DashAI.back.models.hugging_face.m2m100_transformer import M2M100Transformer +from DashAI.back.models.hugging_face.minilm_transformer import MiniLMTransformer from DashAI.back.models.hugging_face.mistral_model import MistralModel from DashAI.back.models.hugging_face.mixtral_model import MixtralModel from DashAI.back.models.hugging_face.modernbert_transformer import ModernBertTransformer +from DashAI.back.models.hugging_face.multilingual_bert_transformer import ( + MultilingualBertTransformer, +) from DashAI.back.models.hugging_face.nllb_transformer import NllbTransformer +from DashAI.back.models.hugging_face.opus_mt_en_de_transformer import ( + OpusMtEnDeTransformer, +) from DashAI.back.models.hugging_face.opus_mt_en_es_transformer import ( OpusMtEnESTransformer, ) +from DashAI.back.models.hugging_face.opus_mt_en_fr_transformer import ( + OpusMtEnFrTransformer, +) +from DashAI.back.models.hugging_face.opus_mt_en_pt_transformer import ( + OpusMtEnPtTransformer, +) from DashAI.back.models.hugging_face.opus_mt_es_en_transformer import ( OpusMtEsENTransformer, ) +from DashAI.back.models.hugging_face.opus_mt_fr_en_transformer import ( + OpusMtFrEnTransformer, +) from DashAI.back.models.hugging_face.pixart_sigma_model import PixArtSigmaModel from DashAI.back.models.hugging_face.qwen_model import QwenModel +from DashAI.back.models.hugging_face.roberta_transformer import RobertaTransformer from DashAI.back.models.hugging_face.sd15_depth_controlnet_model import ( SD15DepthControlNetModel, ) @@ -164,24 +187,52 @@ from DashAI.back.models.hugging_face.stable_diffusion_xl_model import ( StableDiffusionXLModel, ) +from DashAI.back.models.hugging_face.t5_small_transformer import T5SmallTransformer from DashAI.back.models.hugging_face.tongyi_z_image_model import TongyiZImageModel +from DashAI.back.models.hugging_face.xlm_roberta_transformer import ( + XlmRobertaTransformer, +) +from DashAI.back.models.hugging_face.xlnet_transformer import XlnetTransformer +from DashAI.back.models.scikit_learn.adaboost_classifier import AdaBoostClassifier +from DashAI.back.models.scikit_learn.adaboost_regression import AdaBoostRegression +from DashAI.back.models.scikit_learn.bagging_classifier import BaggingClassifier +from DashAI.back.models.scikit_learn.bayesian_ridge_regression import ( + BayesianRidgeRegression, +) from DashAI.back.models.scikit_learn.bow_text_classification_model import ( BagOfWordsTextClassificationModel, ) from DashAI.back.models.scikit_learn.decision_tree_classifier import ( DecisionTreeClassifier, ) +from DashAI.back.models.scikit_learn.decision_tree_regression import ( + DecisionTreeRegression, +) from DashAI.back.models.scikit_learn.dummy_classifier import DummyClassifier +from DashAI.back.models.scikit_learn.elastic_net_regression import ElasticNetRegression +from DashAI.back.models.scikit_learn.extra_trees_classifier import ExtraTreesClassifier +from DashAI.back.models.scikit_learn.extra_trees_regression import ExtraTreesRegression +from DashAI.back.models.scikit_learn.gaussian_nb import GaussianNB +from DashAI.back.models.scikit_learn.gradient_boosting_classifier import ( + GradientBoostingClassifier, +) from DashAI.back.models.scikit_learn.gradient_boosting_regression import ( GradientBoostingR, ) from DashAI.back.models.scikit_learn.hist_gradient_boosting_classifier import ( HistGradientBoostingClassifier, ) +from DashAI.back.models.scikit_learn.hist_gradient_boosting_regression import ( + HistGradientBoostingRegression, +) from DashAI.back.models.scikit_learn.k_neighbors_classifier import KNeighborsClassifier +from DashAI.back.models.scikit_learn.k_neighbors_regression import KNeighborsRegression +from DashAI.back.models.scikit_learn.lasso_regression import LassoRegression from DashAI.back.models.scikit_learn.linear_regression import LinearRegression +from DashAI.back.models.scikit_learn.linear_svc_classifier import LinearSVCClassifier from DashAI.back.models.scikit_learn.linearSVR import LinearSVR from DashAI.back.models.scikit_learn.logistic_regression import LogisticRegression +from DashAI.back.models.scikit_learn.mlp_classifier import MLPClassifier from DashAI.back.models.scikit_learn.mlp_regression import MLPRegression from DashAI.back.models.scikit_learn.random_forest_classifier import ( RandomForestClassifier, @@ -190,7 +241,12 @@ RandomForestRegression, ) from DashAI.back.models.scikit_learn.ridge_regression import RidgeRegression +from DashAI.back.models.scikit_learn.sgd_classifier import SGDClassifier from DashAI.back.models.scikit_learn.svc import SVC +from DashAI.back.models.scikit_learn.svr import SVR +from DashAI.back.models.scikit_learn.tfidf_logreg_text_classification_model import ( + TfIdfLogRegTextClassificationModel, +) # Optimizers from DashAI.back.optimizers.hyperopt_optimizer import HyperOptOptimizer @@ -240,42 +296,76 @@ def get_initial_components(): TextToTextGenerationTask, ControlNetTask, # Models - SVC, + AdaBoostClassifier, + AlbertTransformer, + AdaBoostRegression, + BaggingClassifier, + BagOfWordsTextClassificationModel, + BertTransformer, + BertinTransformer, + BetoTransformer, + BayesianRidgeRegression, + DebertaV3Transformer, DecisionTreeClassifier, + DecisionTreeRegression, + DistilBertTransformer, DummyClassifier, + ElasticNetRegression, + ElectraTransformer, + ExtraTreesClassifier, + ExtraTreesRegression, + GaussianNB, + GradientBoostingClassifier, GradientBoostingR, HistGradientBoostingClassifier, + HistGradientBoostingRegression, KNeighborsClassifier, - QwenModel, + KNeighborsRegression, + LassoRegression, + LinearRegression, + LinearSVCClassifier, + LinearSVR, LlamaModel, + LogisticRegression, + M2M100Transformer, + MiniLMTransformer, MistralModel, MixtralModel, - SmolLMModel, - StableDiffusionV2Model, - StableDiffusionV3Model, - StableDiffusionXLModel, - SDXLTurboModel, - PixArtSigmaModel, - TongyiZImageModel, - StableDiffusionXLV1ControlNet, - SD15DepthControlNetModel, - SD15OpenPoseControlNetModel, - SD15HEDControlNetModel, - SDXLCannyControlNetModel, - LogisticRegression, + MultilingualBertTransformer, + MLPClassifier, MLPRegression, - RandomForestClassifier, - RandomForestRegression, - DistilBertTransformer, ModernBertTransformer, - DebertaV3Transformer, + NllbTransformer, + OpusMtEnDeTransformer, OpusMtEnESTransformer, + OpusMtEnFrTransformer, + OpusMtEnPtTransformer, OpusMtEsENTransformer, - NllbTransformer, - BagOfWordsTextClassificationModel, + OpusMtFrEnTransformer, + PixArtSigmaModel, + QwenModel, + RandomForestClassifier, + RobertaTransformer, + RandomForestRegression, RidgeRegression, - LinearSVR, - LinearRegression, + SD15DepthControlNetModel, + SD15HEDControlNetModel, + SD15OpenPoseControlNetModel, + SDXLCannyControlNetModel, + SDXLTurboModel, + SGDClassifier, + SmolLMModel, + StableDiffusionV2Model, + StableDiffusionV3Model, + StableDiffusionXLModel, + StableDiffusionXLV1ControlNet, + SVC, + SVR, + T5SmallTransformer, + TfIdfLogRegTextClassificationModel, + TongyiZImageModel, + XlmRobertaTransformer, + XlnetTransformer, # Dataloaders CSVDataLoader, JSONDataLoader, diff --git a/DashAI/back/metrics/classification/log_loss.py b/DashAI/back/metrics/classification/log_loss.py index 37e9f4ab5..094cce5dd 100644 --- a/DashAI/back/metrics/classification/log_loss.py +++ b/DashAI/back/metrics/classification/log_loss.py @@ -86,4 +86,8 @@ def score( from sklearn.metrics import log_loss true_labels, _ = prepare_to_metric(true_labels, probs_pred_labels) - return log_loss(true_labels, probs_pred_labels) + # Pass all expected class indices so log_loss works even when a split + # happens to contain only one class (e.g. small validation sets). + n_classes = probs_pred_labels.shape[1] + labels = list(range(n_classes)) + return log_loss(true_labels, probs_pred_labels, labels=labels) diff --git a/DashAI/back/metrics/classification/roc_auc.py b/DashAI/back/metrics/classification/roc_auc.py index c5c79e77d..5d86d2ef7 100644 --- a/DashAI/back/metrics/classification/roc_auc.py +++ b/DashAI/back/metrics/classification/roc_auc.py @@ -80,14 +80,26 @@ def score( float RoC AUC score between true labels and predicted labels """ + import numpy as np + true_labels, _ = prepare_to_metric(true_labels, probs_pred_labels) - # Use the provided multiclass parameter or determine it using is_multiclass - if multiclass is None: - multiclass = ClassificationMetric.is_multiclass(true_labels) + + # ROC AUC is undefined when only one class is present in y_true. + if len(np.unique(true_labels)) < 2: + return float("nan") from sklearn.metrics import roc_auc_score + n_classes = probs_pred_labels.shape[1] + + if multiclass is None: + multiclass = n_classes > 2 if multiclass: - return roc_auc_score(true_labels, probs_pred_labels, multi_class="ovr") + return roc_auc_score( + true_labels, + probs_pred_labels, + multi_class="ovr", + labels=list(range(n_classes)), + ) else: return roc_auc_score(true_labels, probs_pred_labels[:, 1]) diff --git a/DashAI/back/models/hugging_face/albert_transformer.py b/DashAI/back/models/hugging_face/albert_transformer.py new file mode 100644 index 000000000..15e6d8199 --- /dev/null +++ b/DashAI/back/models/hugging_face/albert_transformer.py @@ -0,0 +1,45 @@ +"""DashAI implementation of ALBERT model for English text classification.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_text_classification_transformer import ( + HuggingFaceTextClassificationTransformer, +) +from DashAI.back.models.hugging_face.distilbert_transformer import ( + DistilBertTransformerSchema, +) + + +class AlbertTransformer(HuggingFaceTextClassificationTransformer): + """Pre-trained ALBERT model for efficient English text classification. + + ALBERT (A Lite BERT) reduces BERT's parameters via cross-layer parameter + sharing and factorised embedding parametrisation, making it significantly + smaller and faster while retaining high accuracy. Requires the + ``sentencepiece`` package for its tokeniser. + + References + ---------- + - [1] Lan, Z. et al. (2020). "ALBERT: A Lite BERT for Self-supervised + Learning of Language Representations." ICLR 2020. + - [2] https://huggingface.co/albert-base-v2 + """ + + DISPLAY_NAME: str = MultilingualString( + en="ALBERT Transformer", + es="Transformer ALBERT", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Parameter-efficient BERT variant for English text classification. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "Variante de BERT eficiente en parámetros para clasificación en inglés. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#00838F" + ICON: str = "Speed" + SCHEMA = DistilBertTransformerSchema + MODEL_NAME: str = "albert-base-v2" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_albert" diff --git a/DashAI/back/models/hugging_face/base_opus_mt_transformer.py b/DashAI/back/models/hugging_face/base_opus_mt_transformer.py new file mode 100644 index 000000000..a3d4f223d --- /dev/null +++ b/DashAI/back/models/hugging_face/base_opus_mt_transformer.py @@ -0,0 +1,261 @@ +"""Shared base class for Helsinki-NLP Opus-MT translation transformers.""" + +import shutil +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, List, Optional, Union + +from sklearn.exceptions import NotFittedError + +from DashAI.back.models.translation_model import TranslationModel +from DashAI.back.models.utils import GPU_OR_CPU_PLACEHOLDER + +if TYPE_CHECKING: + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + +class OpusMtTransformerMixin(TranslationModel): + """Shared implementation for Helsinki-NLP Opus-MT translation wrappers. + + Subclasses must define ``MODEL_NAME`` (the HuggingFace checkpoint ID) and + ``SCHEMA``. ``TEMP_CHECKPOINT_DIR`` defaults to a generic path but should + be overridden with a model-specific directory to avoid collisions between + concurrent training runs of different language pairs. + + All seq2seq training, tokenization, inference, save, and load logic lives + here so each language-pair subclass only needs to set class attributes. + + .. note:: + Requires internet access on first use to download pretrained weights + from the Hugging Face Hub. + """ + + MODEL_NAME: str = "" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_opus_mt" + + def __init__(self, model=None, **kwargs): + """Initialize tokenizer and seq2seq model. + + Parameters + ---------- + model : transformers.PreTrainedModel or None + Pre-loaded model to reuse instead of downloading weights. + **kwargs + Training hyperparameters forwarded to ``validate_and_transform``. + """ + kwargs = self.validate_and_transform(kwargs) + + from transformers import AutoTokenizer + + if not self.MODEL_NAME: + raise ValueError( + f"{self.__class__.__name__} must define a non-empty MODEL_NAME." + ) + + self.model_name = self.MODEL_NAME + self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) + + self.training_args = { + "num_train_epochs": kwargs.get("num_train_epochs", 2), + "learning_rate": kwargs.get("learning_rate", 2e-5), + "weight_decay": kwargs.get("weight_decay", 0.01), + } + self.batch_size = kwargs.get("batch_size", 4) + self.device = kwargs.get("device") or GPU_OR_CPU_PLACEHOLDER + self.log_train_every_n_epochs = kwargs.get("log_train_every_n_epochs", 1) + self.log_train_every_n_steps = kwargs.get("log_train_every_n_steps", None) + self.log_validation_every_n_epochs = kwargs.get( + "log_validation_every_n_epochs", 1 + ) + self.log_validation_every_n_steps = kwargs.get( + "log_validation_every_n_steps", None + ) + + if model is None: + from transformers import AutoModelForSeq2SeqLM + + self.model = AutoModelForSeq2SeqLM.from_pretrained(self.model_name) + else: + self.model = model + + self.num_train_epochs = self.training_args.get("num_train_epochs", 2) + self.fitted = model is not None + + def tokenize_data( + self, x: "DashAIDataset", y: Optional["DashAIDataset"] = None + ) -> "DashAIDataset": + """Tokenize source (and optionally target) dataset for seq2seq training.""" + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + is_y = bool(y) + if not y: + y = DashAIDataset.from_list([{"foo": 0}] * len(x)) + + dataset = [] + input_column_name = x.column_names[0] + output_column_name = y.column_names[0] if is_y else None + + for i, input_sample in enumerate(x): + tokenized_input = self.tokenizer( + input_sample[input_column_name], + truncation=True, + padding="max_length", + max_length=512, + ) + sample = { + "input_ids": tokenized_input["input_ids"], + "attention_mask": tokenized_input["attention_mask"], + } + if is_y: + output_sample = y[i] + tokenized_output = self.tokenizer( + output_sample[output_column_name], + truncation=True, + padding="max_length", + max_length=512, + ) + sample["labels"] = tokenized_output["input_ids"] + dataset.append(sample) + + return DashAIDataset.from_list(dataset) + + def train( + self, + x_train: "DashAIDataset", + y_train: "DashAIDataset", + x_validation: "DashAIDataset" = None, + y_validation: "DashAIDataset" = None, + ): + """Fine-tune the Opus-MT model on translation data.""" + from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments + + from DashAI.back.models.hugging_face.metrics_callback import MetricsCallback + + dataset = self.tokenize_data(x_train, y_train) + dataset.set_format("torch", columns=["input_ids", "attention_mask", "labels"]) + + has_validation_data = x_validation is not None and y_validation is not None + + output_root = Path(self.TEMP_CHECKPOINT_DIR) + output_root.mkdir(parents=True, exist_ok=True) + run_output_dir = tempfile.mkdtemp( + prefix=f"{self.__class__.__name__.lower()}_", + dir=str(output_root), + ) + + training_args_obj = Seq2SeqTrainingArguments( + output_dir=run_output_dir, + save_steps=1, + save_total_limit=1, + per_device_train_batch_size=self.batch_size, + per_device_eval_batch_size=self.batch_size, + use_cpu=self.device.lower() != "gpu", + **self.training_args, + ) + + metrics_callback = MetricsCallback( + model_instance=self, + x_train=x_train, + y_train=y_train, + x_val=x_validation, + y_val=y_validation, + total_epochs=self.num_train_epochs, + log_training_every_n_epochs=self.log_train_every_n_epochs, + log_training_every_n_steps=self.log_train_every_n_steps, + log_val_every_n_epochs=( + self.log_validation_every_n_epochs if has_validation_data else None + ), + log_val_every_n_steps=( + self.log_validation_every_n_steps if has_validation_data else None + ), + ) + + trainer = Seq2SeqTrainer( + model=self.model, + args=training_args_obj, + train_dataset=dataset, + callbacks=[metrics_callback], + ) + + self.fitted = True + try: + trainer.train() + finally: + shutil.rmtree(run_output_dir, ignore_errors=True) + + return self + + def predict(self, x_pred: "DashAIDataset") -> List: + """Translate source texts using the fine-tuned model.""" + if not self.fitted: + raise NotFittedError( + f"This {self.__class__.__name__} instance is not fitted yet. " + "Call 'train' with appropriate arguments before using this estimator." + ) + + dataset = self.tokenize_data(x_pred) + dataset.set_format(type="torch", columns=["input_ids", "attention_mask"]) + + translations = [] + for example in dataset: + inputs = { + k: v.unsqueeze(0).to(self.model.device) for k, v in example.items() + } + outputs = self.model.generate(**inputs) + translated_text = self.tokenizer.decode( + outputs[0], skip_special_tokens=True + ) + translations.append(translated_text) + + return translations + + def prepare_dataset( + self, dataset: "DashAIDataset", is_fit: bool = False + ) -> "DashAIDataset": + """Return the dataset unchanged (no preprocessing required).""" + return dataset + + def save(self, filename: Union[str, "Path"]) -> None: + """Persist model weights and hyperparameters to disk.""" + from transformers import AutoConfig + + save_dir = Path(filename) + if save_dir.exists() and save_dir.is_file(): + save_dir.unlink() + save_dir.mkdir(parents=True, exist_ok=True) + + self.model.save_pretrained(save_dir) + config = AutoConfig.from_pretrained(save_dir) + config.custom_params = { + "num_train_epochs": self.training_args.get("num_train_epochs"), + "batch_size": self.batch_size, + "learning_rate": self.training_args.get("learning_rate"), + "device": self.device, + "weight_decay": self.training_args.get("weight_decay"), + "fitted": self.fitted, + } + config.save_pretrained(save_dir) + + @classmethod + def load(cls, filename: Union[str, "Path"]): + """Restore a model instance from disk.""" + from transformers import AutoConfig, AutoModelForSeq2SeqLM + + model = AutoModelForSeq2SeqLM.from_pretrained(filename) + config = AutoConfig.from_pretrained(filename) + custom_params = getattr(config, "custom_params", {}) + + loaded_model = cls( + model=model, + num_train_epochs=custom_params.get("num_train_epochs"), + batch_size=custom_params.get("batch_size"), + learning_rate=custom_params.get("learning_rate"), + device=custom_params.get("device"), + weight_decay=custom_params.get("weight_decay"), + log_train_every_n_epochs=None, + log_train_every_n_steps=None, + log_validation_every_n_epochs=None, + log_validation_every_n_steps=None, + ) + loaded_model.fitted = custom_params.get("fitted", False) + return loaded_model diff --git a/DashAI/back/models/hugging_face/base_text_classification_transformer.py b/DashAI/back/models/hugging_face/base_text_classification_transformer.py index 6f8dc5707..c378d627f 100644 --- a/DashAI/back/models/hugging_face/base_text_classification_transformer.py +++ b/DashAI/back/models/hugging_face/base_text_classification_transformer.py @@ -30,6 +30,10 @@ class HuggingFaceTextClassificationTransformer(TextClassificationModel): - Training via ``transformers.Trainer`` with DashAI metric callbacks. - Inference returning per-class probability matrices. - Save/load utilities that preserve custom training parameters. + + .. note:: + Requires internet access on first use to download pre-trained weights + from the Hugging Face Hub. """ MODEL_NAME: str = "" diff --git a/DashAI/back/models/hugging_face/bert_transformer.py b/DashAI/back/models/hugging_face/bert_transformer.py new file mode 100644 index 000000000..a02bac698 --- /dev/null +++ b/DashAI/back/models/hugging_face/bert_transformer.py @@ -0,0 +1,45 @@ +"""DashAI implementation of BERT model for English text classification.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_text_classification_transformer import ( + HuggingFaceTextClassificationTransformer, +) +from DashAI.back.models.hugging_face.distilbert_transformer import ( + DistilBertTransformerSchema, +) + + +class BertTransformer(HuggingFaceTextClassificationTransformer): + """Pre-trained BERT model for English text classification. + + BERT (Bidirectional Encoder Representations from Transformers) pre-trains deep + bidirectional representations by jointly conditioning on both left and right + context in all layers. Fine-tuned BERT achieves strong results on a wide range + of text classification tasks. + + References + ---------- + - [1] Devlin, J. et al. (2019). "BERT: Pre-training of Deep Bidirectional + Transformers for Language Understanding." NAACL 2019. + - [2] https://huggingface.co/bert-base-uncased + """ + + DISPLAY_NAME: str = MultilingualString( + en="BERT Transformer", + es="Transformer BERT", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Bidirectional BERT model for English text classification. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "Modelo BERT bidireccional para clasificación de texto en inglés. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#1565C0" + ICON: str = "Psychology" + SCHEMA = DistilBertTransformerSchema + MODEL_NAME: str = "bert-base-uncased" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_bert" diff --git a/DashAI/back/models/hugging_face/bertin_transformer.py b/DashAI/back/models/hugging_face/bertin_transformer.py new file mode 100644 index 000000000..5f0c72d53 --- /dev/null +++ b/DashAI/back/models/hugging_face/bertin_transformer.py @@ -0,0 +1,45 @@ +"""DashAI implementation of BERTIN model for Spanish text classification.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_text_classification_transformer import ( + HuggingFaceTextClassificationTransformer, +) +from DashAI.back.models.hugging_face.distilbert_transformer import ( + DistilBertTransformerSchema, +) + + +class BertinTransformer(HuggingFaceTextClassificationTransformer): + """Pre-trained BERTIN model (Spanish RoBERTa) for Spanish text classification. + + BERTIN is a Spanish RoBERTa model trained on the Spanish portion of mC4 and + additional Spanish corpora. It applies RoBERTa's improved training recipe to + Spanish and typically outperforms BETO on Spanish NLP benchmarks. Requires + the ``sentencepiece`` package for its tokeniser. + + References + ---------- + - [1] de la Rosa, J. et al. (2022). "BERTIN: Efficient Pre-Training of a + Spanish Language Model using Perplexity Sampling." + - [2] https://huggingface.co/bertin-project/bertin-roberta-base-spanish + """ + + DISPLAY_NAME: str = MultilingualString( + en="BERTIN Spanish RoBERTa", + es="BERTIN RoBERTa en Español", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Spanish RoBERTa (BERTIN) pre-trained on large Spanish corpora. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "RoBERTa en español (BERTIN) pre-entrenada en grandes corpus en español. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#AD1457" + ICON: str = "RecordVoiceOver" + SCHEMA = DistilBertTransformerSchema + MODEL_NAME: str = "bertin-project/bertin-roberta-base-spanish" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_bertin" diff --git a/DashAI/back/models/hugging_face/beto_transformer.py b/DashAI/back/models/hugging_face/beto_transformer.py new file mode 100644 index 000000000..7e1d87e93 --- /dev/null +++ b/DashAI/back/models/hugging_face/beto_transformer.py @@ -0,0 +1,45 @@ +"""DashAI implementation of BETO model for Spanish text classification.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_text_classification_transformer import ( + HuggingFaceTextClassificationTransformer, +) +from DashAI.back.models.hugging_face.distilbert_transformer import ( + DistilBertTransformerSchema, +) + + +class BetoTransformer(HuggingFaceTextClassificationTransformer): + """Pre-trained BETO model for Spanish text classification. + + BETO is a Spanish BERT trained on the Spanish Wikipedia and other Spanish + corpora using the whole-word masking strategy. It achieves state-of-the-art + results on several Spanish NLP benchmarks. Particularly useful for tasks + involving Spanish text. + + References + ---------- + - [1] Cañete, J. et al. (2020). "Spanish Pre-Trained BERT Model and + Evaluation Data." PML4DC at ICLR 2020. + - [2] https://huggingface.co/dccuchile/bert-base-spanish-wwm-cased + """ + + DISPLAY_NAME: str = MultilingualString( + en="BETO Spanish BERT", + es="BETO BERT en Español", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Spanish BERT (BETO) pre-trained on Spanish corpora. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "BERT en español (BETO) pre-entrenado en corpus en español. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#C62828" + ICON: str = "RecordVoiceOver" + SCHEMA = DistilBertTransformerSchema + MODEL_NAME: str = "dccuchile/bert-base-spanish-wwm-cased" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_beto" diff --git a/DashAI/back/models/hugging_face/electra_transformer.py b/DashAI/back/models/hugging_face/electra_transformer.py new file mode 100644 index 000000000..a774c8ac6 --- /dev/null +++ b/DashAI/back/models/hugging_face/electra_transformer.py @@ -0,0 +1,45 @@ +"""DashAI implementation of ELECTRA model for English text classification.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_text_classification_transformer import ( + HuggingFaceTextClassificationTransformer, +) +from DashAI.back.models.hugging_face.distilbert_transformer import ( + DistilBertTransformerSchema, +) + + +class ElectraTransformer(HuggingFaceTextClassificationTransformer): + """Pre-trained ELECTRA model for efficient English text classification. + + ELECTRA uses a replaced-token-detection pre-training objective: a generator + produces plausible token replacements while a discriminator is trained to + identify which tokens were replaced. This allows ELECTRA to train on all + input tokens rather than only masked ones, making pre-training more efficient. + + References + ---------- + - [1] Clark, K. et al. (2020). "ELECTRA: Pre-training Text Encoders as + Discriminators Rather Than Generators." ICLR 2020. + - [2] https://huggingface.co/google/electra-small-discriminator + """ + + DISPLAY_NAME: str = MultilingualString( + en="ELECTRA Transformer", + es="Transformer ELECTRA", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Sample-efficient ELECTRA discriminator for text classification. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "Discriminador ELECTRA eficiente en muestras para clasificación de texto. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#558B2F" + ICON: str = "ElectricBolt" + SCHEMA = DistilBertTransformerSchema + MODEL_NAME: str = "google/electra-small-discriminator" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_electra" diff --git a/DashAI/back/models/hugging_face/m2m100_transformer.py b/DashAI/back/models/hugging_face/m2m100_transformer.py new file mode 100644 index 000000000..097a222cc --- /dev/null +++ b/DashAI/back/models/hugging_face/m2m100_transformer.py @@ -0,0 +1,327 @@ +"""M2M100 multilingual translation transformer for DashAI.""" + +import shutil +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, List, Optional, Union + +from sklearn.exceptions import NotFittedError + +from DashAI.back.core.schema_fields import schema_field, string_field +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.opus_mt_en_es_transformer import ( + OpusMtEnESTransformerSchema, +) +from DashAI.back.models.translation_model import TranslationModel +from DashAI.back.models.utils import GPU_OR_CPU_PLACEHOLDER + +if TYPE_CHECKING: + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + +class M2M100TransformerSchema(OpusMtEnESTransformerSchema): + """Schema for the M2M100 multilingual translation model. + + Extends the standard translation schema with ``source_language`` and + ``target_language`` fields, which accept ISO 639-1 language codes + (e.g. ``"en"`` for English, ``"es"`` for Spanish, ``"fr"`` for French). + """ + + source_language: schema_field( + string_field(), + placeholder="en", + description=MultilingualString( + en=( + "Source language ISO 639-1 code (e.g. 'en', 'es', 'fr', 'de'). " + "Supports 100 languages." + ), + es=( + "Código ISO 639-1 del idioma de origen (ej. 'en', 'es', 'fr', 'de'). " + "Soporta 100 idiomas." + ), + ), + alias=MultilingualString(en="Source language", es="Idioma de origen"), + ) # type: ignore + target_language: schema_field( + string_field(), + placeholder="es", + description=MultilingualString( + en=( + "Target language ISO 639-1 code (e.g. 'en', 'es', 'fr', 'de'). " + "Supports 100 languages." + ), + es=( + "Código ISO 639-1 del idioma destino (ej. 'en', 'es', 'fr', 'de'). " + "Soporta 100 idiomas." + ), + ), + alias=MultilingualString(en="Target language", es="Idioma destino"), + ) # type: ignore + + +class M2M100Transformer(TranslationModel): + """M2M100 multilingual seq2seq model for configurable language-pair translation. + + Fine-tunes the ``facebook/m2m100_418M`` checkpoint from Meta AI. The base + model supports direct translation across 100 languages using ISO 639-1 + language codes (e.g. ``"en"``, ``"es"``, ``"fr"``). Unlike pivot-based + systems, M2M100 translates directly between any supported pair. + + Target language generation is guided by ``forced_bos_token_id`` obtained + from ``tokenizer.get_lang_id(target_language)``, identical in principle + to the NLLB approach but using simpler ISO codes. + + References + ---------- + - [1] https://huggingface.co/facebook/m2m100_418M + - [2] Fan et al. (2021). "Beyond English-Centric Multilingual Machine + Translation." JMLR 2021. + """ + + SCHEMA = M2M100TransformerSchema + DISPLAY_NAME: str = MultilingualString( + en="M2M-100 Multilingual Transformer", + es="Transformer Multilingüe M2M-100", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Facebook M2M-100 model for direct translation across 100 languages " + "using ISO 639-1 codes. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "Modelo M2M-100 de Facebook para traducción directa entre 100 idiomas " + "usando códigos ISO 639-1. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#6A1B9A" + ICON: str = "Language" + + def __init__(self, model=None, **kwargs): + kwargs = self.validate_and_transform(kwargs) + + from transformers import AutoTokenizer + + self.model_name = "facebook/m2m100_418M" + self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) + + self.source_language = kwargs.get("source_language", "en") + self.target_language = kwargs.get("target_language", "es") + + if hasattr(self.tokenizer, "src_lang"): + self.tokenizer.src_lang = self.source_language + + self.training_args = { + "num_train_epochs": kwargs.get("num_train_epochs", 2), + "learning_rate": kwargs.get("learning_rate", 2e-5), + "weight_decay": kwargs.get("weight_decay", 0.01), + } + self.batch_size = kwargs.get("batch_size", 4) + self.device = kwargs.get("device") or GPU_OR_CPU_PLACEHOLDER + self.log_train_every_n_epochs = kwargs.get("log_train_every_n_epochs", 1) + self.log_train_every_n_steps = kwargs.get("log_train_every_n_steps", None) + self.log_validation_every_n_epochs = kwargs.get( + "log_validation_every_n_epochs", 1 + ) + self.log_validation_every_n_steps = kwargs.get( + "log_validation_every_n_steps", None + ) + + if model is None: + from transformers import AutoModelForSeq2SeqLM + + self.model = AutoModelForSeq2SeqLM.from_pretrained(self.model_name) + else: + self.model = model + + self.num_train_epochs = self.training_args.get("num_train_epochs", 2) + self.fitted = model is not None + + def tokenize_data( + self, x: "DashAIDataset", y: Optional["DashAIDataset"] = None + ) -> "DashAIDataset": + """Tokenize with src_lang set for M2M100.""" + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + if hasattr(self.tokenizer, "src_lang"): + self.tokenizer.src_lang = self.source_language + + is_y = bool(y) + if not y: + y = DashAIDataset.from_list([{"foo": 0}] * len(x)) + + dataset = [] + input_column_name = x.column_names[0] + output_column_name = y.column_names[0] if is_y else None + + for i, input_sample in enumerate(x): + tokenized_input = self.tokenizer( + input_sample[input_column_name], + truncation=True, + padding="max_length", + max_length=512, + ) + sample = { + "input_ids": tokenized_input["input_ids"], + "attention_mask": tokenized_input["attention_mask"], + } + if is_y: + output_sample = y[i] + tokenized_output = self.tokenizer( + output_sample[output_column_name], + truncation=True, + padding="max_length", + max_length=512, + ) + sample["labels"] = tokenized_output["input_ids"] + dataset.append(sample) + + return DashAIDataset.from_list(dataset) + + def train( + self, + x_train: "DashAIDataset", + y_train: "DashAIDataset", + x_validation: "DashAIDataset" = None, + y_validation: "DashAIDataset" = None, + ): + """Fine-tune M2M100 on the configured language pair.""" + from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments + + from DashAI.back.models.hugging_face.metrics_callback import MetricsCallback + + dataset = self.tokenize_data(x_train, y_train) + dataset.set_format("torch", columns=["input_ids", "attention_mask", "labels"]) + + has_validation_data = x_validation is not None and y_validation is not None + + output_root = Path("DashAI/back/user_models/temp_checkpoints_m2m100") + output_root.mkdir(parents=True, exist_ok=True) + run_output_dir = tempfile.mkdtemp(prefix="m2m100_", dir=str(output_root)) + + training_args_obj = Seq2SeqTrainingArguments( + output_dir=run_output_dir, + save_steps=1, + save_total_limit=1, + per_device_train_batch_size=self.batch_size, + per_device_eval_batch_size=self.batch_size, + use_cpu=self.device.lower() != "gpu", + **self.training_args, + ) + + metrics_callback = MetricsCallback( + model_instance=self, + x_train=x_train, + y_train=y_train, + x_val=x_validation, + y_val=y_validation, + total_epochs=self.num_train_epochs, + log_training_every_n_epochs=self.log_train_every_n_epochs, + log_training_every_n_steps=self.log_train_every_n_steps, + log_val_every_n_epochs=( + self.log_validation_every_n_epochs if has_validation_data else None + ), + log_val_every_n_steps=( + self.log_validation_every_n_steps if has_validation_data else None + ), + ) + + trainer = Seq2SeqTrainer( + model=self.model, + args=training_args_obj, + train_dataset=dataset, + callbacks=[metrics_callback], + ) + + self.fitted = True + try: + trainer.train() + finally: + shutil.rmtree(run_output_dir, ignore_errors=True) + + return self + + def predict(self, x_pred: "DashAIDataset") -> List: + """Translate using forced_bos_token_id for the target language.""" + if not self.fitted: + raise NotFittedError( + f"This {self.__class__.__name__} instance is not fitted yet. " + "Call 'train' before using this estimator." + ) + + dataset = self.tokenize_data(x_pred) + dataset.set_format(type="torch", columns=["input_ids", "attention_mask"]) + + target_bos = self.tokenizer.get_lang_id(self.target_language) + translations = [] + + for example in dataset: + inputs = { + k: v.unsqueeze(0).to(self.model.device) for k, v in example.items() + } + outputs = self.model.generate( + **inputs, + forced_bos_token_id=target_bos, + ) + translated_text = self.tokenizer.decode( + outputs[0], skip_special_tokens=True + ) + translations.append(translated_text) + + return translations + + def prepare_dataset( + self, dataset: "DashAIDataset", is_fit: bool = False + ) -> "DashAIDataset": + """Return the dataset unchanged.""" + return dataset + + def save(self, filename: Union[str, "Path"]) -> None: + """Persist model weights and hyperparameters to disk.""" + from transformers import AutoConfig + + save_dir = Path(filename) + if save_dir.exists() and save_dir.is_file(): + save_dir.unlink() + save_dir.mkdir(parents=True, exist_ok=True) + + self.model.save_pretrained(save_dir) + config = AutoConfig.from_pretrained(save_dir) + config.custom_params = { + "num_train_epochs": self.training_args.get("num_train_epochs"), + "batch_size": self.batch_size, + "learning_rate": self.training_args.get("learning_rate"), + "device": self.device, + "weight_decay": self.training_args.get("weight_decay"), + "source_language": self.source_language, + "target_language": self.target_language, + "fitted": self.fitted, + } + config.save_pretrained(save_dir) + + @classmethod + def load(cls, filename: Union[str, "Path"]): + """Restore an M2M100Transformer instance from disk.""" + from transformers import AutoConfig, AutoModelForSeq2SeqLM + + model = AutoModelForSeq2SeqLM.from_pretrained(filename) + config = AutoConfig.from_pretrained(filename) + custom_params = getattr(config, "custom_params", {}) + + loaded_model = cls( + model=model, + num_train_epochs=custom_params.get("num_train_epochs"), + batch_size=custom_params.get("batch_size"), + learning_rate=custom_params.get("learning_rate"), + device=custom_params.get("device"), + weight_decay=custom_params.get("weight_decay"), + source_language=custom_params.get("source_language", "en"), + target_language=custom_params.get("target_language", "es"), + log_train_every_n_epochs=None, + log_train_every_n_steps=None, + log_validation_every_n_epochs=None, + log_validation_every_n_steps=None, + ) + loaded_model.fitted = custom_params.get("fitted", False) + return loaded_model diff --git a/DashAI/back/models/hugging_face/minilm_transformer.py b/DashAI/back/models/hugging_face/minilm_transformer.py new file mode 100644 index 000000000..5b1347936 --- /dev/null +++ b/DashAI/back/models/hugging_face/minilm_transformer.py @@ -0,0 +1,45 @@ +"""DashAI implementation of MiniLM model for efficient English text classification.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_text_classification_transformer import ( + HuggingFaceTextClassificationTransformer, +) +from DashAI.back.models.hugging_face.distilbert_transformer import ( + DistilBertTransformerSchema, +) + + +class MiniLMTransformer(HuggingFaceTextClassificationTransformer): + """Pre-trained MiniLM model for lightweight English text classification. + + MiniLM is a compressed BERT-like model distilled from a larger teacher + network using deep self-attention distillation. It achieves competitive + performance while being significantly smaller and faster than BERT, making + it a good choice for resource-constrained deployments. + + References + ---------- + - [1] Wang, W. et al. (2020). "MiniLM: Deep Self-Attention Distillation for + Task-Agnostic Compression of Pre-Trained Transformers." NeurIPS 2020. + - [2] https://huggingface.co/microsoft/MiniLM-L12-H384-uncased + """ + + DISPLAY_NAME: str = MultilingualString( + en="MiniLM Transformer", + es="Transformer MiniLM", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Compact, fast MiniLM model for efficient text classification. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "Modelo MiniLM compacto y rápido para clasificación de texto eficiente. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#0277BD" + ICON: str = "Speed" + SCHEMA = DistilBertTransformerSchema + MODEL_NAME: str = "microsoft/MiniLM-L12-H384-uncased" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_minilm" diff --git a/DashAI/back/models/hugging_face/multilingual_bert_transformer.py b/DashAI/back/models/hugging_face/multilingual_bert_transformer.py new file mode 100644 index 000000000..4a7788744 --- /dev/null +++ b/DashAI/back/models/hugging_face/multilingual_bert_transformer.py @@ -0,0 +1,47 @@ +"""DashAI implementation of Multilingual BERT for multilingual text classification.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_text_classification_transformer import ( + HuggingFaceTextClassificationTransformer, +) +from DashAI.back.models.hugging_face.distilbert_transformer import ( + DistilBertTransformerSchema, +) + + +class MultilingualBertTransformer(HuggingFaceTextClassificationTransformer): + """Pre-trained Multilingual BERT for cross-lingual text classification. + + mBERT (Multilingual BERT) is a single BERT model pre-trained on the Wikipedia + text of 104 languages. It uses a shared vocabulary and can be fine-tuned on a + task in one language and applied to another (zero-shot cross-lingual transfer). + + References + ---------- + - [1] Devlin, J. et al. (2019). "BERT: Pre-training of Deep Bidirectional + Transformers for Language Understanding." NAACL 2019. + - [2] https://huggingface.co/bert-base-multilingual-cased + """ + + DISPLAY_NAME: str = MultilingualString( + en="Multilingual BERT Transformer", + es="Transformer BERT Multilingüe", + ) + DESCRIPTION: str = MultilingualString( + en=( + "BERT pre-trained on 104 languages for multilingual text classification. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "BERT pre-entrenado en 104 idiomas para clasificación de texto " + "multilingüe. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#283593" + ICON: str = "Translate" + SCHEMA = DistilBertTransformerSchema + MODEL_NAME: str = "bert-base-multilingual-cased" + TEMP_CHECKPOINT_DIR: str = ( + "DashAI/back/user_models/temp_checkpoints_multilingual_bert" + ) diff --git a/DashAI/back/models/hugging_face/nllb_transformer.py b/DashAI/back/models/hugging_face/nllb_transformer.py index b30619b06..df7114759 100644 --- a/DashAI/back/models/hugging_face/nllb_transformer.py +++ b/DashAI/back/models/hugging_face/nllb_transformer.py @@ -31,12 +31,15 @@ class NllbTransformerSchema(OpusMtEnESTransformerSchema): placeholder="spa_Latn", description=MultilingualString( en=( - "Source language code for NLLB tokenizer. " - "Example: spa_Latn for Spanish." + "Source language code for NLLB tokenizer (e.g. spa_Latn for Spanish, " + "eng_Latn for English). It uses BCP-47 language tags in the format" + "[Examples](https://dl-translate.readthedocs.io/en/latest/available_languages/#nllb-200)" ), es=( - "Código de idioma de origen para el tokenizer NLLB. " - "Ejemplo: spa_Latn para español." + "Código de idioma de origen para el tokenizer NLLB (ej. spa_Latn para " + "español, eng_Latn para inglés). Utiliza etiquetas de idioma BCP-47 " + "en el formato " + "[Ejemplos](https://dl-translate.readthedocs.io/en/latest/available_languages/#nllb-200)" ), ), alias=MultilingualString(en="Source language", es="Idioma de origen"), @@ -46,12 +49,15 @@ class NllbTransformerSchema(OpusMtEnESTransformerSchema): placeholder="eng_Latn", description=MultilingualString( en=( - "Target language code for NLLB generation. " - "Example: eng_Latn for English." + "Target language code for NLLB generation (e.g. eng_Latn for English, " + "fra_Latn for French). It uses BCP-47 language tags in the format " + "[Examples](https://dl-translate.readthedocs.io/en/latest/available_languages/#nllb-200)" ), es=( - "Código de idioma destino para la generación NLLB. " - "Ejemplo: eng_Latn para inglés." + "Código de idioma destino para la generación NLLB (ej. eng_Latn para " + "inglés, fra_Latn para francés). Utiliza etiquetas de idioma BCP-47 " + "en el formato " + "[Ejemplos](https://dl-translate.readthedocs.io/en/latest/available_languages/#nllb-200)" ), ), alias=MultilingualString(en="Target language", es="Idioma destino"), diff --git a/DashAI/back/models/hugging_face/opus_mt_en_de_transformer.py b/DashAI/back/models/hugging_face/opus_mt_en_de_transformer.py new file mode 100644 index 000000000..9bf12c8a0 --- /dev/null +++ b/DashAI/back/models/hugging_face/opus_mt_en_de_transformer.py @@ -0,0 +1,47 @@ +"""OpusMtEnDeTransformer model for English-to-German translation.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_opus_mt_transformer import ( + OpusMtTransformerMixin, +) +from DashAI.back.models.hugging_face.opus_mt_en_es_transformer import ( + OpusMtEnESTransformerSchema, +) + + +class OpusMtEnDeTransformerSchema(OpusMtEnESTransformerSchema): + """Schema for the English-to-German Opus-MT model.""" + + +class OpusMtEnDeTransformer(OpusMtTransformerMixin): + """Pre-trained transformer for English-to-German translation. + + Fine-tunes the Helsinki-NLP ``opus-mt-en-de`` checkpoint, a MarianMT + seq2seq model trained on parallel English-German corpora from the OPUS + collection. + + References + ---------- + - [1] https://huggingface.co/Helsinki-NLP/opus-mt-en-de + - [2] https://opus.nlpl.eu/ + """ + + MODEL_NAME: str = "Helsinki-NLP/opus-mt-en-de" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_opus-mt-en-de" + SCHEMA = OpusMtEnDeTransformerSchema + DISPLAY_NAME: str = MultilingualString( + en="Opus MT En-De Transformer", + es="Transformer Opus MT En-De", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Pre-trained transformer for English-German translation. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "Transformer pre-entrenado para traducción inglés-alemán. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#455A64" + ICON: str = "Translate" diff --git a/DashAI/back/models/hugging_face/opus_mt_en_es_transformer.py b/DashAI/back/models/hugging_face/opus_mt_en_es_transformer.py index 1a4c7bbd3..96ecaab78 100644 --- a/DashAI/back/models/hugging_face/opus_mt_en_es_transformer.py +++ b/DashAI/back/models/hugging_face/opus_mt_en_es_transformer.py @@ -1,10 +1,4 @@ -"""OpusMtEnESTransformer model for english-spanish translation DashAI implementation.""" - -import shutil -from pathlib import Path -from typing import TYPE_CHECKING, List, Optional, Union - -from sklearn.exceptions import NotFittedError +"""OpusMtEnESTransformer model for English-to-Spanish translation.""" from DashAI.back.core.schema_fields import ( BaseSchema, @@ -15,20 +9,18 @@ schema_field, ) from DashAI.back.core.utils import MultilingualString -from DashAI.back.models.translation_model import TranslationModel +from DashAI.back.models.hugging_face.base_opus_mt_transformer import ( + OpusMtTransformerMixin, +) from DashAI.back.models.utils import GPU_OR_CPU, GPU_OR_CPU_PLACEHOLDER -if TYPE_CHECKING: - from pathlib import Path - - from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset - class OpusMtEnESTransformerSchema(BaseSchema): - """opus-mt-en-es is a transformer pre-trained model that allows translation of - texts from English to Spanish. The implementation is based on the Helsinki-NLP - opus-mt-en-es checkpoint, which uses the MarianMT architecture and was trained - on parallel corpora from the OPUS collection. + """Schema for Opus-MT translation models (MarianMT architecture). + + Shared by all Helsinki-NLP Opus-MT language-pair wrappers. Controls + training duration, batch size, learning rate, device, regularization, and + metric-logging frequency. """ num_train_epochs: schema_field( @@ -44,8 +36,8 @@ class OpusMtEnESTransformerSchema(BaseSchema): int_field(ge=1), placeholder=4, description=MultilingualString( - en="The batch size per GPU/TPU core/CPU for training", - es="El tamaño de lote por núcleo GPU/TPU/CPU para entrenamiento", + en="The batch size per GPU/TPU core/CPU for training.", + es="El tamaño de lote por núcleo GPU/TPU/CPU para entrenamiento.", ), alias=MultilingualString(en="Batch size", es="Tamaño de lote"), ) # type: ignore @@ -53,8 +45,8 @@ class OpusMtEnESTransformerSchema(BaseSchema): float_field(ge=0.0), placeholder=2e-5, description=MultilingualString( - en="The initial learning rate for AdamW optimizer", - es="La tasa de aprendizaje inicial para el optimizador AdamW", + en="The initial learning rate for AdamW optimizer.", + es="La tasa de aprendizaje inicial para el optimizador AdamW.", ), alias=MultilingualString(en="Learning rate", es="Tasa de aprendizaje"), ) # type: ignore @@ -63,14 +55,13 @@ class OpusMtEnESTransformerSchema(BaseSchema): placeholder=GPU_OR_CPU_PLACEHOLDER, description=MultilingualString( en=( - "Hardware on which the training is run. If available, GPU is " - "recommended for efficiency reasons. Otherwise, use CPU. " - "If GPU is selected then it will use all gpus available. " + "Hardware on which training is run. GPU is recommended when " + "available. If GPU is selected, all available GPUs are used." ), es=( - "Hardware en el que se ejecuta el entrenamiento. Si está disponible, " - "se recomienda GPU por razones de eficiencia. De lo contrario, use " - "CPU. Si se selecciona GPU, usará todas las GPUs disponibles." + "Hardware en el que se ejecuta el entrenamiento. Se recomienda " + "GPU cuando está disponible. Si se selecciona GPU, se usan " + "todas las GPUs disponibles." ), ), alias=MultilingualString(en="Device", es="Dispositivo"), @@ -80,107 +71,87 @@ class OpusMtEnESTransformerSchema(BaseSchema): placeholder=0.01, description=MultilingualString( en=( - "Weight decay is a regularization technique used in training " - "neural networks to prevent overfitting. In the context of the AdamW " - "optimizer, the 'weight_decay' parameter is the rate at which the " - "weights of all layers are reduced during training, provided that " - "this rate is not zero." + "L2 regularization coefficient applied via the AdamW optimizer " + "to prevent overfitting." ), es=( - "Weight decay es una técnica de regularización usada en el " - "entrenamiento de redes neuronales para prevenir sobreajuste. En el " - "contexto del optimizador AdamW, el parámetro 'weight_decay' es la " - "tasa a la cual los pesos de todas las capas se reducen durante el " - "entrenamiento, siempre que esta tasa no sea cero." + "Coeficiente de regularización L2 aplicado mediante el " + "optimizador AdamW para prevenir sobreajuste." ), ), alias=MultilingualString(en="Weight decay", es="Decaimiento de pesos"), ) # type: ignore - log_train_every_n_epochs: schema_field( none_type(int_field(ge=1)), placeholder=1, description=MultilingualString( - en=( - "Log metrics for train split every n epochs during training. " - "If None, it won't log per epoch." - ), + en=("Log train metrics every N epochs. None disables per-epoch logging."), es=( - "Registrar métricas del split de entrenamiento cada n épocas. " - "Si es None, no registrará por época." + "Registrar métricas de entrenamiento cada N épocas. " + "None desactiva el registro por época." ), ), alias=MultilingualString( en="Log train every N epochs", es="Registrar entrenamiento cada N épocas" ), ) # type: ignore - log_train_every_n_steps: schema_field( none_type(int_field(ge=1)), placeholder=None, description=MultilingualString( - en=( - "Log metrics for train split every n steps during training. " - "If None, it won't log per step." - ), + en=("Log train metrics every N steps. None disables per-step logging."), es=( - "Registrar métricas del split de entrenamiento cada n pasos. " - "Si es None, no registrará por paso." + "Registrar métricas de entrenamiento cada N pasos. " + "None desactiva el registro por paso." ), ), alias=MultilingualString( en="Log train every N steps", es="Registrar entrenamiento cada N pasos" ), ) # type: ignore - log_validation_every_n_epochs: schema_field( none_type(int_field(ge=1)), placeholder=1, description=MultilingualString( en=( - "Log metrics for validation split every n epochs during training. " - "If None, it won't log per epoch." + "Log validation metrics every N epochs. " + "None disables per-epoch logging." ), es=( - "Registrar métricas del split de validación cada n épocas. " - "Si es None, no registrará por época." + "Registrar métricas de validación cada N épocas. " + "None desactiva el registro por época." ), ), alias=MultilingualString( - en="Log validation every N epochs", es="Registrar validación cada N épocas" + en="Log validation every N epochs", + es="Registrar validación cada N épocas", ), ) # type: ignore - log_validation_every_n_steps: schema_field( none_type(int_field(ge=1)), placeholder=None, description=MultilingualString( en=( - "Log metrics for validation split every n steps during training. " - "If None, it won't log per step." + "Log validation metrics every N steps. None disables per-step logging." ), es=( - "Registrar métricas del split de validación cada n pasos. " - "Si es None, no registrará por paso." + "Registrar métricas de validación cada N pasos. " + "None desactiva el registro por paso." ), ), alias=MultilingualString( - en="Log validation every N steps", es="Registrar validación cada N pasos" + en="Log validation every N steps", + es="Registrar validación cada N pasos", ), ) # type: ignore -class OpusMtEnESTransformer(TranslationModel): +class OpusMtEnESTransformer(OpusMtTransformerMixin): """Pre-trained transformer for English-to-Spanish translation. - This model fine-tunes the Helsinki-NLP ``opus-mt-en-es`` checkpoint, which is - based on the MarianMT sequence-to-sequence architecture. The base model was - trained on parallel English-Spanish corpora from the OPUS collection and supports - direct translation without intermediate pivot languages. - - Fine-tuning is performed with the HuggingFace ``Seq2SeqTrainer`` using the AdamW - optimizer. Training and validation metrics are logged at configurable epoch and - step intervals via a custom ``MetricsCallback``. + Fine-tunes the Helsinki-NLP ``opus-mt-en-es`` checkpoint, a MarianMT + seq2seq model trained on parallel English-Spanish corpora from the OPUS + collection. Supports direct translation without pivot languages. References ---------- @@ -188,334 +159,22 @@ class OpusMtEnESTransformer(TranslationModel): - [2] https://opus.nlpl.eu/ """ + MODEL_NAME: str = "Helsinki-NLP/opus-mt-en-es" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_opus-mt-en-es" SCHEMA = OpusMtEnESTransformerSchema DISPLAY_NAME: str = MultilingualString( en="Opus MT En-Es Transformer", es="Transformer Opus MT En-Es", ) DESCRIPTION: str = MultilingualString( - en="Pre-trained transformer for English-Spanish translation.", - es="Transformer pre-entrenado para traducción inglés-español.", + en=( + "Pre-trained transformer for English-Spanish translation. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "Transformer pre-entrenado para traducción inglés-español. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), ) COLOR: str = "#FFA500" ICON: str = "Translate" - - def __init__(self, model=None, **kwargs): - """Initialize the transformer. - - This process includes the instantiation of the pre-trained model and the - associated tokenizer. When ``model`` is ``None`` the Helsinki-NLP - ``opus-mt-en-es`` checkpoint is downloaded from HuggingFace; when a - model object is supplied the tokenizer is reused without re-downloading. - - Parameters - ---------- - model : transformers.PreTrainedModel or None, optional - An already-loaded HuggingFace translation model to reuse. If - ``None``, the pre-trained ``Helsinki-NLP/opus-mt-en-es`` checkpoint - is downloaded and initialised. Default ``None``. - **kwargs : dict - Additional hyperparameters forwarded to ``validate_and_transform`` - and used to configure training arguments (e.g. ``batch_size``, - ``epochs``, ``learning_rate``). - """ - kwargs = self.validate_and_transform(kwargs) - from transformers import AutoTokenizer - - self.model_name = "Helsinki-NLP/opus-mt-en-es" - self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) - if model is None: - self.training_args = kwargs - self.batch_size = kwargs.pop("batch_size", 16) - self.device = kwargs.pop("device") - self.log_train_every_n_epochs = kwargs.pop("log_train_every_n_epochs", 1) - self.log_train_every_n_steps = kwargs.pop("log_train_every_n_steps", None) - self.log_validation_every_n_epochs = kwargs.pop( - "log_validation_every_n_epochs", 1 - ) - self.log_validation_every_n_steps = kwargs.pop( - "log_validation_every_n_steps", None - ) - if model is None: - from transformers import AutoModelForSeq2SeqLM - - self.model = AutoModelForSeq2SeqLM.from_pretrained(self.model_name) - else: - self.model = model - self.num_train_epochs = kwargs.get("num_train_epochs", 2) - self.fitted = model is not None - - def tokenize_data( - self, x: "DashAIDataset", y: Optional["DashAIDataset"] = None - ) -> "DashAIDataset": - """Tokenize input and optional target datasets for seq2seq training. - - Each sample is tokenized with truncation and max-length padding to 512 - tokens. When ``y`` is provided, the target tokens are stored under the - ``labels`` key so the ``Seq2SeqTrainer`` can compute the loss directly. - - Parameters - ---------- - x : DashAIDataset - Source-language dataset. Only the first column is used. - y : DashAIDataset, optional - Target-language dataset. When provided, tokenized targets are added - as ``labels``. When ``None``, only ``input_ids`` and - ``attention_mask`` are returned (inference mode). - - Returns - ------- - DashAIDataset - Tokenized dataset with keys ``input_ids``, ``attention_mask``, and - optionally ``labels``. - """ - from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset - - is_y = bool(y) - if not y: - y = DashAIDataset.from_list([{"foo": 0}] * len(x)) - dataset = [] - input_column_name = x.column_names[0] - output_column_name = y.column_names[0] if is_y else None - - for i, input_sample in enumerate(x): - tokenized_input = self.tokenizer( - input_sample[input_column_name], - truncation=True, - padding="max_length", - max_length=512, - ) - - sample = { - "input_ids": tokenized_input["input_ids"], - "attention_mask": tokenized_input["attention_mask"], - } - - if is_y: - output_sample = y[i] - tokenized_output = self.tokenizer( - output_sample[output_column_name], - truncation=True, - padding="max_length", - max_length=512, - ) - sample["labels"] = tokenized_output["input_ids"] - - dataset.append(sample) - return DashAIDataset.from_list(dataset) - - def train( - self, - x_train: "DashAIDataset", - y_train: "DashAIDataset", - x_validation: "DashAIDataset" = None, - y_validation: "DashAIDataset" = None, - ) -> "OpusMtEnESTransformer": - """Fine-tune the opus-mt-en-es model on English-Spanish translation data. - - Parameters - ---------- - x_train : DashAIDataset - Input English text features for training. - y_train : DashAIDataset - Target Spanish translation labels for training. - x_validation : DashAIDataset, optional - Input English text features for validation. Defaults to None. - y_validation : DashAIDataset, optional - Target Spanish translation labels for validation. Defaults to None. - - Returns - ------- - OpusMtEnESTransformer - The fine-tuned model instance. - """ - from DashAI.back.models.hugging_face.metrics_callback import MetricsCallback - - dataset = self.tokenize_data(x_train, y_train) - dataset.set_format("torch", columns=["input_ids", "attention_mask", "labels"]) - - from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments - - training_args = Seq2SeqTrainingArguments( - output_dir="DashAI/back/user_models/temp_checkpoints_opus-mt-en-es", - save_steps=1, - save_total_limit=1, - per_device_train_batch_size=self.batch_size, - per_device_eval_batch_size=self.batch_size, - use_cpu=self.device.lower() != "gpu", - **self.training_args, - ) - - # Initialize the custom callback with epoch information - metrics_callback = MetricsCallback( - model_instance=self, - x_train=x_train, - y_train=y_train, - x_val=x_validation, - y_val=y_validation, - total_epochs=self.num_train_epochs, - log_training_every_n_epochs=self.log_train_every_n_epochs, - log_training_every_n_steps=self.log_train_every_n_steps, - log_val_every_n_epochs=self.log_validation_every_n_epochs, - log_val_every_n_steps=self.log_validation_every_n_steps, - ) - - trainer = Seq2SeqTrainer( - model=self.model, - args=training_args, - train_dataset=dataset, - callbacks=[metrics_callback], - ) - - self.fitted = True - trainer.train() - shutil.rmtree( - "DashAI/back/user_models/temp_checkpoints_opus-mt-en-es", ignore_errors=True - ) - return self - - def predict(self, x_pred: "DashAIDataset") -> List: - """Translate English source texts to Spanish. - - Parameters - ---------- - x_pred : DashAIDataset - Source-language dataset. Only the first column is used. - - Returns - ------- - list of str - One translated string per input sample, in the same order as - ``x_pred``. - - Raises - ------ - sklearn.exceptions.NotFittedError - If the model has not been fine-tuned yet (``fitted`` is ``False``). - """ - if not self.fitted: - raise NotFittedError( - f"This {self.__class__.__name__} instance is not fitted yet. Call 'fit'" - " with appropriate arguments before using this " - "estimator." - ) - - dataset = self.tokenize_data(x_pred) - dataset.set_format(type="torch", columns=["input_ids", "attention_mask"]) - - translations = [] - - for example in dataset: - inputs = { - k: v.unsqueeze(0).to(self.model.device) for k, v in example.items() - } - outputs = self.model.generate(**inputs) - translated_text = self.tokenizer.decode( - outputs[0], skip_special_tokens=True - ) - translations.append(translated_text) - - return translations - - def prepare_dataset( - self, dataset: "DashAIDataset", is_fit: bool = False - ) -> "DashAIDataset": - """Return the dataset unchanged. - - No pre-processing transformations are required for this model. The - method exists for compatibility with the DashAI model interface. - - Parameters - ---------- - dataset : DashAIDataset - The dataset to be prepared. - is_fit : bool, optional - Whether the call is made during fitting. Unused here. Default - ``False``. - - Returns - ------- - DashAIDataset - The original dataset, unmodified. - """ - try: - # Useless in this case, but we keep it for consistency with other models. - return dataset - except Exception as e: - print(f"Couldn't apply transformations to the dataset for the model: {e}") - - def save(self, filename: Union[str, "Path"]) -> None: - """Store the fine-tuned model and its configuration to disk. - - Saves the model weights via ``save_pretrained`` and embeds the - hyperparameters (epochs, batch size, learning rate, etc.) into the - Hugging Face config so they can be restored by :meth:`load`. - - Parameters - ---------- - filename : str or Path - Directory path where the model files will be written. - """ - save_dir = Path(filename) - if save_dir.exists() and save_dir.is_file(): - save_dir.unlink() - save_dir.mkdir(parents=True, exist_ok=True) - - self.model.save_pretrained(save_dir) - from transformers import AutoConfig - - config = AutoConfig.from_pretrained(save_dir) - - config.custom_params = { - "num_train_epochs": self.training_args.get("num_train_epochs"), - "batch_size": self.batch_size, - "learning_rate": self.training_args.get("learning_rate"), - "device": self.device, - "weight_decay": self.training_args.get("weight_decay"), - "fitted": self.fitted, - } - - config.save_pretrained(save_dir) - - @classmethod - def load(cls, filename: Union[str, "Path"]): - """Restore an OpusMtEnESTransformer instance from disk. - - Reads the Hugging Face config to recover the custom hyperparameters - saved by :meth:`save`, then reconstructs the seq2seq model and wraps - it in a new :class:`OpusMtEnESTransformer` instance. - - Parameters - ---------- - filename : str or Path - Directory path from which the model files will be read. - - Returns - ------- - OpusMtEnESTransformer - The restored model instance with ``fitted`` set to the persisted - value. - """ - from transformers import AutoConfig, AutoModelForSeq2SeqLM - - model = AutoModelForSeq2SeqLM.from_pretrained(filename) - - config = AutoConfig.from_pretrained(filename) - - custom_params = getattr(config, "custom_params", {}) - - loaded_model = cls( - model=model, - num_train_epochs=custom_params.get("num_train_epochs"), - batch_size=custom_params.get("batch_size"), - learning_rate=custom_params.get("learning_rate"), - device=custom_params.get("device"), - weight_decay=custom_params.get("weight_decay"), - log_train_every_n_epochs=None, - log_train_every_n_steps=None, - log_validation_every_n_epochs=None, - log_validation_every_n_steps=None, - ) - loaded_model.fitted = custom_params.get("fitted", False) - - return loaded_model diff --git a/DashAI/back/models/hugging_face/opus_mt_en_fr_transformer.py b/DashAI/back/models/hugging_face/opus_mt_en_fr_transformer.py new file mode 100644 index 000000000..144d7cca2 --- /dev/null +++ b/DashAI/back/models/hugging_face/opus_mt_en_fr_transformer.py @@ -0,0 +1,47 @@ +"""OpusMtEnFrTransformer model for English-to-French translation.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_opus_mt_transformer import ( + OpusMtTransformerMixin, +) +from DashAI.back.models.hugging_face.opus_mt_en_es_transformer import ( + OpusMtEnESTransformerSchema, +) + + +class OpusMtEnFrTransformerSchema(OpusMtEnESTransformerSchema): + """Schema for the English-to-French Opus-MT model.""" + + +class OpusMtEnFrTransformer(OpusMtTransformerMixin): + """Pre-trained transformer for English-to-French translation. + + Fine-tunes the Helsinki-NLP ``opus-mt-en-fr`` checkpoint, a MarianMT + seq2seq model trained on parallel English-French corpora from the OPUS + collection. + + References + ---------- + - [1] https://huggingface.co/Helsinki-NLP/opus-mt-en-fr + - [2] https://opus.nlpl.eu/ + """ + + MODEL_NAME: str = "Helsinki-NLP/opus-mt-en-fr" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_opus-mt-en-fr" + SCHEMA = OpusMtEnFrTransformerSchema + DISPLAY_NAME: str = MultilingualString( + en="Opus MT En-Fr Transformer", + es="Transformer Opus MT En-Fr", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Pre-trained transformer for English-French translation. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "Transformer pre-entrenado para traducción inglés-francés. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#1976D2" + ICON: str = "Translate" diff --git a/DashAI/back/models/hugging_face/opus_mt_en_pt_transformer.py b/DashAI/back/models/hugging_face/opus_mt_en_pt_transformer.py new file mode 100644 index 000000000..9613775d1 --- /dev/null +++ b/DashAI/back/models/hugging_face/opus_mt_en_pt_transformer.py @@ -0,0 +1,47 @@ +"""OpusMtEnPtTransformer model for English-to-Portuguese translation.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_opus_mt_transformer import ( + OpusMtTransformerMixin, +) +from DashAI.back.models.hugging_face.opus_mt_en_es_transformer import ( + OpusMtEnESTransformerSchema, +) + + +class OpusMtEnPtTransformerSchema(OpusMtEnESTransformerSchema): + """Schema for the English-to-Portuguese Opus-MT model.""" + + +class OpusMtEnPtTransformer(OpusMtTransformerMixin): + """Pre-trained transformer for English-to-Portuguese translation. + + Fine-tunes the Helsinki-NLP ``opus-mt-en-pt`` checkpoint, a MarianMT + seq2seq model trained on parallel English-Portuguese corpora from the OPUS + collection. + + References + ---------- + - [1] https://huggingface.co/Helsinki-NLP/opus-mt-en-pt + - [2] https://opus.nlpl.eu/ + """ + + MODEL_NAME: str = "Helsinki-NLP/opus-mt-en-pt" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_opus-mt-en-pt" + SCHEMA = OpusMtEnPtTransformerSchema + DISPLAY_NAME: str = MultilingualString( + en="Opus MT En-Pt Transformer", + es="Transformer Opus MT En-Pt", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Pre-trained transformer for English-Portuguese translation. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "Transformer pre-entrenado para traducción inglés-portugués. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#2E7D32" + ICON: str = "Translate" diff --git a/DashAI/back/models/hugging_face/opus_mt_es_en_transformer.py b/DashAI/back/models/hugging_face/opus_mt_es_en_transformer.py index b93050cd7..8cf69ac80 100644 --- a/DashAI/back/models/hugging_face/opus_mt_es_en_transformer.py +++ b/DashAI/back/models/hugging_face/opus_mt_es_en_transformer.py @@ -1,42 +1,27 @@ -"""OpusMtEsENTransformer model for spanish-english translation DashAI implementation.""" - -import shutil -import tempfile -from pathlib import Path -from typing import TYPE_CHECKING, List, Optional, Union - -from sklearn.exceptions import NotFittedError +"""OpusMtEsENTransformer model for Spanish-to-English translation.""" from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_opus_mt_transformer import ( + OpusMtTransformerMixin, +) from DashAI.back.models.hugging_face.opus_mt_en_es_transformer import ( OpusMtEnESTransformerSchema, ) -from DashAI.back.models.translation_model import TranslationModel -from DashAI.back.models.utils import GPU_OR_CPU_PLACEHOLDER - -if TYPE_CHECKING: - from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset class OpusMtEsENTransformerSchema(OpusMtEnESTransformerSchema): - """opus-mt-es-en is a transformer pre-trained model that allows translation of - texts from Spanish to English. The implementation is based on the Helsinki-NLP - opus-mt-es-en checkpoint, which uses the MarianMT architecture and was trained - on parallel corpora from the OPUS collection. + """Schema for the Spanish-to-English Opus-MT model. + + Inherits all fields from ``OpusMtEnESTransformerSchema``. """ -class OpusMtEsENTransformer(TranslationModel): +class OpusMtEsENTransformer(OpusMtTransformerMixin): """Pre-trained transformer for Spanish-to-English translation. - This model fine-tunes the Helsinki-NLP ``opus-mt-es-en`` checkpoint, which is - based on the MarianMT sequence-to-sequence architecture. The base model was - trained on parallel Spanish-English corpora from the OPUS collection and supports - direct translation without intermediate pivot languages. - - Fine-tuning is performed with the HuggingFace ``Seq2SeqTrainer`` using the AdamW - optimizer. Training and validation metrics are logged at configurable epoch and - step intervals via a custom ``MetricsCallback``. + Fine-tunes the Helsinki-NLP ``opus-mt-es-en`` checkpoint, a MarianMT + seq2seq model trained on parallel Spanish-English corpora from the OPUS + collection. Supports direct translation without pivot languages. References ---------- @@ -44,344 +29,22 @@ class OpusMtEsENTransformer(TranslationModel): - [2] https://opus.nlpl.eu/ """ + MODEL_NAME: str = "Helsinki-NLP/opus-mt-es-en" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_opus-mt-es-en" SCHEMA = OpusMtEsENTransformerSchema DISPLAY_NAME: str = MultilingualString( en="Opus MT Es-En Transformer", es="Transformer Opus MT Es-En", ) DESCRIPTION: str = MultilingualString( - en="Pre-trained transformer for Spanish-English translation.", - es="Transformer pre-entrenado para traducción español-inglés.", + en=( + "Pre-trained transformer for Spanish-English translation. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "Transformer pre-entrenado para traducción español-inglés. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), ) COLOR: str = "#FF8A65" ICON: str = "Translate" - - def __init__(self, model=None, **kwargs): - """Initialize the transformer. - - Downloads the ``Helsinki-NLP/opus-mt-es-en`` tokenizer and, when - ``model`` is ``None``, the seq2seq model weights from HuggingFace. - When a pre-loaded model is supplied, the weights are reused directly - and ``fitted`` is set to ``True``. - - Parameters - ---------- - model : transformers.PreTrainedModel or None, optional - An already-loaded HuggingFace seq2seq model to reuse. If ``None``, - the ``Helsinki-NLP/opus-mt-es-en`` checkpoint is downloaded and - initialised. Default ``None``. - **kwargs : dict - Hyperparameters forwarded to ``validate_and_transform`` and used to - configure training (e.g. ``num_train_epochs``, ``batch_size``, - ``learning_rate``, ``weight_decay``, ``device``). - """ - kwargs = self.validate_and_transform(kwargs) - - from transformers import AutoTokenizer - - self.model_name = "Helsinki-NLP/opus-mt-es-en" - self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) - - self.training_args = { - "num_train_epochs": kwargs.get("num_train_epochs", 2), - "learning_rate": kwargs.get("learning_rate", 2e-5), - "weight_decay": kwargs.get("weight_decay", 0.01), - } - self.batch_size = kwargs.get("batch_size", 4) - self.device = kwargs.get("device") or GPU_OR_CPU_PLACEHOLDER - self.log_train_every_n_epochs = kwargs.get("log_train_every_n_epochs", 1) - self.log_train_every_n_steps = kwargs.get("log_train_every_n_steps", None) - self.log_validation_every_n_epochs = kwargs.get( - "log_validation_every_n_epochs", 1 - ) - self.log_validation_every_n_steps = kwargs.get( - "log_validation_every_n_steps", None - ) - - if model is None: - from transformers import AutoModelForSeq2SeqLM - - self.model = AutoModelForSeq2SeqLM.from_pretrained(self.model_name) - else: - self.model = model - - self.num_train_epochs = self.training_args.get("num_train_epochs", 2) - self.fitted = model is not None - - def tokenize_data( - self, x: "DashAIDataset", y: Optional["DashAIDataset"] = None - ) -> "DashAIDataset": - """Tokenize input and optional target datasets for seq2seq training. - - Each sample is tokenized with truncation and max-length padding to 512 - tokens. When ``y`` is provided, the target tokens are stored under the - ``labels`` key so the ``Seq2SeqTrainer`` can compute the loss directly. - - Parameters - ---------- - x : DashAIDataset - Source-language dataset. Only the first column is used. - y : DashAIDataset, optional - Target-language dataset. When provided, tokenized targets are added - as ``labels``. When ``None``, only ``input_ids`` and - ``attention_mask`` are returned (inference mode). - - Returns - ------- - DashAIDataset - Tokenized dataset with keys ``input_ids``, ``attention_mask``, and - optionally ``labels``. - """ - from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset - - is_y = bool(y) - if not y: - y = DashAIDataset.from_list([{"foo": 0}] * len(x)) - - dataset = [] - input_column_name = x.column_names[0] - output_column_name = y.column_names[0] if is_y else None - - for i, input_sample in enumerate(x): - tokenized_input = self.tokenizer( - input_sample[input_column_name], - truncation=True, - padding="max_length", - max_length=512, - ) - - sample = { - "input_ids": tokenized_input["input_ids"], - "attention_mask": tokenized_input["attention_mask"], - } - - if is_y: - output_sample = y[i] - tokenized_output = self.tokenizer( - output_sample[output_column_name], - truncation=True, - padding="max_length", - max_length=512, - ) - sample["labels"] = tokenized_output["input_ids"] - - dataset.append(sample) - - return DashAIDataset.from_list(dataset) - - def train( - self, - x_train: "DashAIDataset", - y_train: "DashAIDataset", - x_validation: "DashAIDataset" = None, - y_validation: "DashAIDataset" = None, - ) -> "OpusMtEsENTransformer": - """Fine-tune the opus-mt-es-en model on Spanish-English translation data. - - Parameters - ---------- - x_train : DashAIDataset - Input Spanish text features for training. - y_train : DashAIDataset - Target English translation labels for training. - x_validation : DashAIDataset, optional - Input Spanish text features for validation. Default ``None``. - y_validation : DashAIDataset, optional - Target English translation labels for validation. Default ``None``. - - Returns - ------- - OpusMtEsENTransformer - The fine-tuned model instance. - """ - from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments - - from DashAI.back.models.hugging_face.metrics_callback import MetricsCallback - - dataset = self.tokenize_data(x_train, y_train) - dataset.set_format("torch", columns=["input_ids", "attention_mask", "labels"]) - - has_validation_data = x_validation is not None and y_validation is not None - - output_root = Path("DashAI/back/user_models/temp_checkpoints_opus-mt-es-en") - output_root.mkdir(parents=True, exist_ok=True) - run_output_dir = tempfile.mkdtemp(prefix="opus_mt_es_en_", dir=str(output_root)) - - training_args = Seq2SeqTrainingArguments( - output_dir=run_output_dir, - save_steps=1, - save_total_limit=1, - per_device_train_batch_size=self.batch_size, - per_device_eval_batch_size=self.batch_size, - use_cpu=self.device.lower() != "gpu", - **self.training_args, - ) - - metrics_callback = MetricsCallback( - model_instance=self, - x_train=x_train, - y_train=y_train, - x_val=x_validation, - y_val=y_validation, - total_epochs=self.num_train_epochs, - log_training_every_n_epochs=self.log_train_every_n_epochs, - log_training_every_n_steps=self.log_train_every_n_steps, - log_val_every_n_epochs=( - self.log_validation_every_n_epochs if has_validation_data else None - ), - log_val_every_n_steps=( - self.log_validation_every_n_steps if has_validation_data else None - ), - ) - - trainer = Seq2SeqTrainer( - model=self.model, - args=training_args, - train_dataset=dataset, - callbacks=[metrics_callback], - ) - - self.fitted = True - try: - trainer.train() - finally: - shutil.rmtree(run_output_dir, ignore_errors=True) - - return self - - def predict(self, x_pred: "DashAIDataset") -> List: - """Translate Spanish source texts to English. - - Parameters - ---------- - x_pred : DashAIDataset - Source-language dataset. Only the first column is used. - - Returns - ------- - list of str - One translated string per input sample, in the same order as - ``x_pred``. - - Raises - ------ - sklearn.exceptions.NotFittedError - If the model has not been fine-tuned yet (``fitted`` is ``False``). - """ - if not self.fitted: - raise NotFittedError( - f"This {self.__class__.__name__} instance is not fitted yet. Call 'fit'" - " with appropriate arguments before using this estimator." - ) - - dataset = self.tokenize_data(x_pred) - dataset.set_format(type="torch", columns=["input_ids", "attention_mask"]) - - translations = [] - - for example in dataset: - inputs = { - k: v.unsqueeze(0).to(self.model.device) for k, v in example.items() - } - outputs = self.model.generate(**inputs) - translated_text = self.tokenizer.decode( - outputs[0], skip_special_tokens=True - ) - translations.append(translated_text) - - return translations - - def prepare_dataset( - self, dataset: "DashAIDataset", is_fit: bool = False - ) -> "DashAIDataset": - """Return the dataset unchanged. - - No pre-processing transformations are required for this model. The - method exists for compatibility with the DashAI model interface. - - Parameters - ---------- - dataset : DashAIDataset - The dataset to be prepared. - is_fit : bool, optional - Whether the call is made during fitting. Unused here. Default - ``False``. - - Returns - ------- - DashAIDataset - The original dataset, unmodified. - """ - return dataset - - def save(self, filename: Union[str, "Path"]) -> None: - """Store the fine-tuned model and its configuration to disk. - - Saves the model weights via ``save_pretrained`` and embeds the - hyperparameters (epochs, batch size, learning rate, etc.) into the - HuggingFace config so they can be restored by :meth:`load`. - - Parameters - ---------- - filename : str or Path - Directory path where the model files will be written. If a file - exists at that path it is removed and replaced by a directory. - """ - from transformers import AutoConfig - - save_dir = Path(filename) - if save_dir.exists() and save_dir.is_file(): - save_dir.unlink() - save_dir.mkdir(parents=True, exist_ok=True) - - self.model.save_pretrained(save_dir) - config = AutoConfig.from_pretrained(save_dir) - config.custom_params = { - "num_train_epochs": self.training_args.get("num_train_epochs"), - "batch_size": self.batch_size, - "learning_rate": self.training_args.get("learning_rate"), - "device": self.device, - "weight_decay": self.training_args.get("weight_decay"), - "fitted": self.fitted, - } - config.save_pretrained(save_dir) - - @classmethod - def load(cls, filename: Union[str, "Path"]): - """Restore an OpusMtEsENTransformer instance from disk. - - Reads the HuggingFace config to recover the custom hyperparameters - saved by :meth:`save`, then reconstructs the seq2seq model and wraps - it in a new :class:`OpusMtEsENTransformer` instance. - - Parameters - ---------- - filename : str or Path - Directory path from which the model files will be read. - - Returns - ------- - OpusMtEsENTransformer - The restored model instance with ``fitted`` set to the persisted - value. - """ - from transformers import AutoConfig, AutoModelForSeq2SeqLM - - model = AutoModelForSeq2SeqLM.from_pretrained(filename) - config = AutoConfig.from_pretrained(filename) - custom_params = getattr(config, "custom_params", {}) - - loaded_model = cls( - model=model, - num_train_epochs=custom_params.get("num_train_epochs"), - batch_size=custom_params.get("batch_size"), - learning_rate=custom_params.get("learning_rate"), - device=custom_params.get("device"), - weight_decay=custom_params.get("weight_decay"), - log_train_every_n_epochs=None, - log_train_every_n_steps=None, - log_validation_every_n_epochs=None, - log_validation_every_n_steps=None, - ) - loaded_model.fitted = custom_params.get("fitted", False) - return loaded_model diff --git a/DashAI/back/models/hugging_face/opus_mt_fr_en_transformer.py b/DashAI/back/models/hugging_face/opus_mt_fr_en_transformer.py new file mode 100644 index 000000000..419c2702a --- /dev/null +++ b/DashAI/back/models/hugging_face/opus_mt_fr_en_transformer.py @@ -0,0 +1,47 @@ +"""OpusMtFrEnTransformer model for French-to-English translation.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_opus_mt_transformer import ( + OpusMtTransformerMixin, +) +from DashAI.back.models.hugging_face.opus_mt_en_es_transformer import ( + OpusMtEnESTransformerSchema, +) + + +class OpusMtFrEnTransformerSchema(OpusMtEnESTransformerSchema): + """Schema for the French-to-English Opus-MT model.""" + + +class OpusMtFrEnTransformer(OpusMtTransformerMixin): + """Pre-trained transformer for French-to-English translation. + + Fine-tunes the Helsinki-NLP ``opus-mt-fr-en`` checkpoint, a MarianMT + seq2seq model trained on parallel French-English corpora from the OPUS + collection. + + References + ---------- + - [1] https://huggingface.co/Helsinki-NLP/opus-mt-fr-en + - [2] https://opus.nlpl.eu/ + """ + + MODEL_NAME: str = "Helsinki-NLP/opus-mt-fr-en" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_opus-mt-fr-en" + SCHEMA = OpusMtFrEnTransformerSchema + DISPLAY_NAME: str = MultilingualString( + en="Opus MT Fr-En Transformer", + es="Transformer Opus MT Fr-En", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Pre-trained transformer for French-English translation. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "Transformer pre-entrenado para traducción francés-inglés. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#0097A7" + ICON: str = "Translate" diff --git a/DashAI/back/models/hugging_face/roberta_transformer.py b/DashAI/back/models/hugging_face/roberta_transformer.py new file mode 100644 index 000000000..a2aa028d3 --- /dev/null +++ b/DashAI/back/models/hugging_face/roberta_transformer.py @@ -0,0 +1,45 @@ +"""DashAI implementation of RoBERTa model for English text classification.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_text_classification_transformer import ( + HuggingFaceTextClassificationTransformer, +) +from DashAI.back.models.hugging_face.distilbert_transformer import ( + DistilBertTransformerSchema, +) + + +class RobertaTransformer(HuggingFaceTextClassificationTransformer): + """Pre-trained RoBERTa model for English text classification. + + RoBERTa (Robustly Optimised BERT Pre-training Approach) improves upon BERT + by training longer with larger mini-batches, removing the next-sentence + prediction objective, and using dynamic masking. It achieves consistently + higher performance on NLP benchmarks than BERT. + + References + ---------- + - [1] Liu, Y. et al. (2019). "RoBERTa: A Robustly Optimized BERT Pretraining + Approach." arXiv:1907.11692. + - [2] https://huggingface.co/roberta-base + """ + + DISPLAY_NAME: str = MultilingualString( + en="RoBERTa Transformer", + es="Transformer RoBERTa", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Robustly optimised BERT for English text classification. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "BERT optimizado robustamente para clasificación de texto en inglés. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#E65100" + ICON: str = "SmartToy" + SCHEMA = DistilBertTransformerSchema + MODEL_NAME: str = "roberta-base" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_roberta" diff --git a/DashAI/back/models/hugging_face/t5_small_transformer.py b/DashAI/back/models/hugging_face/t5_small_transformer.py new file mode 100644 index 000000000..c0da3da66 --- /dev/null +++ b/DashAI/back/models/hugging_face/t5_small_transformer.py @@ -0,0 +1,306 @@ +"""T5SmallTransformer model for English-to-{German, French, Romanian} translation.""" + +import shutil +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, List, Optional, Union + +from sklearn.exceptions import NotFittedError + +from DashAI.back.core.schema_fields import enum_field, schema_field +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.opus_mt_en_es_transformer import ( + OpusMtEnESTransformerSchema, +) +from DashAI.back.models.translation_model import TranslationModel +from DashAI.back.models.utils import GPU_OR_CPU_PLACEHOLDER + +if TYPE_CHECKING: + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + +_T5_SUPPORTED_LANGUAGES = ["German", "French", "Romanian"] + + +class T5SmallTransformerSchema(OpusMtEnESTransformerSchema): + """Schema for the T5-small translation model. + + Extends the standard translation schema with a ``target_language`` field + restricted to the languages covered by T5's pre-training tasks: + German, French, and Romanian (source is always English). + """ + + target_language: schema_field( + enum_field(_T5_SUPPORTED_LANGUAGES), + placeholder="German", + description=MultilingualString( + en=( + "Target language for translation. " + "Supported: 'German', 'French', 'Romanian'. " + "T5-small translates from English only." + ), + es=( + "Idioma destino para la traducción. " + "Soportados: 'German', 'French', 'Romanian'. " + "T5-small traduce solo desde inglés." + ), + ), + alias=MultilingualString(en="Target language", es="Idioma destino"), + ) # type: ignore + + +class T5SmallTransformer(TranslationModel): + """T5-small seq2seq model for English-to-{German, French, Romanian} translation. + + Fine-tunes the ``t5-small`` checkpoint from Google. Translation direction is + controlled by a task prefix prepended to each source sentence, e.g. + ``"translate English to German: "``. + + Supported target languages: German, French, Romanian (T5 pre-training scope). + The source language is always English. + + References + ---------- + - [1] https://huggingface.co/t5-small + - [2] Raffel et al. (2020). "Exploring the Limits of Transfer Learning with a + Unified Text-to-Text Transformer." JMLR 2020. + """ + + SCHEMA = T5SmallTransformerSchema + DISPLAY_NAME: str = MultilingualString( + en="T5-Small Translation Transformer", + es="Transformer de Traducción T5-Small", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Google T5-small model for English-to-{German, French, Romanian} " + "translation using task prefixes. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "Modelo T5-small de Google para traducción inglés-{alemán, francés, " + "rumano} usando prefijos de tarea. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#00695C" + ICON: str = "Language" + + def __init__(self, model=None, **kwargs): + kwargs = self.validate_and_transform(kwargs) + + from transformers import AutoTokenizer + + self.model_name = "t5-small" + self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) + + self.target_language = kwargs.get("target_language", "German") + + self.training_args = { + "num_train_epochs": kwargs.get("num_train_epochs", 2), + "learning_rate": kwargs.get("learning_rate", 2e-5), + "weight_decay": kwargs.get("weight_decay", 0.01), + } + self.batch_size = kwargs.get("batch_size", 4) + self.device = kwargs.get("device") or GPU_OR_CPU_PLACEHOLDER + self.log_train_every_n_epochs = kwargs.get("log_train_every_n_epochs", 1) + self.log_train_every_n_steps = kwargs.get("log_train_every_n_steps", None) + self.log_validation_every_n_epochs = kwargs.get( + "log_validation_every_n_epochs", 1 + ) + self.log_validation_every_n_steps = kwargs.get( + "log_validation_every_n_steps", None + ) + + if model is None: + from transformers import AutoModelForSeq2SeqLM + + self.model = AutoModelForSeq2SeqLM.from_pretrained(self.model_name) + else: + self.model = model + + self.num_train_epochs = self.training_args.get("num_train_epochs", 2) + self.fitted = model is not None + + def _make_prefix(self) -> str: + return f"translate English to {self.target_language}: " + + def tokenize_data( + self, x: "DashAIDataset", y: Optional["DashAIDataset"] = None + ) -> "DashAIDataset": + """Prepend the T5 task prefix and tokenize source/target texts.""" + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + prefix = self._make_prefix() + is_y = bool(y) + if not y: + y = DashAIDataset.from_list([{"foo": 0}] * len(x)) + + dataset = [] + input_column_name = x.column_names[0] + output_column_name = y.column_names[0] if is_y else None + + for i, input_sample in enumerate(x): + text = prefix + input_sample[input_column_name] + tokenized_input = self.tokenizer( + text, + truncation=True, + padding="max_length", + max_length=512, + ) + sample = { + "input_ids": tokenized_input["input_ids"], + "attention_mask": tokenized_input["attention_mask"], + } + if is_y: + output_sample = y[i] + tokenized_output = self.tokenizer( + output_sample[output_column_name], + truncation=True, + padding="max_length", + max_length=512, + ) + sample["labels"] = tokenized_output["input_ids"] + dataset.append(sample) + + return DashAIDataset.from_list(dataset) + + def train( + self, + x_train: "DashAIDataset", + y_train: "DashAIDataset", + x_validation: "DashAIDataset" = None, + y_validation: "DashAIDataset" = None, + ): + """Fine-tune T5-small on the configured translation direction.""" + from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments + + from DashAI.back.models.hugging_face.metrics_callback import MetricsCallback + + dataset = self.tokenize_data(x_train, y_train) + dataset.set_format("torch", columns=["input_ids", "attention_mask", "labels"]) + + has_validation_data = x_validation is not None and y_validation is not None + + output_root = Path("DashAI/back/user_models/temp_checkpoints_t5_small") + output_root.mkdir(parents=True, exist_ok=True) + run_output_dir = tempfile.mkdtemp(prefix="t5_small_", dir=str(output_root)) + + training_args_obj = Seq2SeqTrainingArguments( + output_dir=run_output_dir, + save_steps=1, + save_total_limit=1, + per_device_train_batch_size=self.batch_size, + per_device_eval_batch_size=self.batch_size, + use_cpu=self.device.lower() != "gpu", + **self.training_args, + ) + + metrics_callback = MetricsCallback( + model_instance=self, + x_train=x_train, + y_train=y_train, + x_val=x_validation, + y_val=y_validation, + total_epochs=self.num_train_epochs, + log_training_every_n_epochs=self.log_train_every_n_epochs, + log_training_every_n_steps=self.log_train_every_n_steps, + log_val_every_n_epochs=( + self.log_validation_every_n_epochs if has_validation_data else None + ), + log_val_every_n_steps=( + self.log_validation_every_n_steps if has_validation_data else None + ), + ) + + trainer = Seq2SeqTrainer( + model=self.model, + args=training_args_obj, + train_dataset=dataset, + callbacks=[metrics_callback], + ) + + self.fitted = True + try: + trainer.train() + finally: + shutil.rmtree(run_output_dir, ignore_errors=True) + + return self + + def predict(self, x_pred: "DashAIDataset") -> List: + """Translate from English to the configured target language.""" + if not self.fitted: + raise NotFittedError( + f"This {self.__class__.__name__} instance is not fitted yet. " + "Call 'train' before using this estimator." + ) + + dataset = self.tokenize_data(x_pred) + dataset.set_format(type="torch", columns=["input_ids", "attention_mask"]) + + translations = [] + for example in dataset: + inputs = { + k: v.unsqueeze(0).to(self.model.device) for k, v in example.items() + } + outputs = self.model.generate(**inputs) + translated_text = self.tokenizer.decode( + outputs[0], skip_special_tokens=True + ) + translations.append(translated_text) + + return translations + + def prepare_dataset( + self, dataset: "DashAIDataset", is_fit: bool = False + ) -> "DashAIDataset": + """Return the dataset unchanged.""" + return dataset + + def save(self, filename: Union[str, "Path"]) -> None: + """Persist model weights and hyperparameters to disk.""" + from transformers import AutoConfig + + save_dir = Path(filename) + if save_dir.exists() and save_dir.is_file(): + save_dir.unlink() + save_dir.mkdir(parents=True, exist_ok=True) + + self.model.save_pretrained(save_dir) + config = AutoConfig.from_pretrained(save_dir) + config.custom_params = { + "num_train_epochs": self.training_args.get("num_train_epochs"), + "batch_size": self.batch_size, + "learning_rate": self.training_args.get("learning_rate"), + "device": self.device, + "weight_decay": self.training_args.get("weight_decay"), + "target_language": self.target_language, + "fitted": self.fitted, + } + config.save_pretrained(save_dir) + + @classmethod + def load(cls, filename: Union[str, "Path"]): + """Restore a T5SmallTransformer instance from disk.""" + from transformers import AutoConfig, AutoModelForSeq2SeqLM + + model = AutoModelForSeq2SeqLM.from_pretrained(filename) + config = AutoConfig.from_pretrained(filename) + custom_params = getattr(config, "custom_params", {}) + + loaded_model = cls( + model=model, + num_train_epochs=custom_params.get("num_train_epochs"), + batch_size=custom_params.get("batch_size"), + learning_rate=custom_params.get("learning_rate"), + device=custom_params.get("device"), + weight_decay=custom_params.get("weight_decay"), + target_language=custom_params.get("target_language", "German"), + log_train_every_n_epochs=None, + log_train_every_n_steps=None, + log_validation_every_n_epochs=None, + log_validation_every_n_steps=None, + ) + loaded_model.fitted = custom_params.get("fitted", False) + return loaded_model diff --git a/DashAI/back/models/hugging_face/xlm_roberta_transformer.py b/DashAI/back/models/hugging_face/xlm_roberta_transformer.py new file mode 100644 index 000000000..68257a344 --- /dev/null +++ b/DashAI/back/models/hugging_face/xlm_roberta_transformer.py @@ -0,0 +1,47 @@ +"""DashAI implementation of XLM-RoBERTa model for multilingual text classification.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_text_classification_transformer import ( + HuggingFaceTextClassificationTransformer, +) +from DashAI.back.models.hugging_face.distilbert_transformer import ( + DistilBertTransformerSchema, +) + + +class XlmRobertaTransformer(HuggingFaceTextClassificationTransformer): + """Pre-trained XLM-RoBERTa model for multilingual text classification. + + XLM-RoBERTa is a multilingual version of RoBERTa trained on 2.5 TB of + filtered CommonCrawl data covering 100 languages. It achieves strong + performance on cross-lingual classification without language-specific + fine-tuning. Requires the ``sentencepiece`` package for its tokeniser. + + References + ---------- + - [1] Conneau, A. et al. (2020). "Unsupervised Cross-lingual Representation + Learning at Scale." ACL 2020. + - [2] https://huggingface.co/xlm-roberta-base + """ + + DISPLAY_NAME: str = MultilingualString( + en="XLM-RoBERTa Transformer", + es="Transformer XLM-RoBERTa", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Multilingual RoBERTa for cross-lingual text classification " + "(100 languages). " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "RoBERTa multilingüe para clasificación de texto entre idiomas " + "(100 idiomas). " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#6A1B9A" + ICON: str = "Language" + SCHEMA = DistilBertTransformerSchema + MODEL_NAME: str = "xlm-roberta-base" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_xlm_roberta" diff --git a/DashAI/back/models/hugging_face/xlnet_transformer.py b/DashAI/back/models/hugging_face/xlnet_transformer.py new file mode 100644 index 000000000..bb3c4c451 --- /dev/null +++ b/DashAI/back/models/hugging_face/xlnet_transformer.py @@ -0,0 +1,45 @@ +"""DashAI implementation of XLNet model for English text classification.""" + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.hugging_face.base_text_classification_transformer import ( + HuggingFaceTextClassificationTransformer, +) +from DashAI.back.models.hugging_face.distilbert_transformer import ( + DistilBertTransformerSchema, +) + + +class XlnetTransformer(HuggingFaceTextClassificationTransformer): + """Pre-trained XLNet model for English text classification. + + XLNet is an autoregressive language model that maximises the expected + log-likelihood over all permutations of the factorisation order. Unlike BERT, + XLNet does not rely on a corrupted input and can model bidirectional context + without masking, often outperforming BERT on various NLP tasks. + + References + ---------- + - [1] Yang, Z. et al. (2019). "XLNet: Generalised Autoregressive Pretraining + for Language Understanding." NeurIPS 2019. + - [2] https://huggingface.co/xlnet-base-cased + """ + + DISPLAY_NAME: str = MultilingualString( + en="XLNet Transformer", + es="Transformer XLNet", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Autoregressive XLNet model for English text classification. " + "Downloads weights from Hugging Face on first use (internet required)." + ), + es=( + "Modelo XLNet autorregresivo para clasificación de texto en inglés. " + "Descarga pesos de Hugging Face en el primer uso (requiere internet)." + ), + ) + COLOR: str = "#37474F" + ICON: str = "AutoAwesome" + SCHEMA = DistilBertTransformerSchema + MODEL_NAME: str = "xlnet-base-cased" + TEMP_CHECKPOINT_DIR: str = "DashAI/back/user_models/temp_checkpoints_xlnet" diff --git a/DashAI/back/models/scikit_learn/adaboost_classifier.py b/DashAI/back/models/scikit_learn/adaboost_classifier.py new file mode 100644 index 000000000..dabe5b3ae --- /dev/null +++ b/DashAI/back/models/scikit_learn/adaboost_classifier.py @@ -0,0 +1,130 @@ +from sklearn.ensemble import AdaBoostClassifier as _AdaBoostClassifier + +from DashAI.back.core.schema_fields import ( + BaseSchema, + none_type, + optimizer_float_field, + optimizer_int_field, + schema_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.scikit_learn.sklearn_like_classifier import ( + SklearnLikeClassifier, +) +from DashAI.back.models.tabular_classification_model import TabularClassificationModel + + +class AdaBoostClassifierSchema(BaseSchema): + """Schema that configures the AdaBoost Classifier. + + AdaBoost (Adaptive Boosting) fits a sequence of weak learners on repeatedly + re-weighted versions of the training data. Misclassified samples receive + increased weight so that subsequent learners focus on harder examples. The + underlying implementation is ``sklearn.ensemble.AdaBoostClassifier``. + """ + + n_estimators: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 50, + "lower_bound": 10, + "upper_bound": 500, + }, + description=MultilingualString( + en=( + "The maximum number of estimators at which boosting is terminated. " + "In case of perfect fit, the learning procedure is stopped early." + ), + es=( + "El número máximo de estimadores en el que se termina el boosting. " + "En caso de ajuste perfecto, el procedimiento de aprendizaje se " + "detiene antes." + ), + ), + alias=MultilingualString(en="N estimators", es="N estimadores"), + ) # type: ignore + + learning_rate: schema_field( + optimizer_float_field(ge=0.01), + placeholder={ + "optimize": False, + "fixed_value": 1.0, + "lower_bound": 0.01, + "upper_bound": 2.0, + }, + description=MultilingualString( + en=( + "Weight applied to each classifier at each boosting iteration. " + "A higher learning rate increases the contribution of each classifier." + ), + es=( + "Peso aplicado a cada clasificador en cada iteración de boosting. " + "Una tasa de aprendizaje mayor incrementa la contribución de cada " + "clasificador." + ), + ), + alias=MultilingualString(en="Learning rate", es="Tasa de aprendizaje"), + ) # type: ignore + + random_state: schema_field( + union_type(optimizer_int_field(ge=0), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The seed of the pseudo-random number generator. Pass an int for " + "reproducible output, or None to not set a specific seed." + ), + es=( + "La semilla del generador de números pseudoaleatorios. Pase un int " + "para salida reproducible, o None para no fijar una semilla." + ), + ), + alias=MultilingualString(en="Random state", es="Estado aleatorio"), + ) # type: ignore + + +class AdaBoostClassifier( + TabularClassificationModel, SklearnLikeClassifier, _AdaBoostClassifier +): + """AdaBoost classifier that adapts to misclassified samples iteratively. + + AdaBoost fits a sequence of weak classifiers (decision stumps by default) on + re-weighted training data, giving more weight to misclassified examples at each + round. The final prediction is a weighted majority vote of all weak classifiers. + AdaBoost is sensitive to noisy data and outliers. + + Key hyperparameters include ``n_estimators``, ``learning_rate``, and + ``random_state``. The implementation wraps scikit-learn's + ``AdaBoostClassifier``. + + References + ---------- + - [1] Freund, Y. & Schapire, R.E. (1997). "A Decision-Theoretic Generalization + of On-Line Learning and an Application to Boosting." Journal of Computer + and System Sciences, 55(1), 119-139. + - [2] https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html + """ + + SCHEMA = AdaBoostClassifierSchema + DISPLAY_NAME: str = MultilingualString( + en="AdaBoost Classifier", + es="Clasificador AdaBoost", + ) + DESCRIPTION: str = MultilingualString( + en="Adaptive boosting that focuses on misclassified samples.", + es="Boosting adaptivo que se enfoca en muestras mal clasificadas.", + ) + COLOR: str = "#FFA726" + ICON: str = "Bolt" + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/adaboost_regression.py b/DashAI/back/models/scikit_learn/adaboost_regression.py new file mode 100644 index 000000000..56fae8664 --- /dev/null +++ b/DashAI/back/models/scikit_learn/adaboost_regression.py @@ -0,0 +1,137 @@ +from sklearn.ensemble import AdaBoostRegressor as _AdaBoostRegressor + +from DashAI.back.core.schema_fields import ( + BaseSchema, + enum_field, + none_type, + optimizer_float_field, + optimizer_int_field, + schema_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.regression_model import RegressionModel +from DashAI.back.models.scikit_learn.sklearn_like_regressor import SklearnLikeRegressor + + +class AdaBoostRegressionSchema(BaseSchema): + """Schema that configures the AdaBoost Regressor. + + AdaBoost Regressor fits a sequence of weak regressors on re-weighted versions + of the training data, concentrating on samples with high residuals. The + underlying implementation is ``sklearn.ensemble.AdaBoostRegressor``. + """ + + n_estimators: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 50, + "lower_bound": 10, + "upper_bound": 500, + }, + description=MultilingualString( + en=( + "The maximum number of estimators at which boosting is terminated. " + "In case of perfect fit, the learning procedure stops early." + ), + es=( + "El número máximo de estimadores en el que se termina el boosting. " + "En caso de ajuste perfecto, el procedimiento se detiene antes." + ), + ), + alias=MultilingualString(en="N estimators", es="N estimadores"), + ) # type: ignore + + learning_rate: schema_field( + optimizer_float_field(ge=0.01), + placeholder={ + "optimize": False, + "fixed_value": 1.0, + "lower_bound": 0.01, + "upper_bound": 2.0, + }, + description=MultilingualString( + en=( + "Weight applied to each regressor at each boosting iteration. " + "There is a trade-off between learning_rate and n_estimators." + ), + es=( + "Peso aplicado a cada regresor en cada iteración de boosting. " + "Existe un trade-off entre learning_rate y n_estimators." + ), + ), + alias=MultilingualString(en="Learning rate", es="Tasa de aprendizaje"), + ) # type: ignore + + loss: schema_field( + enum_field(enum=["linear", "square", "exponential"]), + placeholder="linear", + description=MultilingualString( + en=( + "The loss function to use when updating the weights after each " + "boosting iteration." + ), + es=( + "La función de pérdida a usar al actualizar los pesos después de " + "cada iteración de boosting." + ), + ), + alias=MultilingualString(en="Loss", es="Pérdida"), + ) # type: ignore + + random_state: schema_field( + union_type(optimizer_int_field(ge=0), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The seed of the pseudo-random number generator. Pass an int for " + "reproducible output, or None to not set a specific seed." + ), + es=( + "La semilla del generador de números pseudoaleatorios. Pase un int " + "para salida reproducible, o None para no fijar una semilla." + ), + ), + alias=MultilingualString(en="Random state", es="Estado aleatorio"), + ) # type: ignore + + +class AdaBoostRegression(RegressionModel, SklearnLikeRegressor, _AdaBoostRegressor): + """AdaBoost regressor that focuses on samples with high prediction errors. + + AdaBoostRegressor fits weak regressors (decision stumps by default) sequentially + on re-weighted training data, assigning higher weights to samples with larger + errors. The final prediction is a weighted median of all weak regressors. + + Key hyperparameters include ``n_estimators``, ``learning_rate``, and ``loss``. + The implementation wraps scikit-learn's ``AdaBoostRegressor``. + + References + ---------- + - [1] Drucker, H. (1997). "Improving Regressors using Boosting Techniques." + ICML, 107-115. + - [2] https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostRegressor.html + """ + + SCHEMA = AdaBoostRegressionSchema + DISPLAY_NAME: str = MultilingualString( + en="AdaBoost Regression", + es="Regresión AdaBoost", + ) + DESCRIPTION: str = MultilingualString( + en="Adaptive boosting that focuses on samples with large residuals.", + es="Boosting adaptivo que se enfoca en muestras con grandes residuos.", + ) + COLOR: str = "#FFA726" + ICON: str = "Bolt" + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/bagging_classifier.py b/DashAI/back/models/scikit_learn/bagging_classifier.py new file mode 100644 index 000000000..a1de4708f --- /dev/null +++ b/DashAI/back/models/scikit_learn/bagging_classifier.py @@ -0,0 +1,164 @@ +from sklearn.ensemble import BaggingClassifier as _BaggingClassifier + +from DashAI.back.core.schema_fields import ( + BaseSchema, + bool_field, + none_type, + optimizer_float_field, + optimizer_int_field, + schema_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.scikit_learn.sklearn_like_classifier import ( + SklearnLikeClassifier, +) +from DashAI.back.models.tabular_classification_model import TabularClassificationModel + + +class BaggingClassifierSchema(BaseSchema): + """Schema that configures the Bagging Classifier. + + Bagging (Bootstrap Aggregating) fits base classifiers on random subsets of the + training data (drawn with replacement) and aggregates their predictions by + majority vote. It reduces variance and helps avoid overfitting. The underlying + implementation is ``sklearn.ensemble.BaggingClassifier``. + """ + + n_estimators: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 10, + "lower_bound": 5, + "upper_bound": 100, + }, + description=MultilingualString( + en="The number of base estimators in the ensemble.", + es="El número de estimadores base en el conjunto.", + ), + alias=MultilingualString(en="N estimators", es="N estimadores"), + ) # type: ignore + + max_samples: schema_field( + optimizer_float_field(gt=0.0, le=1.0), + placeholder={ + "optimize": False, + "fixed_value": 1.0, + "lower_bound": 0.1, + "upper_bound": 1.0, + }, + description=MultilingualString( + en=( + "Fraction of training samples drawn for each base estimator " + "(0 < max_samples ≤ 1.0)." + ), + es=( + "Fracción de muestras de entrenamiento para cada estimador base " + "(0 < max_samples ≤ 1.0)." + ), + ), + alias=MultilingualString(en="Max samples", es="Máximas muestras"), + ) # type: ignore + + max_features: schema_field( + optimizer_float_field(gt=0.0, le=1.0), + placeholder={ + "optimize": False, + "fixed_value": 1.0, + "lower_bound": 0.1, + "upper_bound": 1.0, + }, + description=MultilingualString( + en=( + "Fraction of features drawn for each base estimator " + "(0 < max_features ≤ 1.0)." + ), + es=( + "Fracción de características para cada estimador base " + "(0 < max_features ≤ 1.0)." + ), + ), + alias=MultilingualString(en="Max features", es="Máximas características"), + ) # type: ignore + + bootstrap: schema_field( + bool_field(), + placeholder=True, + description=MultilingualString( + en="Whether to draw samples with replacement.", + es="Si se extraen muestras con reemplazo.", + ), + alias=MultilingualString(en="Bootstrap", es="Bootstrap"), + ) # type: ignore + + bootstrap_features: schema_field( + bool_field(), + placeholder=False, + description=MultilingualString( + en="Whether to draw features with replacement.", + es="Si se extraen características con reemplazo.", + ), + alias=MultilingualString( + en="Bootstrap features", es="Bootstrap características" + ), + ) # type: ignore + + random_state: schema_field( + union_type(optimizer_int_field(ge=0), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The seed of the pseudo-random number generator. Pass an int for " + "reproducible output, or None to not set a specific seed." + ), + es=( + "La semilla del generador de números pseudoaleatorios. Pase un int " + "para salida reproducible, o None para no fijar una semilla." + ), + ), + alias=MultilingualString(en="Random state", es="Estado aleatorio"), + ) # type: ignore + + +class BaggingClassifier( + TabularClassificationModel, SklearnLikeClassifier, _BaggingClassifier +): + """Bagging classifier that aggregates predictions from bootstrap subsets. + + Bagging builds multiple base classifiers, each trained on a bootstrap sample of + the training data. The final class prediction is determined by majority voting. + Bagging is particularly effective with high-variance, low-bias estimators such + as decision trees. + + Key hyperparameters include ``n_estimators``, ``max_samples``, + ``max_features``, and ``bootstrap``. The implementation wraps scikit-learn's + ``BaggingClassifier``. + + References + ---------- + - [1] Breiman, L. (1996). "Bagging Predictors." Machine Learning, 24(2), 123-140. + - [2] https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.BaggingClassifier.html + """ + + SCHEMA = BaggingClassifierSchema + DISPLAY_NAME: str = MultilingualString( + en="Bagging Classifier", + es="Clasificador Bagging", + ) + DESCRIPTION: str = MultilingualString( + en="Bootstrap aggregating ensemble to reduce variance.", + es="Conjunto de bootstrap aggregating para reducir la varianza.", + ) + COLOR: str = "#26C6DA" + ICON: str = "Inventory" + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/bayesian_ridge_regression.py b/DashAI/back/models/scikit_learn/bayesian_ridge_regression.py new file mode 100644 index 000000000..b0f7e124b --- /dev/null +++ b/DashAI/back/models/scikit_learn/bayesian_ridge_regression.py @@ -0,0 +1,173 @@ +from sklearn.linear_model import BayesianRidge as _BayesianRidge + +from DashAI.back.core.schema_fields import ( + BaseSchema, + bool_field, + optimizer_float_field, + optimizer_int_field, + schema_field, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.regression_model import RegressionModel +from DashAI.back.models.scikit_learn.sklearn_like_model import ( + CategoricalEncodingStrategy, +) +from DashAI.back.models.scikit_learn.sklearn_like_regressor import SklearnLikeRegressor + + +class BayesianRidgeRegressionSchema(BaseSchema): + """Schema that configures the Bayesian Ridge Regression model. + + Bayesian Ridge estimates the parameters of a regression model using Bayesian + inference. It includes regularisation parameters that are estimated from the + data rather than set by the user. The underlying implementation is + ``sklearn.linear_model.BayesianRidge``. + """ + + max_iter: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 300, + "lower_bound": 50, + "upper_bound": 1000, + }, + description=MultilingualString( + en="Maximum number of iterations over the complete dataset.", + es="Número máximo de iteraciones sobre el conjunto de datos completo.", + ), + alias=MultilingualString(en="Max iterations", es="Máximas iteraciones"), + ) # type: ignore + + tol: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 1e-3, + "lower_bound": 1e-6, + "upper_bound": 1e-1, + }, + description=MultilingualString( + en="Stop the algorithm if the weight update is smaller than tol.", + es=("Detener el algoritmo si la actualización de pesos es menor que tol."), + ), + alias=MultilingualString(en="Tolerance", es="Tolerancia"), + ) # type: ignore + + alpha_1: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 1e-6, + "lower_bound": 1e-10, + "upper_bound": 1e-2, + }, + description=MultilingualString( + en="Shape parameter for the Gamma distribution prior over alpha.", + es=("Parámetro de forma para la distribución Gamma previa sobre alfa."), + ), + alias=MultilingualString(en="Alpha 1", es="Alfa 1"), + ) # type: ignore + + alpha_2: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 1e-6, + "lower_bound": 1e-10, + "upper_bound": 1e-2, + }, + description=MultilingualString( + en="Rate parameter for the Gamma distribution prior over alpha.", + es=("Parámetro de tasa para la distribución Gamma previa sobre alfa."), + ), + alias=MultilingualString(en="Alpha 2", es="Alfa 2"), + ) # type: ignore + + lambda_1: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 1e-6, + "lower_bound": 1e-10, + "upper_bound": 1e-2, + }, + description=MultilingualString( + en="Shape parameter for the Gamma distribution prior over lambda.", + es=("Parámetro de forma para la distribución Gamma previa sobre lambda."), + ), + alias=MultilingualString(en="Lambda 1", es="Lambda 1"), + ) # type: ignore + + lambda_2: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 1e-6, + "lower_bound": 1e-10, + "upper_bound": 1e-2, + }, + description=MultilingualString( + en="Rate parameter for the Gamma distribution prior over lambda.", + es=("Parámetro de tasa para la distribución Gamma previa sobre lambda."), + ), + alias=MultilingualString(en="Lambda 2", es="Lambda 2"), + ) # type: ignore + + fit_intercept: schema_field( + bool_field(), + placeholder=True, + description=MultilingualString( + en=( + "Whether to calculate the intercept for this model. If False, " + "the data is expected to be already centred." + ), + es=( + "Si se calcula el intercepto para este modelo. Si es False, " + "se espera que los datos ya estén centrados." + ), + ), + alias=MultilingualString(en="Fit intercept", es="Ajustar intercepto"), + ) # type: ignore + + +class BayesianRidgeRegression(RegressionModel, SklearnLikeRegressor, _BayesianRidge): + """Bayesian Ridge regression with automatic regularisation estimation. + + BayesianRidge places Gamma priors over the regularisation parameters and + estimates them from the data using the Expectation-Maximisation algorithm. + This avoids the need for cross-validation to select ``alpha`` and provides + predictive uncertainty estimates. It tends to be robust to over-fitting. + + Key hyperparameters include ``max_iter``, ``tol``, and the Gamma prior + parameters ``alpha_1``, ``alpha_2``, ``lambda_1``, ``lambda_2``. The + implementation wraps scikit-learn's ``BayesianRidge``. + + References + ---------- + - [1] MacKay, D.J.C. (1992). "Bayesian Interpolation." Neural Computation, 4(3). + - [2] https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.BayesianRidge.html + """ + + SCHEMA = BayesianRidgeRegressionSchema + DISPLAY_NAME: str = MultilingualString( + en="Bayesian Ridge Regression", + es="Regresión Ridge Bayesiana", + ) + DESCRIPTION: str = MultilingualString( + en="Bayesian regression with automatic regularisation estimation.", + es="Regresión bayesiana con estimación automática de regularización.", + ) + COLOR: str = "#7E57C2" + ICON: str = "Psychology" + CATEGORICAL_ENCODING = CategoricalEncodingStrategy.ONE_HOT + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/decision_tree_regression.py b/DashAI/back/models/scikit_learn/decision_tree_regression.py new file mode 100644 index 000000000..c8a441bac --- /dev/null +++ b/DashAI/back/models/scikit_learn/decision_tree_regression.py @@ -0,0 +1,174 @@ +from sklearn.tree import DecisionTreeRegressor as _DecisionTreeRegressor + +from DashAI.back.core.schema_fields import ( + BaseSchema, + none_type, + optimizer_float_field, + optimizer_int_field, + schema_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.regression_model import RegressionModel +from DashAI.back.models.scikit_learn.sklearn_like_regressor import SklearnLikeRegressor + + +class DecisionTreeRegressionSchema(BaseSchema): + """Schema that configures the Decision Tree Regressor. + + The Decision Tree Regressor builds a tree by recursively splitting the feature + space to minimise MSE (or MAE) at each node. The underlying implementation is + ``sklearn.tree.DecisionTreeRegressor``. + """ + + max_depth: schema_field( + union_type(optimizer_int_field(ge=1), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The maximum depth of the tree. If None, nodes are expanded until " + "all leaves are pure or fewer than min_samples_split samples remain." + ), + es=( + "La profundidad máxima del árbol. Si es None, los nodos se expanden " + "hasta que todas las hojas sean puras o queden menos de " + "min_samples_split muestras." + ), + ), + alias=MultilingualString(en="Max depth", es="Profundidad máxima"), + ) # type: ignore + + min_samples_split: schema_field( + optimizer_int_field(ge=2), + placeholder={ + "optimize": False, + "fixed_value": 2, + "lower_bound": 2, + "upper_bound": 20, + }, + description=MultilingualString( + en="Minimum number of samples required to split an internal node.", + es="Número mínimo de muestras requeridas para dividir un nodo interno.", + ), + alias=MultilingualString( + en="Min samples split", es="Mínimas muestras de división" + ), + ) # type: ignore + + min_samples_leaf: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 1, + "lower_bound": 1, + "upper_bound": 20, + }, + description=MultilingualString( + en="Minimum number of samples required to be at a leaf node.", + es="Número mínimo de muestras requeridas para estar en una hoja.", + ), + alias=MultilingualString( + en="Min samples leaf", es="Mínimas muestras para hoja" + ), + ) # type: ignore + + max_leaf_nodes: schema_field( + union_type(optimizer_int_field(ge=2), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "Grow a tree with at most max_leaf_nodes in best-first fashion. " + "If None, unlimited leaf nodes." + ), + es=( + "Crecer un árbol con a lo sumo max_leaf_nodes de manera best-first. " + "Si es None, nodos hoja ilimitados." + ), + ), + alias=MultilingualString(en="Max leaf nodes", es="Máximos nodos hoja"), + ) # type: ignore + + min_impurity_decrease: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 0.0, + "lower_bound": 0.0, + "upper_bound": 0.5, + }, + description=MultilingualString( + en=( + "A node is split if the split induces a decrease of the impurity " + "greater than or equal to this value." + ), + es=( + "Un nodo se divide si la división induce una disminución de la " + "impureza mayor o igual a este valor." + ), + ), + alias=MultilingualString( + en="Min impurity decrease", es="Disminución mínima de impureza" + ), + ) # type: ignore + + random_state: schema_field( + union_type(optimizer_int_field(ge=0), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The seed of the pseudo-random number generator. Pass an int for " + "reproducible output, or None to not set a specific seed." + ), + es=( + "La semilla del generador de números pseudoaleatorios. Pase un int " + "para salida reproducible, o None para no fijar una semilla." + ), + ), + alias=MultilingualString(en="Random state", es="Estado aleatorio"), + ) # type: ignore + + +class DecisionTreeRegression( + RegressionModel, SklearnLikeRegressor, _DecisionTreeRegressor +): + """Decision tree regressor that recursively partitions the feature space. + + DecisionTreeRegressor builds a binary tree by choosing the split that most + reduces the MSE (default) at each internal node. Leaf nodes predict the mean + of the training targets in the region. Decision trees are fast, interpretable, + and require no feature scaling, but tend to overfit without pruning. + + Key hyperparameters include ``max_depth``, ``min_samples_split``, + ``min_samples_leaf``, and ``max_leaf_nodes``. The implementation wraps + scikit-learn's ``DecisionTreeRegressor``. + + References + ---------- + - [1] Breiman, L. et al. (1984). Classification and Regression Trees. Wadsworth. + - [2] https://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeRegressor.html + """ + + SCHEMA = DecisionTreeRegressionSchema + DISPLAY_NAME: str = MultilingualString( + en="Decision Tree Regression", + es="Regresión Árbol de Decisión", + ) + DESCRIPTION: str = MultilingualString( + en="Interpretable tree-based regressor that partitions the feature space.", + es=( + "Regresor basado en árbol interpretable que particiona el espacio " + "de características." + ), + ) + COLOR: str = "#66BB6A" + ICON: str = "AccountTree" + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/elastic_net_regression.py b/DashAI/back/models/scikit_learn/elastic_net_regression.py new file mode 100644 index 000000000..cf475d1ea --- /dev/null +++ b/DashAI/back/models/scikit_learn/elastic_net_regression.py @@ -0,0 +1,173 @@ +from sklearn.linear_model import ElasticNet as _ElasticNet + +from DashAI.back.core.schema_fields import ( + BaseSchema, + bool_field, + none_type, + optimizer_float_field, + optimizer_int_field, + schema_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.regression_model import RegressionModel +from DashAI.back.models.scikit_learn.sklearn_like_model import ( + CategoricalEncodingStrategy, +) +from DashAI.back.models.scikit_learn.sklearn_like_regressor import SklearnLikeRegressor + + +class ElasticNetRegressionSchema(BaseSchema): + """Schema that configures the Elastic Net Regression model. + + Elastic Net combines L1 and L2 penalties, inheriting Lasso's sparse solutions + and Ridge's grouping effect. The ``l1_ratio`` controls the balance between + both penalties. The underlying implementation is + ``sklearn.linear_model.ElasticNet``. + """ + + alpha: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 1.0, + "lower_bound": 0.0001, + "upper_bound": 10.0, + }, + description=MultilingualString( + en=( + "Regularisation strength multiplier. alpha=0 is OLS; " + "increasing alpha increases regularisation." + ), + es=( + "Multiplicador de fuerza de regularización. alpha=0 es MCO; " + "aumentar alpha incrementa la regularización." + ), + ), + alias=MultilingualString(en="Alpha", es="Alfa"), + ) # type: ignore + + l1_ratio: schema_field( + optimizer_float_field(ge=0.0, le=1.0), + placeholder={ + "optimize": False, + "fixed_value": 0.5, + "lower_bound": 0.0, + "upper_bound": 1.0, + }, + description=MultilingualString( + en=( + "The mixing parameter. l1_ratio=0 is pure Ridge; " + "l1_ratio=1 is pure Lasso." + ), + es=( + "El parámetro de mezcla. l1_ratio=0 es Ridge puro; " + "l1_ratio=1 es Lasso puro." + ), + ), + alias=MultilingualString(en="L1 ratio", es="Ratio L1"), + ) # type: ignore + + fit_intercept: schema_field( + bool_field(), + placeholder=True, + description=MultilingualString( + en=( + "Whether to calculate the intercept for this model. If False, " + "the data is expected to be already centred." + ), + es=( + "Si se calcula el intercepto para este modelo. Si es False, " + "se espera que los datos ya estén centrados." + ), + ), + alias=MultilingualString(en="Fit intercept", es="Ajustar intercepto"), + ) # type: ignore + + max_iter: schema_field( + optimizer_int_field(ge=100), + placeholder={ + "optimize": False, + "fixed_value": 1000, + "lower_bound": 100, + "upper_bound": 10000, + }, + description=MultilingualString( + en="The maximum number of iterations.", + es="El número máximo de iteraciones.", + ), + alias=MultilingualString(en="Max iterations", es="Máximas iteraciones"), + ) # type: ignore + + tol: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 1e-4, + "lower_bound": 1e-6, + "upper_bound": 1e-1, + }, + description=MultilingualString( + en="The tolerance for the optimisation.", + es="La tolerancia para la optimización.", + ), + alias=MultilingualString(en="Tolerance", es="Tolerancia"), + ) # type: ignore + + random_state: schema_field( + union_type(optimizer_int_field(ge=0), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The seed of the pseudo-random number generator. Pass an int for " + "reproducible output, or None to not set a specific seed." + ), + es=( + "La semilla del generador de números pseudoaleatorios. Pase un int " + "para salida reproducible, o None para no fijar una semilla." + ), + ), + alias=MultilingualString(en="Random state", es="Estado aleatorio"), + ) # type: ignore + + +class ElasticNetRegression(RegressionModel, SklearnLikeRegressor, _ElasticNet): + """Elastic Net regression combining L1 and L2 penalties. + + Elastic Net minimises ``||y - Xw||^2 / (2*n) + alpha * l1_ratio * ||w||_1 + + alpha * (1 - l1_ratio) * 0.5 * ||w||^2``. The blend of L1 and L2 penalties + overcomes Lasso's limitation with correlated features while still producing + sparse solutions. Useful when there are many correlated features. + + Key hyperparameters include ``alpha``, ``l1_ratio``, ``fit_intercept``, and + ``max_iter``. The implementation wraps scikit-learn's ``ElasticNet``. + + References + ---------- + - [1] Zou, H. & Hastie, T. (2005). "Regularization and Variable Selection + via the Elastic Net." JRSS-B, 67(2), 301-320. + - [2] https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.ElasticNet.html + """ + + SCHEMA = ElasticNetRegressionSchema + DISPLAY_NAME: str = MultilingualString( + en="Elastic Net Regression", + es="Regresión Elastic Net", + ) + DESCRIPTION: str = MultilingualString( + en="Linear regression combining L1 and L2 regularisation.", + es="Regresión lineal que combina regularización L1 y L2.", + ) + COLOR: str = "#26A69A" + ICON: str = "Hub" + CATEGORICAL_ENCODING = CategoricalEncodingStrategy.ONE_HOT + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/extra_trees_classifier.py b/DashAI/back/models/scikit_learn/extra_trees_classifier.py new file mode 100644 index 000000000..9f3879280 --- /dev/null +++ b/DashAI/back/models/scikit_learn/extra_trees_classifier.py @@ -0,0 +1,174 @@ +from sklearn.ensemble import ExtraTreesClassifier as _ExtraTreesClassifier + +from DashAI.back.core.schema_fields import ( + BaseSchema, + bool_field, + none_type, + optimizer_int_field, + schema_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.scikit_learn.sklearn_like_classifier import ( + SklearnLikeClassifier, +) +from DashAI.back.models.tabular_classification_model import TabularClassificationModel + + +class ExtraTreesClassifierSchema(BaseSchema): + """Schema that configures the Extra-Trees Classifier. + + Extra-Trees (Extremely Randomised Trees) builds an ensemble of decision trees + with fully random feature thresholds, which further reduces variance at the + cost of a slightly higher bias compared to Random Forests. The underlying + implementation is ``sklearn.ensemble.ExtraTreesClassifier``. + """ + + n_estimators: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 100, + "lower_bound": 50, + "upper_bound": 500, + }, + description=MultilingualString( + en="The number of trees in the forest.", + es="El número de árboles en el bosque.", + ), + alias=MultilingualString(en="N estimators", es="N estimadores"), + ) # type: ignore + + max_depth: schema_field( + union_type(optimizer_int_field(ge=1), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The maximum depth of the tree. If None, nodes are expanded until " + "all leaves are pure or contain fewer than min_samples_split samples." + ), + es=( + "La profundidad máxima del árbol. Si es None, los nodos se expanden " + "hasta que todas las hojas sean puras o tengan menos de " + "min_samples_split muestras." + ), + ), + alias=MultilingualString(en="Max depth", es="Profundidad máxima"), + ) # type: ignore + + min_samples_split: schema_field( + optimizer_int_field(ge=2), + placeholder={ + "optimize": False, + "fixed_value": 2, + "lower_bound": 2, + "upper_bound": 10, + }, + description=MultilingualString( + en="The minimum number of samples required to split an internal node.", + es="El número mínimo de muestras requeridas para dividir un nodo interno.", + ), + alias=MultilingualString( + en="Min samples split", es="Mínimas muestras de división" + ), + ) # type: ignore + + min_samples_leaf: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 1, + "lower_bound": 1, + "upper_bound": 10, + }, + description=MultilingualString( + en="The minimum number of samples required to be at a leaf node.", + es="El número mínimo de muestras requeridas para estar en una hoja.", + ), + alias=MultilingualString( + en="Min samples leaf", es="Mínimas muestras para hoja" + ), + ) # type: ignore + + bootstrap: schema_field( + bool_field(), + placeholder=False, + description=MultilingualString( + en=( + "Whether bootstrap samples are used when building trees. " + "If False, the whole dataset is used for each tree." + ), + es=( + "Si se usan muestras bootstrap al construir los árboles. " + "Si es False, se usa todo el conjunto de datos para cada árbol." + ), + ), + alias=MultilingualString(en="Bootstrap", es="Bootstrap"), + ) # type: ignore + + random_state: schema_field( + union_type(optimizer_int_field(ge=0), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The seed of the pseudo-random number generator. Pass an int for " + "reproducible output, or None to not set a specific seed." + ), + es=( + "La semilla del generador de números pseudoaleatorios. Pase un int " + "para salida reproducible, o None para no fijar una semilla." + ), + ), + alias=MultilingualString(en="Random state", es="Estado aleatorio"), + ) # type: ignore + + +class ExtraTreesClassifier( + TabularClassificationModel, SklearnLikeClassifier, _ExtraTreesClassifier +): + """Extra-Trees classifier using fully randomised decision tree splits. + + Extremely Randomised Trees differ from Random Forests in how splits are chosen: + instead of searching for the best threshold per feature, Extra-Trees picks + thresholds at random. This introduces additional randomness that, combined with + bootstrap aggregation, further reduces variance. Extra-Trees are typically faster + to train than Random Forests. + + Key hyperparameters include ``n_estimators``, ``max_depth``, + ``min_samples_split``, ``min_samples_leaf``, and ``bootstrap``. The + implementation wraps scikit-learn's ``ExtraTreesClassifier``. + + References + ---------- + - [1] Geurts, P., Ernst, D. & Wehenkel, L. (2006). "Extremely randomized trees." + Machine Learning, 63(1), 3-42. https://doi.org/10.1007/s10994-006-6226-1 + - [2] https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html + """ + + SCHEMA = ExtraTreesClassifierSchema + DISPLAY_NAME: str = MultilingualString( + en="Extra-Trees Classifier", + es="Clasificador Extra-Trees", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Ensemble of fully randomised decision trees for fast, " + "low-variance classification." + ), + es=( + "Conjunto de árboles de decisión completamente aleatorizados " + "para clasificación rápida." + ), + ) + COLOR: str = "#66BB6A" + ICON: str = "Park" + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/extra_trees_regression.py b/DashAI/back/models/scikit_learn/extra_trees_regression.py new file mode 100644 index 000000000..6dec5bd42 --- /dev/null +++ b/DashAI/back/models/scikit_learn/extra_trees_regression.py @@ -0,0 +1,169 @@ +from sklearn.ensemble import ExtraTreesRegressor as _ExtraTreesRegressor + +from DashAI.back.core.schema_fields import ( + BaseSchema, + bool_field, + none_type, + optimizer_int_field, + schema_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.regression_model import RegressionModel +from DashAI.back.models.scikit_learn.sklearn_like_regressor import SklearnLikeRegressor + + +class ExtraTreesRegressionSchema(BaseSchema): + """Schema that configures the Extra-Trees Regressor. + + Extra-Trees (Extremely Randomised Trees) builds an ensemble of decision tree + regressors with fully random feature thresholds, further reducing variance + compared to Random Forests. The underlying implementation is + ``sklearn.ensemble.ExtraTreesRegressor``. + """ + + n_estimators: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 100, + "lower_bound": 50, + "upper_bound": 500, + }, + description=MultilingualString( + en="The number of trees in the forest.", + es="El número de árboles en el bosque.", + ), + alias=MultilingualString(en="N estimators", es="N estimadores"), + ) # type: ignore + + max_depth: schema_field( + union_type(optimizer_int_field(ge=1), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The maximum depth of the tree. If None, nodes are expanded until " + "all leaves are pure or fewer than min_samples_split samples remain." + ), + es=( + "La profundidad máxima del árbol. Si es None, los nodos se expanden " + "hasta que todas las hojas sean puras o queden menos de " + "min_samples_split muestras." + ), + ), + alias=MultilingualString(en="Max depth", es="Profundidad máxima"), + ) # type: ignore + + min_samples_split: schema_field( + optimizer_int_field(ge=2), + placeholder={ + "optimize": False, + "fixed_value": 2, + "lower_bound": 2, + "upper_bound": 10, + }, + description=MultilingualString( + en="Minimum number of samples required to split an internal node.", + es="Número mínimo de muestras requeridas para dividir un nodo interno.", + ), + alias=MultilingualString( + en="Min samples split", es="Mínimas muestras de división" + ), + ) # type: ignore + + min_samples_leaf: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 1, + "lower_bound": 1, + "upper_bound": 10, + }, + description=MultilingualString( + en="Minimum number of samples required to be at a leaf node.", + es="Número mínimo de muestras requeridas para estar en una hoja.", + ), + alias=MultilingualString( + en="Min samples leaf", es="Mínimas muestras para hoja" + ), + ) # type: ignore + + bootstrap: schema_field( + bool_field(), + placeholder=False, + description=MultilingualString( + en=( + "Whether bootstrap samples are used when building trees. " + "If False, the whole dataset is used for each tree." + ), + es=( + "Si se usan muestras bootstrap al construir los árboles. " + "Si es False, se usa todo el conjunto de datos para cada árbol." + ), + ), + alias=MultilingualString(en="Bootstrap", es="Bootstrap"), + ) # type: ignore + + random_state: schema_field( + union_type(optimizer_int_field(ge=0), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The seed of the pseudo-random number generator. Pass an int for " + "reproducible output, or None to not set a specific seed." + ), + es=( + "La semilla del generador de números pseudoaleatorios. Pase un int " + "para salida reproducible, o None para no fijar una semilla." + ), + ), + alias=MultilingualString(en="Random state", es="Estado aleatorio"), + ) # type: ignore + + +class ExtraTreesRegression(RegressionModel, SklearnLikeRegressor, _ExtraTreesRegressor): + """Extra-Trees regressor using fully randomised decision tree splits. + + Extremely Randomised Trees pick thresholds at random instead of searching for + the optimal split, introducing extra randomness that further reduces variance. + Combined with averaging over many trees, Extra-Trees can achieve very low + generalisation error on regression tasks while being fast to train. + + Key hyperparameters include ``n_estimators``, ``max_depth``, + ``min_samples_split``, and ``bootstrap``. The implementation wraps + scikit-learn's ``ExtraTreesRegressor``. + + References + ---------- + - [1] Geurts, P., Ernst, D. & Wehenkel, L. (2006). "Extremely randomized trees." + Machine Learning, 63(1), 3-42. + - [2] https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesRegressor.html + """ + + SCHEMA = ExtraTreesRegressionSchema + DISPLAY_NAME: str = MultilingualString( + en="Extra-Trees Regression", + es="Regresión Extra-Trees", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Ensemble of fully randomised decision trees for fast, " + "low-variance regression." + ), + es=( + "Conjunto de árboles de decisión completamente aleatorizados " + "para regresión rápida y de baja varianza." + ), + ) + COLOR: str = "#26A69A" + ICON: str = "Park" + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/gaussian_nb.py b/DashAI/back/models/scikit_learn/gaussian_nb.py new file mode 100644 index 000000000..3fc5c0e33 --- /dev/null +++ b/DashAI/back/models/scikit_learn/gaussian_nb.py @@ -0,0 +1,89 @@ +from sklearn.naive_bayes import GaussianNB as _GaussianNB + +from DashAI.back.core.schema_fields import ( + BaseSchema, + optimizer_float_field, + schema_field, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.scikit_learn.sklearn_like_classifier import ( + SklearnLikeClassifier, +) +from DashAI.back.models.tabular_classification_model import TabularClassificationModel + + +class GaussianNBSchema(BaseSchema): + """Schema that configures the Gaussian Naïve Bayes Classifier. + + Gaussian Naïve Bayes assumes that the continuous features in each class follow + a Gaussian (normal) distribution and applies Bayes' theorem with the strong + independence assumption between features. The underlying implementation is + ``sklearn.naive_bayes.GaussianNB``. + """ + + var_smoothing: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 1e-9, + "lower_bound": 1e-12, + "upper_bound": 1e-3, + }, + description=MultilingualString( + en=( + "Portion of the largest variance of all features that is added to " + "variances for calculation stability." + ), + es=( + "Porción de la mayor varianza de todas las características que se " + "añade a las varianzas para estabilidad del cálculo." + ), + ), + alias=MultilingualString(en="Var smoothing", es="Suavizado de varianza"), + ) # type: ignore + + +class GaussianNB(TabularClassificationModel, SklearnLikeClassifier, _GaussianNB): + """Gaussian Naïve Bayes classifier based on Bayes' theorem. + + GaussianNB models the likelihood of features as Gaussian distributions per + class. It estimates the mean and variance of each feature in each class from + the training data, then uses Bayes' theorem to compute the posterior class + probability. Despite the strong independence assumption, it often performs well + and is very fast. + + Key hyperparameter: ``var_smoothing`` (additive variance for numerical + stability). The implementation wraps scikit-learn's ``GaussianNB``. + + References + ---------- + - [1] https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.GaussianNB.html + """ + + SCHEMA = GaussianNBSchema + DISPLAY_NAME: str = MultilingualString( + en="Gaussian Naïve Bayes", + es="Naïve Bayes Gaussiano", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Probabilistic classifier based on Bayes' theorem " + "with Gaussian likelihoods." + ), + es=( + "Clasificador probabilístico basado en el teorema de Bayes " + "con verosimilitudes gaussianas." + ), + ) + COLOR: str = "#AB47BC" + ICON: str = "Functions" + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/gradient_boosting_classifier.py b/DashAI/back/models/scikit_learn/gradient_boosting_classifier.py new file mode 100644 index 000000000..601a1b9ff --- /dev/null +++ b/DashAI/back/models/scikit_learn/gradient_boosting_classifier.py @@ -0,0 +1,200 @@ +from sklearn.ensemble import GradientBoostingClassifier as _GradientBoostingClassifier + +from DashAI.back.core.schema_fields import ( + BaseSchema, + enum_field, + none_type, + optimizer_float_field, + optimizer_int_field, + schema_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.scikit_learn.sklearn_like_classifier import ( + SklearnLikeClassifier, +) +from DashAI.back.models.tabular_classification_model import TabularClassificationModel + + +class GradientBoostingClassifierSchema(BaseSchema): + """Schema that configures the Gradient Boosting Classifier. + + Gradient Boosting is a sequential ensemble classification method that fits a new + decision tree at each stage to the negative gradient of a differentiable loss + function. The underlying implementation is + ``sklearn.ensemble.GradientBoostingClassifier``. + """ + + loss: schema_field( + enum_field(enum=["log_loss", "exponential"]), + placeholder="log_loss", + description=MultilingualString( + en=( + "The loss function to be optimized. 'log_loss' refers to binomial and " + "multinomial deviance; 'exponential' is equivalent to AdaBoost." + ), + es=( + "La función de pérdida a optimizar. 'log_loss' refiere a la desviación " + "binomial y multinomial; 'exponential' es equivalente a AdaBoost." + ), + ), + alias=MultilingualString(en="Loss", es="Pérdida"), + ) # type: ignore + + learning_rate: schema_field( + optimizer_float_field(ge=0.01), + placeholder={ + "optimize": False, + "fixed_value": 0.1, + "lower_bound": 0.01, + "upper_bound": 1.0, + }, + description=MultilingualString( + en="Learning rate shrinks the contribution of each tree.", + es="La tasa de aprendizaje reduce la contribución de cada árbol.", + ), + alias=MultilingualString(en="Learning rate", es="Tasa de aprendizaje"), + ) # type: ignore + + n_estimators: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 100, + "lower_bound": 10, + "upper_bound": 500, + }, + description=MultilingualString( + en="The number of boosting stages to be run.", + es="El número de etapas de boosting a ejecutar.", + ), + alias=MultilingualString(en="N estimators", es="N estimadores"), + ) # type: ignore + + max_depth: schema_field( + union_type(optimizer_int_field(ge=1), none_type(int)), + placeholder=3, + description=MultilingualString( + en="Maximum depth of the individual regression estimators.", + es="Profundidad máxima de los estimadores de regresión individuales.", + ), + alias=MultilingualString(en="Max depth", es="Profundidad máxima"), + ) # type: ignore + + min_samples_split: schema_field( + optimizer_int_field(ge=2), + placeholder={ + "optimize": False, + "fixed_value": 2, + "lower_bound": 2, + "upper_bound": 20, + }, + description=MultilingualString( + en="The minimum number of samples required to split an internal node.", + es="El número mínimo de muestras requeridas para dividir un nodo interno.", + ), + alias=MultilingualString( + en="Min samples split", es="Mínimas muestras de división" + ), + ) # type: ignore + + min_samples_leaf: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 1, + "lower_bound": 1, + "upper_bound": 20, + }, + description=MultilingualString( + en="The minimum number of samples required to be at a leaf node.", + es="El número mínimo de muestras requeridas para estar en una hoja.", + ), + alias=MultilingualString( + en="Min samples leaf", es="Mínimas muestras para hoja" + ), + ) # type: ignore + + subsample: schema_field( + optimizer_float_field(ge=0.1, le=1.0), + placeholder={ + "optimize": False, + "fixed_value": 1.0, + "lower_bound": 0.1, + "upper_bound": 1.0, + }, + description=MultilingualString( + en=( + "The fraction of samples to be used for fitting each base learner. " + "Values less than 1.0 lead to stochastic gradient boosting." + ), + es=( + "La fracción de muestras usadas para ajustar cada aprendiz base. " + "Valores menores a 1.0 llevan al gradient boosting estocástico." + ), + ), + alias=MultilingualString(en="Subsample", es="Submuestreo"), + ) # type: ignore + + random_state: schema_field( + union_type(optimizer_int_field(ge=0), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The seed of the pseudo-random number generator. Pass an int for " + "reproducible output, or None to not set a specific seed." + ), + es=( + "La semilla del generador de números pseudoaleatorios. Pase un int " + "para salida reproducible, o None para no fijar una semilla." + ), + ), + alias=MultilingualString(en="Random state", es="Estado aleatorio"), + ) # type: ignore + + +class GradientBoostingClassifier( + TabularClassificationModel, SklearnLikeClassifier, _GradientBoostingClassifier +): + """Gradient boosting classifier that builds an ensemble of trees sequentially. + + Gradient Boosting builds an additive model stage by stage. At each stage a + shallow decision tree is fitted to the negative gradient of the chosen loss + function. A ``learning_rate`` shrinkage factor scales each tree's contribution, + trading slower learning for better generalisation. + + Key hyperparameters include ``n_estimators``, ``learning_rate``, ``max_depth``, + ``subsample``, and ``loss``. The implementation wraps scikit-learn's + ``GradientBoostingClassifier``. + + References + ---------- + - [1] Friedman, J.H. (2001). "Greedy function approximation: a gradient boosting + machine." Annals of Statistics, 29(5), 1189-1232. + - [2] https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html + """ + + SCHEMA = GradientBoostingClassifierSchema + DISPLAY_NAME: str = MultilingualString( + en="Gradient Boosting Classifier", + es="Clasificador Gradient Boosting", + ) + DESCRIPTION: str = MultilingualString( + en="Ensemble that builds trees sequentially to correct previous errors.", + es=( + "Conjunto que construye árboles secuencialmente para corregir " + "errores previos." + ), + ) + COLOR: str = "#4CAF50" + ICON: str = "AutoGraph" + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/hist_gradient_boosting_regression.py b/DashAI/back/models/scikit_learn/hist_gradient_boosting_regression.py new file mode 100644 index 000000000..d9a8c435c --- /dev/null +++ b/DashAI/back/models/scikit_learn/hist_gradient_boosting_regression.py @@ -0,0 +1,173 @@ +from sklearn.ensemble import ( + HistGradientBoostingRegressor as _HistGradientBoostingRegressor, +) + +from DashAI.back.core.schema_fields import ( + BaseSchema, + none_type, + optimizer_float_field, + optimizer_int_field, + schema_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.regression_model import RegressionModel +from DashAI.back.models.scikit_learn.sklearn_like_regressor import SklearnLikeRegressor + + +class HistGradientBoostingRegressionSchema(BaseSchema): + """Schema that configures the Histogram-based Gradient Boosting Regressor. + + Histogram-based Gradient Boosting is a fast sequential ensemble method that + bins features into histograms before building each tree, greatly reducing cost + on large datasets. The underlying implementation is + ``sklearn.ensemble.HistGradientBoostingRegressor``. + """ + + learning_rate: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 0.1, + "lower_bound": 0.01, + "upper_bound": 1.0, + }, + description=MultilingualString( + en=( + "The learning rate (shrinkage). Used as a multiplicative factor " + "for leaf values. Use 1 for no shrinkage." + ), + es=( + "La tasa de aprendizaje (shrinkage). Se usa como factor multiplicativo " + "para los valores de las hojas. Use 1 para no aplicar shrinkage." + ), + ), + alias=MultilingualString(en="Learning rate", es="Tasa de aprendizaje"), + ) # type: ignore + + max_iter: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 100, + "lower_bound": 50, + "upper_bound": 500, + }, + description=MultilingualString( + en="Maximum number of iterations (trees) of the boosting process.", + es="Número máximo de iteraciones (árboles) del proceso de boosting.", + ), + alias=MultilingualString(en="Max iterations", es="Máximas iteraciones"), + ) # type: ignore + + max_depth: schema_field( + union_type(optimizer_int_field(ge=1), none_type(int)), + placeholder=None, + description=MultilingualString( + en=("Maximum depth of each tree. If None, depth is not constrained."), + es=( + "Profundidad máxima de cada árbol. Si es None, la profundidad " + "no está restringida." + ), + ), + alias=MultilingualString(en="Max depth", es="Profundidad máxima"), + ) # type: ignore + + max_leaf_nodes: schema_field( + union_type(optimizer_int_field(ge=2), none_type(int)), + placeholder=31, + description=MultilingualString( + en=( + "Maximum number of leaves for each tree. Must be strictly greater " + "than 1. If None, no maximum limit." + ), + es=( + "Número máximo de hojas para cada árbol. Debe ser estrictamente " + "mayor que 1. Si es None, no hay límite." + ), + ), + alias=MultilingualString(en="Max leaf nodes", es="Máximos nodos hoja"), + ) # type: ignore + + min_samples_leaf: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 20, + "lower_bound": 1, + "upper_bound": 100, + }, + description=MultilingualString( + en="Minimum number of samples required to be at a leaf node.", + es="Número mínimo de muestras requeridas para estar en una hoja.", + ), + alias=MultilingualString( + en="Min samples leaf", es="Mínimas muestras para hoja" + ), + ) # type: ignore + + l2_regularization: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 0.0, + "lower_bound": 0.0, + "upper_bound": 1.0, + }, + description=MultilingualString( + en="The L2 regularisation parameter. Use 0 for no regularisation.", + es=( + "El parámetro de regularización L2. " + "Use 0 para no aplicar regularización." + ), + ), + alias=MultilingualString(en="L2 regularization", es="Regularización L2"), + ) # type: ignore + + +class HistGradientBoostingRegression( + RegressionModel, SklearnLikeRegressor, _HistGradientBoostingRegressor +): + """Histogram-based gradient boosting regressor for large datasets. + + This regressor discretises features into integer-valued bins before tree + construction. The histogram representation reduces candidate split points and + memory footprint, enabling efficient training on datasets with tens of thousands + of samples or more. It natively supports missing values and is inspired by + LightGBM. + + Key hyperparameters include ``learning_rate``, ``max_iter``, ``max_depth``, + ``max_leaf_nodes``, ``min_samples_leaf``, and ``l2_regularization``. The + implementation wraps scikit-learn's ``HistGradientBoostingRegressor``. + + References + ---------- + - [1] Ke, G. et al. (2017). "LightGBM: A Highly Efficient Gradient Boosting + Decision Tree." NeurIPS 30. + - [2] https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.HistGradientBoostingRegressor.html + """ + + SCHEMA = HistGradientBoostingRegressionSchema + DISPLAY_NAME: str = MultilingualString( + en="Histogram Gradient Boosting Regression", + es="Regresión Gradient Boosting con Histogramas", + ) + DESCRIPTION: str = MultilingualString( + en="Fast gradient boosting regression using histogram-based algorithms.", + es=( + "Regresión gradient boosting rápida usando algoritmos basados " + "en histogramas." + ), + ) + COLOR: str = "#9575CD" + ICON: str = "RocketLaunch" + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/k_neighbors_regression.py b/DashAI/back/models/scikit_learn/k_neighbors_regression.py new file mode 100644 index 000000000..56d379f73 --- /dev/null +++ b/DashAI/back/models/scikit_learn/k_neighbors_regression.py @@ -0,0 +1,143 @@ +from sklearn.neighbors import KNeighborsRegressor as _KNeighborsRegressor + +from DashAI.back.core.schema_fields import ( + BaseSchema, + enum_field, + optimizer_int_field, + schema_field, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.regression_model import RegressionModel +from DashAI.back.models.scikit_learn.sklearn_like_model import ( + CategoricalEncodingStrategy, +) +from DashAI.back.models.scikit_learn.sklearn_like_regressor import SklearnLikeRegressor + + +class KNeighborsRegressionSchema(BaseSchema): + """Schema that configures the K-Nearest Neighbours Regressor. + + KNeighborsRegressor predicts the target by averaging the targets of the + ``n_neighbors`` nearest training samples. The underlying implementation is + ``sklearn.neighbors.KNeighborsRegressor``. + """ + + n_neighbors: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 5, + "lower_bound": 1, + "upper_bound": 50, + }, + description=MultilingualString( + en="Number of neighbours to use for the prediction.", + es="Número de vecinos a usar para la predicción.", + ), + alias=MultilingualString(en="N neighbors", es="N vecinos"), + ) # type: ignore + + weights: schema_field( + enum_field(enum=["uniform", "distance"]), + placeholder="uniform", + description=MultilingualString( + en=( + "Weight function used in prediction. 'uniform' weights all " + "neighbours equally; 'distance' weights by inverse distance." + ), + es=( + "Función de pesos usada en la predicción. 'uniform' pondera igual " + "todos los vecinos; 'distance' pondera por distancia inversa." + ), + ), + alias=MultilingualString(en="Weights", es="Pesos"), + ) # type: ignore + + algorithm: schema_field( + enum_field(enum=["auto", "ball_tree", "kd_tree", "brute"]), + placeholder="auto", + description=MultilingualString( + en=( + "Algorithm used to compute nearest neighbours. 'auto' selects the " + "best based on the values passed to fit." + ), + es=( + "Algoritmo para computar los vecinos más cercanos. 'auto' selecciona " + "el mejor en función de los valores pasados a fit." + ), + ), + alias=MultilingualString(en="Algorithm", es="Algoritmo"), + ) # type: ignore + + leaf_size: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 30, + "lower_bound": 5, + "upper_bound": 100, + }, + description=MultilingualString( + en=( + "Leaf size passed to BallTree or KDTree. Affects query speed " + "and memory required to store the tree." + ), + es=( + "Tamaño de hoja pasado a BallTree o KDTree. Afecta la velocidad " + "de consulta y la memoria requerida para almacenar el árbol." + ), + ), + alias=MultilingualString(en="Leaf size", es="Tamaño de hoja"), + ) # type: ignore + + metric: schema_field( + enum_field(enum=["minkowski", "euclidean", "manhattan", "chebyshev"]), + placeholder="minkowski", + description=MultilingualString( + en="Distance metric to use for the neighbour search.", + es="Métrica de distancia para la búsqueda de vecinos.", + ), + alias=MultilingualString(en="Metric", es="Métrica"), + ) # type: ignore + + +class KNeighborsRegression(RegressionModel, SklearnLikeRegressor, _KNeighborsRegressor): + """K-Nearest Neighbours regressor that averages the targets of nearest samples. + + KNeighborsRegressor predicts the target value by computing the (weighted) + mean of the ``n_neighbors`` closest training points. It is a non-parametric + method: no training phase is needed, and predictions can capture non-linear + patterns. Performance degrades in high-dimensional spaces. + + Key hyperparameters include ``n_neighbors``, ``weights``, ``algorithm``, and + ``metric``. The implementation wraps scikit-learn's ``KNeighborsRegressor``. + + References + ---------- + - [1] https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsRegressor.html + """ + + SCHEMA = KNeighborsRegressionSchema + DISPLAY_NAME: str = MultilingualString( + en="K-Nearest Neighbours Regression", + es="Regresión K-Vecinos Más Cercanos", + ) + DESCRIPTION: str = MultilingualString( + en="Non-parametric regression that predicts by averaging nearest neighbours.", + es=( + "Regresión no paramétrica que predice promediando los vecinos más cercanos." + ), + ) + COLOR: str = "#FFA726" + ICON: str = "ScatterPlot" + CATEGORICAL_ENCODING = CategoricalEncodingStrategy.ONE_HOT + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/lasso_regression.py b/DashAI/back/models/scikit_learn/lasso_regression.py new file mode 100644 index 000000000..aea7e1421 --- /dev/null +++ b/DashAI/back/models/scikit_learn/lasso_regression.py @@ -0,0 +1,153 @@ +from sklearn.linear_model import Lasso as _Lasso + +from DashAI.back.core.schema_fields import ( + BaseSchema, + bool_field, + none_type, + optimizer_float_field, + optimizer_int_field, + schema_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.regression_model import RegressionModel +from DashAI.back.models.scikit_learn.sklearn_like_model import ( + CategoricalEncodingStrategy, +) +from DashAI.back.models.scikit_learn.sklearn_like_regressor import SklearnLikeRegressor + + +class LassoRegressionSchema(BaseSchema): + """Schema that configures the Lasso Regression model. + + Lasso (Least Absolute Shrinkage and Selection Operator) adds an L1 penalty on + the absolute values of coefficients, driving some of them exactly to zero, + which performs implicit feature selection. The underlying implementation is + ``sklearn.linear_model.Lasso``. + """ + + alpha: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 1.0, + "lower_bound": 0.0001, + "upper_bound": 10.0, + }, + description=MultilingualString( + en=( + "Regularisation strength. Larger values specify stronger " + "regularisation. alpha=0 is equivalent to OLS." + ), + es=( + "Fuerza de regularización. Valores más grandes especifican " + "regularización más fuerte. alpha=0 es equivalente a MCO." + ), + ), + alias=MultilingualString(en="Alpha", es="Alfa"), + ) # type: ignore + + fit_intercept: schema_field( + bool_field(), + placeholder=True, + description=MultilingualString( + en=( + "Whether to calculate the intercept for this model. If False, " + "the data is expected to be already centred." + ), + es=( + "Si se calcula el intercepto para este modelo. Si es False, " + "se espera que los datos ya estén centrados." + ), + ), + alias=MultilingualString(en="Fit intercept", es="Ajustar intercepto"), + ) # type: ignore + + max_iter: schema_field( + optimizer_int_field(ge=100), + placeholder={ + "optimize": False, + "fixed_value": 1000, + "lower_bound": 100, + "upper_bound": 10000, + }, + description=MultilingualString( + en="The maximum number of iterations.", + es="El número máximo de iteraciones.", + ), + alias=MultilingualString(en="Max iterations", es="Máximas iteraciones"), + ) # type: ignore + + tol: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 1e-4, + "lower_bound": 1e-6, + "upper_bound": 1e-1, + }, + description=MultilingualString( + en="The tolerance for the optimisation.", + es="La tolerancia para la optimización.", + ), + alias=MultilingualString(en="Tolerance", es="Tolerancia"), + ) # type: ignore + + random_state: schema_field( + union_type(optimizer_int_field(ge=0), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The seed of the pseudo-random number generator. Pass an int for " + "reproducible output, or None to not set a specific seed." + ), + es=( + "La semilla del generador de números pseudoaleatorios. Pase un int " + "para salida reproducible, o None para no fijar una semilla." + ), + ), + alias=MultilingualString(en="Random state", es="Estado aleatorio"), + ) # type: ignore + + +class LassoRegression(RegressionModel, SklearnLikeRegressor, _Lasso): + """Lasso regression with L1 regularisation for sparse coefficient solutions. + + Lasso minimises the OLS objective plus an L1 penalty + ``||y - Xw||^2 / (2*n) + alpha * ||w||_1``. The L1 term sets many + coefficients exactly to zero, performing automatic feature selection. Lasso is + particularly useful when there are many features but only a few are expected to + be relevant. + + Key hyperparameters include ``alpha``, ``fit_intercept``, ``max_iter``, and + ``tol``. The implementation wraps scikit-learn's ``Lasso``. + + References + ---------- + - [1] Tibshirani, R. (1996). "Regression Shrinkage and Selection via the Lasso." + Journal of the Royal Statistical Society B, 58(1), 267-288. + - [2] https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Lasso.html + """ + + SCHEMA = LassoRegressionSchema + DISPLAY_NAME: str = MultilingualString( + en="Lasso Regression", + es="Regresión Lasso", + ) + DESCRIPTION: str = MultilingualString( + en="Linear regression with L1 regularisation for feature selection.", + es="Regresión lineal con regularización L1 para selección de características.", + ) + COLOR: str = "#29B6F6" + ICON: str = "SelectAll" + CATEGORICAL_ENCODING = CategoricalEncodingStrategy.ONE_HOT + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/linear_svc_classifier.py b/DashAI/back/models/scikit_learn/linear_svc_classifier.py new file mode 100644 index 000000000..1bd00c135 --- /dev/null +++ b/DashAI/back/models/scikit_learn/linear_svc_classifier.py @@ -0,0 +1,251 @@ +from sklearn.svm import LinearSVC as _LinearSVC + +from DashAI.back.core.schema_fields import ( + BaseSchema, + bool_field, + enum_field, + none_type, + optimizer_float_field, + optimizer_int_field, + schema_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.scikit_learn.sklearn_like_classifier import ( + SklearnLikeClassifier, +) +from DashAI.back.models.scikit_learn.sklearn_like_model import ( + CategoricalEncodingStrategy, +) +from DashAI.back.models.tabular_classification_model import TabularClassificationModel + + +class LinearSVCClassifierSchema(BaseSchema): + """Schema that configures the Linear SVC Classifier. + + LinearSVC implements a linear Support Vector Classification trained with a + linear kernel. It is faster than kernel SVC for large datasets. Because + LinearSVC does not natively expose class probabilities, it is calibrated with + Platt scaling (CalibratedClassifierCV). The underlying implementation is + ``sklearn.svm.LinearSVC``. + """ + + C: schema_field( # noqa: N815 + optimizer_float_field(ge=1e-4), + placeholder={ + "optimize": False, + "fixed_value": 1.0, + "lower_bound": 0.01, + "upper_bound": 100.0, + }, + description=MultilingualString( + en=( + "Regularisation parameter. The strength of the regularisation is " + "inversely proportional to C. Must be strictly positive." + ), + es=( + "Parámetro de regularización. La fuerza de la regularización es " + "inversamente proporcional a C. Debe ser estrictamente positivo." + ), + ), + alias=MultilingualString(en="C", es="C"), + ) # type: ignore + + loss: schema_field( + enum_field(enum=["squared_hinge", "hinge"]), + placeholder="squared_hinge", + description=MultilingualString( + en=( + "Specifies the loss function. 'squared_hinge' is the default; " + "'hinge' is the standard SVM loss." + ), + es=( + "Especifica la función de pérdida. 'squared_hinge' es el " + "predeterminado; 'hinge' es la pérdida estándar de SVM." + ), + ), + alias=MultilingualString(en="Loss", es="Pérdida"), + ) # type: ignore + + max_iter: schema_field( + optimizer_int_field(ge=100), + placeholder={ + "optimize": False, + "fixed_value": 1000, + "lower_bound": 100, + "upper_bound": 10000, + }, + description=MultilingualString( + en="The maximum number of iterations to be run.", + es="El número máximo de iteraciones a ejecutar.", + ), + alias=MultilingualString(en="Max iterations", es="Máximas iteraciones"), + ) # type: ignore + + tol: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 1e-4, + "lower_bound": 1e-6, + "upper_bound": 1e-1, + }, + description=MultilingualString( + en="Tolerance for stopping criteria.", + es="Tolerancia para el criterio de parada.", + ), + alias=MultilingualString(en="Tolerance", es="Tolerancia"), + ) # type: ignore + + fit_intercept: schema_field( + bool_field(), + placeholder=True, + description=MultilingualString( + en=( + "Whether to calculate the intercept for this model. If False, " + "the data is expected to be already centred." + ), + es=( + "Si se calcula el intercepto para este modelo. Si es False, " + "se espera que los datos ya estén centrados." + ), + ), + alias=MultilingualString(en="Fit intercept", es="Ajustar intercepto"), + ) # type: ignore + + random_state: schema_field( + union_type(optimizer_int_field(ge=0), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The seed of the pseudo-random number generator. Pass an int for " + "reproducible output, or None to not set a specific seed." + ), + es=( + "La semilla del generador de números pseudoaleatorios. Pase un int " + "para salida reproducible, o None para no fijar una semilla." + ), + ), + alias=MultilingualString(en="Random state", es="Estado aleatorio"), + ) # type: ignore + + +class LinearSVCClassifier( + TabularClassificationModel, SklearnLikeClassifier, _LinearSVC +): + """Linear SVC classifier with Platt-scaling calibration for class probabilities. + + LinearSVC uses a linear kernel and is trained with coordinate descent, making + it considerably faster than kernel SVC on large datasets. Because LinearSVC + does not expose ``predict_proba`` natively, this wrapper fits a + ``CalibratedClassifierCV`` with sigmoid calibration so that probability + estimates are available to the DashAI evaluation pipeline. + + Key hyperparameters include ``C`` (regularisation), ``loss``, ``max_iter``, + and ``fit_intercept``. The implementation wraps scikit-learn's ``LinearSVC``. + + References + ---------- + - [1] https://scikit-learn.org/stable/modules/generated/sklearn.svm.LinearSVC.html + - [2] https://scikit-learn.org/stable/modules/calibration.html + """ + + SCHEMA = LinearSVCClassifierSchema + DISPLAY_NAME: str = MultilingualString( + en="Linear SVC", + es="SVC Lineal", + ) + DESCRIPTION: str = MultilingualString( + en="Fast linear support vector classifier with probability calibration.", + es=( + "Clasificador de vectores de soporte lineal rápido con " + "calibración de probabilidades." + ), + ) + COLOR: str = "#FF7043" + ICON: str = "LinearScale" + CATEGORICAL_ENCODING = CategoricalEncodingStrategy.ONE_HOT + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) + self._calibrated = None + + def __sklearn_is_fitted__(self) -> bool: + return self._calibrated is not None + + def train(self, x_train, y_train, x_validation=None, y_validation=None): + """Train using CalibratedClassifierCV to expose predict_proba. + + Parameters + ---------- + x_train : DashAIDataset + The input features for training. + y_train : DashAIDataset + The target labels for training. + x_validation : DashAIDataset, optional + Unused (sklearn models ignore validation split). + y_validation : DashAIDataset, optional + Unused. + + Returns + ------- + self + """ + from sklearn.calibration import CalibratedClassifierCV + from sklearn.svm import LinearSVC as _LinearSVCRaw + + x_processed = self.prepare_dataset(x_train, is_fit=True).to_pandas() + y_processed = self.prepare_output(y_train, is_fit=True).to_pandas() + y_arr = y_processed.values.ravel() + + params = { + k: getattr(self, k) + for k in ["C", "loss", "max_iter", "tol", "fit_intercept", "random_state"] + if hasattr(self, k) + } + base = _LinearSVCRaw(**params) + self._calibrated = CalibratedClassifierCV(base, method="sigmoid", cv=3) + self._calibrated.fit(x_processed, y_arr) + return self + + def predict(self, x_pred) -> "ndarray": # noqa: F821 + """Return class-probability matrix using the calibrated model. + + Parameters + ---------- + x_pred : DashAIDataset or pd.DataFrame + Input data. + + Returns + ------- + np.ndarray + Class probability matrix. + """ + import pandas as pd + + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + if isinstance(x_pred, DashAIDataset): + try: + x_prepared = self.prepare_dataset(x_pred, is_fit=False) + except ValueError: + x_prepared = x_pred + x_pred = x_prepared.to_pandas() + elif isinstance(x_pred, pd.DataFrame): + pass + + from sklearn.exceptions import NotFittedError + + if self._calibrated is None: + raise NotFittedError( + f"This {self.__class__.__name__} instance is not fitted yet. " + "Call 'train' with appropriate arguments before using this estimator." + ) + return self._calibrated.predict_proba(x_pred) diff --git a/DashAI/back/models/scikit_learn/mlp_classifier.py b/DashAI/back/models/scikit_learn/mlp_classifier.py new file mode 100644 index 000000000..bb7b4030b --- /dev/null +++ b/DashAI/back/models/scikit_learn/mlp_classifier.py @@ -0,0 +1,187 @@ +from sklearn.neural_network import MLPClassifier as _MLPClassifier + +from DashAI.back.core.schema_fields import ( + BaseSchema, + enum_field, + none_type, + optimizer_float_field, + optimizer_int_field, + schema_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.scikit_learn.sklearn_like_classifier import ( + SklearnLikeClassifier, +) +from DashAI.back.models.tabular_classification_model import TabularClassificationModel + + +class MLPClassifierSchema(BaseSchema): + """Schema that configures the MLP Classifier. + + The Multi-layer Perceptron Classifier is a feedforward neural network trained + with backpropagation. It supports multiple hidden layers and several activation + functions. The underlying implementation is + ``sklearn.neural_network.MLPClassifier``. + """ + + hidden_layer_size: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 100, + "lower_bound": 10, + "upper_bound": 500, + }, + description=MultilingualString( + en=( + "Number of neurons in the single hidden layer. The model uses one " + "hidden layer of this size." + ), + es=( + "Número de neuronas en la capa oculta única. El modelo utiliza una " + "capa oculta de este tamaño." + ), + ), + alias=MultilingualString(en="Hidden layer size", es="Tamaño de capa oculta"), + ) # type: ignore + + activation: schema_field( + enum_field(enum=["relu", "tanh", "logistic", "identity"]), + placeholder="relu", + description=MultilingualString( + en="Activation function for the hidden layer.", + es="Función de activación para la capa oculta.", + ), + alias=MultilingualString(en="Activation", es="Activación"), + ) # type: ignore + + solver: schema_field( + enum_field(enum=["adam", "lbfgs", "sgd"]), + placeholder="adam", + description=MultilingualString( + en=( + "The solver for weight optimisation. 'adam' works well for large " + "datasets; 'lbfgs' converges faster on small datasets; 'sgd' " + "requires more tuning." + ), + es=( + "El solucionador para la optimización de pesos. 'adam' funciona bien " + "para datasets grandes; 'lbfgs' converge más rápido en datasets " + "pequeños; 'sgd' requiere más ajuste." + ), + ), + alias=MultilingualString(en="Solver", es="Solucionador"), + ) # type: ignore + + alpha: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 0.0001, + "lower_bound": 1e-6, + "upper_bound": 1.0, + }, + description=MultilingualString( + en="L2 regularisation term (penalty parameter).", + es="Término de regularización L2 (parámetro de penalización).", + ), + alias=MultilingualString(en="Alpha", es="Alfa"), + ) # type: ignore + + learning_rate_init: schema_field( + optimizer_float_field(ge=1e-6), + placeholder={ + "optimize": False, + "fixed_value": 0.001, + "lower_bound": 1e-5, + "upper_bound": 0.1, + }, + description=MultilingualString( + en="The initial learning rate used for weight updates.", + es="La tasa de aprendizaje inicial usada para actualizar los pesos.", + ), + alias=MultilingualString( + en="Learning rate init", es="Tasa de aprendizaje inicial" + ), + ) # type: ignore + + max_iter: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 200, + "lower_bound": 50, + "upper_bound": 1000, + }, + description=MultilingualString( + en=( + "Maximum number of iterations. The solver iterates until " + "convergence or this limit." + ), + es=( + "Número máximo de iteraciones. El solucionador itera hasta " + "convergencia o este límite." + ), + ), + alias=MultilingualString(en="Max iterations", es="Máximas iteraciones"), + ) # type: ignore + + random_state: schema_field( + union_type(optimizer_int_field(ge=0), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The seed of the pseudo-random number generator. Pass an int for " + "reproducible output, or None to not set a specific seed." + ), + es=( + "La semilla del generador de números pseudoaleatorios. Pase un int " + "para salida reproducible, o None para no fijar una semilla." + ), + ), + alias=MultilingualString(en="Random state", es="Estado aleatorio"), + ) # type: ignore + + +class MLPClassifier(TabularClassificationModel, SklearnLikeClassifier, _MLPClassifier): + """Multi-layer Perceptron classifier trained with backpropagation. + + MLPClassifier is a fully-connected feedforward neural network. The network + uses a single hidden layer whose size is controlled by ``hidden_layer_size``. + Training uses backpropagation with the selected ``solver``. Supports ReLU, + tanh, logistic, and identity activations. + + Key hyperparameters include ``hidden_layer_size``, ``activation``, ``solver``, + ``alpha`` (L2 regularisation), ``learning_rate_init``, and ``max_iter``. The + implementation wraps scikit-learn's ``MLPClassifier``. + + References + ---------- + - [1] https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html + """ + + SCHEMA = MLPClassifierSchema + DISPLAY_NAME: str = MultilingualString( + en="MLP Classifier", + es="Clasificador MLP", + ) + DESCRIPTION: str = MultilingualString( + en="Multi-layer perceptron neural network for tabular classification.", + es="Red neuronal perceptrón multicapa para clasificación tabular.", + ) + COLOR: str = "#EF5350" + ICON: str = "AccountTree" + + def __init__(self, **kwargs) -> None: + """Initialise the model, converting hidden_layer_size to a tuple for sklearn. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values; ``hidden_layer_size`` is converted to a tuple + ``(hidden_layer_size,)`` before being forwarded to sklearn's MLPClassifier. + """ + hidden_size = kwargs.pop("hidden_layer_size", 100) + kwargs["hidden_layer_sizes"] = (hidden_size,) + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/sgd_classifier.py b/DashAI/back/models/scikit_learn/sgd_classifier.py new file mode 100644 index 000000000..45b6ec2eb --- /dev/null +++ b/DashAI/back/models/scikit_learn/sgd_classifier.py @@ -0,0 +1,270 @@ +from sklearn.linear_model import SGDClassifier as _SGDClassifier + +from DashAI.back.core.schema_fields import ( + BaseSchema, + enum_field, + none_type, + optimizer_float_field, + optimizer_int_field, + schema_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.scikit_learn.sklearn_like_classifier import ( + SklearnLikeClassifier, +) +from DashAI.back.models.scikit_learn.sklearn_like_model import ( + CategoricalEncodingStrategy, +) +from DashAI.back.models.tabular_classification_model import TabularClassificationModel + + +class SGDClassifierSchema(BaseSchema): + """Schema that configures the SGD Classifier. + + SGDClassifier implements regularised linear classifiers (SVM, logistic + regression, etc.) with Stochastic Gradient Descent training. The loss function + determines the model type. Because not all loss functions expose + ``predict_proba``, this wrapper always uses CalibratedClassifierCV for + consistent probability estimates. The underlying implementation is + ``sklearn.linear_model.SGDClassifier``. + """ + + loss: schema_field( + enum_field( + enum=[ + "hinge", + "log_loss", + "modified_huber", + "squared_hinge", + "perceptron", + ] + ), + placeholder="hinge", + description=MultilingualString( + en=( + "The loss function to use. 'hinge' gives a linear SVM; 'log_loss' " + "gives logistic regression; 'modified_huber' is smoother; " + "'squared_hinge' is like hinge but quadratically penalised; " + "'perceptron' is the linear loss used by the perceptron algorithm." + ), + es=( + "La función de pérdida a usar. 'hinge' da un SVM lineal; 'log_loss' " + "da regresión logística; 'modified_huber' es más suave; " + "'squared_hinge' es como hinge pero penalizado cuadráticamente; " + "'perceptron' es la pérdida lineal usada por el algoritmo perceptrón." + ), + ), + alias=MultilingualString(en="Loss", es="Pérdida"), + ) # type: ignore + + alpha: schema_field( + optimizer_float_field(ge=1e-6), + placeholder={ + "optimize": False, + "fixed_value": 0.0001, + "lower_bound": 1e-6, + "upper_bound": 1.0, + }, + description=MultilingualString( + en=( + "Regularisation parameter. Higher values result in stronger " + "regularisation." + ), + es=( + "Parámetro de regularización. Valores más altos resultan en " + "regularización más fuerte." + ), + ), + alias=MultilingualString(en="Alpha", es="Alfa"), + ) # type: ignore + + max_iter: schema_field( + optimizer_int_field(ge=1), + placeholder={ + "optimize": False, + "fixed_value": 1000, + "lower_bound": 100, + "upper_bound": 5000, + }, + description=MultilingualString( + en="The maximum number of passes over the training data (epochs).", + es="El número máximo de pasadas sobre los datos de entrenamiento (épocas).", + ), + alias=MultilingualString(en="Max iterations", es="Máximas iteraciones"), + ) # type: ignore + + tol: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 1e-3, + "lower_bound": 1e-6, + "upper_bound": 1e-1, + }, + description=MultilingualString( + en=("The stopping criterion. Training stops when loss > best_loss - tol."), + es=( + "El criterio de parada. El entrenamiento se detiene cuando " + "pérdida > mejor_pérdida - tol." + ), + ), + alias=MultilingualString(en="Tolerance", es="Tolerancia"), + ) # type: ignore + + learning_rate: schema_field( + enum_field(enum=["constant", "optimal", "invscaling", "adaptive"]), + placeholder="optimal", + description=MultilingualString( + en=( + "The learning rate schedule. 'optimal' uses 1/(alpha*(t+t0)); " + "'constant' keeps eta0 constant; 'invscaling' decreases as " + "1/t^power; 'adaptive' halves the rate when training stops." + ), + es=( + "El programa de tasa de aprendizaje. 'optimal' usa " + "1/(alpha*(t+t0)); 'constant' mantiene eta0 constante; " + "'invscaling' decrece como 1/t^power; 'adaptive' reduce a la " + "mitad la tasa cuando el entrenamiento deja de mejorar." + ), + ), + alias=MultilingualString(en="Learning rate", es="Tasa de aprendizaje"), + ) # type: ignore + + random_state: schema_field( + union_type(optimizer_int_field(ge=0), none_type(int)), + placeholder=None, + description=MultilingualString( + en=( + "The seed of the pseudo-random number generator. Pass an int for " + "reproducible output, or None to not set a specific seed." + ), + es=( + "La semilla del generador de números pseudoaleatorios. Pase un int " + "para salida reproducible, o None para no fijar una semilla." + ), + ), + alias=MultilingualString(en="Random state", es="Estado aleatorio"), + ) # type: ignore + + +class SGDClassifier(TabularClassificationModel, SklearnLikeClassifier, _SGDClassifier): + """SGD classifier with probability calibration for consistent predict_proba output. + + SGDClassifier supports multiple loss functions that correspond to different + linear models (SVM with 'hinge', logistic regression with 'log_loss', etc.). + Stochastic Gradient Descent allows efficient training on large datasets. Because + not all loss functions expose ``predict_proba`` natively, this wrapper + consistently calibrates the model with ``CalibratedClassifierCV``. + + Key hyperparameters include ``loss``, ``alpha``, ``max_iter``, ``tol``, and + ``learning_rate``. The implementation wraps scikit-learn's ``SGDClassifier``. + + References + ---------- + - [1] https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.SGDClassifier.html + """ + + SCHEMA = SGDClassifierSchema + DISPLAY_NAME: str = MultilingualString( + en="SGD Classifier", + es="Clasificador SGD", + ) + DESCRIPTION: str = MultilingualString( + en="Linear classifier trained with stochastic gradient descent.", + es="Clasificador lineal entrenado con descenso de gradiente estocástico.", + ) + COLOR: str = "#78909C" + ICON: str = "TrendingDown" + CATEGORICAL_ENCODING = CategoricalEncodingStrategy.ONE_HOT + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) + self._calibrated = None + + def __sklearn_is_fitted__(self) -> bool: + return self._calibrated is not None + + def train(self, x_train, y_train, x_validation=None, y_validation=None): + """Train using CalibratedClassifierCV to guarantee predict_proba availability. + + Parameters + ---------- + x_train : DashAIDataset + The input features for training. + y_train : DashAIDataset + The target labels for training. + x_validation : DashAIDataset, optional + Unused (sklearn models ignore validation split). + y_validation : DashAIDataset, optional + Unused. + + Returns + ------- + self + """ + from sklearn.calibration import CalibratedClassifierCV + from sklearn.linear_model import SGDClassifier as _SGDClassifierRaw + + x_processed = self.prepare_dataset(x_train, is_fit=True).to_pandas() + y_processed = self.prepare_output(y_train, is_fit=True).to_pandas() + y_arr = y_processed.values.ravel() + + params = { + k: getattr(self, k) + for k in [ + "loss", + "alpha", + "max_iter", + "tol", + "learning_rate", + "random_state", + ] + if hasattr(self, k) + } + base = _SGDClassifierRaw(**params) + self._calibrated = CalibratedClassifierCV(base, method="sigmoid", cv=3) + self._calibrated.fit(x_processed, y_arr) + return self + + def predict(self, x_pred) -> "ndarray": # noqa: F821 + """Return class-probability matrix using the calibrated model. + + Parameters + ---------- + x_pred : DashAIDataset or pd.DataFrame + Input data. + + Returns + ------- + np.ndarray + Class probability matrix. + """ + import pandas as pd + + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + if isinstance(x_pred, DashAIDataset): + try: + x_prepared = self.prepare_dataset(x_pred, is_fit=False) + except ValueError: + x_prepared = x_pred + x_pred = x_prepared.to_pandas() + elif isinstance(x_pred, pd.DataFrame): + pass + + from sklearn.exceptions import NotFittedError + + if self._calibrated is None: + raise NotFittedError( + f"This {self.__class__.__name__} instance is not fitted yet. " + "Call 'train' with appropriate arguments before using this estimator." + ) + return self._calibrated.predict_proba(x_pred) diff --git a/DashAI/back/models/scikit_learn/svr.py b/DashAI/back/models/scikit_learn/svr.py new file mode 100644 index 000000000..266a44c32 --- /dev/null +++ b/DashAI/back/models/scikit_learn/svr.py @@ -0,0 +1,161 @@ +from sklearn.svm import SVR as _SVR + +from DashAI.back.core.schema_fields import ( + BaseSchema, + enum_field, + optimizer_float_field, + optimizer_int_field, + schema_field, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.regression_model import RegressionModel +from DashAI.back.models.scikit_learn.sklearn_like_model import ( + CategoricalEncodingStrategy, +) +from DashAI.back.models.scikit_learn.sklearn_like_regressor import SklearnLikeRegressor + + +class SVRSchema(BaseSchema): + """Schema that configures the Support Vector Regressor. + + SVR (Support Vector Regression) finds a function that deviates from the + observed targets by at most ``epsilon`` while being as flat as possible. It + uses kernel functions to handle non-linear relationships. The underlying + implementation is ``sklearn.svm.SVR``. + """ + + kernel: schema_field( + enum_field(enum=["rbf", "linear", "poly", "sigmoid"]), + placeholder="rbf", + description=MultilingualString( + en=( + "Specifies the kernel type to be used in the algorithm. " + "'rbf' is the default radial basis function." + ), + es=( + "Especifica el tipo de kernel a usar. " + "'rbf' es la función de base radial predeterminada." + ), + ), + alias=MultilingualString(en="Kernel", es="Kernel"), + ) # type: ignore + + C: schema_field( # noqa: N815 + optimizer_float_field(ge=1e-4), + placeholder={ + "optimize": False, + "fixed_value": 1.0, + "lower_bound": 0.01, + "upper_bound": 100.0, + }, + description=MultilingualString( + en=( + "Regularisation parameter. Inversely proportional to the " + "strength of the regularisation." + ), + es=( + "Parámetro de regularización. Inversamente proporcional a la " + "fuerza de la regularización." + ), + ), + alias=MultilingualString(en="C", es="C"), + ) # type: ignore + + epsilon: schema_field( + optimizer_float_field(ge=0.0), + placeholder={ + "optimize": False, + "fixed_value": 0.1, + "lower_bound": 0.0, + "upper_bound": 1.0, + }, + description=MultilingualString( + en=( + "Specifies the epsilon-tube within which no penalty is associated " + "in the training loss function." + ), + es=( + "Especifica el tubo epsilon dentro del cual no se asocia penalización " + "en la función de pérdida de entrenamiento." + ), + ), + alias=MultilingualString(en="Epsilon", es="Épsilon"), + ) # type: ignore + + gamma: schema_field( + enum_field(enum=["scale", "auto"]), + placeholder="scale", + description=MultilingualString( + en=( + "Kernel coefficient for 'rbf', 'poly' and 'sigmoid'. " + "'scale' uses 1/(n_features * X.var()); 'auto' uses 1/n_features." + ), + es=( + "Coeficiente del kernel para 'rbf', 'poly' y 'sigmoid'. " + "'scale' usa 1/(n_features * X.var()); 'auto' usa 1/n_features." + ), + ), + alias=MultilingualString(en="Gamma", es="Gamma"), + ) # type: ignore + + max_iter: schema_field( + optimizer_int_field(ge=-1), + placeholder={ + "optimize": False, + "fixed_value": -1, + "lower_bound": 100, + "upper_bound": 10000, + }, + description=MultilingualString( + en=("Hard limit on iterations within solver. -1 means no limit."), + es=( + "Límite en iteraciones dentro del solucionador. " + "-1 significa sin límite." + ), + ), + alias=MultilingualString(en="Max iterations", es="Máximas iteraciones"), + ) # type: ignore + + +class SVR(RegressionModel, SklearnLikeRegressor, _SVR): + """Support Vector Regressor using kernel-based function estimation. + + SVR seeks a function that deviates from the targets by at most ``epsilon`` + (the insensitive tube) while maintaining flatness (controlled by ``C``). + Kernel functions allow SVR to capture non-linear relationships. The RBF kernel + is effective in many practical scenarios. + + Key hyperparameters include ``kernel``, ``C``, ``epsilon``, ``gamma``, and + ``max_iter``. The implementation wraps scikit-learn's ``SVR``. + + References + ---------- + - [1] Vapnik, V.N. (1995). The Nature of Statistical Learning Theory. Springer. + - [2] https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVR.html + """ + + SCHEMA = SVRSchema + DISPLAY_NAME: str = MultilingualString( + en="Support Vector Regression", + es="Regresión de Vectores de Soporte", + ) + DESCRIPTION: str = MultilingualString( + en="Kernel-based SVR that finds a function within an epsilon-insensitive tube.", + es=( + "SVR basado en kernel que encuentra una función dentro de un tubo " + "insensible a épsilon." + ), + ) + COLOR: str = "#EF5350" + ICON: str = "ControlPoint" + CATEGORICAL_ENCODING = CategoricalEncodingStrategy.ONE_HOT + + def __init__(self, **kwargs) -> None: + """Initialise the model by forwarding all kwargs to the parent class. + + Parameters + ---------- + **kwargs : dict + Hyperparameter values forwarded to the parent sklearn wrapper. + """ + super().__init__(**kwargs) diff --git a/DashAI/back/models/scikit_learn/tfidf_logreg_text_classification_model.py b/DashAI/back/models/scikit_learn/tfidf_logreg_text_classification_model.py new file mode 100644 index 000000000..6e15a5e9c --- /dev/null +++ b/DashAI/back/models/scikit_learn/tfidf_logreg_text_classification_model.py @@ -0,0 +1,186 @@ +"""DashAI TF-IDF + Logistic Regression text classification model.""" + +from typing import TYPE_CHECKING, Union + +from DashAI.back.core.schema_fields import ( + BaseSchema, + bool_field, + enum_field, + float_field, + int_field, + schema_field, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.models.text_classification_model import TextClassificationModel + +if TYPE_CHECKING: + from pathlib import Path + + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + +class TfIdfLogRegTextClassificationModelSchema(BaseSchema): + """Configuration schema for TF-IDF + Logistic Regression text classifier.""" + + ngram_min_n: schema_field( + int_field(ge=1), + placeholder=1, + description=MultilingualString( + en="Minimum n-gram size for the TF-IDF vectorizer (≥ 1).", + es="Tamaño mínimo de n-grama para el vectorizador TF-IDF (≥ 1).", + ), + alias=MultilingualString(en="Min n-gram", es="N-grama mínimo"), + ) # type: ignore + ngram_max_n: schema_field( + int_field(ge=1), + placeholder=1, + description=MultilingualString( + en="Maximum n-gram size for the TF-IDF vectorizer (≥ 1).", + es="Tamaño máximo de n-grama para el vectorizador TF-IDF (≥ 1).", + ), + alias=MultilingualString(en="Max n-gram", es="N-grama máximo"), + ) # type: ignore + use_idf: schema_field( + bool_field(), + placeholder=True, + description=MultilingualString( + en="Enable inverse-document-frequency re-weighting.", + es="Activar re-ponderación por frecuencia inversa de documento.", + ), + alias=MultilingualString(en="Use IDF", es="Usar IDF"), + ) # type: ignore + sublinear_tf: schema_field( + bool_field(), + placeholder=False, + description=MultilingualString( + en=("Apply sublinear TF scaling (replace TF with 1 + log(TF))."), + es=("Aplicar escalado sublineal de TF (reemplazar TF con 1 + log(TF))."), + ), + alias=MultilingualString(en="Sublinear TF", es="TF sublineal"), + ) # type: ignore + C: schema_field( + float_field(gt=0.0), + placeholder=1.0, + description=MultilingualString( + en=( + "Regularization parameter for logistic regression. " + "Smaller values mean stronger regularization." + ), + es=( + "Parámetro de regularización para regresión logística. " + "Valores más pequeños significan mayor regularización." + ), + ), + alias=MultilingualString(en="C (Regularization)", es="C (Regularización)"), + ) # type: ignore + max_iter: schema_field( + int_field(ge=100), + placeholder=1000, + description=MultilingualString( + en="Maximum number of iterations for the logistic regression solver.", + es=("Número máximo de iteraciones para el solver de regresión logística."), + ), + alias=MultilingualString(en="Max iterations", es="Iteraciones máximas"), + ) # type: ignore + solver: schema_field( + enum_field(["lbfgs", "liblinear", "saga"]), + placeholder="lbfgs", + description=MultilingualString( + en="Optimization algorithm for logistic regression.", + es="Algoritmo de optimización para regresión logística.", + ), + alias=MultilingualString(en="Solver", es="Solver"), + ) # type: ignore + + +class TfIdfLogRegTextClassificationModel(TextClassificationModel): + """TF-IDF vectorizer combined with Logistic Regression for text classification. + + This model converts raw text into TF-IDF feature vectors using scikit-learn's + ``TfidfVectorizer`` with a configurable n-gram range and IDF weighting, then + trains a ``LogisticRegression`` classifier on the resulting sparse matrix. + It is a strong baseline for text classification tasks, particularly when + training data is limited or computational resources are constrained. + + References + ---------- + - [1] https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html + - [2] https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html + """ + + DISPLAY_NAME: str = MultilingualString( + en="TF-IDF + Logistic Regression", + es="TF-IDF + Regresión Logística", + ) + DESCRIPTION: str = MultilingualString( + en=( + "TF-IDF vectorizer combined with logistic regression " + "for text classification." + ), + es=( + "Vectorizador TF-IDF combinado con regresión logística " + "para clasificación de texto." + ), + ) + COLOR: str = "#00695C" + ICON: str = "Article" + SCHEMA = TfIdfLogRegTextClassificationModelSchema + + def __init__(self, **kwargs) -> None: + from sklearn.feature_extraction.text import TfidfVectorizer + from sklearn.linear_model import LogisticRegression + from sklearn.preprocessing import LabelEncoder + + self.vectorizer = TfidfVectorizer( + ngram_range=(kwargs["ngram_min_n"], kwargs["ngram_max_n"]), + use_idf=kwargs["use_idf"], + sublinear_tf=kwargs["sublinear_tf"], + ) + self.classifier = LogisticRegression( + C=kwargs["C"], + max_iter=kwargs["max_iter"], + solver=kwargs["solver"], + ) + self.label_encoder = LabelEncoder() + + def train( + self, + x, + y, + x_validation=None, + y_validation=None, + ): + input_col = x.column_names[0] + output_col = y.column_names[0] + + X_tfidf = self.vectorizer.fit_transform(x[input_col]) + y_enc = self.label_encoder.fit_transform(y[output_col]) + self.classifier.fit(X_tfidf, y_enc) + + def predict(self, x): + input_col = x.column_names[0] + X_tfidf = self.vectorizer.transform(x[input_col]) + return self.classifier.predict_proba(X_tfidf) + + def prepare_output(self, dataset: "DashAIDataset", is_fit: bool = False): + from datasets import Dataset as HFDataset + + from DashAI.back.dataloaders.classes.dashai_dataset import to_dashai_dataset + + col = dataset.column_names[0] + if is_fit: + encoded = self.label_encoder.fit_transform(dataset[col]).tolist() + else: + encoded = self.label_encoder.transform(dataset[col]).tolist() + return to_dashai_dataset(HFDataset.from_dict({col: encoded})) + + def save(self, filename: Union[str, "Path"]) -> None: + import joblib + + joblib.dump(self, filename) + + @staticmethod + def load(filename: Union[str, "Path"]): + import joblib + + return joblib.load(filename) diff --git a/DashAI/back/tasks/regression_task.py b/DashAI/back/tasks/regression_task.py index ccff6c780..1ea8c31b4 100644 --- a/DashAI/back/tasks/regression_task.py +++ b/DashAI/back/tasks/regression_task.py @@ -23,14 +23,8 @@ class RegressionTask(BaseTask): """ DESCRIPTION: str = MultilingualString( - en=( - "Predict continuous numeric values from structured tabular data " - "using trained regression models." - ), - es=( - "Predice valores numéricos continuos a partir de datos tabulares " - "estructurados usando modelos de regresión." - ), + en="Predict continuous numeric values from tabular data.", + es="Predice valores numéricos continuos a partir de datos tabulares.", ) DISPLAY_NAME: str = MultilingualString(en="Regression", es="Regresión") diff --git a/DashAI/back/tasks/tabular_classification_task.py b/DashAI/back/tasks/tabular_classification_task.py index 9e80757d6..3b941b512 100644 --- a/DashAI/back/tasks/tabular_classification_task.py +++ b/DashAI/back/tasks/tabular_classification_task.py @@ -22,13 +22,10 @@ class TabularClassificationTask(ClassificationTask): """ DESCRIPTION: str = MultilingualString( - en=( - "Predict categorical labels from structured tabular data " - "(rows and columns) using trained classification models." - ), + en="Predict categorical labels from tabular data (rows and columns).", es=( "Predice etiquetas categóricas a partir de datos tabulares " - "estructurados (filas y columnas) usando modelos de clasificación." + "(filas y columnas)." ), ) DISPLAY_NAME: str = MultilingualString( diff --git a/DashAI/back/tasks/text_classification_task.py b/DashAI/back/tasks/text_classification_task.py index cade0f4c4..a1eeb12f3 100644 --- a/DashAI/back/tasks/text_classification_task.py +++ b/DashAI/back/tasks/text_classification_task.py @@ -22,6 +22,25 @@ class TextClassificationTask(ClassificationTask): a predicted class label for each sample. """ + SCORING_PROFILES = { + "text_balanced": { + "description": "Balanced", + "weights": {"Accuracy": 0.3, "F1": 0.4, "ROCAUC": 0.3}, + }, + "text_detectPositives": { + "description": "Detect Positives", + "weights": {"Recall": 0.6, "F1": 0.3, "Precision": 0.1}, + }, + "text_avoidFalseAlarms": { + "description": "Avoid False Alarms", + "weights": {"Precision": 0.6, "F1": 0.3, "Recall": 0.1}, + }, + "text_probabilityQuality": { + "description": "Probability Quality", + "weights": {"ROCAUC": 0.5, "LogLoss": 0.5}, + }, + } + metadata: dict = { "inputs_types": [Text], "outputs_types": [Categorical], @@ -31,13 +50,12 @@ class TextClassificationTask(ClassificationTask): DESCRIPTION: str = MultilingualString( en=( - "Assign predefined labels to text inputs using NLP models. " - "Common uses: sentiment analysis, spam detection, intent recognition." + "Classify text into predefined categories. " + "E.g.: sentiment, spam, intent detection." ), es=( - "Asigna etiquetas predefinidas a textos mediante modelos de PLN. " - "Usos comunes: análisis de sentimientos, detección de spam, " - "reconocimiento de intenciones." + "Clasifica textos en categorías predefinidas. " + "Ej.: sentimiento, spam, intención." ), ) DISPLAY_NAME: str = MultilingualString( diff --git a/DashAI/back/tasks/translation_task.py b/DashAI/back/tasks/translation_task.py index fb32fa65e..98e17542c 100644 --- a/DashAI/back/tasks/translation_task.py +++ b/DashAI/back/tasks/translation_task.py @@ -42,14 +42,8 @@ class TranslationTask(BaseTask): "outputs_cardinality": 1, } DESCRIPTION: str = MultilingualString( - en=( - "Convert text from one language to another while " - "preserving meaning and context." - ), - es=( - "Convierte texto de un idioma a otro preservando " - "el significado y el contexto." - ), + en="Convert text from one language to another preserving meaning.", + es="Convierte texto de un idioma a otro preservando el significado.", ) DISPLAY_NAME: str = MultilingualString(en="Translation", es="Traducción") diff --git a/DashAI/front/src/App.jsx b/DashAI/front/src/App.jsx index f92a875a6..7bbc3db4c 100644 --- a/DashAI/front/src/App.jsx +++ b/DashAI/front/src/App.jsx @@ -28,7 +28,7 @@ function App() { } /> } /> } /> } /> @@ -41,16 +41,13 @@ function App() { path="/app/models/sessions/new/:taskName" element={} /> - } - /> } /> - } /> + } /> } /> + } /> } /> } /> - - - - - + + + ); } diff --git a/DashAI/front/src/components/ResponsiveAppBar.jsx b/DashAI/front/src/components/ResponsiveAppBar.jsx index 577df3c2a..d829b16b3 100644 --- a/DashAI/front/src/components/ResponsiveAppBar.jsx +++ b/DashAI/front/src/components/ResponsiveAppBar.jsx @@ -13,8 +13,9 @@ import HomeIcon from "@mui/icons-material/HomeOutlined"; import { useTranslation } from "react-i18next"; import LanguageSelector from "./LanguageSelector"; import { ColorModeContext } from "../contexts/ThemeContext"; -import Brightness4Icon from "@mui/icons-material/Brightness4"; -import Brightness7Icon from "@mui/icons-material/Brightness7"; +import DarkModeOutlinedIcon from "@mui/icons-material/DarkModeOutlined"; +import LightModeOutlinedIcon from "@mui/icons-material/LightModeOutlined"; +import Tooltip from "@mui/material/Tooltip"; import HardwareMonitorButton from "./hardware/HardwareMonitorButton"; import NavbarTourButton from "./tour/NavbarTourButton"; @@ -41,8 +42,8 @@ function ResponsiveAppBar() { const iconBtnSx = React.useMemo( () => ({ - width: 28, - height: 28, + width: 32, + height: 32, borderRadius: "4px", border: `1px solid ${theme.palette.divider}`, color: theme.palette.text.secondary, @@ -86,9 +87,8 @@ function ResponsiveAppBar() { }} > - - {theme.palette.mode === "dark" ? ( - - ) : ( - - )} - + + {theme.palette.mode === "dark" ? ( + + ) : ( + + )} + + diff --git a/DashAI/front/src/components/configurableObject/FormTooltip.jsx b/DashAI/front/src/components/configurableObject/FormTooltip.jsx index 1c54a1d24..ed6a5d905 100644 --- a/DashAI/front/src/components/configurableObject/FormTooltip.jsx +++ b/DashAI/front/src/components/configurableObject/FormTooltip.jsx @@ -1,7 +1,9 @@ import React from "react"; -import { Tooltip, IconButton, Typography } from "@mui/material"; +import { Tooltip, IconButton, Typography, Link } from "@mui/material"; import HelpOutlineIcon from "@mui/icons-material/HelpOutline"; import PropTypes from "prop-types"; +import ReactMarkdown from "react-markdown"; + /** * This component renders a tooltip containing a description for each parameter in a form, * providing users with additional information to better understand the purpose of each input field. @@ -10,7 +12,31 @@ import PropTypes from "prop-types"; function FormTooltip({ contentStr = "", error }) { return ( {contentStr}} + title={ + + ( + {children} + ), + a: ({ href, children }) => ( + e.stopPropagation()} + > + {children} + + ), + }} + > + {contentStr} + + + } placement="bottom" arrow > diff --git a/DashAI/front/src/components/custom/ComponentDetailsPanel.jsx b/DashAI/front/src/components/custom/ComponentDetailsPanel.jsx new file mode 100644 index 000000000..f2ef101da --- /dev/null +++ b/DashAI/front/src/components/custom/ComponentDetailsPanel.jsx @@ -0,0 +1,237 @@ +import React from "react"; +import PropTypes from "prop-types"; +import { Box, Stack, Typography, Chip, Divider, Link } from "@mui/material"; +import { useTranslation } from "react-i18next"; +import SideBar from "../threeSectionLayout/panelContainers/SideBar"; +import { useTheme } from "@mui/material/styles"; + +function getLabel(component) { + return component.display_name || component.name; +} + +function getDescription(component) { + return component.description ?? component.schema?.description ?? ""; +} + +const URL_PATTERN = /(\[([^\]]+)\]\((https?:\/\/[^)]+)\))|(https?:\/\/\S+)/g; +const TRAILING_PUNCT = /[.,;:!?)\]>'"]+$/; + +function DescriptionText({ text }) { + if (!text) return null; + const parts = []; + let last = 0; + let match; + URL_PATTERN.lastIndex = 0; + while ((match = URL_PATTERN.exec(text)) !== null) { + if (match.index > last) parts.push(text.slice(last, match.index)); + if (match[1]) { + // Markdown link [label](url) — closing ) already excluded by the pattern + parts.push( + + {match[2]} + , + ); + } else { + // Bare URL — strip trailing punctuation that belongs to surrounding prose + const rawUrl = match[0]; + const cleanUrl = rawUrl.replace(TRAILING_PUNCT, ""); + const trailing = rawUrl.slice(cleanUrl.length); + parts.push( + + {cleanUrl} + , + ); + if (trailing) parts.push(trailing); + } + last = match.index + match[0].length; + } + if (last < text.length) parts.push(text.slice(last)); + return parts; +} + +function ComponentDetailsPanel({ + component, + getIcon, + extraSections, + categoryKey = "type", +}) { + const { t } = useTranslation("custom"); + const theme = useTheme(); + + return ( + + + {/* Title */} + + + {t("componentDetails")} + + + + {/* Content */} + {!component || !component.name ? ( + + + {t("selectAnItemToShowInfo")} + + + ) : ( + + + + {getIcon?.(component) && ( + + {getIcon(component)} + + )} + + + {component.display_name || component.name} + + {component[categoryKey] && ( + + {component[categoryKey]} + + )} + + + + + + {t("description")} + + + {getDescription(component) ? ( + + ) : ( + t("noDescriptionAvailable") + )} + + + + {(component.schema?.tags || component.metadata?.tags || []) + .length > 0 && ( + + + {t("tags")} + + + {(component.schema?.tags || component.metadata?.tags).map( + (tag) => ( + + ), + )} + + + )} + + {extraSections && + extraSections.map((section) => ( + + + + {section.title} + + {section.content} + + ))} + + + )} + + + ); +} + +ComponentDetailsPanel.propTypes = { + component: PropTypes.shape({ + name: PropTypes.string, + display_name: PropTypes.string, + type: PropTypes.string, + description: PropTypes.string, + schema: PropTypes.object, + metadata: PropTypes.object, + }), + getIcon: PropTypes.func, + categoryKey: PropTypes.string, + extraSections: PropTypes.arrayOf( + PropTypes.shape({ + title: PropTypes.string.isRequired, + content: PropTypes.node.isRequired, + }), + ), +}; + +export default ComponentDetailsPanel; diff --git a/DashAI/front/src/components/custom/ComponentSelector.jsx b/DashAI/front/src/components/custom/ComponentSelector.jsx new file mode 100644 index 000000000..e2b2ad2bb --- /dev/null +++ b/DashAI/front/src/components/custom/ComponentSelector.jsx @@ -0,0 +1,347 @@ +import React, { useMemo, useState } from "react"; +import PropTypes from "prop-types"; +import { + Box, + Stack, + TextField, + InputAdornment, + IconButton, + Chip, + Typography, + Collapse, + Paper, +} from "@mui/material"; +import { + Search as SearchIcon, + Clear as ClearIcon, + ExpandMore as ExpandMoreIcon, + Check as CheckIcon, +} from "@mui/icons-material"; +import { useTranslation } from "react-i18next"; + +const ALL_CATEGORY = "All"; + +function getLabel(component) { + return component.display_name || component.name; +} + +function getDescription(component, fallback = "") { + return component.description ?? fallback; +} + +function ComponentSelector({ + components, + selected = null, + onSelect, + categoryKey = "type", + searchPlaceholder, + emptyText, + getIcon, +}) { + const { t } = useTranslation("custom"); + const [search, setSearch] = useState(""); + const [activeCategory, setActiveCategory] = useState(ALL_CATEGORY); + + const categories = useMemo(() => { + const set = new Set(); + components.forEach((c) => { + const cat = c[categoryKey]; + if (cat) set.add(cat); + }); + return [ALL_CATEGORY, ...Array.from(set).sort()]; + }, [components, categoryKey]); + + const [expanded, setExpanded] = useState(() => new Set(categories)); + + React.useEffect(() => { + setExpanded(new Set(categories)); + }, [categories]); + + const toggleCategory = (cat) => { + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(cat)) next.delete(cat); + else next.add(cat); + return next; + }); + }; + + const filtered = useMemo(() => { + const q = search.trim().toLowerCase(); + return components.filter((c) => { + const matchesCategory = + activeCategory === ALL_CATEGORY || c[categoryKey] === activeCategory; + if (!matchesCategory) return false; + if (q === "") return true; + const label = getLabel(c).toLowerCase(); + const desc = getDescription(c, t("noDescriptionAvailable")).toLowerCase(); + return label.includes(q) || desc.includes(q); + }); + }, [components, search, activeCategory, categoryKey]); + + const grouped = useMemo(() => { + const groups = {}; + filtered.forEach((c) => { + const cat = c[categoryKey] || "Other"; + if (!groups[cat]) groups[cat] = []; + groups[cat].push(c); + }); + return groups; + }, [filtered, categoryKey]); + + const counts = useMemo(() => { + const map = {}; + components.forEach((c) => { + const cat = c[categoryKey]; + if (cat) map[cat] = (map[cat] || 0) + 1; + }); + return map; + }, [components, categoryKey]); + + const handleSelect = (component) => onSelect?.(component); + + return ( + + setSearch(e.target.value)} + slotProps={{ + input: { + startAdornment: ( + + + + ), + endAdornment: search ? ( + + setSearch("")}> + + + + ) : null, + }, + }} + /> + + + {categories.map((cat) => { + const isActive = activeCategory === cat; + const count = cat === ALL_CATEGORY ? components.length : counts[cat]; + return ( + setActiveCategory(cat)} + size="small" + /> + ); + })} + + + + + {Object.entries(grouped).map(([cat, items]) => { + const isOpen = expanded.has(cat); + return ( + + toggleCategory(cat)} + sx={{ + display: "flex", + alignItems: "center", + justifyContent: "space-between", + py: 1, + cursor: "pointer", + color: "text.secondary", + "&:hover": { color: "text.primary" }, + }} + > + + + {cat} + + + + + + + + {items.map((component) => { + const isSelected = selected?.name === component.name; + const icon = getIcon?.(component); + return ( + handleSelect(component)} + sx={{ + p: 1.5, + display: "flex", + gap: 1.5, + alignItems: "flex-start", + cursor: "pointer", + border: 1, + borderColor: isSelected + ? "primary.main" + : "divider", + bgcolor: isSelected + ? "action.selected" + : "background.paper", + transition: "border-color 0.15s, background 0.15s", + "&:hover": { + borderColor: "primary.light", + }, + }} + > + {icon && ( + + {icon} + + )} + + + {getLabel(component)} + + + {getDescription( + component, + t("noDescriptionAvailable"), + )} + + + {isSelected && ( + + )} + + ); + })} + + + + ); + })} + + {filtered.length === 0 && ( + + + + {emptyText ?? t("noItemsFound")} + + + {t("tryAdjustingSearch")} + + + )} + + + + + + {t("componentsAvailable", { count: filtered.length })} + + {selected && ( + } + label={getLabel(selected)} + color="primary" + variant="outlined" + size="small" + /> + )} + + + ); +} + +ComponentSelector.propTypes = { + components: PropTypes.arrayOf( + PropTypes.shape({ + name: PropTypes.string.isRequired, + type: PropTypes.string, + description: PropTypes.string, + schema: PropTypes.object, + }), + ).isRequired, + selected: PropTypes.shape({ name: PropTypes.string }), + onSelect: PropTypes.func.isRequired, + categoryKey: PropTypes.string, + searchPlaceholder: PropTypes.string, + emptyText: PropTypes.string, + getIcon: PropTypes.func, +}; + +export default ComponentSelector; diff --git a/DashAI/front/src/components/custom/contexts/DatasetsAndNotebooksContext.jsx b/DashAI/front/src/components/custom/contexts/DatasetsAndNotebooksContext.jsx index 771498120..4fdbd8775 100644 --- a/DashAI/front/src/components/custom/contexts/DatasetsAndNotebooksContext.jsx +++ b/DashAI/front/src/components/custom/contexts/DatasetsAndNotebooksContext.jsx @@ -48,6 +48,7 @@ export const DatasetsAndNotebooksProvider = ({ children }) => { const [selectedOption, setSelectedOption] = useState(OptionsEnum.NEW); // "datasets" or "notebooks" const [rightBarContent, setRightBarContent] = useState(null); + const [uploadDataloader, setUploadDataloader] = useState(null); const [datasetInfo, setDatasetInfo] = useState(null); const [datasetTab, setDatasetTab] = useState(0); @@ -88,6 +89,8 @@ export const DatasetsAndNotebooksProvider = ({ children }) => { setDatasetInfo, datasetTab, setDatasetTab, + uploadDataloader, + setUploadDataloader, }; return ( diff --git a/DashAI/front/src/components/datasets/DatasetModal.jsx b/DashAI/front/src/components/datasets/DatasetModal.jsx index 50478c287..2c3193d3f 100644 --- a/DashAI/front/src/components/datasets/DatasetModal.jsx +++ b/DashAI/front/src/components/datasets/DatasetModal.jsx @@ -265,7 +265,7 @@ function DatasetModal({ open, setOpen, updateDatasets }) { {steps.map((step, index) => ( index} + completed={false} disabled={activeStep < index} > diff --git a/DashAI/front/src/components/explainers/InlineExplainerCreator.jsx b/DashAI/front/src/components/explainers/InlineExplainerCreator.jsx index d24543e8d..01c1a69d7 100644 --- a/DashAI/front/src/components/explainers/InlineExplainerCreator.jsx +++ b/DashAI/front/src/components/explainers/InlineExplainerCreator.jsx @@ -315,7 +315,7 @@ export default function InlineExplainerCreator({ {steps.map((step, index) => ( index} + completed={false} disabled={activeStep < index} > diff --git a/DashAI/front/src/components/explainers/NewGlobalExplainerModal.jsx b/DashAI/front/src/components/explainers/NewGlobalExplainerModal.jsx index 48dbd0211..9ac225d5b 100644 --- a/DashAI/front/src/components/explainers/NewGlobalExplainerModal.jsx +++ b/DashAI/front/src/components/explainers/NewGlobalExplainerModal.jsx @@ -295,7 +295,7 @@ export default function NewGlobalExplainerModal({ {steps.map((step, index) => ( index} + completed={false} disabled={activeStep < index} > diff --git a/DashAI/front/src/components/explainers/NewLocalExplainerModal.jsx b/DashAI/front/src/components/explainers/NewLocalExplainerModal.jsx index a85e559fe..c22c30637 100644 --- a/DashAI/front/src/components/explainers/NewLocalExplainerModal.jsx +++ b/DashAI/front/src/components/explainers/NewLocalExplainerModal.jsx @@ -301,7 +301,7 @@ export default function NewLocalExplainerModal({ {steps.map((step, index) => ( index} + completed={false} disabled={activeStep < index} > diff --git a/DashAI/front/src/components/explorations/ExplorationEditor.jsx b/DashAI/front/src/components/explorations/ExplorationEditor.jsx index 1b9b41dfe..db8e07797 100644 --- a/DashAI/front/src/components/explorations/ExplorationEditor.jsx +++ b/DashAI/front/src/components/explorations/ExplorationEditor.jsx @@ -230,7 +230,7 @@ function ExplorationEditor({ open = true, handleCloseDialog = () => {} }) { {steps.map((step, index) => ( index} + completed={false} disabled={activeStep < index} > diff --git a/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx b/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx index 5de494f3e..e227bafad 100644 --- a/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx +++ b/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx @@ -37,8 +37,8 @@ const renderOption = (props, option, _, ownerState) => { title={ {option.tooltip} diff --git a/DashAI/front/src/components/generative/ChatBubble.jsx b/DashAI/front/src/components/generative/ChatBubble.jsx index ac72d8292..735706689 100644 --- a/DashAI/front/src/components/generative/ChatBubble.jsx +++ b/DashAI/front/src/components/generative/ChatBubble.jsx @@ -1,4 +1,4 @@ -import { Box } from "@mui/material"; +import { Box, Typography } from "@mui/material"; import { ChatAvatar } from "./ChatAvatar"; import { ChatTimestamp } from "./ChatTimeStamp"; import { MessageContent } from "./MessageContent"; @@ -25,18 +25,18 @@ export function ChatBubble({ {!isUser && sender && ( - {sender} - + )} + + + + {step === 0 + ? t("generative:label.selectModel") + : t("generative:label.configureSession")} + + + {step === 0 + ? t("generative:label.pickAModelGroupedByTask") + : t("generative:label.nameAndDescribeYourSession")} + + + + + {step === 0 ? ( + loadingModels ? ( + + + + ) : ( + + ) + ) : ( + + + + + )} + + + + + {step === 0 ? ( + + ) : ( + + )} + + + ); +} diff --git a/DashAI/front/src/components/generative/CreateSessionContext.jsx b/DashAI/front/src/components/generative/CreateSessionContext.jsx new file mode 100644 index 000000000..e544f2883 --- /dev/null +++ b/DashAI/front/src/components/generative/CreateSessionContext.jsx @@ -0,0 +1,223 @@ +import { + createContext, + useCallback, + useContext, + useEffect, + useMemo, + useState, +} from "react"; +import { useNavigate, useParams } from "react-router-dom"; +import { useFormik } from "formik"; +import { useSnackbar } from "notistack"; +import { useTranslation } from "react-i18next"; +import { + createGenerativeSession, + getRelatedComponents, +} from "../../api/generativeTask"; +import { generateSequentialName } from "../../utils/nameGenerator"; +import { + buildYupSchema, + formatTaskNameForSession, + preprocessSchema, +} from "./utils"; +import { useGenerative } from "./GenerativeContext"; + +const CreateSessionContext = createContext(null); + +export const useCreateSession = () => useContext(CreateSessionContext); + +export function CreateSessionProvider({ children }) { + const navigate = useNavigate(); + const { modelName } = useParams(); + const { enqueueSnackbar } = useSnackbar(); + const { t } = useTranslation(["generative", "common"]); + const { tasks, sessions: existingSessions, setSessions } = useGenerative(); + + const step = modelName ? 1 : 0; + const [models, setModels] = useState([]); + const [loadingModels, setLoadingModels] = useState(true); + const [selectedModel, setSelectedModel] = useState(null); + const [submitting, setSubmitting] = useState(false); + + // Load all generative models grouped by their compatible task. + // Re-fetches when language changes so display_name/description are translated. + useEffect(() => { + if (!tasks || tasks.length === 0) return; + let cancelled = false; + setLoadingModels(true); + + Promise.all( + tasks.map((task) => + getRelatedComponents(task.name).then((components) => + components.map((c) => ({ + ...c, + task_name: task.name, + task_display_name: task.display_name || task.name, + })), + ), + ), + ) + .then((perTaskLists) => { + if (cancelled) return; + // Deduplicate by model name (a model may appear under several tasks). + const seen = new Set(); + const flat = []; + perTaskLists.flat().forEach((m) => { + if (seen.has(m.name)) return; + seen.add(m.name); + flat.push(m); + }); + setModels(flat); + }) + .catch((err) => { + console.error("Failed to load generative models", err); + enqueueSnackbar(t("generative:error.failedToLoadModels"), { + variant: "error", + }); + }) + .finally(() => { + if (!cancelled) setLoadingModels(false); + }); + + return () => { + cancelled = true; + }; + }, [tasks, enqueueSnackbar, t]); + + const processedProperties = useMemo( + () => + selectedModel?.schema?.properties + ? preprocessSchema(selectedModel.schema.properties) + : {}, + [selectedModel], + ); + + const validationSchema = useMemo( + () => + Object.keys(processedProperties).length > 0 + ? buildYupSchema(processedProperties) + : null, + [processedProperties], + ); + + const formik = useFormik({ + initialValues: { name: "", description: "" }, + validationSchema, + enableReinitialize: false, + validate: (values) => { + const errors = {}; + if (!values.name || values.name.trim() === "") { + errors.name = t("generative:error.nameRequired"); + } + return errors; + }, + onSubmit: async (values) => { + if (!selectedModel) return; + setSubmitting(true); + try { + const created = await createGenerativeSession({ + name: values.name, + description: values.description, + task_name: selectedModel.task_name, + model_name: selectedModel.name, + parameters: values, + }); + setSessions((prev) => [...prev, created]); + enqueueSnackbar(t("generative:message.sessionCreatedSuccess"), { + variant: "success", + }); + navigate(`/app/generative/sessions/${created.id}`); + } catch (error) { + console.error("Error creating session:", error); + const detail = error?.response?.data?.detail || ""; + if ( + error?.response?.status === 409 || + detail.includes("already exists") + ) { + enqueueSnackbar(t("generative:error.sessionNameExists"), { + variant: "error", + }); + } else { + enqueueSnackbar(t("generative:error.failedToCreateSession"), { + variant: "error", + }); + } + } finally { + setSubmitting(false); + } + }, + }); + + // When a model is selected, seed formik with its parameter defaults and a + // freshly computed default session name. Computing the name here (rather than + // in an effect) avoids the render-ordering race where the "fill empty name" + // effect fires before resetForm has cleared the previous model's name. + const handleSelectModel = useCallback( + (model) => { + setSelectedModel(model); + const props = model?.schema?.properties + ? preprocessSchema(model.schema.properties) + : {}; + const paramDefaults = Object.keys(props).reduce((acc, key) => { + acc[key] = props[key].placeholder ?? ""; + return acc; + }, {}); + const { defaultName } = generateSequentialName({ + base: `${formatTaskNameForSession(model.task_name)}_Session`, + items: existingSessions, + getName: (session) => session.name, + filter: (session) => session.task_name === model.task_name, + }); + formik.resetForm({ + values: { + name: defaultName, + description: "", + ...paramDefaults, + }, + }); + }, + [existingSessions], + ); + + // Sync selectedModel from URL param on load and after language-triggered + // model refetch so display_name / description reflect the active language. + useEffect(() => { + if (!modelName || models.length === 0) return; + const match = models.find((m) => m.name === modelName); + if (match) handleSelectModel(match); + }, [modelName, models]); + + const handleNext = () => { + if (step === 0 && selectedModel) + navigate(`/app/generative/sessions/new/${selectedModel.name}`); + }; + + const handleBack = () => { + if (step === 1) navigate("/app/generative/sessions/new"); + else navigate("/app/generative"); + }; + + const handleCreate = () => { + formik.submitForm(); + }; + + const value = { + step, + models, + loadingModels, + selectedModel, + handleSelectModel, + formik, + processedProperties, + submitting, + handleNext, + handleBack, + handleCreate, + }; + + return ( + + {children} + + ); +} diff --git a/DashAI/front/src/components/generative/CreateSessionLanding.jsx b/DashAI/front/src/components/generative/CreateSessionLanding.jsx new file mode 100644 index 000000000..91d52f45b --- /dev/null +++ b/DashAI/front/src/components/generative/CreateSessionLanding.jsx @@ -0,0 +1,27 @@ +import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome"; +import { useNavigate } from "react-router-dom"; +import { useTranslation } from "react-i18next"; +import SelectOptionMenu from "../threeSectionLayout/SelectOptionMenu"; + +export default function CreateSessionLanding() { + const navigate = useNavigate(); + const { t } = useTranslation(["generative", "common"]); + + return ( + navigate("/app/generative/sessions/new")} + title={t("generative:label.generativeModule")} + subtitle={t("generative:label.createNewSessionDescription")} + options={[ + { + name: "new_session", + display_name: t("generative:label.createNewSession"), + description: t("generative:label.createNewSessionDescription"), + Icon: AutoAwesomeIcon, + }, + ]} + searchBar={false} + dataTour="create-session-landing" + > + ); +} diff --git a/DashAI/front/src/components/generative/CreateSessionRight.jsx b/DashAI/front/src/components/generative/CreateSessionRight.jsx new file mode 100644 index 000000000..c155cc8e9 --- /dev/null +++ b/DashAI/front/src/components/generative/CreateSessionRight.jsx @@ -0,0 +1,75 @@ +import { Box, Divider, Typography } from "@mui/material"; +import { useTranslation } from "react-i18next"; +import SideBar from "../threeSectionLayout/panelContainers/SideBar"; +import ComponentDetailsPanel from "../custom/ComponentDetailsPanel"; +import FormSchemaRenderFields from "../shared/FormSchemaRenderFields"; +import { useTheme } from "@mui/material/styles"; +import { useCreateSession } from "./CreateSessionContext"; + +export default function CreateSessionRight() { + const { t } = useTranslation(["generative", "common"]); + const theme = useTheme(); + const { step, selectedModel, formik, processedProperties } = + useCreateSession(); + + if (step === 0) { + return ( + + ); + } + + return ( + + {/* Title */} + + + {t("common:modelParameters")} + + + + {/* Content */} + {Object.keys(processedProperties).length === 0 ? ( + + + {t("generative:label.modelHasNoParameters")} + + + ) : ( + + { + formik.setValues((prev) => ({ ...prev, ...updatedValues })); + }} + onFormSubmit={formik.handleSubmit} + setError={(error) => console.error(error)} + errorsMessage={formik.errors || {}} + spacing={2} + /> + + )} + + ); +} diff --git a/DashAI/front/src/components/generative/GenerativeBreadcrumbs.jsx b/DashAI/front/src/components/generative/GenerativeBreadcrumbs.jsx index 6305e3412..fab27fdd6 100644 --- a/DashAI/front/src/components/generative/GenerativeBreadcrumbs.jsx +++ b/DashAI/front/src/components/generative/GenerativeBreadcrumbs.jsx @@ -7,23 +7,37 @@ import Box from "@mui/material/Box"; import { useNavigate, useLocation, useParams } from "react-router-dom"; import { useTranslation } from "react-i18next"; import { useGenerative } from "./GenerativeContext"; +import { useCreateSession } from "./CreateSessionContext"; export default function GenerativeBreadcrumbs() { const navigate = useNavigate(); const location = useLocation(); const params = useParams(); const { t } = useTranslation(["generative", "common"]); - const { tasks, sessions } = useGenerative(); + const { sessions } = useGenerative(); + const createSession = useCreateSession(); const rootCrumb = { label: t("common:generative"), path: "/app/generative" }; const getBreadcrumbs = () => { const path = location.pathname; - if (path.startsWith("/app/generative/sessions/new/") && params.taskName) { - const task = tasks.find((tk) => tk.name === params.taskName); - const taskLabel = task?.display_name ?? params.taskName; - return [rootCrumb, { label: taskLabel, path: null, current: true }]; + if (path.startsWith("/app/generative/sessions/new")) { + const modelNameParam = params.modelName; + const selectedModel = createSession?.selectedModel; + const modelLabel = selectedModel?.display_name || modelNameParam; + const crumbs = [ + rootCrumb, + { + label: t("generative:label.selectModelCrumb"), + path: modelNameParam ? "/app/generative/sessions/new" : null, + current: !modelNameParam, + }, + ]; + if (modelNameParam) { + crumbs.push({ label: modelLabel, path: null, current: true }); + } + return crumbs; } if (path.startsWith("/app/generative/sessions/") && params.id) { diff --git a/DashAI/front/src/components/generative/GenerativeChat.jsx b/DashAI/front/src/components/generative/GenerativeChat.jsx index 3e9b1690e..338edb3e7 100644 --- a/DashAI/front/src/components/generative/GenerativeChat.jsx +++ b/DashAI/front/src/components/generative/GenerativeChat.jsx @@ -208,7 +208,6 @@ export default function GenerativeChat() { flexDirection="column" justifyContent="flex-start" alignItems="center" - gap={1} width={"100%"} height={"100%"} sx={{ overflow: "hidden", minHeight: 0 }} @@ -223,7 +222,6 @@ export default function GenerativeChat() { alignItems: "center", borderRadius: 1, opacity: 0.5, - mb: 0.8, }} > {sessionInfo?.name ? sessionInfo.name : "Untitled Session"}{" "} @@ -254,7 +254,7 @@ export default function GenerativeChat() { - + {/* Chat display */} {message.type === "history" ? ( - + Parameters updated: {message.changedMessage} diff --git a/DashAI/front/src/components/generative/ModelCard.jsx b/DashAI/front/src/components/generative/ModelCard.jsx deleted file mode 100644 index cfc7dd465..000000000 --- a/DashAI/front/src/components/generative/ModelCard.jsx +++ /dev/null @@ -1,141 +0,0 @@ -import { useEffect, useRef, useState } from "react"; -import { Box, Link, Paper, Typography } from "@mui/material"; -import { alpha, useTheme } from "@mui/material/styles"; -import CheckCircleIcon from "@mui/icons-material/CheckCircle"; -import { useTranslation } from "react-i18next"; - -const DESCRIPTION_LINE_CLAMP = 3; -const URL_SPLIT_REGEX = /(https?:\/\/[^\s<>"']+[^\s<>"'.,;:!?)\]}])/g; -const URL_TEST_REGEX = /^https?:\/\//; - -function renderDescription(text) { - return text.split(URL_SPLIT_REGEX).map((part, i) => { - if (URL_TEST_REGEX.test(part)) { - return ( - e.stopPropagation()} - sx={{ wordBreak: "break-all" }} - > - {part} - - ); - } - return part; - }); -} - -export default function ModelCard({ - model, - color, - isSelected, - onClick, - dataTour, -}) { - const theme = useTheme(); - const [expanded, setExpanded] = useState(false); - const [isClamped, setIsClamped] = useState(false); - const { t } = useTranslation(["generative"]); - const descRef = useRef(null); - - useEffect(() => { - const el = descRef.current; - if (el) { - setIsClamped(el.scrollHeight > el.clientHeight + 1); - } - }, [model.description]); - - const handleToggleExpand = (e) => { - e.stopPropagation(); - setExpanded((prev) => !prev); - }; - - return ( - - - - {model.display_name ? model.display_name : model.name} - - {isSelected && ( - - )} - - - - {model.description - ? renderDescription(model.description) - : t("generative:label.noDescriptionAvailable")} - - - {(isClamped || expanded) && ( - - {expanded - ? t("generative:label.showLess") - : t("generative:label.readMore")} - - )} - - ); -} diff --git a/DashAI/front/src/components/generative/ModelGrid.jsx b/DashAI/front/src/components/generative/ModelGrid.jsx deleted file mode 100644 index 020ebcce0..000000000 --- a/DashAI/front/src/components/generative/ModelGrid.jsx +++ /dev/null @@ -1,68 +0,0 @@ -import { Box, Grid, Pagination } from "@mui/material"; -import ModelCard from "./ModelCard"; - -export const MODEL_COLORS = [ - "#1976d2", - "#7b1fa2", - "#2e7d32", - "#e65100", - "#0288d1", - "#c62828", -]; - -export const MODELS_PER_PAGE = 4; - -export default function ModelGrid({ - models, - selectedModelName, - page, - onSelect, - onPageChange, -}) { - const pageModels = models.slice( - (page - 1) * MODELS_PER_PAGE, - page * MODELS_PER_PAGE, - ); - - return ( - <> - - {pageModels.map((model, indexOnPage) => { - const globalIndex = (page - 1) * MODELS_PER_PAGE + indexOnPage; - const color = - model.color ?? MODEL_COLORS[globalIndex % MODEL_COLORS.length]; - return ( - - onSelect(model, globalIndex)} - dataTour={ - model.name.toLowerCase().includes("qwen") - ? "model-card-qwen" - : undefined - } - /> - - ); - })} - - - {models.length > MODELS_PER_PAGE && ( - - onPageChange(value)} - color="primary" - size="small" - /> - - )} - - ); -} diff --git a/DashAI/front/src/components/generative/ParamsBar.jsx b/DashAI/front/src/components/generative/ParamsBar.jsx index 4e199e067..acf075cc2 100644 --- a/DashAI/front/src/components/generative/ParamsBar.jsx +++ b/DashAI/front/src/components/generative/ParamsBar.jsx @@ -123,119 +123,108 @@ export default function ParamsBar({ onToggle }) { - - - {t("common:modelParameters")} - + + {t("common:modelParameters")} + - {/* Parameter History Modal */} - {selectedSessionId && ( - { - getHistory(); - setHistoryInfoVisible(true); - }} - sx={{ - display: "flex", - alignItems: "center", - justifyContent: "center", - }} - > - - - )} - - {/* Divider */} - - {selectedSessionId ? ( - -
- - {/* Render the parameter fields */} - { - formik.setValues((prevValues) => ({ - ...prevValues, - ...updatedValues, - })); - }} - onFormSubmit={formik.handleSubmit} - setError={(error) => console.error(error)} - errorsMessage={formik.errors || {}} - spacing={1} - /> - - - - -
- - ) : ( - { + getHistory(); + setHistoryInfoVisible(true); + }} sx={{ - flex: 1, display: "flex", alignItems: "center", justifyContent: "center", - p: 2, }} > - - {t("generative:label.selectSessionToViewParameters")} - - + + )} - - {/* Parameter History Modal */} - + + {selectedSessionId ? ( + +
+ + {/* Render the parameter fields */} + { + formik.setValues((prevValues) => ({ + ...prevValues, + ...updatedValues, + })); + }} + onFormSubmit={formik.handleSubmit} + setError={(error) => console.error(error)} + errorsMessage={formik.errors || {}} + spacing={1} + /> + + + + +
+ + ) : ( + + + {t("generative:label.selectSessionToViewParameters")} + + + )} + + {/* Parameter History Modal */} + ); } diff --git a/DashAI/front/src/components/generative/SelectModelMenu.jsx b/DashAI/front/src/components/generative/SelectModelMenu.jsx deleted file mode 100644 index 76532b6f3..000000000 --- a/DashAI/front/src/components/generative/SelectModelMenu.jsx +++ /dev/null @@ -1,232 +0,0 @@ -import { useEffect, useState, useMemo, useRef } from "react"; -import { useNavigate } from "react-router-dom"; -import { Box, Button, Typography } from "@mui/material"; -import { useFormik } from "formik"; -import { useSnackbar } from "notistack"; -import { useTranslation } from "react-i18next"; -import { useTourContext } from "../tour/TourProvider"; -import { - getRelatedComponents, - createGenerativeSession, -} from "../../api/generativeTask"; -import { - preprocessSchema, - buildYupSchema, - formatTaskNameForSession, -} from "./utils"; -import { generateSequentialName } from "../../utils/nameGenerator"; -import { useGenerative } from "./GenerativeContext"; -import ModelGrid from "./ModelGrid"; -import SessionForm from "./SessionForm"; - -export default function SelectModelMenu() { - const navigate = useNavigate(); - const { - selectedTaskName, - selectedDisplayName, - sessions: existingSessions, - setSessions, - } = useGenerative(); - - const [relatedComponents, setRelatedComponents] = useState([]); - const [selectedModel, setSelectedModel] = useState(null); - const [validationSchema, setValidationSchema] = useState(null); - const [page, setPage] = useState(1); - const [nameError, setNameError] = useState(false); - const [nameErrorMessage, setNameErrorMessage] = useState(""); - - const { t } = useTranslation(["generative", "common"]); - const { enqueueSnackbar } = useSnackbar(); - const tourContext = useTourContext(); - const hasAdvancedTourRef = useRef(false); - - const { defaultName } = useMemo(() => { - if (!selectedTaskName) return { defaultName: "" }; - return generateSequentialName({ - base: `${formatTaskNameForSession(selectedTaskName)}_Session`, - items: existingSessions, - getName: (session) => session.name, - filter: (session) => session.task_name === selectedTaskName, - }); - }, [selectedTaskName, existingSessions]); - - useEffect(() => { - if (!selectedTaskName) return; - getRelatedComponents(selectedTaskName).then((components) => { - setRelatedComponents(components); - setPage(1); - setSelectedModel((prev) => - prev ? (components.find((c) => c.name === prev.name) ?? prev) : null, - ); - }); - }, [selectedTaskName, t]); - - useEffect(() => { - if (!selectedModel?.schema?.properties) return; - const processedProps = preprocessSchema(selectedModel.schema.properties); - setValidationSchema(buildYupSchema(processedProps)); - const initialValues = Object.keys(processedProps).reduce( - (acc, key) => { - acc[key] = processedProps[key].placeholder || ""; - return acc; - }, - { name: defaultName || "", description: "" }, - ); - formik.setValues(initialValues); - }, [selectedModel, defaultName]); - - const formik = useFormik({ - initialValues: { name: "", description: "" }, - validationSchema, - enableReinitialize: true, - onSubmit: async (values) => { - if (!values.name || values.name.trim() === "") { - setNameError(true); - setNameErrorMessage(t("generative:error.nameRequired")); - return; - } - try { - const createdSession = await createGenerativeSession({ - name: values.name, - description: values.description, - task_name: selectedTaskName, - model_name: selectedModel?.name || "", - parameters: values, - }); - setSessions((prev) => [...prev, createdSession]); - navigate(`/app/generative/sessions/${createdSession.id}`); - enqueueSnackbar(t("generative:message.sessionCreatedSuccess"), { - variant: "success", - }); - if (tourContext?.run && tourContext?.stepIndex === 5) { - const waitForElement = () => { - const el = document.querySelector( - '[data-tour="sessions-left-panel"]', - ); - if (el) setTimeout(() => tourContext.nextStep(), 100); - else setTimeout(waitForElement, 100); - }; - setTimeout(waitForElement, 100); - } - } catch (error) { - console.error("Error creating session:", error); - const errorDetail = error?.response?.data?.detail || ""; - if ( - error?.response?.status === 409 || - errorDetail.includes("already exists") - ) { - enqueueSnackbar(t("generative:error.sessionNameExists"), { - variant: "error", - }); - } else { - enqueueSnackbar(t("generative:error.failedToCreateSession"), { - variant: "error", - }); - } - } - }, - }); - - const handleNameInputChange = (event) => { - formik.handleChange(event); - const empty = event.target.value.trim() === ""; - setNameError(empty); - setNameErrorMessage(empty ? t("generative:error.nameRequired") : ""); - }; - - const handleModelSelect = (model) => { - setSelectedModel(model); - if ( - tourContext?.run && - tourContext?.stepIndex === 3 && - !hasAdvancedTourRef.current - ) { - hasAdvancedTourRef.current = true; - const waitForElement = () => { - const el = document.querySelector('[data-tour="model-parameters"]'); - if (el) { - const distanceToViewportCenter = Math.abs( - el.getBoundingClientRect().top - window.innerHeight / 2, - ); - const scrollWaitMs = Math.min( - 800, - Math.max(300, Math.round(distanceToViewportCenter * 0.7)), - ); - - el.scrollIntoView({ behavior: "smooth", block: "center" }); - setTimeout(() => tourContext.nextStep(), scrollWaitMs); - return; - } - - setTimeout(waitForElement, 100); - }; - setTimeout(waitForElement, 200); - } - }; - - const processedProperties = selectedModel?.schema?.properties - ? preprocessSchema(selectedModel.schema.properties) - : {}; - - return ( - - - {selectedDisplayName}:{" "} - {t("generative:label.selectModelAndConfigureParameters")} - - - - {t("generative:label.selectModel")} - - - - - - - {!selectedModel && ( - - - - - )} - - {selectedModel?.schema && ( - navigate("/app/generative")} - /> - )} - - ); -} diff --git a/DashAI/front/src/components/generative/SelectTaskMenu.jsx b/DashAI/front/src/components/generative/SelectTaskMenu.jsx deleted file mode 100644 index bf28530fc..000000000 --- a/DashAI/front/src/components/generative/SelectTaskMenu.jsx +++ /dev/null @@ -1,57 +0,0 @@ -import { useTranslation } from "react-i18next"; -import { useNavigate } from "react-router-dom"; -import SelectOptionMenu from "../threeSectionLayout/SelectOptionMenu"; -import { useGenerative } from "./GenerativeContext"; -import { useTourContext } from "../tour/TourProvider"; -import { - ChatBubbleOutline as TextToTextIcon, - Image as TextToImageIcon, - Tune as ControlNetIcon, - AutoAwesome as DefaultGenerativeIcon, -} from "@mui/icons-material"; - -const GENERATIVE_TASK_ICONS = { - TextToTextGenerationTask: TextToTextIcon, - TextToImageGenerationTask: TextToImageIcon, - ControlNetTask: ControlNetIcon, -}; - -export default function SelectTaskMenu() { - const { t } = useTranslation(["generative", "common"]); - const navigate = useNavigate(); - const { tasks } = useGenerative(); - const tourContext = useTourContext(); - - const goToNextStep = (taskName) => { - navigate(`/app/generative/sessions/new/${taskName}`); - - if (tourContext?.run && tourContext?.stepIndex === 2) { - const waitForElement = () => { - const element = document.querySelector('[data-tour="model-selection"]'); - if (element) { - tourContext.nextStep(); - } else { - setTimeout(waitForElement, 100); - } - }; - setTimeout(waitForElement, 100); - } - }; - - return ( - ({ - name: task.name, - display_name: task.display_name, - description: task.description, - Icon: GENERATIVE_TASK_ICONS[task.name] || DefaultGenerativeIcon, - }))} - searchBar={false} - dataTour="task-selection" - dataTourTarget="TextToTextGenerationTask" - /> - ); -} diff --git a/DashAI/front/src/components/generative/SessionBox.jsx b/DashAI/front/src/components/generative/SessionBox.jsx index 3e5eb746c..596abaea2 100644 --- a/DashAI/front/src/components/generative/SessionBox.jsx +++ b/DashAI/front/src/components/generative/SessionBox.jsx @@ -42,18 +42,18 @@ export default function SessionBox({ > {name ? name : t("generative:label.untitledSession")} {modelName} diff --git a/DashAI/front/src/components/generative/SessionForm.jsx b/DashAI/front/src/components/generative/SessionForm.jsx deleted file mode 100644 index 0703833bd..000000000 --- a/DashAI/front/src/components/generative/SessionForm.jsx +++ /dev/null @@ -1,96 +0,0 @@ -import { Box, Button, TextField, Typography } from "@mui/material"; -import { useTranslation } from "react-i18next"; -import FormSchemaRenderFields from "../shared/FormSchemaRenderFields"; - -export default function SessionForm({ - formik, - processedProperties, - nameError, - nameErrorMessage, - onNameChange, - onBack, -}) { - const { t } = useTranslation(["generative", "common"]); - - return ( -
- - - - - {t("common:parameters")} - - { - formik.setValues((prevValues) => ({ - ...prevValues, - ...updatedValues, - })); - }} - onFormSubmit={formik.handleSubmit} - setError={(error) => console.error(error)} - errorsMessage={formik.errors} - spacing={2} - /> - - - - {t("generative:label.nameYourSession")} - - - - - - - - - - - - -
- ); -} diff --git a/DashAI/front/src/components/generative/mediaInput/MediaOnlyPlaceholder.jsx b/DashAI/front/src/components/generative/mediaInput/MediaOnlyPlaceholder.jsx index 10eb59877..43ff663de 100644 --- a/DashAI/front/src/components/generative/mediaInput/MediaOnlyPlaceholder.jsx +++ b/DashAI/front/src/components/generative/mediaInput/MediaOnlyPlaceholder.jsx @@ -1,4 +1,4 @@ -import { Box, Stack } from "@mui/material"; +import { Box, Stack, Typography } from "@mui/material"; import { useTranslation } from "react-i18next"; import { MEDIA_KINDS, @@ -38,12 +38,12 @@ export function MediaOnlyPlaceholder({ > {hasAnyMedia ? ( <> - + {t( "generative:label.attachMediaToContinue", "Attach media to continue", )} - + {MEDIA_ORDER.map((kind) => { const { icon, tooltipKey } = MEDIA_KINDS[kind]; @@ -65,12 +65,12 @@ export function MediaOnlyPlaceholder({ ) : ( - + {t( "generative:label.noInputAvailable", "No input available for this task", )} - + )} ); diff --git a/DashAI/front/src/components/generative/mediaInput/MediaPreviewList.jsx b/DashAI/front/src/components/generative/mediaInput/MediaPreviewList.jsx index 93a63790a..0df66d7f5 100644 --- a/DashAI/front/src/components/generative/mediaInput/MediaPreviewList.jsx +++ b/DashAI/front/src/components/generative/mediaInput/MediaPreviewList.jsx @@ -1,4 +1,4 @@ -import { Box, IconButton } from "@mui/material"; +import { Box, IconButton, Typography } from "@mui/material"; import CloseIcon from "@mui/icons-material/Close"; export function MediaPreviewList({ activeKinds, previewsByKind, onRemove }) { @@ -26,7 +26,9 @@ export function MediaPreviewList({ activeKinds, previewsByKind, onRemove }) { }} /> ) : ( - {kind} #{index + 1} - + )} - + {language || "code"} diff --git a/DashAI/front/src/components/hardware/HardwareMonitorButton.jsx b/DashAI/front/src/components/hardware/HardwareMonitorButton.jsx index 11eb0d3e3..b9d0bc592 100644 --- a/DashAI/front/src/components/hardware/HardwareMonitorButton.jsx +++ b/DashAI/front/src/components/hardware/HardwareMonitorButton.jsx @@ -31,8 +31,8 @@ export default function HardwareMonitorButton() { onClick={handleClick} aria-label={t("common:hardwareMonitor.title")} sx={{ - width: 28, - height: 28, + width: 32, + height: 32, borderRadius: "4px", border: `1px solid ${theme.palette.divider}`, color: theme.palette.text.secondary, @@ -42,7 +42,7 @@ export default function HardwareMonitorButton() { }, }} > - + { ? theme.palette.grey[900] : theme.palette.grey[100], p: 2, - fontSize: "0.875rem", overflow: "auto", maxHeight: "200px", }} > - {displayJob.error_msg} + + {displayJob.error_msg} + diff --git a/DashAI/front/src/components/models/AddModelDialog.jsx b/DashAI/front/src/components/models/AddModelDialog.jsx index 965bbd261..751b1f3c3 100644 --- a/DashAI/front/src/components/models/AddModelDialog.jsx +++ b/DashAI/front/src/components/models/AddModelDialog.jsx @@ -294,7 +294,7 @@ function AddModelDialog({ {steps.map((label) => ( - + {label} ))} diff --git a/DashAI/front/src/components/models/CreateSessionSteps.jsx b/DashAI/front/src/components/models/CreateSessionSteps.jsx index 37b39f96f..c96e1d079 100644 --- a/DashAI/front/src/components/models/CreateSessionSteps.jsx +++ b/DashAI/front/src/components/models/CreateSessionSteps.jsx @@ -1,17 +1,17 @@ import { useState, useMemo, useEffect, useRef } from "react"; import PropTypes from "prop-types"; -import { useLocation, useNavigate } from "react-router-dom"; -import { Box, Stepper, Step, StepLabel } from "@mui/material"; +import { Box, Button, Typography } from "@mui/material"; import { useSnackbar } from "notistack"; import { useFormik } from "formik"; import { useTourContext } from "../tour/TourProvider"; import SetNameAndDatasetStep from "./SetNameAndDatasetStep"; import PrepareDatasetStep from "./modelSession/PrepareDatasetStep"; -import FormSchemaButtonGroup from "../shared/FormSchemaButtonGroup"; +import DatasetAutocomplete from "../notebooks/notebookCreation/DatasetAutocomplete"; import { createModelSession } from "../../api/modelSession"; import { getComponents } from "../../api/component"; import { generateSequentialName } from "../../utils/nameGenerator"; import { useTranslation } from "react-i18next"; +import { useModels } from "./ModelsContext"; function CreateSessionSteps({ backHome, @@ -21,18 +21,11 @@ function CreateSessionSteps({ existingSessions = [], preselectedDatasetId = null, }) { - const navigate = useNavigate(); - const location = useLocation(); - const taskName = selectedTask?.name; - const baseSessionPath = taskName - ? `/app/models/sessions/new/${taskName}` - : "/app/models"; - const preparePath = `${baseSessionPath}/prepare`; - const activeStep = location.pathname.startsWith(preparePath) ? 1 : 0; const { enqueueSnackbar } = useSnackbar(); const { t } = useTranslation(["models", "common"]); const tourContext = useTourContext(); const hasAdvancedTourRef = useRef(false); + const { setSessionRightContent } = useModels(); const [selectedDataset, setSelectedDataset] = useState( preselectedDatasetId @@ -40,8 +33,24 @@ function CreateSessionSteps({ : null, ); + const [newExp, setNewExp] = useState({ + name: "", + dataset: null, + task_name: selectedTask?.name || "", + input_columns: [], + output_columns: [], + train_metrics: [], + validation_metrics: [], + test_metrics: [], + splits: {}, + runs: [], + }); + + const [nextEnabled, setNextEnabled] = useState(false); + const handleDatasetChange = (newDataset) => { setSelectedDataset(newDataset); + setNewExp((prev) => ({ ...prev, dataset: newDataset })); if ( tourContext?.run && tourContext?.stepIndex === 5 && @@ -51,12 +60,10 @@ function CreateSessionSteps({ hasAdvancedTourRef.current = true; const waitForElement = () => { const element = document.querySelector( - '[data-tour="models-next-button"]', + '[data-tour="models-validation-alert"]', ); if (element) { - setTimeout(() => { - tourContext.nextStep(); - }, 100); + tourContext.nextStep(); } else { setTimeout(waitForElement, 100); } @@ -65,26 +72,6 @@ function CreateSessionSteps({ } }; - const [newExp, setNewExp] = useState({ - name: "", - dataset: null, - task_name: selectedTask?.name || "", - input_columns: [], - output_columns: [], - train_metrics: [], - validation_metrics: [], - test_metrics: [], - splits: {}, - runs: [], - }); - - const [nextEnabled, setNextEnabled] = useState(false); - - const steps = [ - t("models:label.selectDataset"), - t("models:label.prepareDataset"), - ]; - const { defaultName } = useMemo(() => { if (!selectedTask) { return { defaultName: "" }; @@ -110,34 +97,7 @@ function CreateSessionSteps({ }, enableReinitialize: true, onSubmit: async (values) => { - if (activeStep === 0) { - setNewExp((prev) => ({ - ...prev, - name: values.name.trim(), - dataset: selectedDataset, - task_name: selectedTask?.name || "", - })); - navigate(preparePath); - setNextEnabled(false); - - if (tourContext?.run && tourContext?.stepIndex === 6) { - const waitForElement = () => { - const element = document.querySelector( - '[data-tour="models-validation-alert"]', - ); - if (element) { - setTimeout(() => { - tourContext.nextStep(); - }, 100); - } else { - setTimeout(waitForElement, 100); - } - }; - setTimeout(waitForElement, 300); - } - } else if (activeStep === 1) { - await createSession(); - } + await createSession(values.name.trim()); }, }); @@ -147,23 +107,30 @@ function CreateSessionSteps({ } }, [selectedTask, defaultName, formik]); - const isNextEnabled = (() => { - if (activeStep === 0) { - const isNameValid = formik.values.name.trim().length >= 4; - const isDatasetValid = selectedDataset !== null; - return isNameValid && isDatasetValid; - } - return nextEnabled; - })(); + useEffect(() => { + if (selectedDataset) return; + setSessionRightContent( + + + {t("models:label.selectDatasetFirst")} + + , + ); + return () => setSessionRightContent(null); + }, [selectedDataset]); const getNameError = () => { - if (!selectedDataset) { - return null; - } - const currentName = formik.values.name.trim(); - if (!currentName) { - return t("models:error.nameRequired"); + if (!currentName || currentName.length < 4) { + return null; } const nameExists = existingSessions.some( @@ -180,15 +147,13 @@ function CreateSessionSteps({ const nameError = getNameError(); - const handleBack = () => { - if (activeStep === 0) { - backHome(); - } else { - navigate(baseSessionPath); - } - }; + const isNextEnabled = + formik.values.name.trim().length >= 4 && + !nameError && + selectedDataset !== null && + nextEnabled; - const createSession = async () => { + const createSession = async (sessionName) => { try { setNextEnabled(false); @@ -211,19 +176,15 @@ function CreateSessionSteps({ const hasTest = newExp.splits.test !== undefined && newExp.splits.test !== 0; - const trainMetrics = hasTrain ? allMetricNames : []; - const validationMetrics = hasValidation ? allMetricNames : []; - const testMetrics = hasTest ? allMetricNames : []; - const response = await createModelSession( - newExp.dataset.id, - newExp.task_name, - newExp.name, + selectedDataset.id, + selectedTask?.name || newExp.task_name, + sessionName, newExp.input_columns, newExp.output_columns, - trainMetrics, - validationMetrics, - testMetrics, + hasTrain ? allMetricNames : [], + hasValidation ? allMetricNames : [], + hasTest ? allMetricNames : [], JSON.stringify(newExp.splits), ); @@ -231,6 +192,8 @@ function CreateSessionSteps({ variant: "success", }); + formik.resetForm(); + if (tourContext?.run) { tourContext.stopTour(); sessionStorage.setItem("startModelsSessionTour", "true"); @@ -248,66 +211,78 @@ function CreateSessionSteps({ }; return ( - <> - - - {activeStep === 0 && ( - - )} - {activeStep === 1 && ( - - )} - + + + + {t("models:label.prepareDataset")} + + + {t("models:label.selectDatasetAndPrepare")} + + - - + + + {selectedDataset && ( + - + )} - + > + + + + ); } + CreateSessionSteps.propTypes = { backHome: PropTypes.func.isRequired, selectedTask: PropTypes.object.isRequired, diff --git a/DashAI/front/src/components/models/ModelCenterContent.jsx b/DashAI/front/src/components/models/ModelCenterContent.jsx index a2349718c..c3cfed07b 100644 --- a/DashAI/front/src/components/models/ModelCenterContent.jsx +++ b/DashAI/front/src/components/models/ModelCenterContent.jsx @@ -93,7 +93,7 @@ export default function ModelsCenterContent() { flexDirection: "column", width: "100%", height: "100%", - overflow: "auto", + overflow: "hidden", px: 2, pt: 2, }} diff --git a/DashAI/front/src/components/models/ModelComparisonTable.jsx b/DashAI/front/src/components/models/ModelComparisonTable.jsx index 4a0ae5072..92349e750 100644 --- a/DashAI/front/src/components/models/ModelComparisonTable.jsx +++ b/DashAI/front/src/components/models/ModelComparisonTable.jsx @@ -315,17 +315,21 @@ function ModelComparisonTable({ const isBest = bestScore !== null && Math.abs(score - bestScore) < 1e-6; const tooltipContent = ( - - + + {t("models:label.score")}: {score.toFixed(1)}/100 - + {breakdown.map(({ metric_name, value, normalized_weight }, i) => ( - + {i === 0 ? "=" : "+"} {metric_name} ({value.toFixed(4)}) ×{" "} {(normalized_weight * 100).toFixed(0)}% - + ))} - + ); return ( diff --git a/DashAI/front/src/components/models/ModelsBreadcrumbs.jsx b/DashAI/front/src/components/models/ModelsBreadcrumbs.jsx index b8243d526..f0aaee6b7 100644 --- a/DashAI/front/src/components/models/ModelsBreadcrumbs.jsx +++ b/DashAI/front/src/components/models/ModelsBreadcrumbs.jsx @@ -37,22 +37,6 @@ export default function ModelsBreadcrumbs() { if (path.startsWith("/app/models/sessions/new/") && params.taskName) { const task = tasks.find((tk) => tk.name === params.taskName); const taskLabel = taskDisplayName(task) ?? params.taskName; - - if (path.endsWith("/prepare")) { - return [ - rootCrumb, - { - label: taskLabel, - path: `/app/models/sessions/new/${params.taskName}`, - }, - { - label: t("models:label.prepareDataset"), - path: null, - current: true, - }, - ]; - } - return [ rootCrumb, { diff --git a/DashAI/front/src/components/models/ModelsContext.jsx b/DashAI/front/src/components/models/ModelsContext.jsx index a94f1c50f..198c160a8 100644 --- a/DashAI/front/src/components/models/ModelsContext.jsx +++ b/DashAI/front/src/components/models/ModelsContext.jsx @@ -76,6 +76,7 @@ export function ModelsProvider({ children }) { const [selectedOption, setSelectedOption] = useState(OptionsEnum.NEW); const [datasetInfo, setDatasetInfo] = useState(null); const [datasetTab, setDatasetTab] = useState(0); + const [sessionRightContent, setSessionRightContent] = useState(null); const selectModel = useCallback((model) => { setSelectedModel(model); @@ -152,6 +153,8 @@ export function ModelsProvider({ children }) { setDatasetInfo, datasetTab, setDatasetTab, + sessionRightContent, + setSessionRightContent, }; return ( diff --git a/DashAI/front/src/components/models/ModelsRightBar.jsx b/DashAI/front/src/components/models/ModelsRightBar.jsx index 12e138a3a..df89936dd 100644 --- a/DashAI/front/src/components/models/ModelsRightBar.jsx +++ b/DashAI/front/src/components/models/ModelsRightBar.jsx @@ -32,6 +32,7 @@ export default function ModelsRightBar({ onToggle }) { closeConfig, datasetInfo, setDatasetTab, + sessionRightContent, } = useModels(); const fetchModels = React.useCallback(async () => { @@ -102,6 +103,30 @@ export default function ModelsRightBar({ onToggle }) { } }; + if (sessionRightContent) { + return ( + + + + {t("models:label.configureSession")} + + + + {sessionRightContent} + + + ); + } + return ( { - setSelectedDataset(newDataset); - if (onDatasetChange) { - onDatasetChange(newDataset); - } - }; return ( - - - - {t("models:label.selectDatasetForSession")} - - - - - - - {t("models:label.nameYourSession")} - - - - + + + ); } SetNameAndDatasetStep.propTypes = { formik: PropTypes.object.isRequired, - selectedDataset: PropTypes.object, - setSelectedDataset: PropTypes.func.isRequired, - datasets: PropTypes.array.isRequired, nameError: PropTypes.string, - selectedTask: PropTypes.object, - onDatasetChange: PropTypes.func, }; export default SetNameAndDatasetStep; diff --git a/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx b/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx index 035afce27..aba9c81c7 100644 --- a/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx +++ b/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx @@ -21,6 +21,7 @@ import { useSnackbar } from "notistack"; import { getColorByColumnType } from "../../../utils"; import { useTranslation } from "react-i18next"; import { Trans } from "react-i18next"; +import { useModels } from "../ModelsContext"; /** * Step of the experiment modal: Set the input and output columns to use for clasification * and the splits for training, validation and testing @@ -28,7 +29,8 @@ import { Trans } from "react-i18next"; * @param {function} setNewExp updates the Eperimento Modal state (newExp) * @param {function} setNextEnabled function to enable or disable the "Next" button in the modal */ -function PrepareDatasetStep({ newExp, setNewExp, setNextEnabled }) { +function PrepareDatasetStep({ newExp, setNewExp, setNextEnabled, dataset }) { + const { setSessionRightContent } = useModels(); const [datasetInfo, setDatasetInfo] = useState({}); const [datasetTypes, setDatasetTypes] = useState({}); const { enqueueSnackbar } = useSnackbar(); @@ -87,11 +89,14 @@ function PrepareDatasetStep({ newExp, setNewExp, setNextEnabled }) { const [splitsReady, setSplitsReady] = useState(false); const getDatasetInfo = async () => { + if (!dataset?.id) return; setInfoLoading(true); + setInputColumnNames([]); + setOutputColumnNames([]); try { const [fetchedDatasetInfo, fetchedDatasetTypes] = await Promise.all([ - getDatasetInfoRequest(newExp.dataset.id), - getDatasetTypesRequest(newExp.dataset.id), + getDatasetInfoRequest(dataset.id), + getDatasetTypesRequest(dataset.id), ]); setDatasetInfo(fetchedDatasetInfo); setDatasetTypes(fetchedDatasetTypes); @@ -201,7 +206,7 @@ function PrepareDatasetStep({ newExp, setNewExp, setNextEnabled }) { try { const validation = await validateColumnsRequest( newExp.task_name, - newExp.dataset.id, + dataset.id, inputColumnNames, outputColumnNames, ); @@ -305,9 +310,60 @@ function PrepareDatasetStep({ newExp, setNewExp, setNextEnabled }) { useEffect(() => { getDatasetInfo(); + }, [dataset?.id]); + + useEffect(() => { getTaskRequirements(); }, []); + // Push SplitDatasetRows (or loading spinner) into the right bar + useEffect(() => { + if (infoLoading) { + setSessionRightContent( + + + , + ); + return () => setSessionRightContent(null); + } + setSessionRightContent( + , + ); + return () => setSessionRightContent(null); + }, [ + infoLoading, + datasetInfo, + rowsPartitionsIndex, + rowsPartitionsPercentage, + splitType, + shuffle, + stratify, + seed, + ]); + const renderTypesAsChips = (typesList) => { if (!typesList || typesList.length === 0) { return {t("common:any")}; @@ -463,24 +519,6 @@ function PrepareDatasetStep({ newExp, setNewExp, setNextEnabled }) { infoLoading || (datasetInfo.column_names || []).length === 0 } /> - - ) : ( @@ -507,5 +545,6 @@ PrepareDatasetStep.propTypes = { }), setNewExp: PropTypes.func.isRequired, setNextEnabled: PropTypes.func.isRequired, + dataset: PropTypes.object.isRequired, }; export default PrepareDatasetStep; diff --git a/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx b/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx index 821bcfaa0..c02e73e5d 100644 --- a/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx +++ b/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx @@ -2,17 +2,62 @@ import React, { useEffect, useState } from "react"; import PropTypes from "prop-types"; import { parseRangeToIndex } from "../../../utils/parseRange"; import { + Box, + FormHelperText, Grid, + Paper, + Stack, TextField, + ToggleButton, + ToggleButtonGroup, Typography, - FormControlLabel, - Radio, - RadioGroup, - FormHelperText, } from "@mui/material"; import BooleanInput from "../../configurableObject/Inputs/BooleanInput"; +import FormSchemaFieldCard from "../../shared/FormSchemaFieldCard"; import { useTranslation } from "react-i18next"; +/** + * Splits card shell — same Paper/header visual as FormSchemaFieldCard but WITHOUT + * the label-hiding CSS so Train / Validation / Test TextField labels stay visible. + */ +function SplitsCard({ label, description, errorMessage, children }) { + return ( + + + + {label} + + + + {children} + + {(description || errorMessage) && ( + + + {errorMessage ?? description} + + + )} + + ); +} + function SplitDatasetRows({ datasetInfo, rowsPartitionsIndex, @@ -30,7 +75,7 @@ function SplitDatasetRows({ seed, setSeed, }) { - const { t } = useTranslation(["experiments"]); + const { t } = useTranslation(["experiments", "common"]); const totalRows = datasetInfo.total_rows; const trainDatasetPercentage = (datasetInfo.train_size / totalRows).toFixed( @@ -46,28 +91,16 @@ function SplitDatasetRows({ validationDatasetPercentage > 0 || testDatasetPercentage > 0; - const checkSplit = (train, validation, test) => { - const sum = train + validation + test; - const tolerance = 0.0001; // Allow small floating point errors - return Math.abs(sum - 1) < tolerance; - }; + const checkSplit = (train, validation, test) => + Math.abs(train + validation + test - 1) < 0.0001; - // handle rows numbers change state - const disabledTextFieldStyle = { - "& .MuiInputBase-input.Mui-disabled": { - WebkitTextFillColor: "#999", - }, - "& .MuiInputLabel-root.Mui-disabled": { - color: "#bbb", - }, - }; const [randomSplitError, setRandomSplitError] = useState(false); const [randomSplitErrorText, setRandomSplitErrorText] = useState(""); const [manualSplitError, setManualSplitError] = useState(false); const [manualSplitErrorText, setManualSplitErrorText] = useState(""); - const handleSplitTypeChange = (event) => { - const newType = event.target.value; + const handleSplitTypeChange = (_e, newType) => { + if (!newType) return; setSplitType(newType); if (newType === SPLIT_TYPES.PREDEFINED) { @@ -76,15 +109,12 @@ function SplitDatasetRows({ if (newType === SPLIT_TYPES.RANDOM) { const newSplit = { train: 0.6, test: 0.2, validation: 0.2 }; setRowsPartitionsPercentage(newSplit); - - // Validate the random split const hasZero = newSplit.train === 0; const sumsToOne = checkSplit( newSplit.train, newSplit.validation, newSplit.test, ); - if (hasZero) { setRandomSplitErrorText( t("experiments:error.trainSplitMustBeGreaterThanZero"), @@ -100,16 +130,10 @@ function SplitDatasetRows({ if (newType === SPLIT_TYPES.MANUAL) { const newIndex = { train: [], test: [], validation: [] }; setRowsPartitionsIndex(newIndex); - - // Validate the manual split - if (newIndex.train.length === 0) { - setManualSplitErrorText( - t("experiments:error.trainSplitMustHaveAtLeastOneRow"), - ); - setManualSplitError(true); - } else { - setManualSplitError(false); - } + setManualSplitErrorText( + t("experiments:error.trainSplitMustHaveAtLeastOneRow"), + ); + setManualSplitError(true); } }; @@ -120,23 +144,8 @@ function SplitDatasetRows({ if (splitType === SPLIT_TYPES.MANUAL) { try { const rowsIndex = parseRangeToIndex(value, totalRows); - let updatedIndex = { ...rowsPartitionsIndex }; - - switch (id) { - case "train": - updatedIndex.train = rowsIndex; - break; - case "validation": - updatedIndex.validation = rowsIndex; - break; - case "test": - updatedIndex.test = rowsIndex; - break; - } - + const updatedIndex = { ...rowsPartitionsIndex, [id]: rowsIndex }; setRowsPartitionsIndex(updatedIndex); - - // Validate after update if (updatedIndex.train.length === 0) { setManualSplitErrorText( t("experiments:error.trainSplitMustHaveAtLeastOneRow"), @@ -150,31 +159,15 @@ function SplitDatasetRows({ setManualSplitError(true); } } else { - let newSplit = { ...rowsPartitionsPercentage }; const numValue = parseFloat(value) || 0; - - switch (id) { - case "train": - newSplit = { ...newSplit, train: numValue }; - break; - case "validation": - newSplit = { ...newSplit, validation: numValue }; - break; - case "test": - newSplit = { ...newSplit, test: numValue }; - break; - } - + const newSplit = { ...rowsPartitionsPercentage, [id]: numValue }; setRowsPartitionsPercentage(newSplit); - - // Check if any value is 0 or if sum is not 1 const hasZero = newSplit.train === 0; const sumsToOne = checkSplit( newSplit.train, newSplit.validation, newSplit.test, ); - if (hasZero) { setRandomSplitErrorText( t("experiments:error.trainSplitMustBeGreaterThanZero"), @@ -191,17 +184,12 @@ function SplitDatasetRows({ const handleShuffleChange = (value) => { setShuffle(value); - if (!value) { - setStratify(false); - } + if (!value) setStratify(false); }; const handleStratifyChange = (value) => { - if (shuffle) { - setStratify(value); - } else { - setStratify(false); - } + if (shuffle) setStratify(value); + else setStratify(false); }; const handleSeedChange = (event) => { @@ -218,7 +206,6 @@ function SplitDatasetRows({ }, [hasPredefinedSplits]); useEffect(() => { - // check if splits doesnt have errors and arent empty if (splitType === SPLIT_TYPES.PREDEFINED) { setSplitsReady(true); } else if ( @@ -244,229 +231,208 @@ function SplitDatasetRows({ splitType, ]); + const splitOptions = [ + { + value: SPLIT_TYPES.PREDEFINED, + label: t("experiments:label.predefined"), + disabled: !hasPredefinedSplits, + }, + { value: SPLIT_TYPES.RANDOM, label: t("experiments:label.random") }, + { value: SPLIT_TYPES.MANUAL, label: t("experiments:label.manual") }, + ]; + + const splitFields = [ + { id: "train", label: t("common:train") }, + { id: "validation", label: t("common:validation") }, + { id: "test", label: t("common:test") }, + ]; + return ( - - - - + + {/* Split type selector */} + + + + {t("experiments:label.splitType")} + + + + + {splitOptions.map((opt) => ( + + {opt.label} + + ))} + + {t("experiments:label.selectHowToDivideDataset")} - - - - } - label={ - hasPredefinedSplits - ? t("experiments:label.usePredefinedSplitsFromDataset") - : t( - "experiments:label.usePredefinedSplitsFromDatasetNotAvailable", - ) - } - sx={{ my: 1 }} - disabled={!hasPredefinedSplits} - /> - {splitType === SPLIT_TYPES.PREDEFINED && ( - - - - - - - - - - - - )} - } - label={t("experiments:label.useRandomRowsBySpecifyingPortion")} - sx={{ my: 1 }} - /> - {splitType === SPLIT_TYPES.RANDOM && ( - <> - - - - - - - - + + + + {/* Predefined */} + {splitType === SPLIT_TYPES.PREDEFINED && ( + + + {[ + { id: "train", value: trainDatasetPercentage }, + { id: "validation", value: validationDatasetPercentage }, + { id: "test", value: testDatasetPercentage }, + ].map(({ id, value }) => ( + - {randomSplitError && ( - - {randomSplitErrorText} + ))} + + + )} + + {/* Random */} + {splitType === SPLIT_TYPES.RANDOM && ( + <> + + + {splitFields.map(({ id, label }) => ( + + - )} - - - - - + ))} - - )} - } - label={t( - "experiments:label.useManualSplittingBySpecifyingRowIndexes", - )} - sx={{ my: 1 }} - /> - {splitType === SPLIT_TYPES.MANUAL && ( - <> - - - - - - - - + + + + + + + + + + + + + + + )} + + {/* Manual */} + {splitType === SPLIT_TYPES.MANUAL && ( + + + {splitFields.map(({ id, label }) => ( + - {manualSplitError && ( - - {manualSplitErrorText} - - )} - - - )} - - + ))} + + + )} + ); } SplitDatasetRows.propTypes = { - datasetInfo: PropTypes.shape({ - test_size: PropTypes.number, - total_columns: PropTypes.number, - total_rows: PropTypes.number, - train_size: PropTypes.number, - val_size: PropTypes.number, - }), - rowsPartitionsIndex: PropTypes.shape({ - train: PropTypes.arrayOf(PropTypes.number), - validation: PropTypes.arrayOf(PropTypes.number), - test: PropTypes.arrayOf(PropTypes.number), - }), + datasetInfo: PropTypes.object.isRequired, + rowsPartitionsIndex: PropTypes.object.isRequired, setRowsPartitionsIndex: PropTypes.func.isRequired, - rowsPartitionsPercentage: PropTypes.shape({ - train: PropTypes.number, - validation: PropTypes.number, - test: PropTypes.number, - }), + rowsPartitionsPercentage: PropTypes.object.isRequired, setRowsPartitionsPercentage: PropTypes.func.isRequired, setSplitsReady: PropTypes.func.isRequired, splitType: PropTypes.string.isRequired, diff --git a/DashAI/front/src/components/notebooks/ColumnSelector.jsx b/DashAI/front/src/components/notebooks/ColumnSelector.jsx index 6df58337c..6ff78a76a 100644 --- a/DashAI/front/src/components/notebooks/ColumnSelector.jsx +++ b/DashAI/front/src/components/notebooks/ColumnSelector.jsx @@ -194,6 +194,15 @@ function ColumnSelector({ [getValidColumnIds, rowSelectionModel, inputCardinality], ); + const handleSelectAllRows = useCallback(() => { + const validIds = getValidColumnIds(); + const allValidSelected = + validIds.length > 0 && + validIds.every((id) => rowSelectionModel.includes(id)); + + handleSelection(allValidSelected ? {} : toMRT(validIds)); + }, [getValidColumnIds, rowSelectionModel]); + // Effect to update selection data and validation whenever rowSelectionModel changes useEffect(() => { if (rows.length > 0) { @@ -270,9 +279,9 @@ function ColumnSelector({ enableFullScreenToggle: false, enableHiding: false, enablePagination: true, - muiPaginationProps: { rowsPerPageOptions: [5, 10, 20] }, + muiPaginationProps: { rowsPerPageOptions: [10, 15, 20] }, initialState: { - pagination: { pageSize: 5, pageIndex: 0 }, + pagination: { pageSize: 10, pageIndex: 0 }, density: "compact", }, mrtTheme: { @@ -292,6 +301,15 @@ function ColumnSelector({ } : {}, }), + muiSelectAllCheckboxProps: () => ({ + checked: + getValidColumnIds().length > 0 && + getValidColumnIds().every((id) => rowSelectionModel.includes(id)), + indeterminate: + rowSelectionModel.length > 0 && + rowSelectionModel.length < getValidColumnIds().length, + onChange: handleSelectAllRows, + }), localization, }); diff --git a/DashAI/front/src/components/notebooks/DataBreadcrumbs.jsx b/DashAI/front/src/components/notebooks/DataBreadcrumbs.jsx index 9fbc441bb..7e19195dc 100644 --- a/DashAI/front/src/components/notebooks/DataBreadcrumbs.jsx +++ b/DashAI/front/src/components/notebooks/DataBreadcrumbs.jsx @@ -13,14 +13,17 @@ export default function DataBreadcrumbs() { const location = useLocation(); const params = useParams(); const { t } = useTranslation(["datasets", "common"]); - const { datasets, notebooks } = useDatasetsAndNotebooks(); + const { datasets, notebooks, uploadDataloader } = useDatasetsAndNotebooks(); const rootCrumb = { label: t("common:datasets"), path: "/app/data" }; const getBreadcrumbs = () => { const path = location.pathname; - if (path.startsWith("/app/data/datasets/new/configure")) { + if (path.startsWith("/app/data/datasets/new/") && params.dataloaderName) { + const dataloaderLabel = uploadDataloader + ? uploadDataloader.display_name || uploadDataloader.name + : params.dataloaderName; return [ rootCrumb, { @@ -28,7 +31,7 @@ export default function DataBreadcrumbs() { path: "/app/data/datasets/new", }, { - label: t("datasets:label.configureAndUpload"), + label: dataloaderLabel, path: null, current: true, }, diff --git a/DashAI/front/src/components/notebooks/NoteBox.jsx b/DashAI/front/src/components/notebooks/NoteBox.jsx index 596cced8f..b94f1d58e 100644 --- a/DashAI/front/src/components/notebooks/NoteBox.jsx +++ b/DashAI/front/src/components/notebooks/NoteBox.jsx @@ -10,12 +10,10 @@ export default function NoteBox({ message, className = "", ...props }) { className={className} {...props} sx={{ - mt: 2, p: 2, bgcolor: theme.palette.background.box, borderRadius: 1, border: `1px solid ${theme.palette.ui.divider}`, - mb: 2, }} > { cancelled = true; }; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [explorersAndConverters]); + }, [explorersAndConverters, t]); // Clear search when the selected notebook changes useEffect(() => { @@ -287,7 +286,6 @@ export default function RightBar({ notebook, onToggle }) { notebook, }; }), - // eslint-disable-next-line react-hooks/exhaustive-deps [explorers, datasetColumns, notebook?.id], ); @@ -303,7 +301,6 @@ export default function RightBar({ notebook, onToggle }) { notebook, }; }), - // eslint-disable-next-line react-hooks/exhaustive-deps [converters, datasetColumns, notebook?.id], ); diff --git a/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx b/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx index 34db7a56c..0fb739890 100644 --- a/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx +++ b/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx @@ -60,7 +60,7 @@ export default function ParameterStepConverter({ variant="h6" sx={{ fontWeight: 700, color: "primary.main", mb: 1 }} > - {t("datasets:label.configureParametersStep", { step: 2 })} + {t("datasets:label.configureParameters")} - - {t("datasets:label.selectScopeStep", { step: 1 })} - {error && ( - + {error} )} - + {isLoading ? t("common:loading") : columnType || t("common:unknown")} @@ -196,11 +188,10 @@ export default function EditableColumnHeader({ > - + {columnType || t("common:unknown")} @@ -239,11 +226,7 @@ export default function EditableColumnHeader({ {encoderError && ( - + {encoderError} )} diff --git a/DashAI/front/src/components/notebooks/dataset/tabs/CategoricalTab.jsx b/DashAI/front/src/components/notebooks/dataset/tabs/CategoricalTab.jsx index 77fb19edd..f6594ac7c 100644 --- a/DashAI/front/src/components/notebooks/dataset/tabs/CategoricalTab.jsx +++ b/DashAI/front/src/components/notebooks/dataset/tabs/CategoricalTab.jsx @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useState } from "react"; import { Box, Typography, CardContent } from "@mui/material"; import { useTheme } from "@mui/material/styles"; import TitleIcon from "@mui/icons-material/Title"; @@ -21,6 +21,7 @@ import { useTranslation } from "react-i18next"; export const CategoricalTab = ({ categoricalStats }) => { const { t } = useTranslation(["datasets", "common"]); const theme = useTheme(); + const [activeIndices, setActiveIndices] = useState({}); return ( @@ -85,6 +86,7 @@ export const CategoricalTab = ({ categoricalStats }) => { /> { /> + activeBar={false} + onMouseEnter={(_, index) => + setActiveIndices((prev) => ({ + ...prev, + [column]: index, + })) + } + onMouseLeave={() => + setActiveIndices((prev) => ({ + ...prev, + [column]: null, + })) + } + > + {stats.top_5.map((_, index) => { + const activeIndex = activeIndices[column] ?? null; + return ( + + ); + })} + diff --git a/DashAI/front/src/components/notebooks/dataset/tabs/NumericTab.jsx b/DashAI/front/src/components/notebooks/dataset/tabs/NumericTab.jsx index c0cccbf3f..db96c69b2 100644 --- a/DashAI/front/src/components/notebooks/dataset/tabs/NumericTab.jsx +++ b/DashAI/front/src/components/notebooks/dataset/tabs/NumericTab.jsx @@ -83,11 +83,11 @@ export const NumericTab = ({ numericStats }) => { {/* Distribution Metrics */} {t("datasets:label.distributionMetrics")} @@ -126,11 +126,11 @@ export const NumericTab = ({ numericStats }) => { {/* Shape Indicators */} {t("datasets:label.shapeIndicators")} diff --git a/DashAI/front/src/components/notebooks/dataset/tabs/OverviewTab.jsx b/DashAI/front/src/components/notebooks/dataset/tabs/OverviewTab.jsx index dccf4ce2c..673075203 100644 --- a/DashAI/front/src/components/notebooks/dataset/tabs/OverviewTab.jsx +++ b/DashAI/front/src/components/notebooks/dataset/tabs/OverviewTab.jsx @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useState } from "react"; import { Box, Typography, Card, CardContent, Chip } from "@mui/material"; import { useTheme } from "@mui/material/styles"; import { @@ -9,6 +9,7 @@ import { YAxis, Tooltip, Bar, + Cell, } from "recharts"; import DatasetTable from "../DatasetTable"; import ExportableCard from "../ExportableCard"; @@ -27,6 +28,7 @@ const OverviewTab = ({ const { t } = useTranslation(["datasets", "common"]); const theme = useTheme(); + const [activeBarIndex, setActiveBarIndex] = useState(null); const missingData = Object.entries(nan).map(([col, count]) => ({ column: col, missing: count, @@ -134,6 +136,7 @@ const OverviewTab = ({ /> + activeBar={false} + onMouseEnter={(_, index) => setActiveBarIndex(index)} + onMouseLeave={() => setActiveBarIndex(null)} + > + {missingData.map((_, index) => ( + + ))} + diff --git a/DashAI/front/src/components/notebooks/dataset/tabs/TextTab.jsx b/DashAI/front/src/components/notebooks/dataset/tabs/TextTab.jsx index eb0d73015..742ba6215 100644 --- a/DashAI/front/src/components/notebooks/dataset/tabs/TextTab.jsx +++ b/DashAI/front/src/components/notebooks/dataset/tabs/TextTab.jsx @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useState } from "react"; import { Box, Typography, @@ -27,6 +27,7 @@ import { useTranslation } from "react-i18next"; export const TextTab = ({ textStats }) => { const theme = useTheme(); const { t } = useTranslation(["datasets", "common"]); + const [activeIndices, setActiveIndices] = useState({}); return ( {Object.entries(textStats).map(([column, stats]) => { @@ -139,11 +140,11 @@ export const TextTab = ({ textStats }) => { {t("datasets:label.lengthMetrics")} @@ -196,6 +197,7 @@ export const TextTab = ({ textStats }) => { { /> + activeBar={false} + onMouseEnter={(_, index) => + setActiveIndices((prev) => ({ + ...prev, + [column]: index, + })) + } + onMouseLeave={() => + setActiveIndices((prev) => ({ + ...prev, + [column]: null, + })) + } + > + {lengthData.map((entry, index) => { + const activeIndex = activeIndices[column] ?? null; + return ( + + ); + })} + diff --git a/DashAI/front/src/components/notebooks/datasetCreation/ConfigureAndUploadDatasetStep.jsx b/DashAI/front/src/components/notebooks/datasetCreation/ConfigureAndUploadDatasetStep.jsx index f89ea00ce..72e3bee39 100644 --- a/DashAI/front/src/components/notebooks/datasetCreation/ConfigureAndUploadDatasetStep.jsx +++ b/DashAI/front/src/components/notebooks/datasetCreation/ConfigureAndUploadDatasetStep.jsx @@ -1,12 +1,11 @@ -import { useState, useEffect, useCallback, useRef } from "react"; -import { Grid, CircularProgress } from "@mui/material"; -import FormSchemaButtonGroup from "../../shared/FormSchemaButtonGroup"; +import { useState, useEffect, useCallback, useMemo, useRef } from "react"; +import { Box, Button, Grid, TextField } from "@mui/material"; import Upload from "./Upload"; import { useSnackbar } from "notistack"; import { enqueueDatasetJob as enqueueDatasetRequest } from "../../../api/job"; import { forceRefreshNow } from "../../../utils/jobPoller"; import { useTourContext } from "../../tour/TourProvider"; - +import { generateSequentialName } from "../../../utils/nameGenerator"; import { createDataset } from "../../../api/datasets"; import { useTranslation } from "react-i18next"; import { useTheme } from "@mui/material/styles"; @@ -20,7 +19,14 @@ export default function ConfigureAndUploadDatasetStep({ formValues, onPreviewError, formHasErrors, + existingDatasets = [], }) { + const { defaultName } = useMemo( + () => generateSequentialName({ base: "Dataset", items: existingDatasets }), + [existingDatasets], + ); + const [datasetName, setDatasetName] = useState(""); + const lastAutoFilledRef = useRef(null); const [uploadEnabled, setUploadEnabled] = useState(false); const [uploading, setUploading] = useState(false); const [previewError, setPreviewError] = useState(false); @@ -34,6 +40,14 @@ export default function ConfigureAndUploadDatasetStep({ const { t } = useTranslation(["common", "datasets"]); const theme = useTheme(); + useEffect(() => { + if (!defaultName) return; + if (!datasetName || datasetName === lastAutoFilledRef.current) { + setDatasetName(defaultName); + lastAutoFilledRef.current = defaultName; + } + }, [defaultName]); + useEffect(() => { if (onPreviewError) { onPreviewError(previewError); @@ -67,7 +81,7 @@ export default function ConfigureAndUploadDatasetStep({ // Merge values coming from the form schema and the onValuesChange callback const params = { ...refValues, ...(formValues || {}) }; - const name = params.name || datasetFileToUpload.file.name; + const name = datasetName.trim() || datasetFileToUpload.file.name; params["name"] = name; // Ensure dataloader is passed as a string (backend expects the dataloader name) @@ -176,7 +190,24 @@ export default function ConfigureAndUploadDatasetStep({ ]); return ( - + + + setDatasetName(e.target.value)} + fullWidth + /> + + - {/* Form buttons */} - - {uploading ? ( - - ) : ( - - )} - - + + + + + ); } diff --git a/DashAI/front/src/components/notebooks/datasetCreation/DataloaderConfigBar.jsx b/DashAI/front/src/components/notebooks/datasetCreation/DataloaderConfigBar.jsx index ff970efb9..9e1339339 100644 --- a/DashAI/front/src/components/notebooks/datasetCreation/DataloaderConfigBar.jsx +++ b/DashAI/front/src/components/notebooks/datasetCreation/DataloaderConfigBar.jsx @@ -1,12 +1,15 @@ -import { Box, Typography } from "@mui/material"; +import { Box, TextField, Typography } from "@mui/material"; import PropTypes from "prop-types"; -import { useCallback, useMemo, useRef, useState } from "react"; +import { useCallback, useRef, useState } from "react"; +import { useTheme } from "@mui/material/styles"; import FormSchema from "../../shared/FormSchema"; import FormSchemaContainer from "../../shared/FormSchemaContainer"; -import { generateSequentialName } from "../../../utils/nameGenerator"; import FormInputWrapper from "../../configurableObject/Inputs/FormInputWrapper"; import InputWithDebounce from "../../shared/InputWithDebounce"; +import { generateSequentialName } from "../../../utils/nameGenerator"; +import FormSchemaFieldCard from "../../shared/FormSchemaFieldCard"; import { useTranslation } from "react-i18next"; +import SideBar from "../../threeSectionLayout/panelContainers/SideBar"; /** * Right sidebar component for configuring dataloader parameters @@ -22,22 +25,12 @@ export default function DataloaderConfigBar({ selectedDataloader, formSubmitRef, setError, - existingDatasets = [], onValuesChange, }) { const [inferenceRows, setInferenceRows] = useState(1000); - // Track FormSchema values separately so we can merge with inference_rows const schemaValuesRef = useRef({}); const { t } = useTranslation(["common", "datasets"]); - - const { defaultName } = useMemo( - () => - generateSequentialName({ - base: "Dataset", - items: existingDatasets, - }), - [existingDatasets], - ); + const theme = useTheme(); // Handler for when FormSchema values change - merge with inference_rows const handleFormSchemaValuesChange = useCallback(() => { @@ -51,7 +44,7 @@ export default function DataloaderConfigBar({ // Handler for when inference_rows changes - merge with schema values const handleInferenceRowsChange = useCallback( (val) => { - const numeric = val ? Number(val) : undefined; + const numeric = val ? Math.max(2, Number(val)) : 2; setInferenceRows(numeric); if (onValuesChange) { onValuesChange({ ...schemaValuesRef.current, inference_rows: numeric }); @@ -70,7 +63,7 @@ export default function DataloaderConfigBar({ justifyContent="center" alignItems="center" bgcolor="background.box" - borderRadius={2} + borderBottom={`0.1px solid ${(theme) => theme.palette.divider}`} p={3} > - {/* Inferred configuration - displayed above the autogenerated schema form - This section is not part of the dataloader schema and can be used to - expose inferred/auxiliary parameters (like inference_rows) to the UI. - */} - - - - {t("datasets:label.inferredConfiguration")} - - - - - + + + + {t("datasets:label.dataloaderConfiguration")} + + - {/* Header */} + - - {t("datasets:label.dataloaderConfiguration")} - - - - {/* Configuration Form */} - - - {selectedDataloader} - - - - - + + + + + + + - + ); } @@ -184,6 +137,5 @@ DataloaderConfigBar.propTypes = { selectedDataloader: PropTypes.string, formSubmitRef: PropTypes.shape({ current: PropTypes.any }), setError: PropTypes.func, - existingDatasets: PropTypes.array, onValuesChange: PropTypes.func, }; diff --git a/DashAI/front/src/components/notebooks/datasetCreation/PreviewDatasetTable.jsx b/DashAI/front/src/components/notebooks/datasetCreation/PreviewDatasetTable.jsx index 2db0b818b..2b3e7941d 100644 --- a/DashAI/front/src/components/notebooks/datasetCreation/PreviewDatasetTable.jsx +++ b/DashAI/front/src/components/notebooks/datasetCreation/PreviewDatasetTable.jsx @@ -245,11 +245,10 @@ export default function PreviewDatasetTable({ ) : ( handleStartEdit(field)} sx={{ fontWeight: 600, - fontSize: "0.875rem", cursor: "pointer", transition: "all 0.2s", "&:hover": { diff --git a/DashAI/front/src/components/notebooks/datasetCreation/SelectDataloaderStep.jsx b/DashAI/front/src/components/notebooks/datasetCreation/SelectDataloaderStep.jsx index 9e327db34..1a5271182 100644 --- a/DashAI/front/src/components/notebooks/datasetCreation/SelectDataloaderStep.jsx +++ b/DashAI/front/src/components/notebooks/datasetCreation/SelectDataloaderStep.jsx @@ -1,9 +1,6 @@ -import { useEffect, useState } from "react"; -import { useSnackbar } from "notistack"; -import { getComponents as getComponentsRequest } from "../../../api/component"; -import ItemSelectorWithInfo from "../../custom/ItemSelectorWithInfo"; -import { Grid } from "@mui/material"; -import FormSchemaButtonGroup from "../../shared/FormSchemaButtonGroup"; +import { useEffect } from "react"; +import ComponentSelector from "../../custom/ComponentSelector"; +import { Box, Button, CircularProgress, Stack } from "@mui/material"; import { useTourContext } from "../../tour/TourProvider"; import { useTranslation } from "react-i18next"; @@ -13,41 +10,19 @@ import { useTranslation } from "react-i18next"; * @param {function} goToPrevStep - Function to navigate back to the previous step in the dataset creation flow. * @param {object} selectedDataloader - The currently selected dataloader * @param {function} setSelectedDataloader - Function to update the selected dataloader + * @param {Array} dataloaders - List of available dataloaders (fetched by parent) + * @param {boolean} loadingDataloaders - Whether dataloaders are still loading */ export default function SelectDataloaderStep({ goToNextStep, goToPrevStep, selectedDataloader, setSelectedDataloader, + dataloaders = [], + loadingDataloaders = false, }) { const tourContext = useTourContext(); - const { enqueueSnackbar } = useSnackbar(); const { t } = useTranslation(["datasets", "common"]); - const [dataloaders, setDataloaders] = useState([]); - const [loading, setLoading] = useState(true); - - async function getCompatibleDataloaders() { - setLoading(true); - try { - const dataloaders = await getComponentsRequest({ - selectTypes: ["DataLoader"], - }); - setDataloaders(dataloaders); - } catch (error) { - enqueueSnackbar(t("datasets:error.fetchingDataloaders"), { - variant: "error", - }); - if (error.response) { - console.error("Response error:", error.message); - } else if (error.request) { - console.error("Request error", error.request); - } else { - console.error("Unknown Error", error.message); - } - } finally { - setLoading(false); - } - } const handleNext = () => { if (tourContext?.run) { @@ -65,7 +40,7 @@ export default function SelectDataloaderStep({ }; useEffect(() => { - if (!loading && tourContext?.run) { + if (!loadingDataloaders && tourContext?.run) { setTimeout(() => { const cards = document.querySelectorAll('[role="button"]'); cards.forEach((card) => { @@ -76,27 +51,31 @@ export default function SelectDataloaderStep({ }); }, 100); } - }, [loading, tourContext]); + }, [loadingDataloaders, tourContext]); - // fetches the available dataloaders - useEffect(() => { - getCompatibleDataloaders(); - }, [t]); return ( - - {/* List of dataloaders */} - - {!loading && ( - { + + + {loadingDataloaders ? ( + + + + ) : ( + ({ + ...d, + category: d.metadata?.category, + }))} + categoryKey="category" + selected={selectedDataloader || null} + onSelect={(item) => { setSelectedDataloader(item); if ( tourContext?.run && @@ -105,24 +84,36 @@ export default function SelectDataloaderStep({ tourContext.nextStep(); } }} - data-tour="csv-dataloader-option" + searchPlaceholder={t("datasets:searchDataloaders", { + defaultValue: "Search data loaders...", + })} /> )} - - - - - + + + + + + + ); } diff --git a/DashAI/front/src/components/notebooks/datasetCreation/Upload.jsx b/DashAI/front/src/components/notebooks/datasetCreation/Upload.jsx index 15ea33a80..d506b2663 100644 --- a/DashAI/front/src/components/notebooks/datasetCreation/Upload.jsx +++ b/DashAI/front/src/components/notebooks/datasetCreation/Upload.jsx @@ -356,7 +356,12 @@ function Upload({ container direction="column" rowSpacing={1} - sx={{ width: "100%", bgcolor: theme.palette.ui.box }} + sx={{ + width: "100%", + bgcolor: theme.palette.ui.box, + p: 2, + borderRadius: 2, + }} data-tour="upload-area" > {/* state text */} diff --git a/DashAI/front/src/components/notebooks/datasetCreation/UploadDatasetSteps.jsx b/DashAI/front/src/components/notebooks/datasetCreation/UploadDatasetSteps.jsx index 063bf46af..631165778 100644 --- a/DashAI/front/src/components/notebooks/datasetCreation/UploadDatasetSteps.jsx +++ b/DashAI/front/src/components/notebooks/datasetCreation/UploadDatasetSteps.jsx @@ -1,15 +1,17 @@ import { useState, useRef, useEffect } from "react"; -import { useLocation, useNavigate } from "react-router-dom"; +import { useLocation, useNavigate, useParams } from "react-router-dom"; import SelectDataloaderStep from "./SelectDataloaderStep"; import ConfigureAndUploadDatasetStep from "./ConfigureAndUploadDatasetStep"; import DataloaderConfigBar from "./DataloaderConfigBar"; -import CustomLayout from "../../custom/CustomLayout"; +import { Box, Typography } from "@mui/material"; +import ComponentDetailsPanel from "../../custom/ComponentDetailsPanel"; import { useTranslation } from "react-i18next"; import { useDatasetsAndNotebooks } from "../../custom/contexts/DatasetsAndNotebooksContext"; import { useTourContext } from "../../tour/TourProvider"; +import { getComponents as getComponentsRequest } from "../../../api/component"; +import { useSnackbar } from "notistack"; const UPLOAD_BASE_PATH = "/app/data/datasets/new"; -const UPLOAD_CONFIGURE_PATH = `${UPLOAD_BASE_PATH}/configure`; export default function UploadDatasetSteps({ backHome }) { const { @@ -17,22 +19,58 @@ export default function UploadDatasetSteps({ backHome }) { addDatasetOptimistically, startDatasetPolling, setRightBarContent, + setUploadDataloader, } = useDatasetsAndNotebooks(); const navigate = useNavigate(); - const location = useLocation(); - const step = location.pathname.startsWith(UPLOAD_CONFIGURE_PATH) ? 1 : 0; - const [selectedDataloader, setSelectedDataloader] = useState({}); + const { dataloaderName } = useParams(); + const step = dataloaderName ? 1 : 0; + + const [selectedDataloader, setSelectedDataloader] = useState(); + const [dataloaders, setDataloaders] = useState([]); + const [loadingDataloaders, setLoadingDataloaders] = useState(true); const [formValues, setFormValues] = useState({}); const [error, setError] = useState(false); const [previewError, setPreviewError] = useState(false); const { t } = useTranslation(["datasets"]); + const { enqueueSnackbar } = useSnackbar(); const tourContext = useTourContext(); const formSubmitRef = useRef(null); + useEffect(() => { + async function fetchDataloaders() { + setLoadingDataloaders(true); + try { + const list = await getComponentsRequest({ + selectTypes: ["DataLoader"], + }); + setDataloaders(list); + } catch (error) { + enqueueSnackbar(t("datasets:error.fetchingDataloaders"), { + variant: "error", + }); + } finally { + setLoadingDataloaders(false); + } + } + fetchDataloaders(); + }, [t]); + + // Sync selected dataloader from URL param, or redirect if unknown + useEffect(() => { + if (!dataloaderName || loadingDataloaders || dataloaders.length === 0) + return; + const match = dataloaders.find((d) => d.name === dataloaderName); + if (match) { + setSelectedDataloader(match); + } else { + navigate(UPLOAD_BASE_PATH, { replace: true }); + } + }, [dataloaderName, loadingDataloaders, dataloaders, navigate]); + const goToNextStep = () => { - navigate(UPLOAD_CONFIGURE_PATH); + navigate(`${UPLOAD_BASE_PATH}/${selectedDataloader.name}`); }; const goToPrevStep = () => { @@ -40,10 +78,20 @@ export default function UploadDatasetSteps({ backHome }) { backHome(); return; } - navigate(UPLOAD_BASE_PATH); }; + const getTitle = () => { + switch (step) { + case 0: + return t("datasets:label.selectDataloader"); + case 1: + return t("datasets:label.uploadDataset"); + default: + return t("datasets:label.createDataset"); + } + }; + const getSubtitle = () => { switch (step) { case 0: @@ -55,28 +103,40 @@ export default function UploadDatasetSteps({ backHome }) { } }; + // Sync selected dataloader to context so DataBreadcrumbs can show display_name useEffect(() => { - if (step === 1 && Object.keys(selectedDataloader).length === 0) { - navigate(UPLOAD_BASE_PATH, { replace: true }); + if (setUploadDataloader) { + setUploadDataloader(selectedDataloader?.name ? selectedDataloader : null); } - }, [step, selectedDataloader, navigate]); + }, [selectedDataloader, setUploadDataloader]); + + // Clear right sidebar and dataloader on unmount + useEffect(() => { + return () => { + if (setRightBarContent) setRightBarContent(null); + if (setUploadDataloader) setUploadDataloader(null); + }; + }, [setRightBarContent, setUploadDataloader]); // Update the right sidebar based on current step useEffect(() => { - if (setRightBarContent) { - if (step === 1 && Object.entries(selectedDataloader).length !== 0) { - setRightBarContent( - , - ); - } else { - setRightBarContent(null); - } + if (!setRightBarContent) return; + + if (step === 0) { + setRightBarContent( + , + ); + } else if (step === 1 && selectedDataloader?.name) { + setRightBarContent( + , + ); + } else { + setRightBarContent(null); } }, [step, selectedDataloader, datasets, setRightBarContent]); @@ -93,20 +153,34 @@ export default function UploadDatasetSteps({ backHome }) { }; return ( - + + + {getTitle()} + + + {getSubtitle()} + + + {step === 0 && ( )} - {step === 1 && Object.entries(selectedDataloader).length !== 0 && ( + {step === 1 && selectedDataloader?.name && ( )} - + ); } diff --git a/DashAI/front/src/components/notebooks/explorer/ExplorerDetailsModal.jsx b/DashAI/front/src/components/notebooks/explorer/ExplorerDetailsModal.jsx index 9bd82e845..432995214 100644 --- a/DashAI/front/src/components/notebooks/explorer/ExplorerDetailsModal.jsx +++ b/DashAI/front/src/components/notebooks/explorer/ExplorerDetailsModal.jsx @@ -193,13 +193,7 @@ export default function ExplorerDetailsModal({ {t("common:created")} - + {formatDate(explorer.created)} diff --git a/DashAI/front/src/components/notebooks/explorer/plotLayout/ColorscaleSelector.jsx b/DashAI/front/src/components/notebooks/explorer/plotLayout/ColorscaleSelector.jsx index 0d69fa74c..2032285bb 100644 --- a/DashAI/front/src/components/notebooks/explorer/plotLayout/ColorscaleSelector.jsx +++ b/DashAI/front/src/components/notebooks/explorer/plotLayout/ColorscaleSelector.jsx @@ -102,12 +102,11 @@ export default function ColorscaleSelector({ value, onChange }) { {/* Stop Index Label */} - {i + 1} - + - {name} + + {name} + - {key} + + {key} + - {formatValue(value)} + + {formatValue(value)} + ))} diff --git a/DashAI/front/src/components/notebooks/explorerCreation/ParameterStepExplorer.jsx b/DashAI/front/src/components/notebooks/explorerCreation/ParameterStepExplorer.jsx index d0739604a..50c299e1d 100644 --- a/DashAI/front/src/components/notebooks/explorerCreation/ParameterStepExplorer.jsx +++ b/DashAI/front/src/components/notebooks/explorerCreation/ParameterStepExplorer.jsx @@ -50,9 +50,7 @@ export default function ParameterStepExplorer({ variant="h6" sx={{ fontWeight: 700, color: "primary.main", mb: 1 }} > - {t("datasets:label.configureParametersStep", { - step: 2, - })} + {t("datasets:label.configureParameters")} {/* Content */} - - {t("datasets:label.selectScopeStep", { - step: 1, - })} - - - {t("datasets:label.selectedDataset")} - diff --git a/DashAI/front/src/components/notebooks/notebookCreation/UploadNotebookSteps.jsx b/DashAI/front/src/components/notebooks/notebookCreation/UploadNotebookSteps.jsx index b08c78edb..90e6fac0b 100644 --- a/DashAI/front/src/components/notebooks/notebookCreation/UploadNotebookSteps.jsx +++ b/DashAI/front/src/components/notebooks/notebookCreation/UploadNotebookSteps.jsx @@ -1,12 +1,9 @@ -import { useState, useMemo, useEffect } from "react"; -import { Typography, TextField, Box } from "@mui/material"; +import { useState, useMemo, useEffect, useRef } from "react"; +import { Typography, TextField, Box, Button } from "@mui/material"; import { useFormik } from "formik"; -import CustomLayout from "../../custom/CustomLayout"; -import FormSchemaButtonGroup from "../../shared/FormSchemaButtonGroup"; import DatasetAutocomplete from "./DatasetAutocomplete"; import { createNotebook } from "../../../api/notebook"; import { useSnackbar } from "notistack"; -import { generateSequentialName } from "../../../utils/nameGenerator"; import NoteBox from "../NoteBox"; import { useTourContext } from "../../tour/TourProvider"; import { useTranslation } from "react-i18next"; @@ -27,17 +24,15 @@ export default function UploadNotebookSteps({ const tourContext = useTourContext(); const { t } = useTranslation(["datasets", "common"]); - const { defaultName } = useMemo(() => { - if (!selectedDataset) { - return { defaultName: "" }; - } + const defaultName = useMemo(() => { + const maxId = existingNotebooks.reduce( + (max, nb) => Math.max(max, nb.id ?? 0), + 0, + ); + return `Notebook_${maxId + 1}`; + }, [existingNotebooks]); - return generateSequentialName({ - base: `Notebook_${selectedDataset.name}`, - items: existingNotebooks, - filter: (notebook) => notebook.dataset_id === selectedDataset.id, - }); - }, [selectedDataset, existingNotebooks]); + const lastAutoFilledRef = useRef(null); const formik = useFormik({ initialValues: { @@ -79,117 +74,109 @@ export default function UploadNotebookSteps({ }); useEffect(() => { - if (selectedDataset && defaultName && !formik.values.name.trim()) { - formik.setValues({ - name: defaultName, - description: formik.values.description, - }); - } - }, [ - selectedDataset, - defaultName, - formik.values.name, - formik.values.description, - ]); - - const getNameError = () => { - if (!selectedDataset) { - return null; - } - + if (!defaultName) return; const currentName = formik.values.name.trim(); - if (!currentName) { - return t("common:nameRequired"); + if (!currentName || currentName === lastAutoFilledRef.current) { + formik.setFieldValue("name", defaultName); + lastAutoFilledRef.current = defaultName; } - return null; - }; + }, [defaultName]); - const nameError = getNameError(); + const nameError = formik.values.name.trim() ? null : t("common:nameRequired"); + const isValid = selectedDataset && !nameError; return ( - - - + + {t("datasets:label.createNewNotebook")} + + + {t("datasets:label.createNewNotebookDescription")} + + + + - {t("datasets:label.selectDatasetForNotebook")} - - - + + + + + + + + + - {t("datasets:label.nameYourNotebook")} - - {/* Notebook name */} - - {/* Notebook description */} - - - + + - + ); } diff --git a/DashAI/front/src/components/notebooks/tool/ConfigureToolModal.jsx b/DashAI/front/src/components/notebooks/tool/ConfigureToolModal.jsx index 9464d5e82..3596ce13a 100644 --- a/DashAI/front/src/components/notebooks/tool/ConfigureToolModal.jsx +++ b/DashAI/front/src/components/notebooks/tool/ConfigureToolModal.jsx @@ -121,7 +121,7 @@ export default function ConfigureToolModal({ {steps.map((label) => ( - + {label} ))} diff --git a/DashAI/front/src/components/pipelines/CustomNode.jsx b/DashAI/front/src/components/pipelines/CustomNode.jsx index d49542e0e..9e765079b 100644 --- a/DashAI/front/src/components/pipelines/CustomNode.jsx +++ b/DashAI/front/src/components/pipelines/CustomNode.jsx @@ -123,7 +123,8 @@ const CustomNode = ({ data, isConnectable }) => { }} > {data.name || data.label} diff --git a/DashAI/front/src/components/predictions/DatasetSelector.jsx b/DashAI/front/src/components/predictions/DatasetSelector.jsx index 77b083825..268ef3799 100644 --- a/DashAI/front/src/components/predictions/DatasetSelector.jsx +++ b/DashAI/front/src/components/predictions/DatasetSelector.jsx @@ -74,9 +74,9 @@ function DatasetSelector({ {selectedDataset && ( <> - + {t("prediction:label.predictionInfo")} - + {t("prediction:label.inputColumns")}: diff --git a/DashAI/front/src/components/predictions/PredictionsTable.jsx b/DashAI/front/src/components/predictions/PredictionsTable.jsx index ef3007573..2252e0076 100644 --- a/DashAI/front/src/components/predictions/PredictionsTable.jsx +++ b/DashAI/front/src/components/predictions/PredictionsTable.jsx @@ -92,7 +92,7 @@ function PredictionsTable({ predictions, onItemClick, onItemDelete }) { {t("prediction:label.manualInput")} diff --git a/DashAI/front/src/components/shared/FormSchemaButtonGroup.jsx b/DashAI/front/src/components/shared/FormSchemaButtonGroup.jsx index ed03723ae..9046a6cdd 100644 --- a/DashAI/front/src/components/shared/FormSchemaButtonGroup.jsx +++ b/DashAI/front/src/components/shared/FormSchemaButtonGroup.jsx @@ -1,4 +1,4 @@ -import { Button, ButtonGroup } from "@mui/material"; +import { Button, Box } from "@mui/material"; import PropTypes from "prop-types"; import { useTranslation } from "react-i18next"; @@ -29,7 +29,17 @@ function FormSchemaButtonGroup({ : undefined); return ( - + {onCancel && ( )} - + ); } diff --git a/DashAI/front/src/components/shared/FormSchemaDialog.jsx b/DashAI/front/src/components/shared/FormSchemaDialog.jsx index 24386ac33..966397b7d 100644 --- a/DashAI/front/src/components/shared/FormSchemaDialog.jsx +++ b/DashAI/front/src/components/shared/FormSchemaDialog.jsx @@ -38,7 +38,7 @@ function FormSchemaDialog({ }, }} > - + - {count} - + {open ? ( {title} - {totalCount} - + {/* Groups - Scrollable */} @@ -176,7 +177,7 @@ export default function GroupedCollapsibleList({ {groupName} - {items?.length || 0} - + {/* Group Items */} diff --git a/DashAI/front/src/components/threeSectionLayout/ItemBox.jsx b/DashAI/front/src/components/threeSectionLayout/ItemBox.jsx index e81fe5a97..f63164f35 100644 --- a/DashAI/front/src/components/threeSectionLayout/ItemBox.jsx +++ b/DashAI/front/src/components/threeSectionLayout/ItemBox.jsx @@ -135,20 +135,15 @@ const ItemBox = forwardRef(function ItemBox( }} /> ) : ( - + {editedName} )} {description ? description : ""} diff --git a/DashAI/front/src/components/threeSectionLayout/OptionBox.jsx b/DashAI/front/src/components/threeSectionLayout/OptionBox.jsx index b2684ca9a..bed9019e3 100644 --- a/DashAI/front/src/components/threeSectionLayout/OptionBox.jsx +++ b/DashAI/front/src/components/threeSectionLayout/OptionBox.jsx @@ -94,8 +94,8 @@ export default function OptionBox({ {/* Title */} ))} - → - + ); diff --git a/DashAI/front/src/components/threeSectionLayout/SelectOptionMenu.jsx b/DashAI/front/src/components/threeSectionLayout/SelectOptionMenu.jsx index eaecd88a7..64d4b3e5d 100644 --- a/DashAI/front/src/components/threeSectionLayout/SelectOptionMenu.jsx +++ b/DashAI/front/src/components/threeSectionLayout/SelectOptionMenu.jsx @@ -64,7 +64,6 @@ export default function SelectOptionMenu({ ( @@ -86,7 +85,7 @@ export default function SelectOptionMenu({ return ( diff --git a/DashAI/front/src/components/threeSectionLayout/panelContainers/SideBar.jsx b/DashAI/front/src/components/threeSectionLayout/panelContainers/SideBar.jsx index 11a5c1bcc..3d42ba7c2 100644 --- a/DashAI/front/src/components/threeSectionLayout/panelContainers/SideBar.jsx +++ b/DashAI/front/src/components/threeSectionLayout/panelContainers/SideBar.jsx @@ -1,21 +1,21 @@ import { Box } from "@mui/material"; +import { useTheme } from "@mui/material/styles"; + +export default function SideBar({ children, ...props }) { + const theme = useTheme(); -export default function SideBar({ children }) { return ( {children} diff --git a/DashAI/front/src/components/threeSectionLayout/panels/LeftPanel.jsx b/DashAI/front/src/components/threeSectionLayout/panels/LeftPanel.jsx index 927217220..488122339 100644 --- a/DashAI/front/src/components/threeSectionLayout/panels/LeftPanel.jsx +++ b/DashAI/front/src/components/threeSectionLayout/panels/LeftPanel.jsx @@ -22,8 +22,7 @@ export default function LeftPanel({ children, "data-tour": dataTour }) { transform: "translateY(-50%)", bgcolor: "primary.main", color: "primary.contrastText", - border: "1px solid", - borderColor: "primary.dark", + // border: "1px solid", boxShadow: 2, width: 18, height: 32, diff --git a/DashAI/front/src/components/threeSectionLayout/panels/RightPanel.jsx b/DashAI/front/src/components/threeSectionLayout/panels/RightPanel.jsx index 18917666f..2db74b4ed 100644 --- a/DashAI/front/src/components/threeSectionLayout/panels/RightPanel.jsx +++ b/DashAI/front/src/components/threeSectionLayout/panels/RightPanel.jsx @@ -25,8 +25,6 @@ export default function RightPanel({ transform: "translateY(-50%)", bgcolor: "primary.main", color: "primary.contrastText", - border: "1px solid", - borderColor: "primary.dark", boxShadow: 2, width: 18, height: 32, diff --git a/DashAI/front/src/components/tour/CustomTooltip.jsx b/DashAI/front/src/components/tour/CustomTooltip.jsx index 408d33693..863060361 100644 --- a/DashAI/front/src/components/tour/CustomTooltip.jsx +++ b/DashAI/front/src/components/tour/CustomTooltip.jsx @@ -36,7 +36,7 @@ export const CustomTooltip = ({ lineHeight: "1.6", color: "#333", "& h3": { - fontSize: "18px", + fontSize: "22px", fontWeight: "bold", marginBottom: "8px", marginTop: 0, @@ -136,8 +136,8 @@ export const CustomTooltip = ({ {continuous && ( - + ); diff --git a/DashAI/front/src/constants/tours/datasetViewTour.js b/DashAI/front/src/constants/tours/datasetViewTour.js index d65e919a5..863d3ae6e 100644 --- a/DashAI/front/src/constants/tours/datasetViewTour.js +++ b/DashAI/front/src/constants/tours/datasetViewTour.js @@ -52,6 +52,21 @@ export const datasetViewTourSteps = [ disableCloseOnEsc: true, disableBackButton: true, }, + { + target: '[data-tour="models-dataset-selection"]', + content: ( + +
+

+

+
+ + ), + placement: "bottom", + disableBeacon: true, + disableOverlayClose: true, + disableBackButton: true, + }, { target: '[data-tour="create-notebook-button"]', content: ( diff --git a/DashAI/front/src/constants/tours/datasetsTour.js b/DashAI/front/src/constants/tours/datasetsTour.js index 2eff86dba..e35a4337f 100644 --- a/DashAI/front/src/constants/tours/datasetsTour.js +++ b/DashAI/front/src/constants/tours/datasetsTour.js @@ -65,7 +65,7 @@ export const datasetsTourSteps = [ onMouseOut={(e) => (e.target.style.backgroundColor = "#ef9f27")} >

-

+

@@ -148,9 +148,7 @@ export const datasetsTourSteps = [ -

+

), diff --git a/DashAI/front/src/constants/tours/experimentsTour.js b/DashAI/front/src/constants/tours/experimentsTour.js index d9da1dc17..d265ec011 100644 --- a/DashAI/front/src/constants/tours/experimentsTour.js +++ b/DashAI/front/src/constants/tours/experimentsTour.js @@ -523,7 +523,7 @@ export const experimentsTourSteps = [ padding: "8px", borderRadius: "4px", marginTop: "10px", - fontSize: "0.9em", + fontSize: "14px", }} > ⏱️ Training may take a few moments depending on your dataset size and diff --git a/DashAI/front/src/constants/tours/modelsTour.js b/DashAI/front/src/constants/tours/modelsTour.js index 8217b4d6b..287256f6d 100644 --- a/DashAI/front/src/constants/tours/modelsTour.js +++ b/DashAI/front/src/constants/tours/modelsTour.js @@ -110,24 +110,6 @@ export const modelsTourSteps = [ disableBackButton: true, maxWidth: "320px", }, - { - target: '[data-tour="models-next-button"]', - content: ( - -
-

-

- -

-
- - ), - placement: "top", - disableBeacon: true, - spotlightClicks: true, - isInteractive: true, - disableBackButton: true, - }, { target: '[data-tour="models-validation-alert"]', content: ( @@ -222,7 +204,7 @@ export const modelsTourSteps = [ ), - placement: "right", + placement: "left", disableBeacon: true, maxWidth: "320px", }, @@ -239,9 +221,7 @@ export const modelsTourSteps = [

-

+

), diff --git a/DashAI/front/src/hooks/useSchema.js b/DashAI/front/src/hooks/useSchema.js index 2ad3ade53..a0ad03982 100644 --- a/DashAI/front/src/hooks/useSchema.js +++ b/DashAI/front/src/hooks/useSchema.js @@ -14,6 +14,8 @@ export default function useSchema({ modelName = null } = {}) { const { t } = useTranslation(); useEffect(() => { + setModel(null); + const getModel = async () => { try { setLoading(true); diff --git a/DashAI/front/src/pages/datasets/DatasetsContent.jsx b/DashAI/front/src/pages/datasets/DatasetsContent.jsx index d55739d84..733429002 100644 --- a/DashAI/front/src/pages/datasets/DatasetsContent.jsx +++ b/DashAI/front/src/pages/datasets/DatasetsContent.jsx @@ -28,6 +28,7 @@ export default function DatasetsContent() { notebooks, selectedNotebookId, selectedDatasetId, + selectedOption, rightBarContent, step, selectDataset, @@ -119,7 +120,10 @@ export default function DatasetsContent() { - ) : selectedDatasetId ? ( + ) : selectedDatasetId || + (step === 1 && + selectedOption === OptionsEnum.NOTEBOOK && + location.state?.preselectedDatasetId) ? ( { const path = location.pathname; - if (path.startsWith("/app/generative/sessions/new/") && params.taskName) { - const task = tasks.find((tk) => tk.name === params.taskName); - setSelectedTaskName(params.taskName); - setSelectedDisplayName(task?.display_name ?? null); + if (isCreating) { setSelectedSessionId(null); + setSelectedTaskName(""); + setSelectedDisplayName(null); setStepIndex(1); return; } @@ -65,7 +68,7 @@ export default function GenerativeContent() { setSelectedDisplayName(null); setStepIndex(0); } - }, [location.pathname, params.id, params.taskName, tasks, sessions]); + }, [location.pathname, params.id, tasks, sessions, isCreating]); useEffect(() => { setDisabled?.( @@ -74,39 +77,34 @@ export default function GenerativeContent() { ); }, [stepIndex, selectedSessionId, setDisabled, t]); - return ( + const renderCenter = () => { + if (selectedSessionId) return ; + if (isCreating) return ; + return ; + }; + + const renderRight = () => { + if (isCreating) return ; + return ; + }; + + const layout = ( - - {selectedSessionId ? ( - - ) : stepIndex === 0 ? ( - - ) : ( - - - - - )} - - + {renderCenter()} - + {renderRight()} ); + + return isCreating ? ( + {layout} + ) : ( + layout + ); } diff --git a/DashAI/front/src/pages/home/Home.jsx b/DashAI/front/src/pages/home/Home.jsx index dbb294c69..5dcc846f9 100644 --- a/DashAI/front/src/pages/home/Home.jsx +++ b/DashAI/front/src/pages/home/Home.jsx @@ -268,18 +268,15 @@ function Home() { : theme.palette.background.box, }} > - + {t("home:label.welcomeDashboardAI")} diff --git a/DashAI/front/src/pages/plugins/components/PluginsCard.jsx b/DashAI/front/src/pages/plugins/components/PluginsCard.jsx index 137743296..be5031bdd 100644 --- a/DashAI/front/src/pages/plugins/components/PluginsCard.jsx +++ b/DashAI/front/src/pages/plugins/components/PluginsCard.jsx @@ -97,7 +97,7 @@ function PluginsCard({ - + {t("models:label.generalMetrics")} @@ -29,7 +29,7 @@ function ResultsGraphsSwitch({ showCustomMetrics, handleToggleMetrics }) { /> - + {t("models:label.customMetrics")} diff --git a/DashAI/front/src/styles/theme.js b/DashAI/front/src/styles/theme.js index bd6fac82b..1dc4c92d5 100644 --- a/DashAI/front/src/styles/theme.js +++ b/DashAI/front/src/styles/theme.js @@ -168,57 +168,49 @@ const getTheme = (mode) => ({ typography: { fontFamily: '"IBM Plex Sans", sans-serif', - // --- SANS SERIF --- - pageTitle: { - fontSize: "18px", - fontWeight: 600, - letterSpacing: "-0.01em", - }, // Main titles - - cardTitle: { - fontSize: "15px", - fontWeight: 600, - letterSpacing: "-0.01em", - }, // Module/Card titles - - navItem: { - fontSize: "12.5px", - fontWeight: 400, - }, // Sidebar links/Navigation - - description: { + // --- ESCALA DE TITULARES --- + h1: { fontSize: "28px", fontWeight: 700, letterSpacing: "-0.01em" }, + h2: { fontSize: "22px", fontWeight: 600, letterSpacing: "-0.01em" }, + h3: { fontSize: "20px", fontWeight: 600, letterSpacing: "-0.01em" }, + h4: { fontSize: "17px", fontWeight: 600, letterSpacing: "-0.01em" }, + h5: { fontSize: "16px", fontWeight: 600 }, + h6: { fontSize: "14px", fontWeight: 600 }, + + // --- ESCALA DE CUERPO --- + subtitle1: { fontSize: "17px", fontWeight: 400 }, // Parámetros principales + subtitle2: { fontSize: "16px", fontWeight: 400 }, // Texto destacado + body1: { fontSize: "14px", fontWeight: 400, lineHeight: 1.6 }, // Cuerpo / párrafos + body2: { fontSize: "12px", fontWeight: 400, lineHeight: 1.5 }, // Texto auxiliar / labels + caption: { fontSize: "12px", fontWeight: 400 }, // Información secundaria, captions + code: { + fontFamily: '"IBM Plex Mono", monospace', fontSize: "12px", - fontWeight: 300, - lineHeight: 1.65, - color: "rgba(171, 178, 191, 0.45)", - }, // Explanatory text + fontWeight: 400, + }, // Code snippets, valores técnicos - // --- MONOSPACE --- + // --- VARIANTES UI FUNCIONALES --- + navItem: { fontSize: "12px", fontWeight: 400 }, // Sidebar links/Navigation tabLabel: { fontFamily: '"IBM Plex Mono", monospace', fontSize: "10px", letterSpacing: "0.12em", textTransform: "uppercase", }, // Navigation tabs - sectionLabel: { fontFamily: '"IBM Plex Mono", monospace', fontSize: "9px", letterSpacing: "0.2em", textTransform: "uppercase", }, // Sidebar section headers - statusBadge: { fontFamily: '"IBM Plex Mono", monospace', fontSize: "8.5px", letterSpacing: "0.12em", textTransform: "uppercase", }, - - // Others button: { - fontSize: "15px", - fontWeight: 400, + fontSize: "14px", + fontWeight: 500, letterSpacing: "-0.01em", textTransform: "uppercase", }, diff --git a/DashAI/front/src/types/component.ts b/DashAI/front/src/types/component.ts index 6234fe108..9bedf444b 100644 --- a/DashAI/front/src/types/component.ts +++ b/DashAI/front/src/types/component.ts @@ -8,4 +8,6 @@ export interface IComponent { schema: IParameterJsonSchema; metadata: ITaskMetadataParameters; description: string; + display_name?: string; + color?: string; } diff --git a/DashAI/front/src/utils/i18n/locales/en/common.json b/DashAI/front/src/utils/i18n/locales/en/common.json index 90e7e9af0..18b7a6bc5 100644 --- a/DashAI/front/src/utils/i18n/locales/en/common.json +++ b/DashAI/front/src/utils/i18n/locales/en/common.json @@ -215,5 +215,7 @@ "expand": "Expand", "optimize": "Optimize", "viewMore": "View more", - "viewLess": "View less" + "viewLess": "View less", + "switchToDarkMode": "Switch to dark mode", + "switchToLightMode": "Switch to light mode" } diff --git a/DashAI/front/src/utils/i18n/locales/en/custom.json b/DashAI/front/src/utils/i18n/locales/en/custom.json index 1a2187d41..7ca90eba3 100644 --- a/DashAI/front/src/utils/i18n/locales/en/custom.json +++ b/DashAI/front/src/utils/i18n/locales/en/custom.json @@ -1,5 +1,14 @@ { "inferenceMethods": "Inference Methods", "selectAnItemToShowInfo": "Select an item to see the description.", - "selectInferenceMethods": "Select the inference methods you want to apply" + "selectInferenceMethods": "Select the inference methods you want to apply", + "search": "Search", + "noItemsFound": "No components found", + "tryAdjustingSearch": "Try adjusting your search or filters", + "componentsAvailable_one": "{{count}} component available", + "componentsAvailable_other": "{{count}} components available", + "componentDetails": "Component Details", + "description": "Description", + "tags": "Tags", + "noDescriptionAvailable": "No description available." } diff --git a/DashAI/front/src/utils/i18n/locales/en/datasets.json b/DashAI/front/src/utils/i18n/locales/en/datasets.json index fe7184973..489c06130 100644 --- a/DashAI/front/src/utils/i18n/locales/en/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/en/datasets.json @@ -197,7 +197,7 @@ "ifYourDatasetHaveSplits": "If your dataset have splits, upload it as a zip file", "indices": "Indices (comma separated, or type 'all')", "inferenceRows": "Inference Rows", - "inferenceRowsDescription": "Number of rows used for preview/type inference (quick override).", + "inferenceRowsDescription": "Number of rows used for preview/type inference (minimum 2).", "inferredConfiguration": "Type Inference Configuration", "insightConstantColumn": "This column has only one unique value. It provides no information for analysis.", "insightHighCardinality": "This categorical column has more than 100 unique values. Consider grouping or encoding.", diff --git a/DashAI/front/src/utils/i18n/locales/en/datasetsTour.json b/DashAI/front/src/utils/i18n/locales/en/datasetsTour.json index 7a674db45..bce30e637 100644 --- a/DashAI/front/src/utils/i18n/locales/en/datasetsTour.json +++ b/DashAI/front/src/utils/i18n/locales/en/datasetsTour.json @@ -5,6 +5,7 @@ "dataLoaderConfig": "<0><0>DataLoader Configuration<1>Here you can configure how the dataset should be loaded:<2><0><0>Name: Give your dataset a meaningful name<1><0>Separator: The character that separates values (comma for CSV)<2><0>Other options: Advanced settings for specific needs<3>💡 <0>Pro tip: The default settings work well for most CSV files, so you can usually leave them as they are!", "datasetModule": "<0><0>Dataset Module<1>This is where you manage your data and create interactive notebooks for analysis. Let's see how to get started!", "downloadSample": "<0><0>Download Sample Dataset<1>To get started quickly, let's download a sample dataset.<2><0>Download Personality_Dataset.csv<3>💡 <0>Tip: The file will be saved to your Downloads folder by default.<4>Once downloaded, click \"Next\" to learn how to upload it!", + "notebookDatasetSelection": "<0><0>Select a Dataset<1>Choose the dataset you want to explore in this notebook. The dataset linked to this notebook will be used for all visualizations and transformations.", "finishProcess": "<0><0>Finish the Process<1>Click \"Create Notebook\" to start working with your data in an interactive environment.<2>You'll be able to visualize, transform, and prepare your data for modeling.", "importantNote": "<0><0>Important Note<1>Pay attention to this information<2>This ensures your original data remains intact while you experiment.", "nextSteps": "<0><0>Next Steps: Create a Notebook<1>Now click \"New Notebook\" to open the dataset in an interactive environment.<2>In a notebook, you can analyze, visualize, and transform your data.", diff --git a/DashAI/front/src/utils/i18n/locales/en/experiments.json b/DashAI/front/src/utils/i18n/locales/en/experiments.json index 359dfc2d4..211b84a1b 100644 --- a/DashAI/front/src/utils/i18n/locales/en/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/en/experiments.json @@ -60,10 +60,19 @@ "parameterModification": "Parameter Modification", "prepareDataset": "Prepare dataset", "recommendPreprocessMissingValues": "It's recommended to preprocess the dataset to handle these missing values before training a model.", + "manual": "Manual", + "predefined": "Predefined", + "random": "Random", + "rowIndexes": "Row Indexes", + "rowIndexesDescription": "Specify row ranges for each split using comma-separated values or ranges (e.g. 0-100, 200).", "seed": "Seed", + "splits": "Splits", + "splitsDescription": "Proportion of data assigned to each subset. Values must be between 0 and 1 and sum to 1.", + "stratifyRequiresShuffle": "Requires shuffle to be enabled", "selectDataset": "Select dataset", "selectDatasetColumns": "Indicate which columns of the dataset will be used as input and output.", "selectDatasetTitle": "Select a dataset for the selected task", + "splitType": "Split type", "selectHowToDivideDataset": "Select how to divide the dataset into training, validation and test subsets.", "selectInputOutputColumnsDescription": "Select column names from the lists.", "selectMetrics": "Select metrics", diff --git a/DashAI/front/src/utils/i18n/locales/en/generative.json b/DashAI/front/src/utils/i18n/locales/en/generative.json index dcca069fa..f0b46ed1e 100644 --- a/DashAI/front/src/utils/i18n/locales/en/generative.json +++ b/DashAI/front/src/utils/i18n/locales/en/generative.json @@ -11,6 +11,7 @@ "failedToFetchSessionInfo": "Failed to fetch session information", "failedToFetchSessions": "Failed to fetch sessions", "failedToFetchTasks": "Failed to fetch generative tasks", + "failedToLoadModels": "Failed to load models", "failedToUpdateSession": "Failed to update session", "nameRequired": "Name is required", "processError": "The process has failed. Deleting it... {{error}}", @@ -24,7 +25,16 @@ "showLess": "Show less", "readMore": "Read more", "generativeModule": "Generative Module", + "createNewSession": "Create a New Session", + "createNewSessionDescription": "Pick a model, configure its parameters, and start chatting.", + "configureSession": "Configure session", + "modelHasNoParameters": "This model has no configurable parameters.", + "nameAndDescribeYourSession": "Name your session. Tweak parameters on the right. You can change them at any time during the session.", "nameYourSession": "Name your session", + "newSession": "New Session", + "pickAModelGroupedByTask": "Pick a model. Models are grouped by task.", + "searchModels": "Search models...", + "startBySelectingATask": "Start by selecting a generative task.", "noSessionsFound": "No sessions found", "parameterChangeEvent": "Parameters updated: <1>", "parameterChangeHistory": "Parameter change history for the current session", @@ -45,9 +55,9 @@ "attachVideo": "Attach video", "attachMedia": "Attach media", "attachMediaToContinue": "Attach media to continue", - "noInputAvailable": "No input available for this task" + "noInputAvailable": "No input available for this task", + "selectModelCrumb": "Select Model" }, - "message": { "sessionCreatedSuccess": "Session successfully created.", "sessionDeleted": "Session deleted successfully", diff --git a/DashAI/front/src/utils/i18n/locales/en/models.json b/DashAI/front/src/utils/i18n/locales/en/models.json index f53441bee..cb0e25a2e 100644 --- a/DashAI/front/src/utils/i18n/locales/en/models.json +++ b/DashAI/front/src/utils/i18n/locales/en/models.json @@ -66,6 +66,7 @@ "chooseTaskForSessionWithDataset": "Choose the machine learning task for your session with dataset \"{{datasetName}}\".", "configureModel": "Configure Model", "configureOptimizer": "Configure Optimizer", + "configureSession": "Configure Session", "configureTasksTrainCompareModels": "Configure tasks, train and compare models in organized sessions. Select a task to begin your modeling workflow.", "customMetrics": "Custom Metrics", "datasetPredictions": "Dataset Predictions", @@ -95,6 +96,7 @@ "modelComparison": "Model Comparison", "modelsModule": "Models Module", "nameYourSession": "Name Your Session", + "selectDatasetAndPrepare": "Name your session, select a dataset and configure its columns and splits.", "noCompatibleModelsFound": "No compatible models found", "noDatasetPredictionsYet": "No dataset predictions yet", "noGlobalExplainersYet": "No global explainers yet", @@ -113,7 +115,9 @@ "predictions": "Predictions", "predictionsCount_one": "• <1>{{count}} prediction", "predictionsCount_other": "• <1>{{count}} predictions", + "divideColumnsAndSplits": "Divide columns and configure dataset splits", "prepareDataset": "Prepare Dataset", + "selectDatasetFirst": "Select a dataset to configure its columns and row splits.", "retrainConfirmDetails": "Are you sure you want to re-train run \"<1>{{runName}}\"?", "retrainModel": "Re-train Model?", "retrainWillDeleteOperations": "This run has existing operations that will be deleted", @@ -162,7 +166,11 @@ "profile_regression_fit": "Model Fit", "profile_regression_error": "Error Balanced", "profile_translation_quality": "Translation Quality", - "profile_translation_balanced": "Translation Balanced" + "profile_translation_balanced": "Translation Balanced", + "profile_text_balanced": "Balanced", + "profile_text_detectPositives": "Detect Positives", + "profile_text_avoidFalseAlarms": "Avoid False Alarms", + "profile_text_probabilityQuality": "Probability Quality" }, "message": { "allRunsCompleted": "{{experiment}} has completed all its runs.", diff --git a/DashAI/front/src/utils/i18n/locales/es/common.json b/DashAI/front/src/utils/i18n/locales/es/common.json index d9a4f55af..86cf2af56 100644 --- a/DashAI/front/src/utils/i18n/locales/es/common.json +++ b/DashAI/front/src/utils/i18n/locales/es/common.json @@ -215,5 +215,7 @@ "expand": "Expandir", "optimize": "Optimizar", "viewMore": "Ver más", - "viewLess": "Ver menos" + "viewLess": "Ver menos", + "switchToDarkMode": "Cambiar a modo oscuro", + "switchToLightMode": "Cambiar a modo claro" } diff --git a/DashAI/front/src/utils/i18n/locales/es/custom.json b/DashAI/front/src/utils/i18n/locales/es/custom.json index a9da76dfa..3b26167f3 100644 --- a/DashAI/front/src/utils/i18n/locales/es/custom.json +++ b/DashAI/front/src/utils/i18n/locales/es/custom.json @@ -1,5 +1,14 @@ { "inferenceMethods": "Métodos de Inferencia", "selectAnItemToShowInfo": "Seleccione un elemento para ver la descripción.", - "selectInferenceMethods": "Seleccione los métodos de inferencia que desea aplicar" + "selectInferenceMethods": "Seleccione los métodos de inferencia que desea aplicar", + "search": "Buscar", + "noItemsFound": "No se encontraron componentes", + "tryAdjustingSearch": "Intenta ajustar tu búsqueda o filtros", + "componentsAvailable_one": "{{count}} componente disponible", + "componentsAvailable_other": "{{count}} componentes disponibles", + "componentDetails": "Detalles del Componente", + "description": "Descripción", + "tags": "Etiquetas", + "noDescriptionAvailable": "No hay descripción disponible." } diff --git a/DashAI/front/src/utils/i18n/locales/es/datasets.json b/DashAI/front/src/utils/i18n/locales/es/datasets.json index c44881eeb..27b22f00c 100644 --- a/DashAI/front/src/utils/i18n/locales/es/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/es/datasets.json @@ -200,7 +200,7 @@ "ifYourDatasetHaveSplits": "Si su dataset tiene divisiones, súbalo como un archivo zip", "indices": "Índices (separados por comas, o escriba 'all')", "inferenceRows": "Filas de Inferencia", - "inferenceRowsDescription": "Número de filas utilizadas para vista previa/inferencia de tipo (anulación rápida).", + "inferenceRowsDescription": "Número de filas utilizadas para vista previa/inferencia de tipo (mínimo 2).", "inferredConfiguration": "Configuración de Inferencia de Tipo", "insightConstantColumn": "Esta columna tiene un solo valor único. No aporta información para el análisis.", "insightHighCardinality": "Esta columna categórica tiene más de 100 valores únicos. Considera agrupar o codificar.", diff --git a/DashAI/front/src/utils/i18n/locales/es/datasetsTour.json b/DashAI/front/src/utils/i18n/locales/es/datasetsTour.json index 7daefc386..87a090a71 100644 --- a/DashAI/front/src/utils/i18n/locales/es/datasetsTour.json +++ b/DashAI/front/src/utils/i18n/locales/es/datasetsTour.json @@ -5,6 +5,7 @@ "dataLoaderConfig": "<0><0>Configuración del DataLoader<1>Aquí puedes configurar cómo se debe cargar el dataset:<2><0><0>Nombre: Dale a tu dataset un nombre significativo<1><0>Separador: El carácter que separa los valores (coma para CSV)<2><0>Otras opciones: Configuraciones avanzadas para necesidades específicas<3>💡 <0>Consejo: La configuración predeterminada funciona bien para la mayoría de los archivos CSV, ¡así que generalmente puedes dejarla como está!", "datasetModule": "<0><0>Módulo de Datasets<1>El módulo de Datasets permite gestionar datos y crear Cuadernos interactivos para análisis. ¡Veamos cómo empezar!", "downloadSample": "<0><0>Descargar Dataset de Ejemplo<1>Para comenzar rápidamente, descarguemos un dataset de ejemplo.<2><0>Descargar Personality_Dataset.csv<3>💡 <0>Consejo: El archivo se guardará en tu carpeta de Descargas de forma predeterminada.<4>¡Una vez descargado, haz clic en \"Siguiente\" para aprender a cargarlo!", + "notebookDatasetSelection": "<0><0>Seleccionar un Dataset<1>Elige el dataset que deseas explorar en este Cuaderno. El dataset vinculado se usará para todas las visualizaciones y transformaciones.", "finishProcess": "<0><0>Finalizar el Proceso<1>Haz clic en \"Crear Cuaderno\" para comenzar a trabajar con tus datos en un entorno interactivo.<2>Podrás visualizar, transformar y preparar tus datos para el modelado.", "importantNote": "<0><0>Nota Importante<1>Presta atención a esta información<2>Esto asegura que tus datos originales permanezcan intactos mientras experimentas.", "nextSteps": "<0><0>Próximos Pasos: Crear un Cuaderno<1>Ahora haz clic en \"Nuevo Cuaderno\" para abrir el dataset en un entorno interactivo.<2>En un Cuaderno, puedes analizar, visualizar y transformar tus datos.", diff --git a/DashAI/front/src/utils/i18n/locales/es/experiments.json b/DashAI/front/src/utils/i18n/locales/es/experiments.json index d1ea6cace..eb6fb956f 100644 --- a/DashAI/front/src/utils/i18n/locales/es/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/es/experiments.json @@ -60,10 +60,19 @@ "parameterModification": "Modificación de Parámetros", "prepareDataset": "Preparar dataset", "recommendPreprocessMissingValues": "Se recomienda preprocesar el dataset para manejar estos valores faltantes antes de entrenar un modelo.", + "manual": "Manual", + "predefined": "Predefinido", + "random": "Aleatorio", + "rowIndexes": "Índices de filas", + "rowIndexesDescription": "Especifica rangos de filas para cada partición usando valores separados por comas o rangos (ej. 0-100, 200).", "seed": "Semilla", + "splits": "Particiones", + "splitsDescription": "Proporción de datos asignada a cada subconjunto. Los valores deben estar entre 0 y 1 y sumar 1.", + "stratifyRequiresShuffle": "Requiere que barajar esté activado", "selectDataset": "Seleccionar dataset", "selectDatasetColumns": "Indique qué columnas del dataset se usarán como entrada y salida.", "selectDatasetTitle": "Seleccione un dataset para la tarea seleccionada", + "splitType": "Tipo de partición", "selectHowToDivideDataset": "Seleccione cómo dividir el dataset en subconjuntos de entrenamiento, validación y prueba.", "selectInputOutputColumnsDescription": "Seleccione nombres de columnas de las listas.", "selectMetrics": "Seleccionar métricas", diff --git a/DashAI/front/src/utils/i18n/locales/es/generative.json b/DashAI/front/src/utils/i18n/locales/es/generative.json index cd94325cb..f4a8ff980 100644 --- a/DashAI/front/src/utils/i18n/locales/es/generative.json +++ b/DashAI/front/src/utils/i18n/locales/es/generative.json @@ -11,6 +11,7 @@ "failedToFetchSessionInfo": "Error al obtener información de la sesión", "failedToFetchSessions": "Error al obtener sesiones", "failedToFetchTasks": "Error al obtener tareas generativas", + "failedToLoadModels": "Error al cargar los modelos", "failedToUpdateSession": "Error al actualizar la sesión", "nameRequired": "Se requiere un nombre", "processError": "El proceso ha fallado. Eliminándolo... {{error}}", @@ -24,7 +25,16 @@ "showLess": "Mostrar menos", "readMore": "Leer más", "generativeModule": "Módulo Generativo", + "createNewSession": "Crear una Nueva Sesión", + "createNewSessionDescription": "Elige un modelo, configura sus parámetros y empieza a conversar.", + "configureSession": "Configurar sesión", + "modelHasNoParameters": "Este modelo no tiene parámetros configurables.", + "nameAndDescribeYourSession": "Nombra tu sesión. Ajusta los parámetros a la derecha. Puedes cambiarlos en cualquier momento durante la sesión.", "nameYourSession": "Nombre su sesión", + "newSession": "Nueva Sesión", + "pickAModelGroupedByTask": "Elige un modelo. Los modelos están agrupados por tarea.", + "searchModels": "Buscar modelos...", + "startBySelectingATask": "Comienza seleccionando una tarea generativa.", "noSessionsFound": "No se encontraron sesiones", "parameterChangeEvent": "Parámetros actualizados: <1>", "parameterChangeHistory": "Historial de cambios de parámetros para la sesión actual", @@ -45,9 +55,9 @@ "attachVideo": "Adjuntar video", "attachMedia": "Adjuntar medio", "attachMediaToContinue": "Adjunta medios para continuar", - "noInputAvailable": "No hay entrada disponible para esta tarea" + "noInputAvailable": "No hay entrada disponible para esta tarea", + "selectModelCrumb": "Select Model" }, - "message": { "sessionCreatedSuccess": "Sesión creada exitosamente.", "sessionDeleted": "Sesión eliminada exitosamente", diff --git a/DashAI/front/src/utils/i18n/locales/es/models.json b/DashAI/front/src/utils/i18n/locales/es/models.json index d17b19c95..e0655c7b8 100644 --- a/DashAI/front/src/utils/i18n/locales/es/models.json +++ b/DashAI/front/src/utils/i18n/locales/es/models.json @@ -66,6 +66,7 @@ "chooseTaskForSessionWithDataset": "Elija la tarea de aprendizaje automático para su sesión con el dataset \"{{datasetName}}\".", "configureModel": "Configurar Modelo", "configureOptimizer": "Configurar Optimizador", + "configureSession": "Configurar Sesión", "configureTasksTrainCompareModels": "Configure tareas, entrene y compare modelos en sesiones organizadas. Seleccione una tarea para comenzar su flujo de trabajo de modelado.", "customMetrics": "Métricas Personalizadas", "datasetPredictions": "Predicciones de Dataset", @@ -95,6 +96,7 @@ "modelComparison": "Comparación de Modelos", "modelsModule": "Módulo de Modelos", "nameYourSession": "Nombre su Sesión", + "selectDatasetAndPrepare": "Nombra tu sesión, selecciona un dataset y configura sus columnas y particiones.", "noCompatibleModelsFound": "No se encontraron modelos compatibles", "noDatasetPredictionsYet": "Aún no hay predicciones de dataset", "noGlobalExplainersYet": "Aún no hay explicadores globales", @@ -113,7 +115,9 @@ "predictions": "Predicciones", "predictionsCount_one": "• <1>{{count}} predicción", "predictionsCount_other": "• <1>{{count}} predicciones", + "divideColumnsAndSplits": "Divide columnas y configura las particiones del dataset", "prepareDataset": "Preparar Dataset", + "selectDatasetFirst": "Selecciona un dataset para configurar sus columnas y particiones de filas.", "retrainConfirmDetails": "¿Está seguro de que desea re-entrenar la ejecución \"<1>{{runName}}\"?", "retrainModel": "¿Re-entrenar Modelo?", "retrainWillDeleteOperations": "Esta ejecución tiene operaciones existentes que serán eliminadas", @@ -162,7 +166,11 @@ "profile_regression_fit": "Ajuste del Modelo", "profile_regression_error": "Error Balanceado", "profile_translation_quality": "Calidad de Traducción", - "profile_translation_balanced": "Traducción Balanceada" + "profile_translation_balanced": "Traducción Balanceada", + "profile_text_balanced": "Equilibrio", + "profile_text_detectPositives": "Detectar Positivos", + "profile_text_avoidFalseAlarms": "Evitar Falsas Alarmas", + "profile_text_probabilityQuality": "Calidad de Probabilidades" }, "message": { "allRunsCompleted": "{{experiment}} ha completado todas sus ejecuciones.", diff --git a/docs/docs/build/contributing.md b/docs/docs/build/contributing.md index 977672512..5d361a701 100644 --- a/docs/docs/build/contributing.md +++ b/docs/docs/build/contributing.md @@ -15,7 +15,7 @@ The easiest way to extend DashAI is by building and publishing a plugin — a se package that adds new models, tasks, metrics, explorers, or other components to the platform without touching the core codebase. -See the [Plugin Development](./plugin-development/overview) guide to get started. +See the [Plugin Development](/build/plugin-development/overview) guide to get started. ## Fork and Submit a Pull Request @@ -23,7 +23,7 @@ To contribute code directly to the core project: 1. Fork the [DashAI repository on GitHub](https://github.com/DashAISoftware/DashAI). 2. Create a branch for your change. -3. Follow the [Dev Setup](./dev-setup) and [Testing](./testing) guides to run the project +3. Follow the [Dev Setup](/build/dev-setup) and [Testing](/build/testing) guides to run the project locally and verify your changes. 4. Open a pull request against the `develop` branch with a clear description of what you changed and why. diff --git a/docs/docs/build/plugin-development/develop.md b/docs/docs/build/plugin-development/develop.md index 0a23efa9d..59a7b2397 100644 --- a/docs/docs/build/plugin-development/develop.md +++ b/docs/docs/build/plugin-development/develop.md @@ -9,8 +9,8 @@ sidebar_label: Developing a Plugin Before developing a plugin, ensure you understand: -1. **[What is a Plugin?](./overview)** — High-level overview of plugin concepts and capabilities -2. **[Plugin Structure](./structure)** — How plugins are organized and what DashAI requires +1. **[What is a Plugin?](/build/plugin-development/overview)** — High-level overview of plugin concepts and capabilities +2. **[Plugin Structure](/build/plugin-development/structure)** — How plugins are organized and what DashAI requires --- @@ -82,7 +82,7 @@ class MyCustomModel(TabularClassificationModel): 1. Create the plugin folder inside the `plugins` directory (if using Option 2 above) 2. Write your Python classes extending appropriate DashAI base classes 3. Create any required JSON configuration files -4. Add the `pyproject.toml` with proper entry points (see [Plugin Structure](./structure)) +4. Add the `pyproject.toml` with proper entry points (see [Plugin Structure](/build/plugin-development/structure)) 5. Write a README describing your plugin --- @@ -93,7 +93,7 @@ class MyCustomModel(TabularClassificationModel): Study working examples to understand best practices: -- **[Real-world example](./overview):** `dashai-phi-model-package` adds Microsoft Phi models +- **[Real-world example](/build/plugin-development/overview):** `dashai-phi-model-package` adds Microsoft Phi models - **Community plugins:** [pypi.org/search/?q=dashai](https://pypi.org/search/?q=dashai) ### Test Your Plugin During Development diff --git a/docs/docs/build/plugin-development/structure.md b/docs/docs/build/plugin-development/structure.md index 96e747f6d..a4aefe540 100644 --- a/docs/docs/build/plugin-development/structure.md +++ b/docs/docs/build/plugin-development/structure.md @@ -7,7 +7,7 @@ sidebar_label: Plugin Structure ## Overview -If you haven't already, start with [What is a Plugin?](./overview) to understand the core concept. +If you haven't already, start with [What is a Plugin?](/build/plugin-development/overview) to understand the core concept. This page details how plugins are structured, what files and configurations are required, and how DashAI discovers and loads them. @@ -84,6 +84,6 @@ keywords = [ ## Next Steps -- See [Developing a Plugin](./develop) for step-by-step implementation guidance -- Check [Plugin Overview](./overview) for a complete working example -- Learn how to [upload your plugin to PyPI](./upload) +- See [Developing a Plugin](/build/plugin-development/develop) for step-by-step implementation guidance +- Check [Plugin Overview](/build/plugin-development/overview) for a complete working example +- Learn how to [upload your plugin to PyPI](/build/plugin-development/upload) diff --git a/docs/docs/build/plugin-development/upload.md b/docs/docs/build/plugin-development/upload.md index 1a5cb2fa9..5f9a90eb3 100644 --- a/docs/docs/build/plugin-development/upload.md +++ b/docs/docs/build/plugin-development/upload.md @@ -11,8 +11,8 @@ Once your plugin is developed and tested, you can share it with the DashAI commu Before uploading, ensure you have completed: -1. **[Plugin Structure](./structure)** — Your plugin has the correct folder and configuration format -2. **[Developing a Plugin](./develop)** — Your plugin is fully implemented and tested locally +1. **[Plugin Structure](/build/plugin-development/structure)** — Your plugin has the correct folder and configuration format +2. **[Developing a Plugin](/build/plugin-development/develop)** — Your plugin is fully implemented and tested locally --- diff --git a/docs/docs/deep-dive/architecture.md b/docs/docs/deep-dive/architecture.md index 4c796be59..f3df7b444 100644 --- a/docs/docs/deep-dive/architecture.md +++ b/docs/docs/deep-dive/architecture.md @@ -46,13 +46,13 @@ so that API endpoints can receive them automatically. ## Further Reading -| Topic | Page | -| ---------------------------------------------------- | ---------------------------------------- | -| REST API structure, router map, dependency injection | [API](./api) | -| Component types, registry, and configurable objects | [Components](./components) | -| SQLite schema, ORM tables, and data storage | [Database](./database) | -| Huey job queue and job types | [Job System](./job-system) | -| Notebook sessions, explorers, and converters | [Notebook](./notebook) | -| End-to-end training and exploration walkthroughs | [Workflow Examples](./workflow-examples) | -| Column semantic types and inference | [Semantic Types](./semantic-types) | -| Core dataset primitive, splits, and data lifecycle | [DashAIDataset](./dashai-dataset) | +| Topic | Page | +| ---------------------------------------------------- | ------------------------------------------------- | +| REST API structure, router map, dependency injection | [API](/deep-dive/api) | +| Component types, registry, and configurable objects | [Components](/deep-dive/components) | +| SQLite schema, ORM tables, and data storage | [Database](/deep-dive/database) | +| Huey job queue and job types | [Job System](/deep-dive/job-system) | +| Notebook sessions, explorers, and converters | [Notebook](/deep-dive/notebook) | +| End-to-end training and exploration walkthroughs | [Workflow Examples](/deep-dive/workflow-examples) | +| Column semantic types and inference | [Semantic Types](/deep-dive/semantic-types) | +| Core dataset primitive, splits, and data lifecycle | [DashAIDataset](/deep-dive/dashai-dataset) | diff --git a/docs/docs/deep-dive/dashai-dataset.md b/docs/docs/deep-dive/dashai-dataset.md index 47ea64e15..400c832f1 100644 --- a/docs/docs/deep-dive/dashai-dataset.md +++ b/docs/docs/deep-dive/dashai-dataset.md @@ -10,7 +10,7 @@ sidebar_position: 9 `DashAIDataset` is DashAI's core dataset primitive. It extends the HuggingFace `Dataset` class with two additional responsibilities: -- **Semantic type metadata** — a `_types` dictionary (`Dict[str, DashAIDataType]`) that maps every column name to its DashAI semantic type (see [Semantic Types](./semantic-types)). This metadata is persisted inside the Apache Arrow schema so it survives save/load round-trips. +- **Semantic type metadata** — a `_types` dictionary (`Dict[str, DashAIDataType]`) that maps every column name to its DashAI semantic type (see [Semantic Types](/deep-dive/semantic-types)). This metadata is persisted inside the Apache Arrow schema so it survives save/load round-trips. - **Split metadata** — a `splits` dictionary that records which row indices belong to which split (`train`, `test`, `validation`), plus aggregate statistics computed during upload. Every piece of data that flows through DashAI — upload, notebook transformations, model training, predictions — is represented as a `DashAIDataset`. diff --git a/docs/docs/deep-dive/replicability.md b/docs/docs/deep-dive/replicability.md index 3c33ce911..ba1f68e2e 100644 --- a/docs/docs/deep-dive/replicability.md +++ b/docs/docs/deep-dive/replicability.md @@ -46,7 +46,7 @@ Full pipeline export (a portable "recipe" capturing the full preprocessing + mod :::info What is a Notebook? A Notebook in DashAI is a working session with a mutable copy of a dataset. It groups Explorers (visualizations) and Converters (transformations) applied to that copy. The -original dataset is never modified. See [System Design → Notebook](./notebook) +original dataset is never modified. See [System Design → Notebook](/deep-dive/notebook) for details. ::: diff --git a/docs/docs/learn/tutorials/Models/overview.md b/docs/docs/learn/tutorials/Models/overview.md index 6b76da9a0..da77c3ed0 100644 --- a/docs/docs/learn/tutorials/Models/overview.md +++ b/docs/docs/learn/tutorials/Models/overview.md @@ -77,11 +77,11 @@ Other tasks have their own corresponding model catalogs. This section is divided into the following pages: -- **[Train a Model](./train)** — How to create a session, configure input/output columns, +- **[Train a Model](/learn/tutorials/Models/train)** — How to create a session, configure input/output columns, define data splits, add models, set hyperparameters, and run training. -- **[Predictions](./predictions)** — How to generate predictions using trained models, +- **[Predictions](/learn/tutorials/Models/predictions)** — How to generate predictions using trained models, both from a full dataset and from manually entered data. -- **[Explainability](./explainability)** — How to use global and local explainers to +- **[Explainability](/learn/tutorials/Models/explainability)** — How to use global and local explainers to understand model behavior. -- **[Model Comparison](./comparison)** — How to compare metrics across models using +- **[Model Comparison](/learn/tutorials/Models/comparison)** — How to compare metrics across models using tables and charts. diff --git a/docs/docs/learn/tutorials/Models/train.md b/docs/docs/learn/tutorials/Models/train.md index 38b88c86f..b86656274 100644 --- a/docs/docs/learn/tutorials/Models/train.md +++ b/docs/docs/learn/tutorials/Models/train.md @@ -190,12 +190,12 @@ Metrics are only available after training is complete. If a model shows ### EXPLAINABILITY Shows global and local explainers attached to this model. See the -[Explainability](./explainability) page for details. +[Explainability](/learn/tutorials/Models/explainability) page for details. ### PREDICTIONS Shows dataset predictions and manual predictions generated from this model. See the -[Predictions](./predictions) page for details. +[Predictions](/learn/tutorials/Models/predictions) page for details. ### HYPERPARAMETERS diff --git a/docs/i18n/es/docusaurus-plugin-content-docs/current/build/contributing.md b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/contributing.md new file mode 100644 index 000000000..b6a22a30d --- /dev/null +++ b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/contributing.md @@ -0,0 +1,44 @@ +--- +title: Contribuir +sidebar_label: Contribuir +sidebar_position: 5 +--- + +# Contribuir a DashAI + +Hay varias formas de contribuir a DashAI, independientemente de tu experiencia o +nivel de conocimiento. + +## Subir un Plugin + +La forma más sencilla de extender DashAI es construir y publicar un plugin — un +paquete autocontenido que agrega nuevos modelos, tareas, métricas, exploradores u +otros componentes a la plataforma sin modificar el código central. + +Consulta la guía de [Desarrollo de Plugins](/build/plugin-development/overview) para comenzar. + +## Hacer un Fork y Enviar un Pull Request + +Para contribuir código directamente al proyecto central: + +1. Haz un fork del [repositorio DashAI en GitHub](https://github.com/DashAISoftware/DashAI). +2. Crea una rama para tu cambio. +3. Sigue las guías de [Configuración de Desarrollo](/build/dev-setup) y [Testing](/build/testing) para + ejecutar el proyecto localmente y verificar tus cambios. +4. Abre un pull request contra la rama `develop` con una descripción clara de qué + cambiaste y por qué. + +Por favor, verifica si existe un issue relacionado o abre uno antes de comenzar +trabajo significativo, para evitar esfuerzo duplicado. + +## Reportar un Problema + +¿Encontraste un bug o tienes una solicitud de funcionalidad? Abre un issue en el +[rastreador de issues de GitHub](https://github.com/DashAISoftware/DashAI/issues). Incluye la mayor +cantidad de detalle posible — pasos para reproducir, comportamiento esperado vs. real, y tu +entorno. + +## Contáctanos + +Para preguntas, propuestas o cualquier otra consulta, puedes contactar al equipo a través +del formulario de contacto en [dash-ai.com](https://www.dash-ai.com/). diff --git a/docs/i18n/es/docusaurus-plugin-content-docs/current/build/dev-setup.md b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/dev-setup.md new file mode 100644 index 000000000..3eedf7e1b --- /dev/null +++ b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/dev-setup.md @@ -0,0 +1,126 @@ +--- +title: Configuración de Desarrollo +sidebar_label: Configuración de Desarrollo +sidebar_position: 2 +--- + +# Configuración de Desarrollo + +## Requisitos Previos + +- Python 3.10 a 3.13 +- Node.js (LTS) y Yarn 3.5.0 +- Git + +## 1. Clonar el Repositorio + +```bash +git clone https://github.com/DashAISoftware/DashAI.git +cd DashAI +git checkout develop +``` + +## 2. Configuración del Backend + +Crea y activa un entorno de Python (conda o venv): + +```bash +conda create -n dashai python=3.10 +conda activate dashai +``` + +Instala el paquete en modo editable con las dependencias de desarrollo: + +```bash +pip install -r requirements.txt +pip install -e . +pip install -r requirements-dev.txt +pre-commit install +``` + +## 3. Configuración del Frontend + +```bash +cd DashAI/front +yarn install +``` + +## Ejecutar en Desarrollo + +**Backend** (desde la raíz del repositorio): + +```bash +python -m DashAI +# o +dashai --no-browser --logging-level INFO +``` + +**Frontend** (servidor de desarrollo con recarga en caliente): + +```bash +cd DashAI/front +yarn start +``` + +El backend corre en `http://localhost:8000` y el servidor de desarrollo del frontend en `http://localhost:3000`. + +## Linting y Formateo + +**Python** (usando Ruff): + +```bash +ruff check . --fix +ruff format . +``` + +**Frontend** (ESLint + Prettier): + +```bash +cd DashAI/front +yarn lint +``` + +## Hooks de Pre-commit + +DashAI usa hooks de pre-commit para mantener la calidad del código: + +```bash +# Ejecutar todos los hooks manualmente +pre-commit run --all-files + +# Ejecutar sobre archivos en staging (ocurre automáticamente en git commit) +pre-commit run +``` + +## Estructura del Proyecto + +``` +DashAI/ +├── DashAI/ +│ ├── __main__.py # Punto de entrada CLI (Typer) +│ ├── back/ # Backend FastAPI +│ │ ├── app.py # Application factory +│ │ ├── container.py # Kink DI container +│ │ ├── initial_components.py # Registro de componentes al iniciar +│ │ ├── api/ # Routers y schemas de request/response +│ │ ├── converters/ # Componentes Converter +│ │ ├── dataloaders/ # Componentes DataLoader +│ │ ├── dependencies/ # Registry, motor de base de datos, cola de jobs +│ │ ├── explainability/ # Componentes Explainer +│ │ ├── exploration/ # Componentes Explorer +│ │ ├── job/ # Implementaciones de Jobs +│ │ ├── metrics/ # Componentes Metric +│ │ ├── models/ # Componentes de modelos ML +│ │ ├── optimizers/ # Componentes optimizadores de hiperparámetros +│ │ ├── plugins/ # Sistema de carga de plugins +│ │ ├── tasks/ # Componentes Task +│ │ └── types/ # Definiciones de tipos compartidos +│ └── front/ # Frontend React +│ └── src/ +│ ├── api/ # Cliente HTTP +│ ├── components/ # Componentes de UI +│ └── pages/ # Componentes de páginas +├── docs/ # Sitio de documentación (Docusaurus) +├── tests/ # Tests del backend +└── alembic/ # Migraciones de base de datos +``` diff --git a/docs/i18n/es/docusaurus-plugin-content-docs/current/build/migrations.md b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/migrations.md new file mode 100644 index 000000000..d5b5ddc0a --- /dev/null +++ b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/migrations.md @@ -0,0 +1,67 @@ +--- +title: Migraciones de Base de Datos +sidebar_label: Migraciones de Base de Datos +sidebar_position: 3 +--- + +# Migraciones de Base de Datos + +DashAI usa **Alembic** para las migraciones del esquema de base de datos. Las migraciones se ejecutan automáticamente al iniciar la aplicación. + +## Aplicar Migraciones + +Para ejecutar todas las migraciones pendientes manualmente (desde el directorio `DashAI/`): + +```bash +alembic upgrade head +``` + +## Crear una Nueva Migración + +Después de modificar los modelos SQLAlchemy en `back/dependencies/database/`: + +```bash +alembic revision --autogenerate -m "description of changes" +``` + +El archivo de migración generado se guarda en `alembic/versions/` y debe ser incluido en el repositorio. + +:::tip +Siempre revisa el archivo de migración generado automáticamente antes de aplicarlo. Alembic no siempre detecta correctamente cambios complejos (como renombrado de columnas o cambios en restricciones). +::: + +## Revertir + +Revertir la migración más reciente: + +```bash +alembic downgrade -1 +``` + +Revertir a una revisión específica: + +```bash +alembic downgrade +``` + +## Verificar Estado + +```bash +# Migración aplicada actualmente +alembic current + +# Historial completo de migraciones +alembic history +``` + +## Verificaciones en CI + +El pipeline de CI ejecuta verificaciones de migraciones en cada PR: + +- `alembic upgrade head` — verificar que la migración se aplica correctamente +- `alembic downgrade -1` / `alembic upgrade head` — verificar reversibilidad +- Verificación de consistencia del esquema — verificar que el esquema final coincide con los modelos SQLAlchemy + +## Ubicación de la Base de Datos + +La base de datos principal se almacena en `~/.DashAI/db.sqlite`. La cola de jobs usa una base de datos separada en `~/.DashAI/job_queue.db`. diff --git a/docs/i18n/es/docusaurus-plugin-content-docs/current/build/plugin-development/develop.md b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/plugin-development/develop.md new file mode 100644 index 000000000..773c3dc0b --- /dev/null +++ b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/plugin-development/develop.md @@ -0,0 +1,132 @@ +--- +title: Desarrollar un Plugin +sidebar_label: Desarrollar un Plugin +--- + +# Desarrollar un Plugin + +## Requisitos Previos + +Antes de desarrollar un plugin, asegúrate de entender: + +1. **[¿Qué es un Plugin?](/build/plugin-development/overview)** — Visión general de los conceptos y capacidades de los plugins +2. **[Estructura de un Plugin](/build/plugin-development/structure)** — Cómo se organizan los plugins y qué requiere DashAI + +--- + +## Configura tu Entorno de Desarrollo + +Para crear un plugin, necesitas acceso a las clases Python de DashAI. Hay dos enfoques: + +### Opción 1: DashAI como Biblioteca Instalada + +Instala DashAI como paquete en tu entorno virtual de Python: + +```bash +pip install dashai +``` + +**Nota:** Este enfoque es más sencillo pero limita tu capacidad de probar el plugin interactivamente dentro de DashAI. + +### Opción 2: Clonar el Repositorio (Recomendado) + +Clona el repositorio de DashAI para obtener capacidades completas de desarrollo y prueba: + +```bash +git clone https://github.com/DashAISoftware/DashAI.git +cd DashAI +``` + +Crea una carpeta `plugins` en la raíz del repositorio para el desarrollo de tu plugin: + +```text +DashAI/ +├── DashAI/ +├── plugins/ ← Crea esta carpeta +├── tests/ +├── ... +``` + +Este enfoque te permite probar tu plugin en tiempo real mientras lo desarrollas. + +--- + +## Paso 1: Identifica Qué Quieres Construir + +Los plugins pueden extender DashAI con: + +- **Modelos de Machine Learning** (`TabularClassificationModel`, `RegressionModel`, `TextGenerationModel`, etc.) +- **Data Loaders** (soporte para formatos de dataset adicionales) +- **Data Converters** (preprocesamiento, ingeniería de características, transformaciones) +- **Explorers** (herramientas de visualización y análisis de datos) +- **Explainers** (herramientas de interpretabilidad de modelos) +- **Tareas Personalizadas** (nuevos tipos de problemas de ML) +- **Métricas Personalizadas** (métricas de evaluación) + +Para garantizar la compatibilidad, las clases de tu plugin **deben extender la clase base apropiada de DashAI**: + +```python +# Ejemplo: Crear un modelo de clasificación tabular +from DashAI.back.models.tabular_classification_model import TabularClassificationModel + +class MyCustomModel(TabularClassificationModel): + def train(self, X, y): + # Tu lógica de entrenamiento + pass +``` + +--- + +## Paso 2: Implementa tu Plugin + +1. Crea la carpeta del plugin dentro del directorio `plugins` (si usas la Opción 2) +2. Escribe tus clases Python extendiendo las clases base apropiadas de DashAI +3. Crea los archivos de configuración JSON necesarios +4. Agrega el `pyproject.toml` con los entry points correctos (ver [Estructura de un Plugin](/build/plugin-development/structure)) +5. Escribe un README describiendo tu plugin + +--- + +## Recomendaciones + +### Revisa Plugins Existentes + +Estudia ejemplos funcionales para entender las mejores prácticas: + +- **[Ejemplo real](/build/plugin-development/overview):** `dashai-phi-model-package` agrega los modelos Microsoft Phi +- **Plugins de la comunidad:** [pypi.org/search/?q=dashai](https://pypi.org/search/?q=dashai) + +### Prueba tu Plugin Durante el Desarrollo + +1. Crea los archivos Python y JSON localmente (Opción 2: clonar el repositorio) +2. Colócalos en la carpeta `/plugins` +3. Inicia DashAI y verifica que tus componentes aparecen y funcionan correctamente +4. Itera hasta estar seguro de la implementación + +### Verifica Conflictos de Dependencias + +Después de instalar nuevas dependencias, verifica que no existan conflictos: + +```bash +pip check +``` + +**Sin conflictos:** + +```text +No broken requirements found. +``` + +**Con conflictos:** + +```text +fastapi 0.106.0 has requirement starlette<0.28.0,>=0.27.0, but you have starlette 0.20.0. +``` + +Resuelve cualquier conflicto antes de publicar tu plugin. + +**Es importante que al instalar una nueva librería para un plugin o paquete, esta sea incluida entre las dependencias del plugin o paquete.** + +5. **Usa prints en el flujo de los componentes agregados** + + Incluir prints en el flujo del componente que creaste puede ser muy útil para verificar el correcto funcionamiento del componente en el software. diff --git a/docs/i18n/es/docusaurus-plugin-content-docs/current/build/plugin-development/overview.md b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/plugin-development/overview.md new file mode 100644 index 000000000..421825a57 --- /dev/null +++ b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/plugin-development/overview.md @@ -0,0 +1,215 @@ +--- +title: Visión General de Plugins +sidebar_label: Visión General de Plugins +--- + +# ¿Qué es un Plugin? + +Un **plugin** es un paquete de extensión que agrega nueva funcionalidad a DashAI sin modificar la aplicación central. Los plugins te permiten: + +- Agregar nuevos **modelos de Machine Learning** (clasificación, regresión, generación, etc.) +- Crear **data loaders** personalizados para soportar formatos de dataset adicionales +- Implementar nuevos **data converters** para preprocesamiento y transformación +- Agregar **explorers** especializados para análisis de datos +- Desarrollar **explainers** personalizados para la interpretabilidad de modelos +- Extender **tareas** para soportar nuevos tipos de problemas de ML +- Definir **métricas** personalizadas para evaluación + +Los plugins se distribuyen como paquetes Python en [PyPI](https://pypi.org) y son descubiertos e instalados automáticamente por DashAI cuando usas el módulo **Plugins**. Esto facilita que la comunidad extienda DashAI con funcionalidades específicas de dominio y experimentales sin esperar lanzamientos oficiales. + +:::tip Convención de Nombres para Plugins +Todos los plugins de DashAI **deben** usar el prefijo `dashai-` en el nombre de su paquete (ej. `dashai-my-model-package`) para que la aplicación pueda descubrirlos y cargarlos automáticamente. Ver más plugins de la comunidad: [pypi.org/search/?q=dashai](https://pypi.org/search/?q=dashai). +::: + +--- + +## Ejemplo Real: Modelos Microsoft Phi + +Aquí hay un ejemplo concreto de un plugin de DashAI en acción. + +**dashai-phi-model-package** agrega los modelos de lenguaje Microsoft Phi para generación de texto a DashAI. + +**Disponible en PyPI:** [dashai-phi-model-package](https://pypi.org/project/dashai-phi-model-package/) + +### Estructura del Paquete + +```bash +dashai_phi_model_package/ +├── dashai_phi_model_package/ +│ ├── __init__.py +│ └── phi_model.py +├── pyproject.toml +└── README.md +``` + +
+pyproject.toml + +```toml +[project] +name = "dashai_phi_model_package" +version = "0.0.2" + +dependencies = ['llama-cpp-python>=0.2.90', 'huggingface-hub>=0.29.1'] + +authors = [{ name = "DashAI team" }, { email = "dashaisoftware@gmail.com" }] + +keywords = ["DashAI", "Model"] + +description = "Phi Model for DashAI" +readme = "README.md" +requires-python = ">=3.8" + +[project.entry-points.'dashai.plugins'] +PhiModel = 'dashai_phi_model_package.phi_model:PhiModel' + +[project.urls] +Homepage = "https://github.com/DashAISoftware/DashAI" +Issues = "https://github.com/DashAISoftware/DashAI/issues" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" +``` + +
+ +La sección `[project.entry-points.'dashai.plugins']` es clave — le indica a DashAI qué clases registrar cuando el plugin está instalado. Cada entrada mapea un nombre arbitrario a una ruta de importación `module:ClassName`. + +
+phi_model.py + +```python +from typing import List + +from llama_cpp import Llama + +from DashAI.back.core.schema_fields import ( + BaseSchema, + enum_field, + float_field, + int_field, + schema_field, +) +from DashAI.back.models.hugging_face.llama_utils import is_gpu_available_for_llama_cpp +from DashAI.back.models.text_to_text_generation_model import ( + TextToTextGenerationTaskModel, +) + +if is_gpu_available_for_llama_cpp(): + DEVICE_ENUM = ["gpu", "cpu"] + DEVICE_PLACEHOLDER = "gpu" +else: + DEVICE_ENUM = ["cpu"] + DEVICE_PLACEHOLDER = "cpu" + + +class PhiSchema(BaseSchema): + """Schema for Phi model.""" + + model_name: schema_field( + enum_field( + enum=[ + "microsoft/Phi-3-mini-4k-instruct-gguf", + "microsoft/phi-4-gguf", + ] + ), + placeholder="microsoft/Phi-3-mini-4k-instruct-gguf", + description="The specific Phi model version to use.", + ) # type: ignore + + max_tokens: schema_field( + int_field(ge=1), + placeholder=100, + description="Maximum number of tokens to generate.", + ) # type: ignore + + temperature: schema_field( + float_field(ge=0.0, le=1.0), + placeholder=0.7, + description=( + "Sampling temperature. Higher values make the output more random, while " + "lower values make it more focused and deterministic." + ), + ) # type: ignore + + frequency_penalty: schema_field( + float_field(ge=0.0, le=2.0), + placeholder=0.1, + description=( + "Penalty for repeated tokens in the output. Higher values reduce the " + "likelihood of repetition, encouraging more diverse text generation." + ), + ) # type: ignore + + context_window: schema_field( + int_field(ge=1), + placeholder=512, + description=( + "Maximum number of tokens the model can process in a single forward pass " + "(context window size)." + ), + ) # type: ignore + + device: schema_field( + enum_field(enum=DEVICE_ENUM), + placeholder=DEVICE_PLACEHOLDER, + description="The device to use for model inference.", + ) # type: ignore + + +class PhiModel(TextToTextGenerationTaskModel): + """Phi model for text generation using llama.cpp library.""" + + SCHEMA = PhiSchema + + def __init__(self, **kwargs): + kwargs = self.validate_and_transform(kwargs) + self.model_name = kwargs.get( + "model_name", "microsoft/Phi-3-mini-4k-instruct-gguf" + ) + self.max_tokens = kwargs.pop("max_tokens", 100) + self.temperature = kwargs.pop("temperature", 0.7) + self.frequency_penalty = kwargs.pop("frequency_penalty", 0.1) + self.n_ctx = kwargs.pop("context_window", 512) + + model_filenames = { + "microsoft/Phi-3-mini-4k-instruct-gguf": "*4.gguf", + "microsoft/phi-4-gguf": "phi-4-IQ3_M.gguf", + } + + self.filename = model_filenames.get( + self.model_name, "Phi-3-mini-4k-instruct-q4.gguf" + ) + + self.model = Llama.from_pretrained( + repo_id=self.model_name, + filename=self.filename, + verbose=True, + n_ctx=self.n_ctx, + n_gpu_layers=-1 if kwargs.get("device", "gpu") == "gpu" else 0, + ) + + def generate(self, prompt: list[dict[str, str]]) -> List[str]: + output = self.model.create_chat_completion( + messages=prompt, + max_tokens=self.max_tokens, + temperature=self.temperature, + frequency_penalty=self.frequency_penalty, + ) + + generated_text = output["choices"][0]["message"]["content"] + return [generated_text] +``` + +
+ +## Conceptos Clave Ilustrados + +| Concepto | Dónde buscar | +| --------------------------------- | ------------------------------------------------------------ | +| Registro de entry points | `pyproject.toml` → `[project.entry-points.'dashai.plugins']` | +| Extender una clase base de DashAI | `PhiModel(TextToTextGenerationTaskModel)` | +| Definir parámetros | `PhiSchema` con `schema_field()` | +| Carga del modelo | `__init__` con `validate_and_transform` | +| Generar salida | `generate()` retornando `List[str]` | diff --git a/docs/i18n/es/docusaurus-plugin-content-docs/current/build/plugin-development/structure.md b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/plugin-development/structure.md new file mode 100644 index 000000000..e1b22e172 --- /dev/null +++ b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/plugin-development/structure.md @@ -0,0 +1,89 @@ +--- +title: Estructura de un Plugin +sidebar_label: Estructura de un Plugin +--- + +# Estructura de un Plugin + +## Visión General + +Si aún no lo hiciste, comienza con [¿Qué es un Plugin?](/build/plugin-development/overview) para entender el concepto central. + +Esta página detalla cómo se estructuran los plugins, qué archivos y configuraciones son necesarios, y cómo DashAI los descubre y carga. + +--- + +## ¿De Qué se Compone un Plugin? + +Un plugin es un paquete Python con una estructura estandarizada. Consta de 4 partes principales: + +1. **src**: Esta carpeta contiene la carpeta con el **nombre del plugin o paquete**. Esto es importante para una correcta organización y descubrimiento. + + La carpeta con el nombre del plugin o paquete debe contener todos los archivos **Python** y **JSON** necesarios para extender el software. + +2. **pyproject.toml**: Archivo de configuración. Determina cómo se crea el paquete, qué metadatos contiene y qué clases registrar como entry points del plugin. + +3. **readme.md**: Contiene la descripción extendida del paquete o plugin. + +4. **LICENSE**: Determina el tipo de licencia que tendrá el paquete. + +--- + +## Estructura de Carpetas Recomendada + +```text +dashai-my-plugin +│ LICENSE +│ pyproject.toml +│ readme.md +└───src + └───dashai_my_plugin + │ example_model.py + │ ExampleModel.json +``` + +--- + +## Configuración Requerida + +Para que el software integre el plugin al instalarlo, debe cumplir los siguientes requisitos: + +### 1. Entry Points en pyproject.toml + +Tu **pyproject.toml** DEBE contener un **entrypoint** por cada clase Python que quieras agregar a DashAI: + +```toml +[project.entry-points.'dashai.plugins'] +ExampleModel = 'dashai_my_plugin.example_model:ExampleModel' +``` + +Esto le indica a DashAI qué clases registrar y hacer disponibles en la UI. + +### 2. Sección de Keywords + +Tu **pyproject.toml** DEBE incluir una sección **keywords**. Estas etiquetas se muestran al presentar el plugin en el módulo **Plugins** de DashAI. + +Los únicos tags válidos son: + +```toml +"DashAI", "Model", "Task", "Dataloaders", "Converter", "Explainer" +``` + +### Ejemplo de Sección Keywords + +```toml +[project] +keywords = [ + "DashAI", + "Model", + "Dataloaders" +] +``` + +--- + +## Próximos Pasos + +- Ver [Desarrollar un Plugin](/build/plugin-development/develop) para una guía de implementación paso a paso +- Revisar [Visión General de Plugins](/build/plugin-development/overview) para un ejemplo completo y funcional +- Aprender cómo [subir tu plugin a PyPI](/build/plugin-development/upload) diff --git a/docs/i18n/es/docusaurus-plugin-content-docs/current/build/plugin-development/upload.md b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/plugin-development/upload.md new file mode 100644 index 000000000..8786ef0a1 --- /dev/null +++ b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/plugin-development/upload.md @@ -0,0 +1,129 @@ +--- +title: Subir un Plugin +sidebar_label: Subir un Plugin +--- + +# Subir un Plugin a PyPI + +Una vez que tu plugin está desarrollado y probado, puedes compartirlo con la comunidad de DashAI en [PyPI](https://pypi.org/). + +## Requisitos Previos + +Antes de subir, asegúrate de haber completado: + +1. **[Estructura de un Plugin](/build/plugin-development/structure)** — Tu plugin tiene el formato correcto de carpetas y configuración +2. **[Desarrollar un Plugin](/build/plugin-development/develop)** — Tu plugin está completamente implementado y probado localmente + +--- + +## Publicar tu Plugin en PyPI + +Esta guía usa **twine** para subir tu paquete, aunque hay otros métodos disponibles. + +### Paso 1: Construir tu Paquete + +Instala las herramientas de build: + +```bash +python -m pip install --upgrade build +``` + +Construye tu paquete plugin: + +```bash +python -m build +``` + +Esto crea dos archivos de distribución en la carpeta `dist/`: + +```text +dist/ +├── dashai_my_plugin-0.0.1-py3-none-any.whl +└── dashai_my_plugin-0.0.1.tar.gz +``` + +### Paso 2: Obtener un Token de API de PyPI + +1. Crea una [cuenta en PyPI](https://pypi.org/account/register/) (si aún no tienes una) +2. Ve a tu [página de tokens de API](https://pypi.org/manage/account/#api-tokens) +3. Haz clic en "Add API token" +4. Guarda el token en un lugar seguro (lo necesitarás en el siguiente paso) + +### Paso 3: Subir a Test PyPI (Recomendado Primero) + +Antes de subir al PyPI de producción, prueba tu paquete en [Test PyPI](https://test.pypi.org/): + +Instala twine: + +```bash +python -m pip install --upgrade twine +``` + +Sube a Test PyPI: + +```bash +python -m twine upload --repository testpypi dist/* +``` + +Cuando se te solicite, usa: + +- **Username:** `__token__` +- **Password:** `` + +Visita `https://test.pypi.org/project/dashai-my-plugin/` para verificar que tu paquete aparece correctamente. + +### Paso 4: Subir a PyPI de Producción + +Una vez completadas las pruebas, sube al PyPI oficial: + +```bash +python -m twine upload --repository pypi dist/* +``` + +Cuando se te solicite, usa: + +- **Username:** `__token__` +- **Password:** `` + +¡Tu plugin ya está disponible en PyPI! Los usuarios pueden instalarlo con: + +```bash +pip install dashai-my-plugin +``` + +--- + +## Notas Importantes + +### Convención de Nombres + +Asegúrate de que tu paquete use el prefijo `dashai-` (ej., `dashai-my-plugin`) para que DashAI lo descubra automáticamente al instalarse. + +### Metadatos del Paquete + +Tu **pyproject.toml** debe incluir: + +- Descripción y keywords claras +- Entry points para las clases del plugin +- Links a la página principal y al repositorio +- Información de licencia + +### Versionado + +Sigue el [Versionado Semántico](https://semver.org/): + +- `0.0.1` para lanzamientos iniciales +- `0.1.0` para adiciones de funcionalidades menores +- `1.0.0` para lanzamientos estables con estabilidad de API + +--- + +## Compartir tu Plugin + +Después de publicar, comparte tu plugin con la comunidad: + +1. Agrega el topic `dashai-plugin` a tu repositorio de GitHub +2. Anúncialo en [GitHub Discussions](https://github.com/DashAISoftware/DashAI/discussions) +3. Considera agregar documentación o un tutorial + +¡Feliz publicación! 🚀 diff --git a/docs/i18n/es/docusaurus-plugin-content-docs/current/build/testing.md b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/testing.md new file mode 100644 index 000000000..1c77d1636 --- /dev/null +++ b/docs/i18n/es/docusaurus-plugin-content-docs/current/build/testing.md @@ -0,0 +1,89 @@ +--- +title: Testing +sidebar_label: Testing +sidebar_position: 4 +--- + +# Testing + +## Tests del Backend + +DashAI usa **pytest** para los tests del backend. + +### Ejecutar Todos los Tests + +```bash +pytest -v +``` + +### Ejecutar un Archivo de Test Específico + +```bash +pytest tests/back/api/test_components_api.py -v +``` + +### Ejecutar un Test por Nombre + +```bash +pytest tests/back/api/test_components_api.py::test_name -v +``` + +### Ejecutar con Cobertura + +```bash +pytest --cov=DashAI --cov-report=html +``` + +## Tests del Frontend + +```bash +cd DashAI/front +yarn test +``` + +## Estructura de Tests + +``` +tests/ +├── back/ +│ ├── api/ # Tests de endpoints de la API +│ ├── job/ # Tests de ejecución de jobs +│ ├── models/ # Tests de componentes de modelos +│ ├── converters/ # Tests de converters +│ ├── explorers/ # Tests de explorers +│ └── ... +└── docs/ # Tests del generador de documentación +``` + +## CI + +GitHub Actions ejecuta la suite completa de tests en cada PR y push sobre: + +- **Versiones de Python**: 3.10, 3.11, 3.12, 3.13 +- **Sistemas operativos**: Ubuntu, Windows, macOS + +Verificaciones adicionales en CI: + +- **pre-commit**: Linting con Ruff, formateo y otros hooks +- **db-migrations**: Verificaciones de upgrade/downgrade/reversibilidad con Alembic +- **docs**: Build de Docusaurus + +## Escribir Tests + +Los tests del backend siguen las convenciones de pytest: + +```python +def test_create_dataset(client, tmp_path): + # Arrange + csv_file = tmp_path / "test.csv" + csv_file.write_text("a,b\n1,2\n3,4") + + # Act + response = client.post("/api/v1/dataset/", files={"file": open(csv_file)}) + + # Assert + assert response.status_code == 201 + assert response.json()["name"] == "test" +``` + +Se provee un fixture `client` que crea una aplicación FastAPI de prueba con una base de datos SQLite en memoria. diff --git a/docs/i18n/es/docusaurus-plugin-content-docs/current/deep-dive/architecture.md b/docs/i18n/es/docusaurus-plugin-content-docs/current/deep-dive/architecture.md index 15d328817..77ff02c3e 100644 --- a/docs/i18n/es/docusaurus-plugin-content-docs/current/deep-dive/architecture.md +++ b/docs/i18n/es/docusaurus-plugin-content-docs/current/deep-dive/architecture.md @@ -31,13 +31,13 @@ La inyección de dependencias es gestionada por **Kink**. El contenedor DI (`bac ## Lecturas Adicionales -| Tema | Página | -| ------------------------------------------------------------------- | --------------------------------------------------- | -| Estructura de la API REST, mapa de rutas, inyección de dependencias | [API](./api) | -| Tipos de componentes, registro y objetos configurables | [Componentes](./components) | -| Esquema SQLite, tablas ORM y almacenamiento de datos | [Base de Datos](./database) | -| Cola de trabajos Huey y tipos de trabajos | [Sistema de Trabajos](./job-system) | -| Sesiones de Notebook, exploradores y converters | [Notebook](./notebook) | -| Recorridos completos de entrenamiento y exploración | [Ejemplos de Flujo de Trabajo](./workflow-examples) | -| Tipos semánticos de columnas e inferencia | [Tipos Semánticos](./semantic-types) | -| Primitivo central de datos, splits y ciclo de vida del dato | [DashAIDataset](./dashai-dataset) | +| Tema | Página | +| ------------------------------------------------------------------- | ------------------------------------------------------------ | +| Estructura de la API REST, mapa de rutas, inyección de dependencias | [API](/deep-dive/api) | +| Tipos de componentes, registro y objetos configurables | [Componentes](/deep-dive/components) | +| Esquema SQLite, tablas ORM y almacenamiento de datos | [Base de Datos](/deep-dive/database) | +| Cola de trabajos Huey y tipos de trabajos | [Sistema de Trabajos](/deep-dive/job-system) | +| Sesiones de Notebook, exploradores y converters | [Notebook](/deep-dive/notebook) | +| Recorridos completos de entrenamiento y exploración | [Ejemplos de Flujo de Trabajo](/deep-dive/workflow-examples) | +| Tipos semánticos de columnas e inferencia | [Tipos Semánticos](/deep-dive/semantic-types) | +| Primitivo central de datos, splits y ciclo de vida del dato | [DashAIDataset](/deep-dive/dashai-dataset) | diff --git a/docs/i18n/es/docusaurus-plugin-content-docs/current/deep-dive/dashai-dataset.md b/docs/i18n/es/docusaurus-plugin-content-docs/current/deep-dive/dashai-dataset.md index ed1f08c2e..0baccfdae 100644 --- a/docs/i18n/es/docusaurus-plugin-content-docs/current/deep-dive/dashai-dataset.md +++ b/docs/i18n/es/docusaurus-plugin-content-docs/current/deep-dive/dashai-dataset.md @@ -10,7 +10,7 @@ sidebar_position: 9 `DashAIDataset` es el primitivo central de datos de DashAI. Extiende la clase `Dataset` de HuggingFace con dos responsabilidades adicionales: -- **Metadatos de tipo semántico** — un diccionario `_types` (`Dict[str, DashAIDataType]`) que mapea cada nombre de columna a su tipo semántico DashAI (ver [Tipos Semánticos](./semantic-types)). Estos metadatos se persisten dentro del esquema Apache Arrow para sobrevivir ciclos de guardado/carga. +- **Metadatos de tipo semántico** — un diccionario `_types` (`Dict[str, DashAIDataType]`) que mapea cada nombre de columna a su tipo semántico DashAI (ver [Tipos Semánticos](/deep-dive/semantic-types)). Estos metadatos se persisten dentro del esquema Apache Arrow para sobrevivir ciclos de guardado/carga. - **Metadatos de splits** — un diccionario `splits` que registra qué índices de fila pertenecen a qué split (`train`, `test`, `validation`), junto con estadísticas agregadas calculadas durante la carga del dataset. Cada pieza de datos que fluye por DashAI — carga, transformaciones en el notebook, entrenamiento de modelos, predicciones — se representa como un `DashAIDataset`. diff --git a/docs/i18n/es/docusaurus-plugin-content-docs/current/deep-dive/replicability.md b/docs/i18n/es/docusaurus-plugin-content-docs/current/deep-dive/replicability.md index 77b45d2bf..ccaf4af18 100644 --- a/docs/i18n/es/docusaurus-plugin-content-docs/current/deep-dive/replicability.md +++ b/docs/i18n/es/docusaurus-plugin-content-docs/current/deep-dive/replicability.md @@ -44,15 +44,15 @@ La exportación completa de la pipeline (una "receta" portátil que captura toda ::: :::info ¿Qué es un Notebook? -Un Notebook en DashAI es una sesión de trabajo con una copia mutable de un dataset. Agrupa Exploradores (visualizaciones) y Converters (transformaciones) aplicados a esa copia. El dataset original nunca se modifica. Consulta [Diseño del Sistema → Notebook](./notebook) para más detalles. +Un Notebook en DashAI es una sesión de trabajo con una copia mutable de un dataset. Agrupa Exploradores (visualizaciones) y Converters (transformaciones) aplicados a esa copia. El dataset original nunca se modifica. Consulta [Diseño del Sistema → Notebook](/deep-dive/notebook) para más detalles. ::: ## Detalles del Almacenamiento de Datos -| Artefacto | Ubicación de almacenamiento | -| --------------------- | ---------------------------------------------------------------------------------------------------- | -| Datasets | Archivos Apache Arrow IPC en `~/.DashAI/` | -| Modelos entrenados | Archivos pickle/joblib en `~/.DashAI/runs/{run_id}/` | -| Gráficos de optimización | Objetos Plotly serializados junto a la ejecución | -| Métricas | Tabla `Metric` en `db.sqlite` | +| Artefacto | Ubicación de almacenamiento | +| ------------------------- | ---------------------------------------------------------------------------------------------------- | +| Datasets | Archivos Apache Arrow IPC en `~/.DashAI/` | +| Modelos entrenados | Archivos pickle/joblib en `~/.DashAI/runs/{run_id}/` | +| Gráficos de optimización | Objetos Plotly serializados junto a la ejecución | +| Métricas | Tabla `Metric` en `db.sqlite` | | Resultados de exploración | Imágenes PNG generadas por Exploradores dentro de un Notebook, referenciadas por la tabla `Explorer` | diff --git a/docs/i18n/es/docusaurus-plugin-content-docs/current/learn/tutorials/Models/overview.md b/docs/i18n/es/docusaurus-plugin-content-docs/current/learn/tutorials/Models/overview.md index 16ed1268c..fda6bb8c3 100644 --- a/docs/i18n/es/docusaurus-plugin-content-docs/current/learn/tutorials/Models/overview.md +++ b/docs/i18n/es/docusaurus-plugin-content-docs/current/learn/tutorials/Models/overview.md @@ -22,12 +22,12 @@ El módulo de Modelos es el entorno de DashAI para entrenar, evaluar, comparar y Al abrir el módulo de Modelos, el área principal muestra los tipos de tareas disponibles con una descripción de cada una. Usa la barra de búsqueda para filtrar tareas por nombre. -| Tarea | Descripción | -| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Clasificación Tabular** | Predice una etiqueta categórica a partir de datos tabulares estructurados (filas y columnas). | -| **Clasificación de Texto** | Asigna categorías o etiquetas predefinidas a documentos de texto según su contenido. Útil para análisis de sentimientos, filtrado de spam, categorización de temas. | -| **Regresión** | Predice un valor numérico continuo a partir de datos tabulares estructurados. | -| **Traducción** | Convierte texto de un idioma a otro preservando el significado y el contexto (tarea de NLP). | +| Tarea | Descripción | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Clasificación Tabular** | Predice una etiqueta categórica a partir de datos tabulares estructurados (filas y columnas). | +| **Clasificación de Texto** | Asigna categorías o etiquetas predefinidas a documentos de texto según su contenido. Útil para análisis de sentimientos, filtrado de spam, categorización de temas. | +| **Regresión** | Predice un valor numérico continuo a partir de datos tabulares estructurados. | +| **Traducción** | Convierte texto de un idioma a otro preservando el significado y el contexto (tarea de NLP). | Cada tarea impone requisitos específicos sobre los tipos de columnas para entrada y salida — estos se validan automáticamente al configurar la sesión. @@ -58,7 +58,7 @@ Los modelos disponibles varían según la tarea. Para Clasificación Tabular, po Esta sección se divide en las siguientes páginas: -- **[Entrenar un Modelo](./train)** — Cómo crear una sesión, configurar columnas de entrada/salida, definir divisiones de datos, añadir modelos, establecer hiperparámetros y ejecutar el entrenamiento. -- **[Predicciones](./predictions)** — Cómo generar predicciones usando modelos entrenados, tanto desde un dataset completo como desde datos ingresados manualmente. -- **[Explicabilidad](./explainability)** — Cómo usar explicadores globales y locales para entender el comportamiento del modelo. -- **[Comparación de Modelos](./comparison)** — Cómo comparar métricas entre modelos usando tablas y gráficos. +- **[Entrenar un Modelo](/learn/tutorials/Models/train)** — Cómo crear una sesión, configurar columnas de entrada/salida, definir divisiones de datos, añadir modelos, establecer hiperparámetros y ejecutar el entrenamiento. +- **[Predicciones](/learn/tutorials/Models/predictions)** — Cómo generar predicciones usando modelos entrenados, tanto desde un dataset completo como desde datos ingresados manualmente. +- **[Explicabilidad](/learn/tutorials/Models/explainability)** — Cómo usar explicadores globales y locales para entender el comportamiento del modelo. +- **[Comparación de Modelos](/learn/tutorials/Models/comparison)** — Cómo comparar métricas entre modelos usando tablas y gráficos. diff --git a/docs/i18n/es/docusaurus-plugin-content-docs/current/learn/tutorials/Models/train.md b/docs/i18n/es/docusaurus-plugin-content-docs/current/learn/tutorials/Models/train.md index 2de62a69c..6ef6152ef 100644 --- a/docs/i18n/es/docusaurus-plugin-content-docs/current/learn/tutorials/Models/train.md +++ b/docs/i18n/es/docusaurus-plugin-content-docs/current/learn/tutorials/Models/train.md @@ -52,18 +52,18 @@ Cada tarea impone requisitos de tipo específicos. Para Clasificación Tabular, Define cómo DashAI divide el dataset en subconjuntos de entrenamiento, validación y prueba. Hay tres opciones disponibles: -| Opción | Descripción | -| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| **Use predefined splits** | Usa divisiones train/validación/test ya definidas en el archivo del dataset. Solo disponible si el dataset fue cargado con estructura pre-dividida. | -| **Random split by proportion** | Asigna filas aleatoriamente a cada subconjunto según las proporciones que especifiques. El valor por defecto es Train: 0.6, Validation: 0.2, Test: 0.2. | -| **Manual split by row indices** | Especifica manualmente los índices de fila de inicio y fin para cada subconjunto. | +| Opción | Descripción | +| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Use predefined splits** | Usa divisiones train/validación/test ya definidas en el archivo del dataset. Solo disponible si el dataset fue cargado con estructura pre-dividida. | +| **Random split by proportion** | Asigna filas aleatoriamente a cada subconjunto según las proporciones que especifiques. El valor por defecto es Train: 0.6, Validation: 0.2, Test: 0.2. | +| **Manual split by row indices** | Especifica manualmente los índices de fila de inicio y fin para cada subconjunto. | Al usar **Random split**, hay tres opciones adicionales disponibles: -| Opción | Descripción | -| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------- | -| **Shuffle** | Mezcla aleatoriamente las filas antes de dividir. Recomendado para evitar sesgo de orden en los datos. Habilitado por defecto. | -| **Stratify** | Asegura que cada división preserve las mismas proporciones de clases que el dataset completo. Útil para datasets desbalanceados. | +| Opción | Descripción | +| ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Shuffle** | Mezcla aleatoriamente las filas antes de dividir. Recomendado para evitar sesgo de orden en los datos. Habilitado por defecto. | +| **Stratify** | Asegura que cada división preserve las mismas proporciones de clases que el dataset completo. Útil para datasets desbalanceados. | | **Seed** | Una semilla aleatoria fija para reproducibilidad. El valor por defecto es `42`. Establece un valor específico para asegurar que siempre se produzca la misma división. | Haz clic en **CREATE SESSION** para finalizar la configuración. La sesión se abre inmediatamente. @@ -120,12 +120,12 @@ Una tabla resumen que muestra todos los modelos en la sesión con columnas: Cada modelo tiene una tarjeta expandible que muestra su nombre, algoritmo, insignia de estado actual y botones de acción: -| Botón | Descripción | -| ---------------- | --------------------------------------------------------------------------------------------- | -| **EDIT** | Reabre el modal de configuración para cambiar el nombre del modelo o los hiperparámetros. | -| **TRAIN** | Inicia el entrenamiento de este modelo. Cambia a **RE-TRAIN** después de la primera ejecución. | -| **Insignia de estado** | Muestra el estado actual: **Not Started**, **Finalizado** o **Error**. | -| 🗑 | Elimina el modelo de la sesión. | +| Botón | Descripción | +| ---------------------- | ---------------------------------------------------------------------------------------------- | +| **EDIT** | Reabre el modal de configuración para cambiar el nombre del modelo o los hiperparámetros. | +| **TRAIN** | Inicia el entrenamiento de este modelo. Cambia a **RE-TRAIN** después de la primera ejecución. | +| **Insignia de estado** | Muestra el estado actual: **Not Started**, **Finalizado** o **Error**. | +| 🗑 | Elimina el modelo de la sesión. | **Para entrenar un modelo individual:** haz clic en **TRAIN** en su tarjeta. @@ -151,11 +151,11 @@ Las métricas solo están disponibles después de que el entrenamiento está com ### EXPLAINABILITY -Muestra los explicadores globales y locales adjuntos a este modelo. Consulta la página [Explicabilidad](./explainability) para más detalles. +Muestra los explicadores globales y locales adjuntos a este modelo. Consulta la página [Explicabilidad](/learn/tutorials/Models/explainability) para más detalles. ### PREDICTIONS -Muestra las predicciones de dataset y las predicciones manuales generadas desde este modelo. Consulta la página [Predicciones](./predictions) para más detalles. +Muestra las predicciones de dataset y las predicciones manuales generadas desde este modelo. Consulta la página [Predicciones](/learn/tutorials/Models/predictions) para más detalles. ### HYPERPARAMETERS @@ -172,10 +172,10 @@ Muestra los valores exactos de hiperparámetros usados en la última ejecución ## Solución de Problemas -| Síntoma | Causa probable | Solución | -| ------------------------------------------------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------- | -| El banner de compatibilidad muestra una advertencia | Los tipos de columnas no coinciden con los requisitos de la tarea | Revisa los tipos de columnas en el Explorador de Datasets y vuelve a cargar si es necesario | -| El botón NEXT no está activo en la configuración de sesión | Los campos requeridos están vacíos | Asegúrate de que se seleccionó un dataset y se ingresó un nombre de sesión | -| El modelo muestra insignia **Error** después del entrenamiento | Valores de hiperparámetros inválidos o problema con los datos | Haz clic en **EDIT** para revisar los parámetros, o revisa la Job Queue para detalles del error | -| No hay métricas disponibles después del entrenamiento | Modelo entrenado con datos incompatibles | Revisa la selección de columnas de entrada/salida y vuelve a entrenar | -| RUN ALL no es visible | No se han añadido modelos todavía | Añade al menos un modelo antes de usar RUN ALL | +| Síntoma | Causa probable | Solución | +| -------------------------------------------------------------- | ----------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| El banner de compatibilidad muestra una advertencia | Los tipos de columnas no coinciden con los requisitos de la tarea | Revisa los tipos de columnas en el Explorador de Datasets y vuelve a cargar si es necesario | +| El botón NEXT no está activo en la configuración de sesión | Los campos requeridos están vacíos | Asegúrate de que se seleccionó un dataset y se ingresó un nombre de sesión | +| El modelo muestra insignia **Error** después del entrenamiento | Valores de hiperparámetros inválidos o problema con los datos | Haz clic en **EDIT** para revisar los parámetros, o revisa la Job Queue para detalles del error | +| No hay métricas disponibles después del entrenamiento | Modelo entrenado con datos incompatibles | Revisa la selección de columnas de entrada/salida y vuelve a entrenar | +| RUN ALL no es visible | No se han añadido modelos todavía | Añade al menos un modelo antes de usar RUN ALL | diff --git a/docs/scripts/generate_components.py b/docs/scripts/generate_components.py index 9759e0fcc..328d9df95 100644 --- a/docs/scripts/generate_components.py +++ b/docs/scripts/generate_components.py @@ -528,7 +528,7 @@ def _render_component_mdx(info) -> str: comp_type = class_lookup.get(comp) if comp_type: comp_dir = _type_to_dir(comp_type) - lines.append(f"- [`{comp}`](../{comp_dir}/{comp})") + lines.append(f"- [`{comp}`](/components/{comp_dir}/{comp})") else: lines.append(f"- `{comp}`") lines.append("") diff --git a/installer/installer.iss b/installer/installer.iss index d53848f29..38f743732 100644 --- a/installer/installer.iss +++ b/installer/installer.iss @@ -4,7 +4,7 @@ ; pyinstaller -D -n DashAI-launcher-cpu --clean --add-data "DashAI/front/build;DashAI/front/build" --add-data "%CONDA_PREFIX%\Lib\site-packages\transformers;transformers" --add-binary "%CONDA_PREFIX%\Lib\site-packages\llama_cpp\lib\*;llama_cpp/lib" --additional-hooks-dir=hooks DashAI/__main__.py [Setup] AppName=DashAI -AppVersion=0.9.2 +AppVersion=0.9.3 AppPublisher=DashAI Software AppPublisherURL=https://dash-ai.com DefaultDirName={pf}\DashAI diff --git a/setup.py b/setup.py index 6bde33622..821d55b8f 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def load_requirements(filename): setup( name="DashAI", - version="0.9.2", + version="0.9.3", license="MIT", description=( "DashAI: a graphical toolbox for training, evaluating and deploying " diff --git a/tests/back/api/test_components_api.py b/tests/back/api/test_components_api.py index f41a29e50..9d501dbf0 100644 --- a/tests/back/api/test_components_api.py +++ b/tests/back/api/test_components_api.py @@ -191,7 +191,7 @@ def test_get_component_by_id(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -318,7 +318,7 @@ def test_get_components_select_only_dataloaders(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -328,7 +328,7 @@ def test_get_components_select_only_dataloaders(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -338,7 +338,7 @@ def test_get_components_select_only_dataloaders(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -443,7 +443,7 @@ def test_get_components_ignore_models(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -453,7 +453,7 @@ def test_get_components_ignore_models(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -463,7 +463,7 @@ def test_get_components_ignore_models(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -481,7 +481,7 @@ def test_get_components_ignore_tasks_and_models(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -491,7 +491,7 @@ def test_get_components_ignore_tasks_and_models(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -501,7 +501,7 @@ def test_get_components_ignore_tasks_and_models(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -605,7 +605,7 @@ def test_get_components_dataloader_component_parent(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -615,7 +615,7 @@ def test_get_components_dataloader_component_parent(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -650,7 +650,7 @@ def test_get_components_by_type_and_task(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -660,7 +660,7 @@ def test_get_components_by_type_and_task(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -694,7 +694,7 @@ def test_get_components_select_and_ignore_by_type(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -704,7 +704,7 @@ def test_get_components_select_and_ignore_by_type(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -714,7 +714,7 @@ def test_get_components_select_and_ignore_by_type(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -735,7 +735,7 @@ def test_get_components_select_type_and_parent(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, @@ -745,7 +745,7 @@ def test_get_components_select_type_and_parent(client: TestClient): "type": "DataLoader", "configurable_object": True, "schema": {}, - "metadata": None, + "metadata": {"category": "File Uploading"}, "description": None, "display_name": None, "color": None, diff --git a/tests/back/models/test_tabular_class_models.py b/tests/back/models/test_tabular_class_models.py index 96413440b..3d99798d0 100644 --- a/tests/back/models/test_tabular_class_models.py +++ b/tests/back/models/test_tabular_class_models.py @@ -16,10 +16,20 @@ split_indexes, to_dashai_dataset, ) +from DashAI.back.models.scikit_learn.adaboost_classifier import AdaBoostClassifier +from DashAI.back.models.scikit_learn.bagging_classifier import BaggingClassifier +from DashAI.back.models.scikit_learn.extra_trees_classifier import ExtraTreesClassifier +from DashAI.back.models.scikit_learn.gaussian_nb import GaussianNB +from DashAI.back.models.scikit_learn.gradient_boosting_classifier import ( + GradientBoostingClassifier, +) from DashAI.back.models.scikit_learn.k_neighbors_classifier import KNeighborsClassifier +from DashAI.back.models.scikit_learn.linear_svc_classifier import LinearSVCClassifier +from DashAI.back.models.scikit_learn.mlp_classifier import MLPClassifier from DashAI.back.models.scikit_learn.random_forest_classifier import ( RandomForestClassifier, ) +from DashAI.back.models.scikit_learn.sgd_classifier import SGDClassifier from DashAI.back.models.scikit_learn.svc import SVC from DashAI.back.types.categorical import Categorical from DashAI.back.types.utils import save_types_in_arrow_metadata @@ -119,6 +129,65 @@ def fixture_model_params() -> dict: "tol": 0.001, "verbose": False, }, + "gradient_boosting": { + "loss": "log_loss", + "learning_rate": 0.1, + "n_estimators": 5, + "max_depth": 2, + "min_samples_split": 2, + "min_samples_leaf": 1, + "subsample": 1.0, + "random_state": 42, + }, + "extra_trees": { + "n_estimators": 5, + "max_depth": None, + "min_samples_split": 2, + "min_samples_leaf": 1, + "bootstrap": False, + "random_state": 42, + }, + "adaboost": { + "n_estimators": 5, + "learning_rate": 1.0, + "random_state": 42, + }, + "bagging": { + "n_estimators": 5, + "max_samples": 1.0, + "max_features": 1.0, + "bootstrap": True, + "bootstrap_features": False, + "random_state": 42, + }, + "gaussian_nb": { + "var_smoothing": 1e-9, + }, + "mlp": { + "hidden_layer_size": 10, + "activation": "relu", + "solver": "adam", + "alpha": 0.0001, + "learning_rate_init": 0.001, + "max_iter": 50, + "random_state": 42, + }, + "linear_svc": { + "C": 1.0, + "loss": "squared_hinge", + "max_iter": 100, + "tol": 1e-4, + "fit_intercept": True, + "random_state": 42, + }, + "sgd": { + "loss": "hinge", + "alpha": 0.0001, + "max_iter": 100, + "tol": 1e-3, + "learning_rate": "optimal", + "random_state": 42, + }, } @@ -205,3 +274,134 @@ def test_get_schema_from_model_class(): assert model_schema["type"] == "object" assert "properties" in model_schema assert isinstance(model_schema["properties"], dict) + + +def test_check_is_fitted_new_classifiers( + divided_dataset: Tuple[DatasetDict, DatasetDict], model_params: dict +): + gb_model = GradientBoostingClassifier(**model_params["gradient_boosting"]) + gb_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + + et_model = ExtraTreesClassifier(**model_params["extra_trees"]) + et_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + + ab_model = AdaBoostClassifier(**model_params["adaboost"]) + ab_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + + bag_model = BaggingClassifier(**model_params["bagging"]) + bag_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + + gnb_model = GaussianNB(**model_params["gaussian_nb"]) + gnb_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + + mlp_model = MLPClassifier(**model_params["mlp"]) + mlp_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + + lsvc_model = LinearSVCClassifier(**model_params["linear_svc"]) + lsvc_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + + sgd_model = SGDClassifier(**model_params["sgd"]) + sgd_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + + try: + check_is_fitted(gb_model) + check_is_fitted(et_model) + check_is_fitted(ab_model) + check_is_fitted(bag_model) + check_is_fitted(gnb_model) + check_is_fitted(mlp_model) + check_is_fitted(lsvc_model) + check_is_fitted(sgd_model) + except Exception as e: + pytest.fail( + f"Unexpected error in test_check_is_fitted_new_classifiers: {repr(e)}" + ) + + +def test_predict_new_classifiers( + divided_dataset: Tuple[DatasetDict, DatasetDict], model_params: dict +): + gb_model = GradientBoostingClassifier(**model_params["gradient_boosting"]) + gb_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + y_pred_gb = gb_model.predict(divided_dataset[0]["test"]) + + et_model = ExtraTreesClassifier(**model_params["extra_trees"]) + et_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + y_pred_et = et_model.predict(divided_dataset[0]["test"]) + + ab_model = AdaBoostClassifier(**model_params["adaboost"]) + ab_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + y_pred_ab = ab_model.predict(divided_dataset[0]["test"]) + + bag_model = BaggingClassifier(**model_params["bagging"]) + bag_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + y_pred_bag = bag_model.predict(divided_dataset[0]["test"]) + + gnb_model = GaussianNB(**model_params["gaussian_nb"]) + gnb_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + y_pred_gnb = gnb_model.predict(divided_dataset[0]["test"]) + + mlp_model = MLPClassifier(**model_params["mlp"]) + mlp_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + y_pred_mlp = mlp_model.predict(divided_dataset[0]["test"]) + + lsvc_model = LinearSVCClassifier(**model_params["linear_svc"]) + lsvc_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + y_pred_lsvc = lsvc_model.predict(divided_dataset[0]["test"]) + + sgd_model = SGDClassifier(**model_params["sgd"]) + sgd_model.train(divided_dataset[0]["train"], divided_dataset[1]["train"]) + y_pred_sgd = sgd_model.predict(divided_dataset[0]["test"]) + + n_test = divided_dataset[0]["test"].num_rows + for y_pred in ( + y_pred_gb, + y_pred_et, + y_pred_ab, + y_pred_bag, + y_pred_gnb, + y_pred_mlp, + y_pred_lsvc, + y_pred_sgd, + ): + assert isinstance(y_pred, np.ndarray) + assert len(y_pred) == n_test + + +def test_not_fitted_new_classifiers( + divided_dataset: Tuple[DatasetDict, DatasetDict], model_params: dict +): + with pytest.raises(NotFittedError): + GradientBoostingClassifier(**model_params["gradient_boosting"]).predict( + divided_dataset[0]["test"] + ) + + with pytest.raises(NotFittedError): + LinearSVCClassifier(**model_params["linear_svc"]).predict( + divided_dataset[0]["test"] + ) + + with pytest.raises(NotFittedError): + SGDClassifier(**model_params["sgd"]).predict(divided_dataset[0]["test"]) + + +def test_get_schema_from_new_classifier_classes(): + new_models = ( + GradientBoostingClassifier, + ExtraTreesClassifier, + AdaBoostClassifier, + BaggingClassifier, + GaussianNB, + MLPClassifier, + LinearSVCClassifier, + SGDClassifier, + ) + for model_cls in new_models: + schema = model_cls.get_schema() + assert isinstance(schema, dict), f"{model_cls.__name__} schema is not a dict" + assert schema.get("type") == "object", ( + f"{model_cls.__name__} schema type != object" + ) + assert isinstance(schema.get("properties"), dict), ( + f"{model_cls.__name__} schema has no properties dict" + )