uptuna

Author: Wonters

mlruns/0/meta.yaml

@@ -1,6 +1,6 @@
-artifact_location: file:///app/mlruns/0
-creation_time: 1744912772300
+artifact_location: file:///workspace/sentimental_analyses/mlruns/0
+creation_time: 1744984721760
 experiment_id: '0'
-last_update_time: 1744912772300
+last_update_time: 1744984721760
 lifecycle_stage: active
 name: Default

src/mixins.py

@@ -1,9 +1,13 @@
 import mlflow
 import torch
 from tqdm import tqdm
+import numpy as np
 import gc
 import time
 import logging
+import optuna
+import random
+from torch.utils.data import Subset, DataLoader
 
 logger = logging.getLogger(__name__)
 
@@ -17,6 +21,40 @@ class TorchModelTrainMixin:
     lr: float = 2e-5
     device: torch.device
 
+    def get_sampled_dataloader(self, frac=0.1):
+        dataset_size = len(self.dataset)
+        sample_size = int(frac * dataset_size)
+        indices = random.sample(range(dataset_size), sample_size)
+        sampled_dataset = Subset(self.dataset, indices)
+        return DataLoader(sampled_dataset, batch_size=self.batch_size, shuffle=True)
+
+    def optuna_train(self, run_name:str = "", n_trials:int=30, frac=0.1):
+        self.init_mlflow(run_name)
+        self.dataloader = self.get_sampled_dataloader(frac=frac)
+        self.x_val = self.x_val.sample(frac=frac, random_state=42)
+        self.y_val = self.y_val.sample(frac=frac, random_state=42)
+        self.x_train = self.x_train.sample(frac=frac, random_state=42)
+        self.y_train = self.y_train.sample(frac=frac, random_state=42)
+        study = optuna.create_study(direction="maximize",
+                                    pruner=optuna.pruners.MedianPruner(n_startup_trials=5, n_warmup_steps=1))
+        study.optimize(self.objective, n_trials=n_trials)
+        
+
+    def objective(self, trial):
+        lr = trial.suggest_loguniform('lr', 1e-6, 1e-3)
+        gamma = trial.suggest_float('gamma', 0.1, 0.9)
+        step_size = trial.suggest_int('step_size', 2, 10)
+        with mlflow.start_run(nested=True):
+            mlflow.log_params({
+                "lr": lr,
+                "gamma": gamma,
+                "step_size": step_size
+            })
+            self.optimizer = torch.optim.Adam(self.model.parameters(), lr=lr)
+            self.scheduler = torch.optim.lr_scheduler.StepLR(self.optimizer, step_size=step_size, gamma=gamma)
+            acc = self.train()
+        return acc
+
     def _train_batch(self, x, y):
         inputs = self.tokenizer(x, return_tensors="pt", truncation=True, padding=True)
         if isinstance(inputs, dict) and inputs["input_ids"]:
@@ -38,7 +76,7 @@ def _train_batch(self, x, y):
         acc = correct / len(labels)
         loss.backward()
         self.optimizer.step()
-        logger.info(loss.item())
+        logger.info(f"loss {loss.item()}")
         mlflow.log_metric("loss", loss.item())
         mlflow.log_metric("acc", acc)
         mlflow.log_metric("time", time.time())
@@ -47,17 +85,22 @@ def _train_batch(self, x, y):
         if torch.backends.mps.is_available():
             torch.mps.empty_cache()
         time.sleep(0.2)
+        return acc
 
     def train(self):
-        self.init_mlflow()
+        try:
+            self.init_mlflow()
+        except Exception:
+            logger.info("mlflow run already started, you had launched train with optuna")
+            pass
         self.model.train()
         self.model.to(self.device)
+        acc = []
         for epoch in tqdm(range(self.epoch)):
             for tweets, labels in tqdm(self.dataloader):
                 try:
-                    self._train_batch(tweets, labels.float())
+                    acc.append(self._train_batch(tweets, labels.float()))
                 except RuntimeError as e:
-                    raise e
                     logger.error(e)
                     del tweets, labels, self.optimizer
                     gc.collect()
@@ -79,6 +122,7 @@ def train(self):
                     )
             self.scheduler.step()
         super().train()
+        return sum(acc)/len(acc)
 
 
 class SklearnModelTrainMixin:

src/ml.py

@@ -11,20 +11,21 @@
 import torch
 from tqdm import tqdm
 from torch.utils.data import DataLoader
+import torch.nn.functional as F
 from sklearn.linear_model import LogisticRegression
 from lightgbm import LGBMClassifier
 from sklearn.model_selection import train_test_split, cross_val_score
 from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
 from sklearn.metrics import confusion_matrix, classification_report
+from sklearn.ensemble import RandomForestClassifier
 import seaborn as sns
 from transformers import AutoTokenizer, AutoModelForSequenceClassification
 import pandas as pd
-from sklearn.ensemble import RandomForestClassifier
 from skopt import BayesSearchCV, gp_minimize
 from skopt.space import Real, Categorical
 from skopt.utils import use_named_args
 import logging
-import torch.nn.functional as F
+from transformers import PreTrainedModel
 import mlflow
 from mlflow.data.pandas_dataset import from_pandas
 from mlflow.models import infer_signature
@@ -44,7 +45,7 @@
     DEVICE = torch.device("cpu")
     logger.info("Using CPU")
 
-DEVICE = torch.device("cpu")
+#DEVICE = torch.device("cpu")
 
 SENTIMENT_LABELS = {
     0: "😡 unsatisfy",
@@ -67,7 +68,7 @@ class BaseModelABC(ABC):
     dataset_class = None
     tokenizer_class = None
     batch_size = 32
-    artifact_uri = "file:///app/mlruns"
+    #artifact_uri = "file:///app/mlruns"
 
     def __init__(self, dataset: pd.DataFrame):
         self.original_dataset = dataset
@@ -146,7 +147,7 @@ def confusion_matrix(self):
             plt.ylabel("Cluster réels")
             plt.savefig(f.name)
             plt.close()
-            mlflow.log_artifact(f.name, "confusion_matrix.png")
+            mlflow.log_artifact(f.name)#, "confusion_matrix.png")
 
     def train(self):
         """
@@ -160,12 +161,22 @@ def train(self):
         signature = infer_signature(self.x_train, self.predict(self.x_train))
         dataset = from_pandas(self.original_dataset.loc[self.x_train.index], source="local")
         mlflow.log_input(dataset, context="tweet-dataset")
-        model_info = mlflow.sklearn.log_model(
-            sk_model=self.model,
-            artifact_path=self.name,
-            signature=signature,
-            registered_model_name=f"{self.name}-quickstart",
-        )
+        if isinstance(self.model, PreTrainedModel):
+            mlflow.transformers.log_model(
+                transformers_model=self.checkpoint,
+                artifact_path=self.name,
+                task="text-classification",  # important !
+                tokenizer=self.tokenizer,
+                signature=signature,
+                registered_model_name=f"{self.name}-quickstart"
+            )
+        else:
+            mlflow.sklearn.log_model(
+                sk_model=self.model,
+                artifact_path=self.name,
+                signature=signature,
+                registered_model_name=f"{self.name}-quickstart",
+            )
         mlflow.end_run()
 
     def predict(self, x: Union[pd.Series, np.ndarray]):
@@ -341,9 +352,9 @@ class BertModel(TorchBaseModel):
     model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
     dataset_class = TweetDataset
     epoch = 1
-    batch_size = 200
+    batch_size = 100
     out_features = 2
-    lr = 2e-5
+    lr = 2.561e-4
     device = DEVICE
 
     def load_checkpoint(self):
@@ -362,7 +373,7 @@ def load_checkpoint(self):
             )
         self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.lr)
         self.scheduler = torch.optim.lr_scheduler.StepLR(
-            self.optimizer, step_size=5, gamma=0.1
+            self.optimizer, step_size=8, gamma=0.248
         )
         self.criterion = torch.nn.CrossEntropyLoss()
 
@@ -371,14 +382,24 @@ def save(self):
         self.tokenizer.save_pretrained(self.checkpoint)
 
     def predict(self, x: list):
-        inputs = self.preprocessing(x)
-        inputs = inputs.to(self.device)
-        with torch.no_grad():
-            outputs = self.model(**inputs)
-            probs = F.softmax(outputs.logits, dim=1)
-            predicted_class = torch.argmax(probs, dim=1).tolist()
+        predicted_class = []
+        for i in range(0, len(x), self.batch_size):
+            inputs = self.preprocessing(x[i:i+self.batch_size])
+            inputs = inputs.to(self.device)
+            with torch.no_grad():
+                outputs = self.model(**inputs)
+                probs = F.softmax(outputs.logits, dim=1)
+                predicted_class.extend(torch.argmax(probs, dim=1).tolist())
         return predicted_class
 
+class RobertaModel(BertModel):
+    """
+    Using a roberta base sentiment to predict tweet sentiments
+    """
+    model_name = "cardiffnlp/twitter-roberta-base-sentiment"
+    tokenizer_name = "cardiffnlp/twitter-roberta-base-sentiment"
+    checkpoint = "checkpoints/roberta"
+
 
 class LSTMModel(TorchBaseModel):
     checkpoint = "checkpoints/lstm"

src/tests/tests.py

@@ -6,10 +6,12 @@
     LogisticRegressionModel,
     load_data,
     BertModel,
+    RobertaModel,
     LSTMModel,
     RandomForestModel,
     LightGBMModel,
 )
+from transformers import PreTrainedModel
 from ..server import app
 
 
@@ -58,6 +60,19 @@ def test_predict(self):
     def test_confusion_matrix(self):
         self.model.confusion_matrix()
 
+    def test_optuna_train(self):
+        self.model.optuna_train(n_trials=5)
+
+class TestRobertaModel(BaseTest):
+    class_model = RobertaModel
+
+    def test_optuna_train(self):
+        self.model.optuna_train(n_trials=5)
+
+    def test_predict(self):
+        result = self.model.predict(list(self.x_test))
+        assert result == [1, 1, 0, 0, 0, 0]
+
 
 class TestLSTMModel(BaseTest):
     class_model = LSTMModel

train.py

@@ -1,4 +1,4 @@
-from src.ml import load_data, RandomForestModel, LogisticRegressionModel, BertModel
+from src.ml import load_data, RandomForestModel, LogisticRegressionModel, BertModel, RobertaModel, LSTMModel
 import logging
 from rich.logging import RichHandler
 
@@ -7,5 +7,5 @@
 )
 file = "../data/training.1600000.processed.noemoticon.csv"
 original_df = load_data(file)
-model = BertModel(original_df)
-model.train()
+model = LSTMModel(original_df)
+model.optuna_train(n_trials=5, frac=0.01)