add loguru and improve embedding use or simple text #39

Author: MNIKIEMA

policy_analysis/pyproject.toml

@@ -13,6 +13,7 @@ dependencies = [
     "sentence-transformers>=5.1.2",
     "torch>=2.9.1",
     "matplotlib>=3.10.7",
+    "loguru>=0.7.3",
 ]
 
 [project.optional-dependencies]

policy_analysis/src/policy_analysis/policies_clustering/clusterings.py

@@ -1,3 +1,4 @@
+from loguru import logger
 from sklearn.base import BaseEstimator
 from sklearn.pipeline import Pipeline
 from sklearn.decomposition import TruncatedSVD
@@ -61,22 +62,28 @@ def build_hdbscan_pipeline(
 if __name__ == "__main__":
     import pandas as pd
     from pathlib import Path
+    from sentence_transformers import SentenceTransformer
+    from umap import UMAP
     root = Path().cwd()
     fp = root / "data/conclusions&pollitiques_synthetiques.jsonl"
+    model_name = "all-MiniLM-L6-v2"
+    model = SentenceTransformer(model_name)
     df = pd.read_json(fp, lines=True)
     texts = df["response"].tolist()
+    embeddings = model.encode(texts, show_progress_bar=True)
+    logger.info(f"Embeddings shape: {embeddings.shape}")
 
     pipe = build_hdbscan_pipeline(
         embedding="sbert",
         n_components=5,
         min_cluster_size=2
     )
 
-    pipe.fit(texts)
+    pipe.fit(embeddings)
 
+    umap_model = UMAP(n_components=2, n_neighbors=15, random_state=42,
+                    metric="cosine", verbose=True)
     labels = pipe.named_steps["cluster"].labels_
-    X_2d = pipe.named_steps["umap"].transform(
-        pipe.named_steps["embed"].transform(texts)
-    )
+    reduced_embeddings_2d = umap_model.fit_transform(embeddings)
 
-    plot_clusters_2d(X_2d, labels)
+    plot_clusters_2d(reduced_embeddings_2d, labels)

policy_analysis/src/policy_analysis/policies_clustering/embeddings.py

@@ -40,6 +40,8 @@ def fit(self, X, y=None):
         return self
 
     def transform(self, X):
+        if isinstance(X, np.ndarray):
+            return X
         embeddings = self.model.encode(
             X,
             batch_size=self.batch_size,