Skip to content

Commit 0a0ad74

Browse files
MNIKIEMAMNIKIEMA
committed
add text field
1 parent 0b9bbef commit 0a0ad74

File tree

2 files changed

+48
-43
lines changed

2 files changed

+48
-43
lines changed

‎policy_analysis/notebooks/BERTopic.ipynb‎

Lines changed: 29 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@
6666
},
6767
{
6868
"cell_type": "code",
69-
"execution_count": 3,
69+
"execution_count": null,
7070
"metadata": {
7171
"colab": {
7272
"base_uri": "https://localhost:8080/"
@@ -187,16 +187,26 @@
187187
}
188188
],
189189
"source": [
190-
"from pathlib import Path\n",
191-
"dataset_name = \"madoss/wsl_library_filtered\"\n",
190+
"dataset_name = \"EdouardCallet/wsl-policy-10k\"\n",
192191
"dataset_split = \"train\"\n",
193192
"dataset = load_dataset(dataset_name, split=dataset_split)\n",
194-
"\n",
195-
"df = dataset.select_columns([\"text\"]).to_pandas()\n",
193+
"text_field = \"single_policy_item\"\n",
194+
"dataset = dataset.filter(lambda example: bool(example[text_field]))\n",
195+
"df: pd.DataFrame = dataset.select_columns([text_field]).to_pandas()\n",
196196
"print(df.shape)\n",
197197
"df.head()"
198198
]
199199
},
200+
{
201+
"cell_type": "code",
202+
"execution_count": null,
203+
"metadata": {},
204+
"outputs": [],
205+
"source": [
206+
"print(df.isna().sum())\n",
207+
"df = df.dropna()"
208+
]
209+
},
200210
{
201211
"cell_type": "markdown",
202212
"metadata": {
@@ -290,7 +300,7 @@
290300
},
291301
{
292302
"cell_type": "code",
293-
"execution_count": 7,
303+
"execution_count": null,
294304
"metadata": {},
295305
"outputs": [
296306
{
@@ -308,7 +318,7 @@
308318
"import matplotlib.pyplot as plt\n",
309319
"from wordcloud import WordCloud\n",
310320
"\n",
311-
"text = \" \".join(df['text'].tolist())\n",
321+
"text = \" \".join(df[text_field].tolist())\n",
312322
"\n",
313323
"wordcloud = WordCloud(width = 600, height = 400, \n",
314324
" background_color ='white', \n",
@@ -326,7 +336,7 @@
326336
},
327337
{
328338
"cell_type": "code",
329-
"execution_count": 8,
339+
"execution_count": null,
330340
"metadata": {
331341
"colab": {
332342
"base_uri": "https://localhost:8080/"
@@ -351,13 +361,13 @@
351361
"# Construction du vocabulaire\n",
352362
"all_vocab = collections.Counter()\n",
353363
"tokenizer = CountVectorizer().build_tokenizer()\n",
354-
"for doc in tqdm(df['text'].tolist()):\n",
364+
"for doc in tqdm(df[text_field].tolist()):\n",
355365
" all_vocab.update(tokenizer(doc))"
356366
]
357367
},
358368
{
359369
"cell_type": "code",
360-
"execution_count": 10,
370+
"execution_count": null,
361371
"metadata": {},
362372
"outputs": [
363373
{
@@ -378,7 +388,7 @@
378388
"most_common_words = all_vocab.most_common(top_n)\n",
379389
"\n",
380390
"# Separate the words and their frequencies\n",
381-
"words, frequencies = zip(*most_common_words)\n",
391+
"words, frequencies = zip(*most_common_words, strict=True)\n",
382392
"\n",
383393
"# Create a bar plot\n",
384394
"\n",
@@ -444,31 +454,11 @@
444454
},
445455
{
446456
"cell_type": "code",
447-
"execution_count": 15,
457+
"execution_count": null,
448458
"metadata": {
449459
"id": "OvnyNuYJzy_J"
450460
},
451-
"outputs": [
452-
{
453-
"ename": "OutOfMemoryError",
454-
"evalue": "CUDA out of memory. Tried to allocate 12.00 MiB. GPU 0 has a total capacity of 3.68 GiB of which 5.25 MiB is free. Including non-PyTorch memory, this process has 3.66 GiB memory in use. Of the allocated memory 3.58 GiB is allocated by PyTorch, and 4.72 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)",
455-
"output_type": "error",
456-
"traceback": [
457-
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
458-
"\u001b[31mOutOfMemoryError\u001b[39m Traceback (most recent call last)",
459-
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[15]\u001b[39m\u001b[32m, line 10\u001b[39m\n\u001b[32m 5\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mbertopic\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mrepresentation\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m MaximalMarginalRelevance\n\u001b[32m 7\u001b[39m \u001b[38;5;66;03m# Preparation des modèles\u001b[39;00m\n\u001b[32m 8\u001b[39m \n\u001b[32m 9\u001b[39m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m10\u001b[39m embedding_model = \u001b[43mSentenceTransformer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mEmbeddings\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 11\u001b[39m \u001b[38;5;66;03m# UMAP\u001b[39;00m\n\u001b[32m 12\u001b[39m umap_model = UMAP(n_components=\u001b[32m8\u001b[39m, n_neighbors=\u001b[32m10\u001b[39m, random_state=\u001b[32m42\u001b[39m,\n\u001b[32m 13\u001b[39m metric=\u001b[33m\"\u001b[39m\u001b[33mcosine\u001b[39m\u001b[33m\"\u001b[39m, verbose=\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
460-
"\u001b[36mFile \u001b[39m\u001b[32m~/OpenSource/13_democratiser_sobriete/policy_analysis/.venv/lib/python3.12/site-packages/sentence_transformers/SentenceTransformer.py:367\u001b[39m, in \u001b[36mSentenceTransformer.__init__\u001b[39m\u001b[34m(self, model_name_or_path, modules, device, prompts, default_prompt_name, similarity_fn_name, cache_folder, trust_remote_code, revision, local_files_only, token, use_auth_token, truncate_dim, model_kwargs, tokenizer_kwargs, config_kwargs, model_card_data, backend)\u001b[39m\n\u001b[32m 364\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[32m 365\u001b[39m \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m367\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 368\u001b[39m \u001b[38;5;28mself\u001b[39m.is_hpu_graph_enabled = \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[32m 370\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.default_prompt_name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m.default_prompt_name \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.prompts:\n",
461-
"\u001b[36mFile \u001b[39m\u001b[32m~/OpenSource/13_democratiser_sobriete/policy_analysis/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py:1371\u001b[39m, in \u001b[36mModule.to\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1368\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 1369\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m1371\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_apply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconvert\u001b[49m\u001b[43m)\u001b[49m\n",
462-
"\u001b[36mFile \u001b[39m\u001b[32m~/OpenSource/13_democratiser_sobriete/policy_analysis/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py:930\u001b[39m, in \u001b[36mModule._apply\u001b[39m\u001b[34m(self, fn, recurse)\u001b[39m\n\u001b[32m 928\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m recurse:\n\u001b[32m 929\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.children():\n\u001b[32m--> \u001b[39m\u001b[32m930\u001b[39m \u001b[43mmodule\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_apply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfn\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 932\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mcompute_should_use_set_data\u001b[39m(tensor, tensor_applied) -> \u001b[38;5;28mbool\u001b[39m:\n\u001b[32m 933\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m torch._has_compatible_shallow_copy_type(tensor, tensor_applied):\n\u001b[32m 934\u001b[39m \u001b[38;5;66;03m# If the new tensor has compatible tensor type as the existing tensor,\u001b[39;00m\n\u001b[32m 935\u001b[39m \u001b[38;5;66;03m# the current behavior is to change the tensor in-place using `.data =`,\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 940\u001b[39m \u001b[38;5;66;03m# global flag to let the user control whether they want the future\u001b[39;00m\n\u001b[32m 941\u001b[39m \u001b[38;5;66;03m# behavior of overwriting the existing tensor or not.\u001b[39;00m\n",
463-
"\u001b[36mFile \u001b[39m\u001b[32m~/OpenSource/13_democratiser_sobriete/policy_analysis/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py:930\u001b[39m, in \u001b[36mModule._apply\u001b[39m\u001b[34m(self, fn, recurse)\u001b[39m\n\u001b[32m 928\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m recurse:\n\u001b[32m 929\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.children():\n\u001b[32m--> \u001b[39m\u001b[32m930\u001b[39m \u001b[43mmodule\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_apply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfn\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 932\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mcompute_should_use_set_data\u001b[39m(tensor, tensor_applied) -> \u001b[38;5;28mbool\u001b[39m:\n\u001b[32m 933\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m torch._has_compatible_shallow_copy_type(tensor, tensor_applied):\n\u001b[32m 934\u001b[39m \u001b[38;5;66;03m# If the new tensor has compatible tensor type as the existing tensor,\u001b[39;00m\n\u001b[32m 935\u001b[39m \u001b[38;5;66;03m# the current behavior is to change the tensor in-place using `.data =`,\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 940\u001b[39m \u001b[38;5;66;03m# global flag to let the user control whether they want the future\u001b[39;00m\n\u001b[32m 941\u001b[39m \u001b[38;5;66;03m# behavior of overwriting the existing tensor or not.\u001b[39;00m\n",
464-
" \u001b[31m[... skipping similar frames: Module._apply at line 930 (3 times)]\u001b[39m\n",
465-
"\u001b[36mFile \u001b[39m\u001b[32m~/OpenSource/13_democratiser_sobriete/policy_analysis/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py:930\u001b[39m, in \u001b[36mModule._apply\u001b[39m\u001b[34m(self, fn, recurse)\u001b[39m\n\u001b[32m 928\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m recurse:\n\u001b[32m 929\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.children():\n\u001b[32m--> \u001b[39m\u001b[32m930\u001b[39m \u001b[43mmodule\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_apply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfn\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 932\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mcompute_should_use_set_data\u001b[39m(tensor, tensor_applied) -> \u001b[38;5;28mbool\u001b[39m:\n\u001b[32m 933\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m torch._has_compatible_shallow_copy_type(tensor, tensor_applied):\n\u001b[32m 934\u001b[39m \u001b[38;5;66;03m# If the new tensor has compatible tensor type as the existing tensor,\u001b[39;00m\n\u001b[32m 935\u001b[39m \u001b[38;5;66;03m# the current behavior is to change the tensor in-place using `.data =`,\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 940\u001b[39m \u001b[38;5;66;03m# global flag to let the user control whether they want the future\u001b[39;00m\n\u001b[32m 941\u001b[39m \u001b[38;5;66;03m# behavior of overwriting the existing tensor or not.\u001b[39;00m\n",
466-
"\u001b[36mFile \u001b[39m\u001b[32m~/OpenSource/13_democratiser_sobriete/policy_analysis/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py:957\u001b[39m, in \u001b[36mModule._apply\u001b[39m\u001b[34m(self, fn, recurse)\u001b[39m\n\u001b[32m 953\u001b[39m \u001b[38;5;66;03m# Tensors stored in modules are graph leaves, and we don't want to\u001b[39;00m\n\u001b[32m 954\u001b[39m \u001b[38;5;66;03m# track autograd history of `param_applied`, so we have to use\u001b[39;00m\n\u001b[32m 955\u001b[39m \u001b[38;5;66;03m# `with torch.no_grad():`\u001b[39;00m\n\u001b[32m 956\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m torch.no_grad():\n\u001b[32m--> \u001b[39m\u001b[32m957\u001b[39m param_applied = \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mparam\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 958\u001b[39m p_should_use_set_data = compute_should_use_set_data(param, param_applied)\n\u001b[32m 960\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mtorch\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_subclasses\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mfake_tensor\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m FakeTensor\n",
467-
"\u001b[36mFile \u001b[39m\u001b[32m~/OpenSource/13_democratiser_sobriete/policy_analysis/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py:1357\u001b[39m, in \u001b[36mModule.to.<locals>.convert\u001b[39m\u001b[34m(t)\u001b[39m\n\u001b[32m 1350\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m convert_to_format \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m t.dim() \u001b[38;5;129;01min\u001b[39;00m (\u001b[32m4\u001b[39m, \u001b[32m5\u001b[39m):\n\u001b[32m 1351\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m t.to(\n\u001b[32m 1352\u001b[39m device,\n\u001b[32m 1353\u001b[39m dtype \u001b[38;5;28;01mif\u001b[39;00m t.is_floating_point() \u001b[38;5;129;01mor\u001b[39;00m t.is_complex() \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m 1354\u001b[39m non_blocking,\n\u001b[32m 1355\u001b[39m memory_format=convert_to_format,\n\u001b[32m 1356\u001b[39m )\n\u001b[32m-> \u001b[39m\u001b[32m1357\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mt\u001b[49m\u001b[43m.\u001b[49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1358\u001b[39m \u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1359\u001b[39m \u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mt\u001b[49m\u001b[43m.\u001b[49m\u001b[43mis_floating_point\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mt\u001b[49m\u001b[43m.\u001b[49m\u001b[43mis_complex\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 1360\u001b[39m \u001b[43m \u001b[49m\u001b[43mnon_blocking\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1361\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1362\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 1363\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mstr\u001b[39m(e) == \u001b[33m\"\u001b[39m\u001b[33mCannot copy out of meta tensor; no data!\u001b[39m\u001b[33m\"\u001b[39m:\n",
468-
"\u001b[31mOutOfMemoryError\u001b[39m: CUDA out of memory. Tried to allocate 12.00 MiB. GPU 0 has a total capacity of 3.68 GiB of which 5.25 MiB is free. Including non-PyTorch memory, this process has 3.66 GiB memory in use. Of the allocated memory 3.58 GiB is allocated by PyTorch, and 4.72 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)"
469-
]
470-
}
471-
],
461+
"outputs": [],
472462
"source": [
473463
"from umap import UMAP\n",
474464
"import hdbscan\n",
@@ -496,7 +486,7 @@
496486
" vectorizer_model=vectorizer_model,\n",
497487
" verbose=True,\n",
498488
" representation_model= MaximalMarginalRelevance(diversity=0.3),\n",
499-
").fit(df[\"text\"].tolist(), embeddings=embeddings)"
489+
").fit(df[text_field].tolist(), embeddings=embeddings)"
500490
]
501491
},
502492
{
@@ -4540,7 +4530,7 @@
45404530
}
45414531
],
45424532
"source": [
4543-
"hierarchical_topics = topic_model.hierarchical_topics(df[\"text\"].tolist())"
4533+
"hierarchical_topics = topic_model.hierarchical_topics(df[text_field].tolist())"
45444534
]
45454535
},
45464536
{
@@ -15258,11 +15248,11 @@
1525815248
},
1525915249
{
1526015250
"cell_type": "code",
15261-
"execution_count": 24,
15251+
"execution_count": null,
1526215252
"metadata": {},
1526315253
"outputs": [],
1526415254
"source": [
15265-
"def fit_bertopic_model(docs=df[\"policy\"].tolist(), embeddings=embeddings, vocab=vocab, stopword=stopword, \n",
15255+
"def fit_bertopic_model(docs=df[text_field].tolist(), embeddings=embeddings, vocab=vocab, stopword=stopword, \n",
1526615256
" embedding_model_name=Embeddings, representation_model=None, y=None):\n",
1526715257
" \"\"\"\n",
1526815258
" Fits a BERTopic model with custom configurations.\n",
@@ -15544,7 +15534,7 @@
1554415534
},
1554515535
{
1554615536
"cell_type": "code",
15547-
"execution_count": 26,
15537+
"execution_count": null,
1554815538
"metadata": {
1554915539
"id": "XupETxy89uqd"
1555015540
},
@@ -15563,7 +15553,7 @@
1556315553
"color_key = {str(topic): next(colors) for topic in set(topic_model.topics_) if topic != -1}\n",
1556415554
"\n",
1556515555
"# Transformer les embeddings en dataframe\n",
15566-
"docs = df[\"policy\"].tolist()\n",
15556+
"docs = df[text_field].tolist()\n",
1556715557
"data = pd.DataFrame({\"x\": reduced_embeddings_2d[:, 0], \"y\": reduced_embeddings_2d[:, 1],\n",
1556815558
" \"Topic\": [str(t) for t in topic_model.topics_]})\n",
1556915559
"\n",

0 commit comments

Comments
 (0)