Merge pull request #136 from unicef/feature/name-parser

Add ML name parser

Author: sergey-misuk-valor

.gitignore

@@ -33,3 +33,5 @@ black.txt
 flake8
 act.*
 root/
+/src/country_workspace/data/name_parser/input/
+/src/country_workspace/data/name_parser/tmp/

pyproject.toml

@@ -46,13 +46,19 @@ dependencies = [
   "hope-flex-fields>=0.6.2",
   "hope-smart-export>=0.3",
   "hope-smart-import>=0.3",
+  "notebook>=7.4.3",
+  "numpy>=2.3",
   "openpyxl>=3.1.5",
+  "pandas>=2.3",
   "pillow>=11.2.1",
+  "pre-commit>=4.2",
   "psycopg2-binary>=2.9.9",
   "python-redis-lock[django]>=4",
   "redis",
+  "scikit-learn>=1.7",
   "sentry-sdk>=2.7.1",
   "social-auth-app-django",
+  "torch>=2.7.1",
   "unicef-security>=1.5.1",
 ]
 scripts.celery-monitor = "country_workspace.__monitor__:run"

src/country_workspace/contrib/name_parser/__init__.py

@@ -0,0 +1,146 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a2dc0c03c80eea0b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "COUNTRY_CODE = \"UKR\"\n",
+    "NAME_TYPES = [\"given_name\", \"middle_name\", \"family_name\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1e6195d85d2779fb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "name_df = pd.read_csv(f\"../../../data/name_parser/input/{COUNTRY_CODE}.csv\", sep=\";\", usecols=NAME_TYPES)\n",
+    "name_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f960c133714b1dad",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "name_df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9bbd412ff9cb84b8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "name_dfs = []\n",
+    "for name in NAME_TYPES:\n",
+    "    name_df = name_df[name].rename(\"name\").to_frame()\n",
+    "    name_df[\"type\"] = name\n",
+    "    name_dfs.append(name_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7dc5ff11aad0049",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "name_df = pd.concat(name_dfs)\n",
+    "name_df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6e49f394945497a4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "name_df[\"name\"] = name_df[\"name\"].str.replace(r\"[^-\\w' ]\", \"\", regex=True)\n",
+    "name_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4296d2039d6ed809",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "name_df[\"name\"] = name_df[\"name\"].str.strip().str.title()\n",
+    "name_df = name_df.dropna()\n",
+    "name_df = name_df.drop_duplicates()\n",
+    "name_df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ff915f8d1f7d5409",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "name_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "afda3a63d02324b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "name_df.to_csv(f\"../../../data/name_parser/tmp/{COUNTRY_CODE}.csv\", index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ffbaef8d4acd17cd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}