Merge pull request #34 from xsqian/main

change the prompt to generate data with a longer key to avoid the dupicates

Author: EdmondIguazio

notebook.ipynb

@@ -142,7 +142,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "76f05c23312057a3",
    "metadata": {},
    "outputs": [],
@@ -155,23 +155,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "7e7503f4-8830-4972-a986-10f4c29624b9",
    "metadata": {
     "tags": []
    },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "True"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "import dotenv\n",
     "import os\n",
@@ -185,7 +174,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "d33fc614-2898-4b12-b34f-62181651a9b5",
    "metadata": {},
    "outputs": [],
@@ -197,7 +186,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "9e2a5fd0-f4ff-486a-9d22-e5b80b7eb2a4",
    "metadata": {},
    "outputs": [],
@@ -310,7 +299,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
    "id": "adc25213-d4d6-411d-a9b0-edf91007ba33",
    "metadata": {},
    "outputs": [],

project_setup.py

@@ -248,7 +248,7 @@ def _set_function(
 def _set_calls_generation_functions(
     project: mlrun.projects.MlrunProject,
     node_name: str = None,
-    image: str = ".mlrun-project-image-zzz"
+    image: str = ".mlrun-project-image"
 ):
     # Client and agent data generator
     _set_function(
@@ -288,7 +288,7 @@ def _set_calls_analysis_functions(
     gpus: int,
     node_name: str = None,
     node_selector: dict = None,
-    image: str = ".mlrun-project-image-zzz"
+    image: str = ".mlrun-project-image"
 ):
     # DB management:
     _set_function(

src/calls_analysis/db_management.py

@@ -32,6 +32,7 @@
     insert,
     select,
     update,
+    delete,
 )
 from sqlalchemy.orm import (
     Mapped,
@@ -225,6 +226,8 @@ def create_tables():
     """
     # Create an engine:
     engine = DBEngine()
+    # Create the schema's tables if it already exits, drop it
+    # Base.metadata.drop_all(engine.engine)
     # Create the schema's tables
     Base.metadata.create_all(engine.engine)
 
@@ -240,6 +243,8 @@ def insert_clients(clients: list):
 
     # Insert the new calls into the table and commit:
     with session.begin() as sess:
+        sess.execute(delete(Call))
+        sess.execute(delete(Client))
         sess.execute(insert(Client), clients)
 
     engine.update_db()
@@ -254,6 +259,8 @@ def insert_agents(agents: list):
 
     # Insert the new calls into the table and commit:
     with session.begin() as sess:
+        sess.execute(delete(Call))
+        sess.execute(delete(Agent))
         sess.execute(insert(Agent), agents)
 
     engine.update_db()
@@ -273,6 +280,7 @@ def insert_calls(
 
     # Insert the new calls into the table and commit:
     with session.begin() as sess:
+        sess.execute(delete(Call))
         sess.execute(insert(Call), records)
 
     engine.update_db()

src/workflows/calls_generation.py

@@ -68,7 +68,7 @@ def pipeline(
                     f"last_name: in {language}, no special characters",
                     "phone_number",
                     "email",
-                    "client_id: no leading zeros",
+                    "client_id: no leading zeros, at least 8 digits long, only numbers, this is a primay key field for the database, avoid duplicates as much as possible",
                     "client_city: Enter city, state in the US (e.g., Austin, TX), Not only Texas",
                     "latitude: That correspond to the city",
                     "longitude: That correspond to the city",
@@ -103,7 +103,7 @@ def pipeline(
                 "fields": [
                     f"first_name: in {language}, no special characters",
                     f"last_name: in {language}, no special characters",
-                    "agent_id: no leading zeros",
+                    "agent_id: no leading zeros, at least 8 digits long, only numbers, this is a primay key field for the database, avoid duplicates as much as possible",
                 ],
             },
             returns=["agents: file"],