release updates for 1.3.4 (#61)

* Changes the activity log caller function logic to collect data before the thread

* Adds unkown user creation logic for comments

* Changes websocket notification positioning for project deletion

* Fixes run on 10 issue

* Adds missing source_code import to file import

* Changes missing columns to query

* PR comments

* submodule change

* Submodule change

* Submodule change

Author: JWittmeyer

controller/attribute/util.py

@@ -50,24 +50,12 @@ def add_log_to_attribute_logs(
 
 def prepare_sample_records_doc_bin(attribute_id: str, project_id: str) -> str:
 
-    sample_record_ids = [
-        record_item
-        for record_item, in record.get_attribute_calculation_sample_records(project_id)
-    ]
-
-    missing_columns = [
-        attribute_item.name
-        for attribute_item in attribute.get_all(project_id)
-        if attribute_item.data_type != DataTypes.TEXT.value
-    ]
-    missing_columns_str = ",\n".join(
-        ["'" + k + "',r.data->'" + k + "'" for k in missing_columns]
-    )
+    sample_records = record.get_attribute_calculation_sample_records(project_id)
 
     sample_records_doc_bin = tokenization.get_doc_bin_table_to_json(
         project_id=project_id,
-        missing_columns=missing_columns_str,
-        record_ids=sample_record_ids,
+        missing_columns=record.get_missing_columns_str(project_id),
+        record_ids=[r[0] for r in sample_records],
     )
     project_item = project.get(project_id)
     org_id = str(project_item.organization_id)

controller/payload/payload_scheduler.py

@@ -707,7 +707,7 @@ def prepare_sample_records_doc_bin(
 
     sample_records_doc_bin = get_doc_bin_table_to_json(
         project_id=project_id,
-        missing_columns=get_missing_columns_tokenization(project_id),
+        missing_columns=record.get_missing_columns_str(project_id),
         record_ids=[r[0] for r in sample_records],
     )
     project_item = project.get(project_id)
@@ -790,16 +790,3 @@ def run_labeling_function_exec_env(
     s3.delete_object(org_id, project_id + "/" + prefixed_knowledge_base)
 
     return calculated_labels, container_logs, code_has_errors
-
-
-def get_missing_columns_tokenization(project_id: str) -> str:
-    missing_columns = [
-        attribute_item.name
-        for attribute_item in attribute.get_all(project_id)
-        if attribute_item.data_type != enums.DataTypes.TEXT.value
-    ]
-    missing_columns_str = ",\n".join(
-        ["'" + k + "',r.data->'" + k + "'" for k in missing_columns]
-    )
-
-    return missing_columns_str

controller/transfer/project_transfer_manager.py

@@ -205,6 +205,9 @@ def import_file(
             logs=attribute_item.get(
                 "logs",
             ),
+            source_code=attribute_item.get(
+                "source_code",
+            ),
             project_id=project_id,
         )
         attribute_ids_by_old_id[
@@ -399,6 +402,12 @@ def import_file(
             if created_by not in used_user_ids:
                 used_user_ids.append(created_by)
 
+    if "comments" in data:
+        for comment_item in data.get("comments"):
+            created_by = comment_item.get("created_by")
+            if created_by not in used_user_ids:
+                used_user_ids.append(created_by)
+
     if used_user_ids:
         existing_users = user.get_by_id_list(used_user_ids)
         existing_users = [str(u.id) for u in existing_users]
@@ -787,19 +796,19 @@ def import_file(
             xftype = comment_item.get("xftype")
             if xftype == enums.CommentCategory.RECORD.value:
                 new_xfkey = record_ids.get(old_xfkey)
-            if xftype == enums.CommentCategory.LABELING_TASK.value:
+            elif xftype == enums.CommentCategory.LABELING_TASK.value:
                 new_xfkey = labeling_task_ids.get(old_xfkey)
-            if xftype == enums.CommentCategory.ATTRIBUTE.value:
+            elif xftype == enums.CommentCategory.ATTRIBUTE.value:
                 new_xfkey = attribute_ids_by_old_id.get(old_xfkey)
-            if xftype == enums.CommentCategory.LABEL.value:
+            elif xftype == enums.CommentCategory.LABEL.value:
                 new_xfkey = labeling_task_labels_ids.get(old_xfkey)
-            if xftype == enums.CommentCategory.DATA_SLICE.value:
+            elif xftype == enums.CommentCategory.DATA_SLICE.value:
                 new_xfkey = data_slice_ids.get(old_xfkey)
-            if xftype == enums.CommentCategory.EMBEDDING.value:
+            elif xftype == enums.CommentCategory.EMBEDDING.value:
                 new_xfkey = embedding_ids.get(old_xfkey)
-            if xftype == enums.CommentCategory.HEURISTIC.value:
+            elif xftype == enums.CommentCategory.HEURISTIC.value:
                 new_xfkey = information_source_ids.get(old_xfkey)
-            if xftype == enums.CommentCategory.KNOWLEDGE_BASE.value:
+            elif xftype == enums.CommentCategory.KNOWLEDGE_BASE.value:
                 new_xfkey = knowledge_base_ids.get(old_xfkey)
             if not new_xfkey:
                 continue
@@ -808,7 +817,7 @@ def import_file(
                 xfkey=new_xfkey,
                 xftype=comment_item.get("xftype"),
                 comment=comment_item.get("comment"),
-                created_by=import_user_id,
+                created_by=comment_item.get("created_by"),
                 project_id=project_id,
                 order_key=comment_item.get("order_key"),
                 is_markdown=comment_item.get("is_markdown"),

graphql_api/mutation/project.py

@@ -73,10 +73,10 @@ def mutate(self, info, project_id: str):
         notification.create_notification(
             enums.NotificationType.PROJECT_DELETED, user.id, None, project_item.name
         )
+        manager.delete_project(project_id)
         notification.send_organization_update(
             project_id, f"project_deleted:{project_id}", True, organization_id
         )
-        manager.delete_project(project_id)
         return DeleteProject(ok=True)
 
 

submodules/model

@@ -1,5 +1,7 @@
+import inspect
 import os
 import traceback
+from typing import Dict, Union
 from controller.misc import config_service
 from submodules.model import models, events
 from util import daemon, service_requests
@@ -22,15 +24,31 @@ def register_user(user: models.User):
 
 
 def post_event(user: models.User, event: events.Event):
-    daemon.run(__post_thread, user, event)
+    caller_dict = __get_caller_data()
+    daemon.run(__post_thread, user, event, caller_dict)
 
-def __post_thread(user: models.User, event: events.Event):
+
+def __post_thread(
+    user: models.User, event: events.Event, caller_dict: Dict[str, Union[str, int]]
+):
     try:
         url = f"{BASE_URI}/track/{user.id}/{event.event_name()}"
         event.IsManaged = config_service.get_config_value("is_managed")
         add_user_activity_entry(
-            str(user.id), {"eventName": event.event_name(), **event.__dict__}
+            str(user.id),
+            {"eventName": event.event_name(), **event.__dict__},
+            caller_dict,
         )
         service_requests.post_call_or_raise(url, event.__dict__)
     except Exception:
         print(traceback.format_exc(), flush=True)
+
+
+def __get_caller_data() -> Dict[str, Union[str, int]]:
+    parent = inspect.stack()[2]
+    # not this function (index 0), not the parent (post_event - index 1) but the one before (index 2)
+    return {
+        "caller_function": parent.function,
+        "caller_file": parent.filename,
+        "caller_lineno": parent.lineno,
+    }

util/doc_ock.py

@@ -5,28 +5,15 @@
 from util import daemon
 import os
 from datetime import datetime
-import inspect
 import json
 
 BACKUP_FILE_PATH = "user_activity_backup.tmp"
 __thread_running = False
 
 
-def get_caller_data() -> Dict[str, Union[str, int]]:
-    # not this, not the parent (add_user_actvity) but the one before
-    parent = inspect.stack()[2]
-    if parent.function == "post_event" and parent.filename == "./util/doc_ock.py":
-        # one step higher
-        parent = inspect.stack()[3]
-    return {
-        "caller_function": parent.function,
-        "caller_file": parent.filename,
-        "caller_lineno": parent.lineno,
-    }
-
-
-def add_user_activity_entry(user_id: str, activity: Any) -> None:
-    caller_dict = get_caller_data()
+def add_user_activity_entry(
+    user_id: str, activity: Any, caller_dict: Dict[str, Union[str, int]]
+) -> None:
     if isinstance(activity, str):
         activity = {**caller_dict, "activity": activity}
     else: