Merge with dev

Author: JWittmeyer

alembic/versions/ad13850a7245_ui_flip_project_icon.py

@@ -30,7 +30,10 @@ def upgrade():
 
     update_dataset_sql = """
         UPDATE cognition.project
-        SET icon = 'IconBolt';
+        SET icon = 'IconBolt';"""
+    connection.execute(update_dataset_sql)
+
+    update_dataset_sql = """
         UPDATE public.user
         SET use_new_cognition_ui = FALSE; """
     connection.execute(update_dataset_sql)

controller/misc/manager.py

@@ -1,13 +1,21 @@
 from typing import Any, Dict, List
 from fast_api.types import ServiceVersionResult
-from submodules.model.global_objects import customer_button
+from submodules.model.global_objects import (
+    customer_button,
+    admin_queries as admin_queries_db_go,
+)
 from datetime import datetime
 import os
 from controller.auth import kratos
 from submodules.model.util import sql_alchemy_to_dict
 from submodules.model import enums
 import requests
 from urllib.parse import urlparse, urlencode, parse_qsl, parse_qs
+from util.tmp_export_file_cleanup import add_cleanup_task
+from uuid import uuid4
+import pandas as pd
+from submodules.model.util import ensure_sql_text
+from submodules.model.business_objects import general
 
 import base64
 
@@ -169,3 +177,16 @@ def __patch_url(url: str, **kwargs):
         ._replace(query=urlencode(dict(parse_qsl(urlparse(url).query), **kwargs)))
         .geturl()
     )
+
+
+def create_admin_query_excel(
+    query: enums.AdminQueries, parameters: Dict[str, Any]
+) -> str:
+
+    q = admin_queries_db_go.get_result_admin_query(query, parameters, as_query=True)
+
+    df = pd.read_sql(ensure_sql_text(q), con=general.get_bind())
+    tmp_filename = f"tmp/feedback_{uuid4()}.xlsx"
+    df.to_excel(tmp_filename, index=False)
+    add_cleanup_task(tmp_filename, 5)
+    return tmp_filename

controller/user/manager.py

@@ -1,11 +1,12 @@
-from typing import Dict, Optional
+from typing import Dict, Optional, Any
 from submodules.model import User, daemon, enums
 from submodules.model.business_objects import user, general
 from controller.auth import kratos
 from submodules.model.exceptions import EntityNotFoundException
 from controller.organization import manager as organization_manager
 from datetime import datetime, timedelta
 from util.decorator import param_throttle
+from submodules.model.util import is_string_true_value
 
 
 def get_user(user_id: str) -> User:
@@ -70,20 +71,13 @@ def update_user_role(user_id: str, role: str) -> User:
     return user_item
 
 
-def update_user_language_display(user_id: str, language_display: str) -> User:
+def update_user_field(user_id: str, field: str, value: Any) -> User:
     user_item = user.get(user_id)
     if not user_item:
         raise ValueError("User not found")
-    user_item.language_display = language_display
-    general.commit()
-    return user_item
-
-
-def set_use_new_ui(user_id: str, value: bool) -> User:
-    user_item = user.get(user_id)
-    if not user_item:
-        raise ValueError("User not found")
-    user_item.use_new_cognition_ui = value
+    if field == "use_new_cognition_ui":
+        value = is_string_true_value(value)
+    setattr(user_item, field, value)
     general.commit()
     return user_item
 

fast_api/models.py

@@ -508,5 +508,9 @@ class CheckInviteUsersBody(BaseModel):
     emails: List[StrictStr]
 
 
+class AdminQueryFilterBody(BaseModel):
+    parameters: Optional[Dict[str, Any]] = None
+
+
 class RecordDeletion(BaseModel):
     record_ids: List[str]

fast_api/routes/misc.py

@@ -1,6 +1,6 @@
 from exceptions.exceptions import AuthManagerError
 from fastapi import APIRouter, Body, Request, status
-from fastapi.responses import PlainTextResponse
+from fastapi.responses import PlainTextResponse, FileResponse
 from fast_api.models import (
     CancelTaskBody,
     CheckInviteUsersBody,
@@ -9,6 +9,7 @@
     ModelProviderDownloadModelBody,
     CreateCustomerButton,
     UpdateCustomerButton,
+    AdminQueryFilterBody,
 )
 from fast_api.routes.client_response import (
     pack_json_result,
@@ -22,12 +23,16 @@
 from controller.model_provider import manager as model_provider_manager
 from controller.task_master import manager as task_master_manager
 from submodules.model import enums
-from submodules.model.global_objects import customer_button as customer_button_db_go
+from submodules.model.global_objects import (
+    customer_button as customer_button_db_go,
+    admin_queries as admin_queries_db_go,
+)
 from submodules.model.util import sql_alchemy_to_dict
 from submodules.model.enums import (
     try_parse_enum_value,
     CustomerButtonType,
     CustomerButtonLocation,
+    AdminQueries,
 )
 from submodules.model.business_objects import task_queue as task_queue_bo
 
@@ -296,3 +301,33 @@ def check_valid_emails(request: Request, body: CheckInviteUsersBody = Body(...))
         raise AuthManagerError("Full admin access required")
     data = auth.check_valid_emails(body.emails)
     return pack_json_result(data)
+
+
+# post to allow for a body to be sent
+@router.post("/admin-query/{query}")
+def get_admin_queries(
+    request: Request, query: AdminQueries, body: AdminQueryFilterBody = Body(...)
+):
+    auth.check_admin_access(request.state.info)
+    if not auth.check_is_full_admin(request):
+        raise AuthManagerError("Full admin access required")
+    data = admin_queries_db_go.get_result_admin_query(query, body.parameters)
+    return pack_json_result(data)
+
+
+@router.post("/admin-query/{query}/download")
+def get_admin_query_excel(
+    request: Request, query: AdminQueries, body: AdminQueryFilterBody = Body(...)
+):
+    auth.check_admin_access(request.state.info)
+    if not auth.check_is_full_admin(request):
+        raise AuthManagerError("Full admin access required")
+
+    file_path = manager.create_admin_query_excel(query, body.parameters)
+
+    if file_path is None:
+        return GENERIC_FAILURE_RESPONSE
+    return FileResponse(
+        file_path,
+        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    )

fast_api/routes/organization.py

@@ -247,19 +247,11 @@ def archive_admin_message(
     return get_silent_success()
 
 
-# in use cognition-ui (08.01.25)
-@router.put("/set-language-display/{language}")
-def set_language_display(request: Request, language: str):
-    user_id = auth_manager.get_user_id_by_info(request.state.info)
-    user_manager.update_user_language_display(user_id, language)
-    return get_silent_success()
-
-
 # in use cognition-ui (23.06.25)
-@router.put("/set-use-new-ui/{value}")
-def set_use_new_ui(request: Request, value: bool):
+@router.put("/update-user-field/{field}/{value}")
+def set_language_display(request: Request, field: str, value: str):
     user_id = auth_manager.get_user_id_by_info(request.state.info)
-    user_manager.set_use_new_ui(user_id, value)
+    user_manager.update_user_field(user_id, field, value)
     return get_silent_success()
 
 

submodules/model

@@ -1,5 +1,6 @@
 from submodules.model.business_objects import upload_task
-import os, shutil
+import os
+import shutil
 
 
 def clean_up_database() -> None:

util/clean_up.py

@@ -0,0 +1,56 @@
+from datetime import datetime, timedelta
+from submodules.model.daemon import run_without_db_token
+import glob
+import os
+import traceback
+import time
+
+__cleanup_tasks = None
+
+
+def add_cleanup_task(tmp_filename: str, minutes: int) -> None:
+    global __cleanup_tasks
+    if __cleanup_tasks is None:
+        add_existing_feedback_and_consumption_files_to_cleanup(tmp_filename)
+        run_without_db_token(start_cleanup)
+    else:
+        __cleanup_tasks[tmp_filename] = datetime.now() + timedelta(minutes=minutes)
+
+
+TMP_PATTERN = [
+    "tmp/feedback_*.xlsx",
+    "tmp/consumption_*.zip",
+    "tmp/consumption_detailed_*.zip",
+    "tmp/consumption_summary_*.xlsx",
+]
+
+
+def add_existing_feedback_and_consumption_files_to_cleanup(tmp_filename: str) -> None:
+    # failsafe for shutdown before deletion time ran out
+    # should only be called for feedback_cleanup_tasks = None
+    global __cleanup_tasks
+    if __cleanup_tasks is not None:
+        raise ValueError("someone didn't read the comment")
+    __cleanup_tasks = {}
+    for pattern in TMP_PATTERN:
+        for filename in glob.glob(pattern):
+            minutes = 1  # after a restart they most likely are not valid anymore so we delete them sooner
+            if filename == tmp_filename:
+                minutes = 60
+            __cleanup_tasks[filename] = datetime.now() + timedelta(minutes=minutes)
+
+
+def start_cleanup() -> None:
+    global __cleanup_tasks
+    while __cleanup_tasks is not None and len(__cleanup_tasks) > 0:
+        try:
+            for filename, delete_time in list(__cleanup_tasks.items()):
+                if datetime.now() > delete_time:
+                    if os.path.isfile(filename):
+                        os.remove(filename)
+                    del __cleanup_tasks[filename]
+        except Exception:
+            print("Error in cleanup task", flush=True)
+            print(traceback.format_exc(), flush=True)
+        finally:
+            time.sleep(60)