feat: completion logs view

Author: moeakwak

backend/api/models/doc/__init__.py

@@ -281,6 +281,7 @@ class AskLogDocument(Document):
     time: datetime.datetime = Field(default_factory=lambda: datetime.datetime.utcnow())
     meta: Union[OpenaiWebAskLogMeta, OpenaiApiAskLogMeta] = Field(discriminator='source')
     user_id: int
+    conversation_id: Optional[uuid.UUID] = None
     queueing_time: Optional[float]
     ask_time: Optional[float]
 

backend/api/routers/chat.py

@@ -325,7 +325,7 @@ async def reply(response: AskResponse):
         # stream 传输
         async for data in manager.complete(text_content=ask_request.text_content,
                                            conversation_id=ask_request.conversation_id,
-                                           parent_id=ask_request.parent,
+                                           parent_message_id=ask_request.parent,
                                            model=model,
                                            plugin_ids=ask_request.openai_web_plugin_ids if ask_request.new_conversation else None,
                                            attachments=ask_request.openai_web_attachments,
@@ -476,7 +476,7 @@ async def reply(response: AskResponse):
                         str(ask_message.id): ask_message,
                         str(message.id): message
                     },
-                    current_node=str(message.id),
+                    current_node=message.id,
                     current_model=message.model
                 )
 
@@ -572,6 +572,7 @@ async def reply(response: AskResponse):
                 user_id=user.id,
                 queueing_time=queueing_time,
                 ask_time=ask_time,
+                conversation_id=conversation_id,
             ).create()
 
     websocket.scope["ask_websocket_close_code"] = websocket_code

backend/api/routers/logs.py

@@ -0,0 +1,61 @@
+from datetime import datetime
+from datetime import datetime
+
+from fastapi import APIRouter, Depends
+
+import api.globals as g
+from api.conf import Config, Credentials
+from api.models.db import User
+from api.models.doc import AskLogDocument
+from api.schemas import LogFilterOptions
+from api.users import current_super_user
+from utils.logger import get_logger
+
+logger = get_logger(__name__)
+config = Config()
+credentials = Credentials()
+
+router = APIRouter()
+
+
+def read_last_n_lines(file_path, n, exclude_key_words=None):
+    if exclude_key_words is None:
+        exclude_key_words = []
+    try:
+        with open(file_path, "r") as f:
+            lines = f.readlines()[::-1]
+    except FileNotFoundError:
+        return [f"File not found: {file_path}"]
+    last_n_lines = []
+    for line in lines:
+        if len(last_n_lines) >= n:
+            break
+        if any([line.find(key_word) != -1 for key_word in exclude_key_words]):
+            continue
+        last_n_lines.append(line)
+    return last_n_lines[::-1]
+
+
+@router.post("/logs/server", tags=["logs"])
+async def get_server_logs(_user: User = Depends(current_super_user), options: LogFilterOptions = LogFilterOptions()):
+    lines = read_last_n_lines(
+        g.server_log_filename,
+        options.max_lines,
+        options.exclude_keywords
+    )
+    return lines
+
+
+@router.get("/logs/completions", tags=["logs"], response_model=list[AskLogDocument])
+async def get_completion_logs(start_time: datetime = None, end_time: datetime = None, max_results: int = 100,
+                              _user: User = Depends(current_super_user)):
+    criteria = []
+    if start_time:
+        criteria.append(AskLogDocument.time >= start_time)
+    if end_time:
+        criteria.append(AskLogDocument.time <= end_time)
+    if not criteria:
+        logs = await AskLogDocument.find_all().sort(-AskLogDocument.time).limit(max_results).to_list()
+    else:
+        logs = await AskLogDocument.find(*criteria).sort(-AskLogDocument.time).limit(max_results).to_list()
+    return logs

backend/api/routers/system.py

@@ -1,10 +1,7 @@
-import csv
 import random
 from datetime import datetime, timedelta, timezone
-from typing import Optional
-
 import httpx
-from fastapi import APIRouter, Depends, UploadFile, File
+from fastapi import APIRouter, Depends
 from fastapi_cache.decorator import cache
 from sqlalchemy import select
 
@@ -78,36 +75,6 @@ async def get_system_info(_user: User = Depends(current_super_user)):
     return result
 
 
-FAKE_REQ_START_TIMESTAMP = 1672502400  # 2023-01-01 00:00:00
-
-
-def make_fake_requests_count(total=100, max=500):
-    result = {}
-    start_stage = FAKE_REQ_START_TIMESTAMP // config.stats.request_counts_interval
-    for i in range(total):
-        result[start_stage + i] = [random.randint(0, max), [1]]
-    return result
-
-
-def make_fake_ask_records(total=100, days=2):
-    result = []
-    model_names = list(api.enums.models.OpenaiWebChatModels)
-    for i in range(total):
-        ask_time = random.random() * 60 + 1
-        total_time = ask_time + random.random() * 30
-        result.append([
-            [
-                # random.randint(1, 10),  # user_id
-                1,
-                model_names[random.randint(0, len(model_names) - 1)],  # model_name
-                ask_time,
-                total_time
-            ],
-            FAKE_REQ_START_TIMESTAMP + random.random() * 60 * 60 * 24 * days,  # ask_time
-        ])
-    return result
-
-
 @router.get("/system/stats/request", tags=["system"], response_model=list[RequestLogAggregation])
 @cache(expire=30)
 async def get_request_statistics(
@@ -208,34 +175,6 @@ async def get_ask_statistics(
     return result
 
 
-def read_last_n_lines(file_path, n, exclude_key_words=None):
-    if exclude_key_words is None:
-        exclude_key_words = []
-    try:
-        with open(file_path, "r") as f:
-            lines = f.readlines()[::-1]
-    except FileNotFoundError:
-        return [f"File not found: {file_path}"]
-    last_n_lines = []
-    for line in lines:
-        if len(last_n_lines) >= n:
-            break
-        if any([line.find(key_word) != -1 for key_word in exclude_key_words]):
-            continue
-        last_n_lines.append(line)
-    return last_n_lines[::-1]
-
-
-@router.post("/system/logs/server", tags=["system"])
-async def get_server_logs(_user: User = Depends(current_super_user), options: LogFilterOptions = LogFilterOptions()):
-    lines = read_last_n_lines(
-        g.server_log_filename,
-        options.max_lines,
-        options.exclude_keywords
-    )
-    return lines
-
-
 @router.get("/system/config", tags=["system"], response_model=ConfigModel)
 async def get_config(_user: User = Depends(current_super_user)):
     return config.model()
@@ -276,54 +215,3 @@ async def sync_openai_web_conversations(_user: User = Depends(current_super_user
         else:
             raise exception
     return None
-
-
-# @router.post("/system/import-users", tags=["system"])
-async def import_users(file: UploadFile = File(...), _user: User = Depends(current_super_user)):
-    """
-    解析csv文件，导入用户
-    csv字段：
-    """
-    raise NotImplementedError()
-
-    headers = ["id", "username", "nickname", "email", "active_time", "chat_status", "can_use_paid", "max_conv_count",
-               "available_ask_count", "is_superuser", "is_active", "is_verified", "hashed_password", "can_use_gpt4",
-               "available_gpt4_ask_count"]
-    content = await file.read()
-    content = content.decode("utf-8")
-    reader = csv.DictReader(content.splitlines())
-    # check headers
-    for field in headers:
-        if field not in reader.fieldnames:
-            raise InvalidParamsException(f"Invalid csv file, missing field: {field}")
-    async with get_async_session_context() as session:
-        async with get_user_db_context(session) as user_db:
-            async with get_user_manager_context(user_db) as user_manager:
-                for row in reader:
-                    user_create = UserCreate(
-                        username=row["username"],
-                        nickname=row["nickname"],
-                        email=f"{row['username']}@example.com",
-                        password="12345678",
-                        remark=row["email"]
-                    )
-                    await user_manager._check_username_unique(user_create.username)
-                    user_dict = user_create.dict()
-
-                    del user_dict["password"]
-                    user_dict["hashed_password"] = row["hashed_password"]
-
-                    user_setting = UserSettingSchema(
-                        credits=0,
-                        openai_web=OpenaiWebSourceSettingSchema.default(),
-                        openai_api=OpenaiApiSourceSettingSchema.default(),
-                    )
-                    user_setting.openai_web.available_models = ["gpt_3_5", "gpt_4"]
-                    if not row["can_use_gpt4"]:
-                        user_setting.openai_web.available_models = ["gpt_3_5"]
-                    user_setting.openai_web.per_model_ask_count.gpt_3_5 = max(
-                        int(row["available_ask_count"]) - int(row["available_gpt4_ask_count"]), 0)
-                    user_setting.openai_web.per_model_ask_count.gpt_4 = int(row["available_gpt4_ask_count"])
-                    user_setting.openai_web.max_conv_count = int(row["max_conv_count"])
-
-                    await user_manager.create_with_user_dict(user_dict, user_setting)

backend/api/sources/openai_api.py

@@ -61,7 +61,7 @@ def reset_session(self):
         self.session = make_session()
 
     async def complete(self, text_content: str, conversation_id: uuid.UUID = None,
-                       parent_id: uuid.UUID = None, model: OpenaiApiChatModels = None,
+                       parent_message_id: uuid.UUID = None, model: OpenaiApiChatModels = None,
                        context_message_count: int = -1, extra_args: Optional[dict] = None, **_kwargs):
 
         assert config.openai_api.enabled, "openai_api is not enabled"
@@ -73,7 +73,7 @@ async def complete(self, text_content: str, conversation_id: uuid.UUID = None,
             id=message_id,
             role="user",
             create_time=now_time,
-            parent=parent_id,
+            parent=parent_message_id,
             children=[],
             content=OpenaiApiChatMessageTextContent(content_type="text", text=text_content),
             metadata=OpenaiApiChatMessageMetadata(
@@ -84,16 +84,16 @@ async def complete(self, text_content: str, conversation_id: uuid.UUID = None,
         messages = []
 
         if not conversation_id:
-            assert parent_id is None, "parent_id must be None when conversation_id is None"
+            assert parent_message_id is None, "parent_id must be None when conversation_id is None"
             messages = [new_message]
         else:
             conv_history = await OpenaiApiConversationHistoryDocument.get(conversation_id)
             if not conv_history:
                 raise ValueError("conversation_id not found")
             if conv_history.source != ChatSourceTypes.openai_api:
                 raise ValueError(f"{conversation_id} is not api conversation")
-            if not conv_history.mapping.get(str(parent_id)):
-                raise ValueError(f"{parent_id} is not a valid parent of {conversation_id}")
+            if not conv_history.mapping.get(str(parent_message_id)):
+                raise ValueError(f"{parent_message_id} is not a valid parent of {conversation_id}")
 
             # 从 current_node 开始往前找 context_message_count 个 message
             if not conv_history.current_node:

backend/main.py

@@ -23,7 +23,7 @@
 from api.middlewares import AccessLoggerMiddleware, StatisticsMiddleware
 from api.models.db import User
 from api.response import CustomJSONResponse, handle_exception_response
-from api.routers import users, conv, chat, system, status, files
+from api.routers import users, conv, chat, system, status, files, logs
 from api.schemas import UserCreate, UserSettingSchema
 from api.sources import OpenaiWebChatManager
 from api.users import get_user_manager_context
@@ -121,6 +121,7 @@ async def lifespan(app: FastAPI):
 app.include_router(conv.router)
 app.include_router(chat.router)
 app.include_router(system.router)
+app.include_router(logs.router)
 app.include_router(status.router)
 app.include_router(files.router)
 

frontend/src/api/logs.ts

@@ -0,0 +1,21 @@
+import axios from 'axios';
+
+import {
+  AskLogAggregation,
+  AskLogDocument,
+  ConfigModel,
+  CredentialsModel,
+  LogFilterOptions,
+  RequestLogAggregation,
+  SystemInfo,
+} from '@/types/schema';
+
+import ApiUrl from './url';
+
+export function getServerLogsApi(options: LogFilterOptions | null) {
+  return axios.post(ApiUrl.ServerLogs, options);
+}
+
+export function getCompletionLogsApi(start_time?: string, end_time?: string, limit = 100) {
+  return axios.get<AskLogDocument[]>(ApiUrl.CompletionLogs, { params: { start_time, end_time, limit } });
+}

frontend/src/api/system.ts

@@ -27,10 +27,6 @@ export function getAskStatisticsApi(granularity: number) {
   });
 }
 
-export function getServerLogsApi(options: LogFilterOptions | null) {
-  return axios.post(ApiUrl.ServerLogs, options);
-}
-
 export function getSystemConfig() {
   return axios.get<ConfigModel>(ApiUrl.SystemConfig);
 }

frontend/src/api/url.ts

@@ -17,9 +17,11 @@ enum ApiUrl {
   SystemInfo = '/system/info',
   SystemRequestStatistics = '/system/stats/request',
   SystemAskStatistics = '/system/stats/ask',
-  ServerLogs = '/system/logs/server',
   SystemActionSyncOpenaiWebConversations = '/system/action/sync-openai-web-conv',
 
+  ServerLogs = '/logs/server',
+  CompletionLogs = '/logs/completions',
+
   SystemConfig = '/system/config',
   SystemCredentials = '/system/credentials',
 

frontend/src/locales/en-US.json

@@ -184,7 +184,9 @@
     "readFileResult": "Result",
     "updateTime": "Update Time",
     "executeCode": "Execute Code",
-    "siteTitleFull": "ChatGPT Web Share"
+    "siteTitleFull": "ChatGPT Web Share",
+    "completionLogs": "Completion Logs",
+    "time": "Time"
   },
   "errors": {
     "403": "403 error: access denied",
@@ -343,7 +345,8 @@
       "enabled_models": "Enabled Models",
       "model_code_mapping": "Model Code Mapping",
       "file_upload_strategy": "File Upload Strategy"
-    }
+    },
+    "conversation_id": "Conversatin ID"
   },
   "models": {
     "gpt_3_5": "GPT-3.5",