feat: 添加插件系统支持并优化代码结构

重构插件管理器以支持更灵活的插件加载机制，添加依赖分析和循环检测功能
新增webhook处理接口和认证工具函数
优化知识库处理逻辑，添加并发控制和错误日志
更新依赖版本和文档说明

Author: ch-liuzhide

pyproject.toml

@@ -1,4 +1,3 @@
-from importlib.resources import Resource
 from typing import List, Optional
 
 from core.auth import Action, Resource, get_tenant_with_permissions

server/api/dashboard/router.py

@@ -1,10 +1,17 @@
-from typing import List
+from typing import Any, Dict, List
 
-from core.auth import Action, Resource, get_tenant_with_permissions
+from whiskerrag_utils import get_all_registered_with_metadata
+
+from core.auth import (
+    Action,
+    Resource,
+    get_tenant_with_permissions,
+    validate_key_string,
+)
 from core.log import logger
 from core.plugin_manager import PluginManager
 from core.response import ResponseModel
-from fastapi import APIRouter, HTTPException
+from fastapi import APIRouter, Body, HTTPException, Path
 from pydantic import BaseModel
 from whiskerrag_types.model import (
     Knowledge,
@@ -13,7 +20,7 @@
     PageResponse,
     Tenant,
 )
-from whiskerrag_utils.registry import RegisterTypeEnum, get_registry_list
+from whiskerrag_utils.registry import RegisterTypeEnum
 
 from .utils import gen_knowledge_list
 
@@ -142,7 +149,7 @@ async def delete_knowledge(
         raise HTTPException(
             status_code=404, detail=f"Knowledge {knowledge_id} not found"
         )
-    await db_engine.delete_knowledge(tenant.tenant_id, [knowledge_id])
+    await db_engine.delete_knowledge(tenant.tenant_id, [knowledge_id], True)
     return ResponseModel(
         success=True, message=f"Knowledge {knowledge_id} deleted successfully"
     )
@@ -157,12 +164,49 @@ async def get_embedding_models_list(
     tenant: Tenant = get_tenant_with_permissions(Resource.PUBLIC, []),
 ):
     try:
-        registries = get_registry_list()
-        embedding_registry = registries.get(RegisterTypeEnum.EMBEDDING)
-        if not embedding_registry:
+        registries = get_all_registered_with_metadata(RegisterTypeEnum.EMBEDDING)
+        if not registries:
             raise KeyError("Embedding registry not found")
-        keys = [str(key) for key in embedding_registry._dict.keys()]
-        return ResponseModel(success=True, data=keys, message="Success")
+        # 取 keys 和 metadata ，组成 dict
+        models = []
+        for key, model_cls in registries.items():
+            models.append(
+                {
+                    "name": key,
+                    "metadata": model_cls.get("metadata", {}),
+                }
+            )
+        return ResponseModel(success=True, data=models, message="Success")
     except KeyError as e:
         logger.error(f"[get_embedding_models_list][error], error={str(e)}")
         raise HTTPException(status_code=404, detail=f"Registry not found: {str(e)}")
+
+
+@router.post(
+    "/{webhook_type}/{source}/{auth_info}/{knowledge_base_id}",
+    status_code=200,
+    summary="通用webhook处理器",
+    description="处理不同类型的webhook：knowledge, deployment, notification等",
+    response_model_by_alias=False,
+)
+async def handle_webhook(
+    webhook_type: str = Path(..., description="webhook type"),
+    source: str = Path(..., description="webhook source"),
+    auth_info: str = Path(..., description="auth info"),
+    knowledge_base_id: str = Path(..., description="knowledge base id"),
+    body: Dict[str, Any] = Body(..., description="webhook payload"),
+):
+    db_engine = PluginManager().dbPlugin
+    (is_valid, tenant, error) = await validate_key_string(
+        auth_info, Resource.KNOWLEDGE, [Action.ALL]
+    )
+    if not is_valid or not tenant:
+        raise HTTPException(status_code=401, detail=error)
+    res = await db_engine.handle_webhook(
+        tenant=tenant,
+        webhook_type=webhook_type,
+        source=source,
+        knowledge_base_id=knowledge_base_id,
+        payload=body,
+    )
+    return ResponseModel(success=True, data=res)

server/api/knowledge/router.py

@@ -56,10 +56,14 @@ async def _process_single_knowledge(
         )
     if not saved_knowledge:
         return new_knowledge
+
     elif saved_knowledge.file_sha != new_knowledge.file_sha:
+        # 仅删除当前文件，不删除子文件
         await db_engine.delete_knowledge(
             tenant.tenant_id, [saved_knowledge.knowledge_id]
         )
+        # 旧文件与新文件不同，需要删除旧文件，重新添加新文件,并更新knowledge_id,避免指向旧文件的 parent_id 丢失
+        new_knowledge.knowledge_id = saved_knowledge.knowledge_id
         return new_knowledge
     return None
 
@@ -71,17 +75,22 @@ async def gen_knowledge_list(
         return []
     db_engine = PluginManager().dbPlugin
     pre_add_knowledge_list: List[Knowledge] = []
+
+    # 创建信号量来控制并发数量为4
+    semaphore = asyncio.Semaphore(4)
+
+    async def _process_with_semaphore(record: KnowledgeCreateUnion):
+        async with semaphore:
+            return await _process_single_knowledge(record, tenant, db_engine)
+
     try:
-        tasks = [
-            _process_single_knowledge(record, tenant, db_engine)
-            for record in user_input
-        ]
+        tasks = [_process_with_semaphore(record) for record in user_input]
         results = await asyncio.gather(*tasks)
         for knowledge in results:
             if knowledge is not None:
                 pre_add_knowledge_list.append(knowledge)
         return pre_add_knowledge_list
 
     except Exception as e:
-        print(f"Error generating knowledge list: {e}")
+        logger.error(f"Error generating knowledge list: {e}")
         raise

server/api/knowledge/utils.py

@@ -1,4 +1,8 @@
 import logging
+from fastapi import Header, HTTPException
+from whiskerrag_types.model import Tenant, APIKey, Resource, Action
+from typing import Optional, List, Callable, Tuple
+from fastapi import Depends, HTTPException, Request
 from datetime import datetime, timezone
 from typing import Callable, List, Optional, Tuple
 
@@ -163,9 +167,128 @@ async def dependency(
         request: Request,
         header_auth: Optional[str] = Header(None, alias="Authorization"),
     ):
+        if not header_auth:
+            raise HTTPException(
+                status_code=401,
+                detail="Authorization header is missing. Please include a valid API key or secret key.",
+            )
         return await authenticate_request(request, header_auth, resource, actions)
 
     return Depends(dependency)
 
 
-__all__ = [get_tenant_with_permissions, Resource, Action]
+async def authenticate_by_key_string(
+    key_string: str,
+    resource: Resource = Resource.PUBLIC,
+    actions: List[Action] = [],
+) -> Tenant:
+    """
+    根据 ak 或 sk 字符串进行鉴权
+
+    Args:
+        key_string: API key 或 Secret key 字符串 (以 ak- 或 sk- 开头)
+        resource: 需要访问的资源
+        actions: 需要执行的操作列表
+
+    Returns:
+        Tenant: 验证通过的租户对象
+
+    Raises:
+        HTTPException: 鉴权失败时抛出异常
+    """
+    if not key_string:
+        raise HTTPException(status_code=401, detail="Key string is missing")
+
+    # 验证 key 格式
+    if not (key_string.startswith("ak-") or key_string.startswith("sk-")):
+        raise HTTPException(
+            status_code=401, detail="Key must start with 'ak-' or 'sk-'"
+        )
+
+    # 构造 Authorization header 格式进行复用现有逻辑
+    auth_header = f"Bearer {key_string}"
+
+    # 根据 key 类型选择认证方法
+    authenticate = authenticate_ak if key_string.startswith("ak-") else authenticate_sk
+
+    is_auth, tenant, api_key, error = await authenticate(auth_header)
+    if not is_auth:
+        raise HTTPException(status_code=403, detail=error)
+
+    # 检查 API key 权限
+    if key_string.startswith("ak-"):
+        if not await verify_permissions(api_key, resource, actions):
+            raise HTTPException(status_code=403, detail="Permission denied")
+    else:
+        # SK 情况下，记录访问日志
+        logger.info(f"Access granted for resource: {resource} with SK")
+
+    # 设置租户上下文
+    set_tenant_id(tenant.tenant_id)
+
+    return tenant
+
+
+async def authenticate_multiple_keys(
+    keys: List[str],
+    resource: Resource = Resource.PUBLIC,
+    actions: List[Action] = [],
+) -> List[Tuple[str, Optional[Tenant], Optional[str]]]:
+    """
+    批量验证多个 key
+
+    Args:
+        keys: key 字符串列表
+        resource: 需要访问的资源
+        actions: 需要执行的操作列表
+
+    Returns:
+        List[Tuple[str, Optional[Tenant], Optional[str]]]:
+        每个元素为 (key, tenant_or_none, error_message_or_none)
+    """
+    results = []
+
+    for key in keys:
+        try:
+            tenant = await authenticate_by_key_string(key, resource, actions)
+            results.append((key, tenant, None))
+        except HTTPException as e:
+            results.append((key, None, e.detail))
+        except Exception as e:
+            results.append((key, None, str(e)))
+
+    return results
+
+
+async def validate_key_string(
+    key_string: str,
+    resource: Resource = Resource.PUBLIC,
+    actions: List[Action] = [],
+) -> Tuple[bool, Optional[Tenant], Optional[str]]:
+    """
+    验证 key 字符串是否有效（不抛出异常的版本）
+
+    Args:
+        key_string: API key 或 Secret key 字符串
+        resource: 需要访问的资源
+        actions: 需要执行的操作列表
+
+    Returns:
+        Tuple[bool, Optional[Tenant], Optional[str]]:
+        (是否验证成功, 租户对象或None, 错误信息或None)
+    """
+    try:
+        tenant = await authenticate_by_key_string(key_string, resource, actions)
+        return True, tenant, None
+    except HTTPException as e:
+        return False, None, e.detail
+    except Exception as e:
+        return False, None, str(e)
+
+
+__all__ = [
+    get_tenant_with_permissions,
+    authenticate_multiple_keys,
+    validate_key_string,
+    Action,
+]