feat: 增加流程,任务和插件相关的运营统计数据 --story=130062520

bkflow/statistics/admin.py

@@ -0,0 +1,121 @@
+"""
+TencentBlueKing is pleased to support the open source community by making
+蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available.
+Copyright (C) 2024 THL A29 Limited,
+a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+We undertake not to change the open source license (MIT license) applicable
+
+to the current version of the project delivered to anyone in the future.
+"""
+
+from django.contrib import admin
+
+from bkflow.statistics.models import (
+    DailyStatisticsSummary,
+    PluginExecutionSummary,
+    TaskflowExecutedNodeStatistics,
+    TaskflowStatistics,
+    TemplateNodeStatistics,
+    TemplateStatistics,
+)
+
+
+@admin.register(TemplateNodeStatistics)
+class TemplateNodeStatisticsAdmin(admin.ModelAdmin):
+    list_display = ["id", "template_id", "space_id", "component_code", "node_name", "is_remote", "created_at"]
+    list_filter = ["space_id", "is_remote", "is_sub"]
+    search_fields = ["component_code", "node_name", "template_id"]
+    readonly_fields = ["created_at"]
+
+
+@admin.register(TemplateStatistics)
+class TemplateStatisticsAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+        "template_id",
+        "space_id",
+        "template_name",
+        "atom_total",
+        "subprocess_total",
+        "is_enabled",
+        "updated_at",
+    ]
+    list_filter = ["space_id", "is_enabled"]
+    search_fields = ["template_name", "template_id"]
+    readonly_fields = ["created_at", "updated_at"]
+
+
+@admin.register(TaskflowStatistics)
+class TaskflowStatisticsAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+        "task_id",
+        "space_id",
+        "template_id",
+        "create_method",
+        "is_success",
+        "final_state",
+        "elapsed_time",
+        "create_time",
+    ]
+    list_filter = ["space_id", "create_method", "trigger_method", "is_success", "final_state"]
+    search_fields = ["task_id", "instance_id"]
+    readonly_fields = ["created_at", "updated_at"]
+
+
+@admin.register(TaskflowExecutedNodeStatistics)
+class TaskflowExecutedNodeStatisticsAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+        "task_id",
+        "node_id",
+        "component_code",
+        "status",
+        "state",
+        "elapsed_time",
+        "started_time",
+    ]
+    list_filter = ["space_id", "status", "state", "is_skip", "is_retry"]
+    search_fields = ["component_code", "node_name", "task_id"]
+    readonly_fields = ["created_at"]
+
+
+@admin.register(DailyStatisticsSummary)
+class DailyStatisticsSummaryAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+        "date",
+        "space_id",
+        "task_created_count",
+        "task_success_count",
+        "task_failed_count",
+        "avg_task_elapsed_time",
+    ]
+    list_filter = ["space_id", "date"]
+    readonly_fields = ["created_at", "updated_at"]
+
+
+@admin.register(PluginExecutionSummary)
+class PluginExecutionSummaryAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+        "period_type",
+        "period_start",
+        "space_id",
+        "component_code",
+        "execution_count",
+        "success_count",
+        "avg_elapsed_time",
+    ]
+    list_filter = ["space_id", "period_type", "is_remote"]
+    search_fields = ["component_code"]
+    readonly_fields = ["created_at", "updated_at"]

bkflow/statistics/apps.py

@@ -0,0 +1,37 @@
+"""
+TencentBlueKing is pleased to support the open source community by making
+蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available.
+Copyright (C) 2024 THL A29 Limited,
+a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+We undertake not to change the open source license (MIT license) applicable
+
+to the current version of the project delivered to anyone in the future.
+"""
+
+from django.apps import AppConfig
+
+
+class AnalysisStatisticsConfig(AppConfig):
+    name = "bkflow.statistics"
+    verbose_name = "Statistics"
+
+    def ready(self):
+        # 注册信号处理器
+        try:
+            from bkflow.statistics.signals import register_statistics_signals
+
+            register_statistics_signals()
+        except Exception as e:
+            import logging
+
+            logger = logging.getLogger("root")
+            logger.warning(f"[AnalysisStatistics] Failed to register signals: {e}")

bkflow/statistics/collectors/__init__.py

@@ -0,0 +1,28 @@
+"""
+TencentBlueKing is pleased to support the open source community by making
+蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available.
+Copyright (C) 2024 THL A29 Limited,
+a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+We undertake not to change the open source license (MIT license) applicable
+
+to the current version of the project delivered to anyone in the future.
+"""
+
+from bkflow.statistics.collectors.base import BaseStatisticsCollector
+from bkflow.statistics.collectors.task_collector import TaskStatisticsCollector
+from bkflow.statistics.collectors.template_collector import TemplateStatisticsCollector
+
+__all__ = [
+    "BaseStatisticsCollector",
+    "TemplateStatisticsCollector",
+    "TaskStatisticsCollector",
+]

bkflow/statistics/collectors/base.py

@@ -0,0 +1,87 @@
+"""
+TencentBlueKing is pleased to support the open source community by making
+蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available.
+Copyright (C) 2024 THL A29 Limited,
+a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+We undertake not to change the open source license (MIT license) applicable
+
+to the current version of the project delivered to anyone in the future.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Tuple
+
+from bkflow.statistics.conf import StatisticsConfig
+
+logger = logging.getLogger("celery")
+
+
+class BaseStatisticsCollector(ABC):
+    """统计采集器基类"""
+
+    def __init__(self):
+        self.db_alias = StatisticsConfig.get_db_alias()
+        self.engine_id = StatisticsConfig.get_engine_id()
+
+    @abstractmethod
+    def collect(self):
+        """执行采集"""
+        raise NotImplementedError
+
+    @staticmethod
+    def count_pipeline_tree_nodes(pipeline_tree: dict) -> Tuple[int, int, int]:
+        """
+        统计 pipeline_tree 中的节点数量
+
+        Args:
+            pipeline_tree: pipeline 树结构
+
+        Returns:
+            tuple: (atom_total, subprocess_total, gateways_total)
+        """
+        activities = pipeline_tree.get("activities", {})
+        gateways = pipeline_tree.get("gateways", {})
+
+        atom_total = 0
+        subprocess_total = 0
+
+        for act_id, act in activities.items():
+            act_type = act.get("type", "")
+            if act_type == "ServiceActivity":
+                atom_total += 1
+            elif act_type == "SubProcess":
+                subprocess_total += 1
+                # 递归统计子流程内的节点
+                sub_pipeline = act.get("pipeline", {})
+                if sub_pipeline:
+                    sub_atom, sub_subproc, _ = BaseStatisticsCollector.count_pipeline_tree_nodes(sub_pipeline)
+                    atom_total += sub_atom
+                    subprocess_total += sub_subproc
+
+        gateways_total = len(gateways)
+
+        return atom_total, subprocess_total, gateways_total
+
+    @staticmethod
+    def parse_datetime(time_str):
+        """解析时间字符串"""
+        if not time_str:
+            return None
+        from django.utils.dateparse import parse_datetime
+
+        # 处理多种可能的时间格式
+        if isinstance(time_str, str):
+            # 替换空格分隔的时区格式
+            time_str = time_str.replace(" +", "+").replace(" -", "-")
+            return parse_datetime(time_str)
+        return time_str