feat(backend): dba工具箱-实例级别主库故障切换 #9538

# Reviewed, transaction id: 36770

Author: ygcyao

dbm-ui/backend/db_meta/api/cluster/nosqlcomm/create_cluster.py

@@ -21,7 +21,6 @@
 from backend.db_meta.api import common
 from backend.db_meta.enums import AccessLayer, ClusterEntryType, ClusterPhase, ClusterStatus, ClusterType, MachineType
 from backend.db_meta.models import Cluster, ClusterEntry, ProxyInstance, StorageInstance
-from backend.db_services.dbbase.constants import IP_PORT_DIVIDER
 from backend.flow.utils.redis.redis_module_operate import RedisCCTopoOperator
 
 from ....exceptions import (
@@ -59,6 +58,7 @@ def create_twemproxy_cluster(
     2. proxy 不能有已绑定的后端
     3. 必须只有 1 个 master
     """
+    from backend.db_services.dbbase.constants import IP_PORT_DIVIDER
 
     ip_port_storages = {"{}{}{}".format(s["ip"], IP_PORT_DIVIDER, s["port"]): s for s in storages}
 

dbm-ui/backend/db_services/bigdata/influxdb/query.py

@@ -18,7 +18,6 @@
 from backend.db_meta.models.group import Group, GroupInstance
 from backend.db_meta.models.instance import StorageInstance
 from backend.db_services.bigdata.resources.query import BigDataBaseListRetrieveResource
-from backend.db_services.dbbase.constants import IP_PORT_DIVIDER
 from backend.db_services.dbbase.resources import query
 from backend.db_services.dbbase.resources.register import register_resource_decorator
 from backend.db_services.ipchooser.query.resource import ResourceQueryHelper
@@ -129,6 +128,7 @@ def _to_instance(
         cls, instance: dict, restart_map: dict, group_id_map: dict, group_name_map: dict, host_id__host_map: dict
     ) -> dict:
         """实例序列化"""
+        from backend.db_services.dbbase.constants import IP_PORT_DIVIDER
 
         instance_id = instance["id"]
         restart_at = restart_map.get(instance_id, "")

dbm-ui/backend/db_services/cmdb/biz.py

@@ -12,9 +12,12 @@
 import logging
 from typing import Dict, List
 
+from django.db.models import Count
+
 from backend.components import CCApi
 from backend.components.dbconfig.constants import DEPLOY_FILE_NAME, ConfType, LevelName
-from backend.db_meta.models import AppCache, DBModule
+from backend.db_meta.models import AppCache, Cluster, DBModule, Machine, ProxyInstance, StorageInstance
+from backend.db_services.cmdb.constants import ModuleCountType
 from backend.db_services.cmdb.exceptions import BkAppAttrAlreadyExistException
 from backend.db_services.dbconfig.dataclass import DBBaseConfig, DBConfigLevelData
 from backend.db_services.dbconfig.handlers import DBConfigHandler
@@ -222,3 +225,85 @@ def get_or_create_set_with_name(bk_biz_id: int, bk_set_name: str) -> int:
             raise err
         else:
             return bk_set_id
+
+
+def filter_by_biz_name(data: list, biz_name: str) -> list:
+    return [biz for biz in data if biz["bk_biz_name"] == biz_name]
+
+
+def filter_by_module_name(data: list, module_name: str) -> list:
+    result = []
+    for biz in data:
+        filtered_modules = [module for module in biz["modules"] if module["module_name"] == module_name]
+        if filtered_modules:
+            new_biz = biz.copy()
+            new_biz["modules"] = filtered_modules
+            result.append(new_biz)
+    return result
+
+
+def process_count_obj(queryset, nested_data):
+    count_obj = queryset.values("db_module_id", "bk_biz_id").annotate(count=Count("db_module_id")).order_by("-count")
+    for obj in count_obj:
+        bk_biz_id = obj["bk_biz_id"]
+        db_module_id = obj["db_module_id"]
+        count = obj["count"]
+        nested_data[bk_biz_id]["count"] += count
+        nested_data[bk_biz_id]["modules"][db_module_id] = count
+
+
+def list_biz_module_trees(
+    cluster_types: str, bk_biz_name: str, module_name: str, count_type: str, instance_inner_role: str
+) -> List[Dict]:
+    """
+    获取业务与模块维度集群数量
+    :param cluster_types: 以逗号分隔的集群类型字符串
+    :param bk_biz_name: 业务名称
+    :param module_name: 模块名称
+    :param count_type: 计数类型（如 "cluster" 或 "instance"）
+    :param instance_inner_role: 是否过滤为后端主实例
+    :return: 包含业务和模块维度的计数列表
+    """
+    cluster_type_list = cluster_types.split(",")
+
+    if count_type == ModuleCountType.CLUSTER.value:
+        queryset = Cluster.objects.filter(cluster_type__in=cluster_type_list)
+    elif count_type == ModuleCountType.INSTANCE.value:
+        filters = {"cluster_type__in": cluster_type_list}
+        if instance_inner_role:
+            filters["instance_inner_role"] = instance_inner_role
+        queryset = StorageInstance.objects.filter(**filters)
+    elif count_type == ModuleCountType.MACHINE.value:
+        queryset = Machine.objects.filter(cluster_type__in=cluster_type_list)
+    else:
+        raise ValueError(f"Unsupported count_type: {count_type}")
+
+    nested_data = collections.defaultdict(lambda: {"count": 0, "modules": collections.defaultdict(int)})
+    process_count_obj(queryset, nested_data)
+
+    if count_type == ModuleCountType.INSTANCE.value:
+        proxy_queryset = ProxyInstance.objects.filter(cluster_type__in=cluster_type_list)
+        process_count_obj(proxy_queryset, nested_data)
+
+    db_module_map = DBModule.db_module_map()
+    id_to_name = AppCache.id_to_name()
+
+    final_data = [
+        {
+            "bk_biz_name": id_to_name.get(bk_biz_id, ""),
+            "bk_biz_id": bk_biz_id,
+            "count": data["count"],
+            "modules": [
+                {"module_name": db_module_map.get(module_id, ""), "module_id": module_id, "count": count}
+                for module_id, count in data["modules"].items()
+            ],
+        }
+        for bk_biz_id, data in nested_data.items()
+    ]
+
+    if bk_biz_name:
+        final_data = filter_by_biz_name(final_data, bk_biz_name)
+    if module_name:
+        final_data = filter_by_module_name(final_data, module_name)
+
+    return final_data

dbm-ui/backend/db_services/cmdb/constants.py

@@ -8,6 +8,17 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
+from django.utils.translation import gettext as _
+
+from blue_krill.data_types.enum import EnumField, StructuredEnum
 
 MAX_DB_MODULE_LIMIT = 63
 MAX_DB_APP_ABBR_LIMIT = 63
+
+
+class ModuleCountType(str, StructuredEnum):
+    """module统计类型"""
+
+    CLUSTER = EnumField("cluster", _("集群"))
+    INSTANCE = EnumField("instance", _("实例"))
+    MACHINE = EnumField("machine", _("主机"))

dbm-ui/backend/db_services/cmdb/serializers.py

@@ -11,8 +11,8 @@
 from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
-from backend.db_meta.enums import ClusterType
-from backend.db_services.cmdb.constants import MAX_DB_APP_ABBR_LIMIT, MAX_DB_MODULE_LIMIT
+from backend.db_meta.enums import ClusterType, InstanceInnerRole
+from backend.db_services.cmdb.constants import MAX_DB_APP_ABBR_LIMIT, MAX_DB_MODULE_LIMIT, ModuleCountType
 from backend.iam_app.dataclass import ResourceEnum
 from backend.iam_app.dataclass.actions import ActionEnum
 
@@ -80,3 +80,25 @@ class ListNodesSerializer(TopoSerializer):
     page = serializers.IntegerField(help_text=_("页数"))
     module_id = serializers.IntegerField(help_text=_("模块ID"), required=False)
     set_id = serializers.IntegerField(help_text=_("集群ID"), required=False)
+
+
+class ListBIZModulesSLZ(serializers.Serializer):
+    cluster_types = serializers.CharField(help_text=_("集群类型(逗号分隔)"))
+    bk_biz_name = serializers.CharField(help_text=_("业务名称"), required=False)
+    module_name = serializers.CharField(help_text=_("模块名称"), required=False)
+    count_type = serializers.ChoiceField(choices=ModuleCountType.get_choices(), default=ModuleCountType.CLUSTER.value)
+    instance_inner_role = serializers.ChoiceField(
+        help_text=_("实例角色"), choices=InstanceInnerRole.get_choices(), required=False
+    )
+
+
+class BIZModuleSLZ(serializers.Serializer):
+    class ModuleInstanceCountSLZ(serializers.Serializer):
+        module_name = serializers.CharField(help_text=_("模块名"))
+        module_id = serializers.IntegerField(help_text=_("模块ID"))
+        count = serializers.IntegerField(help_text=_("实例数量"))
+
+    bk_biz_name = serializers.CharField(help_text=_("业务名"))
+    bk_biz_id = serializers.IntegerField(help_text=_("业务ID"))
+    count = serializers.IntegerField(help_text=_("实例数量"))
+    modules = serializers.ListField(help_text=_("模块信息"), child=ModuleInstanceCountSLZ())

dbm-ui/backend/db_services/cmdb/views.py

@@ -99,3 +99,23 @@ def set_db_app_abbr(self, request, bk_biz_id):
     @action(methods=["GET"], detail=True)
     def list_cc_obj_user(self, request, bk_biz_id):
         return Response(biz.list_cc_obj_user(bk_biz_id))
+
+    @common_swagger_auto_schema(
+        operation_summary=_("业务模块树信息"),
+        query_serializer=serializers.ListBIZModulesSLZ(),
+        responses={status.HTTP_200_OK: serializers.BIZModuleSLZ(label=_("业务模块树信息"), many=True)},
+        tags=[SWAGGER_TAG],
+    )
+    @action(methods=["GET"], detail=False, serializer_class=serializers.ListBIZModulesSLZ)
+    def list_biz_module_trees(self, request):
+        validated_data = self.params_validate(self.get_serializer_class())
+        cluster_types = validated_data.get("cluster_types")
+        bk_biz_name = validated_data.get("bk_biz_name")
+        module_name = validated_data.get("module_name")
+        count_type = validated_data.get("count_type")
+        instance_inner_role = validated_data.get("instance_inner_role")
+        serializer = serializers.BIZModuleSLZ(
+            biz.list_biz_module_trees(cluster_types, bk_biz_name, module_name, count_type, instance_inner_role),
+            many=True,
+        )
+        return Response(serializer.data)

dbm-ui/backend/db_services/dbbase/cluster/handlers.py

@@ -18,6 +18,7 @@
 from django.forms import model_to_dict
 from django.utils.translation import ugettext_lazy as _
 
+from backend.configuration.constants import DBType
 from backend.db_meta.enums import AccessLayer, ClusterType, InstanceInnerRole, InstanceStatus
 from backend.db_meta.exceptions import ClusterNotExistException, InstanceNotExistException
 from backend.db_meta.models import Cluster, ProxyInstance, StorageInstance, StorageInstanceTuple
@@ -368,3 +369,25 @@ def get_cluster_service_handler(bk_biz_id: int, db_type: str = "dbbase"):
         handler = ClusterServiceHandler(bk_biz_id)
 
     return handler
+
+
+def retrieve_resources(self, request, serializer_class, resource_method_name):
+    """
+    通用方法来处理不同资源类型的请求。
+    """
+    from backend.db_services.dbbase.constants import DB_RESOURCE_MAP
+
+    query_params = self.params_validate(serializer_class)
+    bk_biz_id = query_params.pop("bk_biz_id", None)
+    db_type = query_params.pop("db_type", None)
+    RetrieveResource = DB_RESOURCE_MAP[db_type]
+
+    # 为 MySQL 和 SQLServer 设置集群类型
+    if db_type in [DBType.MySQL.value, DBType.Sqlserver.value]:
+        RetrieveResource.cluster_types = ClusterType.db_type_cluster_types_map().get(db_type)
+
+    # 动态调用资源方法获取数据
+    resource_method = getattr(RetrieveResource, resource_method_name)
+    data = self.paginator.paginate_list(request, bk_biz_id, resource_method, query_params)
+
+    return self.get_paginated_response(data)

dbm-ui/backend/db_services/dbbase/constants.py

@@ -10,6 +10,22 @@
 """
 from django.utils.translation import ugettext_lazy as _
 
+from backend.configuration.constants import DBType
+from backend.db_services.bigdata.doris.query import DorisListRetrieveResource
+from backend.db_services.bigdata.es.query import ESListRetrieveResource
+from backend.db_services.bigdata.hdfs.query import HDFSListRetrieveResource
+from backend.db_services.bigdata.influxdb.query import InfluxDBListRetrieveResource
+from backend.db_services.bigdata.kafka.query import KafkaListRetrieveResource
+from backend.db_services.bigdata.pulsar.query import PulsarListRetrieveResource
+from backend.db_services.bigdata.riak.query import RiakListRetrieveResource
+from backend.db_services.bigdata.vm.query import VMListRetrieveResource
+from backend.db_services.mongodb.resources.query import MongoDBListRetrieveResource
+from backend.db_services.mysql.resources.tendbcluster.query import (
+    ListRetrieveResource as TendDBClusterRetrieveResource,
+)
+from backend.db_services.mysql.resources.tendbha.query import ListRetrieveResource as TendDBHAHaRetrieveResource
+from backend.db_services.redis.resources.redis_cluster.query import RedisListRetrieveResource
+from backend.db_services.sqlserver.resources.query import SqlserverListRetrieveResource
 from blue_krill.data_types.enum import EnumField, StructuredEnum
 
 ES_DEFAULT_PORT = 9200
@@ -44,3 +60,20 @@ class ResourceType(str, StructuredEnum):
 
     SPOTTY_HOST = EnumField("spotty_host", _("污点主机"))
     RESOURCE_RECORD = EnumField("resource_record", _("资源操作记录"))
+
+
+DB_RESOURCE_MAP = {
+    DBType.TenDBCluster.value: TendDBHAHaRetrieveResource,
+    DBType.MySQL.value: TendDBClusterRetrieveResource,
+    DBType.Redis.value: RedisListRetrieveResource,
+    DBType.MongoDB.value: MongoDBListRetrieveResource,
+    DBType.Kafka.value: KafkaListRetrieveResource,
+    DBType.Hdfs.value: HDFSListRetrieveResource,
+    DBType.Es.value: ESListRetrieveResource,
+    DBType.Pulsar.value: PulsarListRetrieveResource,
+    DBType.InfluxDB.value: InfluxDBListRetrieveResource,
+    DBType.Riak.value: RiakListRetrieveResource,
+    DBType.Sqlserver.value: SqlserverListRetrieveResource,
+    DBType.Doris.value: DorisListRetrieveResource,
+    DBType.Vm.value: VMListRetrieveResource,
+}