fix(mysql): 标准化流程优化, crontab 安装保留业务自定义任务 #7509

Author: xfwduke

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/standardize_mysql.go

@@ -98,9 +98,10 @@ func (c *StandardizeMySQLComp) Init() error {
 }
 
 func (c *StandardizeMySQLComp) ClearOldCrontab() error {
-	err := osutil.RemoveUserCrontab("mysql")
+	err := osutil.CleanLocalCrontab()
 	if err != nil {
 		logger.Error("clear mysql crontab failed: %s", err.Error())
+		return err
 	} else {
 		logger.Info("clear mysql crontab success")
 	}

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/standardize_proxy.go

@@ -11,9 +11,10 @@ type StandardizeProxyComp struct {
 }
 
 func (c *StandardizeProxyComp) ClearOldCrontab() error {
-	err := osutil.RemoveUserCrontab("mysql")
+	err := osutil.CleanLocalCrontab()
 	if err != nil {
 		logger.Error("clear mysql crontab failed: %s", err.Error())
+		return err
 	} else {
 		logger.Info("clear mysql crontab success")
 	}

dbm-services/mysql/db-tools/dbactuator/pkg/util/osutil/crontab.go

@@ -202,16 +202,18 @@ func CleanLocalCrontab() error {
 		slaveSync          = "/home/mysql/monitor/master_slave_sync_check.pl"
 		tbinlodumperStatus = "tbinlogdumper_status.pl"
 		prometheus         = "prometheus"
+		filebeatWatcher    = "/home/mysql/filebeat-deploy/monitor_filebeat_watcher"
 	)
 	cleanCrontabs := []string{
 		getStatusPL, dbBackupOld, dbBackupNew, dbBackupMulti, dbBackupXtrabackup, rotateLog,
-		proxyStatus, slaveSync, tbinlodumperStatus, prometheus, rotateLogGo, dbBackupMultiGo,
+		proxyStatus, slaveSync, tbinlodumperStatus, prometheus, rotateLogGo, dbBackupMultiGo, filebeatWatcher,
 	}
 
 	existCrontabs, err := CrontabsExist(cleanCrontabs)
 	if err != nil {
 		return err
 	}
+	logger.Info("find crontabs %s", existCrontabs)
 	// 如果不存在需要清理的crontab 直接返回成功
 	if len(existCrontabs) <= 0 {
 		return nil

dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_ha_standardize_flow.py

@@ -67,6 +67,8 @@ def standardize(self):
 
         cluster_objects = Cluster.objects.filter(
             pk__in=cluster_ids, bk_biz_id=bk_biz_id, cluster_type=ClusterType.TenDBHA.value
+        ).prefetch_related(
+            "proxyinstance_set", "storageinstance_set", "proxyinstance_set__machine", "storageinstance_set__machine"
         )
         if cluster_objects.count() != len(cluster_ids):
             raise DBMetaException(
@@ -78,7 +80,7 @@ def standardize(self):
             data=self.data,
             need_random_pass_cluster_ids=list(set(self.data["infos"]["cluster_ids"])),
         )
-        standardize_pipe.add_sub_pipeline(self._build_trans_module_sub(clusters=cluster_objects))
+
         standardize_pipe.add_sub_pipeline(self._build_instantiate_mysql_config_sub(clusters=cluster_objects))
 
         # 为了代码方便这里稍微特殊点
@@ -111,6 +113,9 @@ def standardize(self):
                 self._build_storage_sub(ips=storage_ips),
             ]
         )
+
+        standardize_pipe.add_sub_pipeline(self._build_trans_module_sub(clusters=cluster_objects))
+
         logger.info(_("构建TenDBHA集群标准化流程成功"))
         standardize_pipe.run_pipeline(is_drop_random_user=True)
 

dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_cluster_standardize_flow.py

@@ -65,6 +65,12 @@ def standardize(self):
 
         cluster_objects = Cluster.objects.filter(
             pk__in=cluster_ids, bk_biz_id=bk_biz_id, cluster_type=ClusterType.TenDBCluster.value
+        ).prefetch_related(
+            "proxyinstance_set",
+            "storageinstance_set",
+            "proxyinstance_set__machine",
+            "storageinstance_set__machine",
+            "proxyinstance_set__tendbclusterspiderext",
         )
         if cluster_objects.count() != len(cluster_ids):
             raise DBMetaException(
@@ -77,7 +83,6 @@ def standardize(self):
             need_random_pass_cluster_ids=list(set(self.data["infos"]["cluster_ids"])),
         )
 
-        standardize_pipe.add_sub_pipeline(self._build_trans_module_sub(clusters=cluster_objects))
         standardize_pipe.add_sub_pipeline(self._build_instantiate_config_sub(clusters=cluster_objects))
 
         spider_master_ips = {}
@@ -123,6 +128,8 @@ def standardize(self):
         if spider_mnt_ips:
             standardize_pipe.add_sub_pipeline(self._build_spider_mnt_sub(ips=spider_mnt_ips))
 
+        standardize_pipe.add_sub_pipeline(self._build_trans_module_sub(clusters=cluster_objects))
+
         logger.info(_("构建TenDBCluster集群标准化流程成功"))
         standardize_pipe.run_pipeline(is_drop_random_user=True)
 

dbm-ui/backend/flow/engine/bamboo/scene/tendbsingle/standardize.py

@@ -54,7 +54,7 @@ def standardize(self):
 
         cluster_objects = Cluster.objects.filter(
             pk__in=cluster_ids, bk_biz_id=bk_biz_id, cluster_type=ClusterType.TenDBSingle.value
-        )
+        ).prefetch_related("storageinstance_set", "storageinstance_set__machine")
         if cluster_objects.count() != len(cluster_ids):
             raise DBMetaException(
                 message="input {} clusters, but found {}".format(len(cluster_ids), cluster_objects.count())
@@ -65,7 +65,7 @@ def standardize(self):
             data=self.data,
             need_random_pass_cluster_ids=list(set(self.data["infos"]["cluster_ids"])),
         )
-        standardize_pipe.add_sub_pipeline(self._build_trans_module_sub(clusters=cluster_objects))
+
         standardize_pipe.add_sub_pipeline(self._build_instantiate_mysql_config_sub(clusters=cluster_objects))
 
         storage_ips = {}
@@ -84,6 +84,7 @@ def standardize(self):
                 self._build_storage_sub(ips=storage_ips),
             ]
         )
+        standardize_pipe.add_sub_pipeline(self._build_trans_module_sub(clusters=cluster_objects))
         logger.info(_("构建TenDBSingle集群标准化流程成功"))
         standardize_pipe.run_pipeline(is_drop_random_user=True)
 

dbm-ui/backend/flow/plugins/components/collections/mysql/cluster_standardize_trans_module.py

@@ -11,6 +11,7 @@
 from django.utils.translation import ugettext as _
 from pipeline.component_framework.component import Component
 
+from backend.db_meta.enums import TenDBClusterSpiderRole
 from backend.db_meta.models import Cluster
 from backend.flow.plugins.components.collections.common.base_service import BaseService
 from backend.flow.utils.mysql.mysql_module_operate import MysqlCCTopoOperator
@@ -28,8 +29,15 @@ def _execute(self, data, parent_data) -> bool:
             cluster_obj.storageinstance_set.all(), is_increment=True
         )
         MysqlCCTopoOperator(cluster_obj).transfer_instances_to_cluster_module(
-            cluster_obj.proxyinstance_set.all(), is_increment=True
+            cluster_obj.proxyinstance_set.exclude(
+                tendbclusterspiderext__spider_role__in=[
+                    TenDBClusterSpiderRole.SPIDER_MNT,
+                    TenDBClusterSpiderRole.SPIDER_SLAVE_MNT,
+                ]
+            ),
+            is_increment=True,
         )
+
         self.log_info(_("[{}] CC 模块标准化完成".format(kwargs["node_name"])))
         return True
 

dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py

@@ -1607,7 +1607,9 @@ def get_deploy_mysql_monitor_payload(self, **kwargs) -> dict:
                 )
         # 增加对安装spider监控的适配
         elif machine.machine_type == MachineType.SPIDER.value:
-            for instance in ProxyInstance.objects.filter(machine__ip=kwargs["ip"], cluster__id__in=cluster_ids):
+            for instance in ProxyInstance.objects.filter(
+                machine__ip=kwargs["ip"], cluster__id__in=cluster_ids
+            ).prefetch_related("tendbclusterspiderext"):
                 cluster = instance.cluster.get()
                 instances_info.append(
                     {