feat: 项目和集群数据同步值storage到消息队列

bcs-services/bcs-cluster-manager/conf/bcs-cluster-manager.json.template

@@ -40,6 +40,11 @@
         "templateURL": "${bcsSopsTemplateURL}",
         "frontURL": "${bcsSopsFrontURL}"
     },
+    "storage": {
+        "host": "${bcsStorageHost}",
+        "token": "${bcsStorageToken}",
+        "debug": ${debug}
+    },
     "cmdb": {
         "enable": ${bcsCmdbEnable},
         "appCode": "${bcsAppCode}",

bcs-services/bcs-cluster-manager/internal/actions/cluster/create.go

@@ -25,6 +25,7 @@ import (
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/actions"
 	autils "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/actions/utils"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/cloudprovider"
+	com "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/cloudprovider/common"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/common"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/lock"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/remote/encrypt"
@@ -516,6 +517,35 @@ func (ca *CreateAction) Handle(ctx context.Context, req *cmproto.CreateClusterRe
 		blog.Errorf("create cluster[%s] CreateOperationLog failed: %v", cls.ClusterID, err)
 	}
 
+	// sync cluster data to storage
+	err = com.StorageCli.SyncClusterData(cls.ClusterID, map[string]interface{}{
+		"clusterID":       cls.ClusterID,
+		"clusterName":     cls.ClusterName,
+		"provider":        cls.Provider,
+		"region":          cls.Region,
+		"vpcID":           cls.VpcID,
+		"projectID":       cls.ProjectID,
+		"businessID":      cls.BusinessID,
+		"environment":     cls.Environment,
+		"engineType":      cls.EngineType,
+		"clusterType":     cls.ClusterType,
+		"labels":          cls.Labels,
+		"creator":         cls.Creator,
+		"createTime":      cls.CreateTime,
+		"systemID":        cls.SystemID,
+		"manageType":      cls.ManageType,
+		"status":          cls.Status,
+		"networkType":     cls.NetworkType,
+		"moduleID":        cls.ModuleID,
+		"isCommonCluster": cls.IsCommonCluster,
+		"description":     cls.Description,
+		"clusterCategory": cls.ClusterCategory,
+		"isShared":        cls.IsShared,
+	})
+	if err != nil {
+		blog.Errorf("create cluster[%s] sync cluster data to storage failed: %v", cls.ClusterID, err)
+	}
+
 	ca.resp.Data = cls
 	ca.resp.Task = ca.task
 	ca.setResp(common.BcsErrClusterManagerSuccess, common.BcsErrClusterManagerSuccessStr)

bcs-services/bcs-cluster-manager/internal/actions/cluster/delete.go

@@ -26,6 +26,7 @@ import (
 	cmproto "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/api/clustermanager"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/actions"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/cloudprovider"
+	com "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/cloudprovider/common"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/clusterops"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/common"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/store"
@@ -460,6 +461,11 @@ func (da *DeleteAction) Handle(ctx context.Context, req *cmproto.DeleteClusterRe
 		blog.Errorf("delete cluster[%s] CreateOperationLog failed: %v", da.cluster.ClusterID, err)
 	}
 
+	err = com.StorageCli.DelClusterData(da.cluster.ClusterID)
+	if err != nil {
+		blog.Errorf("delete cluster[%s] del storage data failed: %v", da.cluster.ClusterID, err)
+	}
+
 	da.resp.Data = da.cluster
 	da.resp.Task = da.tasks
 	da.setResp(common.BcsErrClusterManagerSuccess, common.BcsErrClusterManagerSuccessStr)

bcs-services/bcs-cluster-manager/internal/app/app.go

@@ -469,6 +469,21 @@ func (cm *ClusterManager) initBKOpsClient() error {
 	return nil
 }
 
+// init bk-ops client
+func (cm *ClusterManager) initStorageClient() error {
+	err := common.SetStorageClient(common.StorageOptions{
+		Host:  cm.opt.StorageManager.Host,
+		Token: cm.opt.StorageManager.Token,
+		Debug: cm.opt.StorageManager.Debug,
+	})
+	if err != nil {
+		blog.Errorf("initBKOpsClient failed: %v", err)
+		return err
+	}
+
+	return nil
+}
+
 // init iam client for perm
 func (cm *ClusterManager) initIAMClient() error {
 	var err error
@@ -1164,6 +1179,11 @@ func (cm *ClusterManager) Init() error {
 	if err != nil {
 		return err
 	}
+	// init storage client
+	err = cm.initStorageClient()
+	if err != nil {
+		return err
+	}
 	// init cloud template config
 	err = cm.initCloudTemplateConfig()
 	if err != nil {

bcs-services/bcs-cluster-manager/internal/cloudprovider/common/storage_client.go

@@ -0,0 +1,127 @@
+/*
+ * Tencent is pleased to support the open source community by making Blueking Container Service available.
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ * http://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package common
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
+	"github.com/parnurzeal/gorequest"
+)
+
+// StorageOptions storage options
+type StorageOptions struct {
+	Host  string
+	Token string
+	Debug bool
+}
+
+// StorageClient storage client
+type StorageClient struct {
+	host        string
+	token       string
+	serverDebug bool
+}
+
+// StorageCli global storage client
+var StorageCli *StorageClient
+
+// SetStorageClient set storage client
+func SetStorageClient(options StorageOptions) error {
+	cli, err := NewStorageClient(options)
+	if err != nil {
+		return err
+	}
+
+	StorageCli = cli
+	return nil
+}
+
+// NewStorageClient create bcs storage client
+func NewStorageClient(options StorageOptions) (*StorageClient, error) {
+	c := &StorageClient{
+		host:        options.Host,
+		token:       options.Token,
+		serverDebug: options.Debug,
+	}
+
+	return c, nil
+}
+
+// SyncClusterData sync cluster data to storage
+func (s *StorageClient) SyncClusterData(clusterID string, data map[string]interface{}) error {
+	if s == nil {
+		return ErrServerNotInit
+	}
+
+	reqUrl := fmt.Sprintf("%s/bcsapi/v4/storage/clusters/%s", s.host, clusterID)
+	respData := &StorageResponse{}
+
+	_, _, errs := gorequest.New().
+		Timeout(defaultTimeOut).
+		Put(reqUrl).
+		Set("Content-Type", "application/json").
+		Set("Accept", "application/json").
+		Set("Authorization", fmt.Sprintf("Bearer %s", s.token)).
+		SetDebug(s.serverDebug).
+		Send(SyncClusterDataRequest{Data: data}).
+		EndStruct(&respData)
+	if len(errs) > 0 {
+		blog.Errorf("call api SyncClusterData failed: %v", errs[0])
+		return errs[0]
+	}
+
+	if !respData.Result {
+		blog.Errorf("call api SyncClusterData failed: %v", respData.Message)
+		return errors.New(respData.Message)
+	}
+
+	// successfully request
+	blog.Infof("call api SyncClusterData with url(%s) successfully", reqUrl)
+
+	return nil
+}
+
+// DelClusterData del storage cluster data
+func (s *StorageClient) DelClusterData(clusterID string) error {
+	if s == nil {
+		return ErrServerNotInit
+	}
+
+	reqUrl := fmt.Sprintf("%s/bcsapi/v4/storage/clusters/%s", s.host, clusterID)
+	respData := &StorageResponse{}
+
+	_, _, errs := gorequest.New().
+		Timeout(defaultTimeOut).
+		Delete(reqUrl).
+		Set("Content-Type", "application/json").
+		Set("Accept", "application/json").
+		Set("Authorization", fmt.Sprintf("Bearer %s", s.token)).
+		SetDebug(s.serverDebug).
+		EndStruct(&respData)
+	if len(errs) > 0 {
+		blog.Errorf("call api DelClusterData failed: %v", errs[0])
+		return errs[0]
+	}
+
+	if !respData.Result {
+		blog.Errorf("call api DelClusterData failed: %v", respData.Message)
+		return errors.New(respData.Message)
+	}
+
+	// successfully request
+	blog.Infof("call api DelClusterData with url(%s) successfully", reqUrl)
+
+	return nil
+}

bcs-services/bcs-cluster-manager/internal/cloudprovider/common/types.go

@@ -305,3 +305,16 @@ type OperateTaskResponse struct {
 	Result  bool   `json:"result"`
 	Message string `json:"message"`
 }
+
+// SyncClusterDataRequest sync cluster data to storage request
+type SyncClusterDataRequest struct {
+	Data map[string]interface{} `json:"data"`
+}
+
+// StorageResponse sync cluster data to storage response
+type StorageResponse struct {
+	Result  bool        `json:"result"`
+	Code    int         `json:"code"`
+	Message string      `json:"message"`
+	Data    interface{} `json:"data"`
+}

bcs-services/bcs-cluster-manager/internal/options/options.go

@@ -172,6 +172,13 @@ type UserConfig struct {
 	Token       string `json:"token"`
 }
 
+// StorageConfig storage config
+type StorageConfig struct {
+	Host  string `json:"host"`
+	Token string `json:"token"`
+	Debug bool   `json:"debug"`
+}
+
 // AlarmConfig for alarm interface
 type AlarmConfig struct {
 	Server        string `json:"server"`
@@ -308,6 +315,7 @@ type ClusterManagerOptions struct {
 	Mongo              MongoConfig           `json:"mongo"`
 	Broker             BrokerConfig          `json:"broker"`
 	BKOps              BKOpsConfig           `json:"bkOps"`
+	StorageManager     StorageConfig         `json:"storage"`
 	Cmdb               CmdbConfig            `json:"cmdb"`
 	TenCmdb            TenCmdbConfig         `json:"tenCmdb"`
 	NodeMan            NodeManConfig         `json:"nodeman"`

bcs-services/bcs-project-manager/internal/actions/project/create.go

@@ -22,6 +22,7 @@ import (
 	"github.com/Tencent/bk-bcs/bcs-services/pkg/bcs-auth/middleware"
 
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-project-manager/internal/component/bcscc"
+	"github.com/Tencent/bk-bcs/bcs-services/bcs-project-manager/internal/component/bcsstorage"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-project-manager/internal/logging"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-project-manager/internal/store"
 	pm "github.com/Tencent/bk-bcs/bcs-services/bcs-project-manager/internal/store/project"
@@ -66,6 +67,16 @@ func (ca *CreateAction) Do(ctx context.Context, req *proto.CreateProjectRequest)
 	if err != nil {
 		return nil, errorx.NewDBErr(err.Error())
 	}
+
+	// 向 bcs storage 写入数据
+	go func() {
+		if err := bcsstorage.SyncProjectData(p); err != nil {
+			logging.Error("[ALARM-CC-PROJECT] create project %s/%s in storage failed, err: %s",
+				p.ProjectID, p.ProjectCode, err.Error())
+		}
+
+	}()
+
 	// 向 bcs cc 写入数据
 	go func() {
 		if err := bcscc.CreateProject(p); err != nil {

bcs-services/bcs-project-manager/internal/component/bcsstorage/bcsstorage.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-project-manager/internal/config"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-project-manager/internal/logging"
+	pm "github.com/Tencent/bk-bcs/bcs-services/bcs-project-manager/internal/store/project"
 )
 
 var (
@@ -41,6 +42,19 @@ type QuotaInfo struct {
 	Data MultiClusterResourceQuota `json:"data"` // 多集群资源配额数据
 }
 
+// SyncProjectDataRequest sync project data to storage request
+type SyncProjectDataRequest struct {
+	Data map[string]interface{} `json:"data"`
+}
+
+// StorageResponse sync project data to storage response
+type StorageResponse struct {
+	Result  bool        `json:"result"`
+	Code    int         `json:"code"`
+	Message string      `json:"message"`
+	Data    interface{} `json:"data"`
+}
+
 // GetMultiClusterResourceQuota 根据集群ID和配额名称获取多集群资源配额信息
 // 参数:
 //   - clusterID: 集群ID
@@ -93,3 +107,61 @@ func GetMultiClusterResourceQuota(clusterID, name string) (*MultiClusterResource
 
 	return nil, fmt.Errorf("GetMultiClusterResourceQuota failed")
 }
+
+// SyncProjectData 同步项目数据
+func SyncProjectData(p *pm.Project) error {
+	var (
+		// 构造请求URL
+		reqURL = fmt.Sprintf("%s/bcsapi/v4/storage/projects/%s", config.GlobalConf.Storage.Host, p.ProjectID)
+
+		// 响应数据结构
+		respData = &StorageResponse{}
+	)
+
+	// 发起HTTP PUT请求同步项目数据到storage
+	_, _, errs := gorequest.New().
+		Timeout(defaultTimeOut).
+		Put(reqURL).
+		Set("Content-Type", "application/json").
+		Set("Accept", "application/json").
+		Set("Authorization", fmt.Sprintf(`Bearer %s`, config.GlobalConf.Storage.Token)).
+		Send(SyncProjectDataRequest{Data: map[string]interface{}{
+			"createTime":  p.CreateTime,
+			"creator":     p.Creator,
+			"Managers":    p.Managers,
+			"projectID":   p.ProjectID,
+			"name":        p.Name,
+			"projectCode": p.ProjectCode,
+			"useBKRes":    p.UseBKRes,
+			"description": p.Description,
+			"isOffline":   p.IsOffline,
+			"kind":        p.Kind,
+			"businessID":  p.BusinessID,
+			"isSecret":    p.IsSecret,
+			"projectType": p.ProjectType,
+			"deployType":  p.DeployType,
+			"bgID":        p.BGID,
+			"bgName":      p.BGName,
+			"deptID":      p.DeptID,
+			"deptName":    p.DeptName,
+			"centerID":    p.CenterID,
+			"centerName":  p.CenterName,
+			"labels":      p.Labels,
+			"annotations": p.Annotations,
+		}}).
+		EndStruct(&respData)
+	if len(errs) > 0 {
+		logging.Error("SyncProjectData err: %v", errs[0])
+		return errs[0]
+	}
+
+	// 检查响应结果
+	if !respData.Result {
+		logging.Error("SyncProjectData failed: %s", respData.Message)
+		return fmt.Errorf(respData.Message)
+	}
+
+	logging.Info("SyncProjectData %s successfully", reqURL)
+
+	return nil
+}