feat:support data visibility

Author: duandaa

server/controller/db/mysql/migration/rawsql/init.sql

@@ -2826,3 +2826,13 @@ CREATE TABLE IF NOT EXISTS ch_user (
     updated_at              TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
 )ENGINE=innodb DEFAULT CHARSET=utf8;
 TRUNCATE TABLE ch_user;
+
+CREATE TABLE IF NOT EXISTS ch_statistic_tag (
+    `db`                      VARCHAR(128) NOT NULL,
+    `table`                   VARCHAR(256) NOT NULL,
+    `type`                    VARCHAR(128) NOT NULL,
+    `name`                    VARCHAR(256) NOT NULL,
+    `updated_at`              TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+    PRIMARY KEY (`db`, `table`, `type`, `name`)
+)ENGINE=innodb DEFAULT CHARSET=utf8;
+TRUNCATE TABLE ch_statistic_tag;

server/controller/db/mysql/migration/rawsql/issu/6.6.1.13.sql

@@ -0,0 +1,13 @@
+CREATE TABLE IF NOT EXISTS ch_statistic_tag (
+    `db`                      VARCHAR(128) NOT NULL,
+    `table`                   VARCHAR(256) NOT NULL,
+    `type`                    VARCHAR(128) NOT NULL,
+    `name`                    VARCHAR(256) NOT NULL,
+    `updated_at`              TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+    PRIMARY KEY (`db`, `table`, `type`, `name`)
+)ENGINE=innodb DEFAULT CHARSET=utf8;
+TRUNCATE TABLE ch_statistic_tag;
+
+-- update db_version to latest, remeber update DB_VERSION_EXPECT in migrate/version.go
+UPDATE db_version SET version='6.6.1.13';
+-- modify end

server/controller/db/mysql/migration/version.go

@@ -18,5 +18,5 @@ package migration
 
 const (
 	DB_VERSION_TABLE    = "db_version"
-	DB_VERSION_EXPECTED = "6.6.1.12"
+	DB_VERSION_EXPECTED = "6.6.1.13"
 )

server/controller/db/mysql/model/ch_model.go

@@ -680,3 +680,11 @@ type ChUser struct {
 	Name      string    `gorm:"column:name;type:varchar(256)" json:"NAME"`
 	UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime:now,type:timestamp" json:"UPDATED_AT"`
 }
+
+type ChStatisticTag struct {
+	Name      string    `gorm:"primaryKey;column:name;type:varchar(256)" json:"NAME"`
+	Db        string    `gorm:"primaryKey;column:db;type:varchar(128)" json:"DB"`
+	Table     string    `gorm:"primaryKey;column:table;type:varchar(256)" json:"TABLE"`
+	Type      string    `gorm:"primaryKey;column:type;type:varchar(128)" json:"TYPE"`
+	UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime:now,type:timestamp" json:"UPDATED_AT"`
+}

server/controller/tagrecorder/ch_statistic_tag.go

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2024 Yunshan Networks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tagrecorder
+
+import (
+	"github.com/deepflowio/deepflow/server/controller/db/mysql"
+	mysqlmodel "github.com/deepflowio/deepflow/server/controller/db/mysql/model"
+)
+
+type ChStatisticTag struct {
+	UpdaterComponent[mysqlmodel.ChStatisticTag, StatisticTagKey]
+}
+
+func NewChStatisticTag() *ChStatisticTag {
+	updater := &ChStatisticTag{
+		newUpdaterComponent[mysqlmodel.ChStatisticTag, StatisticTagKey](
+			RESOURCE_TYPE_STATISTIC_TAG,
+		),
+	}
+	updater.updaterDG = updater
+	return updater
+}
+
+func (r *ChStatisticTag) generateNewData(db *mysql.DB) (map[StatisticTagKey]mysqlmodel.ChStatisticTag, bool) {
+
+	return nil, false
+}
+
+func (r *ChStatisticTag) generateKey(dbItem mysqlmodel.ChStatisticTag) StatisticTagKey {
+	return StatisticTagKey{Db: dbItem.Db, Table: dbItem.Table, Type: dbItem.Type, Name: dbItem.Name}
+}
+
+func (r *ChStatisticTag) generateUpdateInfo(oldItem, newItem mysqlmodel.ChStatisticTag) (map[string]interface{}, bool) {
+
+	return nil, false
+}

server/controller/tagrecorder/const.go

@@ -119,6 +119,8 @@ const (
 	RESOURCE_TYPE_CH_LABEL_NAME                         = "ch_prometheus_label_name"
 	RESOURCE_TYPE_CH_METRIC_NAME                        = "ch_prometheus_metric_name"
 	RESOURCE_TYPE_CH_PROMETHEUS_TARGET_LABEL_LAYOUT     = "ch_prometheus_target_label_layout"
+
+	RESOURCE_TYPE_STATISTIC_TAG = "ch_statistic_tag"
 )
 
 const (
@@ -192,6 +194,8 @@ const (
 
 	CH_APP_LABEL_LIVE_VIEW    = "app_label_live_view"
 	CH_TARGET_LABEL_LIVE_VIEW = "target_label_live_view"
+
+	CH_STATISTIC_TAG = "statistic_tag"
 )
 
 const (
@@ -880,6 +884,17 @@ const (
 		SQL_SOURCE_MYSQL +
 		SQL_LIFETIME +
 		SQL_LAYOUT_FLAT
+	CREATE_STATISTIC_TAG_SQL = SQL_CREATE_DICT +
+		"(\n" +
+		"    `db` String,\n" +
+		"    `table` String,\n" +
+		"    `type` String,\n" +
+		"    `name` String\n" +
+		")\n" +
+		"PRIMARY KEY db, table, type, name\n" +
+		SQL_SOURCE_MYSQL +
+		SQL_LIFETIME +
+		SQL_LAYOUT_FLAT
 )
 
 const (
@@ -989,6 +1004,8 @@ var CREATE_SQL_MAP = map[string]string{
 
 	CH_APP_LABEL_LIVE_VIEW:    CREATE_APP_LABEL_LIVE_VIEW_SQL,
 	CH_TARGET_LABEL_LIVE_VIEW: CREATE_TARGET_LABEL_LIVE_VIEW_SQL,
+
+	CH_STATISTIC_TAG: CREATE_STATISTIC_TAG_SQL,
 }
 
 var VTAP_TYPE_TO_DEVICE_TYPE = map[int]int{

server/controller/tagrecorder/constraint.go

@@ -31,12 +31,12 @@ type MySQLChModel interface {
 		mysqlmodel.ChDevice | mysqlmodel.ChIPRelation | mysqlmodel.ChPodGroup | mysqlmodel.ChNetwork | mysqlmodel.ChPod | mysqlmodel.ChPodCluster |
 		mysqlmodel.ChPodNode | mysqlmodel.ChPodNamespace | mysqlmodel.ChTapType | mysqlmodel.ChVTap | mysqlmodel.ChPodK8sLabels | mysqlmodel.ChNodeType | mysqlmodel.ChGProcess | mysqlmodel.ChPodK8sAnnotation | mysqlmodel.ChPodK8sAnnotations |
 		mysqlmodel.ChPodServiceK8sAnnotation | mysqlmodel.ChPodServiceK8sAnnotations |
-		mysqlmodel.ChPodK8sEnv | mysqlmodel.ChPodK8sEnvs | mysqlmodel.ChPodService | mysqlmodel.ChChost | mysqlmodel.ChPolicy | mysqlmodel.ChNpbTunnel
+		mysqlmodel.ChPodK8sEnv | mysqlmodel.ChPodK8sEnvs | mysqlmodel.ChPodService | mysqlmodel.ChChost | mysqlmodel.ChPolicy | mysqlmodel.ChNpbTunnel | mysqlmodel.ChStatisticTag
 }
 
 // ch资源的组合key
 type ChModelKey interface {
 	PrometheusTargetLabelKey | PrometheusAPPLabelKey | OSAPPTagKey | OSAPPTagsKey | CloudTagsKey | CloudTagKey | IntEnumTagKey | StringEnumTagKey | VtapPortKey | IPResourceKey | K8sLabelKey | PortIDKey | PortIPKey | PortDeviceKey | IDKey | DeviceKey |
 		IPRelationKey | TapTypeKey | K8sLabelsKey | NodeTypeKey | K8sAnnotationKey | K8sAnnotationsKey |
-		K8sEnvKey | K8sEnvsKey | PolicyKey
+		K8sEnvKey | K8sEnvsKey | PolicyKey | StatisticTagKey
 }

server/controller/tagrecorder/key.go

@@ -138,3 +138,9 @@ type PolicyKey struct {
 	ACLGID     int
 	TunnelType int
 }
+type StatisticTagKey struct {
+	Db    string
+	Table string
+	Type  string
+	Name  string
+}