sink: make sink statistics consistent and flush-aware

downstreamadapter/sink/blackhole/sink.go

@@ -19,7 +19,7 @@ import (
 	"github.com/pingcap/log"
 	"github.com/pingcap/ticdc/pkg/common"
 	commonEvent "github.com/pingcap/ticdc/pkg/common/event"
-	"github.com/pingcap/ticdc/pkg/metrics"
+	"github.com/pingcap/ticdc/pkg/statistics"
 	"github.com/pingcap/ticdc/utils/chann"
 	"go.uber.org/zap"
 )
@@ -28,13 +28,13 @@ import (
 // Including DDL and DML.
 type Sink struct {
 	eventCh    *chann.UnlimitedChannel[*commonEvent.DMLEvent, any]
-	statistics *metrics.Statistics
+	statistics *statistics.Statistics
 }
 
 func New(changefeedID common.ChangeFeedID, keyspaceID uint32) (*Sink, error) {
 	return &Sink{
 		eventCh:    chann.NewUnlimitedChannelDefault[*commonEvent.DMLEvent](),
-		statistics: metrics.NewStatistics(changefeedID, keyspaceID, "sink"),
+		statistics: statistics.New(changefeedID, keyspaceID),
 	}, nil
 }
 
@@ -54,6 +54,7 @@ func (s *Sink) AddDMLEvent(event *commonEvent.DMLEvent) {
 	// ref: https://github.com/pingcap/ticdc/blob/da834db76e0662ff15ef12645d1f37bfa6506d83/tests/integration_tests/lossy_ddl/run.sh#L23
 	// Use zap.Stringer to call String() method which applies log redaction
 	log.Debug("BlackHoleSink: WriteEvents", zap.Stringer("dml", event))
+	s.statistics.TrackDMLEvent(event)
 	s.eventCh.Push(event)
 }
 
@@ -104,8 +105,8 @@ func (s *Sink) Run(ctx context.Context) error {
 				log.Info("blackhole sink event channel closed")
 				return nil
 			}
-			err := s.statistics.RecordBatchExecution(func() (int, int64, error) {
-				return int(event.Len()), event.GetSize(), nil
+			err := s.statistics.RecordBatchExecution(int(event.Len()), func() error {
+				return nil
 			})
 			if err != nil {
 				return err

downstreamadapter/sink/cloudstorage/dml_writers.go

@@ -23,8 +23,8 @@ import (
 	"github.com/pingcap/ticdc/pkg/cloudstorage"
 	commonType "github.com/pingcap/ticdc/pkg/common"
 	commonEvent "github.com/pingcap/ticdc/pkg/common/event"
-	"github.com/pingcap/ticdc/pkg/metrics"
 	"github.com/pingcap/ticdc/pkg/sink/codec/common"
+	"github.com/pingcap/ticdc/pkg/statistics"
 	"github.com/pingcap/ticdc/utils/chann"
 	"github.com/pingcap/tidb/pkg/objstore/storeapi"
 	"go.uber.org/atomic"
@@ -34,7 +34,7 @@ import (
 // dmlWriters coordinates encoding and output shard writers.
 type dmlWriters struct {
 	changefeedID commonType.ChangeFeedID
-	statistics   *metrics.Statistics
+	statistics   *statistics.Statistics
 
 	// msgCh is the only unbounded queue in the storage sink pipeline.
 	// External callers push tasks into it, addTasks consumes it, and
@@ -55,7 +55,7 @@ func newDMLWriters(
 	config *cloudstorage.Config,
 	encoderConfig *common.Config,
 	extension string,
-	statistics *metrics.Statistics,
+	statistics *statistics.Statistics,
 	columnSelector *columnselector.ColumnSelectors,
 ) (*dmlWriters, error) {
 	messageCh := chann.NewUnlimitedChannelDefault[*task]()

downstreamadapter/sink/cloudstorage/sink.go

@@ -29,6 +29,7 @@ import (
 	"github.com/pingcap/ticdc/pkg/config"
 	"github.com/pingcap/ticdc/pkg/errors"
 	"github.com/pingcap/ticdc/pkg/metrics"
+	"github.com/pingcap/ticdc/pkg/statistics"
 	"github.com/pingcap/ticdc/pkg/util"
 	"github.com/pingcap/tidb/pkg/meta/model"
 	"github.com/pingcap/tidb/pkg/objstore/storeapi"
@@ -65,7 +66,7 @@ type sink struct {
 	lastSendCheckpointTsTime time.Time
 
 	cron       *cron.Cron
-	statistics *metrics.Statistics
+	statistics *statistics.Statistics
 
 	isNormal    *atomic.Bool
 	cleanupJobs []func() /* only for test */
@@ -135,7 +136,7 @@ func New(
 	if err != nil {
 		return nil, err
 	}
-	statistics := metrics.NewStatistics(changefeedID, keyspaceID, "cloudstorage")
+	statistics := statistics.New(changefeedID, keyspaceID)
 	defer func() {
 		if err != nil {
 			statistics.Close()
@@ -205,6 +206,7 @@ func (s *sink) AddDMLEvent(event *commonEvent.DMLEvent) {
 			zap.String("dispatcher", event.GetDispatcherID().String()))
 		return
 	}
+	s.statistics.TrackDMLEvent(event)
 	s.dmlWriters.addDMLEvent(event)
 }
 

downstreamadapter/sink/cloudstorage/writer.go

@@ -25,7 +25,7 @@ import (
 	"github.com/pingcap/ticdc/pkg/cloudstorage"
 	"github.com/pingcap/ticdc/pkg/common"
 	"github.com/pingcap/ticdc/pkg/errors"
-	pmetrics "github.com/pingcap/ticdc/pkg/metrics"
+	"github.com/pingcap/ticdc/pkg/statistics"
 	"github.com/pingcap/tidb/pkg/objstore/storeapi"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
@@ -45,7 +45,7 @@ type writer struct {
 	// the channel does not need to be closed explicitly.
 	flushCh chan flushTask
 
-	statistics        *pmetrics.Statistics
+	statistics        *statistics.Statistics
 	filePathGenerator *cloudstorage.FilePathGenerator
 
 	metricFlushBytes    prometheus.Observer
@@ -74,7 +74,7 @@ func newWriter(
 	storage storeapi.Storage,
 	config *cloudstorage.Config,
 	extension string,
-	statistics *pmetrics.Statistics,
+	statistics *statistics.Statistics,
 	spoolBuffer *spool.Spool,
 ) *writer {
 	var (
@@ -223,20 +223,20 @@ func (d *writer) writeDataFile(ctx context.Context, dataFilePath, indexFilePath
 	changefeed := d.changeFeedID.Name()
 	start := time.Now()
 
-	err := d.statistics.RecordBatchExecution(func() (int, int64, error) {
+	err := d.statistics.RecordBatchExecution(payload.rowsCount, func() error {
 		if d.config.FlushConcurrency <= 1 {
 			err := d.storage.WriteFile(ctx, dataFilePath, payload.data)
 			if err != nil {
-				return 0, 0, err
+				return err
 			}
-			return payload.rowsCount, payload.nBytes, nil
+			return nil
 		}
 
 		writer, err := d.storage.Create(ctx, dataFilePath, &storeapi.WriterOption{
 			Concurrency: d.config.FlushConcurrency,
 		})
 		if err != nil {
-			return 0, 0, err
+			return err
 		}
 
 		_, err = writer.Write(ctx, payload.data)
@@ -247,16 +247,16 @@ func (d *writer) writeDataFile(ctx context.Context, dataFilePath, indexFilePath
 					zap.String("keyspace", keyspace), zap.String("changefeed", changefeed),
 					zap.String("path", dataFilePath), zap.Error(closeErr))
 			}
-			return 0, 0, err
+			return err
 		}
 
 		if err = writer.Close(ctx); err != nil {
 			log.Error("failed to close concurrency writer",
 				zap.String("keyspace", keyspace), zap.String("changefeed", changefeed),
 				zap.String("path", dataFilePath), zap.Error(err))
-			return 0, 0, err
+			return err
 		}
-		return payload.rowsCount, payload.nBytes, nil
+		return nil
 	})
 	if err != nil {
 		return err

downstreamadapter/sink/cloudstorage/writer_test.go

@@ -31,9 +31,9 @@ import (
 	commonType "github.com/pingcap/ticdc/pkg/common"
 	commonEvent "github.com/pingcap/ticdc/pkg/common/event"
 	"github.com/pingcap/ticdc/pkg/config"
-	"github.com/pingcap/ticdc/pkg/metrics"
 	"github.com/pingcap/ticdc/pkg/pdutil"
 	"github.com/pingcap/ticdc/pkg/sink/codec/common"
+	"github.com/pingcap/ticdc/pkg/statistics"
 	"github.com/pingcap/ticdc/pkg/util"
 	"github.com/pingcap/tidb/pkg/meta/model"
 	"github.com/pingcap/tidb/pkg/objstore/objectio"
@@ -59,7 +59,7 @@ func testWriter(ctx context.Context, t *testing.T, dir string) *writer {
 	require.NoError(t, err)
 
 	changefeedID := commonType.NewChangefeedID4Test("test", t.Name())
-	statistics := metrics.NewStatistics(changefeedID, commonType.DefaultKeyspaceID, t.Name())
+	statistics := statistics.New(changefeedID, commonType.DefaultKeyspaceID)
 	spoolBuffer := newTestSpool(t, changefeedID, cfg)
 	d := newWriter(1, changefeedID, storage,
 		cfg, ".json", statistics, spoolBuffer)
@@ -479,7 +479,7 @@ func TestWriterStoresPendingMessagesInSpoolBeforeFlush(t *testing.T) {
 	cfg.FlushInterval = time.Hour
 
 	changefeedID := commonType.NewChangefeedID4Test("test", "spool-pending")
-	statistics := metrics.NewStatistics(changefeedID, commonType.DefaultKeyspaceID, t.Name())
+	statistics := statistics.New(changefeedID, commonType.DefaultKeyspaceID)
 	setPDClockForTest(t, pdutil.NewClock4Test())
 
 	spoolBuffer := newTestSpool(t, changefeedID, cfg)
@@ -648,7 +648,7 @@ func TestWriterIndexWriteError(t *testing.T) {
 	cfg.FlushInterval = time.Hour
 
 	changefeedID := commonType.NewChangefeedID4Test("test", "writer-error-metric")
-	statistics := metrics.NewStatistics(changefeedID, commonType.DefaultKeyspaceID, t.Name())
+	statistics := statistics.New(changefeedID, commonType.DefaultKeyspaceID)
 	setPDClockForTest(t, pdutil.NewClock4Test())
 	spoolBuffer := newTestSpool(t, changefeedID, cfg)
 	d := newWriter(1, changefeedID, storage, cfg, ".json", statistics, spoolBuffer)
@@ -714,7 +714,7 @@ func TestWriterDataFileCloseError(t *testing.T) {
 	cfg.FlushInterval = time.Hour
 
 	changefeedID := commonType.NewChangefeedID4Test("test", "writer-close-error")
-	statistics := metrics.NewStatistics(changefeedID, commonType.DefaultKeyspaceID, t.Name())
+	statistics := statistics.New(changefeedID, commonType.DefaultKeyspaceID)
 	setPDClockForTest(t, pdutil.NewClock4Test())
 	spoolBuffer := newTestSpool(t, changefeedID, cfg)
 	d := newWriter(1, changefeedID, storage, cfg, ".json", statistics, spoolBuffer)

downstreamadapter/sink/kafka/sink.go

@@ -31,6 +31,7 @@ import (
 	codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common"
 	"github.com/pingcap/ticdc/pkg/sink/kafka"
 	"github.com/pingcap/ticdc/pkg/sink/kafka/claimcheck"
+	"github.com/pingcap/ticdc/pkg/statistics"
 	"github.com/pingcap/ticdc/pkg/util"
 	"github.com/pingcap/ticdc/utils/chann"
 	"go.uber.org/atomic"
@@ -51,7 +52,7 @@ type sink struct {
 	metricsCollector kafka.MetricsCollector
 
 	comp       components
-	statistics *metrics.Statistics
+	statistics *statistics.Statistics
 
 	protocol      config.Protocol
 	partitionRule helper.DDLDispatchRule
@@ -170,7 +171,7 @@ func newWithComponents(
 	protocol config.Protocol,
 	comp components,
 ) (*sink, error) {
-	statistics := metrics.NewStatistics(changefeedID, keyspaceID, "sink")
+	statistics := statistics.New(changefeedID, keyspaceID)
 	var (
 		err           error
 		asyncProducer kafka.AsyncProducer
@@ -244,6 +245,7 @@ func (s *sink) IsNormal() bool {
 }
 
 func (s *sink) AddDMLEvent(event *commonEvent.DMLEvent) {
+	s.statistics.TrackDMLEvent(event)
 	s.eventChan.Push(event)
 }
 
@@ -429,18 +431,26 @@ func (s *sink) sendMessages(ctx context.Context) error {
 			}
 			for _, message := range future.Messages {
 				start := time.Now()
-				if err = s.statistics.RecordBatchExecution(func() (int, int64, error) {
-					message.SetPartitionKey(future.Key.PartitionKey)
-					if err = s.dmlProducer.AsyncSend(
-						ctx,
-						future.Key.Topic,
-						future.Key.Partition,
-						message); err != nil {
-						return 0, 0, err
+				rows := message.GetRowsCount()
+				callback := message.Callback
+				message.Callback = func() {
+					_ = s.statistics.RecordBatchExecution(rows, func() error {
+						return nil
+					})
+					if callback != nil {
+						callback()
 					}
-					return message.GetRowsCount(), int64(message.Length()), nil
-				}); err != nil {
-					return err
+				}
+
+				message.SetPartitionKey(future.Key.PartitionKey)
+				if err = s.dmlProducer.AsyncSend(
+					ctx,
+					future.Key.Topic,
+					future.Key.Partition,
+					message); err != nil {
+					return s.statistics.RecordBatchExecution(rows, func() error {
+						return err
+					})
 				}
 				metricSendMessageDuration.Observe(time.Since(start).Seconds())
 			}

downstreamadapter/sink/mysql/sink.go

@@ -28,6 +28,7 @@ import (
 	"github.com/pingcap/ticdc/pkg/errors"
 	"github.com/pingcap/ticdc/pkg/metrics"
 	"github.com/pingcap/ticdc/pkg/sink/mysql"
+	"github.com/pingcap/ticdc/pkg/statistics"
 	"github.com/pingcap/tidb/pkg/parser/ast"
 	"go.uber.org/atomic"
 	"go.uber.org/zap"
@@ -54,7 +55,7 @@ type Sink struct {
 	// Compatibility callers built through NewMySQLSink use one shared pool.
 	dmlDB      *sql.DB
 	controlDB  *sql.DB
-	statistics *metrics.Statistics
+	statistics *statistics.Statistics
 
 	conflictDetector *causality.ConflictDetector
 
@@ -158,7 +159,7 @@ func newMySQLSinkWithDBs(
 	progressInterval time.Duration,
 	keyspaceID uint32,
 ) *Sink {
-	stat := metrics.NewStatistics(changefeedID, keyspaceID, "TxnSink")
+	stat := statistics.New(changefeedID, keyspaceID)
 
 	var activeActiveSyncStatsCollector *mysql.ActiveActiveSyncStatsCollector
 	if enableActiveActive && cfg.IsTiDB && cfg.ActiveActiveSyncStatsInterval > 0 {
@@ -320,6 +321,7 @@ func (s *Sink) SetTableSchemaStore(tableSchemaStore *commonEvent.TableSchemaStor
 }
 
 func (s *Sink) AddDMLEvent(event *commonEvent.DMLEvent) {
+	s.statistics.TrackDMLEvent(event)
 	s.conflictDetector.Add(event)
 }