Use batches to call CreateTimeSeries (#617)

* Use batches for call to CreateTimeSeries

* Refactor exportTimeSeries

* Add tests

Author: knightpp

exporter/metric/metric.go

@@ -51,6 +51,10 @@ import (
 )
 
 const (
+	// The number of timeserieses to send to GCM in a single request. This
+	// is a hard limit in the GCM API, so we never want to exceed 200.
+	sendBatchSize = 200
+
 	cloudMonitoringMetricDescriptorNameFormat = "workload.googleapis.com/%s"
 )
 
@@ -221,31 +225,34 @@ func (me *metricExporter) createMetricDescriptorIfNeeded(ctx context.Context, md
 // exportTimeSeries create TimeSeries from the records in cps.
 // res should be the common resource among all TimeSeries, such as instance id, application name and so on.
 func (me *metricExporter) exportTimeSeries(ctx context.Context, rm metricdata.ResourceMetrics) error {
-	tss := []*monitoringpb.TimeSeries{}
-	mr := me.resourceToMonitoredResourcepb(rm.Resource)
-	var aggError error
+	tss, aggErr := me.recordsToTspbs(rm)
+	if len(tss) == 0 {
+		return aggErr
+	}
 
-	extraLabels := me.extraLabelsFromResource(rm.Resource)
-	for _, scope := range rm.ScopeMetrics {
-		for _, metrics := range scope.Metrics {
-			ts, err := me.recordToTspb(metrics, mr, scope.Scope, extraLabels)
-			aggError = multierr.Append(aggError, err)
-			tss = append(tss, ts...)
+	name := fmt.Sprintf("projects/%s", me.o.projectID)
+
+	var createErrors []error
+	for i := 0; i < len(tss); i += sendBatchSize {
+		j := i + sendBatchSize
+		if j >= len(tss) {
+			j = len(tss)
 		}
-	}
 
-	if len(tss) == 0 {
-		return aggError
-	}
+		// TODO: When this exporter is rewritten, support writing to multiple
+		// projects based on the "gcp.project.id" resource.
+		req := &monitoringpb.CreateTimeSeriesRequest{
+			Name:       name,
+			TimeSeries: tss[i:j],
+		}
 
-	// TODO: When this exporter is rewritten, support writing to multiple
-	// projects based on the "gcp.project.id" resource.
-	req := &monitoringpb.CreateTimeSeriesRequest{
-		Name:       fmt.Sprintf("projects/%s", me.o.projectID),
-		TimeSeries: tss,
+		err := me.client.CreateTimeSeries(ctx, req)
+		if err != nil {
+			createErrors = append(createErrors, err)
+		}
 	}
 
-	return multierr.Append(aggError, me.client.CreateTimeSeries(ctx, req))
+	return multierr.Append(aggErr, multierr.Combine(createErrors...))
 }
 
 func (me *metricExporter) extraLabelsFromResource(res *resource.Resource) *attribute.Set {
@@ -495,6 +502,28 @@ func (me *metricExporter) recordToTspb(m metricdata.Metrics, mr *monitoredrespb.
 	return tss, aggErr
 }
 
+func (me *metricExporter) recordsToTspbs(rm metricdata.ResourceMetrics) ([]*monitoringpb.TimeSeries, error) {
+	mr := me.resourceToMonitoredResourcepb(rm.Resource)
+	extraLabels := me.extraLabelsFromResource(rm.Resource)
+
+	var (
+		tss    []*monitoringpb.TimeSeries
+		errors []error
+	)
+	for _, scope := range rm.ScopeMetrics {
+		for _, metrics := range scope.Metrics {
+			ts, err := me.recordToTspb(metrics, mr, scope.Scope, extraLabels)
+			if err != nil {
+				errors = append(errors, err)
+			}
+
+			tss = append(tss, ts...)
+		}
+	}
+
+	return tss, multierr.Combine(errors...)
+}
+
 func sanitizeUTF8(s string) string {
 	return strings.ToValidUTF8(s, "�")
 }

exporter/metric/metric_test.go

@@ -1073,3 +1073,99 @@ func TestMetricTypeToDisplayName(t *testing.T) {
 		})
 	}
 }
+
+func TestBatchingExport(t *testing.T) {
+	setup := func(t *testing.T) (metric.Exporter, *cloudmock.MetricsTestServer) {
+		testServer, err := cloudmock.NewMetricTestServer()
+		//nolint:errcheck
+		go testServer.Serve()
+		t.Cleanup(testServer.Shutdown)
+
+		assert.NoError(t, err)
+
+		clientOpts := []option.ClientOption{
+			option.WithEndpoint(testServer.Endpoint),
+			option.WithoutAuthentication(),
+			option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())),
+		}
+
+		exporter, err := New(
+			WithProjectID("PROJECT_ID_NOT_REAL"),
+			WithMonitoringClientOptions(clientOpts...),
+			WithMetricDescriptorTypeFormatter(formatter),
+		)
+		assert.NoError(t, err)
+
+		t.Cleanup(func() {
+			ctx := context.Background()
+			err := exporter.Shutdown(ctx)
+			assert.NoError(t, err)
+		})
+
+		return exporter, testServer
+	}
+
+	createMetrics := func(n int) []metricdata.Metrics {
+		inputMetrics := make([]metricdata.Metrics, n)
+		for i := 0; i < n; i++ {
+			inputMetrics[i] = metricdata.Metrics{Name: "testing", Data: metricdata.Histogram{
+				DataPoints: []metricdata.HistogramDataPoint{
+					{},
+				},
+			}}
+		}
+
+		return inputMetrics
+	}
+
+	for _, tc := range []struct {
+		desc                  string
+		numMetrics            int
+		expectedCreateTSCalls int
+	}{
+		{desc: "0 metrics"},
+		{
+			desc:                  "150 metrics",
+			numMetrics:            150,
+			expectedCreateTSCalls: 1,
+		},
+		{
+			desc:                  "200 metrics",
+			numMetrics:            200,
+			expectedCreateTSCalls: 1,
+		},
+		{
+			desc:                  "201 metrics",
+			numMetrics:            201,
+			expectedCreateTSCalls: 2,
+		},
+		{
+			desc:                  "500 metrics",
+			numMetrics:            500,
+			expectedCreateTSCalls: 3,
+		},
+		{
+			desc:                  "1199 metrics",
+			numMetrics:            1199,
+			expectedCreateTSCalls: 6,
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			exporter, testServer := setup(t)
+			input := createMetrics(tc.numMetrics)
+			ctx := context.Background()
+
+			err := exporter.Export(ctx, metricdata.ResourceMetrics{
+				ScopeMetrics: []metricdata.ScopeMetrics{
+					{
+						Metrics: input,
+					},
+				},
+			})
+			assert.NoError(t, err)
+
+			gotCalls := testServer.CreateTimeSeriesRequests()
+			assert.Equal(t, tc.expectedCreateTSCalls, len(gotCalls))
+		})
+	}
+}