Skip to content

Upgrade aws-sdk-go dependency for cloudwatch from v1 to v2 #1304

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Upgrade aws-sdk-go dependency for cloudwatch from v1 to v2 #1304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ go 1.17
require (
github.com/VividCortex/gohistogram v1.0.0
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5
github.com/aws/aws-sdk-go v1.40.45
github.com/aws/aws-sdk-go-v2 v1.9.1
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1
github.com/casbin/casbin/v2 v2.37.0
Expand Down
3 changes: 0 additions & 3 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,6 @@ github.com/armon/go-metrics v0.4.0 h1:yCQqn7dwca4ITXb+CbubHmedzaQYHhNhrEXLYUeEe8
github.com/armon/go-metrics v0.4.0/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go v1.40.45 h1:QN1nsY27ssD/JmW4s83qmSb+uL6DG4GmCDzjmJB4xUI=
github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-sdk-go-v2 v1.9.1 h1:ZbovGV/qo40nrOJ4q8G33AGICzaPI45FHQWJ9650pF4=
github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1 h1:w/fPGB0t5rWwA43mux4e9ozFSH5zF1moQemlA131PWc=
Expand Down Expand Up @@ -571,7 +569,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
Expand Down
42 changes: 24 additions & 18 deletions metrics/cloudwatch/cloudwatch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,20 @@ import (
"sync"
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"

"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/generic"
"github.com/go-kit/kit/metrics/internal/lv"
"github.com/go-kit/log"
)

type CloudWatchAPI interface {
PutMetricData(context.Context, *cloudwatch.PutMetricDataInput) (*cloudwatch.PutMetricDataOutput, error)
}

const (
maxConcurrentRequests = 20
maxValuesInABatch = 150
Expand All @@ -32,7 +36,7 @@ type CloudWatch struct {
mtx sync.RWMutex
sem chan struct{}
namespace string
svc cloudwatchiface.CloudWatchAPI
svc CloudWatchAPI
counters *lv.Space
gauges *lv.Space
histograms *lv.Space
Expand Down Expand Up @@ -85,7 +89,7 @@ func WithConcurrentRequests(n int) Option {
// Namespace is applied to all created metrics and maps to the CloudWatch namespace.
// Callers must ensure that regular calls to Send are performed, either
// manually or with one of the helper methods.
func New(namespace string, svc cloudwatchiface.CloudWatchAPI, options ...Option) *CloudWatch {
func New(namespace string, svc CloudWatchAPI, options ...Option) *CloudWatch {
cw := &CloudWatch{
sem: nil, // set below
namespace: namespace,
Expand Down Expand Up @@ -157,11 +161,11 @@ func (cw *CloudWatch) Send() error {
defer cw.mtx.RUnlock()
now := time.Now()

var datums []*cloudwatch.MetricDatum
var datums []types.MetricDatum

cw.counters.Reset().Walk(func(name string, lvs lv.LabelValues, values []float64) bool {
value := sum(values)
datums = append(datums, &cloudwatch.MetricDatum{
datums = append(datums, types.MetricDatum{
MetricName: aws.String(name),
Dimensions: makeDimensions(lvs...),
Value: aws.Float64(value),
Expand All @@ -175,7 +179,7 @@ func (cw *CloudWatch) Send() error {
return true
}

datum := &cloudwatch.MetricDatum{
datum := types.MetricDatum{
MetricName: aws.String(name),
Dimensions: makeDimensions(lvs...),
Timestamp: aws.Time(now),
Expand All @@ -192,8 +196,8 @@ func (cw *CloudWatch) Send() error {
if len(datum.Values) == maxValuesInABatch {
break
}
datum.Values = append(datum.Values, aws.Float64(value))
datum.Counts = append(datum.Counts, aws.Float64(float64(count)))
datum.Values = append(datum.Values, value)
datum.Counts = append(datum.Counts, float64(count))
}

datums = append(datums, datum)
Expand All @@ -217,7 +221,7 @@ func (cw *CloudWatch) Send() error {

for _, perc := range cw.percentiles {
value := histogram.Quantile(perc)
datums = append(datums, &cloudwatch.MetricDatum{
datums = append(datums, types.MetricDatum{
MetricName: aws.String(fmt.Sprintf("%s_%s", name, formatPerc(perc))),
Dimensions: makeDimensions(lvs...),
Value: aws.Float64(value),
Expand All @@ -227,22 +231,24 @@ func (cw *CloudWatch) Send() error {
return true
})

var batches [][]*cloudwatch.MetricDatum
var batches [][]types.MetricDatum
for len(datums) > 0 {
var batch []*cloudwatch.MetricDatum
var batch []types.MetricDatum
lim := min(len(datums), maxConcurrentRequests)
batch, datums = datums[:lim], datums[lim:]
batches = append(batches, batch)
}

var errors = make(chan error, len(batches))
for _, batch := range batches {
go func(batch []*cloudwatch.MetricDatum) {
go func(batch []types.MetricDatum) {
cw.sem <- struct{}{}
defer func() {
<-cw.sem
}()
_, err := cw.svc.PutMetricData(&cloudwatch.PutMetricDataInput{

// using TODO context, should be passed in ideally
_, err := cw.svc.PutMetricData(context.TODO(), &cloudwatch.PutMetricDataInput{
Namespace: aws.String(cw.namespace),
MetricData: batch,
})
Expand Down Expand Up @@ -349,10 +355,10 @@ func (h *Histogram) Observe(value float64) {
h.obs(h.name, h.lvs, value)
}

func makeDimensions(labelValues ...string) []*cloudwatch.Dimension {
dimensions := make([]*cloudwatch.Dimension, len(labelValues)/2)
func makeDimensions(labelValues ...string) []types.Dimension {
dimensions := make([]types.Dimension, len(labelValues)/2)
for i, j := 0, 0; i < len(labelValues); i, j = i+2, j+1 {
dimensions[j] = &cloudwatch.Dimension{
dimensions[j] = types.Dimension{
Name: aws.String(labelValues[i]),
Value: aws.String(labelValues[i+1]),
}
Expand Down
15 changes: 8 additions & 7 deletions metrics/cloudwatch/cloudwatch_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
package cloudwatch

import (
"context"
"errors"
"fmt"
"strconv"
"sync"
"testing"

"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"

"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/teststat"
Expand All @@ -20,20 +21,20 @@ const metricNameToGenerateError = "metric_name_used_to_throw_an_error"
var errTest = errors.New("test error")

type mockCloudWatch struct {
cloudwatchiface.CloudWatchAPI
CloudWatchAPI
mtx sync.RWMutex
valuesReceived map[string][]float64
dimensionsReceived map[string][]*cloudwatch.Dimension
dimensionsReceived map[string][]types.Dimension
}

func newMockCloudWatch() *mockCloudWatch {
return &mockCloudWatch{
valuesReceived: map[string][]float64{},
dimensionsReceived: map[string][]*cloudwatch.Dimension{},
dimensionsReceived: map[string][]types.Dimension{},
}
}

func (mcw *mockCloudWatch) PutMetricData(input *cloudwatch.PutMetricDataInput) (*cloudwatch.PutMetricDataOutput, error) {
func (mcw *mockCloudWatch) PutMetricData(_ context.Context, input *cloudwatch.PutMetricDataInput) (*cloudwatch.PutMetricDataOutput, error) {
mcw.mtx.Lock()
defer mcw.mtx.Unlock()
for _, datum := range input.MetricData {
Expand All @@ -43,7 +44,7 @@ func (mcw *mockCloudWatch) PutMetricData(input *cloudwatch.PutMetricDataInput) (

if len(datum.Values) > 0 {
for _, v := range datum.Values {
mcw.valuesReceived[*datum.MetricName] = append(mcw.valuesReceived[*datum.MetricName], *v)
mcw.valuesReceived[*datum.MetricName] = append(mcw.valuesReceived[*datum.MetricName], v)
}
} else {
mcw.valuesReceived[*datum.MetricName] = append(mcw.valuesReceived[*datum.MetricName], *datum.Value)
Expand Down
Loading