add region label to operational metrics (#835)

Author: leonorfmartins

cmd/exporter/config/config.go

@@ -26,6 +26,7 @@ type Config struct {
 		Azure struct {
 			Services       StringSliceFlag
 			SubscriptionId string
+			Region         string
 		}
 	}
 	Collector struct {

pkg/aws/aws.go

@@ -138,7 +138,13 @@ func newWithDependencies(ctx context.Context, config *Config, awsClient client.C
 
 		switch service {
 		case serviceS3:
-			collector := s3.New(config.ScrapeInterval, awsClient)
+			collector, err := s3.New(ctx, config.ScrapeInterval, awsClient)
+			if err != nil {
+				logger.LogAttrs(ctx, slog.LevelError, "Error creating collector",
+					slog.String("service", service),
+					slog.String("message", err.Error()))
+				continue
+			}
 			collectors = append(collectors, collector)
 		case serviceEC2:
 			collector, err := ec2Collector.New(ctx, &ec2Collector.Config{

pkg/aws/aws_test.go

@@ -78,6 +78,7 @@ func Test_NewWithDependencies(t *testing.T) {
 				{RegionName: stringPtr("us-east-1")},
 			},
 			setupMockClient: func(m *mock_client.MockClient) {
+				m.EXPECT().DescribeRegions(gomock.Any(), false).Return(nil, nil)
 			},
 			setupRegionClients: map[string]client.Client{},
 			expectedCollectors: 1,
@@ -92,6 +93,9 @@ func Test_NewWithDependencies(t *testing.T) {
 				{RegionName: stringPtr("us-east-1")},
 				{RegionName: stringPtr("us-west-2")},
 			},
+			setupMockClient: func(m *mock_client.MockClient) {
+				m.EXPECT().DescribeRegions(gomock.Any(), false).Return(nil, nil)
+			},
 			setupRegionClients: map[string]client.Client{
 				"us-east-1": &mockRegionClient{}, // EC2 collector needs region map
 				"us-west-2": &mockRegionClient{},
@@ -119,6 +123,9 @@ func Test_NewWithDependencies(t *testing.T) {
 			regions: []types.Region{
 				{RegionName: stringPtr("us-east-1")},
 			},
+			setupMockClient: func(m *mock_client.MockClient) {
+				m.EXPECT().DescribeRegions(gomock.Any(), false).Return(nil, nil).Times(2)
+			},
 			setupRegionClients: map[string]client.Client{},
 			expectedCollectors: 2, // Both should create collectors
 			validateAWS: func(t *testing.T, aws *AWS) {
@@ -489,7 +496,7 @@ func Test_CollectMetrics(t *testing.T) {
 				c.EXPECT().Describe(gomock.Any()).Return(nil).AnyTimes()
 			}
 			aws := &AWS{
-				Config:           nil,
+				Config:           &Config{Region: "us-east-1"},
 				collectors:       []provider.Collector{},
 				logger:           logger,
 				ctx:              t.Context(),

pkg/aws/client/billing.go

@@ -10,6 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/costexplorer"
 	"github.com/aws/aws-sdk-go-v2/service/costexplorer/types"
 	cost "github.com/grafana/cloudcost-exporter/pkg/aws/services/costexplorer"
+	"github.com/grafana/cloudcost-exporter/pkg/utils"
 )
 
 type billing struct {
@@ -94,24 +95,30 @@ func parseBillingData(outputs []*costexplorer.GetCostAndUsageOutput) *BillingDat
 	return billingData
 }
 
-// getRegionFromKey returns the region from the key. If the key is requests, it will return an empty string because there is no region associated with it.
+// getRegionFromKey returns the region from the key. Keys without a recognisable
+// region prefix are returned as "unknown".
 func getRegionFromKey(key string) string {
+	region := utils.RegionUnknown
 	if key == "Requests-Tier1" || key == "Requests-Tier2" {
-		return ""
+		return region
 	}
 
 	split := strings.Split(key, "-")
 	if len(split) < 2 {
-		slog.Warn("Could not find region in key", "key", key)
-		return ""
+		return region
 	}
 
 	billingRegion := split[0]
 	if region, ok := BillingToRegionMap[billingRegion]; ok {
 		return region
 	}
-	slog.Warn("Could not find mapped region", "key", key, "billingRegion", billingRegion)
-	return ""
+
+	// Per AWS S3 documentation, usage types for us-east-1 may appear without a
+	// region prefix (e.g. "TimedStorage-ByteHrs" instead of "USE1-TimedStorage-ByteHrs").
+	if billingRegion == "TimedStorage" {
+		return BillingToRegionMap["USE1"]
+	}
+	return region
 }
 
 // getComponentFromKey returns the component from the key. If the component does not contain a region, it will return
@@ -125,6 +132,10 @@ func getComponentFromKey(key string) string {
 	if len(split) < 2 {
 		return val
 	}
+	// Handle known S3 components that appear without a region prefix (no-prefix = us-east-1).
+	if split[0] == "TimedStorage" {
+		return "TimedStorage"
+	}
 	val = split[1]
 	// Check to see if the value is a region. If so, set val to empty string to skip the dimension
 	// Currently this is such a minor part of our bill that it's not worth it.

pkg/aws/client/billing_test.go

@@ -55,12 +55,59 @@ func Test_getRegionFromKey(t *testing.T) {
 		key, want := record[0], record[1]
 		got := getRegionFromKey(key)
 		mappedWant := BillingToRegionMap[want]
+		if want == "unknown" {
+			mappedWant = "unknown"
+		}
 		if mappedWant != got {
 			t.Fatalf("getRegionFromKey(%s) = %v, want %v", key, got, want)
 		}
 	}
 }
 
+func Test_getRegionFromKey_unknown(t *testing.T) {
+	tests := map[string]struct {
+		key  string
+		want string
+	}{
+		"bare Requests-Tier1 has no region": {
+			key:  "Requests-Tier1",
+			want: "unknown",
+		},
+		"bare Requests-Tier2 has no region": {
+			key:  "Requests-Tier2",
+			want: "unknown",
+		},
+		"Global prefix (free-tier credits) has no region": {
+			key:  "Global-Bucket-Hrs-FreeTier",
+			want: "unknown",
+		},
+		"DataTransfer prefix without region code has no region": {
+			key:  "DataTransfer-In-Bytes",
+			want: "unknown",
+		},
+		"DataTransfer-Out prefix without region code has no region": {
+			key:  "DataTransfer-Out-Bytes",
+			want: "unknown",
+		},
+		"completely unrecognised prefix has no region": {
+			key:  "SomeNewPrefix-Bytes",
+			want: "unknown",
+		},
+		"key too short to contain a region": {
+			key:  "NoHyphen",
+			want: "unknown",
+		},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			if got := getRegionFromKey(tt.key); got != tt.want {
+				t.Errorf("getRegionFromKey(%q) = %q, want %q", tt.key, got, tt.want)
+			}
+		})
+	}
+}
+
 func TestS3BillingData_AddRegion(t *testing.T) {
 	type args struct {
 		key   string

pkg/aws/client/compute.go

@@ -176,3 +176,12 @@ func ClusterNameFromInstance(instance types.Instance) string {
 	}
 	return ""
 }
+
+// RegionsFromEC2Types converts a slice of EC2 Region types to a slice of region name strings.
+func RegionsFromEC2Types(regions []types.Region) []string {
+	result := make([]string, 0, len(regions))
+	for _, r := range regions {
+		result = append(result, *r.RegionName)
+	}
+	return result
+}

pkg/aws/client/testdata/dimensions.csv

@@ -10,8 +10,8 @@ EUC1-DataTransfer-Out-Bytes,EUC1,DataTransfer,Out,Bytes,
 EUC1-Requests-Tier1,EUC1,Requests-Tier1,Tier1,,
 EUC1-Requests-Tier2,EUC1,Requests-Tier2,Tier2,,
 EUC1-TimedStorage-ByteHrs,EUC1,TimedStorage,ByteHrs,,
-Requests-Tier1,,,,,
-Requests-Tier2,,,,,
+Requests-Tier1,unknown,,,,
+Requests-Tier2,unknown,,,,
 USE1-USE2-AWS-Out-Bytes,USE1,,AWS,Out,Bytes
 USE2-DataTransfer-Out-Bytes,USE2,DataTransfer,Out,Bytes,
 USE2-Requests-Tier1,USE2,Requests-Tier1,Tier1,,
@@ -22,8 +22,8 @@ USW2-Requests-Tier1,USW2,Requests-Tier1,Tier1,,
 USW2-Requests-Tier2,USW2,Requests-Tier2,Tier2,,
 USW2-TimedStorage-ByteHrs,USW2,TimedStorage,ByteHrs,,
 APS1-DataTransfer-In-Bytes,APS1,DataTransfer,In,Bytes,
-DataTransfer-In-Bytes,DataTransfer,In,Bytes,,
-DataTransfer-Out-Bytes,DataTransfer,Out,Bytes,,
+DataTransfer-In-Bytes,unknown,In,Bytes,,
+DataTransfer-Out-Bytes,unknown,Out,Bytes,,
 USE2-DataTransfer-In-Bytes,USE2,DataTransfer,In,Bytes,
 USW2-DataTransfer-In-Bytes,USW2,DataTransfer,In,Bytes,
 APS1-USE1-AWS-Out-Bytes,APS1,,AWS,Out,Bytes
@@ -50,4 +50,6 @@ EUN1-USE1-AWS-Out-Bytes,EUN1,,AWS,Out,Bytes
 SAE1-USE1-AWS-Out-Bytes,SAE1,,AWS,Out,Bytes
 APS2-TimedStorage-ByteHrs,APS2,TimedStorage,ByteHrs,,
 EUN1-TimedStorage-ByteHrs,EUN1,TimedStorage,ByteHrs,,
-SAE1-TimedStorage-ByteHrs,SAE1,TimedStorage,ByteHrs,,
+SAE1-TimedStorage-ByteHrs,SAE1,TimedStorage,ByteHrs,,
+TimedStorage-ByteHrs,USE1,TimedStorage,ByteHrs,,
+Global-Bucket-Hrs-FreeTier,unknown,Bucket,Hrs,FreeTier,

pkg/aws/ec2/ec2.go

@@ -64,7 +64,7 @@ var (
 
 // Collector is a prometheus collector that collects metrics from AWS EKS clusters.
 type Collector struct {
-	Regions            []ec2Types.Region
+	regions            []ec2Types.Region
 	ScrapeInterval     time.Duration
 	computePricingMap  *ComputePricingMap
 	storagePricingMap  *StoragePricingMap
@@ -124,7 +124,7 @@ func New(ctx context.Context, config *Config) (*Collector, error) {
 
 	return &Collector{
 		ScrapeInterval:     config.ScrapeInterval,
-		Regions:            config.Regions,
+		regions:            config.Regions,
 		logger:             logger,
 		awsRegionClientMap: config.RegionMap,
 		computePricingMap:  computeMap,
@@ -136,7 +136,7 @@ func New(ctx context.Context, config *Config) (*Collector, error) {
 func (c *Collector) Collect(ctx context.Context, ch chan<- prometheus.Metric) error {
 	c.logger.LogAttrs(ctx, slog.LevelInfo, "calling collect")
 
-	numOfRegions := len(c.Regions)
+	numOfRegions := len(c.regions)
 
 	wgInstances := sync.WaitGroup{}
 	wgInstances.Add(numOfRegions)
@@ -146,7 +146,7 @@ func (c *Collector) Collect(ctx context.Context, ch chan<- prometheus.Metric) er
 	wgVolumes.Add(numOfRegions)
 	volumeCh := make(chan []ec2Types.Volume, numOfRegions)
 
-	for _, region := range c.Regions {
+	for _, region := range c.regions {
 		regionName := *region.RegionName
 
 		regionClient, ok := c.awsRegionClientMap[regionName]
@@ -374,6 +374,10 @@ func (c *Collector) Name() string {
 	return subsystem
 }
 
+func (c *Collector) Regions() []string {
+	return utils.RegionsFromMap(c.awsRegionClientMap)
+}
+
 func (c *Collector) Register(_ provider.Registry) error {
 	return nil
 }

pkg/aws/ec2/ec2_test.go

@@ -394,7 +394,7 @@ func Test_FetchVolumesData(t *testing.T) {
 			Times(1)
 
 		wg := sync.WaitGroup{}
-		wg.Add(len(collector.Regions))
+		wg.Add(len(collector.regions))
 		ch := make(chan []ec2Types.Volume)
 		go collector.fetchVolumesData(t.Context(), c, regionName, ch)
 		go func() {

pkg/aws/elb/elb.go

@@ -40,7 +40,7 @@ var (
 )
 
 type Collector struct {
-	Regions            []ec2Types.Region
+	regions            []ec2Types.Region
 	ScrapeInterval     time.Duration
 	pricingMap         *ELBPricingMap
 	awsRegionClientMap map[string]client.Client
@@ -82,7 +82,7 @@ type elbProduct struct {
 
 func New(config *Config) *Collector {
 	return &Collector{
-		Regions:            config.Regions,
+		regions:            config.Regions,
 		ScrapeInterval:     config.ScrapeInterval,
 		awsRegionClientMap: config.RegionClients,
 		logger:             config.Logger,
@@ -104,7 +104,7 @@ func (c *Collector) Collect(ctx context.Context, ch chan<- prometheus.Metric) er
 	c.logger.Info("Starting ELB collection")
 
 	if c.shouldScrape() {
-		if err := c.pricingMap.refresh(ctx, c.awsRegionClientMap, c.Regions); err != nil {
+		if err := c.pricingMap.refresh(ctx, c.awsRegionClientMap, c.regions); err != nil {
 			c.logger.Error("Failed to refresh pricing", "error", err)
 			return err
 		}
@@ -153,6 +153,10 @@ func (c *Collector) Name() string {
 	return subsystem
 }
 
+func (c *Collector) Regions() []string {
+	return utils.RegionsFromMap(c.awsRegionClientMap)
+}
+
 func (c *Collector) shouldScrape() bool {
 	return time.Now().After(c.NextScrape)
 }