implement hyperdisk-balanced tiered costs

docs/metrics/gcp/gke.md

@@ -60,8 +60,10 @@ To map the sku to the disk type, we can use the following mapping:
 
 Cloudcost Exporter needs to support the following hyperdisk pricing dimensions:
 - [x] provisioned space
-- [ ] Network throughput
-- [ ] IOps
+- [x] Network throughput
+- [x] IOps
 - [ ] high availability
 
-[#344](https://github.com/grafana/cloudcost-exporter/pull/344) introduced experimental support for provisioned space for [hyperdisk class](https://cloud.google.com/compute/disks-image-pricing#persistentdisk) 
+[#344](https://github.com/grafana/cloudcost-exporter/pull/344) introduced experimental support for provisioned space for [hyperdisk class](https://cloud.google.com/compute/disks-image-pricing#persistentdisk).
+
+For Hyperdisk Balanced, `persistent_volume_usd_per_hour` includes all three cost dimensions (capacity + IOPS + throughput). The first 3000 IOPS and 140 MBps are free; only provisioned amounts above those baselines incur cost.

pkg/google/gke/disk.go

@@ -22,28 +22,32 @@ const (
 type Disk struct {
 	Cluster string
 
-	Project     string
-	name        string // Name of the disk as it appears in the GCP console. Used as a backup if the name can't be extracted from the description
-	zone        string
-	labels      map[string]string
-	description map[string]string
-	diskType    string // type is a reserved word, which is why we're using diskType
-	Size        int64
-	users       []string
+	Project               string
+	name                  string // Name of the disk as it appears in the GCP console. Used as a backup if the name can't be extracted from the description
+	zone                  string
+	labels                map[string]string
+	description           map[string]string
+	diskType              string // type is a reserved word, which is why we're using diskType
+	Size                  int64
+	ProvisionedIops       int64
+	ProvisionedThroughput int64
+	users                 []string
 }
 
 func NewDisk(disk *compute.Disk, project string) *Disk {
 	clusterName := disk.Labels[client.GkeClusterLabel]
 	d := &Disk{
-		Cluster:     clusterName,
-		Project:     project,
-		name:        disk.Name,
-		zone:        disk.Zone,
-		diskType:    disk.Type,
-		labels:      disk.Labels,
-		description: make(map[string]string),
-		Size:        disk.SizeGb,
-		users:       disk.Users,
+		Cluster:               clusterName,
+		Project:               project,
+		name:                  disk.Name,
+		zone:                  disk.Zone,
+		diskType:              disk.Type,
+		labels:                disk.Labels,
+		description:           make(map[string]string),
+		Size:                  disk.SizeGb,
+		ProvisionedIops:       disk.ProvisionedIops,
+		ProvisionedThroughput: disk.ProvisionedThroughput,
+		users:                 disk.Users,
 	}
 	err := extractLabelsFromDesc(disk.Description, d.description)
 	if err != nil {

pkg/google/gke/gke.go

@@ -209,7 +209,7 @@ func (c *Collector) Collect(ctx context.Context, ch chan<- prometheus.Metric) er
 					d.UseStatus(),
 				}
 
-				price, err := c.pricingMap.GetCostOfStorage(d.Region(), d.StorageClass())
+				prices, err := c.pricingMap.GetCostOfStorage(d.Region(), d.StorageClass())
 				if err != nil {
 					c.logger.LogAttrs(ctx,
 						slog.LevelError,
@@ -225,7 +225,7 @@ func (c *Collector) Collect(ctx context.Context, ch chan<- prometheus.Metric) er
 				ch <- prometheus.MustNewConstMetric(
 					persistentVolumeHourlyCostDesc,
 					prometheus.GaugeValue,
-					float64(d.Size)*price,
+					computeDiskCost(d, prices),
 					labelValues...,
 				)
 			}

pkg/google/gke/pricing_map.go

@@ -134,10 +134,19 @@ func NewMachineTypePricing() *FamilyPricing {
 	}
 }
 
+const (
+	// HyperdiskBalancedFreeIOPS is the number of IOPS included at no charge with
+	// each Hyperdisk Balanced volume.
+	HyperdiskBalancedFreeIOPS = 3000
+	// HyperdiskBalancedFreeThroughputMBps is the throughput (in MBps) included at
+	// no charge with each Hyperdisk Balanced volume.
+	HyperdiskBalancedFreeThroughputMBps = 140
+)
+
 type StoragePrices struct {
 	ProvisionedSpaceGiB float64
 	Throughput          float64
-	IOops               float64
+	IOps                float64
 }
 
 // StoragePricing is a map where the key is the storage type and the value is the price
@@ -170,26 +179,45 @@ func (pm *PricingMap) GetCostOfInstance(instance *client.MachineSpec) (float64,
 	return computePrices.Cpu, computePrices.Ram, nil
 }
 
-func (pm *PricingMap) GetCostOfStorage(region, storageClass string) (float64, error) {
+// computeDiskCost returns the total hourly cost for a disk given its pricing.
+// For non-hyperdisk types, IOps and Throughput rates are zero so only capacity
+// contributes. For Hyperdisk Balanced, IOPS and throughput above the free tier
+// are included.
+func computeDiskCost(d *Disk, p *StoragePrices) float64 {
+	cost := float64(d.Size) * p.ProvisionedSpaceGiB
+
+	if p.IOps > 0 && d.ProvisionedIops > HyperdiskBalancedFreeIOPS {
+		cost += float64(d.ProvisionedIops-HyperdiskBalancedFreeIOPS) * p.IOps
+	}
+	if p.Throughput > 0 && d.ProvisionedThroughput > HyperdiskBalancedFreeThroughputMBps {
+		cost += float64(d.ProvisionedThroughput-HyperdiskBalancedFreeThroughputMBps) * p.Throughput
+	}
+
+	return cost
+}
+
+func (pm *PricingMap) GetCostOfStorage(region, storageClass string) (*StoragePrices, error) {
 	if len(pm.storage) == 0 {
-		return 0, ErrRegionNotFound
+		return nil, ErrRegionNotFound
 	}
 	if _, ok := pm.storage[region]; !ok {
-		return 0, fmt.Errorf("%w: %s", ErrRegionNotFound, region)
+		return nil, fmt.Errorf("%w: %s", ErrRegionNotFound, region)
 	}
 	if _, ok := pm.storage[region].Storage[storageClass]; !ok {
-		return 0, fmt.Errorf("%w: %s", ErrFamilyTypeNotFound, storageClass)
+		return nil, fmt.Errorf("%w: %s", ErrFamilyTypeNotFound, storageClass)
 	}
-	return pm.storage[region].Storage[storageClass].ProvisionedSpaceGiB, nil
+	return pm.storage[region].Storage[storageClass], nil
 }
 
 var (
 	storageClasses = map[string]string{
-		"Storage PD Capacity":         "pd-standard",
-		"SSD backed PD Capacity":      "pd-ssd",
-		"Balanced PD Capacity":        "pd-balanced",
-		"Extreme PD Capacity":         "pd-extreme",
-		"Hyperdisk Balanced Capacity": "hyperdisk-balanced",
+		"Storage PD Capacity":           "pd-standard",
+		"SSD backed PD Capacity":        "pd-ssd",
+		"Balanced PD Capacity":          "pd-balanced",
+		"Extreme PD Capacity":           "pd-extreme",
+		"Hyperdisk Balanced Capacity":   "hyperdisk-balanced",
+		"Hyperdisk Balanced IOPS":       "hyperdisk-balanced",
+		"Hyperdisk Balanced Throughput": "hyperdisk-balanced",
 	}
 )
 
@@ -254,7 +282,6 @@ func (pm *PricingMap) ParseSkus(skus []*billingpb.Sku) error {
 				// In GKE you can only provision the following classes: https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver#create_a_storageclass
 				// For extreme disks, we are ignoring the cost of IOPs, which would be a significant cost(could double cost of disk)
 				// TODO(pokom): Add support for other storage classes
-				// TODO(pokom): Add support for IOps operations
 				if _, ok := pm.storage[data.Region]; !ok {
 					pm.storage[data.Region] = NewStoragePricing()
 				}
@@ -269,32 +296,40 @@ func (pm *PricingMap) ParseSkus(skus []*billingpb.Sku) error {
 						break
 					}
 				}
-				if storageClass == "" {
-					slog.Warn("Storage class not found, skipping", "description", data.Description)
+				if strings.Contains(data.Description, "Confidential") {
+					slog.Info("Storage class contains Confidential, skipping", "description", data.Description)
 					continue
 				}
-				if strings.Contains(data.Description, "Confidential") {
-					slog.Info("Storage class contains Confidential, skipping", "storageClass", storageClass, "description", data.Description)
+				if storageClass == "" {
+					slog.Warn("Storage class not found, skipping", "description", data.Description)
 					continue
 				}
 				// First time seen, need to initialize the StoragePrices for the storageClass
 				if _, ok := pm.storage[data.Region].Storage[storageClass]; !ok {
 					pm.storage[data.Region].Storage[storageClass] = &StoragePrices{}
 				}
-				if pm.storage[data.Region].Storage[storageClass].ProvisionedSpaceGiB != 0.0 {
-					slog.Warn("Storage class already exists in region", "storageClass", storageClass, "region", data.Region)
-					continue
+				hourlyRate := float64(data.Price) * 1e-9 / utils.HoursInMonth
+				sp := pm.storage[data.Region].Storage[storageClass]
+				switch {
+				case strings.Contains(data.Description, "IOPS") || strings.Contains(data.Description, "Iops"):
+					if sp.IOps != 0.0 {
+						slog.Warn("IOps price already exists in region", "storageClass", storageClass, "region", data.Region)
+						continue
+					}
+					sp.IOps = hourlyRate
+				case strings.Contains(data.Description, "Throughput"):
+					if sp.Throughput != 0.0 {
+						slog.Warn("Throughput price already exists in region", "storageClass", storageClass, "region", data.Region)
+						continue
+					}
+					sp.Throughput = hourlyRate
+				default:
+					if sp.ProvisionedSpaceGiB != 0.0 {
+						slog.Warn("Storage class already exists in region", "storageClass", storageClass, "region", data.Region)
+						continue
+					}
+					sp.ProvisionedSpaceGiB = hourlyRate
 				}
-				// Switch statement must go here to handle hyperdisk cases, otherwise what's happening is
-				// The four dimensions get ignored. There is a sku for:
-				// 1. Standard IOPS( Hyperdisk Balanced Storage Pools Standard IOPS - Oregon)
-				// 2. Capacity (Hyperdisk Balanced Capacity in Milan)
-				// 3. Throughput (Hyperdisk Balanced Throughput in Columbus)
-				// 4. High Availability Iops(Hyperdisk Balanced High Availability Iops in Mexico)
-				// The current implementation specifically looks for `Hyperdisk Balanced Capacity` to avoid taking the last price that's found
-				// Then there is one variation of hyperdisks that are priced differently:
-				// 1. Storage Pools Advanced Capacity(Hyperdisk Balanced Storage Pools Advanced Capacity - Mexico)
-				pm.storage[data.Region].Storage[storageClass].ProvisionedSpaceGiB = float64(data.Price) * 1e-9 / utils.HoursInMonth
 			}
 		}
 	}

pkg/google/gke/pricing_map_test.go

@@ -549,27 +549,53 @@ func TestPricingMapParseSkus(t *testing.T) {
 			},
 		},
 		{
-			name: "HyperDisk Pricing",
-			skus: []*billingpb.Sku{{
-				Description:    "Hyperdisk Balanced Capacity",
-				Category:       &billingpb.Category{ResourceFamily: "Storage"},
-				ServiceRegions: []string{"europe-west1"},
-				PricingInfo: []*billingpb.PricingInfo{{
-					PricingExpression: &billingpb.PricingExpression{
-						TieredRates: []*billingpb.PricingExpression_TierRate{{
-							UnitPrice: &money.Money{
-								Nanos: 1e9,
-							},
-						}},
-					},
-				}},
-			}},
+			name: "HyperDisk Pricing with all dimensions",
+			skus: []*billingpb.Sku{
+				{
+					Description:    "Hyperdisk Balanced Capacity",
+					Category:       &billingpb.Category{ResourceFamily: "Storage"},
+					ServiceRegions: []string{"europe-west1"},
+					PricingInfo: []*billingpb.PricingInfo{{
+						PricingExpression: &billingpb.PricingExpression{
+							TieredRates: []*billingpb.PricingExpression_TierRate{{
+								UnitPrice: &money.Money{Nanos: 1e9},
+							}},
+						},
+					}},
+				},
+				{
+					Description:    "Hyperdisk Balanced IOPS",
+					Category:       &billingpb.Category{ResourceFamily: "Storage"},
+					ServiceRegions: []string{"europe-west1"},
+					PricingInfo: []*billingpb.PricingInfo{{
+						PricingExpression: &billingpb.PricingExpression{
+							TieredRates: []*billingpb.PricingExpression_TierRate{{
+								UnitPrice: &money.Money{Nanos: 5e8},
+							}},
+						},
+					}},
+				},
+				{
+					Description:    "Hyperdisk Balanced Throughput",
+					Category:       &billingpb.Category{ResourceFamily: "Storage"},
+					ServiceRegions: []string{"europe-west1"},
+					PricingInfo: []*billingpb.PricingInfo{{
+						PricingExpression: &billingpb.PricingExpression{
+							TieredRates: []*billingpb.PricingExpression_TierRate{{
+								UnitPrice: &money.Money{Nanos: 7e8},
+							}},
+						},
+					}},
+				},
+			},
 			expectedPricingMap: &PricingMap{
 				storage: map[string]*StoragePricing{
 					"europe-west1": {
 						Storage: map[string]*StoragePrices{
 							"hyperdisk-balanced": {
-								ProvisionedSpaceGiB: 1.0 / utils.HoursInMonth,
+								ProvisionedSpaceGiB: float64(1e9) * 1e-9 / utils.HoursInMonth,
+								IOps:                float64(5e8) * 1e-9 / utils.HoursInMonth,
+								Throughput:          float64(7e8) * 1e-9 / utils.HoursInMonth,
 							},
 						},
 					},
@@ -857,3 +883,72 @@ func Test_parseAllProducts(t *testing.T) {
 	}
 	fmt.Printf("%v SKU weren't parsable", counter)
 }
+
+func Test_computeDiskCost(t *testing.T) {
+	tests := []struct {
+		name     string
+		disk     *Disk
+		prices   *StoragePrices
+		expected float64
+	}{
+		{
+			name: "non-hyperdisk: only capacity",
+			disk: &Disk{Size: 100},
+			prices: &StoragePrices{
+				ProvisionedSpaceGiB: 0.01,
+			},
+			expected: 100 * 0.01,
+		},
+		{
+			name: "hyperdisk below free tier",
+			disk: &Disk{
+				Size:                  100,
+				ProvisionedIops:       2000,
+				ProvisionedThroughput: 100,
+			},
+			prices: &StoragePrices{
+				ProvisionedSpaceGiB: 0.01,
+				IOps:                0.001,
+				Throughput:          0.005,
+			},
+			expected: 100 * 0.01, // no IOPS or throughput cost
+		},
+		{
+			name: "hyperdisk at exact free tier boundary",
+			disk: &Disk{
+				Size:                  100,
+				ProvisionedIops:       HyperdiskBalancedFreeIOPS,
+				ProvisionedThroughput: HyperdiskBalancedFreeThroughputMBps,
+			},
+			prices: &StoragePrices{
+				ProvisionedSpaceGiB: 0.01,
+				IOps:                0.001,
+				Throughput:          0.005,
+			},
+			expected: 100 * 0.01, // exactly at boundary, no extra cost
+		},
+		{
+			name: "hyperdisk above free tier",
+			disk: &Disk{
+				Size:                  200,
+				ProvisionedIops:       5000,
+				ProvisionedThroughput: 240,
+			},
+			prices: &StoragePrices{
+				ProvisionedSpaceGiB: 0.01,
+				IOps:                0.001,
+				Throughput:          0.005,
+			},
+			// capacity: 200 * 0.01 = 2.0
+			// iops: (5000-3000) * 0.001 = 2.0
+			// throughput: (240-140) * 0.005 = 0.5
+			expected: 2.0 + 2.0 + 0.5,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := computeDiskCost(tt.disk, tt.prices)
+			require.InDelta(t, tt.expected, got, 1e-9)
+		})
+	}
+}