Cache Azure blob cost query on scrape interval

Author: suntala

docs/metrics/azure/blob.md

@@ -2,7 +2,7 @@
 
 Pass `blob` in `--azure.services` to enable this collector. Matching is case-insensitive.
 
-The collector defines a storage cost `GaugeVec` that the Azure provider includes in its `Describe` and `Collect` fan-out (same gatherer pattern as `azure_aks`). `Collect` calls `StorageCostQuerier.QueryBlobStorage` with a **30-day** lookback (`defaultQueryLookback` in `pkg/azure/blob/cost_query.go`) and sets the gauge from each row. `Config.CostQuerier` supplies the querier; when it is nil the collector uses a no-op querier (no rows). The parent Azure collector forwards `StorageGauge.Collect(ch)` so blob cost metrics share one registration path with the rest of the Azure exporter. Scrape instrumentation publishes `cloudcost_exporter_collector_*` with label `collector="azure_blob"`.
+The collector defines a storage cost `GaugeVec` that the Azure provider includes in its `Describe` and `Collect` fan-out (same gatherer pattern as `azure_aks`). `Collect` calls `StorageCostQuerier.QueryBlobStorage` when `ScrapeInterval` has elapsed since the last successful query (similar billing refresh cadence to `pkg/aws/s3`). Each query uses a **30-day** lookback (`defaultQueryLookback` in `pkg/azure/blob/cost_query.go`). Cached rows are applied to the gauge every scrape. `Config.CostQuerier` supplies the querier; when it is nil the collector uses a no-op querier (no rows). The parent Azure collector forwards `StorageGauge.Collect(ch)` so blob cost metrics share one registration path with the rest of the Azure exporter. Scrape instrumentation publishes `cloudcost_exporter_collector_*` with label `collector="azure_blob"`.
 
 ## Cost metrics
 

pkg/azure/blob/blob.go

@@ -3,6 +3,7 @@ package blob
 import (
 	"context"
 	"log/slog"
+	"sync"
 	"time"
 
 	"github.com/grafana/cloudcost-exporter/pkg/provider"
@@ -49,6 +50,10 @@ type Collector struct {
 	querier        StorageCostQuerier
 	subscriptionID string
 	scrapeInterval time.Duration
+
+	mu          sync.Mutex
+	cachedRows  []StorageCostRow
+	nextRefresh time.Time // QueryBlobStorage when time.Now is on or after this (S3 billing refresh pattern).
 }
 
 // Config holds settings for the blob collector.
@@ -76,21 +81,34 @@ func New(cfg *Config) (*Collector, error) {
 		querier:        q,
 		subscriptionID: cfg.SubscriptionId,
 		scrapeInterval: interval,
+		// First Collect runs a query immediately (same idea as pkg/aws/s3 nextScrape).
+		nextRefresh: time.Now().Add(-interval),
 	}, nil
 }
 
 // Collect queries cost rows, updates the storage vec, then forwards metrics on ch for the parent gatherer.
 func (c *Collector) Collect(ctx context.Context, ch chan<- prometheus.Metric) error {
 	c.logger.LogAttrs(ctx, slog.LevelInfo, "collecting metrics")
-	rows, err := c.querier.QueryBlobStorage(ctx, c.subscriptionID, defaultQueryLookback)
-	if err != nil {
-		return err
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	now := time.Now()
+	if !now.Before(c.nextRefresh) {
+		rows, err := c.querier.QueryBlobStorage(ctx, c.subscriptionID, defaultQueryLookback)
+		if err != nil {
+			return err
+		}
+		c.cachedRows = rows
+		c.nextRefresh = now.Add(c.scrapeInterval)
 	}
+	c.applyRowsToGauge(c.cachedRows)
+	c.metrics.StorageGauge.Collect(ch)
+	return nil
+}
+
+func (c *Collector) applyRowsToGauge(rows []StorageCostRow) {
 	for _, row := range rows {
 		c.metrics.StorageGauge.WithLabelValues(row.Region, row.Class).Set(row.Rate)
 	}
-	c.metrics.StorageGauge.Collect(ch)
-	return nil
 }
 
 // Describe satisfies provider.Collector.

pkg/azure/blob/blob_test.go

@@ -6,6 +6,7 @@ import (
 	"log/slog"
 	"os"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -32,6 +33,39 @@ func (s stubCostQuerier) QueryBlobStorage(context.Context, string, time.Duration
 	return s.rows, s.err
 }
 
+type countingCostQuerier struct {
+	mu   sync.Mutex
+	n    int
+	rows []StorageCostRow
+	err  error
+}
+
+func (c *countingCostQuerier) QueryBlobStorage(context.Context, string, time.Duration) ([]StorageCostRow, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.n++
+	return c.rows, c.err
+}
+
+func (c *countingCostQuerier) calls() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.n
+}
+
+func newCollectorWithCountingQuerier(t *testing.T, rows []StorageCostRow, querierErr error) (*Collector, *countingCostQuerier) {
+	t.Helper()
+	q := &countingCostQuerier{rows: rows, err: querierErr}
+	c, err := New(&Config{
+		Logger:         testLogger,
+		SubscriptionId: "sub",
+		ScrapeInterval: time.Hour,
+		CostQuerier:    q,
+	})
+	require.NoError(t, err)
+	return c, q
+}
+
 func TestCollector_Collect_queryError(t *testing.T) {
 	c, err := New(&Config{
 		Logger:         testLogger,
@@ -42,6 +76,32 @@ func TestCollector_Collect_queryError(t *testing.T) {
 	assert.Error(t, c.Collect(t.Context(), testCollectSink()))
 }
 
+func TestCollector_Collect_costQueryRefresh(t *testing.T) {
+	sampleRows := []StorageCostRow{{Region: "eastus", Class: "Hot", Rate: 0.001}}
+
+	t.Run("skips_until_interval", func(t *testing.T) {
+		c, q := newCollectorWithCountingQuerier(t, sampleRows, nil)
+		require.NoError(t, c.Collect(t.Context(), testCollectSink()))
+		require.NoError(t, c.Collect(t.Context(), testCollectSink()))
+		assert.Equal(t, 1, q.calls(), "second scrape within interval should not call querier")
+	})
+
+	t.Run("refetches_when_next_refresh_elapsed", func(t *testing.T) {
+		c, q := newCollectorWithCountingQuerier(t, sampleRows, nil)
+		require.NoError(t, c.Collect(t.Context(), testCollectSink()))
+		c.nextRefresh = time.Now().Add(-time.Second)
+		require.NoError(t, c.Collect(t.Context(), testCollectSink()))
+		assert.Equal(t, 2, q.calls())
+	})
+
+	t.Run("retries_after_error", func(t *testing.T) {
+		c, q := newCollectorWithCountingQuerier(t, nil, errors.New("query failed"))
+		assert.Error(t, c.Collect(t.Context(), testCollectSink()))
+		assert.Error(t, c.Collect(t.Context(), testCollectSink()))
+		assert.Equal(t, 2, q.calls(), "errors do not advance nextRefresh; querier should run again")
+	})
+}
+
 func TestCollector_Collect_setsGaugeFromQuerier(t *testing.T) {
 	c, err := New(&Config{
 		Logger:         testLogger,