Add fixture/snapshot tests (#101)

* Add fixture/snapshot tests

Add tests that transform vendored API responses (fixtures) into vendored
/metrics output (snapshots), so that we can see how changes to the
exporter implementation affect the actual output of the exporter, in the
form of changes to the snapshots.

I've sourced the fixtures from debug JSON logging of real API responses
from tasks running in AWS. Each task consisted of an ecs_exporter
container and a prometheus container.

My expectation is that fixtures should not be updated unless we become
informed that API responses have materially changed, e.g. because new
features we are using were added, or breaking changes have been made.

I've got two sets of fixtures and snapshots, one for EC2 and one for
Fargate. The API responses differ between EC2 and Fargate
enough (because, among other things, they use two completely different
implementations of the task metadata API, and two different container
runtimes entirely!) that I think it's worth covering them both
independently.

Closes #99.

Signed-off-by: Ian Kerins <[email protected]>

* Fix typo'd metric name

Compare to the sibling metric whose name is spelled correctly:
ecs_network_receive_packets_dropped_total.

This commit shows the snapshot tests in action.

Signed-off-by: Ian Kerins <[email protected]>

* Fix nil pointer dereference with stopped container in EC2

As the updated task stats fixture for EC2 shows, stopped containers in
EC2 have the empty JSON object for their stats, instead of the `null`
that Fargate has, which triggered a nil pointer dereference that is now
fixed.

Signed-off-by: Ian Kerins <[email protected]>

* Load test fixtures at test execution time

Signed-off-by: Ian Kerins <[email protected]>

* Generate snapshot diff using testutil.CollectAndCompare

Signed-off-by: Ian Kerins <[email protected]>

---------

Signed-off-by: Ian Kerins <[email protected]>

Author: isker

README.md

@@ -48,190 +48,10 @@ None.
 
 ## Example output
 
-(With `--web.disable-exporter-metrics` passed, such that standard Go metrics are not included here.)
-
-```
-# HELP ecs_container_cpu_usage_seconds_total Cumulative total container CPU usage in seconds.
-# TYPE ecs_container_cpu_usage_seconds_total counter
-ecs_container_cpu_usage_seconds_total{container_name="ecs-exporter"} 0.028057878
-# HELP ecs_container_memory_limit_bytes Configured container memory limit in bytes, set from the container-level limit in the task definition if any, otherwise the task-level limit.
-# TYPE ecs_container_memory_limit_bytes gauge
-ecs_container_memory_limit_bytes{container_name="ecs-exporter"} 5.36870912e+08
-# HELP ecs_container_memory_page_cache_size_bytes Current container memory page cache size in bytes. This is not a subset of used bytes.
-# TYPE ecs_container_memory_page_cache_size_bytes gauge
-ecs_container_memory_page_cache_size_bytes{container_name="ecs-exporter"} 0
-# HELP ecs_container_memory_usage_bytes Current container memory usage in bytes.
-# TYPE ecs_container_memory_usage_bytes gauge
-ecs_container_memory_usage_bytes{container_name="ecs-exporter"} 4.243456e+06
-# HELP ecs_exporter_build_info A metric with a constant '1' value labeled by version, revision, branch, goversion from which ecs_exporter was built, and the goos and goarch for the build.
-# TYPE ecs_exporter_build_info gauge
-ecs_exporter_build_info{branch="",goarch="arm64",goos="linux",goversion="go1.23.2",revision="unknown",tags="unknown",version=""} 1
-# HELP ecs_network_receive_bytes_total Cumulative total size of network packets received in bytes.
-# TYPE ecs_network_receive_bytes_total counter
-ecs_network_receive_bytes_total{interface="eth1"} 1.1172419e+07
-# HELP ecs_network_receive_errors_total Cumulative total count of network errors in receiving.
-# TYPE ecs_network_receive_errors_total counter
-ecs_network_receive_errors_total{interface="eth1"} 0
-# HELP ecs_network_receive_packets_dropped_total Cumulative total count of network packets dropped in receiving.
-# TYPE ecs_network_receive_packets_dropped_total counter
-ecs_network_receive_packets_dropped_total{interface="eth1"} 0
-# HELP ecs_network_receive_packets_total Cumulative total count of network packets received.
-# TYPE ecs_network_receive_packets_total counter
-ecs_network_receive_packets_total{interface="eth1"} 8084
-# HELP ecs_network_transmit_bytes_total Cumulative total size of network packets transmitted in bytes.
-# TYPE ecs_network_transmit_bytes_total counter
-ecs_network_transmit_bytes_total{interface="eth1"} 178817
-# HELP ecs_network_transmit_dropped_total Cumulative total count of network packets dropped in transmit.
-# TYPE ecs_network_transmit_dropped_total counter
-ecs_network_transmit_dropped_total{interface="eth1"} 0
-# HELP ecs_network_transmit_errors_total Cumulative total count of network errors in transmit.
-# TYPE ecs_network_transmit_errors_total counter
-ecs_network_transmit_errors_total{interface="eth1"} 0
-# HELP ecs_network_transmit_packets_total Cumulative total count of network packets transmitted.
-# TYPE ecs_network_transmit_packets_total counter
-ecs_network_transmit_packets_total{interface="eth1"} 897
-# HELP ecs_task_cpu_limit_vcpus Configured task CPU limit in vCPUs (1 vCPU = 1024 CPU units). This is optional when running on EC2; if no limit is set, this metric has no value.
-# TYPE ecs_task_cpu_limit_vcpus gauge
-ecs_task_cpu_limit_vcpus 0.25
-# HELP ecs_task_ephemeral_storage_allocated_bytes Configured Fargate task ephemeral storage allocated size in bytes.
-# TYPE ecs_task_ephemeral_storage_allocated_bytes gauge
-ecs_task_ephemeral_storage_allocated_bytes 2.1491613696e+10
-# HELP ecs_task_ephemeral_storage_used_bytes Current Fargate task ephemeral storage usage in bytes.
-# TYPE ecs_task_ephemeral_storage_used_bytes gauge
-ecs_task_ephemeral_storage_used_bytes 3.7748736e+07
-# HELP ecs_task_image_pull_start_timestamp_seconds The time at which the task started pulling docker images for its containers.
-# TYPE ecs_task_image_pull_start_timestamp_seconds gauge
-ecs_task_image_pull_start_timestamp_seconds 1.737156015124145e+09
-# HELP ecs_task_image_pull_stop_timestamp_seconds The time at which the task stopped (i.e. completed) pulling docker images for its containers.
-# TYPE ecs_task_image_pull_stop_timestamp_seconds gauge
-ecs_task_image_pull_stop_timestamp_seconds 1.7371560172684324e+09
-# HELP ecs_task_memory_limit_bytes Configured task memory limit in bytes. This is optional when running on EC2; if no limit is set, this metric has no value.
-# TYPE ecs_task_memory_limit_bytes gauge
-ecs_task_memory_limit_bytes 5.36870912e+08
-# HELP ecs_task_metadata_info ECS task metadata, sourced from the task metadata endpoint version 4.
-# TYPE ecs_task_metadata_info gauge
-ecs_task_metadata_info{availability_zone="us-east-1a",cluster="arn:aws:ecs:us-east-1:829490980523:cluster/prom-ecs-exporter-sandbox",desired_status="RUNNING",family="prom-ecs-exporter-sandbox-isker-fix-network-metrics-fargate",known_status="RUNNING",launch_type="FARGATE",revision="1",task_arn="arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/c8387acdc4884a0fa13dae78e68a989f"} 1
-```
-
-## Example task definition
-
-```
-{
-  "ipcMode": null,
-  "executionRoleArn": "arn:aws:iam::ACCOUNT_ID:role/ecsTaskExecutionRole",
-  "containerDefinitions": [
-    {
-      "dnsSearchDomains": null,
-      "environmentFiles": null,
-      "logConfiguration": {
-        "logDriver": "awslogs",
-        "secretOptions": null,
-        "options": {
-          "awslogs-group": "/ecs/ecs-exporter",
-          "awslogs-region": "us-west-2",
-          "awslogs-stream-prefix": "ecs"
-        }
-      },
-      "entryPoint": null,
-      "portMappings": [
-        {
-          "hostPort": 9779,
-          "protocol": "tcp",
-          "containerPort": 9779
-        }
-      ],
-      "command": null,
-      "linuxParameters": null,
-      "cpu": 0,
-      "environment": [],
-      "resourceRequirements": null,
-      "ulimits": null,
-      "dnsServers": null,
-      "mountPoints": [],
-      "workingDirectory": null,
-      "secrets": null,
-      "dockerSecurityOptions": null,
-      "memory": null,
-      "memoryReservation": null,
-      "volumesFrom": [],
-      "stopTimeout": null,
-      "image": "quay.io/prometheuscommunity/ecs-exporter:v0.1.0",
-      "startTimeout": null,
-      "firelensConfiguration": null,
-      "dependsOn": null,
-      "disableNetworking": null,
-      "interactive": null,
-      "healthCheck": null,
-      "essential": true,
-      "links": null,
-      "hostname": null,
-      "extraHosts": null,
-      "pseudoTerminal": null,
-      "user": null,
-      "readonlyRootFilesystem": null,
-      "dockerLabels": null,
-      "systemControls": null,
-      "privileged": null,
-      "name": "ecs-exporter"
-    }
-  ],
-  "placementConstraints": [],
-  "memory": "512",
-  "taskRoleArn": "arn:aws:iam::ACCOUNT_ID:role/ecsTaskExecutionRole",
-  "compatibilities": [
-    "EC2",
-    "FARGATE"
-  ],
-  "taskDefinitionArn": "arn:aws:ecs:us-west-2:ACCOUNT_ID:task-definition/ecs-exporter:1",
-  "family": "ecs-exporter",
-  "requiresAttributes": [
-    {
-      "targetId": null,
-      "targetType": null,
-      "value": null,
-      "name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
-    },
-    {
-      "targetId": null,
-      "targetType": null,
-      "value": null,
-      "name": "ecs.capability.execution-role-awslogs"
-    },
-    {
-      "targetId": null,
-      "targetType": null,
-      "value": null,
-      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
-    },
-    {
-      "targetId": null,
-      "targetType": null,
-      "value": null,
-      "name": "com.amazonaws.ecs.capability.task-iam-role"
-    },
-    {
-      "targetId": null,
-      "targetType": null,
-      "value": null,
-      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
-    },
-    {
-      "targetId": null,
-      "targetType": null,
-      "value": null,
-      "name": "ecs.capability.task-eni"
-    }
-  ],
-  "pidMode": null,
-  "requiresCompatibilities": [
-    "FARGATE"
-  ],
-  "networkMode": "awsvpc",
-  "cpu": "256",
-  "revision": 1,
-  "status": "ACTIVE",
-  "inferenceAccelerators": null,
-  "proxyConfiguration": null,
-  "volumes": []
-}
-```
+Check out the [metrics snapshots](./ecscollector/testdata/snapshots) which
+contain sample metrics emitted by ecs_exporter in the [Prometheus text
+format](https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format)
+you should expect to see on /metrics. Note that these snapshots behave as if
+`--web.disable-exporter-metrics` were passed when running ecs_exporter, such
+that standard [client_golang](https://github.com/prometheus/client_golang)
+metrics are not included.

ecscollector/collector.go

@@ -124,7 +124,7 @@ var (
 		networkLabels, nil)
 
 	networkTxDroppedDesc = prometheus.NewDesc(
-		"ecs_network_transmit_dropped_total",
+		"ecs_network_transmit_packets_dropped_total",
 		"Cumulative total count of network packets dropped in transmit.",
 		networkLabels, nil)
 
@@ -196,7 +196,7 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) {
 		c.logger.Debug("Failed to retrieve metadata", "error", err)
 		return
 	}
-	c.logger.Debug("Got ECS task metadata response", "stats", metadata)
+	c.logger.Debug("Got ECS task metadata response", "metadata", metadata)
 
 	ch <- prometheus.MustNewConstMetric(
 		taskMetadataDesc,
@@ -269,10 +269,10 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) {
 	networks := make(map[string]*container.NetworkStats)
 	for _, container := range metadata.Containers {
 		s := stats[container.ID]
-		if s == nil {
+		if s == nil || s.StatsJSON == nil {
 			// This can happen if the container is stopped; if it's
 			// nonessential, the task goes on.
-			c.logger.Debug("Couldn't find container with ID in stats", "id", container.ID)
+			c.logger.Debug("Couldn't find stats for container", "id", container.ID)
 			continue
 		}
 

ecscollector/collector_test.go

@@ -0,0 +1,140 @@
+// Copyright 2025 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ecscollector
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/prometheus-community/ecs_exporter/ecsmetadata"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+// Create a metadata client that will always receive the given fixture API
+// responses.
+func fixtureClient(taskMetadataPath, taskStatsPath string) (*ecsmetadata.Client, *httptest.Server, error) {
+	taskMetadata, err := os.ReadFile(taskMetadataPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to read task metadata fixture: %w", err)
+	}
+	taskStats, err := os.ReadFile(taskStatsPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to read task stats fixture: %w", err)
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /task", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Add("content-type", "application/json")
+		w.Write(taskMetadata)
+	})
+	mux.HandleFunc("GET /task/stats", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Add("content-type", "application/json")
+		w.Write(taskStats)
+	})
+
+	server := httptest.NewServer(mux)
+	return ecsmetadata.NewClient(server.URL), server, nil
+}
+
+// Renders metrics from the given collector to the prometheus text exposition
+// format.
+func renderMetrics(collector prometheus.Collector) ([]byte, error) {
+	registry := prometheus.NewRegistry()
+	registry.MustRegister(collector)
+
+	// It seems that the only way to really get full /metrics output is with
+	// promhttp. There is testutil.CollectAndFormat but it requires you to
+	// specify every metric name you want in the output, which seems to be not
+	// worth it compared to this.
+	promServer := httptest.NewServer(promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
+	defer promServer.Close()
+	resp, err := http.Get(promServer.URL)
+	if err != nil {
+		return nil, fmt.Errorf("metrics request failed: %w", err)
+	}
+
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("non-200 metrics response: %v", resp.StatusCode)
+	}
+	metrics, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read metrics response body: %w", err)
+	}
+	return metrics, nil
+}
+
+var updateSnapshots = flag.Bool("update-snapshots", false, "update snapshot files")
+
+func assertSnapshot(t *testing.T, collector prometheus.Collector, path string) {
+	if *updateSnapshots {
+		metrics, err := renderMetrics(collector)
+		if err != nil {
+			t.Fatalf("failed to render new snapshot %s: %v", path, err)
+		}
+		dir := filepath.Dir(path)
+		if err := os.MkdirAll(dir, 0750); err != nil {
+			t.Fatalf("failed to create snapshot output directory %s: %v", dir, err)
+		} else if err := os.WriteFile(path, metrics, 0666); err != nil {
+			t.Fatalf("failed to write snapshot file %s: %v", path, err)
+		} else {
+			t.Logf("updated snapshot: %s", path)
+		}
+	}
+
+	file, err := os.Open(path)
+	if errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("snapshot file does not exist, set the -update-snapshots flag to update: %v", err)
+	} else if err != nil {
+		t.Fatalf("failed to open snapshot file: %v", err)
+	} else if err := testutil.CollectAndCompare(collector, file); err != nil {
+		t.Fatalf("snapshot outdated, set the -update-snapshots flag to update\n%v", err)
+	}
+}
+
+func TestFargateMetrics(t *testing.T) {
+	metadataClient, metadataServer, err := fixtureClient(
+		"testdata/fixtures/fargate_task_metadata.json",
+		"testdata/fixtures/fargate_task_stats.json",
+	)
+	if err != nil {
+		t.Fatalf("failed to load test fixtures: %v", err)
+	}
+	defer metadataServer.Close()
+	collector := NewCollector(metadataClient, slog.Default())
+	assertSnapshot(t, collector, "testdata/snapshots/fargate_metrics.txt")
+}
+
+func TestEc2Metrics(t *testing.T) {
+	metadataClient, metadataServer, err := fixtureClient(
+		"testdata/fixtures/ec2_task_metadata.json",
+		"testdata/fixtures/ec2_task_stats.json",
+	)
+	if err != nil {
+		t.Fatalf("failed to load test fixtures: %v", err)
+	}
+	defer metadataServer.Close()
+	collector := NewCollector(metadataClient, slog.Default())
+	assertSnapshot(t, collector, "testdata/snapshots/ec2_metrics.txt")
+}