diff --git a/README.md b/README.md index 7cdde2e..218ac6d 100644 --- a/README.md +++ b/README.md @@ -48,190 +48,10 @@ None. ## Example output -(With `--web.disable-exporter-metrics` passed, such that standard Go metrics are not included here.) - -``` -# HELP ecs_container_cpu_usage_seconds_total Cumulative total container CPU usage in seconds. -# TYPE ecs_container_cpu_usage_seconds_total counter -ecs_container_cpu_usage_seconds_total{container_name="ecs-exporter"} 0.028057878 -# HELP ecs_container_memory_limit_bytes Configured container memory limit in bytes, set from the container-level limit in the task definition if any, otherwise the task-level limit. -# TYPE ecs_container_memory_limit_bytes gauge -ecs_container_memory_limit_bytes{container_name="ecs-exporter"} 5.36870912e+08 -# HELP ecs_container_memory_page_cache_size_bytes Current container memory page cache size in bytes. This is not a subset of used bytes. -# TYPE ecs_container_memory_page_cache_size_bytes gauge -ecs_container_memory_page_cache_size_bytes{container_name="ecs-exporter"} 0 -# HELP ecs_container_memory_usage_bytes Current container memory usage in bytes. -# TYPE ecs_container_memory_usage_bytes gauge -ecs_container_memory_usage_bytes{container_name="ecs-exporter"} 4.243456e+06 -# HELP ecs_exporter_build_info A metric with a constant '1' value labeled by version, revision, branch, goversion from which ecs_exporter was built, and the goos and goarch for the build. -# TYPE ecs_exporter_build_info gauge -ecs_exporter_build_info{branch="",goarch="arm64",goos="linux",goversion="go1.23.2",revision="unknown",tags="unknown",version=""} 1 -# HELP ecs_network_receive_bytes_total Cumulative total size of network packets received in bytes. -# TYPE ecs_network_receive_bytes_total counter -ecs_network_receive_bytes_total{interface="eth1"} 1.1172419e+07 -# HELP ecs_network_receive_errors_total Cumulative total count of network errors in receiving. -# TYPE ecs_network_receive_errors_total counter -ecs_network_receive_errors_total{interface="eth1"} 0 -# HELP ecs_network_receive_packets_dropped_total Cumulative total count of network packets dropped in receiving. -# TYPE ecs_network_receive_packets_dropped_total counter -ecs_network_receive_packets_dropped_total{interface="eth1"} 0 -# HELP ecs_network_receive_packets_total Cumulative total count of network packets received. -# TYPE ecs_network_receive_packets_total counter -ecs_network_receive_packets_total{interface="eth1"} 8084 -# HELP ecs_network_transmit_bytes_total Cumulative total size of network packets transmitted in bytes. -# TYPE ecs_network_transmit_bytes_total counter -ecs_network_transmit_bytes_total{interface="eth1"} 178817 -# HELP ecs_network_transmit_dropped_total Cumulative total count of network packets dropped in transmit. -# TYPE ecs_network_transmit_dropped_total counter -ecs_network_transmit_dropped_total{interface="eth1"} 0 -# HELP ecs_network_transmit_errors_total Cumulative total count of network errors in transmit. -# TYPE ecs_network_transmit_errors_total counter -ecs_network_transmit_errors_total{interface="eth1"} 0 -# HELP ecs_network_transmit_packets_total Cumulative total count of network packets transmitted. -# TYPE ecs_network_transmit_packets_total counter -ecs_network_transmit_packets_total{interface="eth1"} 897 -# HELP ecs_task_cpu_limit_vcpus Configured task CPU limit in vCPUs (1 vCPU = 1024 CPU units). This is optional when running on EC2; if no limit is set, this metric has no value. -# TYPE ecs_task_cpu_limit_vcpus gauge -ecs_task_cpu_limit_vcpus 0.25 -# HELP ecs_task_ephemeral_storage_allocated_bytes Configured Fargate task ephemeral storage allocated size in bytes. -# TYPE ecs_task_ephemeral_storage_allocated_bytes gauge -ecs_task_ephemeral_storage_allocated_bytes 2.1491613696e+10 -# HELP ecs_task_ephemeral_storage_used_bytes Current Fargate task ephemeral storage usage in bytes. -# TYPE ecs_task_ephemeral_storage_used_bytes gauge -ecs_task_ephemeral_storage_used_bytes 3.7748736e+07 -# HELP ecs_task_image_pull_start_timestamp_seconds The time at which the task started pulling docker images for its containers. -# TYPE ecs_task_image_pull_start_timestamp_seconds gauge -ecs_task_image_pull_start_timestamp_seconds 1.737156015124145e+09 -# HELP ecs_task_image_pull_stop_timestamp_seconds The time at which the task stopped (i.e. completed) pulling docker images for its containers. -# TYPE ecs_task_image_pull_stop_timestamp_seconds gauge -ecs_task_image_pull_stop_timestamp_seconds 1.7371560172684324e+09 -# HELP ecs_task_memory_limit_bytes Configured task memory limit in bytes. This is optional when running on EC2; if no limit is set, this metric has no value. -# TYPE ecs_task_memory_limit_bytes gauge -ecs_task_memory_limit_bytes 5.36870912e+08 -# HELP ecs_task_metadata_info ECS task metadata, sourced from the task metadata endpoint version 4. -# TYPE ecs_task_metadata_info gauge -ecs_task_metadata_info{availability_zone="us-east-1a",cluster="arn:aws:ecs:us-east-1:829490980523:cluster/prom-ecs-exporter-sandbox",desired_status="RUNNING",family="prom-ecs-exporter-sandbox-isker-fix-network-metrics-fargate",known_status="RUNNING",launch_type="FARGATE",revision="1",task_arn="arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/c8387acdc4884a0fa13dae78e68a989f"} 1 -``` - -## Example task definition - -``` -{ - "ipcMode": null, - "executionRoleArn": "arn:aws:iam::ACCOUNT_ID:role/ecsTaskExecutionRole", - "containerDefinitions": [ - { - "dnsSearchDomains": null, - "environmentFiles": null, - "logConfiguration": { - "logDriver": "awslogs", - "secretOptions": null, - "options": { - "awslogs-group": "/ecs/ecs-exporter", - "awslogs-region": "us-west-2", - "awslogs-stream-prefix": "ecs" - } - }, - "entryPoint": null, - "portMappings": [ - { - "hostPort": 9779, - "protocol": "tcp", - "containerPort": 9779 - } - ], - "command": null, - "linuxParameters": null, - "cpu": 0, - "environment": [], - "resourceRequirements": null, - "ulimits": null, - "dnsServers": null, - "mountPoints": [], - "workingDirectory": null, - "secrets": null, - "dockerSecurityOptions": null, - "memory": null, - "memoryReservation": null, - "volumesFrom": [], - "stopTimeout": null, - "image": "quay.io/prometheuscommunity/ecs-exporter:v0.1.0", - "startTimeout": null, - "firelensConfiguration": null, - "dependsOn": null, - "disableNetworking": null, - "interactive": null, - "healthCheck": null, - "essential": true, - "links": null, - "hostname": null, - "extraHosts": null, - "pseudoTerminal": null, - "user": null, - "readonlyRootFilesystem": null, - "dockerLabels": null, - "systemControls": null, - "privileged": null, - "name": "ecs-exporter" - } - ], - "placementConstraints": [], - "memory": "512", - "taskRoleArn": "arn:aws:iam::ACCOUNT_ID:role/ecsTaskExecutionRole", - "compatibilities": [ - "EC2", - "FARGATE" - ], - "taskDefinitionArn": "arn:aws:ecs:us-west-2:ACCOUNT_ID:task-definition/ecs-exporter:1", - "family": "ecs-exporter", - "requiresAttributes": [ - { - "targetId": null, - "targetType": null, - "value": null, - "name": "com.amazonaws.ecs.capability.logging-driver.awslogs" - }, - { - "targetId": null, - "targetType": null, - "value": null, - "name": "ecs.capability.execution-role-awslogs" - }, - { - "targetId": null, - "targetType": null, - "value": null, - "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19" - }, - { - "targetId": null, - "targetType": null, - "value": null, - "name": "com.amazonaws.ecs.capability.task-iam-role" - }, - { - "targetId": null, - "targetType": null, - "value": null, - "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18" - }, - { - "targetId": null, - "targetType": null, - "value": null, - "name": "ecs.capability.task-eni" - } - ], - "pidMode": null, - "requiresCompatibilities": [ - "FARGATE" - ], - "networkMode": "awsvpc", - "cpu": "256", - "revision": 1, - "status": "ACTIVE", - "inferenceAccelerators": null, - "proxyConfiguration": null, - "volumes": [] -} -``` +Check out the [metrics snapshots](./ecscollector/testdata/snapshots) which +contain sample metrics emitted by ecs_exporter in the [Prometheus text +format](https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format) +you should expect to see on /metrics. Note that these snapshots behave as if +`--web.disable-exporter-metrics` were passed when running ecs_exporter, such +that standard [client_golang](https://github.com/prometheus/client_golang) +metrics are not included. diff --git a/ecscollector/collector.go b/ecscollector/collector.go index e477816..c696d44 100644 --- a/ecscollector/collector.go +++ b/ecscollector/collector.go @@ -124,7 +124,7 @@ var ( networkLabels, nil) networkTxDroppedDesc = prometheus.NewDesc( - "ecs_network_transmit_dropped_total", + "ecs_network_transmit_packets_dropped_total", "Cumulative total count of network packets dropped in transmit.", networkLabels, nil) @@ -196,7 +196,7 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) { c.logger.Debug("Failed to retrieve metadata", "error", err) return } - c.logger.Debug("Got ECS task metadata response", "stats", metadata) + c.logger.Debug("Got ECS task metadata response", "metadata", metadata) ch <- prometheus.MustNewConstMetric( taskMetadataDesc, @@ -269,10 +269,10 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) { networks := make(map[string]*container.NetworkStats) for _, container := range metadata.Containers { s := stats[container.ID] - if s == nil { + if s == nil || s.StatsJSON == nil { // This can happen if the container is stopped; if it's // nonessential, the task goes on. - c.logger.Debug("Couldn't find container with ID in stats", "id", container.ID) + c.logger.Debug("Couldn't find stats for container", "id", container.ID) continue } diff --git a/ecscollector/collector_test.go b/ecscollector/collector_test.go new file mode 100644 index 0000000..57e54f5 --- /dev/null +++ b/ecscollector/collector_test.go @@ -0,0 +1,140 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ecscollector + +import ( + "errors" + "flag" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/prometheus-community/ecs_exporter/ecsmetadata" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// Create a metadata client that will always receive the given fixture API +// responses. +func fixtureClient(taskMetadataPath, taskStatsPath string) (*ecsmetadata.Client, *httptest.Server, error) { + taskMetadata, err := os.ReadFile(taskMetadataPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to read task metadata fixture: %w", err) + } + taskStats, err := os.ReadFile(taskStatsPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to read task stats fixture: %w", err) + } + + mux := http.NewServeMux() + mux.HandleFunc("GET /task", func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("content-type", "application/json") + w.Write(taskMetadata) + }) + mux.HandleFunc("GET /task/stats", func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("content-type", "application/json") + w.Write(taskStats) + }) + + server := httptest.NewServer(mux) + return ecsmetadata.NewClient(server.URL), server, nil +} + +// Renders metrics from the given collector to the prometheus text exposition +// format. +func renderMetrics(collector prometheus.Collector) ([]byte, error) { + registry := prometheus.NewRegistry() + registry.MustRegister(collector) + + // It seems that the only way to really get full /metrics output is with + // promhttp. There is testutil.CollectAndFormat but it requires you to + // specify every metric name you want in the output, which seems to be not + // worth it compared to this. + promServer := httptest.NewServer(promhttp.HandlerFor(registry, promhttp.HandlerOpts{})) + defer promServer.Close() + resp, err := http.Get(promServer.URL) + if err != nil { + return nil, fmt.Errorf("metrics request failed: %w", err) + } + + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("non-200 metrics response: %v", resp.StatusCode) + } + metrics, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read metrics response body: %w", err) + } + return metrics, nil +} + +var updateSnapshots = flag.Bool("update-snapshots", false, "update snapshot files") + +func assertSnapshot(t *testing.T, collector prometheus.Collector, path string) { + if *updateSnapshots { + metrics, err := renderMetrics(collector) + if err != nil { + t.Fatalf("failed to render new snapshot %s: %v", path, err) + } + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0750); err != nil { + t.Fatalf("failed to create snapshot output directory %s: %v", dir, err) + } else if err := os.WriteFile(path, metrics, 0666); err != nil { + t.Fatalf("failed to write snapshot file %s: %v", path, err) + } else { + t.Logf("updated snapshot: %s", path) + } + } + + file, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + t.Fatalf("snapshot file does not exist, set the -update-snapshots flag to update: %v", err) + } else if err != nil { + t.Fatalf("failed to open snapshot file: %v", err) + } else if err := testutil.CollectAndCompare(collector, file); err != nil { + t.Fatalf("snapshot outdated, set the -update-snapshots flag to update\n%v", err) + } +} + +func TestFargateMetrics(t *testing.T) { + metadataClient, metadataServer, err := fixtureClient( + "testdata/fixtures/fargate_task_metadata.json", + "testdata/fixtures/fargate_task_stats.json", + ) + if err != nil { + t.Fatalf("failed to load test fixtures: %v", err) + } + defer metadataServer.Close() + collector := NewCollector(metadataClient, slog.Default()) + assertSnapshot(t, collector, "testdata/snapshots/fargate_metrics.txt") +} + +func TestEc2Metrics(t *testing.T) { + metadataClient, metadataServer, err := fixtureClient( + "testdata/fixtures/ec2_task_metadata.json", + "testdata/fixtures/ec2_task_stats.json", + ) + if err != nil { + t.Fatalf("failed to load test fixtures: %v", err) + } + defer metadataServer.Close() + collector := NewCollector(metadataClient, slog.Default()) + assertSnapshot(t, collector, "testdata/snapshots/ec2_metrics.txt") +} diff --git a/ecscollector/testdata/fixtures/ec2_task_metadata.json b/ecscollector/testdata/fixtures/ec2_task_metadata.json new file mode 100644 index 0000000..5c7f869 --- /dev/null +++ b/ecscollector/testdata/fixtures/ec2_task_metadata.json @@ -0,0 +1,229 @@ +{ + "Cluster": "prom-ecs-exporter-sandbox", + "TaskARN": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9", + "Family": "prom-ecs-exporter-sandbox-main-ec2", + "Revision": "13", + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "PullStartedAt": "2025-02-27T05:09:52.332595252Z", + "PullStoppedAt": "2025-02-27T05:10:01.206072368Z", + "AvailabilityZone": "us-east-1a", + "LaunchType": "EC2", + "Containers": [ + { + "DockerId": "213e1203f4bb72af185724d937e698d2724acf35b57ec2dd5f3c963adbd2d38c", + "Name": "nonessential", + "DockerName": "ecs-prom-ecs-exporter-sandbox-main-ec2-13-nonessential-9c9ab8aeb0e0dbdca601", + "Image": "alpine", + "ImageID": "sha256:8d591b0b7dea080ea3be9e12ae563eebf9869168ffced1cb25b2470a3d9fe15e", + "Labels": { + "com.amazonaws.ecs.cluster": "prom-ecs-exporter-sandbox", + "com.amazonaws.ecs.container-name": "nonessential", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9", + "com.amazonaws.ecs.task-definition-family": "prom-ecs-exporter-sandbox-main-ec2", + "com.amazonaws.ecs.task-definition-version": "13" + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "STOPPED", + "ExitCode": 0, + "Limits": { + "CPU": 128, + "Memory": 256 + }, + "CreatedAt": "2025-02-27T05:09:54.959587312Z", + "StartedAt": "2025-02-27T05:09:56.392336771Z", + "FinishedAt": "2025-02-27T05:09:56.409399983Z", + "Type": "NORMAL", + "Volumes": [ + { + "Source": "/var/lib/ecs/deps/execute-command/config/amazon-ssm-agent-Orvj12YkCf4DKDu1cHTOVj7smDviWx1T4Kg3Q_IdNYA=.json", + "Destination": "/ecs-execute-command-636764d0-0d77-44c1-96b2-207c74034dff/configuration/amazon-ssm-agent.json" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/config/seelog-gEZ-TIvHAyOLfMC5wiWRofgDMlDzaCZ6zcswnAoop84=.xml", + "Destination": "/ecs-execute-command-636764d0-0d77-44c1-96b2-207c74034dff/configuration/seelog.xml" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/certs/tls-ca-bundle.pem", + "Destination": "/ecs-execute-command-636764d0-0d77-44c1-96b2-207c74034dff/certs/amazon-ssm-agent.crt" + }, + { + "Source": "/var/log/ecs/exec/506f22fab0414cde856201584703fed9/nonessential", + "Destination": "/var/log/amazon/ssm" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/amazon-ssm-agent", + "Destination": "/ecs-execute-command-636764d0-0d77-44c1-96b2-207c74034dff/amazon-ssm-agent" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/ssm-agent-worker", + "Destination": "/ecs-execute-command-636764d0-0d77-44c1-96b2-207c74034dff/ssm-agent-worker" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/ssm-session-worker", + "Destination": "/ecs-execute-command-636764d0-0d77-44c1-96b2-207c74034dff/ssm-session-worker" + } + ], + "ContainerARN": "arn:aws:ecs:us-east-1:829490980523:container/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9/80b5fc27-0113-4b4f-83a4-f3d4b4b2b016", + "Networks": [ + { + "NetworkMode": "bridge", + "IPv4Addresses": [ + "" + ] + } + ] + }, + { + "DockerId": "01cf1f3208005cda71d5ac936ded65d2ecc0a8cc8ff8a82d2e00410bf4fbbd6d", + "Name": "ecs-exporter", + "DockerName": "ecs-prom-ecs-exporter-sandbox-main-ec2-13-ecs-exporter-e2aeb1e6be8998c72300", + "Image": "quay.io/prometheuscommunity/ecs-exporter:main", + "ImageID": "sha256:1585460bf5becf755c9f45fa931283546ca62e2d51bb638010c8958158d144bc", + "Ports": [ + { + "ContainerPort": 9779, + "Protocol": "tcp", + "HostPort": 32768, + "HostIp": "0.0.0.0" + }, + { + "ContainerPort": 9779, + "Protocol": "tcp", + "HostPort": 32768, + "HostIp": "::" + } + ], + "Labels": { + "com.amazonaws.ecs.cluster": "prom-ecs-exporter-sandbox", + "com.amazonaws.ecs.container-name": "ecs-exporter", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9", + "com.amazonaws.ecs.task-definition-family": "prom-ecs-exporter-sandbox-main-ec2", + "com.amazonaws.ecs.task-definition-version": "13" + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": { + "CPU": 128, + "Memory": 256 + }, + "CreatedAt": "2025-02-27T05:10:00.313953836Z", + "StartedAt": "2025-02-27T05:10:02.731563327Z", + "Type": "NORMAL", + "Volumes": [ + { + "Source": "/var/lib/ecs/deps/execute-command/config/seelog-gEZ-TIvHAyOLfMC5wiWRofgDMlDzaCZ6zcswnAoop84=.xml", + "Destination": "/ecs-execute-command-36dfb910-8e80-47b9-8b3a-12c7308123b2/configuration/seelog.xml" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/certs/tls-ca-bundle.pem", + "Destination": "/ecs-execute-command-36dfb910-8e80-47b9-8b3a-12c7308123b2/certs/amazon-ssm-agent.crt" + }, + { + "Source": "/var/log/ecs/exec/506f22fab0414cde856201584703fed9/ecs-exporter", + "Destination": "/var/log/amazon/ssm" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/amazon-ssm-agent", + "Destination": "/ecs-execute-command-36dfb910-8e80-47b9-8b3a-12c7308123b2/amazon-ssm-agent" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/ssm-agent-worker", + "Destination": "/ecs-execute-command-36dfb910-8e80-47b9-8b3a-12c7308123b2/ssm-agent-worker" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/ssm-session-worker", + "Destination": "/ecs-execute-command-36dfb910-8e80-47b9-8b3a-12c7308123b2/ssm-session-worker" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/config/amazon-ssm-agent-Orvj12YkCf4DKDu1cHTOVj7smDviWx1T4Kg3Q_IdNYA=.json", + "Destination": "/ecs-execute-command-36dfb910-8e80-47b9-8b3a-12c7308123b2/configuration/amazon-ssm-agent.json" + } + ], + "LogDriver": "awslogs", + "LogOptions": { + "awslogs-group": "EcsExporterCdkStack-promecsexportersandboxmainec2taskdefinitionpromecsexportersandboxmainec2ecsexporterLogGroup874A22EF-y3iGqSSTf3sz", + "awslogs-region": "us-east-1", + "awslogs-stream": "ecs-exporter/ecs-exporter/506f22fab0414cde856201584703fed9" + }, + "ContainerARN": "arn:aws:ecs:us-east-1:829490980523:container/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9/5fba1957-462a-48b2-9295-8602b69e00be", + "Networks": [ + { + "NetworkMode": "bridge", + "IPv4Addresses": [ + "172.17.0.2" + ] + } + ] + }, + { + "DockerId": "6b80adab0733f579594eccae31e5b0056b9544b805450ad6e278fed7f5e1c5ba", + "Name": "prometheus", + "DockerName": "ecs-prom-ecs-exporter-sandbox-main-ec2-13-prometheus-86f1e9bab7a8e9a65400", + "Image": "prom/prometheus:v3.1.0", + "ImageID": "sha256:f3d60e89ba2d4a402d1c62dccdab300f81579355e0744670c55b9ba282f3b56d", + "Labels": { + "com.amazonaws.ecs.cluster": "prom-ecs-exporter-sandbox", + "com.amazonaws.ecs.container-name": "prometheus", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9", + "com.amazonaws.ecs.task-definition-family": "prom-ecs-exporter-sandbox-main-ec2", + "com.amazonaws.ecs.task-definition-version": "13" + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": { + "CPU": 128, + "Memory": 256 + }, + "CreatedAt": "2025-02-27T05:10:01.22383376Z", + "StartedAt": "2025-02-27T05:10:02.730952683Z", + "Type": "NORMAL", + "Volumes": [ + { + "DockerName": "b4c23c0b1e1cea0ddfeab13122e911c7f52eb67720d3ffb43adba63b817e6a1e", + "Source": "/var/lib/docker/volumes/b4c23c0b1e1cea0ddfeab13122e911c7f52eb67720d3ffb43adba63b817e6a1e/_data", + "Destination": "/prometheus" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/amazon-ssm-agent", + "Destination": "/ecs-execute-command-12f856a9-3af4-4de7-ab82-671147b2a114/amazon-ssm-agent" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/ssm-agent-worker", + "Destination": "/ecs-execute-command-12f856a9-3af4-4de7-ab82-671147b2a114/ssm-agent-worker" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/ssm-session-worker", + "Destination": "/ecs-execute-command-12f856a9-3af4-4de7-ab82-671147b2a114/ssm-session-worker" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/config/amazon-ssm-agent-Orvj12YkCf4DKDu1cHTOVj7smDviWx1T4Kg3Q_IdNYA=.json", + "Destination": "/ecs-execute-command-12f856a9-3af4-4de7-ab82-671147b2a114/configuration/amazon-ssm-agent.json" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/config/seelog-gEZ-TIvHAyOLfMC5wiWRofgDMlDzaCZ6zcswnAoop84=.xml", + "Destination": "/ecs-execute-command-12f856a9-3af4-4de7-ab82-671147b2a114/configuration/seelog.xml" + }, + { + "Source": "/var/lib/ecs/deps/execute-command/certs/tls-ca-bundle.pem", + "Destination": "/ecs-execute-command-12f856a9-3af4-4de7-ab82-671147b2a114/certs/amazon-ssm-agent.crt" + }, + { + "Source": "/var/log/ecs/exec/506f22fab0414cde856201584703fed9/prometheus", + "Destination": "/var/log/amazon/ssm" + } + ], + "ContainerARN": "arn:aws:ecs:us-east-1:829490980523:container/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9/7ae35d49-867b-468f-afef-916925db8dca", + "Networks": [ + { + "NetworkMode": "bridge", + "IPv4Addresses": [ + "172.17.0.3" + ] + } + ] + } + ], + "VPCID": "vpc-0839c743edb0c009e", + "ServiceName": "prom-ecs-exporter-sandbox-main-ec2" +} diff --git a/ecscollector/testdata/fixtures/ec2_task_stats.json b/ecscollector/testdata/fixtures/ec2_task_stats.json new file mode 100644 index 0000000..4d61c40 --- /dev/null +++ b/ecscollector/testdata/fixtures/ec2_task_stats.json @@ -0,0 +1,235 @@ +{ + "01cf1f3208005cda71d5ac936ded65d2ecc0a8cc8ff8a82d2e00410bf4fbbd6d": { + "read": "2025-02-27T05:18:07.856950259Z", + "preread": "2025-02-27T05:18:06.853724878Z", + "pids_stats": { + "current": 33, + "limit": 404 + }, + "blkio_stats": { + "io_service_bytes_recursive": [ + { + "major": 259, + "minor": 0, + "op": "read", + "value": 60960768 + }, + { + "major": 259, + "minor": 0, + "op": "write", + "value": 57344 + } + ], + "io_serviced_recursive": null, + "io_queue_recursive": null, + "io_service_time_recursive": null, + "io_wait_time_recursive": null, + "io_merged_recursive": null, + "io_time_recursive": null, + "sectors_recursive": null + }, + "num_procs": 0, + "storage_stats": {}, + "cpu_stats": { + "cpu_usage": { + "total_usage": 331125000, + "usage_in_kernelmode": 66278000, + "usage_in_usermode": 264846000 + }, + "system_cpu_usage": 1045600000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "precpu_stats": { + "cpu_usage": { + "total_usage": 325963000, + "usage_in_kernelmode": 65245000, + "usage_in_usermode": 260717000 + }, + "system_cpu_usage": 1043620000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "memory_stats": { + "usage": 65249280, + "stats": { + "active_anon": 9170944, + "active_file": 7655424, + "anon": 37691392, + "anon_thp": 0, + "file": 25296896, + "file_dirty": 4096, + "file_mapped": 19968000, + "file_writeback": 0, + "inactive_anon": 28520448, + "inactive_file": 17641472, + "kernel_stack": 524288, + "pgactivate": 6895, + "pgdeactivate": 4638, + "pgfault": 17631, + "pglazyfree": 0, + "pglazyfreed": 0, + "pgmajfault": 1120, + "pgrefill": 6128, + "pgscan": 24707, + "pgsteal": 10033, + "shmem": 0, + "slab": 908520, + "slab_reclaimable": 383648, + "slab_unreclaimable": 524872, + "sock": 0, + "thp_collapse_alloc": 0, + "thp_fault_alloc": 0, + "unevictable": 0, + "workingset_activate": 0, + "workingset_nodereclaim": 0, + "workingset_refault": 0 + }, + "limit": 268435456 + }, + "name": "/ecs-prom-ecs-exporter-sandbox-main-ec2-13-ecs-exporter-e2aeb1e6be8998c72300", + "id": "01cf1f3208005cda71d5ac936ded65d2ecc0a8cc8ff8a82d2e00410bf4fbbd6d", + "networks": { + "eth0": { + "rx_bytes": 101869, + "rx_packets": 284, + "rx_errors": 0, + "rx_dropped": 0, + "tx_bytes": 43958, + "tx_packets": 278, + "tx_errors": 0, + "tx_dropped": 0 + } + }, + "network_rate_stats": { + "rx_bytes_per_sec": 2764.084716796875, + "tx_bytes_per_sec": 3390.065673828125 + } + }, + "213e1203f4bb72af185724d937e698d2724acf35b57ec2dd5f3c963adbd2d38c": {}, + "6b80adab0733f579594eccae31e5b0056b9544b805450ad6e278fed7f5e1c5ba": { + "read": "2025-02-27T05:18:07.855390957Z", + "preread": "2025-02-27T05:18:06.852213453Z", + "pids_stats": { + "current": 25, + "limit": 404 + }, + "blkio_stats": { + "io_service_bytes_recursive": [ + { + "major": 259, + "minor": 0, + "op": "read", + "value": 99926016 + }, + { + "major": 259, + "minor": 0, + "op": "write", + "value": 274432 + } + ], + "io_serviced_recursive": null, + "io_queue_recursive": null, + "io_service_time_recursive": null, + "io_wait_time_recursive": null, + "io_merged_recursive": null, + "io_time_recursive": null, + "sectors_recursive": null + }, + "num_procs": 0, + "storage_stats": {}, + "cpu_stats": { + "cpu_usage": { + "total_usage": 566060000, + "usage_in_kernelmode": 164664000, + "usage_in_usermode": 401395000 + }, + "system_cpu_usage": 1045600000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "precpu_stats": { + "cpu_usage": { + "total_usage": 566060000, + "usage_in_kernelmode": 164664000, + "usage_in_usermode": 401395000 + }, + "system_cpu_usage": 1043620000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "memory_stats": { + "usage": 60981248, + "stats": { + "active_anon": 14704640, + "active_file": 8572928, + "anon": 39268352, + "anon_thp": 0, + "file": 19619840, + "file_dirty": 0, + "file_mapped": 17047552, + "file_writeback": 0, + "inactive_anon": 24563712, + "inactive_file": 11046912, + "kernel_stack": 393216, + "pgactivate": 13496, + "pgdeactivate": 11069, + "pgfault": 18768, + "pglazyfree": 0, + "pglazyfreed": 0, + "pgmajfault": 1133, + "pgrefill": 13249, + "pgscan": 48430, + "pgsteal": 20303, + "shmem": 0, + "slab": 929280, + "slab_reclaimable": 519208, + "slab_unreclaimable": 410072, + "sock": 0, + "thp_collapse_alloc": 0, + "thp_fault_alloc": 0, + "unevictable": 0, + "workingset_activate": 0, + "workingset_nodereclaim": 0, + "workingset_refault": 0 + }, + "limit": 268435456 + }, + "name": "/ecs-prom-ecs-exporter-sandbox-main-ec2-13-prometheus-86f1e9bab7a8e9a65400", + "id": "6b80adab0733f579594eccae31e5b0056b9544b805450ad6e278fed7f5e1c5ba", + "networks": { + "eth0": { + "rx_bytes": 45368, + "rx_packets": 132, + "rx_errors": 0, + "rx_dropped": 0, + "tx_bytes": 13532, + "tx_packets": 118, + "tx_errors": 0, + "tx_dropped": 0 + } + }, + "network_rate_stats": { + "rx_bytes_per_sec": 41.86697006225586, + "tx_bytes_per_sec": 41.86697006225586 + } + } +} diff --git a/ecscollector/testdata/fixtures/fargate_task_metadata.json b/ecscollector/testdata/fixtures/fargate_task_metadata.json new file mode 100644 index 0000000..f22c715 --- /dev/null +++ b/ecscollector/testdata/fixtures/fargate_task_metadata.json @@ -0,0 +1,176 @@ +{ + "Cluster": "arn:aws:ecs:us-east-1:829490980523:cluster/prom-ecs-exporter-sandbox", + "TaskARN": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/bae32def0ab64f06818e8862e58f8d6d", + "Family": "prom-ecs-exporter-sandbox-main-fargate", + "Revision": "9", + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": { + "CPU": 0.25, + "Memory": 512 + }, + "PullStartedAt": "2025-02-27T05:06:03.714437592Z", + "PullStoppedAt": "2025-02-27T05:06:18.599126545Z", + "AvailabilityZone": "us-east-1a", + "LaunchType": "FARGATE", + "Containers": [ + { + "DockerId": "bae32def0ab64f06818e8862e58f8d6d-1585788040", + "Name": "nonessential", + "DockerName": "nonessential", + "Image": "alpine", + "ImageID": "sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff4511df3ef88c", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-east-1:829490980523:cluster/prom-ecs-exporter-sandbox", + "com.amazonaws.ecs.container-name": "nonessential", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/bae32def0ab64f06818e8862e58f8d6d", + "com.amazonaws.ecs.task-definition-family": "prom-ecs-exporter-sandbox-main-fargate", + "com.amazonaws.ecs.task-definition-version": "9" + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "STOPPED", + "ExitCode": 0, + "Limits": { + "CPU": 2 + }, + "CreatedAt": "2025-02-27T05:06:19.200809191Z", + "StartedAt": "2025-02-27T05:06:19.200809191Z", + "FinishedAt": "2025-02-27T05:06:19.219711004Z", + "Type": "NORMAL", + "ContainerARN": "arn:aws:ecs:us-east-1:829490980523:container/prom-ecs-exporter-sandbox/bae32def0ab64f06818e8862e58f8d6d/cafd106c-dc95-4396-a466-a894961efa50", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": [ + "10.0.117.145" + ], + "IPv6Addresses": [ + "2600:1f18:4ae8:400:7ca9:f2:a4c:8285" + ], + "AttachmentIndex": 0, + "MACAddress": "0a:ff:e5:34:fa:c9", + "IPv4SubnetCIDRBlock": "10.0.0.0/17", + "IPv6SubnetCIDRBlock": "2600:1f18:4ae8:400::/64", + "DomainNameServers": [ + "10.0.0.2" + ], + "DomainNameSearchList": [ + "ec2.internal" + ], + "PrivateDNSName": "ip-10-0-117-145.ec2.internal", + "SubnetGatewayIpv4Address": "10.0.0.1/17" + } + ], + "Snapshotter": "overlayfs" + }, + { + "DockerId": "bae32def0ab64f06818e8862e58f8d6d-1819985369", + "Name": "prometheus", + "DockerName": "prometheus", + "Image": "prom/prometheus:v3.1.0", + "ImageID": "sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-east-1:829490980523:cluster/prom-ecs-exporter-sandbox", + "com.amazonaws.ecs.container-name": "prometheus", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/bae32def0ab64f06818e8862e58f8d6d", + "com.amazonaws.ecs.task-definition-family": "prom-ecs-exporter-sandbox-main-fargate", + "com.amazonaws.ecs.task-definition-version": "9" + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": { + "CPU": 2 + }, + "CreatedAt": "2025-02-27T05:06:19.179996393Z", + "StartedAt": "2025-02-27T05:06:19.179996393Z", + "Type": "NORMAL", + "ContainerARN": "arn:aws:ecs:us-east-1:829490980523:container/prom-ecs-exporter-sandbox/bae32def0ab64f06818e8862e58f8d6d/50e269e1-4232-4aed-8bf4-29c4909858f9", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": [ + "10.0.117.145" + ], + "IPv6Addresses": [ + "2600:1f18:4ae8:400:7ca9:f2:a4c:8285" + ], + "AttachmentIndex": 0, + "MACAddress": "0a:ff:e5:34:fa:c9", + "IPv4SubnetCIDRBlock": "10.0.0.0/17", + "IPv6SubnetCIDRBlock": "2600:1f18:4ae8:400::/64", + "DomainNameServers": [ + "10.0.0.2" + ], + "DomainNameSearchList": [ + "ec2.internal" + ], + "PrivateDNSName": "ip-10-0-117-145.ec2.internal", + "SubnetGatewayIpv4Address": "10.0.0.1/17" + } + ], + "Snapshotter": "overlayfs" + }, + { + "DockerId": "bae32def0ab64f06818e8862e58f8d6d-4159844948", + "Name": "ecs-exporter", + "DockerName": "ecs-exporter", + "Image": "quay.io/prometheuscommunity/ecs-exporter:main", + "ImageID": "sha256:d1802fb18cb208eda88d4b23aeff903e72c091c20fcdf02596d6bec4679f676d", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-east-1:829490980523:cluster/prom-ecs-exporter-sandbox", + "com.amazonaws.ecs.container-name": "ecs-exporter", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/bae32def0ab64f06818e8862e58f8d6d", + "com.amazonaws.ecs.task-definition-family": "prom-ecs-exporter-sandbox-main-fargate", + "com.amazonaws.ecs.task-definition-version": "9" + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": { + "CPU": 2 + }, + "CreatedAt": "2025-02-27T05:06:19.394790335Z", + "StartedAt": "2025-02-27T05:06:19.394790335Z", + "Type": "NORMAL", + "LogDriver": "awslogs", + "LogOptions": { + "awslogs-group": "EcsExporterCdkStack-promecsexportersandboxmainfargatetaskdefinitionpromecsexportersandboxmainfargateecsexporterLogGroup44D32D35-DcG8HDbOu1Sl", + "awslogs-region": "us-east-1", + "awslogs-stream": "ecs-exporter/ecs-exporter/bae32def0ab64f06818e8862e58f8d6d" + }, + "ContainerARN": "arn:aws:ecs:us-east-1:829490980523:container/prom-ecs-exporter-sandbox/bae32def0ab64f06818e8862e58f8d6d/a9b9d903-4ca1-4ce2-8138-93094e438c6b", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": [ + "10.0.117.145" + ], + "IPv6Addresses": [ + "2600:1f18:4ae8:400:7ca9:f2:a4c:8285" + ], + "AttachmentIndex": 0, + "MACAddress": "0a:ff:e5:34:fa:c9", + "IPv4SubnetCIDRBlock": "10.0.0.0/17", + "IPv6SubnetCIDRBlock": "2600:1f18:4ae8:400::/64", + "DomainNameServers": [ + "10.0.0.2" + ], + "DomainNameSearchList": [ + "ec2.internal" + ], + "PrivateDNSName": "ip-10-0-117-145.ec2.internal", + "SubnetGatewayIpv4Address": "10.0.0.1/17" + } + ], + "Snapshotter": "overlayfs" + } + ], + "ClockDrift": { + "ClockErrorBound": 0.33292849999999996, + "ReferenceTimestamp": "2025-02-27T05:22:43Z", + "ClockSynchronizationStatus": "SYNCHRONIZED" + }, + "EphemeralStorageMetrics": { + "Utilized": 427, + "Reserved": 20496 + } +} diff --git a/ecscollector/testdata/fixtures/fargate_task_stats.json b/ecscollector/testdata/fixtures/fargate_task_stats.json new file mode 100644 index 0000000..8892403 --- /dev/null +++ b/ecscollector/testdata/fixtures/fargate_task_stats.json @@ -0,0 +1,515 @@ +{ + "bae32def0ab64f06818e8862e58f8d6d-1585788040": null, + "bae32def0ab64f06818e8862e58f8d6d-1819985369": { + "read": "2025-02-27T05:22:49.186680233Z", + "preread": "2025-02-27T05:22:39.185738849Z", + "pids_stats": {}, + "blkio_stats": { + "io_service_bytes_recursive": [ + { + "major": 259, + "minor": 1, + "op": "Read", + "value": 23793664 + }, + { + "major": 259, + "minor": 1, + "op": "Write", + "value": 0 + }, + { + "major": 259, + "minor": 1, + "op": "Sync", + "value": 23793664 + }, + { + "major": 259, + "minor": 1, + "op": "Async", + "value": 0 + }, + { + "major": 259, + "minor": 1, + "op": "Discard", + "value": 0 + }, + { + "major": 259, + "minor": 1, + "op": "Total", + "value": 23793664 + }, + { + "major": 259, + "minor": 0, + "op": "Read", + "value": 65409024 + }, + { + "major": 259, + "minor": 0, + "op": "Write", + "value": 0 + }, + { + "major": 259, + "minor": 0, + "op": "Sync", + "value": 65409024 + }, + { + "major": 259, + "minor": 0, + "op": "Async", + "value": 0 + }, + { + "major": 259, + "minor": 0, + "op": "Discard", + "value": 0 + }, + { + "major": 259, + "minor": 0, + "op": "Total", + "value": 65409024 + } + ], + "io_serviced_recursive": [ + { + "major": 259, + "minor": 1, + "op": "Read", + "value": 295 + }, + { + "major": 259, + "minor": 1, + "op": "Write", + "value": 0 + }, + { + "major": 259, + "minor": 1, + "op": "Sync", + "value": 295 + }, + { + "major": 259, + "minor": 1, + "op": "Async", + "value": 0 + }, + { + "major": 259, + "minor": 1, + "op": "Discard", + "value": 0 + }, + { + "major": 259, + "minor": 1, + "op": "Total", + "value": 295 + }, + { + "major": 259, + "minor": 0, + "op": "Read", + "value": 682 + }, + { + "major": 259, + "minor": 0, + "op": "Write", + "value": 0 + }, + { + "major": 259, + "minor": 0, + "op": "Sync", + "value": 682 + }, + { + "major": 259, + "minor": 0, + "op": "Async", + "value": 0 + }, + { + "major": 259, + "minor": 0, + "op": "Discard", + "value": 0 + }, + { + "major": 259, + "minor": 0, + "op": "Total", + "value": 682 + } + ], + "io_queue_recursive": [], + "io_service_time_recursive": [], + "io_wait_time_recursive": [], + "io_merged_recursive": [], + "io_time_recursive": [], + "sectors_recursive": [] + }, + "num_procs": 0, + "storage_stats": {}, + "cpu_stats": { + "cpu_usage": { + "total_usage": 932439492, + "percpu_usage": [ + 484165457, + 448274035 + ], + "usage_in_kernelmode": 120000000, + "usage_in_usermode": 880000000 + }, + "system_cpu_usage": 2121200000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "precpu_stats": { + "cpu_usage": { + "total_usage": 925246189, + "percpu_usage": [ + 479268399, + 445977790 + ], + "usage_in_kernelmode": 120000000, + "usage_in_usermode": 870000000 + }, + "system_cpu_usage": 2101320000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "memory_stats": { + "usage": 127934464, + "max_usage": 128557056, + "stats": { + "active_anon": 0, + "active_file": 32952320, + "cache": 85426176, + "dirty": 0, + "hierarchical_memory_limit": 536870912, + "hierarchical_memsw_limit": 9223372036854771712, + "inactive_anon": 40820736, + "inactive_file": 52383744, + "mapped_file": 72179712, + "pgfault": 18975, + "pgmajfault": 693, + "pgpgin": 34749, + "pgpgout": 3936, + "rss": 40820736, + "rss_huge": 0, + "total_active_anon": 0, + "total_active_file": 32952320, + "total_cache": 85426176, + "total_dirty": 0, + "total_inactive_anon": 40820736, + "total_inactive_file": 52383744, + "total_mapped_file": 72179712, + "total_pgfault": 18975, + "total_pgmajfault": 693, + "total_pgpgin": 34749, + "total_pgpgout": 3936, + "total_rss": 40820736, + "total_rss_huge": 0, + "total_unevictable": 0, + "total_writeback": 0, + "unevictable": 0, + "writeback": 0 + }, + "limit": 9223372036854771712 + }, + "name": "prometheus", + "id": "bae32def0ab64f06818e8862e58f8d6d-1819985369", + "networks": { + "eth1": { + "rx_bytes": 129045374, + "rx_packets": 88933, + "rx_errors": 0, + "rx_dropped": 0, + "tx_bytes": 347839, + "tx_packets": 3501, + "tx_errors": 0, + "tx_dropped": 0 + } + }, + "network_rate_stats": { + "rx_bytes_per_sec": 2464.6679803462657, + "tx_bytes_per_sec": 1173.289548516495 + } + }, + "bae32def0ab64f06818e8862e58f8d6d-4159844948": { + "read": "2025-02-27T05:22:49.406170373Z", + "preread": "2025-02-27T05:22:39.406479661Z", + "pids_stats": {}, + "blkio_stats": { + "io_service_bytes_recursive": [ + { + "major": 259, + "minor": 1, + "op": "Read", + "value": 28639232 + }, + { + "major": 259, + "minor": 1, + "op": "Write", + "value": 0 + }, + { + "major": 259, + "minor": 1, + "op": "Sync", + "value": 28639232 + }, + { + "major": 259, + "minor": 1, + "op": "Async", + "value": 0 + }, + { + "major": 259, + "minor": 1, + "op": "Discard", + "value": 0 + }, + { + "major": 259, + "minor": 1, + "op": "Total", + "value": 28639232 + }, + { + "major": 259, + "minor": 0, + "op": "Read", + "value": 14655488 + }, + { + "major": 259, + "minor": 0, + "op": "Write", + "value": 4096 + }, + { + "major": 259, + "minor": 0, + "op": "Sync", + "value": 14655488 + }, + { + "major": 259, + "minor": 0, + "op": "Async", + "value": 4096 + }, + { + "major": 259, + "minor": 0, + "op": "Discard", + "value": 0 + }, + { + "major": 259, + "minor": 0, + "op": "Total", + "value": 14659584 + } + ], + "io_serviced_recursive": [ + { + "major": 259, + "minor": 1, + "op": "Read", + "value": 327 + }, + { + "major": 259, + "minor": 1, + "op": "Write", + "value": 0 + }, + { + "major": 259, + "minor": 1, + "op": "Sync", + "value": 327 + }, + { + "major": 259, + "minor": 1, + "op": "Async", + "value": 0 + }, + { + "major": 259, + "minor": 1, + "op": "Discard", + "value": 0 + }, + { + "major": 259, + "minor": 1, + "op": "Total", + "value": 327 + }, + { + "major": 259, + "minor": 0, + "op": "Read", + "value": 157 + }, + { + "major": 259, + "minor": 0, + "op": "Write", + "value": 1 + }, + { + "major": 259, + "minor": 0, + "op": "Sync", + "value": 157 + }, + { + "major": 259, + "minor": 0, + "op": "Async", + "value": 1 + }, + { + "major": 259, + "minor": 0, + "op": "Discard", + "value": 0 + }, + { + "major": 259, + "minor": 0, + "op": "Total", + "value": 158 + } + ], + "io_queue_recursive": [], + "io_service_time_recursive": [], + "io_wait_time_recursive": [], + "io_merged_recursive": [], + "io_time_recursive": [], + "sectors_recursive": [] + }, + "num_procs": 0, + "storage_stats": {}, + "cpu_stats": { + "cpu_usage": { + "total_usage": 322633383, + "percpu_usage": [ + 138347736, + 184285647 + ], + "usage_in_kernelmode": 50000000, + "usage_in_usermode": 180000000 + }, + "system_cpu_usage": 2121630000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "precpu_stats": { + "cpu_usage": { + "total_usage": 242770940, + "percpu_usage": [ + 104627034, + 138143906 + ], + "usage_in_kernelmode": 30000000, + "usage_in_usermode": 140000000 + }, + "system_cpu_usage": 2101780000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "memory_stats": { + "usage": 84111360, + "max_usage": 84238336, + "stats": { + "active_anon": 0, + "active_file": 3379200, + "cache": 42442752, + "dirty": 0, + "hierarchical_memory_limit": 536870912, + "hierarchical_memsw_limit": 9223372036854771712, + "inactive_anon": 39469056, + "inactive_file": 39100416, + "mapped_file": 33927168, + "pgfault": 17358, + "pgmajfault": 297, + "pgpgin": 23430, + "pgpgout": 3367, + "rss": 39469056, + "rss_huge": 0, + "total_active_anon": 0, + "total_active_file": 3379200, + "total_cache": 42442752, + "total_dirty": 0, + "total_inactive_anon": 39469056, + "total_inactive_file": 39100416, + "total_mapped_file": 33927168, + "total_pgfault": 17358, + "total_pgmajfault": 297, + "total_pgpgin": 23430, + "total_pgpgout": 3367, + "total_rss": 39469056, + "total_rss_huge": 0, + "total_unevictable": 0, + "total_writeback": 0, + "unevictable": 0, + "writeback": 0 + }, + "limit": 9223372036854771712 + }, + "name": "ecs-exporter", + "id": "bae32def0ab64f06818e8862e58f8d6d-4159844948", + "networks": { + "eth1": { + "rx_bytes": 129046293, + "rx_packets": 88938, + "rx_errors": 0, + "rx_dropped": 0, + "tx_bytes": 348223, + "tx_packets": 3507, + "tx_errors": 0, + "tx_dropped": 0 + } + }, + "network_rate_stats": { + "rx_bytes_per_sec": 2556.879064581499, + "tx_bytes_per_sec": 1211.8374728018853 + } + } +} diff --git a/ecscollector/testdata/snapshots/ec2_metrics.txt b/ecscollector/testdata/snapshots/ec2_metrics.txt new file mode 100644 index 0000000..7674fff --- /dev/null +++ b/ecscollector/testdata/snapshots/ec2_metrics.txt @@ -0,0 +1,49 @@ +# HELP ecs_container_cpu_usage_seconds_total Cumulative total container CPU usage in seconds. +# TYPE ecs_container_cpu_usage_seconds_total counter +ecs_container_cpu_usage_seconds_total{container_name="ecs-exporter"} 0.331125 +ecs_container_cpu_usage_seconds_total{container_name="prometheus"} 0.56606 +# HELP ecs_container_memory_limit_bytes Configured container memory limit in bytes, set from the container-level limit in the task definition if any, otherwise the task-level limit. +# TYPE ecs_container_memory_limit_bytes gauge +ecs_container_memory_limit_bytes{container_name="ecs-exporter"} 2.68435456e+08 +ecs_container_memory_limit_bytes{container_name="prometheus"} 2.68435456e+08 +# HELP ecs_container_memory_page_cache_size_bytes Current container memory page cache size in bytes. This is not a subset of used bytes. +# TYPE ecs_container_memory_page_cache_size_bytes gauge +ecs_container_memory_page_cache_size_bytes{container_name="ecs-exporter"} 0 +ecs_container_memory_page_cache_size_bytes{container_name="prometheus"} 0 +# HELP ecs_container_memory_usage_bytes Current container memory usage in bytes. +# TYPE ecs_container_memory_usage_bytes gauge +ecs_container_memory_usage_bytes{container_name="ecs-exporter"} 6.524928e+07 +ecs_container_memory_usage_bytes{container_name="prometheus"} 6.0981248e+07 +# HELP ecs_network_receive_bytes_total Cumulative total size of network packets received in bytes. +# TYPE ecs_network_receive_bytes_total counter +ecs_network_receive_bytes_total{interface="eth0"} 45368 +# HELP ecs_network_receive_errors_total Cumulative total count of network errors in receiving. +# TYPE ecs_network_receive_errors_total counter +ecs_network_receive_errors_total{interface="eth0"} 0 +# HELP ecs_network_receive_packets_dropped_total Cumulative total count of network packets dropped in receiving. +# TYPE ecs_network_receive_packets_dropped_total counter +ecs_network_receive_packets_dropped_total{interface="eth0"} 0 +# HELP ecs_network_receive_packets_total Cumulative total count of network packets received. +# TYPE ecs_network_receive_packets_total counter +ecs_network_receive_packets_total{interface="eth0"} 132 +# HELP ecs_network_transmit_bytes_total Cumulative total size of network packets transmitted in bytes. +# TYPE ecs_network_transmit_bytes_total counter +ecs_network_transmit_bytes_total{interface="eth0"} 13532 +# HELP ecs_network_transmit_errors_total Cumulative total count of network errors in transmit. +# TYPE ecs_network_transmit_errors_total counter +ecs_network_transmit_errors_total{interface="eth0"} 0 +# HELP ecs_network_transmit_packets_dropped_total Cumulative total count of network packets dropped in transmit. +# TYPE ecs_network_transmit_packets_dropped_total counter +ecs_network_transmit_packets_dropped_total{interface="eth0"} 0 +# HELP ecs_network_transmit_packets_total Cumulative total count of network packets transmitted. +# TYPE ecs_network_transmit_packets_total counter +ecs_network_transmit_packets_total{interface="eth0"} 118 +# HELP ecs_task_image_pull_start_timestamp_seconds The time at which the task started pulling docker images for its containers. +# TYPE ecs_task_image_pull_start_timestamp_seconds gauge +ecs_task_image_pull_start_timestamp_seconds 1.7406329923325953e+09 +# HELP ecs_task_image_pull_stop_timestamp_seconds The time at which the task stopped (i.e. completed) pulling docker images for its containers. +# TYPE ecs_task_image_pull_stop_timestamp_seconds gauge +ecs_task_image_pull_stop_timestamp_seconds 1.7406330012060723e+09 +# HELP ecs_task_metadata_info ECS task metadata, sourced from the task metadata endpoint version 4. +# TYPE ecs_task_metadata_info gauge +ecs_task_metadata_info{availability_zone="us-east-1a",cluster="prom-ecs-exporter-sandbox",desired_status="RUNNING",family="prom-ecs-exporter-sandbox-main-ec2",known_status="RUNNING",launch_type="EC2",revision="13",task_arn="arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9"} 1 diff --git a/ecscollector/testdata/snapshots/fargate_metrics.txt b/ecscollector/testdata/snapshots/fargate_metrics.txt new file mode 100644 index 0000000..83cf0f0 --- /dev/null +++ b/ecscollector/testdata/snapshots/fargate_metrics.txt @@ -0,0 +1,61 @@ +# HELP ecs_container_cpu_usage_seconds_total Cumulative total container CPU usage in seconds. +# TYPE ecs_container_cpu_usage_seconds_total counter +ecs_container_cpu_usage_seconds_total{container_name="ecs-exporter"} 0.322633383 +ecs_container_cpu_usage_seconds_total{container_name="prometheus"} 0.9324394920000001 +# HELP ecs_container_memory_limit_bytes Configured container memory limit in bytes, set from the container-level limit in the task definition if any, otherwise the task-level limit. +# TYPE ecs_container_memory_limit_bytes gauge +ecs_container_memory_limit_bytes{container_name="ecs-exporter"} 5.36870912e+08 +ecs_container_memory_limit_bytes{container_name="prometheus"} 5.36870912e+08 +# HELP ecs_container_memory_page_cache_size_bytes Current container memory page cache size in bytes. This is not a subset of used bytes. +# TYPE ecs_container_memory_page_cache_size_bytes gauge +ecs_container_memory_page_cache_size_bytes{container_name="ecs-exporter"} 4.2442752e+07 +ecs_container_memory_page_cache_size_bytes{container_name="prometheus"} 8.5426176e+07 +# HELP ecs_container_memory_usage_bytes Current container memory usage in bytes. +# TYPE ecs_container_memory_usage_bytes gauge +ecs_container_memory_usage_bytes{container_name="ecs-exporter"} 8.411136e+07 +ecs_container_memory_usage_bytes{container_name="prometheus"} 1.27934464e+08 +# HELP ecs_network_receive_bytes_total Cumulative total size of network packets received in bytes. +# TYPE ecs_network_receive_bytes_total counter +ecs_network_receive_bytes_total{interface="eth1"} 1.29046293e+08 +# HELP ecs_network_receive_errors_total Cumulative total count of network errors in receiving. +# TYPE ecs_network_receive_errors_total counter +ecs_network_receive_errors_total{interface="eth1"} 0 +# HELP ecs_network_receive_packets_dropped_total Cumulative total count of network packets dropped in receiving. +# TYPE ecs_network_receive_packets_dropped_total counter +ecs_network_receive_packets_dropped_total{interface="eth1"} 0 +# HELP ecs_network_receive_packets_total Cumulative total count of network packets received. +# TYPE ecs_network_receive_packets_total counter +ecs_network_receive_packets_total{interface="eth1"} 88938 +# HELP ecs_network_transmit_bytes_total Cumulative total size of network packets transmitted in bytes. +# TYPE ecs_network_transmit_bytes_total counter +ecs_network_transmit_bytes_total{interface="eth1"} 348223 +# HELP ecs_network_transmit_errors_total Cumulative total count of network errors in transmit. +# TYPE ecs_network_transmit_errors_total counter +ecs_network_transmit_errors_total{interface="eth1"} 0 +# HELP ecs_network_transmit_packets_dropped_total Cumulative total count of network packets dropped in transmit. +# TYPE ecs_network_transmit_packets_dropped_total counter +ecs_network_transmit_packets_dropped_total{interface="eth1"} 0 +# HELP ecs_network_transmit_packets_total Cumulative total count of network packets transmitted. +# TYPE ecs_network_transmit_packets_total counter +ecs_network_transmit_packets_total{interface="eth1"} 3507 +# HELP ecs_task_cpu_limit_vcpus Configured task CPU limit in vCPUs (1 vCPU = 1024 CPU units). This is optional when running on EC2; if no limit is set, this metric has no value. +# TYPE ecs_task_cpu_limit_vcpus gauge +ecs_task_cpu_limit_vcpus 0.25 +# HELP ecs_task_ephemeral_storage_allocated_bytes Configured Fargate task ephemeral storage allocated size in bytes. +# TYPE ecs_task_ephemeral_storage_allocated_bytes gauge +ecs_task_ephemeral_storage_allocated_bytes 2.1491613696e+10 +# HELP ecs_task_ephemeral_storage_used_bytes Current Fargate task ephemeral storage usage in bytes. +# TYPE ecs_task_ephemeral_storage_used_bytes gauge +ecs_task_ephemeral_storage_used_bytes 4.47741952e+08 +# HELP ecs_task_image_pull_start_timestamp_seconds The time at which the task started pulling docker images for its containers. +# TYPE ecs_task_image_pull_start_timestamp_seconds gauge +ecs_task_image_pull_start_timestamp_seconds 1.7406327637144377e+09 +# HELP ecs_task_image_pull_stop_timestamp_seconds The time at which the task stopped (i.e. completed) pulling docker images for its containers. +# TYPE ecs_task_image_pull_stop_timestamp_seconds gauge +ecs_task_image_pull_stop_timestamp_seconds 1.7406327785991266e+09 +# HELP ecs_task_memory_limit_bytes Configured task memory limit in bytes. This is optional when running on EC2; if no limit is set, this metric has no value. +# TYPE ecs_task_memory_limit_bytes gauge +ecs_task_memory_limit_bytes 5.36870912e+08 +# HELP ecs_task_metadata_info ECS task metadata, sourced from the task metadata endpoint version 4. +# TYPE ecs_task_metadata_info gauge +ecs_task_metadata_info{availability_zone="us-east-1a",cluster="arn:aws:ecs:us-east-1:829490980523:cluster/prom-ecs-exporter-sandbox",desired_status="RUNNING",family="prom-ecs-exporter-sandbox-main-fargate",known_status="RUNNING",launch_type="FARGATE",revision="9",task_arn="arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/bae32def0ab64f06818e8862e58f8d6d"} 1 diff --git a/go.mod b/go.mod index bd75873..54655f2 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/klauspost/compress v1.17.9 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mdlayher/socket v0.4.1 // indirect github.com/mdlayher/vsock v1.2.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect