labels-scrape.alloy

/*
Module Components: labels_scrape
Description: Scrapes targets for metrics based on Docker Containers labels

Note: Every argument except for "forward_to" is optional, and does have a defined default value.  However, the values for these
      arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... ").
      This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will
      does not override the value passed in, where coalesce() will return the first non-null value.


Following labels are available:

  metrics.grafana.com/scrape: true

the default scraping scheme is http, only support http now.

  metrics.grafana.com/scheme: http
or
  prometheus.io/scheme: http

the default path to scrape is /metrics, this can be specified as a single value which would override, the scrape path being used
for all ports attached to the target:

  metrics.grafana.com/path: /metrics/some_path

the default port to scrape is the target port, this can be specified as a single value which would override the scrape port being
used for all ports attached to the target, note that even if an target had multiple targets, the relabel_config targets are
deduped before scraping:

  metrics.grafana.com/port: 8080
or
  prometheus.io/port: 8080

the default interval to scrape is 15s, this can be specified as a single value which would override, the scrape interval being used
for all ports attached to the target:

  metrics.grafana.com/interval: 15s
or
  prometheus.io/interval: 15s


the default timeout for scraping is 10s, this can be specified as a single value which would override, the scrape interval being
used for all ports attached to the target:

  metrics.grafana.com/timeout: 10s
or
  prometheus.io/timeout: 10s

the default job is namespace/{{ service name }}  there may be a different job name is required because of a set of dashboards, rules,
etc. to support this there is a job annotation which will override the default value:

  metrics.grafana.com/job: integrations/kubernetes/kube-state-metrics
or
  prometheus.io/job: integrations/kubernetes/kube-state-metrics
*/

declare "labels_scrape" {

	/*****************************************************************
	* ARGUMENTS
	*****************************************************************/
	argument "forward_to" {
		comment = "Must be a list(MetricsReceiver) where collected metrics should be forwarded to"
	}

	argument "label_prefix" {
		// Docs: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
		// i.e. metrics.grafana.com, then again for prometheus.io
		comment  = "The label_prefix to use Auto-Scraping (default: metrics.grafana.com)"
		default  = "metrics.grafana.com"
		optional = true
	}

	argument "__sd_label" {
		optional = true
		comment  = "The logic is used to transform the label_prefix argument into a valid label name by removing unsupported characters."
		default  = string.replace(string.replace(string.replace(coalesce(argument.label_prefix.value, "metrics.grafana.com"), ".", "_"), "/", "_"), "-", "_")
	}

	argument "cluster" {
		optional = true
		default  = "docker-compose"
	}

	argument "namespace" {
		optional = true
		default  = "monitoring-system"
	}

	argument "tenant" {
		comment  = "The tenant to write metrics to.  This does not have to be the tenantId, this is the value to look for in the logs.grafana.com/tenant label, and this can be a regex. (default: (.*))"
		optional = true
		default  = "(.*)"
	}

	argument "keep_metrics" {
		comment  = "A regex of metrics to keep (default: (.+))"
		optional = true
		default  = "(.+)"
	}

	argument "drop_metrics" {
		comment  = "A regex of metrics to drop (default: \"\")"
		optional = true
		default  = ""
	}

	argument "scrape_interval" {
		comment  = "How often to scrape metrics from the targets (default: 60s)"
		optional = true
		default  = "60s"
	}

	argument "scrape_timeout" {
		comment  = "How long before a scrape times out (default: 10s)"
		optional = true
		default  = "10s"
	}

	/*****************************************************************
	* Targets From Docker Discovery
	*****************************************************************/
	discovery.docker "dd_metrics" {
		host             = "unix:///var/run/docker.sock"
		refresh_interval = "30s"

		filter {
			name   = "status"
			values = ["running"]
		}
	}

	/*****************************************************************
	* Discovery Relabelings (pre-scrape)
	*****************************************************************/
	discovery.relabel "dr_label_metrics" {
		targets = discovery.docker.dd_metrics.targets

		/****************************************************************************************************************
		* Handle Discovers From Docker Engine Containers Targets to Keep or Drop
		* https://grafana.com/docs/agent/latest/flow/reference/components/discovery.docker/#exported-fields
		****************************************************************************************************************/
		// allow resources to declare their metrics scraped or not
		// Example Annotation:
		//   metrics.grafana.com/scrape: false
		//
		// the label prometheus.io/service-monitor: "false" is a common label for headless services, when performing endpoint
		// service discovery, if there is both a load-balanced service and headless service, this can result in duplicate
		// scrapes if the name of the service is attached as a label.  any targets with this label or annotation set should be dropped
		rule {
			action        = "replace"
			source_labels = [
				"__meta_docker_container_label_" + argument.__sd_label.value + "_scrape",
				"__meta_docker_container_label_prometheus_io_scrape",
			]
			separator    = ";"
			regex        = "^(?:;*)?(true|false).*$"
			replacement  = "$1"
			target_label = "__tmp_scrape"
		}

		// drop any targets that have scrape: false
		rule {
			action        = "drop"
			source_labels = ["__tmp_scrape"]
			regex         = "false"
		}

		// allow resources to declare the protocol to use when collecting metrics, the default value is "http",
		// Example Annotation:
		//   metrics.grafana.com/scheme: http
		rule {
			action       = "replace"
			replacement  = "http"
			target_label = "__scheme__"
		}

		rule {
			action        = "replace"
			source_labels = [
				"__meta_docker_container_label_" + argument.__sd_label.value + "_scheme",
				"__meta_docker_container_label_prometheus_io_scheme",
			]
			separator    = ";"
			regex        = "^(?:;*)?(https?).*$"
			replacement  = "$1"
			target_label = "__scheme__"
		}

		// allow resources to declare their metrics the tenant their metrics should be sent to,
		// Example Annotation:
		//   metrics.grafana.com/tenant: primary
		//
		// Note: This does not necessarily have to be the actual tenantId, it can be a friendly name as well that is simply used
		//       to determine if the metrics should be gathered for the current tenant
		rule {
			action        = "keep"
			source_labels = [
				"__meta_docker_container_label_" + argument.__sd_label.value + "_tenant",
				"__meta_docker_container_label_prometheus_io_tenant",
			]
			regex = "^(" + argument.tenant.value + ")$"
		}

		rule {
			action        = "replace"
			source_labels = [
				"__meta_docker_container_label_" + argument.__sd_label.value + "_port",
				"__meta_docker_container_label_prometheus_io_port",
			]
			separator    = ";"
			regex        = "^(?:;*)?(\\d+).*$"
			replacement  = "$1"
			target_label = "__tmp_metrics_port"
		}

		// allow resources to declare the port to use when collecting metrics, the default value is the discovered port from
		// Example Annotation:
		//   metrics.grafana.com/port: 9090
		rule {
			action        = "replace"
			source_labels = [
				"__address__",
				"__tmp_metrics_port",
			]
			separator    = ";"
			regex        = "^([^:]+)(?::\\d+)?;(\\d+)$"
			replacement  = "$1:$2"
			target_label = "__address__"
		}

		// allow resources to declare their the path to use when collecting their metrics, the default value is "/metrics",
		// Example Annotation:
		//   metrics.grafana.com/path: /metrics/foo
		rule {
			action        = "replace"
			source_labels = [
				"__meta_docker_container_label_" + argument.__sd_label.value + "_path",
				"__meta_docker_container_label_prometheus_io_path",
			]
			separator    = ";"
			regex        = "^(?:;*)?([^;]+).*$"
			replacement  = "$1"
			target_label = "__metrics_path__"
		}

		// allow resources to declare how often their metrics should be collected, the default value is 15s,
		// the following duration formats are supported (s|m|ms|h|d):
		// Example Annotation:
		//   metrics.grafana.com/interval: 15s
		rule {
			action       = "replace"
			replacement  = argument.scrape_interval.value
			target_label = "__scrape_interval__"
		}

		rule {
			action        = "replace"
			source_labels = [
				"__meta_docker_container_label_" + argument.__sd_label.value + "_interval",
				"__meta_docker_container_label_prometheus_io_interval",
			]
			separator    = ";"
			regex        = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$"
			replacement  = "$1"
			target_label = "__scrape_interval__"
		}

		// allow resources to declare the timeout of the scrape request, the default value is 10s,
		// the following duration formats are supported (s|m|ms|h|d):
		// Example Annotation:
		//   metrics.grafana.com/timeout: 10s
		rule {
			action       = "replace"
			replacement  = argument.scrape_timeout.value
			target_label = "__scrape_timeout__"
		}

		rule {
			action        = "replace"
			source_labels = [
				"__meta_docker_container_label_" + argument.__sd_label.value + "_timeout",
				"__meta_docker_container_label_prometheus_io_timeout",
			]
			separator    = ";"
			regex        = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$"
			replacement  = "$1"
			target_label = "__scrape_timeout__"
		}

		/********************************************
		* Handle Setting Common Labels
		********************************************/

		// set the cluster label
		rule {
			action       = "replace"
			replacement  = argument.cluster.value
			target_label = "cluster"
		}

		// set the namespace label
		rule {
			action       = "replace"
			replacement  = argument.namespace.value
			target_label = "namespace"
		}

		// set a default job label to be the namespace/service_name
		rule {
			action        = "replace"
			source_labels = [
				"__meta_docker_container_label_com_docker_compose_service",
			]
			regex        = "^(?:;*)?([^;]+).*$"
			replacement  = argument.namespace.value + "/$1"
			target_label = "job"
		}

		// allow resources to declare their the job label value to use when collecting their metrics, the default value is "",
		// Example Annotation:
		//   metrics.grafana.com/job: integrations/kubernetes/cadvisor
		rule {
			action        = "replace"
			source_labels = [
				"__meta_docker_container_label_" + argument.__sd_label.value + "_job",
				"__meta_docker_container_label_prometheus_io_job",
			]
			separator    = ";"
			regex        = "^(?:;*)?([^;]+).*$"
			replacement  = "$1"
			target_label = "job"
		}

		rule {
			source_labels = ["__meta_docker_container_name"]
			regex         = "/(.*)"
			target_label  = "pod"
		}

		rule {
			source_labels = ["__meta_docker_container_name"]
			regex         = "/(.*)"
			target_label  = "container"
		}

		rule {
			action        = "replace"
			source_labels = [
				"__meta_docker_container_label_com_docker_compose_service",
			]
			regex        = "^(?:;*)?([^;]+).*$"
			replacement  = "$1"
			target_label = "app"
		}

		rule {
			action        = "replace"
			source_labels = [
				"__meta_docker_container_label_app",
			]
			regex        = "^(?:;*)?([^;]+).*$"
			replacement  = "$1"
			target_label = "app"
		}
	}

	// only keep http targets
	discovery.relabel "dr_keep_http_targets" {
		targets = discovery.relabel.dr_label_metrics.output

		rule {
			action        = "keep"
			source_labels = ["__scheme__"]
			regex         = "http"
		}
	}

	/*****************************************************************
	* Prometheus Scrape Labels Targets
	*****************************************************************/
	prometheus.scrape "pc_docker_metrics" {
		targets = array.concat(
			discovery.relabel.dr_keep_http_targets.output,
		)

		job_name = "label-metrics"

		scrape_classic_histograms = true

		clustering {
			enabled = true
		}

		forward_to = [prometheus.relabel.pr_docker_metrics.receiver]
	}

	/*****************************************************************
	* Prometheus Metric Relabelings (post-scrape)
	*****************************************************************/
	prometheus.relabel "pr_docker_metrics" {
		forward_to = argument.forward_to.value

		// perform generic relabeling using keep_metrics and drop_metrics
		// keep only metrics that match the keep_metrics regex
		rule {
			action        = "keep"
			source_labels = ["__name__"]
			regex         = argument.keep_metrics.value
		}

		// drop metrics that match the drop_metrics regex
		rule {
			action        = "drop"
			source_labels = ["__name__"]
			regex         = argument.drop_metrics.value
		}
	}
}