Add an integration for Istio's service and sidecars

charts/k8s-monitoring/charts/feature-integrations/README.md

@@ -114,6 +114,12 @@ Be sure perform actual integration testing in a live environment in the main [k8
 |-----|------|---------|-------------|
 | grafana | object | `{"instances":[]}` | Scrape metrics/logs from Grafana |
 
+### Integration: Istio
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| istio | object | `{"instances":[]}` | Scrape metrics/logs from Istio sidecar and service. |
+
 ### Integration: Loki
 
 | Key | Type | Default | Description |

charts/k8s-monitoring/charts/feature-integrations/default-allow-lists/istio.yaml

@@ -0,0 +1,43 @@
+---
+# The set of metrics from Istio required for the Istio integration
+- envoy_cluster_upstream_cx_rx_bytes_total
+- envoy_cluster_upstream_cx_tx_bytes_total
+- galley_validation_failed
+- galley_validation_passed
+- go_memstats_heap_alloc_bytes
+- go_memstats_heap_inuse_bytes
+- go_memstats_heap_sys_bytes
+- istio_agent_go_memstats_heap_alloc_bytes
+- istio_agent_go_memstats_heap_inuse_bytes
+- istio_agent_go_memstats_heap_sys_bytes
+- istio_agent_process_cpu_seconds_total
+- istio_agent_process_open_fds
+- istio_agent_process_resident_memory_bytes
+- istio_agent_process_virtual_memory_bytes
+- istio_build
+- istio_request_bytes_sum
+- istio_request_duration_milliseconds_count
+- istio_request_duration_milliseconds_sum
+- istio_requests_total
+- istio_response_bytes_sum
+- istio_tcp_received_bytes_total
+- istio_tcp_sent_bytes_total
+- istiod_uptime_seconds
+- pilot_conflict_inbound_listener
+- pilot_conflict_outbound_listener_tcp_over_current_tcp
+- pilot_info
+- pilot_k8s_cfg_events
+- pilot_total_xds_internal_errors
+- pilot_total_xds_rejects
+- pilot_xds_cds_reject
+- pilot_xds_eds_reject
+- pilot_xds_lds_reject
+- pilot_xds_pushes
+- pilot_xds_rds_reject
+- pilot_xds_write_timeout
+- process_cpu_seconds_total
+- process_open_fds
+- process_resident_memory_bytes
+- process_virtual_memory_bytes
+- sidecar_injection_failure_total
+- sidecar_injection_success_total

charts/k8s-monitoring/charts/feature-integrations/docs/integrations/istio.md

@@ -0,0 +1,62 @@
+# istio
+
+## Values
+
+### Istiod Metrics Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| istiodMetrics.enabled | bool | `true` | Whether to enable metrics collection from Istio . |
+| istiodMetrics.extraMetricProcessingRules | string | `""` | Rule blocks to be added to the prometheus.relabel component for MySQL metrics. These relabeling rules are applied post-scrape against the metrics returned from the scraped target, no `__meta*` labels are present. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#rule-block)) |
+| istiodMetrics.maxCacheSize | string | `100000` | Sets the max_cache_size for prometheus.relabel component. This should be at least 2x-5x your largest scrape target or samples appended rate. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus.relabel/#arguments)) Overrides global.maxCacheSize |
+| istiodMetrics.scrapeInterval | string | `60s` | How frequently to scrape metrics from MySQL Exporter. |
+| istiodMetrics.scrapeTimeout | string | `10s` | The timeout for scraping metrics from MySQL Exporter. |
+| istiodMetrics.tuning.excludeMetrics | list | `[]` | Metrics to drop. Can use regular expressions. |
+| istiodMetrics.tuning.includeMetrics | list | `[]` | Metrics to keep. Can use regular expressions. |
+
+### General Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| jobLabel | string | `"integration/istio"` | The value of the job label for scraped metrics and logs |
+| name | string | `""` | Name for this Istio integration instance. |
+
+### Logs Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| logs.enabled | bool | `true` | Whether to enable special processing of Istio pod logs. |
+
+### Discovery Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| logs.labelSelectors | object | `{}` | Discover MySQL instances based on label selectors. At least one is required. |
+| logs.namespaces | list | `[]` | The namespaces to look for MySQL instances in. Will automatically look for MySQL instances in all namespaces unless specified here |
+
+### Sidecar Metrics Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| sidecarMetrics.enabled | bool | `true` | Whether to enable metrics collection from Istio sidecar containers. |
+| sidecarMetrics.extraMetricProcessingRules | string | `""` | Rule blocks to be added to the prometheus.relabel component for MySQL metrics. These relabeling rules are applied post-scrape against the metrics returned from the scraped target, no `__meta*` labels are present. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#rule-block)) |
+| sidecarMetrics.maxCacheSize | string | `100000` | Sets the max_cache_size for prometheus.relabel component. This should be at least 2x-5x your largest scrape target or samples appended rate. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus.relabel/#arguments)) Overrides global.maxCacheSize |
+| sidecarMetrics.scrapeInterval | string | `60s` | How frequently to scrape metrics from Istio sidecar containers. |
+| sidecarMetrics.scrapeTimeout | string | `10s` | The timeout for scraping metrics from Istio sidecar containers. |
+| sidecarMetrics.sidecarContainerName | string | `"istio-proxy.*"` | The name of the Istio sidecar container. |
+| sidecarMetrics.tuning.excludeMetrics | list | `[]` | Metrics to drop. Can use regular expressions. |
+| sidecarMetrics.tuning.includeMetrics | list | `[]` | Metrics to keep. Can use regular expressions. |
+
+### Metric Processing Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| sidecarMetrics.tuning.useDefaultAllowList | bool | `true` | Filter the list of metrics from Grafana Alloy to the minimal set required for the Grafana Alloy integration. |
+
+### Other Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| istiodMetrics.namespace | string | `""` |  |
+| istiodMetrics.serviceName | string | `"istiod"` |  |
+| sidecarMetrics.labelSelectors | object | `{}` |  |

charts/k8s-monitoring/charts/feature-integrations/integrations/istio-values.yaml

@@ -0,0 +1,112 @@
+---
+# -- Name for this Istio integration instance.
+# @section -- General Settings
+name: ""
+
+# -- The value of the job label for scraped metrics and logs
+# @section -- General Settings
+jobLabel: integration/istio
+
+sidecarMetrics:
+  # -- Whether to enable metrics collection from Istio sidecar containers.
+  # @section -- Sidecar Metrics Settings
+  enabled: true
+
+  labelSelectors: {}
+
+  # -- The name of the Istio sidecar container.
+  # @section -- Sidecar Metrics Settings
+  sidecarContainerName: "istio-proxy.*"
+
+  # -- How frequently to scrape metrics from Istio sidecar containers.
+  # @default -- `60s`
+  # @section -- Sidecar Metrics Settings
+  scrapeInterval:
+
+  # -- The timeout for scraping metrics from Istio sidecar containers.
+  # @default -- `10s`
+  # @section -- Sidecar Metrics Settings
+  scrapeTimeout: ""
+
+  # -- Sets the max_cache_size for prometheus.relabel component.
+  # This should be at least 2x-5x your largest scrape target or samples appended rate.
+  # ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus.relabel/#arguments))
+  # Overrides global.maxCacheSize
+  # @default -- `100000`
+  # @section -- Sidecar Metrics Settings
+  maxCacheSize:
+
+  # Adjustments to the scraped metrics to filter the amount of data sent to storage.
+  tuning:
+    # -- Filter the list of metrics from Grafana Alloy to the minimal set required for the Grafana Alloy integration.
+    # @section -- Metric Processing Settings
+    useDefaultAllowList: true
+    # -- Metrics to keep. Can use regular expressions.
+    # @section -- Sidecar Metrics Settings
+    includeMetrics: []
+    # -- Metrics to drop. Can use regular expressions.
+    # @section -- Sidecar Metrics Settings
+    excludeMetrics: []
+
+  # -- Rule blocks to be added to the prometheus.relabel component for MySQL metrics.
+  # These relabeling rules are applied post-scrape against the metrics returned from the scraped target, no `__meta*` labels are present.
+  # ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#rule-block))
+  # @section -- Sidecar Metrics Settings
+  extraMetricProcessingRules: ""
+
+istiodMetrics:
+  # -- Whether to enable metrics collection from Istio .
+  # @section -- Istiod Metrics Settings
+  enabled: true
+
+  serviceName: istiod
+
+  namespace: ""
+
+  # -- How frequently to scrape metrics from MySQL Exporter.
+  # @default -- `60s`
+  # @section -- Istiod Metrics Settings
+  scrapeInterval:
+
+  # -- The timeout for scraping metrics from MySQL Exporter.
+  # @default -- `10s`
+  # @section -- Istiod Metrics Settings
+  scrapeTimeout: ""
+
+  # -- Sets the max_cache_size for prometheus.relabel component.
+  # This should be at least 2x-5x your largest scrape target or samples appended rate.
+  # ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus.relabel/#arguments))
+  # Overrides global.maxCacheSize
+  # @default -- `100000`
+  # @section -- Istiod Metrics Settings
+  maxCacheSize:
+
+  # Adjustments to the scraped metrics to filter the amount of data sent to storage.
+  tuning:
+    # -- Metrics to keep. Can use regular expressions.
+    # @section -- Istiod Metrics Settings
+    includeMetrics: []
+    # -- Metrics to drop. Can use regular expressions.
+    # @section -- Istiod Metrics Settings
+    excludeMetrics: []
+
+  # -- Rule blocks to be added to the prometheus.relabel component for MySQL metrics.
+  # These relabeling rules are applied post-scrape against the metrics returned from the scraped target, no `__meta*` labels are present.
+  # ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#rule-block))
+  # @section -- Istiod Metrics Settings
+  extraMetricProcessingRules: ""
+
+# Settings for log gathering using the Pod Logs feature
+logs:
+  # -- Whether to enable special processing of Istio pod logs.
+  # @section -- Logs Settings
+  enabled: true
+
+  # -- The namespaces to look for MySQL instances in.
+  # Will automatically look for MySQL instances in all namespaces unless specified here
+  # @section -- Discovery Settings
+  namespaces: []
+
+  # -- Discover MySQL instances based on label selectors. At least one is required.
+  # @section -- Discovery Settings
+  labelSelectors: {}

charts/k8s-monitoring/charts/feature-integrations/schema-mods/definitions/istio-integration.schema.json

@@ -0,0 +1,106 @@
+{
+    "type": "object",
+    "properties": {
+        "istiodMetrics": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "boolean"
+                },
+                "extraMetricProcessingRules": {
+                    "type": "string"
+                },
+                "maxCacheSize": {
+                    "type": "null"
+                },
+                "namespace": {
+                    "type": "string"
+                },
+                "scrapeInterval": {
+                    "type": "null"
+                },
+                "scrapeTimeout": {
+                    "type": "string"
+                },
+                "serviceName": {
+                    "type": "string"
+                },
+                "tuning": {
+                    "type": "object",
+                    "properties": {
+                        "excludeMetrics": {
+                            "type": "array"
+                        },
+                        "includeMetrics": {
+                            "type": "array"
+                        }
+                    }
+                }
+            }
+        },
+        "jobLabel": {
+            "type": "string"
+        },
+        "logs": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "boolean"
+                },
+                "labelSelectors": {
+                    "type": "object"
+                },
+                "namespaces": {
+                    "type": "array"
+                }
+            }
+        },
+        "name": {
+            "type": "string"
+        },
+        "sidecarMetrics": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "boolean"
+                },
+                "extraMetricProcessingRules": {
+                    "type": "string"
+                },
+                "labelSelectors": {
+                    "type": "object"
+                },
+                "maxCacheSize": {
+                    "type": "null"
+                },
+                "scrapeInterval": {
+                    "type": "null"
+                },
+                "scrapeTimeout": {
+                    "type": "string"
+                },
+                "sidecarContainerName": {
+                    "type": "string"
+                },
+                "tuning": {
+                    "type": "object",
+                    "properties": {
+                        "excludeMetrics": {
+                            "type": "array"
+                        },
+                        "includeMetrics": {
+                            "type": "array"
+                        },
+                        "useDefaultAllowList": {
+                            "type": "boolean"
+                        }
+                    }
+                }
+            }
+        },
+        "type": {
+            "type": "string",
+            "const": "istio"
+        }
+    }
+}

charts/k8s-monitoring/charts/feature-integrations/schema-mods/integration-list.json

@@ -9,6 +9,7 @@
           { "$ref": "#/definitions/dcgm-exporter-integration"},
           { "$ref": "#/definitions/etcd-integration"},
           { "$ref": "#/definitions/grafana-integration"},
+          { "$ref": "#/definitions/istio-integration"},
           { "$ref": "#/definitions/loki-integration"},
           { "$ref": "#/definitions/mimir-integration"},
           { "$ref": "#/definitions/mysql-integration"},

charts/k8s-monitoring/charts/feature-integrations/templates/_helpers.tpl

@@ -13,6 +13,9 @@
 {{- define "pod_label" -}}
 {{ printf "__meta_kubernetes_pod_label_%s" (include "escape_label" .) }}
 {{- end }}
+{{- define "pod_annotation" -}}
+{{ printf "__meta_kubernetes_pod_annotation_%s" (include "escape_label" .) }}
+{{- end }}
 
 {{- define "english_list" }}
 {{- if eq (len .) 0 }}

charts/k8s-monitoring/charts/feature-integrations/templates/_integration_istio.tpl

@@ -0,0 +1,8 @@
+{{- define "integrations.istio.validate" }}
+  {{- range $instance := $.Values.istio.instances }}
+    {{- $defaultValues := fromYaml ($.Files.Get "integrations/istio-values.yaml") }}
+    {{- include "integrations.istio.instance.validate" (dict "instance" (mergeOverwrite $defaultValues $instance (dict "type" "integration.istio"))) | nindent 2 }}
+  {{- end }}
+{{- end }}
+
+{{- define "integrations.istio.instance.validate" }}{{- end }}