Network: gateway prometheus metrics

Author: cheina97

cmd/gateway/main.go

@@ -110,7 +110,7 @@ func run(cmd *cobra.Command, _ []string) error {
 		MapperProvider: mapper.LiqoMapperProvider(scheme),
 		Scheme:         scheme,
 		Metrics: server.Options{
-			BindAddress: "0", // Metrics are exposed by "connection" container.
+			BindAddress: connoptions.GwOptions.MetricsAddress,
 		},
 		HealthProbeBindAddress: connoptions.GwOptions.ProbeAddr,
 		LeaderElection:         connoptions.GwOptions.LeaderElection,

cmd/gateway/wireguard/main.go

@@ -33,6 +33,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	ipamv1alpha1 "github.com/liqotech/liqo/apis/ipam/v1alpha1"
@@ -167,6 +168,19 @@ func run(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("unable to init wireguard link: %w", err)
 	}
 
+	// Create the Prometheus collector and register it inside the controller-runtime metrics server.
+	promcollect, err := wireguard.NewPrometheusCollector(mgr.GetClient(), &wireguard.MetricsOptions{
+		RemoteClusterID:  options.GwOptions.RemoteClusterID,
+		Namespace:        options.GwOptions.Namespace,
+		WgImplementation: options.Implementation,
+	})
+	if err != nil {
+		return fmt.Errorf("unable to create prometheus collector: %w", err)
+	}
+	if err := metrics.Registry.Register(promcollect); err != nil {
+		return fmt.Errorf("unable to register prometheus collector: %w", err)
+	}
+
 	// Start the manager.
 	return mgr.Start(cmd.Context())
 }

deployments/liqo/README.md

@@ -74,6 +74,8 @@
 | metricAgent.pod.labels | object | `{}` | Labels for the metricAgent pod. |
 | metricAgent.pod.priorityClassName | string | `""` | PriorityClassName (https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority) for the metricAgent pod. |
 | metricAgent.pod.resources | object | `{"limits":{},"requests":{}}` | Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the metricAgent pod. |
+| metrics.enabled | bool | `false` | Enable/Disable the metrics server in every liqo component. |
+| metrics.prometheusOperator.enabled | bool | `false` | Enable/Disable the creation of a Prometheus servicemonitor/podmonitor for the metrics servers. Turn on this flag when the Prometheus Operator runs in your cluster. |
 | nameOverride | string | `""` | Override the standard name used by Helm and associated to Kubernetes/Liqo resources. |
 | networking.clientResources | list | `[{"apiVersion":"networking.liqo.io/v1alpha1","resource":"wggatewayclients"}]` | Set the list of resources that implement the GatewayClient |
 | networking.enabled | bool | `true` | Use the default Liqo networking module. |

deployments/liqo/templates/liqo-gateway-podmonitor.yaml

@@ -0,0 +1,26 @@
+---
+{{- $gatewayConfig := (merge (dict "name" "gateway" "module" "networking" ) .) -}}
+
+{{- if and (.Values.networking.enabled) (.Values.metrics.enabled) (.Values.metrics.prometheusOperator.enabled) }}
+
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: gateway
+  labels:
+    {{- include "liqo.labels" $gatewayConfig | nindent 4 }}
+spec:
+  podMetricsEndpoints:
+    - port: gw-metrics
+      path: /metrics
+    - port: wg-metrics
+      path: /metrics
+    - port: gv-metrics
+      path: /metrics
+  selector:
+    matchLabels:
+      networking.liqo.io/component: gateway
+  namespaceSelector:
+    any: true
+    
+{{- end }}

deployments/liqo/templates/liqo-wireguard-gateway-client-template.yaml

@@ -46,7 +46,9 @@ spec:
                 - --gateway-uid={{"{{ .GatewayUID }}"}}
                 - --node-name={{"$(NODE_NAME)"}}
                 - --mode=client
+                {{- if .Values.metrics.enabled }}
                 - --metrics-address=:8080
+                {{- end }}
                 - --health-probe-bind-address=:8081
                 - --ping-enabled=true
                 - --ping-loss-threshold={{ .Values.networking.gatewayTemplates.ping.lossThreshold }}
@@ -55,6 +57,11 @@ spec:
                 {{- if gt .Values.networking.gatewayTemplates.replicas 1.0 }}
                 - --leader-election=true
                 {{- end }}
+                {{- if .Values.metrics.enabled }}
+                ports:
+                - containerPort: 8080
+                  name: gw-metrics
+                {{- end }}
                 env:
                 - name: NODE_NAME
                   valueFrom:
@@ -78,9 +85,16 @@ spec:
                 - --mtu={{"{{ .Spec.MTU }}"}}
                 - --endpoint-address={{"{{ index .Spec.Endpoint.Addresses 0 }}"}}
                 - --endpoint-port={{"{{ .Spec.Endpoint.Port }}"}}
+                {{- if .Values.metrics.enabled }}
                 - --metrics-address=:8082
+                {{- end }}
                 - --health-probe-bind-address=:8083
                 - --implementation={{ .Values.networking.gatewayTemplates.wireguard.implementation }}
+                {{- if .Values.metrics.enabled }}
+                ports:
+                - containerPort: 8082
+                  name: wg-metrics
+                {{- end }}
                 securityContext:
                   capabilities:
                     add:
@@ -99,9 +113,16 @@ spec:
                 - --gateway-uid={{"{{ .GatewayUID }}"}}
                 - --node-name={{"$(NODE_NAME)"}}
                 - --mode=server
+                {{- if .Values.metrics.enabled }}
                 - --metrics-address=:8084
+                {{- end }}
                 - --health-probe-bind-address=:8085
                 - --enable-arp=true
+                {{- if .Values.metrics.enabled }}
+                ports:
+                - containerPort: 8084
+                  name: gv-metrics
+                {{- end }} 
                 env:
                 - name: NODE_NAME
                   valueFrom:

deployments/liqo/templates/liqo-wireguard-gateway-server-template.yaml

@@ -64,7 +64,9 @@ spec:
                 - --node-name={{"$(NODE_NAME)"}}
                 - --gateway-uid={{"{{ .GatewayUID }}"}}
                 - --mode=server
+                {{- if .Values.metrics.enabled }}
                 - --metrics-address=:8080
+                {{- end }}
                 - --health-probe-bind-address=:8081
                 - --ping-enabled=true
                 - --ping-loss-threshold={{ .Values.networking.gatewayTemplates.ping.lossThreshold }}
@@ -73,6 +75,11 @@ spec:
                 {{- if gt .Values.networking.gatewayTemplates.replicas 1.0 }}
                 - --leader-election=true
                 {{- end }}
+                {{- if .Values.metrics.enabled }}
+                ports:
+                - containerPort: 8080
+                  name: gw-metrics
+                {{- end }}
                 env:
                 - name: NODE_NAME
                   valueFrom:
@@ -95,9 +102,16 @@ spec:
                 - --mode=server
                 - --mtu={{"{{ .Spec.MTU }}"}}
                 - --listen-port={{"{{ .Spec.Endpoint.Port }}"}}
+                {{- if .Values.metrics.enabled }}
                 - --metrics-address=:8082
+                {{- end }}
                 - --health-probe-bind-address=:8083
                 - --implementation={{ .Values.networking.gatewayTemplates.wireguard.implementation }}
+                {{- if .Values.metrics.enabled }}
+                ports:
+                - containerPort: 8082
+                  name: wg-metrics
+                {{- end }}
                 securityContext:
                   capabilities:
                     add:
@@ -116,9 +130,16 @@ spec:
                 - --node-name={{"$(NODE_NAME)"}}
                 - --gateway-uid={{"{{ .GatewayUID }}"}}
                 - --mode=server
+                {{- if .Values.metrics.enabled }}
                 - --metrics-address=:8084
+                {{- end }}
                 - --health-probe-bind-address=:8085
                 - --enable-arp=true
+                {{- if .Values.metrics.enabled }}
+                ports:
+                - containerPort: 8084
+                  name: gv-metrics
+                {{- end }}
                 env:
                 - name: NODE_NAME
                   valueFrom:

deployments/liqo/values.yaml

@@ -10,6 +10,14 @@ nameOverride: ""
 # -- Override the standard full name used by Helm and associated to Kubernetes/Liqo resources.
 fullnameOverride: ""
 
+metrics:
+  # -- Enable/Disable the metrics server in every liqo component.
+  enabled: false
+  prometheusOperator:
+    # -- Enable/Disable the creation of a Prometheus servicemonitor/podmonitor for the metrics servers.
+    # Turn on this flag when the Prometheus Operator runs in your cluster.
+    enabled: false
+
 apiServer:
   # -- The address that must be used to contact your API server, it needs to be reachable from the clusters that you will peer with (defaults to your master IP).
   address: ""