Controller-Manager: propagate virtual kubelet metrics settings

Author: cheina97

cmd/liqo-controller-manager/main.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Package main contains the main function for the Liqo controller manager.
 package main
 
 import (
@@ -72,6 +73,7 @@ import (
 	liqoerrors "github.com/liqotech/liqo/pkg/utils/errors"
 	"github.com/liqotech/liqo/pkg/utils/mapper"
 	"github.com/liqotech/liqo/pkg/utils/restcfg"
+	"github.com/liqotech/liqo/pkg/vkMachinery"
 	"github.com/liqotech/liqo/pkg/vkMachinery/forge"
 )
 
@@ -96,6 +98,8 @@ func main() {
 	var nodeExtraAnnotations, nodeExtraLabels argsutils.StringMap
 	var kubeletCPURequests, kubeletCPULimits argsutils.Quantity
 	var kubeletRAMRequests, kubeletRAMLimits argsutils.Quantity
+	var kubeletMetricsAddress string
+	var kubeletMetricsEnabled bool
 
 	webhookPort := flag.Uint("webhook-port", 9443, "The port the webhook server binds to")
 	metricsAddr := flag.String("metrics-address", ":8080", "The address the metric endpoint binds to")
@@ -152,6 +156,8 @@ func main() {
 	flag.Var(&kubeletCPULimits, "kubelet-cpu-limits", "CPU limits assigned to the Virtual Kubelet Pod")
 	flag.Var(&kubeletRAMRequests, "kubelet-ram-requests", "RAM requests assigned to the Virtual Kubelet Pod")
 	flag.Var(&kubeletRAMLimits, "kubelet-ram-limits", "RAM limits assigned to the Virtual Kubelet Pod")
+	flag.StringVar(&kubeletMetricsAddress, "kubelet-metrics-address", vkMachinery.MetricsAddress, "The address the kubelet metrics endpoint binds to")
+	flag.BoolVar(&kubeletMetricsEnabled, "kubelet-metrics-enabled", false, "Enable the kubelet metrics endpoint")
 	flag.Var(&nodeExtraAnnotations, "node-extra-annotations", "Extra annotations to add to the Virtual Node")
 	flag.Var(&nodeExtraLabels, "node-extra-labels", "Extra labels to add to the Virtual Node")
 	kubeletIpamServer := flag.String("kubelet-ipam-server", "",
@@ -310,6 +316,8 @@ func main() {
 		LimitsCPU:            kubeletCPULimits.Quantity,
 		LimitsRAM:            kubeletRAMLimits.Quantity,
 		IpamEndpoint:         *kubeletIpamServer,
+		MetricsAddress:       kubeletMetricsAddress,
+		MetricsEnabled:       kubeletMetricsEnabled,
 	}
 
 	resourceOfferReconciler := resourceoffercontroller.NewResourceOfferController(

cmd/virtual-kubelet/root/flag.go

@@ -72,7 +72,8 @@ func InstallFlags(flags *pflag.FlagSet, o *Opts) {
 	flags.BoolVar(&o.EnableStorage, "enable-storage", false, "Enable the Liqo storage reflection")
 	flags.StringVar(&o.VirtualStorageClassName, "virtual-storage-class-name", "liqo", "Name of the virtual storage class")
 	flags.StringVar(&o.RemoteRealStorageClassName, "remote-real-storage-class-name", "", "Name of the real storage class to use for the actual volumes")
-	flags.BoolVar(&o.EnableMetrics, "enable-metrics", false, "Enable the metrics server")
+	flags.BoolVar(&o.EnableMetrics, "metrics-enabled", false, "Enable the metrics server")
+	flags.StringVar(&o.MetricsAddress, "metrics-address", ":8080", "The address to listen to for metrics requests")
 	flags.StringVar(&o.HomeAPIServerHost, "home-api-server-host", "",
 		"Home cluster API server HOST, this parameter is optional and required only to override the default values")
 	flags.StringVar(&o.HomeAPIServerPort, "home-api-server-port", "",

cmd/virtual-kubelet/root/opts.go

@@ -97,6 +97,7 @@ type Opts struct {
 	VirtualStorageClassName    string
 	RemoteRealStorageClassName string
 	EnableMetrics              bool
+	MetricsAddress             string
 
 	HomeAPIServerHost string
 	HomeAPIServerPort string

cmd/virtual-kubelet/root/root.go

@@ -197,7 +197,7 @@ func runRootCommand(ctx context.Context, c *Opts) error {
 	}
 
 	if c.EnableMetrics {
-		metrics.SetupMetricHandler()
+		metrics.SetupMetricHandler(c.MetricsAddress)
 	}
 
 	go func() {

deployments/liqo/README.md

@@ -138,7 +138,7 @@
 | virtualKubelet.metrics.podMonitor.enabled | bool | `false` |  |
 | virtualKubelet.metrics.podMonitor.interval | string | `""` |  |
 | virtualKubelet.metrics.podMonitor.scrapeTimeout | string | `""` |  |
-| virtualKubelet.metrics.port | int | `9090` | port used to expose metrics. |
+| virtualKubelet.metrics.port | int | `5872` | port used to expose metrics. |
 | virtualKubelet.virtualNode.extra.annotations | object | `{}` | virtual node extra annotations |
 | virtualKubelet.virtualNode.extra.labels | object | `{}` | virtual node extra labels |
 | webhook.failurePolicy | string | `"Fail"` | the webhook failure policy, among Ignore and Fail |

deployments/liqo/templates/liqo-controller-manager-deployment.yaml

@@ -26,9 +26,6 @@
 {{- $vkargs = append $vkargs "--certificate-type=aws" }}
 {{- end }}
 {{- end }}
-{{- if not (or (has "--enable-metrics" $vkargs ) (has "--enable-metrics=true" $vkargs ) (has "--enable-metrics=false" $vkargs )) }}
-{{- $vkargs = append $vkargs "--enable-metrics=true" }}
-{{- end}}
 
 apiVersion: apps/v1
 kind: Deployment
@@ -81,6 +78,11 @@ spec:
           - --enable-incoming-peering={{ .Values.discovery.config.incomingPeeringEnabled }}
           - --resource-sharing-percentage={{ .Values.controllerManager.config.resourceSharingPercentage }}
           - --kubelet-image={{ .Values.virtualKubelet.imageName }}{{ include "liqo.suffix" $ctrlManagerConfig }}:{{ include "liqo.version" $ctrlManagerConfig }}
+          {{- if .Values.virtualKubelet.metrics.enabled }}
+          - --kubelet-metrics-address=:{{ .Values.virtualKubelet.metrics.port }}
+          - --kubelet-metrics-enabled={{ .Values.virtualKubelet.metrics.enabled }}
+          {{- else }}
+          {{- end }}
           {{- if .Values.networkManager.externalIPAM.enabled }}
           - --kubelet-ipam-server={{ .Values.networkManager.externalIPAM.url }}
           {{- else if not .Values.networking.internal }}

deployments/liqo/values.yaml

@@ -319,7 +319,7 @@ virtualKubelet:
     # -- expose metrics about virtual kubelet resources.
     enabled: false
     # -- port used to expose metrics.
-    port: 9090
+    port: 5872
     podMonitor:
       # # -- create a prometheus podmonitor.
       enabled: false

pkg/liqo-controller-manager/resourceoffer-controller/resourceoffer_controller_methods.go

@@ -118,7 +118,7 @@ func (r *ResourceOfferReconciler) createVirtualKubeletDeployment(
 
 	// forge the virtual Kubelet
 	vkDeployment, err := forge.VirtualKubeletDeployment(
-		&r.cluster, &remoteClusterIdentity, namespace, r.liqoNamespace,
+		&r.cluster, &remoteClusterIdentity, namespace,
 		r.virtualKubeletOpts, resourceOffer)
 	if err != nil {
 		klog.Error(err)

pkg/virtualKubelet/metrics/metrics.go

@@ -24,11 +24,6 @@ import (
 	"k8s.io/klog/v2"
 )
 
-const (
-	// MetricsPort is the metrics port constant.
-	MetricsPort = ":9090"
-)
-
 var (
 	// ErrorsCounter is the counter of the errors occurred during the reflection.
 	ErrorsCounter *prometheus.CounterVec
@@ -59,7 +54,7 @@ func init() {
 }
 
 // SetupMetricHandler sets up the metric handler.
-func SetupMetricHandler() {
+func SetupMetricHandler(metricsAddress string) {
 	// Register the metrics to the prometheus registry.
 	prometheus.MustRegister(ErrorsCounter)
 	// Register the metrics to the prometheus registry.
@@ -68,10 +63,10 @@ func SetupMetricHandler() {
 	http.Handle("/metrics", promhttp.Handler())
 
 	go func() {
-		klog.Infof("Starting the virtual kubelet Metric Handler listening on %q", MetricsPort)
+		klog.Infof("Starting the virtual kubelet Metric Handler listening on %q", metricsAddress)
 
 		server := &http.Server{
-			Addr:              ":1234",
+			Addr:              metricsAddress,
 			ReadHeaderTimeout: 10 * time.Second,
 		}
 

pkg/vkMachinery/const.go

@@ -39,3 +39,6 @@ var KubeletBaseLabels = map[string]string{
 var ClusterRoleBindingLabels = map[string]string{
 	"app.kubernetes.io/managed-by": "advertisementoperator",
 }
+
+// MetricsAddress is the default address used to expose metrics.
+const MetricsAddress = ":8080"