virtual kubelet: support log/exec on EKS

Author: giorio94

cmd/liqo-controller-manager/main.go

@@ -130,8 +130,6 @@ func main() {
 
 	// Virtual-kubelet parameters
 	kubeletImage := flag.String("kubelet-image", "liqo/virtual-kubelet", "The image of the virtual kubelet to be deployed")
-	disableKubeletCertGeneration := flag.Bool("disable-kubelet-certificate-generation", false,
-		"Whether to disable the virtual kubelet certificate generation by means of an init container (used for logs/exec capabilities)")
 	flag.Var(&kubeletExtraAnnotations, "kubelet-extra-annotations", "Extra annotations to add to the Virtual Kubelet Deployments and Pods")
 	flag.Var(&kubeletExtraLabels, "kubelet-extra-labels", "Extra labels to add to the Virtual Kubelet Deployments and Pods")
 	flag.Var(&kubeletExtraArgs, "kubelet-extra-args", "Extra arguments to add to the Virtual Kubelet Deployments and Pods")
@@ -274,17 +272,16 @@ func main() {
 	}
 
 	virtualKubeletOpts := &forge.VirtualKubeletOpts{
-		ContainerImage:        *kubeletImage,
-		DisableCertGeneration: *disableKubeletCertGeneration,
-		ExtraAnnotations:      kubeletExtraAnnotations.StringMap,
-		ExtraLabels:           kubeletExtraLabels.StringMap,
-		ExtraArgs:             kubeletExtraArgs.StringList,
-		NodeExtraAnnotations:  nodeExtraAnnotations,
-		NodeExtraLabels:       nodeExtraLabels,
-		RequestsCPU:           kubeletCPURequests.Quantity,
-		RequestsRAM:           kubeletRAMRequests.Quantity,
-		LimitsCPU:             kubeletCPULimits.Quantity,
-		LimitsRAM:             kubeletRAMLimits.Quantity,
+		ContainerImage:       *kubeletImage,
+		ExtraAnnotations:     kubeletExtraAnnotations.StringMap,
+		ExtraLabels:          kubeletExtraLabels.StringMap,
+		ExtraArgs:            kubeletExtraArgs.StringList,
+		NodeExtraAnnotations: nodeExtraAnnotations,
+		NodeExtraLabels:      nodeExtraLabels,
+		RequestsCPU:          kubeletCPURequests.Quantity,
+		RequestsRAM:          kubeletRAMRequests.Quantity,
+		LimitsCPU:            kubeletCPULimits.Quantity,
+		LimitsRAM:            kubeletRAMLimits.Quantity,
 	}
 
 	resourceOfferReconciler := resourceoffercontroller.NewResourceOfferController(
@@ -330,8 +327,7 @@ func main() {
 	}
 
 	// Start the handler to approve the virtual kubelet certificate signing requests.
-	csrWatcher := csr.NewWatcher(clientset, *resyncPeriod, labels.Everything(),
-		fields.OneTermEqualSelector("spec.signerName", certificates.KubeletServingSignerName))
+	csrWatcher := csr.NewWatcher(clientset, *resyncPeriod, labels.Everything(), fields.Everything())
 	csrWatcher.RegisterHandler(csr.ApproverHandler(clientset, "LiqoApproval", "This CSR was approved by Liqo",
 		// Approve only the CSRs for a requestor living in a liqo tenant namespace (based on the prefix).
 		// This is far from elegant, but the client-go utility generating the CSRs does not allow to customize the labels.

cmd/virtual-kubelet/root/flag.go

@@ -37,7 +37,7 @@ func InstallFlags(flags *pflag.FlagSet, o *Opts) {
 	flags.StringVar(&o.LiqoIpamServer, "ipam-server", o.LiqoIpamServer, "The address to contact the IPAM module")
 
 	flags.StringVar(&o.NodeIP, "node-ip", o.NodeIP, "The IP address of the virtual kubelet pod, and assigned to the virtual node as internal address")
-	flags.BoolVar(&o.SelfSignedCertificate, "self-signed-certificate", false, "Whether to use a self-signed certificate for the virtual kubelet server")
+	flags.Var(o.CertificateType, "certificate-type", "The type of virtual kubelet server certificate to generate, among kubelet, aws, self-signed")
 	flags.Uint16Var(&o.ListenPort, "listen-port", o.ListenPort, "The port to listen to for requests from the Kubernetes API server")
 	flags.BoolVar(&o.EnableProfiling, "enable-profiling", o.EnableProfiling, "Enable pprof profiling")
 

cmd/virtual-kubelet/root/http.go

@@ -52,10 +52,17 @@ func setupHTTPServer(ctx context.Context, handler workload.PodHandler, localClie
 		return fmt.Errorf("failed to parse node IP %q", cfg.NodeIP)
 	}
 
-	if cfg.SelfSignedCertificate {
+	switch cfg.CertificateType.Value {
+	case CertificateTypeSelfSigned:
 		retriever = newSelfSignedCertificateRetriever(cfg.NodeName, parsedIP)
-	} else {
-		retriever, err = newCertificateRetriever(localClient, cfg.NodeName, parsedIP)
+	default:
+		// Determine the appropriate signer based on the requested certificate type.
+		signer := map[string]string{
+			CertificateTypeKubelet: certificates.KubeletServingSignerName,
+			CertificateTypeAWS:     "beta.eks.amazonaws.com/app-serving",
+		}
+
+		retriever, err = newCertificateRetriever(localClient, signer[cfg.CertificateType.Value], cfg.NodeName, parsedIP)
 		if err != nil {
 			return fmt.Errorf("failed to initialize certificate manager: %w", err)
 		}
@@ -137,7 +144,7 @@ func attachMetricsRoutes(ctx context.Context, mux *http.ServeMux, cl rest.Interf
 // newCertificateManager creates a certificate manager for the kubelet when retrieving a server certificate, or returns an error.
 // This function is inspired by the original kubelet implementation:
 // https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/certificate/kubelet.go
-func newCertificateRetriever(kubeClient kubernetes.Interface, nodeName string, nodeIP net.IP) (crtretriever, error) {
+func newCertificateRetriever(kubeClient kubernetes.Interface, signer, nodeName string, nodeIP net.IP) (crtretriever, error) {
 	const (
 		vkCertsPath   = "/tmp/certs"
 		vkCertsPrefix = "virtual-kubelet"
@@ -163,7 +170,7 @@ func newCertificateRetriever(kubeClient kubernetes.Interface, nodeName string, n
 			return kubeClient, nil
 		},
 		GetTemplate: getTemplate,
-		SignerName:  certificates.KubeletServingSignerName,
+		SignerName:  signer,
 		Usages: []certificates.KeyUsage{
 			// https://tools.ietf.org/html/rfc5280#section-4.2.1.3
 			//

cmd/virtual-kubelet/root/opts.go

@@ -27,6 +27,15 @@ import (
 	argsutils "github.com/liqotech/liqo/pkg/utils/args"
 )
 
+const (
+	// CertificateTypeKubelet -> the kubelet certificate is requested to be signed by kubernetes.io/kubelet-serving.
+	CertificateTypeKubelet = "kubelet"
+	// CertificateTypeAWS -> the kubelet certificate is requested to be signed by beta.eks.amazonaws.com/app-serving.
+	CertificateTypeAWS = "aws"
+	// CertificateTypeSelfSigned -> the kubelet certificate is self signed.
+	CertificateTypeSelfSigned = "self-signed"
+)
+
 // Defaults for root command options.
 const (
 	DefaultNodeName             = "virtual-kubelet"
@@ -59,10 +68,10 @@ type Opts struct {
 	LiqoIpamServer string
 
 	// Sets the addresses to listen for requests from the Kubernetes API server
-	NodeIP                string
-	ListenPort            uint16
-	SelfSignedCertificate bool
-	EnableProfiling       bool
+	NodeIP          string
+	ListenPort      uint16
+	CertificateType *argsutils.StringEnum
+	EnableProfiling bool
 
 	// Number of workers to use to handle pod notifications and resource reflection
 	PodWorkers                   uint
@@ -96,6 +105,7 @@ func NewOpts() *Opts {
 
 		LiqoIpamServer: fmt.Sprintf("%v:%v", consts.NetworkManagerServiceName, consts.NetworkManagerIpamPort),
 
+		CertificateType: argsutils.NewEnum([]string{CertificateTypeKubelet, CertificateTypeAWS, CertificateTypeSelfSigned}, CertificateTypeKubelet),
 		ListenPort:      DefaultListenPort,
 		EnableProfiling: false,
 

deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml

@@ -73,6 +73,14 @@ rules:
   - certificatesigningrequests/status
   verbs:
   - update
+- apiGroups:
+  - certificates.k8s.io
+  resourceNames:
+  - beta.eks.amazonaws.com/app-serving
+  resources:
+  - signers
+  verbs:
+  - approve
 - apiGroups:
   - certificates.k8s.io
   resourceNames:

deployments/liqo/templates/liqo-controller-manager-deployment.yaml

@@ -2,13 +2,19 @@
 {{- $ctrlManagerConfig := (merge (dict "name" "controller-manager" "module" "controller-manager") .) -}}
 {{- $webhookConfig := (merge (dict "name" "webhook" "module" "webhook") .) -}}
 
-{{- /* Enable the API support only in for Kubernetes versions < 1.24 (due to lack of support for third party tokens), if not overridden by the user */ -}}
 {{- $vkargs := .Values.virtualKubelet.extra.args }}
+{{- /* Enable the API support only in for Kubernetes versions < 1.24 (due to lack of support for third party tokens), if not overridden by the user */ -}}
 {{- if semverCompare "< 1.24.0" .Capabilities.KubeVersion.Version }}
 {{- if not (or (has "--enable-apiserver-support" $vkargs ) (has "--enable-apiserver-support=true" $vkargs ) (has "--enable-apiserver-support=false" $vkargs )) }}
 {{- $vkargs = append $vkargs "--enable-apiserver-support=true" }}
 {{- end }}
 {{- end }}
+{{- /* Configure the appropriate certificate generation approach on EKS clusters, if not overridden by the user */ -}}
+{{- if .Values.awsConfig.accessKeyId }}
+{{- if not (or (has "--certificate-type=kubelet" $vkargs ) (has "--certificate-type=aws" $vkargs ) (has "--certificate-type=self-signed" $vkargs )) }}
+{{- $vkargs = append $vkargs "--certificate-type=aws" }}
+{{- end }}
+{{- end }}
 
 apiVersion: apps/v1
 kind: Deployment

pkg/liqo-controller-manager/namespacemap-controller/namespacemap_controller.go

@@ -54,6 +54,7 @@ type NamespaceMapReconciler struct {
 // +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests/status,verbs=update
 // +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests/approval,verbs=update
 // +kubebuilder:rbac:groups=certificates.k8s.io,resourceNames=kubernetes.io/kubelet-serving,resources=signers,verbs=approve
+// +kubebuilder:rbac:groups=certificates.k8s.io,resourceNames=beta.eks.amazonaws.com/app-serving,resources=signers,verbs=approve
 
 // Reconcile adds/removes NamespaceMap finalizer, and checks differences
 // between DesiredMapping and CurrentMapping in order to create/delete the Namespaces if it is necessary.

pkg/liqoctl/install/eks/provider.go

@@ -137,11 +137,5 @@ func (o *Options) Values() map[string]interface{} {
 			"region":          o.region,
 			"clusterName":     o.eksClusterName,
 		},
-
-		"controllerManager": map[string]interface{}{
-			"pod": map[string]interface{}{
-				"extraArgs": []interface{}{"--disable-kubelet-certificate-generation=true"},
-			},
-		},
 	}
 }

pkg/vkMachinery/forge/forge.go

@@ -68,10 +68,6 @@ func forgeVKContainers(
 		args = append(args, stringifyArgument("--node-extra-labels", opts.NodeExtraLabels.String()))
 	}
 
-	if opts.DisableCertGeneration {
-		args = append(args, "--self-signed-certificate")
-	}
-
 	args = append(args, opts.ExtraArgs...)
 
 	return []v1.Container{

pkg/vkMachinery/forge/type.go

@@ -23,17 +23,14 @@ import (
 // VirtualKubeletOpts defines the custom options associated with the virtual kubelet deployment forging.
 type VirtualKubeletOpts struct {
 	// ContainerImage contains the virtual kubelet image name and tag.
-	ContainerImage string
-	// DisableCertGeneration allows to disable the virtual kubelet certificate generation (with the Kubernetes CSR)
-	// by means of the init container (used for logs/exec capabilities).
-	DisableCertGeneration bool
-	ExtraAnnotations      map[string]string
-	ExtraLabels           map[string]string
-	ExtraArgs             []string
-	NodeExtraAnnotations  argsutils.StringMap
-	NodeExtraLabels       argsutils.StringMap
-	RequestsCPU           resource.Quantity
-	LimitsCPU             resource.Quantity
-	RequestsRAM           resource.Quantity
-	LimitsRAM             resource.Quantity
+	ContainerImage       string
+	ExtraAnnotations     map[string]string
+	ExtraLabels          map[string]string
+	ExtraArgs            []string
+	NodeExtraAnnotations argsutils.StringMap
+	NodeExtraLabels      argsutils.StringMap
+	RequestsCPU          resource.Quantity
+	LimitsCPU            resource.Quantity
+	RequestsRAM          resource.Quantity
+	LimitsRAM            resource.Quantity
 }