Skip to content

Commit 6181a42

Browse files
giorio94adamjensenbot
authored andcommitted
virtual kubelet: support log/exec on EKS
1 parent 708f880 commit 6181a42

File tree

10 files changed

+64
-49
lines changed

10 files changed

+64
-49
lines changed

cmd/liqo-controller-manager/main.go

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,6 @@ func main() {
130130

131131
// Virtual-kubelet parameters
132132
kubeletImage := flag.String("kubelet-image", "liqo/virtual-kubelet", "The image of the virtual kubelet to be deployed")
133-
disableKubeletCertGeneration := flag.Bool("disable-kubelet-certificate-generation", false,
134-
"Whether to disable the virtual kubelet certificate generation by means of an init container (used for logs/exec capabilities)")
135133
flag.Var(&kubeletExtraAnnotations, "kubelet-extra-annotations", "Extra annotations to add to the Virtual Kubelet Deployments and Pods")
136134
flag.Var(&kubeletExtraLabels, "kubelet-extra-labels", "Extra labels to add to the Virtual Kubelet Deployments and Pods")
137135
flag.Var(&kubeletExtraArgs, "kubelet-extra-args", "Extra arguments to add to the Virtual Kubelet Deployments and Pods")
@@ -274,17 +272,16 @@ func main() {
274272
}
275273

276274
virtualKubeletOpts := &forge.VirtualKubeletOpts{
277-
ContainerImage: *kubeletImage,
278-
DisableCertGeneration: *disableKubeletCertGeneration,
279-
ExtraAnnotations: kubeletExtraAnnotations.StringMap,
280-
ExtraLabels: kubeletExtraLabels.StringMap,
281-
ExtraArgs: kubeletExtraArgs.StringList,
282-
NodeExtraAnnotations: nodeExtraAnnotations,
283-
NodeExtraLabels: nodeExtraLabels,
284-
RequestsCPU: kubeletCPURequests.Quantity,
285-
RequestsRAM: kubeletRAMRequests.Quantity,
286-
LimitsCPU: kubeletCPULimits.Quantity,
287-
LimitsRAM: kubeletRAMLimits.Quantity,
275+
ContainerImage: *kubeletImage,
276+
ExtraAnnotations: kubeletExtraAnnotations.StringMap,
277+
ExtraLabels: kubeletExtraLabels.StringMap,
278+
ExtraArgs: kubeletExtraArgs.StringList,
279+
NodeExtraAnnotations: nodeExtraAnnotations,
280+
NodeExtraLabels: nodeExtraLabels,
281+
RequestsCPU: kubeletCPURequests.Quantity,
282+
RequestsRAM: kubeletRAMRequests.Quantity,
283+
LimitsCPU: kubeletCPULimits.Quantity,
284+
LimitsRAM: kubeletRAMLimits.Quantity,
288285
}
289286

290287
resourceOfferReconciler := resourceoffercontroller.NewResourceOfferController(
@@ -330,8 +327,7 @@ func main() {
330327
}
331328

332329
// Start the handler to approve the virtual kubelet certificate signing requests.
333-
csrWatcher := csr.NewWatcher(clientset, *resyncPeriod, labels.Everything(),
334-
fields.OneTermEqualSelector("spec.signerName", certificates.KubeletServingSignerName))
330+
csrWatcher := csr.NewWatcher(clientset, *resyncPeriod, labels.Everything(), fields.Everything())
335331
csrWatcher.RegisterHandler(csr.ApproverHandler(clientset, "LiqoApproval", "This CSR was approved by Liqo",
336332
// Approve only the CSRs for a requestor living in a liqo tenant namespace (based on the prefix).
337333
// This is far from elegant, but the client-go utility generating the CSRs does not allow to customize the labels.

cmd/virtual-kubelet/root/flag.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func InstallFlags(flags *pflag.FlagSet, o *Opts) {
3737
flags.StringVar(&o.LiqoIpamServer, "ipam-server", o.LiqoIpamServer, "The address to contact the IPAM module")
3838

3939
flags.StringVar(&o.NodeIP, "node-ip", o.NodeIP, "The IP address of the virtual kubelet pod, and assigned to the virtual node as internal address")
40-
flags.BoolVar(&o.SelfSignedCertificate, "self-signed-certificate", false, "Whether to use a self-signed certificate for the virtual kubelet server")
40+
flags.Var(o.CertificateType, "certificate-type", "The type of virtual kubelet server certificate to generate, among kubelet, aws, self-signed")
4141
flags.Uint16Var(&o.ListenPort, "listen-port", o.ListenPort, "The port to listen to for requests from the Kubernetes API server")
4242
flags.BoolVar(&o.EnableProfiling, "enable-profiling", o.EnableProfiling, "Enable pprof profiling")
4343

cmd/virtual-kubelet/root/http.go

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,17 @@ func setupHTTPServer(ctx context.Context, handler workload.PodHandler, localClie
5252
return fmt.Errorf("failed to parse node IP %q", cfg.NodeIP)
5353
}
5454

55-
if cfg.SelfSignedCertificate {
55+
switch cfg.CertificateType.Value {
56+
case CertificateTypeSelfSigned:
5657
retriever = newSelfSignedCertificateRetriever(cfg.NodeName, parsedIP)
57-
} else {
58-
retriever, err = newCertificateRetriever(localClient, cfg.NodeName, parsedIP)
58+
default:
59+
// Determine the appropriate signer based on the requested certificate type.
60+
signer := map[string]string{
61+
CertificateTypeKubelet: certificates.KubeletServingSignerName,
62+
CertificateTypeAWS: "beta.eks.amazonaws.com/app-serving",
63+
}
64+
65+
retriever, err = newCertificateRetriever(localClient, signer[cfg.CertificateType.Value], cfg.NodeName, parsedIP)
5966
if err != nil {
6067
return fmt.Errorf("failed to initialize certificate manager: %w", err)
6168
}
@@ -137,7 +144,7 @@ func attachMetricsRoutes(ctx context.Context, mux *http.ServeMux, cl rest.Interf
137144
// newCertificateManager creates a certificate manager for the kubelet when retrieving a server certificate, or returns an error.
138145
// This function is inspired by the original kubelet implementation:
139146
// https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/certificate/kubelet.go
140-
func newCertificateRetriever(kubeClient kubernetes.Interface, nodeName string, nodeIP net.IP) (crtretriever, error) {
147+
func newCertificateRetriever(kubeClient kubernetes.Interface, signer, nodeName string, nodeIP net.IP) (crtretriever, error) {
141148
const (
142149
vkCertsPath = "/tmp/certs"
143150
vkCertsPrefix = "virtual-kubelet"
@@ -163,7 +170,7 @@ func newCertificateRetriever(kubeClient kubernetes.Interface, nodeName string, n
163170
return kubeClient, nil
164171
},
165172
GetTemplate: getTemplate,
166-
SignerName: certificates.KubeletServingSignerName,
173+
SignerName: signer,
167174
Usages: []certificates.KeyUsage{
168175
// https://tools.ietf.org/html/rfc5280#section-4.2.1.3
169176
//

cmd/virtual-kubelet/root/opts.go

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,15 @@ import (
2727
argsutils "github.com/liqotech/liqo/pkg/utils/args"
2828
)
2929

30+
const (
31+
// CertificateTypeKubelet -> the kubelet certificate is requested to be signed by kubernetes.io/kubelet-serving.
32+
CertificateTypeKubelet = "kubelet"
33+
// CertificateTypeAWS -> the kubelet certificate is requested to be signed by beta.eks.amazonaws.com/app-serving.
34+
CertificateTypeAWS = "aws"
35+
// CertificateTypeSelfSigned -> the kubelet certificate is self signed.
36+
CertificateTypeSelfSigned = "self-signed"
37+
)
38+
3039
// Defaults for root command options.
3140
const (
3241
DefaultNodeName = "virtual-kubelet"
@@ -59,10 +68,10 @@ type Opts struct {
5968
LiqoIpamServer string
6069

6170
// Sets the addresses to listen for requests from the Kubernetes API server
62-
NodeIP string
63-
ListenPort uint16
64-
SelfSignedCertificate bool
65-
EnableProfiling bool
71+
NodeIP string
72+
ListenPort uint16
73+
CertificateType *argsutils.StringEnum
74+
EnableProfiling bool
6675

6776
// Number of workers to use to handle pod notifications and resource reflection
6877
PodWorkers uint
@@ -96,6 +105,7 @@ func NewOpts() *Opts {
96105

97106
LiqoIpamServer: fmt.Sprintf("%v:%v", consts.NetworkManagerServiceName, consts.NetworkManagerIpamPort),
98107

108+
CertificateType: argsutils.NewEnum([]string{CertificateTypeKubelet, CertificateTypeAWS, CertificateTypeSelfSigned}, CertificateTypeKubelet),
99109
ListenPort: DefaultListenPort,
100110
EnableProfiling: false,
101111

deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,14 @@ rules:
7373
- certificatesigningrequests/status
7474
verbs:
7575
- update
76+
- apiGroups:
77+
- certificates.k8s.io
78+
resourceNames:
79+
- beta.eks.amazonaws.com/app-serving
80+
resources:
81+
- signers
82+
verbs:
83+
- approve
7684
- apiGroups:
7785
- certificates.k8s.io
7886
resourceNames:

deployments/liqo/templates/liqo-controller-manager-deployment.yaml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,19 @@
22
{{- $ctrlManagerConfig := (merge (dict "name" "controller-manager" "module" "controller-manager") .) -}}
33
{{- $webhookConfig := (merge (dict "name" "webhook" "module" "webhook") .) -}}
44

5-
{{- /* Enable the API support only in for Kubernetes versions < 1.24 (due to lack of support for third party tokens), if not overridden by the user */ -}}
65
{{- $vkargs := .Values.virtualKubelet.extra.args }}
6+
{{- /* Enable the API support only in for Kubernetes versions < 1.24 (due to lack of support for third party tokens), if not overridden by the user */ -}}
77
{{- if semverCompare "< 1.24.0" .Capabilities.KubeVersion.Version }}
88
{{- if not (or (has "--enable-apiserver-support" $vkargs ) (has "--enable-apiserver-support=true" $vkargs ) (has "--enable-apiserver-support=false" $vkargs )) }}
99
{{- $vkargs = append $vkargs "--enable-apiserver-support=true" }}
1010
{{- end }}
1111
{{- end }}
12+
{{- /* Configure the appropriate certificate generation approach on EKS clusters, if not overridden by the user */ -}}
13+
{{- if .Values.awsConfig.accessKeyId }}
14+
{{- if not (or (has "--certificate-type=kubelet" $vkargs ) (has "--certificate-type=aws" $vkargs ) (has "--certificate-type=self-signed" $vkargs )) }}
15+
{{- $vkargs = append $vkargs "--certificate-type=aws" }}
16+
{{- end }}
17+
{{- end }}
1218

1319
apiVersion: apps/v1
1420
kind: Deployment

pkg/liqo-controller-manager/namespacemap-controller/namespacemap_controller.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ type NamespaceMapReconciler struct {
5454
// +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests/status,verbs=update
5555
// +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests/approval,verbs=update
5656
// +kubebuilder:rbac:groups=certificates.k8s.io,resourceNames=kubernetes.io/kubelet-serving,resources=signers,verbs=approve
57+
// +kubebuilder:rbac:groups=certificates.k8s.io,resourceNames=beta.eks.amazonaws.com/app-serving,resources=signers,verbs=approve
5758

5859
// Reconcile adds/removes NamespaceMap finalizer, and checks differences
5960
// between DesiredMapping and CurrentMapping in order to create/delete the Namespaces if it is necessary.

pkg/liqoctl/install/eks/provider.go

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -137,11 +137,5 @@ func (o *Options) Values() map[string]interface{} {
137137
"region": o.region,
138138
"clusterName": o.eksClusterName,
139139
},
140-
141-
"controllerManager": map[string]interface{}{
142-
"pod": map[string]interface{}{
143-
"extraArgs": []interface{}{"--disable-kubelet-certificate-generation=true"},
144-
},
145-
},
146140
}
147141
}

pkg/vkMachinery/forge/forge.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,10 +68,6 @@ func forgeVKContainers(
6868
args = append(args, stringifyArgument("--node-extra-labels", opts.NodeExtraLabels.String()))
6969
}
7070

71-
if opts.DisableCertGeneration {
72-
args = append(args, "--self-signed-certificate")
73-
}
74-
7571
args = append(args, opts.ExtraArgs...)
7672

7773
return []v1.Container{

pkg/vkMachinery/forge/type.go

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,14 @@ import (
2323
// VirtualKubeletOpts defines the custom options associated with the virtual kubelet deployment forging.
2424
type VirtualKubeletOpts struct {
2525
// ContainerImage contains the virtual kubelet image name and tag.
26-
ContainerImage string
27-
// DisableCertGeneration allows to disable the virtual kubelet certificate generation (with the Kubernetes CSR)
28-
// by means of the init container (used for logs/exec capabilities).
29-
DisableCertGeneration bool
30-
ExtraAnnotations map[string]string
31-
ExtraLabels map[string]string
32-
ExtraArgs []string
33-
NodeExtraAnnotations argsutils.StringMap
34-
NodeExtraLabels argsutils.StringMap
35-
RequestsCPU resource.Quantity
36-
LimitsCPU resource.Quantity
37-
RequestsRAM resource.Quantity
38-
LimitsRAM resource.Quantity
26+
ContainerImage string
27+
ExtraAnnotations map[string]string
28+
ExtraLabels map[string]string
29+
ExtraArgs []string
30+
NodeExtraAnnotations argsutils.StringMap
31+
NodeExtraLabels argsutils.StringMap
32+
RequestsCPU resource.Quantity
33+
LimitsCPU resource.Quantity
34+
RequestsRAM resource.Quantity
35+
LimitsRAM resource.Quantity
3936
}

0 commit comments

Comments
 (0)