Skip to content

Commit 429d569

Browse files
nasusobanasusoba
authored
feat: user could set airgapped install script path (#140)
Signed-off-by: nasusoba <[email protected]> user could customize etcd proxy image Signed-off-by: nasusoba <[email protected]> add test for resolveEtcdProxyFile
1 parent f7d2499 commit 429d569

20 files changed

+387
-116
lines changed

‎bootstrap/api/v1beta1/conversion.go

+9
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ func (c *KThreesConfig) ConvertTo(dstRaw ctrlconversion.Hub) error {
4343
dst.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider
4444
dst.Spec.ServerConfig.DisableCloudController = restored.Spec.ServerConfig.DisableCloudController
4545
dst.Spec.ServerConfig.SystemDefaultRegistry = restored.Spec.ServerConfig.SystemDefaultRegistry
46+
dst.Spec.ServerConfig.EtcdProxyImage = restored.Spec.ServerConfig.EtcdProxyImage
47+
dst.Spec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.AgentConfig.AirGappedInstallScriptPath
4648
return nil
4749
}
4850

@@ -96,6 +98,8 @@ func (r *KThreesConfigTemplate) ConvertTo(dstRaw ctrlconversion.Hub) error {
9698
dst.Spec.Template.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.Template.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider
9799
dst.Spec.Template.Spec.ServerConfig.DisableCloudController = restored.Spec.Template.Spec.ServerConfig.DisableCloudController
98100
dst.Spec.Template.Spec.ServerConfig.SystemDefaultRegistry = restored.Spec.Template.Spec.ServerConfig.SystemDefaultRegistry
101+
dst.Spec.Template.Spec.ServerConfig.EtcdProxyImage = restored.Spec.Template.Spec.ServerConfig.EtcdProxyImage
102+
dst.Spec.Template.Spec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.Template.Spec.AgentConfig.AirGappedInstallScriptPath
99103
return nil
100104
}
101105

@@ -159,3 +163,8 @@ func Convert_v1beta2_KThreesServerConfig_To_v1beta1_KThreesServerConfig(in *boot
159163

160164
return nil
161165
}
166+
167+
// Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig is an autogenerated conversion function.
168+
func Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(in *bootstrapv1beta2.KThreesAgentConfig, out *KThreesAgentConfig, s conversion.Scope) error { //nolint: stylecheck
169+
return autoConvert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(in, out, s)
170+
}

‎bootstrap/api/v1beta1/zz_generated.conversion.go

+7-10
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bootstrap/api/v1beta2/kthreesconfig_types.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,10 @@ type KThreesServerConfig struct {
124124
// SystemDefaultRegistry defines private registry to be used for all system images
125125
// +optional
126126
SystemDefaultRegistry string `json:"systemDefaultRegistry,omitempty"`
127+
128+
// Customized etcd proxy image for management cluster to communicate with workload cluster etcd (default: "alpine/socat")
129+
// +optional
130+
EtcdProxyImage string `json:"etcdProxyImage,omitempty"`
127131
}
128132

129133
type KThreesAgentConfig struct {
@@ -154,10 +158,16 @@ type KThreesAgentConfig struct {
154158

155159
// AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
156160
// basically supposing that online container registries and k3s install scripts are not reachable.
157-
// User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
161+
// User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh")
158162
// on all nodes in the air-gap environment.
159163
// +optional
160164
AirGapped bool `json:"airGapped,omitempty"`
165+
166+
// AirGappedInstallScriptPath is the path to the install script in the air-gapped environment.
167+
// The install script should be prepared by the user. The value is only
168+
// used when AirGapped is set to true (default: "/opt/install.sh").
169+
// +optional
170+
AirGappedInstallScriptPath string `json:"airGappedInstallScriptPath,omitempty"`
161171
}
162172

163173
// KThreesConfigStatus defines the observed state of KThreesConfig.

‎bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml

+11-1
Original file line numberDiff line numberDiff line change
@@ -326,9 +326,15 @@ spec:
326326
description: |-
327327
AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
328328
basically supposing that online container registries and k3s install scripts are not reachable.
329-
User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
329+
User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh")
330330
on all nodes in the air-gap environment.
331331
type: boolean
332+
airGappedInstallScriptPath:
333+
description: |-
334+
AirGappedInstallScriptPath is the path to the install script in the air-gapped environment.
335+
The install script should be prepared by the user. The value is only
336+
used when AirGapped is set to true (default: "/opt/install.sh").
337+
type: string
332338
kubeProxyArgs:
333339
description: KubeProxyArgs Customized flag for kube-proxy process
334340
items:
@@ -471,6 +477,10 @@ spec:
471477
the ''cloud-provider=external'' kubelet argument. (default:
472478
false)'
473479
type: boolean
480+
etcdProxyImage:
481+
description: 'Customized etcd proxy image for management cluster
482+
to communicate with workload cluster etcd (default: "alpine/socat")'
483+
type: string
474484
httpsListenPort:
475485
description: 'HTTPSListenPort HTTPS listen port (default: 6443)'
476486
type: string

‎bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml

+12-1
Original file line numberDiff line numberDiff line change
@@ -281,9 +281,15 @@ spec:
281281
description: |-
282282
AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
283283
basically supposing that online container registries and k3s install scripts are not reachable.
284-
User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
284+
User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh")
285285
on all nodes in the air-gap environment.
286286
type: boolean
287+
airGappedInstallScriptPath:
288+
description: |-
289+
AirGappedInstallScriptPath is the path to the install script in the air-gapped environment.
290+
The install script should be prepared by the user. The value is only
291+
used when AirGapped is set to true (default: "/opt/install.sh").
292+
type: string
287293
kubeProxyArgs:
288294
description: KubeProxyArgs Customized flag for kube-proxy
289295
process
@@ -432,6 +438,11 @@ spec:
432438
the ''cloud-provider=external'' kubelet argument. (default:
433439
false)'
434440
type: boolean
441+
etcdProxyImage:
442+
description: 'Customized etcd proxy image for management
443+
cluster to communicate with workload cluster etcd (default:
444+
"alpine/socat")'
445+
type: string
435446
httpsListenPort:
436447
description: 'HTTPSListenPort HTTPS listen port (default:
437448
6443)'

‎bootstrap/controllers/kthreesconfig_controller.go

+66-30
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,11 @@ limitations under the License.
1717
package controllers
1818

1919
import (
20+
"bytes"
2021
"context"
2122
"errors"
2223
"fmt"
24+
"html/template"
2325
"time"
2426

2527
"github.com/go-logr/logr"
@@ -249,23 +251,24 @@ func (r *KThreesConfigReconciler) joinControlplane(ctx context.Context, scope *S
249251
}
250252

251253
if scope.Config.Spec.IsEtcdEmbedded() {
252-
etcdProxyFile := bootstrapv1.File{
253-
Path: etcd.EtcdProxyDaemonsetYamlLocation,
254-
Content: etcd.EtcdProxyDaemonsetYaml,
255-
Owner: "root:root",
256-
Permissions: "0640",
254+
etcdProxyFile, err := r.resolveEtcdProxyFile(scope.Config)
255+
if err != nil {
256+
conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
257+
return fmt.Errorf("failed to resolve etcd proxy file: %w", err)
257258
}
258-
files = append(files, etcdProxyFile)
259+
260+
files = append(files, *etcdProxyFile)
259261
}
260262

261263
cpInput := &cloudinit.ControlPlaneInput{
262264
BaseUserData: cloudinit.BaseUserData{
263-
PreK3sCommands: scope.Config.Spec.PreK3sCommands,
264-
PostK3sCommands: scope.Config.Spec.PostK3sCommands,
265-
AdditionalFiles: files,
266-
ConfigFile: workerConfigFile,
267-
K3sVersion: scope.Config.Spec.Version,
268-
AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
265+
PreK3sCommands: scope.Config.Spec.PreK3sCommands,
266+
PostK3sCommands: scope.Config.Spec.PostK3sCommands,
267+
AdditionalFiles: files,
268+
ConfigFile: workerConfigFile,
269+
K3sVersion: scope.Config.Spec.Version,
270+
AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
271+
AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath,
269272
},
270273
}
271274

@@ -320,12 +323,13 @@ func (r *KThreesConfigReconciler) joinWorker(ctx context.Context, scope *Scope)
320323

321324
winput := &cloudinit.WorkerInput{
322325
BaseUserData: cloudinit.BaseUserData{
323-
PreK3sCommands: scope.Config.Spec.PreK3sCommands,
324-
PostK3sCommands: scope.Config.Spec.PostK3sCommands,
325-
AdditionalFiles: files,
326-
ConfigFile: workerConfigFile,
327-
K3sVersion: scope.Config.Spec.Version,
328-
AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
326+
PreK3sCommands: scope.Config.Spec.PreK3sCommands,
327+
PostK3sCommands: scope.Config.Spec.PostK3sCommands,
328+
AdditionalFiles: files,
329+
ConfigFile: workerConfigFile,
330+
K3sVersion: scope.Config.Spec.Version,
331+
AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
332+
AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath,
329333
},
330334
}
331335

@@ -380,6 +384,38 @@ func (r *KThreesConfigReconciler) resolveSecretFileContent(ctx context.Context,
380384
return data, nil
381385
}
382386

387+
func (r *KThreesConfigReconciler) resolveEtcdProxyFile(cfg *bootstrapv1.KThreesConfig) (*bootstrapv1.File, error) {
388+
// Parse the template
389+
tpl, err := template.New("etcd-proxy").Parse(etcd.EtcdProxyDaemonsetYamlTemplate)
390+
if err != nil {
391+
return nil, fmt.Errorf("failed to parse etcd-proxy template: %w", err)
392+
}
393+
394+
// If user has set the systemDefaultRegistry, will prefix the image with it.
395+
systemDefaultRegistry := cfg.Spec.ServerConfig.SystemDefaultRegistry
396+
if systemDefaultRegistry != "" {
397+
systemDefaultRegistry = fmt.Sprintf("%s/", systemDefaultRegistry)
398+
}
399+
400+
// Render the template, the image name will be ${EtcdProxyImage} if the user
401+
// has set it, otherwise it will be ${SystemDefaultRegistry}alpine/socat
402+
var buf bytes.Buffer
403+
err = tpl.Execute(&buf, map[string]string{
404+
"EtcdProxyImage": cfg.Spec.ServerConfig.EtcdProxyImage,
405+
"SystemDefaultRegistry": systemDefaultRegistry,
406+
})
407+
if err != nil {
408+
return nil, fmt.Errorf("failed to render etcd-proxy template: %w", err)
409+
}
410+
411+
return &bootstrapv1.File{
412+
Path: etcd.EtcdProxyDaemonsetYamlLocation,
413+
Content: buf.String(),
414+
Owner: "root:root",
415+
Permissions: "0640",
416+
}, nil
417+
}
418+
383419
func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Context, scope *Scope) (_ ctrl.Result, reterr error) {
384420
// initialize the DataSecretAvailableCondition if missing.
385421
// this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing
@@ -465,23 +501,23 @@ func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Contex
465501
}
466502

467503
if scope.Config.Spec.IsEtcdEmbedded() {
468-
etcdProxyFile := bootstrapv1.File{
469-
Path: etcd.EtcdProxyDaemonsetYamlLocation,
470-
Content: etcd.EtcdProxyDaemonsetYaml,
471-
Owner: "root:root",
472-
Permissions: "0640",
504+
etcdProxyFile, err := r.resolveEtcdProxyFile(scope.Config)
505+
if err != nil {
506+
conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
507+
return ctrl.Result{}, fmt.Errorf("failed to resolve etcd proxy file: %w", err)
473508
}
474-
files = append(files, etcdProxyFile)
509+
files = append(files, *etcdProxyFile)
475510
}
476511

477512
cpinput := &cloudinit.ControlPlaneInput{
478513
BaseUserData: cloudinit.BaseUserData{
479-
PreK3sCommands: scope.Config.Spec.PreK3sCommands,
480-
PostK3sCommands: scope.Config.Spec.PostK3sCommands,
481-
AdditionalFiles: files,
482-
ConfigFile: initConfigFile,
483-
K3sVersion: scope.Config.Spec.Version,
484-
AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
514+
PreK3sCommands: scope.Config.Spec.PreK3sCommands,
515+
PostK3sCommands: scope.Config.Spec.PostK3sCommands,
516+
AdditionalFiles: files,
517+
ConfigFile: initConfigFile,
518+
K3sVersion: scope.Config.Spec.Version,
519+
AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
520+
AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath,
485521
},
486522
Certificates: certificates,
487523
}

‎bootstrap/controllers/kthreesconfig_controller_test.go

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllers
18+
19+
import (
20+
"testing"
21+
22+
. "github.com/onsi/gomega"
23+
24+
bootstrapv1 "github.com/k3s-io/cluster-api-k3s/bootstrap/api/v1beta2"
25+
)
26+
27+
func TestKThreesConfigReconciler_ResolveEtcdProxyFile(t *testing.T) {
28+
g := NewWithT(t)
29+
// If EtcdProxyImage is set, it should override the system default registry
30+
config := &bootstrapv1.KThreesConfig{
31+
Spec: bootstrapv1.KThreesConfigSpec{
32+
ServerConfig: bootstrapv1.KThreesServerConfig{
33+
EtcdProxyImage: "etcd-proxy-image",
34+
SystemDefaultRegistry: "system-default-registry",
35+
},
36+
},
37+
}
38+
r := &KThreesConfigReconciler{}
39+
etcdProxyFile, err := r.resolveEtcdProxyFile(config)
40+
g.Expect(err).ToNot(HaveOccurred())
41+
g.Expect(etcdProxyFile.Content).To(ContainSubstring("etcd-proxy-image"), "generated etcd proxy image should contain EtcdProxyImage")
42+
g.Expect(etcdProxyFile.Content).ToNot(ContainSubstring("system-default-registry"), "system-default-registry should be overwritten by EtcdProxyImage")
43+
44+
// If EtcdProxyImage is not set, the system default registry should be used
45+
config2 := &bootstrapv1.KThreesConfig{
46+
Spec: bootstrapv1.KThreesConfigSpec{
47+
ServerConfig: bootstrapv1.KThreesServerConfig{
48+
SystemDefaultRegistry: "system-default-registry2",
49+
},
50+
},
51+
}
52+
etcdProxyFile, err = r.resolveEtcdProxyFile(config2)
53+
g.Expect(err).ToNot(HaveOccurred())
54+
g.Expect(etcdProxyFile.Content).To(ContainSubstring("system-default-registry2/"), "generated etcd proxy image should be prefixed with SystemDefaultRegistry")
55+
}

‎controlplane/api/v1beta1/conversion.go

+2
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,11 @@ func (in *KThreesControlPlane) ConvertTo(dstRaw ctrlconversion.Hub) error {
9494
dst.Spec.KThreesConfigSpec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.KThreesConfigSpec.ServerConfig.DeprecatedDisableExternalCloudProvider
9595
dst.Spec.KThreesConfigSpec.ServerConfig.DisableCloudController = restored.Spec.KThreesConfigSpec.ServerConfig.DisableCloudController
9696
dst.Spec.KThreesConfigSpec.ServerConfig.SystemDefaultRegistry = restored.Spec.KThreesConfigSpec.ServerConfig.SystemDefaultRegistry
97+
dst.Spec.KThreesConfigSpec.ServerConfig.EtcdProxyImage = restored.Spec.KThreesConfigSpec.ServerConfig.EtcdProxyImage
9798
dst.Spec.MachineTemplate.NodeVolumeDetachTimeout = restored.Spec.MachineTemplate.NodeVolumeDetachTimeout
9899
dst.Spec.MachineTemplate.NodeDeletionTimeout = restored.Spec.MachineTemplate.NodeDeletionTimeout
99100
dst.Status.Version = restored.Status.Version
101+
dst.Spec.KThreesConfigSpec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.KThreesConfigSpec.AgentConfig.AirGappedInstallScriptPath
100102
return nil
101103
}
102104

0 commit comments

Comments
 (0)