Skip to content

Commit 7ddba2f

Browse files
xmudriikubermatic-bot
xmudrii
authored and
kubermatic-bot
committed
Use client-go for applying MachineDeployments and for waiting machine-controller to become Running (#204)
* Use client-go for creating MachineDeployments * Use client-go for waiting for machine-controller * Sync vendor * Rename Interface -> Client * Increase delay and timeout * Plural client argument names
1 parent a49b5d0 commit 7ddba2f

File tree

20 files changed

+1388
-124
lines changed

20 files changed

+1388
-124
lines changed

‎Gopkg.lock‎

Lines changed: 7 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/installer/installation/prerequisites.go‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"github.com/kubermatic/kubeone/pkg/config"
77
"github.com/kubermatic/kubeone/pkg/installer/util"
88
"github.com/kubermatic/kubeone/pkg/ssh"
9-
"github.com/kubermatic/kubeone/pkg/templates/machinecontroller"
109
)
1110

1211
const dockerVersion = "18.09.2"
@@ -23,15 +22,6 @@ func installPrerequisites(ctx *util.Context) error {
2322

2423
func generateConfigurationFiles(ctx *util.Context) error {
2524
ctx.Configuration.AddFile("cfg/cloud-config", ctx.Cluster.Provider.CloudConfig)
26-
27-
if len(ctx.Cluster.Workers) > 0 {
28-
machines, deployErr := machinecontroller.MachineDeployments(ctx.Cluster)
29-
if deployErr != nil {
30-
return fmt.Errorf("failed to create worker machine configuration: %v", deployErr)
31-
}
32-
ctx.Configuration.AddFile("workers.yaml", machines)
33-
}
34-
3525
return nil
3626
}
3727

‎pkg/installer/installation/worker.go‎

Lines changed: 11 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,7 @@ import (
55
"fmt"
66
"time"
77

8-
"github.com/kubermatic/kubeone/pkg/config"
98
"github.com/kubermatic/kubeone/pkg/installer/util"
10-
"github.com/kubermatic/kubeone/pkg/ssh"
119
"github.com/kubermatic/kubeone/pkg/templates/machinecontroller"
1210
)
1311

@@ -16,37 +14,17 @@ func createWorkerMachines(ctx *util.Context) error {
1614
return nil
1715
}
1816

19-
return ctx.RunTaskOnLeader(func(ctx *util.Context, _ *config.HostConfig, conn ssh.Connection) error {
20-
ctx.Logger.Infoln("Waiting for machine-controller to come up…")
21-
22-
cmd := fmt.Sprintf(
23-
`kubectl -n "%s" get pods -l '%s=%s' -o jsonpath='{.items[0].status.phase}'`,
24-
machinecontroller.WebhookNamespace,
25-
machinecontroller.WebhookAppLabelKey,
26-
machinecontroller.WebhookAppLabelValue,
27-
)
28-
if !ctx.Runner.WaitForCondition(cmd, 1*time.Minute, util.IsRunning) {
29-
return errors.New("machine-controller-webhook did not come up")
30-
}
31-
32-
cmd = fmt.Sprintf(
33-
`kubectl -n "%s" get pods -l '%s=%s' -o jsonpath='{.items[0].status.phase}'`,
34-
machinecontroller.MachineControllerNamespace,
35-
machinecontroller.MachineControllerAppLabelKey,
36-
machinecontroller.MachineControllerAppLabelValue,
37-
)
38-
if !ctx.Runner.WaitForCondition(cmd, 1*time.Minute, util.IsRunning) {
39-
return errors.New("machine-controller did not come up")
40-
}
41-
42-
// it can still take a bit before the MC is actually ready
43-
time.Sleep(10 * time.Second)
17+
ctx.Logger.Infoln("Waiting for machine-controller to come up…")
18+
if err := machinecontroller.WaitForWebhook(ctx.Clientset.CoreV1()); err != nil {
19+
return fmt.Errorf("machine-controller-webhook did not come up: %v", err)
20+
}
21+
if err := machinecontroller.WaitForMachineController(ctx.Clientset.CoreV1()); err != nil {
22+
return errors.New("machine-controller did not come up")
23+
}
4424

45-
ctx.Logger.Infoln("Creating worker machines…")
46-
_, _, err := ctx.Runner.Run(`kubectl apply -f ./{{ .WORK_DIR }}/workers.yaml`, util.TemplateVariables{
47-
"WORK_DIR": ctx.WorkDir,
48-
})
25+
// it can still take a bit before the MC is actually ready
26+
time.Sleep(10 * time.Second)
4927

50-
return err
51-
})
28+
ctx.Logger.Infoln("Creating worker machines…")
29+
return machinecontroller.DeployMachineDeployments(ctx)
5230
}

‎pkg/templates/ark/ark.go‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ func Deploy(ctx *util.Context) error {
2929
if ctx.APIExtensionClientset == nil {
3030
return errors.New("kubernetes apiextension clientset not initialized")
3131
}
32+
if ctx.RESTConfig == nil {
33+
return errors.New("kubernetes rest config not initialized")
34+
}
3235

3336
// Kubernetes clientsets
3437
coreClient := ctx.Clientset.CoreV1()
@@ -118,10 +121,10 @@ func Deploy(ctx *util.Context) error {
118121
return nil
119122
}
120123

121-
func ensureBackupStorageLocation(backupLocationInterface arkclientset.BackupStorageLocationInterface, required *arkv1.BackupStorageLocation) error {
122-
existing, err := backupLocationInterface.Get(required.Name, metav1.GetOptions{})
124+
func ensureBackupStorageLocation(backupStorageLocationsClient arkclientset.BackupStorageLocationInterface, required *arkv1.BackupStorageLocation) error {
125+
existing, err := backupStorageLocationsClient.Get(required.Name, metav1.GetOptions{})
123126
if apierrors.IsNotFound(err) {
124-
_, err = backupLocationInterface.Create(required)
127+
_, err = backupStorageLocationsClient.Create(required)
125128
return err
126129
}
127130
if err != nil {
@@ -135,14 +138,14 @@ func ensureBackupStorageLocation(backupLocationInterface arkclientset.BackupStor
135138
return nil
136139
}
137140

138-
_, err = backupLocationInterface.Update(existing)
141+
_, err = backupStorageLocationsClient.Update(existing)
139142
return err
140143
}
141144

142-
func ensureVolumeSnapshotLocation(snapshotLocationInterface arkclientset.VolumeSnapshotLocationInterface, required *arkv1.VolumeSnapshotLocation) error {
143-
existing, err := snapshotLocationInterface.Get(required.Name, metav1.GetOptions{})
145+
func ensureVolumeSnapshotLocation(volumeSnapshotLocationsClient arkclientset.VolumeSnapshotLocationInterface, required *arkv1.VolumeSnapshotLocation) error {
146+
existing, err := volumeSnapshotLocationsClient.Get(required.Name, metav1.GetOptions{})
144147
if apierrors.IsNotFound(err) {
145-
_, err = snapshotLocationInterface.Create(required)
148+
_, err = volumeSnapshotLocationsClient.Create(required)
146149
return err
147150
}
148151
if err != nil {
@@ -156,6 +159,6 @@ func ensureVolumeSnapshotLocation(snapshotLocationInterface arkclientset.VolumeS
156159
return nil
157160
}
158161

159-
_, err = snapshotLocationInterface.Update(existing)
162+
_, err = volumeSnapshotLocationsClient.Update(existing)
160163
return err
161164
}

‎pkg/templates/machinecontroller/deployment.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ package machinecontroller
22

33
import (
44
"errors"
5+
"fmt"
56
"net"
7+
"time"
68

79
"github.com/kubermatic/kubeone/pkg/config"
810
"github.com/kubermatic/kubeone/pkg/installer/util"
@@ -14,6 +16,8 @@ import (
1416
apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
1517
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1618
"k8s.io/apimachinery/pkg/util/intstr"
19+
"k8s.io/apimachinery/pkg/util/wait"
20+
corev1types "k8s.io/client-go/kubernetes/typed/core/v1"
1721
"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
1822
)
1923

@@ -134,6 +138,22 @@ func Deploy(ctx *util.Context) error {
134138
return nil
135139
}
136140

141+
// WaitForMachineController waits for machine-controller-webhook to become running
142+
func WaitForMachineController(corev1Client corev1types.CoreV1Interface) error {
143+
return wait.Poll(5*time.Second, 3*time.Minute, func() (bool, error) {
144+
machineControllerPods, err := corev1Client.Pods(WebhookNamespace).List(metav1.ListOptions{
145+
LabelSelector: fmt.Sprintf("%s=%s", MachineControllerAppLabelKey, MachineControllerAppLabelValue),
146+
})
147+
if err != nil {
148+
return false, err
149+
}
150+
if machineControllerPods.Items[0].Status.Phase == corev1.PodRunning {
151+
return true, nil
152+
}
153+
return false, nil
154+
})
155+
}
156+
137157
func machineControllerServiceAccount() *corev1.ServiceAccount {
138158
return &corev1.ServiceAccount{
139159
TypeMeta: metav1.TypeMeta{

‎pkg/templates/machinecontroller/machines.go‎

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,19 @@ import (
66
"fmt"
77

88
"github.com/kubermatic/kubeone/pkg/config"
9+
"github.com/kubermatic/kubeone/pkg/installer/util"
910
"github.com/kubermatic/kubeone/pkg/templates"
1011

12+
"k8s.io/apimachinery/pkg/api/equality"
13+
apierrors "k8s.io/apimachinery/pkg/api/errors"
1114
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1215
"k8s.io/apimachinery/pkg/labels"
1316
"k8s.io/apimachinery/pkg/runtime"
1417
"k8s.io/apimachinery/pkg/util/intstr"
1518
clustercommon "sigs.k8s.io/cluster-api/pkg/apis/cluster/common"
1619
clusterv1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
20+
clusterclientset "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset"
21+
clustertypes "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1"
1722
)
1823

1924
type providerSpec struct {
@@ -24,20 +29,36 @@ type providerSpec struct {
2429
OperatingSystemSpec interface{} `json:"operatingSystemSpec"`
2530
}
2631

27-
// MachineDeployments returns YAML manifests for MachineDeployments
28-
func MachineDeployments(cluster *config.Cluster) (string, error) {
29-
deployments := make([]interface{}, 0)
32+
// DeployMachineDeployments deploys MachineDeployments that create appropriate machines
33+
func DeployMachineDeployments(ctx *util.Context) error {
34+
if ctx.Clientset == nil {
35+
return errors.New("kubernetes clientset not initialized")
36+
}
37+
if ctx.RESTConfig == nil {
38+
return errors.New("kubernetes rest config not initialized")
39+
}
40+
41+
// Create Cluster-API clientset
42+
clusterapiClientset, err := clusterclientset.NewForConfig(ctx.RESTConfig)
43+
if err != nil {
44+
return err
45+
}
46+
clusterapiClient := clusterapiClientset.ClusterV1alpha1()
3047

31-
for _, workerset := range cluster.Workers {
32-
deployment, err := createMachineDeployment(cluster, workerset)
48+
// Apply MachineDeployments
49+
for _, workerset := range ctx.Cluster.Workers {
50+
deployment, err := createMachineDeployment(ctx.Cluster, workerset)
3351
if err != nil {
34-
return "", err
52+
return err
3553
}
3654

37-
deployments = append(deployments, deployment)
55+
err = ensureMachineDeployment(clusterapiClient.MachineDeployments(deployment.Namespace), deployment)
56+
if err != nil {
57+
return err
58+
}
3859
}
3960

40-
return templates.KubernetesToYAML(deployments)
61+
return nil
4162
}
4263

4364
func createMachineDeployment(cluster *config.Cluster, workerset config.WorkerConfig) (*clusterv1alpha1.MachineDeployment, error) {
@@ -113,6 +134,27 @@ func createMachineDeployment(cluster *config.Cluster, workerset config.WorkerCon
113134
}, nil
114135
}
115136

137+
func ensureMachineDeployment(machineDeploymentsClient clustertypes.MachineDeploymentInterface, required *clusterv1alpha1.MachineDeployment) error {
138+
existing, err := machineDeploymentsClient.Get(required.Name, metav1.GetOptions{})
139+
if apierrors.IsNotFound(err) {
140+
_, err = machineDeploymentsClient.Create(required)
141+
return err
142+
}
143+
if err != nil {
144+
return err
145+
}
146+
147+
modified := false
148+
templates.MergeStringMap(&modified, &existing.ObjectMeta.Annotations, required.ObjectMeta.Annotations)
149+
templates.MergeStringMap(&modified, &existing.ObjectMeta.Labels, required.ObjectMeta.Labels)
150+
if equality.Semantic.DeepEqual(required.Spec, existing.Spec) && !modified {
151+
return nil
152+
}
153+
154+
_, err = machineDeploymentsClient.Update(existing)
155+
return err
156+
}
157+
116158
func machineSpec(cluster *config.Cluster, workerset config.WorkerConfig, provider config.ProviderName) (map[string]interface{}, error) {
117159
var err error
118160

0 commit comments

Comments
 (0)