Skip to content

Commit df6c654

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #1771 from shysank/v1beta1_upgrade_tests
v1beta1 cluster upgrade tests (using clusterctl upgrade)
2 parents 4482f7f + aefd581 commit df6c654

9 files changed

Lines changed: 474 additions & 65 deletions

File tree

‎api/v1beta1/azurecluster_webhook.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,20 @@ func (c *AzureCluster) ValidateUpdate(oldRaw runtime.Object) error {
8585
}
8686

8787
if !reflect.DeepEqual(c.Spec.AzureEnvironment, old.Spec.AzureEnvironment) {
88-
allErrs = append(allErrs,
89-
field.Invalid(field.NewPath("spec", "AzureEnvironment"),
90-
c.Spec.AzureEnvironment, "field is immutable"),
91-
)
88+
// The equality failure could be because of default mismatch between v1alpha3 and v1beta1. This happens because
89+
// the new object `r` will have run through the default webhooks but the old object `old` would not have so.
90+
// This means if the old object was in v1alpha3, it would not get the new defaults set in v1beta1 resulting
91+
// in object inequality. To workaround this, we set the v1beta1 defaults here so that the old object also gets
92+
// the new defaults.
93+
old.setAzureEnvironmentDefault()
94+
95+
// if it's still not equal, return error.
96+
if !reflect.DeepEqual(c.Spec.AzureEnvironment, old.Spec.AzureEnvironment) {
97+
allErrs = append(allErrs,
98+
field.Invalid(field.NewPath("spec", "AzureEnvironment"),
99+
c.Spec.AzureEnvironment, "field is immutable"),
100+
)
101+
}
92102
}
93103

94104
if !reflect.DeepEqual(c.Spec.NetworkSpec.PrivateDNSZoneName, old.Spec.NetworkSpec.PrivateDNSZoneName) {

‎api/v1beta1/azurecluster_webhook_test.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,16 @@ func TestAzureCluster_ValidateUpdate(t *testing.T) {
234234
},
235235
wantErr: true,
236236
},
237+
{
238+
name: "azurecluster azureEnvironment default mismatch",
239+
oldCluster: createValidCluster(),
240+
cluster: func() *AzureCluster {
241+
cluster := createValidCluster()
242+
cluster.Spec.AzureEnvironment = "AzurePublicCloud"
243+
return cluster
244+
}(),
245+
wantErr: false,
246+
},
237247
{
238248
name: "control plane outbound lb is immutable",
239249
oldCluster: &AzureCluster{

‎api/v1beta1/azuremachinetemplate_webhook.go‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,19 @@ func (r *AzureMachineTemplate) ValidateUpdate(oldRaw runtime.Object) error {
6464
old := oldRaw.(*AzureMachineTemplate)
6565

6666
if !reflect.DeepEqual(r.Spec.Template.Spec, old.Spec.Template.Spec) {
67-
allErrs = append(allErrs,
68-
field.Invalid(field.NewPath("AzureMachineTemplate", "spec", "template", "spec"), r, AzureMachineTemplateImmutableMsg),
69-
)
67+
// The equality failure could be because of default mismatch between v1alpha3 and v1beta1. This happens because
68+
// the new object `r` will have run through the default webhooks but the old object `old` would not have so.
69+
// This means if the old object was in v1alpha3, it would not get the new defaults set in v1beta1 resulting
70+
// in object inequality. To workaround this, we set the v1beta1 defaults here so that the old object also gets
71+
// the new defaults.
72+
old.Default()
73+
74+
// if it's still not equal, return error.
75+
if !reflect.DeepEqual(r.Spec.Template.Spec, old.Spec.Template.Spec) {
76+
allErrs = append(allErrs,
77+
field.Invalid(field.NewPath("AzureMachineTemplate", "spec", "template", "spec"), r, AzureMachineTemplateImmutableMsg),
78+
)
79+
}
7080
}
7181

7282
if len(allErrs) == 0 {
@@ -83,5 +93,7 @@ func (r *AzureMachineTemplate) ValidateDelete() error {
8393
// Default implements webhookutil.defaulter so a webhook will be registered for the type.
8494
func (r *AzureMachineTemplate) Default() {
8595
machinetemplatelog.Info("default", "name", r.Name)
86-
r.Spec.Template.Spec.SetDefaults(machinetemplatelog)
96+
r.Spec.Template.Spec.SetDefaultCachingType()
97+
r.Spec.Template.Spec.SetDataDisksDefaults()
98+
r.Spec.Template.Spec.SetIdentityDefaults()
8799
}

‎api/v1beta1/azuremachinetemplate_webhook_test.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,49 @@ func TestAzureMachineTemplate_ValidateUpdate(t *testing.T) {
226226
},
227227
wantErr: false,
228228
},
229+
{
230+
name: "AzureMachineTemplate with default mismatch",
231+
oldTemplate: &AzureMachineTemplate{
232+
Spec: AzureMachineTemplateSpec{
233+
Template: AzureMachineTemplateResource{
234+
Spec: AzureMachineSpec{
235+
VMSize: "size",
236+
FailureDomain: &failureDomain,
237+
OSDisk: OSDisk{
238+
OSType: "type",
239+
DiskSizeGB: to.Int32Ptr(11),
240+
CachingType: "",
241+
},
242+
DataDisks: []DataDisk{},
243+
SSHPublicKey: "",
244+
},
245+
},
246+
},
247+
ObjectMeta: metav1.ObjectMeta{
248+
Name: "OldTemplate",
249+
},
250+
},
251+
template: &AzureMachineTemplate{
252+
Spec: AzureMachineTemplateSpec{
253+
Template: AzureMachineTemplateResource{
254+
Spec: AzureMachineSpec{
255+
VMSize: "size",
256+
FailureDomain: &failureDomain,
257+
OSDisk: OSDisk{
258+
OSType: "type",
259+
DiskSizeGB: to.Int32Ptr(11),
260+
CachingType: "None",
261+
},
262+
DataDisks: []DataDisk{},
263+
},
264+
},
265+
},
266+
ObjectMeta: metav1.ObjectMeta{
267+
Name: "NewTemplate",
268+
},
269+
},
270+
wantErr: false,
271+
},
229272
}
230273

231274
for _, amt := range tests {

‎test/e2e/capi_test.go‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@ package e2e
2020

2121
import (
2222
"context"
23+
"encoding/base64"
2324
"fmt"
24-
"os"
25-
2625
. "github.com/onsi/ginkgo"
2726
. "github.com/onsi/gomega"
2827
corev1 "k8s.io/api/core/v1"
2928
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29+
"os"
3030
e2e_namespace "sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/namespace"
3131
clusterctl "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
3232
capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
@@ -228,4 +228,34 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
228228
}
229229
})
230230
})
231+
232+
if os.Getenv("LOCAL_ONLY") != "true" {
233+
Context("upgrade from v1alpha3 to v1beta1, and scale workload clusters created in v1alpha3", func() {
234+
BeforeEach(func() {
235+
// Unset resource group and vnet env variables, since we capi test creates 2 clusters,
236+
// and will result in both the clusters using the same vnet and resource group.
237+
Expect(os.Unsetenv(AzureResourceGroup)).To(Succeed())
238+
Expect(os.Unsetenv(AzureVNetName)).To(Succeed())
239+
240+
// Set base64 encoded values for v1alpha3 cluster.
241+
Expect(os.Setenv("AZURE_CLIENT_ID_B64", base64.StdEncoding.EncodeToString([]byte(os.Getenv(AzureClientId))))).To(Succeed())
242+
Expect(os.Setenv("AZURE_CLIENT_SECRET_B64", base64.StdEncoding.EncodeToString([]byte(os.Getenv(AzureClientSecret))))).To(Succeed())
243+
Expect(os.Setenv("AZURE_SUBSCRIPTION_ID_B64", base64.StdEncoding.EncodeToString([]byte(os.Getenv("AZURE_SUBSCRIPTION_ID"))))).To(Succeed())
244+
Expect(os.Setenv("AZURE_TENANT_ID_B64", base64.StdEncoding.EncodeToString([]byte(os.Getenv("AZURE_TENANT_ID"))))).To(Succeed())
245+
246+
// Unset windows specific variables
247+
Expect(os.Unsetenv("WINDOWS_WORKER_MACHINE_COUNT")).To(Succeed())
248+
Expect(os.Unsetenv("K8S_FEATURE_GATES")).To(Succeed())
249+
})
250+
capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
251+
return capi_e2e.ClusterctlUpgradeSpecInput{
252+
E2EConfig: e2eConfig,
253+
ClusterctlConfigPath: clusterctlConfigPath,
254+
BootstrapClusterProxy: bootstrapClusterProxy,
255+
ArtifactFolder: artifactFolder,
256+
SkipCleanup: skipCleanup,
257+
}
258+
})
259+
})
260+
}
231261
})

‎test/e2e/config/azure-dev.yaml‎

Lines changed: 94 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,15 @@ providers:
88
- name: cluster-api
99
type: CoreProvider
1010
versions:
11+
- name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
12+
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/core-components.yaml
13+
type: url
14+
contract: v1alpha3
15+
files:
16+
- sourcePath: "../data/shared/v1alpha3/metadata.yaml"
17+
replacements:
18+
- old: "imagePullPolicy: Always"
19+
new: "imagePullPolicy: IfNotPresent"
1120
- name: v1.0.0
1221
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/core-components.yaml
1322
type: url
@@ -16,11 +25,20 @@ providers:
1625
replacements:
1726
- old: "imagePullPolicy: Always"
1827
new: "imagePullPolicy: IfNotPresent"
19-
- old: "--leader-elect"
20-
new: "--leader-elect=false"
28+
29+
2130
- name: kubeadm
2231
type: BootstrapProvider
2332
versions:
33+
- name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
34+
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/bootstrap-components.yaml
35+
type: url
36+
contract: v1alpha3
37+
files:
38+
- sourcePath: "../data/shared/v1alpha3/metadata.yaml"
39+
replacements:
40+
- old: "imagePullPolicy: Always"
41+
new: "imagePullPolicy: IfNotPresent"
2442
- name: v1.0.0
2543
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/bootstrap-components.yaml
2644
type: url
@@ -29,11 +47,19 @@ providers:
2947
replacements:
3048
- old: "imagePullPolicy: Always"
3149
new: "imagePullPolicy: IfNotPresent"
32-
- old: "--leader-elect"
33-
new: "--leader-elect=false"
50+
3451
- name: kubeadm
3552
type: ControlPlaneProvider
3653
versions:
54+
- name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
55+
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/control-plane-components.yaml
56+
type: url
57+
contract: v1alpha3
58+
files:
59+
- sourcePath: "../data/shared/v1alpha3/metadata.yaml"
60+
replacements:
61+
- old: "imagePullPolicy: Always"
62+
new: "imagePullPolicy: IfNotPresent"
3763
- name: v1.0.0
3864
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/control-plane-components.yaml
3965
type: url
@@ -42,66 +68,74 @@ providers:
4268
replacements:
4369
- old: "imagePullPolicy: Always"
4470
new: "imagePullPolicy: IfNotPresent"
45-
- old: "--leader-elect"
46-
new: "--leader-elect=false"
71+
4772
- name: azure
4873
type: InfrastructureProvider
4974
versions:
50-
- name: v1.0.0
75+
- name: v0.4.15 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
76+
value: https://github.com/kubernetes-sigs/cluster-api-provider-azure/releases/download/v0.4.15/infrastructure-components.yaml
77+
type: url
78+
contract: v1alpha3
79+
files:
80+
- sourcePath: "../data/shared/v1alpha3_provider/metadata.yaml"
81+
- sourcePath: "../data/infrastructure-azure/v1alpha3/cluster-template-prow.yaml"
82+
targetName: "cluster-template.yaml"
83+
replacements:
84+
- old: "imagePullPolicy: Always"
85+
new: "imagePullPolicy: IfNotPresent"
86+
- name: v1.0.99 # next; use manifest from source files
5187
value: "${PWD}/config/default"
5288
files:
5389
- sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
90+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow.yaml"
91+
targetName: "cluster-template.yaml"
92+
- sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template.yaml"
93+
targetName: "cluster-template-management.yaml"
94+
- sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-kcp-adoption.yaml"
95+
targetName: "cluster-template-kcp-adoption.yaml"
96+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ipv6.yaml"
97+
targetName: "cluster-template-ipv6.yaml"
98+
- sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml"
99+
targetName: "cluster-template-md-remediation.yaml"
100+
- sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml"
101+
targetName: "cluster-template-kcp-remediation.yaml"
102+
- sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml"
103+
targetName: "cluster-template-kcp-scale-in.yaml"
104+
- sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml"
105+
targetName: "cluster-template-node-drain.yaml"
106+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool.yaml"
107+
targetName: "cluster-template-machine-pool.yaml"
108+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml"
109+
targetName: "cluster-template-nvidia-gpu.yaml"
110+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-private.yaml"
111+
targetName: "cluster-template-private.yaml"
112+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version.yaml"
113+
targetName: "cluster-template-conformance-ci-artifacts.yaml"
114+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-windows.yaml"
115+
targetName: "cluster-template-conformance-ci-artifacts-windows.yaml"
116+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml"
117+
targetName: "cluster-template-conformance-ci-artifacts-windows-containerd-2022.yaml"
118+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version.yaml"
119+
targetName: "cluster-template-conformance-ci-artifacts-windows-containerd.yaml"
120+
- sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds.yaml"
121+
targetName: "cluster-template-conformance-presubmit-artifacts.yaml"
122+
- sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds.yaml"
123+
targetName: "cluster-template-conformance-presubmit-artifacts-windows-containerd.yaml"
124+
- sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds-windows.yaml"
125+
targetName: "cluster-template-conformance-presubmit-artifacts-windows.yaml"
126+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-windows.yaml"
127+
targetName: "cluster-template-windows.yaml"
128+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml"
129+
targetName: "cluster-template-machine-pool-windows.yaml"
130+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml"
131+
targetName: "cluster-template-external-cloud-provider.yaml"
132+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks-multi-tenancy.yaml"
133+
targetName: "cluster-template-aks-multi-tenancy.yaml"
134+
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-custom-vnet.yaml"
135+
targetName: "cluster-template-custom-vnet.yaml"
54136
replacements:
55137
- old: "--v=0"
56138
new: "--v=2"
57-
files:
58-
- sourcePath: "../data/shared/v1beta1/metadata.yaml"
59-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow.yaml"
60-
targetName: "cluster-template.yaml"
61-
- sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template.yaml"
62-
targetName: "cluster-template-management.yaml"
63-
- sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-kcp-adoption.yaml"
64-
targetName: "cluster-template-kcp-adoption.yaml"
65-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ipv6.yaml"
66-
targetName: "cluster-template-ipv6.yaml"
67-
- sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml"
68-
targetName: "cluster-template-md-remediation.yaml"
69-
- sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml"
70-
targetName: "cluster-template-kcp-remediation.yaml"
71-
- sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml"
72-
targetName: "cluster-template-kcp-scale-in.yaml"
73-
- sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml"
74-
targetName: "cluster-template-node-drain.yaml"
75-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool.yaml"
76-
targetName: "cluster-template-machine-pool.yaml"
77-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml"
78-
targetName: "cluster-template-nvidia-gpu.yaml"
79-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-private.yaml"
80-
targetName: "cluster-template-private.yaml"
81-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version.yaml"
82-
targetName: "cluster-template-conformance-ci-artifacts.yaml"
83-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-windows.yaml"
84-
targetName: "cluster-template-conformance-ci-artifacts-windows.yaml"
85-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml"
86-
targetName: "cluster-template-conformance-ci-artifacts-windows-containerd-2022.yaml"
87-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version.yaml"
88-
targetName: "cluster-template-conformance-ci-artifacts-windows-containerd.yaml"
89-
- sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds.yaml"
90-
targetName: "cluster-template-conformance-presubmit-artifacts.yaml"
91-
- sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds.yaml"
92-
targetName: "cluster-template-conformance-presubmit-artifacts-windows-containerd.yaml"
93-
- sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds-windows.yaml"
94-
targetName: "cluster-template-conformance-presubmit-artifacts-windows.yaml"
95-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-windows.yaml"
96-
targetName: "cluster-template-windows.yaml"
97-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml"
98-
targetName: "cluster-template-machine-pool-windows.yaml"
99-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml"
100-
targetName: "cluster-template-external-cloud-provider.yaml"
101-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks-multi-tenancy.yaml"
102-
targetName: "cluster-template-aks-multi-tenancy.yaml"
103-
- sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-custom-vnet.yaml"
104-
targetName: "cluster-template-custom-vnet.yaml"
105139

106140
variables:
107141
KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.22.1}"
@@ -125,6 +159,11 @@ variables:
125159
CLUSTER_IDENTITY_NAME: "cluster-identity"
126160
NODE_DRAIN_TIMEOUT: "60s"
127161
CI_VERSION: ""
162+
# NOTE: INIT_WITH_BINARY and INIT_WITH_KUBERNETES_VERSION are only used by the clusterctl upgrade test to initialize
163+
# the management cluster to be upgraded.
164+
INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/clusterctl-{OS}-{ARCH}"
165+
INIT_WITH_PROVIDERS_CONTRACT: "v1alpha3"
166+
INIT_WITH_KUBERNETES_VERSION: "v1.21.2"
128167

129168
intervals:
130169
default/wait-controllers: ["3m", "10s"]

0 commit comments

Comments
 (0)