Skip to content

Commit 3044c4a

Browse files
sipriyaasipriyaa
committed
management workload domain isolation automation
1 parent 17c4f15 commit 3044c4a

4 files changed

+373
-6
lines changed

‎tests/e2e/e2e_common.go

+7
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,13 @@ var (
461461
envTopologySetupType = "TOPOLOGY_SETUP_TYPE"
462462
)
463463

464+
// For management workload domain isolation
465+
var (
466+
envZonal2StoragePolicyName = "ZONAL2_STORAGECLASS"
467+
envWrkldDomain1ZoneName = "WORKLOAD_1_ZONE_NAME"
468+
topologyDomainIsolation = "Workload_Management_Isolation"
469+
)
470+
464471
// GetAndExpectStringEnvVar parses a string from env variable.
465472
func GetAndExpectStringEnvVar(varName string) string {
466473
varValue := os.Getenv(varName)

‎tests/e2e/mgmt_wrkld_domain_isolation.go

+150
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e
18+
19+
import (
20+
"context"
21+
"fmt"
22+
23+
"github.com/onsi/ginkgo/v2"
24+
"github.com/onsi/gomega"
25+
26+
v1 "k8s.io/api/core/v1"
27+
apierrors "k8s.io/apimachinery/pkg/api/errors"
28+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29+
clientset "k8s.io/client-go/kubernetes"
30+
"k8s.io/kubernetes/test/e2e/framework"
31+
fss "k8s.io/kubernetes/test/e2e/framework/statefulset"
32+
admissionapi "k8s.io/pod-security-admission/api"
33+
)
34+
35+
var _ bool = ginkgo.Describe("[domain-isolation] Management-Workload-Domain-Isolation", func() {
36+
37+
f := framework.NewDefaultFramework("domain-isolation")
38+
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
39+
f.SkipNamespaceCreation = true // tests will create their own namespaces
40+
var (
41+
client clientset.Interface
42+
namespace string
43+
storageProfileId string
44+
vcRestSessionId string
45+
allowedTopologies []v1.TopologySelectorLabelRequirement
46+
storagePolicyName string
47+
replicas int32
48+
topkeyStartIndex int
49+
)
50+
51+
ginkgo.BeforeEach(func() {
52+
ctx, cancel := context.WithCancel(context.Background())
53+
defer cancel()
54+
55+
// making vc connection
56+
client = f.ClientSet
57+
bootstrap()
58+
59+
// reading vc session id
60+
vcRestSessionId = createVcSession4RestApis(ctx)
61+
62+
// reading topology map set for management doamin and workload domain
63+
topologyMap := GetAndExpectStringEnvVar(envTopologyMap)
64+
allowedTopologies = createAllowedTopolgies(topologyMap)
65+
})
66+
67+
ginkgo.AfterEach(func() {
68+
ctx, cancel := context.WithCancel(context.Background())
69+
defer cancel()
70+
71+
ginkgo.By(fmt.Sprintf("Deleting all statefulsets in namespace: %v", namespace))
72+
fss.DeleteAllStatefulSets(ctx, client, namespace)
73+
74+
ginkgo.By(fmt.Sprintf("Deleting service nginx in namespace: %v", namespace))
75+
err := client.CoreV1().Services(namespace).Delete(ctx, servicename, *metav1.NewDeleteOptions(0))
76+
if !apierrors.IsNotFound(err) {
77+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
78+
}
79+
})
80+
81+
/*
82+
Testcase-1
83+
Basic test
84+
Deploy statefulsets with 1 replica on namespace-1 in the supervisor cluster using vsan-zonal policy with
85+
immediate volume binding mode storageclass.
86+
87+
Steps:
88+
1. Create a wcp namespace and tagged it to zone-2 workload zone.
89+
2. Read a zonal storage policy which is tagged to wcp namespace created in step #1 using Immediate Binding mode.
90+
3. Create statefulset with replica count 1.
91+
4. Wait for PVC and PV to reach Bound state.
92+
5. Verify PVC has csi.vsphere.volume-accessible-topology annotation with zone-2
93+
6. Verify PV has node affinity rule for zone-2
94+
7. Verify statefulset pod is in up and running state.
95+
8. Veirfy Pod node annoation.
96+
9. Perform cleanup: Delete Statefulset
97+
10. Perform cleanup: Delete PVC
98+
*/
99+
100+
ginkgo.It("Verifying volume creation and pv affinities when svc namespace is tagged to zonal-2 policy", func() {
101+
ctx, cancel := context.WithCancel(context.Background())
102+
defer cancel()
103+
104+
// statefulset replica count
105+
replicas = 1
106+
107+
// reading zonal storage policy of zone-2 workload domain
108+
storagePolicyName = GetAndExpectStringEnvVar(envZonal2StoragePolicyName)
109+
storageProfileId = e2eVSphere.GetSpbmPolicyID(storagePolicyName)
110+
111+
wrkld1ZoneName := GetAndExpectStringEnvVar(envWrkldDomain1ZoneName)
112+
113+
/*
114+
EX - zone -> zone-1, zone-2, zone-3, zone-4
115+
so topValStartIndex=1 and topValEndIndex=2 will fetch the 1st index value from topology map string
116+
*/
117+
topValStartIndex := 1
118+
topValEndIndex := 2
119+
120+
ginkgo.By("Create a WCP namespace tagged to zone-2")
121+
allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex,
122+
topValEndIndex)
123+
namespace = createTestWcpNsWithZones(
124+
vcRestSessionId, storageProfileId, getSvcId(vcRestSessionId), []string{wrkld1ZoneName})
125+
126+
ginkgo.By("Read wcp namespace tagged zonal storage class")
127+
storageclass, err := client.StorageV1().StorageClasses().Get(ctx, storagePolicyName, metav1.GetOptions{})
128+
if !apierrors.IsNotFound(err) {
129+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
130+
}
131+
132+
ginkgo.By("Creating service")
133+
service := CreateService(namespace, client)
134+
defer func() {
135+
deleteService(namespace, client, service)
136+
}()
137+
138+
ginkgo.By("Creating statefulset")
139+
statefulset := createCustomisedStatefulSets(ctx, client, namespace, true, replicas, false, nil,
140+
false, true, "", "", storageclass, storageclass.Name)
141+
defer func() {
142+
fss.DeleteAllStatefulSets(ctx, client, namespace)
143+
}()
144+
145+
ginkgo.By("Verify svc pv affinity, pvc annotation and pod node affinity")
146+
err = verifyAnnotationsAndNodeAffinityForStatefulsetinSvc(ctx, client, statefulset, namespace,
147+
allowedTopologies)
148+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
149+
})
150+
})

‎tests/e2e/mgmt_wrkld_domain_isolation_utils.go

+206
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e
18+
19+
import (
20+
"context"
21+
"encoding/json"
22+
"fmt"
23+
"math/rand"
24+
"strings"
25+
"time"
26+
27+
"github.com/onsi/gomega"
28+
appsv1 "k8s.io/api/apps/v1"
29+
v1 "k8s.io/api/core/v1"
30+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
31+
clientset "k8s.io/client-go/kubernetes"
32+
"k8s.io/kubernetes/test/e2e/framework"
33+
fnodes "k8s.io/kubernetes/test/e2e/framework/node"
34+
)
35+
36+
/*
37+
This util will verify supervisor pvc annotation, pv affinity rules,
38+
pod node anotation and cns volume metadata
39+
*/
40+
func verifyAnnotationsAndNodeAffinityForStatefulsetinSvc(ctx context.Context, client clientset.Interface,
41+
statefulset *appsv1.StatefulSet, namespace string,
42+
allowedTopologies []v1.TopologySelectorLabelRequirement) error {
43+
// Read topology mapping
44+
allowedTopologiesMap := createAllowedTopologiesMap(allowedTopologies)
45+
topologyMap := GetAndExpectStringEnvVar(envTopologyMap)
46+
_, topologyCategories := createTopologyMapLevel5(topologyMap)
47+
48+
framework.Logf("Reading statefulset pod list for node affinity verification")
49+
ssPodsBeforeScaleDown := GetListOfPodsInSts(client, statefulset)
50+
for _, sspod := range ssPodsBeforeScaleDown.Items {
51+
// Get Pod details
52+
_, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{})
53+
if err != nil {
54+
return fmt.Errorf("failed to get pod %s in namespace %s: %w", sspod.Name, namespace, err)
55+
}
56+
57+
framework.Logf("Verifying PVC annotation and PV affinity rules")
58+
for _, volumespec := range sspod.Spec.Volumes {
59+
if volumespec.PersistentVolumeClaim != nil {
60+
svPvcName := volumespec.PersistentVolumeClaim.ClaimName
61+
pv := getPvFromClaim(client, statefulset.Namespace, svPvcName)
62+
63+
// Get SVC PVC
64+
svcPVC, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, svPvcName, metav1.GetOptions{})
65+
if err != nil {
66+
return fmt.Errorf("failed to get SVC PVC %s in namespace %s: %w", svPvcName, namespace, err)
67+
}
68+
69+
// Get ready and schedulable nodes
70+
nodeList, err := fnodes.GetReadySchedulableNodes(ctx, client)
71+
if err != nil {
72+
return fmt.Errorf("failed to get ready and schedulable nodes: %w", err)
73+
}
74+
if len(nodeList.Items) <= 0 {
75+
return fmt.Errorf("no ready and schedulable nodes found")
76+
}
77+
78+
// Verify SV PVC topology annotations
79+
err = checkPvcTopologyAnnotationOnSvc(svcPVC, allowedTopologiesMap, topologyCategories)
80+
if err != nil {
81+
return fmt.Errorf("topology annotation verification failed for SVC PVC %s: %w", svcPVC.Name, err)
82+
}
83+
84+
// Verify SV PV node affinity details
85+
svcPV := getPvFromClaim(client, namespace, svPvcName)
86+
_, err = verifyVolumeTopologyForLevel5(svcPV, allowedTopologiesMap)
87+
if err != nil {
88+
return fmt.Errorf("topology verification failed for SVC PV %s: %w", svcPV.Name, err)
89+
}
90+
91+
// Verify pod node annotation
92+
_, err = verifyPodLocationLevel5(&sspod, nodeList, allowedTopologiesMap)
93+
if err != nil {
94+
return fmt.Errorf("pod node annotation verification failed for pod %s: %w", sspod.Name, err)
95+
}
96+
97+
// Verify CNS volume metadata
98+
err = verifyVolumeMetadataInCNS(&e2eVSphere, pv.Spec.CSI.VolumeHandle, svPvcName, pv.ObjectMeta.Name, sspod.Name)
99+
if err != nil {
100+
return fmt.Errorf("CNS volume metadata verification failed for pod %s: %w", sspod.Name, err)
101+
}
102+
}
103+
}
104+
}
105+
return nil
106+
}
107+
108+
109+
// Function to check annotation on a Supervisor PVC
110+
func checkPvcTopologyAnnotationOnSvc(svcPVC *v1.PersistentVolumeClaim,
111+
allowedTopologies map[string][]string, categories []string) error {
112+
113+
annotationsMap := svcPVC.Annotations
114+
if accessibleTopoString, exists := annotationsMap[tkgHAccessibleAnnotationKey]; exists {
115+
// Parse the accessible topology string
116+
var accessibleTopologyList []map[string]string
117+
err := json.Unmarshal([]byte(accessibleTopoString), &accessibleTopologyList)
118+
if err != nil {
119+
return fmt.Errorf("failed to parse accessible topology: %v", err)
120+
}
121+
122+
for _, topo := range accessibleTopologyList {
123+
for topoKey, topoVal := range topo {
124+
if allowedVals, ok := allowedTopologies[topoKey]; ok {
125+
// Check if topoVal exists in allowedVals
126+
found := false
127+
for _, val := range allowedVals {
128+
if val == topoVal {
129+
found = true
130+
break
131+
}
132+
}
133+
if !found {
134+
return fmt.Errorf("couldn't find allowed accessible topology: %v on svc pvc: %s, instead found: %v",
135+
allowedVals, svcPVC.Name, topoVal)
136+
}
137+
} else {
138+
category := strings.SplitN(topoKey, "/", 2)
139+
if len(category) > 1 && !containsItem(categories, category[1]) {
140+
return fmt.Errorf("couldn't find key: %s in allowed categories %v", category[1], categories)
141+
}
142+
}
143+
}
144+
}
145+
} else {
146+
return fmt.Errorf("couldn't find annotation key: %s on svc pvc: %s",
147+
tkgHAccessibleAnnotationKey, svcPVC.Name)
148+
}
149+
return nil
150+
}
151+
152+
// Helper function to check if a string exists in a slice
153+
func containsItem(slice []string, item string) bool {
154+
for _, val := range slice {
155+
if val == item {
156+
return true
157+
}
158+
}
159+
return false
160+
}
161+
162+
/*
163+
This util createTestWcpNsWithZones will create a wcp namespace which will be tagged to the zone and
164+
storage policy passed in the util parameters
165+
*/
166+
func createTestWcpNsWithZones(
167+
vcRestSessionId string, storagePolicyId string,
168+
supervisorId string, zoneNames []string) string {
169+
170+
vcIp := e2eVSphere.Config.Global.VCenterHostname
171+
r := rand.New(rand.NewSource(time.Now().Unix()))
172+
173+
namespace := fmt.Sprintf("csi-vmsvcns-%v", r.Intn(10000))
174+
nsCreationUrl := "https://" + vcIp + "/api/vcenter/namespaces/instances/v2"
175+
176+
// Create a string to represent the zones array
177+
var zonesString string
178+
for i, zone := range zoneNames {
179+
if i > 0 {
180+
zonesString += ","
181+
}
182+
zonesString += fmt.Sprintf(`{"name": "%s"}`, zone)
183+
}
184+
185+
reqBody := fmt.Sprintf(`{
186+
"namespace": "%s",
187+
"storage_specs": [
188+
{
189+
"policy": "%s"
190+
}
191+
],
192+
"supervisor": "%s",
193+
"zones": [%s]
194+
}`, namespace, storagePolicyId, supervisorId, zonesString)
195+
196+
// Print the request body for debugging
197+
fmt.Println(reqBody)
198+
199+
// Make the API request
200+
_, statusCode := invokeVCRestAPIPostRequest(vcRestSessionId, nsCreationUrl, reqBody)
201+
202+
// Validate the status code
203+
gomega.Expect(statusCode).Should(gomega.BeNumerically("==", 204))
204+
framework.Logf("Successfully created namespace %v in SVC.", namespace)
205+
return namespace
206+
}

0 commit comments

Comments
 (0)