Skip to content

Commit f069364

Browse files
gdsoumyarociomroman
authored andcommitted
feat: add resource name filtering in k8s probe (#598)
* feat: add resource name filtering in k8s probe Signed-off-by: Soumya Ghosh Dastidar <[email protected]> Signed-off-by: Rocio Roman <[email protected]>
1 parent e461c82 commit f069364

File tree

6 files changed

+115
-36
lines changed

6 files changed

+115
-36
lines changed

chaoslib/litmus/azure-disk-loss/lib/azure-disk-loss.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@ func abortWatcher(experimentsDetails *experimentTypes.ExperimentDetails, attache
275275
}
276276
if diskStatusString != "Attached" {
277277
if err := diskStatus.AttachDisk(experimentsDetails.SubscriptionID, experimentsDetails.ResourceGroup, instanceName, experimentsDetails.ScaleSet, diskList); err != nil {
278-
log.Errorf("failed to attach disk '%v, manual revert required, err: %v", err)
278+
log.Errorf("failed to attach disk, manual revert required, err: %v", err)
279279
} else {
280280
common.SetTargets(*disk.Name, "re-attached", "VirtualDisk", chaosDetails)
281281
}

chaoslib/litmus/node-memory-hog/lib/node-memory-hog.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,7 @@ func calculateMemoryConsumption(experimentsDetails *experimentTypes.ExperimentDe
284284
//Get the percentage of memory under chaos wrt allocatable memory
285285
totalMemoryConsumption = int((float64(memoryForChaos) / float64(memoryAllocatable)) * 100)
286286
if totalMemoryConsumption > 100 {
287-
log.Infof("[Info]: PercentageOfMemoryCapacity To Be Used: %d percent, which is more than 100 percent (%d percent) of Allocatable Memory, so the experiment will only consume upto 100 percent of Allocatable Memory", experimentsDetails.MemoryConsumptionPercentage, totalMemoryConsumption)
287+
log.Infof("[Info]: PercentageOfMemoryCapacity To Be Used: %v percent, which is more than 100 percent (%d percent) of Allocatable Memory, so the experiment will only consume upto 100 percent of Allocatable Memory", experimentsDetails.MemoryConsumptionPercentage, totalMemoryConsumption)
288288
MemoryConsumption = "100%"
289289
} else {
290290
log.Infof("[Info]: PercentageOfMemoryCapacity To Be Used: %v percent, which is %d percent of Allocatable Memory", experimentsDetails.MemoryConsumptionPercentage, totalMemoryConsumption)

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ require (
99
github.com/aws/aws-sdk-go v1.38.59
1010
github.com/containerd/cgroups v1.0.1
1111
github.com/kyokomi/emoji v2.2.4+incompatible
12-
github.com/litmuschaos/chaos-operator v0.0.0-20220929101337-868b2827f820
12+
github.com/litmuschaos/chaos-operator v0.0.0-20221114055503-3d12d34d2032
1313
github.com/pkg/errors v0.9.1
1414
github.com/sirupsen/logrus v1.7.0
1515
github.com/spf13/cobra v1.1.1

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -767,8 +767,8 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9
767767
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
768768
github.com/lightstep/lightstep-tracer-go v0.18.0/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
769769
github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
770-
github.com/litmuschaos/chaos-operator v0.0.0-20220929101337-868b2827f820 h1:xMlb6eMbWzdR/2IB6F095p0NDadccZIkiovJBE9fg9I=
771-
github.com/litmuschaos/chaos-operator v0.0.0-20220929101337-868b2827f820/go.mod h1:CJGiHqC06PQkIBySk/JroB7B2zFebDbkhQ1A6ZbYmHA=
770+
github.com/litmuschaos/chaos-operator v0.0.0-20221114055503-3d12d34d2032 h1:VeVpXvz5JVj28rQZs4DI101b+vVKHIKlUNWGfbDF6V0=
771+
github.com/litmuschaos/chaos-operator v0.0.0-20221114055503-3d12d34d2032/go.mod h1:CJGiHqC06PQkIBySk/JroB7B2zFebDbkhQ1A6ZbYmHA=
772772
github.com/litmuschaos/elves v0.0.0-20201107015738-552d74669e3c/go.mod h1:DsbHGNUq/78NZozWVVI9Q6eBei4I+JjlkkD5aibJ3MQ=
773773
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
774774
github.com/lovoo/gcloud-opentracing v0.3.0/go.mod h1:ZFqk2y38kMDDikZPAK7ynTTGuyt17nSPdS3K5e+ZTBY=

pkg/clients/clientset.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func buildConfigFromFlags(masterUrl, kubeconfigPath string) (*restclient.Config,
8383
if err == nil {
8484
return kubeconfig, nil
8585
}
86-
klog.Warningf("Neither --kubeconfig nor --master was specified. Using the inClusterConfig. Error creating inClusterConfig: ", err)
86+
klog.Warningf("Neither --kubeconfig nor --master was specified. Using the inClusterConfig. Error creating inClusterConfig: %v", err)
8787
}
8888
return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
8989
&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},

pkg/probe/k8sprobe.go

Lines changed: 109 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package probe
22

33
import (
44
"context"
5+
k8serrors "k8s.io/apimachinery/pkg/api/errors"
56
"strings"
67
"time"
78

@@ -42,7 +43,7 @@ func prepareK8sProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Result
4243
// triggerK8sProbe run the k8s probe command
4344
func triggerK8sProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, resultDetails *types.ResultDetails) error {
4445

45-
inputs := probe.K8sProbeInputs
46+
inputs := &probe.K8sProbeInputs
4647

4748
// It parse the templated command and return normal string
4849
// if command doesn't have template, it will return the same command
@@ -56,6 +57,19 @@ func triggerK8sProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets,
5657
return err
5758
}
5859

60+
inputs.ResourceNames, err = parseCommand(inputs.ResourceNames, resultDetails)
61+
if err != nil {
62+
return err
63+
}
64+
65+
parsedResourceNames := []string{}
66+
if inputs.ResourceNames != "" {
67+
parsedResourceNames = strings.Split(inputs.ResourceNames, ",")
68+
for i := range parsedResourceNames {
69+
parsedResourceNames[i] = strings.TrimSpace(parsedResourceNames[i])
70+
}
71+
}
72+
5973
// it will retry for some retry count, in each iterations of try it contains following things
6074
// it contains a timeout per iteration of retry. if the timeout expires without success then it will go to next try
6175
// for a timeout, it will run the command, if it fails wait for the iterval and again execute the command until timeout expires
@@ -77,32 +91,19 @@ func triggerK8sProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets,
7791
return err
7892
}
7993
case "delete":
80-
if err = deleteResource(probe, gvr, clients); err != nil {
94+
if err = deleteResource(probe, gvr, parsedResourceNames, clients); err != nil {
8195
log.Errorf("the %v k8s probe has Failed, err: %v", probe.Name, err)
8296
return err
8397
}
8498
case "present":
85-
resourceList, err := clients.DynamicClient.Resource(gvr).Namespace(inputs.Namespace).List(context.Background(), v1.ListOptions{
86-
FieldSelector: inputs.FieldSelector,
87-
LabelSelector: inputs.LabelSelector,
88-
})
89-
if err != nil {
99+
if err = resourcesPresent(probe, gvr, parsedResourceNames, clients); err != nil {
90100
log.Errorf("the %v k8s probe has Failed, err: %v", probe.Name, err)
91-
return errors.Errorf("unable to list the resources with matching selector, err: %v", err)
92-
} else if len(resourceList.Items) == 0 {
93-
return errors.Errorf("no resource found with provided selectors")
101+
return err
94102
}
95103
case "absent":
96-
resourceList, err := clients.DynamicClient.Resource(gvr).Namespace(inputs.Namespace).List(context.Background(), v1.ListOptions{
97-
FieldSelector: inputs.FieldSelector,
98-
LabelSelector: inputs.LabelSelector,
99-
})
100-
if err != nil {
101-
return errors.Errorf("unable to list the resources with matching selector, err: %v", err)
102-
}
103-
if len(resourceList.Items) != 0 {
104+
if err = resourcesAbsent(probe, gvr, parsedResourceNames, clients); err != nil {
104105
log.Errorf("the %v k8s probe has Failed, err: %v", probe.Name, err)
105-
return errors.Errorf("resource is not deleted yet due to, err: %v", err)
106+
return err
106107
}
107108
default:
108109
return errors.Errorf("operation type '%s' not supported in the k8s probe", inputs.Operation)
@@ -165,21 +166,99 @@ func createResource(probe v1alpha1.ProbeAttributes, gvr schema.GroupVersionResou
165166
}
166167

167168
// deleteResource deletes the resource with matching label & field selector
168-
func deleteResource(probe v1alpha1.ProbeAttributes, gvr schema.GroupVersionResource, clients clients.ClientSets) error {
169-
resourceList, err := clients.DynamicClient.Resource(gvr).Namespace(probe.K8sProbeInputs.Namespace).List(context.Background(), v1.ListOptions{
170-
FieldSelector: probe.K8sProbeInputs.FieldSelector,
171-
LabelSelector: probe.K8sProbeInputs.LabelSelector,
172-
})
173-
if err != nil {
174-
return errors.Errorf("unable to list the resources with matching selector, err: %v", err)
175-
} else if len(resourceList.Items) == 0 {
176-
return errors.Errorf("no resource found with provided selectors")
169+
func deleteResource(probe v1alpha1.ProbeAttributes, gvr schema.GroupVersionResource, parsedResourceNames []string, clients clients.ClientSets) error {
170+
// resource name has higher priority
171+
if len(parsedResourceNames) > 0 {
172+
// check if all resources are available
173+
if err := areResourcesWithNamePresent(probe, gvr, parsedResourceNames, clients); err != nil {
174+
return err
175+
}
176+
// delete resources
177+
for _, res := range parsedResourceNames {
178+
if err = clients.DynamicClient.Resource(gvr).Namespace(probe.K8sProbeInputs.Namespace).Delete(context.Background(), res, v1.DeleteOptions{}); err != nil {
179+
return err
180+
}
181+
}
182+
} else {
183+
resourceList, err := clients.DynamicClient.Resource(gvr).Namespace(probe.K8sProbeInputs.Namespace).List(context.Background(), v1.ListOptions{
184+
FieldSelector: probe.K8sProbeInputs.FieldSelector,
185+
LabelSelector: probe.K8sProbeInputs.LabelSelector,
186+
})
187+
if err != nil {
188+
return errors.Errorf("unable to list the resources with matching selector, err: %v", err)
189+
} else if len(resourceList.Items) == 0 {
190+
return errors.Errorf("no resource found with provided selectors")
191+
}
192+
193+
for index := range resourceList.Items {
194+
if err = clients.DynamicClient.Resource(gvr).Namespace(probe.K8sProbeInputs.Namespace).Delete(context.Background(), resourceList.Items[index].GetName(), v1.DeleteOptions{}); err != nil {
195+
return err
196+
}
197+
}
177198
}
199+
return nil
200+
}
178201

179-
for index := range resourceList.Items {
180-
if err = clients.DynamicClient.Resource(gvr).Namespace(probe.K8sProbeInputs.Namespace).Delete(context.Background(), resourceList.Items[index].GetName(), v1.DeleteOptions{}); err != nil {
202+
func resourcesPresent(probe v1alpha1.ProbeAttributes, gvr schema.GroupVersionResource, parsedResourceNames []string, clients clients.ClientSets) error {
203+
// resource name has higher priority
204+
if len(parsedResourceNames) > 0 {
205+
// check if all resources are available
206+
if err := areResourcesWithNamePresent(probe, gvr, parsedResourceNames, clients); err != nil {
181207
return err
182208
}
209+
} else {
210+
resourceList, err := clients.DynamicClient.Resource(gvr).Namespace(probe.K8sProbeInputs.Namespace).List(context.Background(), v1.ListOptions{
211+
FieldSelector: probe.K8sProbeInputs.FieldSelector,
212+
LabelSelector: probe.K8sProbeInputs.LabelSelector,
213+
})
214+
if err != nil {
215+
log.Errorf("the %v k8s probe has Failed, err: %v", probe.Name, err)
216+
return errors.Errorf("unable to list the resources with matching selector, err: %v", err)
217+
} else if len(resourceList.Items) == 0 {
218+
return errors.Errorf("no resource found with provided selectors")
219+
}
220+
}
221+
return nil
222+
}
223+
224+
func areResourcesWithNamePresent(probe v1alpha1.ProbeAttributes, gvr schema.GroupVersionResource, parsedResourceNames []string, clients clients.ClientSets) error {
225+
for _, res := range parsedResourceNames {
226+
resource, err := clients.DynamicClient.Resource(gvr).Namespace(probe.K8sProbeInputs.Namespace).Get(context.Background(), res, v1.GetOptions{})
227+
if err != nil {
228+
return errors.Errorf("unable to get the resources with name %v, err: %v", res, err)
229+
} else if resource == nil {
230+
return errors.Errorf("unable to get the resources with name %v", res)
231+
}
232+
}
233+
return nil
234+
}
235+
236+
func resourcesAbsent(probe v1alpha1.ProbeAttributes, gvr schema.GroupVersionResource, parsedResourceNames []string, clients clients.ClientSets) error {
237+
// resource name has higher priority
238+
if len(parsedResourceNames) > 0 {
239+
// check if all resources are absent
240+
for _, res := range parsedResourceNames {
241+
resource, err := clients.DynamicClient.Resource(gvr).Namespace(probe.K8sProbeInputs.Namespace).Get(context.Background(), res, v1.GetOptions{})
242+
if err != nil {
243+
// ignore not found error, that is the expected outcome
244+
if !k8serrors.IsNotFound(err) {
245+
return errors.Errorf("unable to get the resources with name %v from k8s, err: %v", res, err)
246+
}
247+
} else if resource != nil {
248+
return errors.Errorf("resource '%v' still exists but is expected to be absent", res)
249+
}
250+
}
251+
} else {
252+
resourceList, err := clients.DynamicClient.Resource(gvr).Namespace(probe.K8sProbeInputs.Namespace).List(context.Background(), v1.ListOptions{
253+
FieldSelector: probe.K8sProbeInputs.FieldSelector,
254+
LabelSelector: probe.K8sProbeInputs.LabelSelector,
255+
})
256+
if err != nil {
257+
return errors.Errorf("unable to list the resources with matching selector, err: %v", err)
258+
}
259+
if len(resourceList.Items) != 0 {
260+
return errors.Errorf("resource with provided selectors still exists, found %v resources with matching selectors", len(resourceList.Items))
261+
}
183262
}
184263
return nil
185264
}

0 commit comments

Comments
 (0)