Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit a95fc81

Browse files
SSanjeeviSSanjeevi
committedMar 17, 2025··
Added Azure VM Chaos by running Powershell script based chaos
Added Azure VM Chaos by running PowerShell script on start of chaos to create the chaos and at end of Chaos runs another PowerShell script with parameters to complete the chaos. This will open way for many custom Chaos which can be run with the option of PowerShell script instead of predefined Azure operations.
1 parent caae228 commit a95fc81

File tree

8 files changed

+626
-1
lines changed

8 files changed

+626
-1
lines changed
 

‎chaoslib/litmus/azure-instance-runscript/azure-instance-runscript.go

+159
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
package lib
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"os/signal"
8+
"strings"
9+
"syscall"
10+
"time"
11+
12+
experimentTypes "github.com/litmuschaos/litmus-go/pkg/azure/instance-runscript/types"
13+
"github.com/litmuschaos/litmus-go/pkg/cerrors"
14+
"github.com/litmuschaos/litmus-go/pkg/clients"
15+
azureCommon "github.com/litmuschaos/litmus-go/pkg/cloud/azure/common"
16+
azureStatus "github.com/litmuschaos/litmus-go/pkg/cloud/azure/instance"
17+
"github.com/litmuschaos/litmus-go/pkg/events"
18+
"github.com/litmuschaos/litmus-go/pkg/log"
19+
"github.com/litmuschaos/litmus-go/pkg/probe"
20+
"github.com/litmuschaos/litmus-go/pkg/telemetry"
21+
"github.com/litmuschaos/litmus-go/pkg/types"
22+
"github.com/litmuschaos/litmus-go/pkg/utils/common"
23+
"github.com/palantir/stacktrace"
24+
"go.opentelemetry.io/otel"
25+
)
26+
27+
var (
28+
err error
29+
inject, abort chan os.Signal
30+
)
31+
32+
// PrepareAzureRunScript will initialize instanceNameList and start chaos injection based on sequence method selected
33+
func PrepareAzureRunScript(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
34+
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "PrepareAzureInstanceRunScriptFault")
35+
defer span.End()
36+
37+
// inject channel is used to transmit signal notifications
38+
inject = make(chan os.Signal, 1)
39+
// Catch and relay certain signal(s) to inject channel
40+
signal.Notify(inject, os.Interrupt, syscall.SIGTERM)
41+
42+
// abort channel is used to transmit signal notifications.
43+
abort = make(chan os.Signal, 1)
44+
signal.Notify(abort, os.Interrupt, syscall.SIGTERM)
45+
46+
// Waiting for the ramp time before chaos injection
47+
if experimentsDetails.RampTime != 0 {
48+
log.Infof("[Ramp]: Waiting for the %vs ramp time before injecting chaos", experimentsDetails.RampTime)
49+
common.WaitForDuration(experimentsDetails.RampTime)
50+
}
51+
52+
// get the instance name or list of instance names
53+
instanceNameList := strings.Split(experimentsDetails.AzureInstanceNames, ",")
54+
if experimentsDetails.AzureInstanceNames == "" || len(instanceNameList) == 0 {
55+
return cerrors.Error{ErrorCode: cerrors.ErrorTypeTargetSelection, Reason: "no instance name found to stop"}
56+
}
57+
58+
// watching for the abort signal and revert the chaos
59+
go abortWatcher(experimentsDetails, instanceNameList)
60+
61+
switch strings.ToLower(experimentsDetails.Sequence) {
62+
case "serial":
63+
if err = injectChaosInSerialMode(ctx, experimentsDetails, instanceNameList, clients, resultDetails, eventsDetails, chaosDetails); err != nil {
64+
return stacktrace.Propagate(err, "could not run chaos in serial mode")
65+
}
66+
case "parallel":
67+
if err = injectChaosInParallelMode(ctx, experimentsDetails, instanceNameList, clients, resultDetails, eventsDetails, chaosDetails); err != nil {
68+
return stacktrace.Propagate(err, "could not run chaos in parallel mode")
69+
}
70+
default:
71+
return cerrors.Error{ErrorCode: cerrors.ErrorTypeGeneric, Reason: fmt.Sprintf("'%s' sequence is not supported", experimentsDetails.Sequence)}
72+
}
73+
74+
// Waiting for the ramp time after chaos injection
75+
if experimentsDetails.RampTime != 0 {
76+
log.Infof("[Ramp]: Waiting for the %vs ramp time after injecting chaos", experimentsDetails.RampTime)
77+
common.WaitForDuration(experimentsDetails.RampTime)
78+
}
79+
return nil
80+
}
81+
82+
// injectChaosInSerialMode will inject the Azure instance termination in serial mode that is one after the other
83+
func injectChaosInSerialMode(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, instanceNameList []string, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
84+
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAzureInstanceRunScriptFaultInSerialMode")
85+
defer span.End()
86+
87+
select {
88+
case <-inject:
89+
//running script the chaos execution, if abort signal received
90+
os.Exit(0)
91+
default:
92+
// ChaosStartTimeStamp contains the start timestamp, when the chaos injection begin
93+
ChaosStartTimeStamp := time.Now()
94+
duration := int(time.Since(ChaosStartTimeStamp).Seconds())
95+
96+
for duration < experimentsDetails.ChaosDuration {
97+
98+
log.Infof("[Info]: Target instanceName list, %v", instanceNameList)
99+
100+
if experimentsDetails.EngineName != "" {
101+
msg := "Injecting " + experimentsDetails.ExperimentName + " chaos on Azure instance"
102+
types.SetEngineEventAttributes(eventsDetails, types.ChaosInject, msg, "Normal", chaosDetails)
103+
events.GenerateEvents(eventsDetails, clients, chaosDetails, "ChaosEngine")
104+
}
105+
106+
//Run script in the instance serially
107+
for i, vmName := range instanceNameList {
108+
109+
//Running start Script the Azure instance
110+
log.Infof("[Chaos]:Running Script the Azure instance: %v", vmName)
111+
if experimentsDetails.ScaleSet == "enable" {
112+
return notImplementedError("scale set instance run script not implemented")
113+
} else {
114+
if err := azureStatus.AzureInstanceRunScript(experimentsDetails.Timeout, experimentsDetails.Delay, experimentsDetails.SubscriptionID, experimentsDetails.ResourceGroup, vmName, experimentDetails.PowershellChaosStartBase64OrPsFilePath, experimentDetails.IsBase64, experimentDetails.PowershellChaosStartParams); err != nil {
115+
return stacktrace.Propagate(err, "unable to run script in the Azure instance")
116+
}
117+
}
118+
119+
// Run the probes during chaos
120+
// the OnChaos probes execution will start in the first iteration and keep running for the entire chaos duration
121+
if len(resultDetails.ProbeDetails) != 0 && i == 0 {
122+
if err = probe.RunProbes(ctx, chaosDetails, clients, resultDetails, "DuringChaos", eventsDetails); err != nil {
123+
return stacktrace.Propagate(err, "failed to run probes")
124+
}
125+
}
126+
127+
// Wait for Chaos interval
128+
log.Infof("[Wait]: Waiting for chaos interval of %vs", experimentsDetails.ChaosInterval)
129+
common.WaitForDuration(experimentsDetails.ChaosInterval)
130+
131+
// Running the end PS Script in the Azure instance
132+
log.Info("[Chaos]: Starting back the Azure instance")
133+
if experimentsDetails.ScaleSet == "enable" {
134+
return notImplementedError("scale set instance run script not implemented")
135+
} else {
136+
if err := azureStatus.AzureInstanceRunScript(experimentsDetails.Timeout, experimentsDetails.Delay, experimentsDetails.SubscriptionID, experimentsDetails.ResourceGroup, vmName, experimentDetails.PowershellChaosEndBase64OrPsFilePath, experimentDetails.IsBase64, experimentDetails.PowershellChaosEndParams); err != nil {
137+
return stacktrace.Propagate(err, "unable to run the script in the Azure instance")
138+
}
139+
}
140+
}
141+
duration = int(time.Since(ChaosStartTimeStamp).Seconds())
142+
}
143+
}
144+
return nil
145+
}
146+
147+
// injectChaosInParallelMode will inject the Azure instance termination in parallel mode that is all at once
148+
func injectChaosInParallelMode(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, instanceNameList []string, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
149+
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAzureInstanceRunScriptFaultInParallelMode")
150+
defer span.End()
151+
return notImplementedError("scale set instance run script not implemented so parallel mode run script also not implemented.")
152+
}
153+
154+
// watching for the abort signal and revert the chaos
155+
func abortWatcher(experimentsDetails *experimentTypes.ExperimentDetails, instanceNameList []string) {
156+
<-abort
157+
158+
return notImplementedError("Abort run script not implemented")
159+
}

‎experiments/azure/instance-runscript/README.md

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
## Experiment Metadata
2+
3+
<table>
4+
<tr>
5+
<th> Name </th>
6+
<th> Description </th>
7+
<th> Documentation Link </th>
8+
</tr>
9+
<tr>
10+
<td> Azure Instance Run PS Script </td>
11+
<td> This experiment runs a Powershell script for creating Chaos and runs another script to complete Chaos within the chaos duration.</td>
12+
<td> <a href="https://litmuschaos.github.io/litmus/experiments/categories/azure/azure-instance-runscript/"> Here </a> </td>
13+
</tr>
14+
</table>

‎experiments/azure/instance-runscript/experiment/azure-instance-runscript.go

+199
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
package experiment
2+
3+
import (
4+
"context"
5+
"os"
6+
7+
"github.com/litmuschaos/chaos-operator/api/litmuschaos/v1alpha1"
8+
litmusLIB "github.com/litmuschaos/litmus-go/chaoslib/litmus/azure-instance-runscript/lib"
9+
experimentEnv "github.com/litmuschaos/litmus-go/pkg/azure/instance-runscript/environment"
10+
experimentTypes "github.com/litmuschaos/litmus-go/pkg/azure/instance-runscript/types"
11+
"github.com/litmuschaos/litmus-go/pkg/clients"
12+
azureCommon "github.com/litmuschaos/litmus-go/pkg/cloud/azure/common"
13+
azureStatus "github.com/litmuschaos/litmus-go/pkg/cloud/azure/instance"
14+
15+
"github.com/litmuschaos/litmus-go/pkg/events"
16+
"github.com/litmuschaos/litmus-go/pkg/log"
17+
"github.com/litmuschaos/litmus-go/pkg/probe"
18+
"github.com/litmuschaos/litmus-go/pkg/result"
19+
"github.com/litmuschaos/litmus-go/pkg/types"
20+
"github.com/litmuschaos/litmus-go/pkg/utils/common"
21+
"github.com/sirupsen/logrus"
22+
)
23+
24+
// AzureInstanceStop inject the azure Instance Run Script chaos
25+
func AzureInstanceStop(ctx context.Context, clients clients.ClientSets) {
26+
27+
var err error
28+
experimentsDetails := experimentTypes.ExperimentDetails{}
29+
resultDetails := types.ResultDetails{}
30+
eventsDetails := types.EventDetails{}
31+
chaosDetails := types.ChaosDetails{}
32+
33+
//Fetching all the ENV passed from the runner pod
34+
log.Infof("[PreReq]: Getting the ENV for the %v experiment", os.Getenv("EXPERIMENT_NAME"))
35+
experimentEnv.GetENV(&experimentsDetails)
36+
37+
// Initialize the chaos attributes
38+
types.InitialiseChaosVariables(&chaosDetails)
39+
40+
// Initialize Chaos Result Parameters
41+
types.SetResultAttributes(&resultDetails, chaosDetails)
42+
43+
if experimentsDetails.EngineName != "" {
44+
// Get values from chaosengine. Bail out upon error, as we haven't entered exp business logic yet
45+
if err = types.GetValuesFromChaosEngine(&chaosDetails, clients, &resultDetails); err != nil {
46+
log.Errorf("Unable to initialize the probes: %v", err)
47+
}
48+
}
49+
50+
//Updating the chaos result in the beginning of experiment
51+
log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)
52+
err = result.ChaosResult(&chaosDetails, clients, &resultDetails, "SOT")
53+
if err != nil {
54+
log.Errorf("Unable to create the chaosresult: %v", err)
55+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
56+
return
57+
}
58+
59+
// Set the chaos result uid
60+
result.SetResultUID(&resultDetails, clients, &chaosDetails)
61+
62+
// Calling AbortWatcher go routine, it will continuously watch for the abort signal and generate the required events and result
63+
go common.AbortWatcherWithoutExit(experimentsDetails.ExperimentName, clients, &resultDetails, &chaosDetails, &eventsDetails)
64+
65+
//DISPLAY THE APP INFORMATION
66+
log.InfoWithValues("The instance information is as follows", logrus.Fields{
67+
"Chaos Duration": experimentsDetails.ChaosDuration,
68+
"Resource Group": experimentsDetails.ResourceGroup,
69+
"Powershell Chaos Start Base64 Or Ps File Path": experimentsDetails.PowershellChaosStartBase64OrPsFilePath,
70+
"Powershell Chaos End Base64 Or Ps File Path": experimentsDetails.PowershellChaosEndBase64OrPsFilePath,
71+
"Instance Name": experimentsDetails.AzureInstanceNames,
72+
"Sequence": experimentsDetails.Sequence,
73+
})
74+
75+
// Setting up Azure Subscription ID
76+
if experimentsDetails.SubscriptionID, err = azureCommon.GetSubscriptionID(); err != nil {
77+
log.Errorf("Failed to get the subscription id: %v", err)
78+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
79+
return
80+
}
81+
82+
// generating the event in chaosresult to marked the verdict as awaited
83+
msg := "experiment: " + experimentsDetails.ExperimentName + ", Result: Awaited"
84+
types.SetResultEventAttributes(&eventsDetails, types.AwaitedVerdict, msg, "Normal", &resultDetails)
85+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosResults"); eventErr != nil {
86+
log.Errorf("Failed to create %v event inside chaosresults", types.AwaitedVerdict)
87+
}
88+
89+
if experimentsDetails.EngineName != "" {
90+
// marking AUT as running, as we already checked the status of application under test
91+
msg := "AUT: Running"
92+
93+
// run the probes in the pre-chaos check
94+
if len(resultDetails.ProbeDetails) != 0 {
95+
96+
err = probe.RunProbes(ctx, &chaosDetails, clients, &resultDetails, "PreChaos", &eventsDetails)
97+
if err != nil {
98+
log.Errorf("Probe Failed: %v", err)
99+
msg := "AUT: Running, Probes: Unsuccessful"
100+
types.SetEngineEventAttributes(&eventsDetails, types.PreChaosCheck, msg, "Warning", &chaosDetails)
101+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine"); eventErr != nil {
102+
log.Errorf("Failed to create %v event inside chaosengine", types.PreChaosCheck)
103+
}
104+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
105+
return
106+
}
107+
msg = "AUT: Running, Probes: Successful"
108+
}
109+
// generating the events for the pre-chaos check
110+
types.SetEngineEventAttributes(&eventsDetails, types.PreChaosCheck, msg, "Normal", &chaosDetails)
111+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine"); eventErr != nil {
112+
log.Errorf("Failed to create %v event inside chaosengine", types.PreChaosCheck)
113+
}
114+
}
115+
116+
//Verify the azure target instance is running (pre-chaos)
117+
if chaosDetails.DefaultHealthCheck {
118+
if err = azureStatus.InstanceStatusCheckByName(experimentsDetails.AzureInstanceNames, experimentsDetails.ScaleSet, experimentsDetails.SubscriptionID, experimentsDetails.ResourceGroup); err != nil {
119+
log.Errorf("Azure instance status check failed: %v", err)
120+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
121+
return
122+
}
123+
log.Info("[Status]: Azure instance(s) is in running state (pre-chaos)")
124+
}
125+
126+
chaosDetails.Phase = types.ChaosInjectPhase
127+
128+
if err = litmusLIB.PrepareAzureRunScript(ctx, &experimentsDetails, clients, &resultDetails, &eventsDetails, &chaosDetails); err != nil {
129+
log.Errorf("Chaos injection failed: %v", err)
130+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
131+
return
132+
}
133+
134+
log.Info("[Confirmation]: Azure Instance Run Script chaos has been injected successfully")
135+
resultDetails.Verdict = v1alpha1.ResultVerdictPassed
136+
137+
chaosDetails.Phase = types.PostChaosPhase
138+
139+
//Verify the azure instance is running (post chaos)
140+
if chaosDetails.DefaultHealthCheck {
141+
if err = azureStatus.InstanceStatusCheckByName(experimentsDetails.AzureInstanceNames, experimentsDetails.ScaleSet, experimentsDetails.SubscriptionID, experimentsDetails.ResourceGroup); err != nil {
142+
log.Errorf("Azure instance status check failed: %v", err)
143+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
144+
return
145+
}
146+
log.Info("[Status]: Azure instance is in running state (post chaos)")
147+
}
148+
149+
if experimentsDetails.EngineName != "" {
150+
// marking AUT as running, as we already checked the status of application under test
151+
msg := "AUT: Running"
152+
153+
// run the probes in the post-chaos check
154+
if len(resultDetails.ProbeDetails) != 0 {
155+
err = probe.RunProbes(ctx, &chaosDetails, clients, &resultDetails, "PostChaos", &eventsDetails)
156+
if err != nil {
157+
log.Errorf("Probes Failed: %v", err)
158+
msg := "AUT: Running, Probes: Unsuccessful"
159+
types.SetEngineEventAttributes(&eventsDetails, types.PostChaosCheck, msg, "Warning", &chaosDetails)
160+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine"); eventErr != nil {
161+
log.Errorf("Failed to create %v event inside chaosengine", types.PostChaosCheck)
162+
}
163+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
164+
return
165+
}
166+
msg = "AUT: Running, Probes: Successful"
167+
}
168+
169+
// generating post chaos event
170+
types.SetEngineEventAttributes(&eventsDetails, types.PostChaosCheck, msg, "Normal", &chaosDetails)
171+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine"); eventErr != nil {
172+
log.Errorf("Failed to create %v event inside chaosengine", types.PostChaosCheck)
173+
}
174+
}
175+
176+
//Updating the chaosResult in the end of experiment
177+
log.Infof("[The End]: Updating the chaos result of %v experiment (EOT)", experimentsDetails.ExperimentName)
178+
err = result.ChaosResult(&chaosDetails, clients, &resultDetails, "EOT")
179+
if err != nil {
180+
log.Errorf("Unable to update the chaosresult: %v", err)
181+
}
182+
183+
// generating the event in chaosresult to marked the verdict as pass/fail
184+
msg = "experiment: " + experimentsDetails.ExperimentName + ", Result: " + string(resultDetails.Verdict)
185+
reason, eventType := types.GetChaosResultVerdictEvent(resultDetails.Verdict)
186+
types.SetResultEventAttributes(&eventsDetails, reason, msg, eventType, &resultDetails)
187+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosResults"); eventErr != nil {
188+
log.Errorf("Failed to create %v event inside chaosresults", reason)
189+
}
190+
191+
if experimentsDetails.EngineName != "" {
192+
msg := experimentsDetails.ExperimentName + " experiment has been " + string(resultDetails.Verdict) + "ed"
193+
types.SetEngineEventAttributes(&eventsDetails, types.Summary, msg, "Normal", &chaosDetails)
194+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine"); eventErr != nil {
195+
log.Errorf("Failed to create %v event inside chaosengine", types.Summary)
196+
}
197+
}
198+
199+
}
There was a problem loading the remainder of the diff.

0 commit comments

Comments
 (0)
Please sign in to comment.