Skip to content

Commit 466144f

Browse files
SSanjeeviSSanjeevi
committed
Added Azure VM Chaos by running Powershell script based chaos
Signed-off-by: Author Name [email protected]
1 parent caae228 commit 466144f

File tree

8 files changed

+645
-0
lines changed

8 files changed

+645
-0
lines changed

‎chaoslib/litmus/azure-instance-runscript/azure-instance-runscript.go

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
package lib
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"os/signal"
8+
"strings"
9+
"syscall"
10+
"time"
11+
12+
instanceRunScript "pkg/azure/instance-runscript"
13+
14+
experimentTypes "github.com/litmuschaos/litmus-go/pkg/azure/instance-runscript/types"
15+
"github.com/litmuschaos/litmus-go/pkg/cerrors"
16+
"github.com/litmuschaos/litmus-go/pkg/clients"
17+
"github.com/litmuschaos/litmus-go/pkg/events"
18+
"github.com/litmuschaos/litmus-go/pkg/log"
19+
"github.com/litmuschaos/litmus-go/pkg/probe"
20+
"github.com/litmuschaos/litmus-go/pkg/telemetry"
21+
"github.com/litmuschaos/litmus-go/pkg/types"
22+
"github.com/litmuschaos/litmus-go/pkg/utils/common"
23+
"github.com/palantir/stacktrace"
24+
"go.opentelemetry.io/otel"
25+
)
26+
27+
var (
28+
err error
29+
inject, abort chan os.Signal
30+
)
31+
32+
// PrepareAzureRunScript will initialize instanceNameList and start chaos injection based on sequence method selected
33+
func PrepareAzureRunScript(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
34+
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "PrepareAzureInstanceRunScriptFault")
35+
defer span.End()
36+
37+
// inject channel is used to transmit signal notifications
38+
inject = make(chan os.Signal, 1)
39+
// Catch and relay certain signal(s) to inject channel
40+
signal.Notify(inject, os.Interrupt, syscall.SIGTERM)
41+
42+
// abort channel is used to transmit signal notifications.
43+
abort = make(chan os.Signal, 1)
44+
signal.Notify(abort, os.Interrupt, syscall.SIGTERM)
45+
46+
// Waiting for the ramp time before chaos injection
47+
if experimentsDetails.RampTime != 0 {
48+
log.Infof("[Ramp]: Waiting for the %vs ramp time before injecting chaos", experimentsDetails.RampTime)
49+
common.WaitForDuration(experimentsDetails.RampTime)
50+
}
51+
52+
// get the instance name or list of instance names
53+
instanceNameList := strings.Split(experimentsDetails.AzureInstanceNames, ",")
54+
if experimentsDetails.AzureInstanceNames == "" || len(instanceNameList) == 0 {
55+
return cerrors.Error{ErrorCode: cerrors.ErrorTypeTargetSelection, Reason: "no instance name found to stop"}
56+
}
57+
58+
// watching for the abort signal and revert the chaos
59+
go abortWatcher(experimentsDetails, instanceNameList)
60+
61+
switch strings.ToLower(experimentsDetails.Sequence) {
62+
case "serial":
63+
if err = injectChaosInSerialMode(ctx, experimentsDetails, instanceNameList, clients, resultDetails, eventsDetails, chaosDetails); err != nil {
64+
return stacktrace.Propagate(err, "could not run chaos in serial mode")
65+
}
66+
case "parallel":
67+
if err = injectChaosInParallelMode(ctx, experimentsDetails, instanceNameList, clients, resultDetails, eventsDetails, chaosDetails); err != nil {
68+
return stacktrace.Propagate(err, "could not run chaos in parallel mode")
69+
}
70+
default:
71+
return cerrors.Error{ErrorCode: cerrors.ErrorTypeGeneric, Reason: fmt.Sprintf("'%s' sequence is not supported", experimentsDetails.Sequence)}
72+
}
73+
74+
// Waiting for the ramp time after chaos injection
75+
if experimentsDetails.RampTime != 0 {
76+
log.Infof("[Ramp]: Waiting for the %vs ramp time after injecting chaos", experimentsDetails.RampTime)
77+
common.WaitForDuration(experimentsDetails.RampTime)
78+
}
79+
return nil
80+
}
81+
82+
// injectChaosInSerialMode will inject the Azure instance termination in serial mode that is one after the other
83+
func injectChaosInSerialMode(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, instanceNameList []string, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
84+
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAzureInstanceRunScriptFaultInSerialMode")
85+
defer span.End()
86+
87+
select {
88+
case <-inject:
89+
//running script the chaos execution, if abort signal received
90+
os.Exit(0)
91+
default:
92+
// ChaosStartTimeStamp contains the start timestamp, when the chaos injection begin
93+
ChaosStartTimeStamp := time.Now()
94+
duration := int(time.Since(ChaosStartTimeStamp).Seconds())
95+
96+
for duration < experimentsDetails.ChaosDuration {
97+
98+
log.Infof("[Info]: Target instanceName list, %v", instanceNameList)
99+
100+
if experimentsDetails.EngineName != "" {
101+
msg := "Injecting " + experimentsDetails.ExperimentName + " chaos on Azure instance"
102+
types.SetEngineEventAttributes(eventsDetails, types.ChaosInject, msg, "Normal", chaosDetails)
103+
events.GenerateEvents(eventsDetails, clients, chaosDetails, "ChaosEngine")
104+
}
105+
106+
//Run script in the instance serially
107+
for i, vmName := range instanceNameList {
108+
109+
//Running start Script the Azure instance
110+
log.Infof("[Chaos]:Running Script the Azure instance: %v", vmName)
111+
if experimentsDetails.ScaleSet == "enable" {
112+
// Yet to implement
113+
} else {
114+
if err := instanceRunScript.AzureInstanceRunScript(experimentsDetails.Timeout, experimentsDetails.Delay, experimentsDetails.SubscriptionID, experimentsDetails.ResourceGroup, vmName, experimentDetails.PowershellChaosStartBase64OrPsFilePath, experimentDetails.IsBase64, experimentDetails.PowershellChaosStartParamNames, experimentDetails.PowershellChaosStartParamValues); err != nil {
115+
return stacktrace.Propagate(err, "unable to run script in the Azure instance")
116+
}
117+
}
118+
119+
// Run the probes during chaos
120+
// the OnChaos probes execution will start in the first iteration and keep running for the entire chaos duration
121+
if len(resultDetails.ProbeDetails) != 0 && i == 0 {
122+
if err = probe.RunProbes(ctx, chaosDetails, clients, resultDetails, "DuringChaos", eventsDetails); err != nil {
123+
return stacktrace.Propagate(err, "failed to run probes")
124+
}
125+
}
126+
127+
// Wait for Chaos interval
128+
log.Infof("[Wait]: Waiting for chaos interval of %vs", experimentsDetails.ChaosInterval)
129+
common.WaitForDuration(experimentsDetails.ChaosInterval)
130+
131+
// Running the end PS Script in the Azure instance
132+
log.Info("[Chaos]: Starting back the Azure instance")
133+
if experimentsDetails.ScaleSet == "enable" {
134+
//scale set instance run script not implemented
135+
} else {
136+
if err := instanceRunScript.AzureInstanceRunScript(experimentsDetails.Timeout, experimentsDetails.Delay, experimentsDetails.SubscriptionID, experimentsDetails.ResourceGroup, vmName, experimentDetails.PowershellChaosEndBase64OrPsFilePath, experimentDetails.IsBase64, experimentDetails.PowershellChaosEndParamNames, experimentDetails.PowershellChaosEndParamValues); err != nil {
137+
return stacktrace.Propagate(err, "unable to run the script in the Azure instance")
138+
}
139+
}
140+
}
141+
duration = int(time.Since(ChaosStartTimeStamp).Seconds())
142+
}
143+
}
144+
return nil
145+
}
146+
147+
// injectChaosInParallelMode will inject the Azure instance termination in parallel mode that is all at once
148+
func injectChaosInParallelMode(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, instanceNameList []string, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
149+
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAzureInstanceRunScriptFaultInParallelMode")
150+
defer span.End()
151+
//scale set instance run script not implemented so parallel mode run script also not implemented
152+
}
153+
154+
// watching for the abort signal and revert the chaos
155+
func abortWatcher(experimentsDetails *experimentTypes.ExperimentDetails, instanceNameList []string) {
156+
<-abort
157+
158+
//Abort run script not implemented
159+
}

‎experiments/azure/instance-runscript/README.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
## Experiment Metadata
2+
3+
<table>
4+
<tr>
5+
<th> Name </th>
6+
<th> Description </th>
7+
<th> Documentation Link </th>
8+
</tr>
9+
<tr>
10+
<td> Azure Instance Run PS Script </td>
11+
<td> This experiment runs a Powershell script for creating Chaos and runs another script to complete Chaos within the chaos duration.</td>
12+
<td> <a href="https://litmuschaos.github.io/litmus/experiments/categories/azure/azure-instance-runscript/"> Here </a> </td>
13+
</tr>
14+
</table>

‎experiments/azure/instance-runscript/experiment/azure-instance-runscript.go

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
package experiment
2+
3+
import (
4+
"context"
5+
"os"
6+
7+
"github.com/litmuschaos/chaos-operator/api/litmuschaos/v1alpha1"
8+
litmusLIB "github.com/litmuschaos/litmus-go/chaoslib/litmus/azure-instance-runscript/lib"
9+
experimentEnv "github.com/litmuschaos/litmus-go/pkg/azure/instance-runscript/environment"
10+
experimentTypes "github.com/litmuschaos/litmus-go/pkg/azure/instance-runscript/types"
11+
"github.com/litmuschaos/litmus-go/pkg/clients"
12+
azureCommon "github.com/litmuschaos/litmus-go/pkg/cloud/azure/common"
13+
azureStatus "github.com/litmuschaos/litmus-go/pkg/cloud/azure/instance"
14+
15+
"github.com/litmuschaos/litmus-go/pkg/events"
16+
"github.com/litmuschaos/litmus-go/pkg/log"
17+
"github.com/litmuschaos/litmus-go/pkg/probe"
18+
"github.com/litmuschaos/litmus-go/pkg/result"
19+
"github.com/litmuschaos/litmus-go/pkg/types"
20+
"github.com/litmuschaos/litmus-go/pkg/utils/common"
21+
"github.com/sirupsen/logrus"
22+
)
23+
24+
// AzureInstanceStop inject the azure Instance Run Script chaos
25+
func AzureInstanceStop(ctx context.Context, clients clients.ClientSets) {
26+
27+
var err error
28+
experimentsDetails := experimentTypes.ExperimentDetails{}
29+
resultDetails := types.ResultDetails{}
30+
eventsDetails := types.EventDetails{}
31+
chaosDetails := types.ChaosDetails{}
32+
33+
//Fetching all the ENV passed from the runner pod
34+
log.Infof("[PreReq]: Getting the ENV for the %v experiment", os.Getenv("EXPERIMENT_NAME"))
35+
experimentEnv.GetENV(&experimentsDetails)
36+
37+
// Initialize the chaos attributes
38+
types.InitialiseChaosVariables(&chaosDetails)
39+
40+
// Initialize Chaos Result Parameters
41+
types.SetResultAttributes(&resultDetails, chaosDetails)
42+
43+
if experimentsDetails.EngineName != "" {
44+
// Get values from chaosengine. Bail out upon error, as we haven't entered exp business logic yet
45+
if err = types.GetValuesFromChaosEngine(&chaosDetails, clients, &resultDetails); err != nil {
46+
log.Errorf("Unable to initialize the probes: %v", err)
47+
}
48+
}
49+
50+
//Updating the chaos result in the beginning of experiment
51+
log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)
52+
err = result.ChaosResult(&chaosDetails, clients, &resultDetails, "SOT")
53+
if err != nil {
54+
log.Errorf("Unable to create the chaosresult: %v", err)
55+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
56+
return
57+
}
58+
59+
// Set the chaos result uid
60+
result.SetResultUID(&resultDetails, clients, &chaosDetails)
61+
62+
// Calling AbortWatcher go routine, it will continuously watch for the abort signal and generate the required events and result
63+
go common.AbortWatcherWithoutExit(experimentsDetails.ExperimentName, clients, &resultDetails, &chaosDetails, &eventsDetails)
64+
65+
//DISPLAY THE APP INFORMATION
66+
log.InfoWithValues("The instance information is as follows", logrus.Fields{
67+
"Chaos Duration": experimentsDetails.ChaosDuration,
68+
"Resource Group": experimentsDetails.ResourceGroup,
69+
"Powershell Chaos Start Base64 Or Ps File Path": experimentsDetails.PowershellChaosStartBase64OrPsFilePath,
70+
"Powershell Chaos End Base64 Or Ps File Path": experimentsDetails.PowershellChaosEndBase64OrPsFilePath,
71+
"Instance Name": experimentsDetails.AzureInstanceNames,
72+
"Sequence": experimentsDetails.Sequence,
73+
})
74+
75+
// Setting up Azure Subscription ID
76+
if experimentsDetails.SubscriptionID, err = azureCommon.GetSubscriptionID(); err != nil {
77+
log.Errorf("Failed to get the subscription id: %v", err)
78+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
79+
return
80+
}
81+
82+
// generating the event in chaosresult to marked the verdict as awaited
83+
msg := "experiment: " + experimentsDetails.ExperimentName + ", Result: Awaited"
84+
types.SetResultEventAttributes(&eventsDetails, types.AwaitedVerdict, msg, "Normal", &resultDetails)
85+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosResults"); eventErr != nil {
86+
log.Errorf("Failed to create %v event inside chaosresults", types.AwaitedVerdict)
87+
}
88+
89+
if experimentsDetails.EngineName != "" {
90+
// marking AUT as running, as we already checked the status of application under test
91+
msg := "AUT: Running"
92+
93+
// run the probes in the pre-chaos check
94+
if len(resultDetails.ProbeDetails) != 0 {
95+
96+
err = probe.RunProbes(ctx, &chaosDetails, clients, &resultDetails, "PreChaos", &eventsDetails)
97+
if err != nil {
98+
log.Errorf("Probe Failed: %v", err)
99+
msg := "AUT: Running, Probes: Unsuccessful"
100+
types.SetEngineEventAttributes(&eventsDetails, types.PreChaosCheck, msg, "Warning", &chaosDetails)
101+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine"); eventErr != nil {
102+
log.Errorf("Failed to create %v event inside chaosengine", types.PreChaosCheck)
103+
}
104+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
105+
return
106+
}
107+
msg = "AUT: Running, Probes: Successful"
108+
}
109+
// generating the events for the pre-chaos check
110+
types.SetEngineEventAttributes(&eventsDetails, types.PreChaosCheck, msg, "Normal", &chaosDetails)
111+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine"); eventErr != nil {
112+
log.Errorf("Failed to create %v event inside chaosengine", types.PreChaosCheck)
113+
}
114+
}
115+
116+
//Verify the azure target instance is running (pre-chaos)
117+
if chaosDetails.DefaultHealthCheck {
118+
if err = azureStatus.InstanceStatusCheckByName(experimentsDetails.AzureInstanceNames, experimentsDetails.ScaleSet, experimentsDetails.SubscriptionID, experimentsDetails.ResourceGroup); err != nil {
119+
log.Errorf("Azure instance status check failed: %v", err)
120+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
121+
return
122+
}
123+
log.Info("[Status]: Azure instance(s) is in running state (pre-chaos)")
124+
}
125+
126+
chaosDetails.Phase = types.ChaosInjectPhase
127+
128+
if err = litmusLIB.PrepareAzureRunScript(ctx, &experimentsDetails, clients, &resultDetails, &eventsDetails, &chaosDetails); err != nil {
129+
log.Errorf("Chaos injection failed: %v", err)
130+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
131+
return
132+
}
133+
134+
log.Info("[Confirmation]: Azure Instance Run Script chaos has been injected successfully")
135+
resultDetails.Verdict = v1alpha1.ResultVerdictPassed
136+
137+
chaosDetails.Phase = types.PostChaosPhase
138+
139+
//Verify the azure instance is running (post chaos)
140+
if chaosDetails.DefaultHealthCheck {
141+
if err = azureStatus.InstanceStatusCheckByName(experimentsDetails.AzureInstanceNames, experimentsDetails.ScaleSet, experimentsDetails.SubscriptionID, experimentsDetails.ResourceGroup); err != nil {
142+
log.Errorf("Azure instance status check failed: %v", err)
143+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
144+
return
145+
}
146+
log.Info("[Status]: Azure instance is in running state (post chaos)")
147+
}
148+
149+
if experimentsDetails.EngineName != "" {
150+
// marking AUT as running, as we already checked the status of application under test
151+
msg := "AUT: Running"
152+
153+
// run the probes in the post-chaos check
154+
if len(resultDetails.ProbeDetails) != 0 {
155+
err = probe.RunProbes(ctx, &chaosDetails, clients, &resultDetails, "PostChaos", &eventsDetails)
156+
if err != nil {
157+
log.Errorf("Probes Failed: %v", err)
158+
msg := "AUT: Running, Probes: Unsuccessful"
159+
types.SetEngineEventAttributes(&eventsDetails, types.PostChaosCheck, msg, "Warning", &chaosDetails)
160+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine"); eventErr != nil {
161+
log.Errorf("Failed to create %v event inside chaosengine", types.PostChaosCheck)
162+
}
163+
result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
164+
return
165+
}
166+
msg = "AUT: Running, Probes: Successful"
167+
}
168+
169+
// generating post chaos event
170+
types.SetEngineEventAttributes(&eventsDetails, types.PostChaosCheck, msg, "Normal", &chaosDetails)
171+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine"); eventErr != nil {
172+
log.Errorf("Failed to create %v event inside chaosengine", types.PostChaosCheck)
173+
}
174+
}
175+
176+
//Updating the chaosResult in the end of experiment
177+
log.Infof("[The End]: Updating the chaos result of %v experiment (EOT)", experimentsDetails.ExperimentName)
178+
err = result.ChaosResult(&chaosDetails, clients, &resultDetails, "EOT")
179+
if err != nil {
180+
log.Errorf("Unable to update the chaosresult: %v", err)
181+
}
182+
183+
// generating the event in chaosresult to marked the verdict as pass/fail
184+
msg = "experiment: " + experimentsDetails.ExperimentName + ", Result: " + string(resultDetails.Verdict)
185+
reason, eventType := types.GetChaosResultVerdictEvent(resultDetails.Verdict)
186+
types.SetResultEventAttributes(&eventsDetails, reason, msg, eventType, &resultDetails)
187+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosResults"); eventErr != nil {
188+
log.Errorf("Failed to create %v event inside chaosresults", reason)
189+
}
190+
191+
if experimentsDetails.EngineName != "" {
192+
msg := experimentsDetails.ExperimentName + " experiment has been " + string(resultDetails.Verdict) + "ed"
193+
types.SetEngineEventAttributes(&eventsDetails, types.Summary, msg, "Normal", &chaosDetails)
194+
if eventErr := events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine"); eventErr != nil {
195+
log.Errorf("Failed to create %v event inside chaosengine", types.Summary)
196+
}
197+
}
198+
199+
}

0 commit comments

Comments
 (0)