Merge pull request #55 from crytic/additional-planner-faults

Additional planner faults

Author: bsamuels453

README.md

@@ -159,6 +159,16 @@ Config:
     - grace_period: 1800s # how long to wait for health checks to pass before marking the test as failed
 ```
 
+#### IOLatency
+Config:
+```yaml
+    - grace_period: 1800s # how long to wait for health checks to pass before marking the test as failed
+      delay: 1000ms # how long the i/o delay should be
+      duration: 1m # how long the fault should last
+      percent: 50 # the percentage of i/o requests impacted.
+```
+
+
 ## Running test suites
 
 Once you've got your configuration set up, you can run Attacknet:
@@ -175,6 +185,14 @@ At this time, health checks will be run in perpetuity once the fault has conclud
 **Dec 15, 2023 version v0.1 (internal)**
 - Initial internal release
 
+**Jan 11, 2023 version v0.2 (internal)**
+- Updated to kurtosis v0.86.1
+- Updated to Go 1.21
+- Grafana port-forwarding has been temporarily disabled
+- Introduces multi-step tests. This allows multiple faults and other actions to be composed into a single test.
+- Introduces the suite planner. The suite planner allows the user to define a set of testing criteria/dimensions, which the planner turns into a suite containing multiple tests.
+- Successful & failed test suites now emit test artifacts summarizing the results of the test.
+- 
 ## Developing (wip)
 
 1. Install pre-commit

pkg/health/checker.go

@@ -14,7 +14,7 @@ import (
 
 type CheckOrchestrator struct {
 	checkerImpl types.GenericNetworkChecker
-	gracePeriod time.Duration
+	gracePeriod *time.Duration
 }
 
 func BuildHealthChecker(cfg *confTypes.ConfigParsed, kubeClient *kubernetes.KubeClient, podsUnderTest []*chaos_mesh.PodUnderTest, healthCheckConfig confTypes.HealthCheckConfig) (*CheckOrchestrator, error) {
@@ -35,7 +35,7 @@ func BuildHealthChecker(cfg *confTypes.ConfigParsed, kubeClient *kubernetes.Kube
 
 func (hc *CheckOrchestrator) RunChecks(ctx context.Context) (*types.HealthCheckResult, error) {
 	start := time.Now()
-	latestAllowable := start.Add(hc.gracePeriod)
+	latestAllowable := start.Add(*hc.gracePeriod)
 	log.Infof("Allowing up to %.0f seconds for health checks to pass on all nodes", hc.gracePeriod.Seconds())
 
 	lastHealthCheckResult := &types.HealthCheckResult{}

pkg/kubernetes/port_forward.go

@@ -111,18 +111,28 @@ func (c *KubeClient) StartPortForwarding(pod string, localPort, remotePort int,
 	}
 	log.Debugf("Starting port-forward to pod/%s:%d", pod, remotePort)
 
-	go func() {
-		if err = portForward.ForwardPorts(); err != nil {
-			panic(stacktrace.Propagate(err, "unable to start port forward session"))
+	portForwardIssueCh := make(chan error, 1)
+	defer close(portForwardIssueCh)
+
+	retries := 3
+	go func(retriesRem int) {
+		for retriesRem > 0 {
+			if err = portForward.ForwardPorts(); err == nil {
+				retriesRem--
+				if retriesRem == 0 {
+					portForwardIssueCh <- stacktrace.Propagate(err, "unable to start port forward session")
+				}
+			}
 		}
-	}()
+	}(retries)
 
 	select {
 	case <-readyCh:
 		log.Debugf("Port-forward established to pod/%s:%d", pod, remotePort)
 	case <-time.After(time.Minute):
 		return nil, errors.New("timed out after waiting to establish port forward")
+	case err = <-portForwardIssueCh:
+		return nil, err
 	}
-
 	return stopCh, nil
 }

pkg/plan/serialization.go

@@ -19,6 +19,7 @@ func SerializeNetworkTopology(nodes []*network.Node, config *network.GenesisConf
 			"dora",
 		},
 		ParallelKeystoreGen: false,
+		Persistent:          true,
 	}
 
 	bs, err := yaml.Marshal(netConfig)

pkg/plan/suite/faults.go

@@ -2,8 +2,11 @@ package suite
 
 import (
 	"attacknet/cmd/pkg/types"
+	"fmt"
 	"github.com/kurtosis-tech/stacktrace"
 	"gopkg.in/yaml.v3"
+	"strings"
+	"time"
 )
 
 // We can't use chaos mesh's types because type-inlining is not supported in yaml.v3, making it so you can't serialize
@@ -55,6 +58,27 @@ type PodChaosWrapper struct {
 	PodChaosFault `yaml:"chaosFaultSpec"`
 }
 
+type IOChaosSpec struct {
+	Selector `yaml:"selector"`
+	Mode     string `yaml:"mode"`
+
+	Action     string         `yaml:"action"`
+	VolumePath string         `yaml:"volumePath"`
+	Delay      *time.Duration `yaml:"delay"`
+	Percent    int            `yaml:"percent"`
+	Duration   *time.Duration `yaml:"duration"`
+}
+
+type IOChaosFault struct {
+	Spec       IOChaosSpec `yaml:"spec"`
+	Kind       string      `yaml:"kind"`
+	ApiVersion string      `yaml:"apiVersion"`
+}
+
+type IOChaosWrapper struct {
+	IOChaosFault `yaml:"chaosFaultSpec"`
+}
+
 func convertFaultSpecToMap(s interface{}) (map[string]interface{}, error) {
 	// convert to map[string]interface{} using yaml intermediate. seriously.
 	bs, err := yaml.Marshal(s)
@@ -128,3 +152,62 @@ func buildPodRestartFault(description string, expressionSelectors []ChaosExpress
 	}
 	return step, nil
 }
+
+func getVolumePathForIOFault(podName string) (string, error) {
+	var nodeType string
+	parts := strings.Split(podName, "-")
+	if parts[0] == "el" {
+		nodeType = "execution"
+	} else {
+		nodeType = "consensus"
+	}
+	if parts[len(parts)-1] == "validator" {
+		return "", stacktrace.NewError("cannot create an i/o latency fault on a validator sidecar pod. Try to target matching clients only: %s", podName)
+	}
+	clientName := parts[2]
+	volumeTarget := fmt.Sprintf("/data/%s/%s-data", clientName, nodeType)
+	return volumeTarget, nil
+}
+
+func buildIOLatencyFault(description string, expressionSelector ChaosExpressionSelector, delay *time.Duration, percent int, duration *time.Duration) ([]types.PlanStep, error) {
+	var steps []types.PlanStep
+
+	for _, podName := range expressionSelector.Values {
+		volumePath, err := getVolumePathForIOFault(podName)
+		if err != nil {
+			return nil, err
+		}
+
+		t := IOChaosWrapper{
+			IOChaosFault: IOChaosFault{
+				Kind:       "IOChaos",
+				ApiVersion: "chaos-mesh.org/v1alpha1",
+				Spec: IOChaosSpec{
+					Duration: duration,
+					Mode:     "all",
+					Selector: Selector{
+						ExpressionSelectors: []ChaosExpressionSelector{expressionSelector},
+					},
+					Action:     "latency",
+					VolumePath: volumePath,
+					Delay:      delay,
+					Percent:    percent,
+				},
+			},
+		}
+
+		faultSpec, err := convertFaultSpecToMap(t)
+		if err != nil {
+			return nil, err
+		}
+
+		step := types.PlanStep{
+			StepType:        types.InjectFault,
+			StepDescription: description,
+			Spec:            faultSpec,
+		}
+		steps = append(steps, step)
+	}
+
+	return steps, nil
+}

pkg/plan/suite/step_builder.go

@@ -4,7 +4,9 @@ import (
 	"attacknet/cmd/pkg/plan/network"
 	"attacknet/cmd/pkg/types"
 	"fmt"
+	"github.com/kurtosis-tech/stacktrace"
 	log "github.com/sirupsen/logrus"
+	"time"
 )
 
 type clientType string
@@ -33,9 +35,9 @@ func composeWaitForFaultCompletionStep() *types.PlanStep {
 	return &types.PlanStep{StepType: types.WaitForFaultCompletion, StepDescription: "wait for faults to terminate"}
 }
 
-func composeNodeClockSkewPlanSteps(nodesSelected []*ChaosTargetSelector, skew, duration string) ([]types.PlanStep, error) {
+func composeNodeClockSkewPlanSteps(targetsSelected []*ChaosTargetSelector, skew, duration string) ([]types.PlanStep, error) {
 	var steps []types.PlanStep
-	for _, target := range nodesSelected {
+	for _, target := range targetsSelected {
 		description := fmt.Sprintf("Inject clock skew on target %s", target.Description)
 
 		skewStep, err := buildClockSkewFault(description, skew, duration, target.Selector)
@@ -48,10 +50,10 @@ func composeNodeClockSkewPlanSteps(nodesSelected []*ChaosTargetSelector, skew, d
 	return steps, nil
 }
 
-func composeNodeRestartSteps(nodesSelected []*ChaosTargetSelector) ([]types.PlanStep, error) {
+func composeNodeRestartSteps(targetsSelected []*ChaosTargetSelector) ([]types.PlanStep, error) {
 	var steps []types.PlanStep
 
-	for _, target := range nodesSelected {
+	for _, target := range targetsSelected {
 		description := fmt.Sprintf("Restart target %s", target.Description)
 		restartStep, err := buildPodRestartFault(description, target.Selector)
 
@@ -63,3 +65,39 @@ func composeNodeRestartSteps(nodesSelected []*ChaosTargetSelector) ([]types.Plan
 
 	return steps, nil
 }
+
+func areExprSelectorsMatchingIdIn(expressionSelectors []ChaosExpressionSelector) error {
+	for _, selector := range expressionSelectors {
+		if selector.Key != "kurtosistech.com/id" {
+			return stacktrace.NewError("i/o latency faults can only be target using pod id: %s", selector.Key)
+		}
+		if selector.Operator != "In" {
+			return stacktrace.NewError("i/o latency faults can only be target using the 'In' operator: %s", selector.Operator)
+		}
+	}
+	return nil
+}
+
+func composeIOLatencySteps(targetsSelected []*ChaosTargetSelector, delay *time.Duration, percent int, duration *time.Duration) ([]types.PlanStep, error) {
+	var steps []types.PlanStep
+
+	for _, target := range targetsSelected {
+		description := fmt.Sprintf("Inject i/o latency on target %s", target.Description)
+		err := areExprSelectorsMatchingIdIn(target.Selector)
+		if err != nil {
+			return nil, err
+		}
+
+		// for i/o faults, we need to create a plan step for each individual pod because the fault spec has to say the data path.
+		for _, selector := range target.Selector {
+			ioLatencySteps, err := buildIOLatencyFault(description, selector, delay, percent, duration)
+			if err != nil {
+				return nil, err
+			}
+			steps = append(steps, ioLatencySteps...)
+		}
+	}
+
+	return steps, nil
+
+}

pkg/plan/suite/suite_builder.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/kurtosis-tech/stacktrace"
 	log "github.com/sirupsen/logrus"
+	"strconv"
 	"time"
 )
 
@@ -72,12 +73,25 @@ func ComposeTestSuite(
 			}
 		}
 	}
-
-	log.Infof("ESTIMATE: Running this test suite will take, at minimum, %d minutes", runtimeEstimate/60)
+	log.Infof("Tests generated: %d", len(tests))
+	log.Infof("ESTIMATE: Running this test suite will take, at minimum, %d minutes based on fault durations.", runtimeEstimate/60)
 
 	return tests, nil
 }
 
+func getDurationValue(key string, m map[string]string) (*time.Duration, error) {
+
+	valueStr, ok := m[key]
+	if !ok {
+		return nil, stacktrace.NewError("missing %s field", key)
+	}
+	duration, err := time.ParseDuration(valueStr)
+	if err != nil {
+		return nil, stacktrace.NewError("unable to convert %s field to a time duration", key)
+	}
+	return &duration, nil
+}
+
 func composeTestForFaultType(
 	faultType FaultTypeEnum,
 	config map[string]string,
@@ -95,28 +109,46 @@ func composeTestForFaultType(
 		if !ok {
 			return nil, stacktrace.NewError("missing duration field for clock skew fault")
 		}
-		grace, ok := config["grace_period"]
-		if !ok {
-			return nil, stacktrace.NewError("missing grace_period field for clock skew fault")
-		}
-		graceDuration, err := time.ParseDuration(grace)
+		graceDuration, err := getDurationValue("grace_period", config)
 		if err != nil {
-			return nil, stacktrace.NewError("unable to convert grace_period field to a time duration for clock skew fault")
+			return nil, err
 		}
 
 		description := fmt.Sprintf("Apply %s clock skew for %s against %d targets. %s", skew, duration, len(targetSelectors), targetingDescription)
 		return composeNodeClockSkewTest(description, targetSelectors, skew, duration, graceDuration)
 	case FaultContainerRestart:
-		grace, ok := config["grace_period"]
-		if !ok {
-			return nil, stacktrace.NewError("missing grace_period field for restsrt fault")
-		}
-		graceDuration, err := time.ParseDuration(grace)
+
+		graceDuration, err := getDurationValue("grace_period", config)
 		if err != nil {
-			return nil, stacktrace.NewError("unable to convert grace_period field to a time duration for clock skew fault")
+			return nil, err
 		}
 		description := fmt.Sprintf("Restarting %d targets. %s", len(targetSelectors), targetingDescription)
 		return composeNodeRestartTest(description, targetSelectors, graceDuration)
+	case FaultIOLatency:
+		grace, err := getDurationValue("grace_period", config)
+		if err != nil {
+			return nil, err
+		}
+		delay, err := getDurationValue("delay", config)
+		if err != nil {
+			return nil, err
+		}
+		faultDuration, err := getDurationValue("duration", config)
+		if err != nil {
+			return nil, err
+		}
+
+		percent, ok := config["percent"]
+		if !ok {
+			return nil, stacktrace.NewError("missing percent field in io latency fault")
+		}
+		percentInt, err := strconv.Atoi(percent)
+		if err != nil {
+			return nil, stacktrace.Propagate(err, "unable to parse io latency fault percent field")
+		}
+		description := fmt.Sprintf("Apply %s i/o latency for %s. Impacting %d pct of i/o calls. against %d targets. %s", delay, faultDuration, percentInt, len(targetSelectors), targetingDescription)
+
+		return composeIOLatencyTest(description, targetSelectors, delay, percentInt, faultDuration, grace)
 	}
 
 	return nil, nil