Merge branch 'master' into verkle-support

Author: pk910

pkg/coordinator/tasks/check_clients_are_healthy/README.md

@@ -36,4 +36,4 @@ These are the default settings for the `check_clients_are_healthy` task:
     skipExecutionCheck: false
     expectUnhealthy: false
     minClientCount: 0
- ```
+```

pkg/coordinator/tasks/check_consensus_proposer_duty/README.md

@@ -11,6 +11,9 @@ The `check_consensus_proposer_duty` task is designed to check for a specific pro
 - **`validatorIndex`**:\
   The index of a specific validator to be checked. If this is set, the task focuses on the validator with this index. If it is `null`, the task does not filter by a specific validator index.
 
+- **`minSlotDistance`**:\
+  The minimum slot distance from the current slot at which to start checking for the validator's proposer duty. A value of 0 indicates the current slot.
+
 - **`maxSlotDistance`**:\
   The maximum number of slots (individual time periods in the blockchain) within which the validator is expected to propose a block. The task succeeds if a matching validator is scheduled for block proposal within this slot distance.
 

pkg/coordinator/tasks/check_consensus_proposer_duty/config.go

@@ -3,6 +3,7 @@ package checkconsensusproposerduty
 type Config struct {
 	ValidatorNamePattern string  `yaml:"validatorNamePattern" json:"validatorNamePattern"`
 	ValidatorIndex       *uint64 `yaml:"validatorIndex" json:"validatorIndex"`
+	MinSlotDistance      uint64  `yaml:"minSlotDistance" json:"minSlotDistance"`
 	MaxSlotDistance      uint64  `yaml:"maxSlotDistance" json:"maxSlotDistance"`
 	FailOnCheckMiss      bool    `yaml:"failOnCheckMiss" json:"failOnCheckMiss"`
 }

pkg/coordinator/tasks/check_consensus_proposer_duty/task.go

@@ -182,6 +182,11 @@ func (t *Task) runProposerDutyCheck(slot uint64) bool {
 			}
 		}
 
+		if t.config.MinSlotDistance > 0 && slot-1-currentSlot < t.config.MinSlotDistance {
+			t.logger.Errorf("slot %v check failed: matching duty too early: in %v slots, min distance: %v", slot, slot-1-currentSlot, t.config.MinSlotDistance)
+			return false
+		}
+
 		return true
 	}
 }

pkg/coordinator/tasks/run_task_matrix/task.go

@@ -127,25 +127,34 @@ func (t *Task) Execute(ctx context.Context) error {
 
 	t.taskCtx = taskCtx
 
-	var taskWaitChan chan bool
-	if !t.config.RunConcurrent {
-		taskWaitChan = make(chan bool, 1)
-	}
+	var currentTaskWaitChan, previousTaskWaitChan chan bool
 
 	// start child tasks
 	for i := range t.tasks {
 		taskWaitGroup.Add(1)
 
+		if !t.config.RunConcurrent {
+			previousTaskWaitChan = currentTaskWaitChan
+			currentTaskWaitChan = make(chan bool)
+		}
+
 		t.taskIdxMap[t.tasks[i]] = i
 
-		go func(i int) {
+		go func(i int, taskWaitChan, prevTaskWaitChan chan bool) {
 			defer taskWaitGroup.Done()
 
-			if taskWaitChan != nil {
-				taskWaitChan <- true
-				defer func() {
-					<-taskWaitChan
-				}()
+			if !t.config.RunConcurrent {
+				if prevTaskWaitChan != nil {
+					// wait for previous task to be executed
+					select {
+					case <-prevTaskWaitChan:
+					case <-ctx.Done():
+						return
+					}
+				}
+
+				// allow next task to run once this finishes
+				defer close(taskWaitChan)
 			}
 
 			task := t.tasks[i]
@@ -158,7 +167,7 @@ func (t *Task) Execute(ctx context.Context) error {
 
 			//nolint:errcheck // ignore
 			t.ctx.Scheduler.ExecuteTask(taskCtx, task, t.watchChildTask)
-		}(i)
+		}(i, currentTaskWaitChan, previousTaskWaitChan)
 	}
 
 	// watch result updates

pkg/coordinator/tasks/run_task_options/README.md

@@ -8,6 +8,9 @@ The `run_task_options` task is designed to execute a single task with configurab
 - **`task`**:\
   The task to be executed. This is defined following the standard task definition format.
 
+- **`propagateResult`**:\
+  This setting controls how the result of the child task influences the result of the `run_task_options` task. If set to `true`, any change in the result of the child task (success or failure) is immediately reflected in the result of the parent `run_task_options` task. If `false`, the child task's result is only propagated to the parent task after the child task has completed its execution.
+
 - **`exitOnResult`**:\
   If set to `true`, the task will cancel the child task as soon as it sets a result, whether it is "success" or "failure." This option is useful for scenarios where immediate response to the child task's result is necessary.
 
@@ -20,6 +23,12 @@ The `run_task_options` task is designed to execute a single task with configurab
 - **`ignoreFailure`**:\
   When `true`, any failure result from the child task is ignored, and the `run_task_options` task will return a success result instead. This is useful for cases where the child task's failure is an acceptable outcome.
 
+- **`retryOnFailure`**:\
+  If set to `true`, the task will retry the execution of the child task if it fails, up to the maximum number of retries specified by `maxRetryCount`.
+
+- **`maxRetryCount`**:\
+  The maximum number of times the child task will be retried if it fails and `retryOnFailure` is true. A value of 0 means no retries.
+
 - **`newVariableScope`**:\
   Determines whether to create a new variable scope for the child task. If `false`, the current scope is passed through, allowing the child task to share the same variable context as the `run_task_options` task.
 
@@ -31,9 +40,12 @@ Default settings for the `run_task_options` task:
 - name: run_task_options
   config:
     task: null
+    propagateResult: false
     exitOnResult: false
     invertResult: false
     expectFailure: false
     ignoreFailure: false
+    retryOnFailure: false
+    maxRetryCount: 0
     newVariableScope: false
 ```

pkg/coordinator/tasks/run_task_options/config.go

@@ -8,10 +8,13 @@ import (
 
 type Config struct {
 	Task             *helper.RawMessage `yaml:"task" json:"tasks"`
+	PropagateResult  bool               `yaml:"propagateResult" json:"propagateResult"`
 	ExitOnResult     bool               `yaml:"exitOnResult" json:"exitOnResult"`
 	InvertResult     bool               `yaml:"invertResult" json:"invertResult"`
 	ExpectFailure    bool               `yaml:"expectFailure" json:"expectFailure"`
 	IgnoreFailure    bool               `yaml:"ignoreFailure" json:"ignoreFailure"`
+	RetryOnFailure   bool               `yaml:"retryOnFailure" json:"retryOnFailure"`
+	MaxRetryCount    uint               `yaml:"maxRetryCount" json:"maxRetryCount"`
 	NewVariableScope bool               `yaml:"newVariableScope" json:"newVariableScope"`
 }
 

pkg/coordinator/tasks/run_task_options/task.go

@@ -80,48 +80,65 @@ func (t *Task) LoadConfig() error {
 		return err2
 	}
 
-	// init child task
-	taskOpts, err := t.ctx.Scheduler.ParseTaskOptions(config.Task)
-	if err != nil {
-		return fmt.Errorf("failed parsing child task config: %w", err)
-	}
-
-	taskVars := t.ctx.Vars
-	if config.NewVariableScope {
-		taskVars = taskVars.NewScope()
-	}
-
-	t.task, err = t.ctx.NewTask(taskOpts, taskVars)
-	if err != nil {
-		return fmt.Errorf("failed initializing child task: %w", err)
-	}
-
 	t.config = config
 
 	return nil
 }
 
 func (t *Task) Execute(ctx context.Context) error {
-	err := t.ctx.Scheduler.ExecuteTask(ctx, t.task, func(ctx context.Context, cancelFn context.CancelFunc, _ types.Task) {
-		t.watchTaskResult(ctx, cancelFn)
-	})
-
-	switch {
-	case t.config.ExpectFailure:
-		if err == nil {
-			return fmt.Errorf("child task succeeded, but should have failed")
-		}
-	case t.config.IgnoreFailure:
+	var taskErr error
+
+	retryCount := uint(0)
+
+	for {
+		// init child task
+		taskOpts, err := t.ctx.Scheduler.ParseTaskOptions(t.config.Task)
 		if err != nil {
-			t.logger.Warnf("child task failed: %w", err)
+			return fmt.Errorf("failed parsing child task config: %w", err)
+		}
+
+		taskVars := t.ctx.Vars
+		if t.config.NewVariableScope {
+			taskVars = taskVars.NewScope()
 		}
-	default:
+
+		t.task, err = t.ctx.NewTask(taskOpts, taskVars)
 		if err != nil {
-			return fmt.Errorf("child task failed: %w", err)
+			return fmt.Errorf("failed initializing child task: %w", err)
 		}
+
+		// execute task
+		taskErr = t.ctx.Scheduler.ExecuteTask(ctx, t.task, func(ctx context.Context, cancelFn context.CancelFunc, _ types.Task) {
+			t.watchTaskResult(ctx, cancelFn)
+		})
+
+		switch {
+		case t.config.RetryOnFailure && retryCount < t.config.MaxRetryCount:
+			if taskErr != nil {
+				retryCount++
+
+				t.logger.Warnf("child task failed: %w (retrying)", taskErr)
+
+				continue
+			}
+		case t.config.ExpectFailure:
+			if taskErr == nil {
+				return fmt.Errorf("child task succeeded, but should have failed")
+			}
+		case t.config.IgnoreFailure:
+			if taskErr != nil {
+				t.logger.Warnf("child task failed: %w", taskErr)
+			}
+		default:
+			if taskErr != nil {
+				return fmt.Errorf("child task failed: %w", taskErr)
+			}
+		}
+
+		break
 	}
 
-	return nil
+	return taskErr
 }
 
 func (t *Task) watchTaskResult(ctx context.Context, cancelFn context.CancelFunc) {
@@ -158,7 +175,9 @@ func (t *Task) watchTaskResult(ctx context.Context, cancelFn context.CancelFunc)
 			}
 		}
 
-		t.ctx.SetResult(taskResult)
+		if t.config.PropagateResult {
+			t.ctx.SetResult(taskResult)
+		}
 
 		if t.config.ExitOnResult {
 			cancelFn()

pkg/coordinator/tasks/run_tasks/README.md

@@ -1,7 +1,7 @@
 ## `run_tasks` Task
 
 ### Description
-The `run_tasks` task is designed for executing a series of tasks sequentially, ensuring each task is completed before starting the next. This setup is essential for tests requiring a specific order of task execution.
+The `run_tasks` task executes a series of specified tasks sequentially. This is particularly useful for scenarios where tasks need to be performed in a specific order, with the outcome of one potentially affecting the subsequent ones.
 
 #### Task Behavior
 - The task starts the child tasks one after the other in the order they are listed.
@@ -15,6 +15,9 @@ An important aspect of this task is that it cancels tasks once they return a res
 - **`tasks`**:\
   An array of tasks to be executed one after the other. Each task is defined according to the standard task structure.
 
+- **`stopChildOnResult`**:\
+  If set to `true`, each child task in the sequence is stopped as soon as it sets a result (either "success" or "failure"). This ensures that once a task has reached a outcome, it does not continue to run unnecessarily, allowing the next task in the sequence to commence.
+
 - **`expectFailure`**:\
   If set to `true`, this option expects each task in the sequence to fail. The task sequence stops with a "failure" result if any task does not fail as expected.
 
@@ -29,6 +32,7 @@ Default settings for the `run_tasks` task:
 - name: run_tasks
   config:
     tasks: []
+    stopChildOnResult: true
     expectFailure: false
     continueOnFailure: false
 ```

pkg/coordinator/tasks/run_tasks/config.go

@@ -8,13 +8,15 @@ import (
 
 type Config struct {
 	Tasks             []helper.RawMessage `yaml:"tasks" json:"tasks"`
+	StopChildOnResult bool                `yaml:"stopChildOnResult" json:"stopChildOnResult"`
 	ExpectFailure     bool                `yaml:"expectFailure" json:"expectFailure"`
 	ContinueOnFailure bool                `yaml:"continueOnFailure" json:"continueOnFailure"`
 }
 
 func DefaultConfig() Config {
 	return Config{
-		Tasks: []helper.RawMessage{},
+		Tasks:             []helper.RawMessage{},
+		StopChildOnResult: true,
 	}
 }