Merge branch 'master' into verkle-support

Author: pk910

cmd/root.go

@@ -26,6 +26,10 @@ var rootCmd = &cobra.Command{
 			logr.Fatal(err)
 		}
 
+		if maxConcurrentTests > 0 {
+			config.Coordinator.MaxConcurrentTests = maxConcurrentTests
+		}
+
 		if logFormat == "json" {
 			logr.SetFormatter(&logrus.JSONFormatter{})
 			logr.Info("Log format set to json")
@@ -37,7 +41,7 @@ var rootCmd = &cobra.Command{
 			logr.SetLevel(logrus.DebugLevel)
 		}
 
-		coord := coordinator.NewCoordinator(config, logr, metricsPort, maxConcurrentTests)
+		coord := coordinator.NewCoordinator(config, logr, metricsPort)
 
 		if err := coord.Run(cmd.Context()); err != nil {
 			logr.Fatal(err)
@@ -51,7 +55,7 @@ var (
 	logFormat          string
 	verbose            bool
 	metricsPort        int
-	maxConcurrentTests int
+	maxConcurrentTests uint64
 	version            bool
 )
 
@@ -70,7 +74,7 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file")
 	rootCmd.PersistentFlags().StringVar(&logFormat, "log-format", "text", "log format (default is text). Valid values are 'text', 'json'")
 	rootCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Verbose output")
-	rootCmd.Flags().IntVar(&maxConcurrentTests, "maxConcurrentTests", 1, "Number of tests to run concurrently")
+	rootCmd.Flags().Uint64Var(&maxConcurrentTests, "maxConcurrentTests", 0, "Number of tests to run concurrently")
 	rootCmd.Flags().IntVarP(&metricsPort, "metrics-port", "", 9090, "Port to serve Prometheus metrics on")
 	rootCmd.Flags().BoolVarP(&version, "version", "", false, "Print version information")
 }

pkg/coordinator/config.go

@@ -4,6 +4,7 @@ import (
 	"os"
 
 	"github.com/ethpandaops/assertoor/pkg/coordinator/clients"
+	"github.com/ethpandaops/assertoor/pkg/coordinator/human-duration"
 	"github.com/ethpandaops/assertoor/pkg/coordinator/names"
 	"github.com/ethpandaops/assertoor/pkg/coordinator/types"
 	web_types "github.com/ethpandaops/assertoor/pkg/coordinator/web/types"
@@ -23,13 +24,25 @@ type Config struct {
 	// Global variables
 	GlobalVars map[string]interface{} `yaml:"globalVars" json:"globalVars"`
 
+	// Coordinator config
+	Coordinator *CoordinatorConfig `yaml:"coordinator" json:"coordinator"`
+
 	// List of Test configurations.
 	Tests []*types.TestConfig `yaml:"tests" json:"tests"`
 
 	// List of yaml files with test configurations
 	ExternalTests []*types.ExternalTestConfig `yaml:"externalTests" json:"externalTests"`
 }
 
+//nolint:revive // ignore
+type CoordinatorConfig struct {
+	// Maximum number of tests executed concurrently
+	MaxConcurrentTests uint64 `yaml:"maxConcurrentTests" json:"maxConcurrentTests"`
+
+	// Test history cleanup delay
+	TestRetentionTime human.Duration `yaml:"testRetentionTime" json:"testRetentionTime"`
+}
+
 // DefaultConfig represents a sane-default configuration.
 func DefaultConfig() *Config {
 	return &Config{
@@ -41,6 +54,7 @@ func DefaultConfig() *Config {
 			},
 		},
 		GlobalVars:    make(map[string]interface{}),
+		Coordinator:   &CoordinatorConfig{},
 		Tests:         []*types.TestConfig{},
 		ExternalTests: []*types.ExternalTestConfig{},
 	}

pkg/coordinator/coordinator.go

@@ -47,7 +47,6 @@ type Coordinator struct {
 	testHistory          []types.Test
 	testRegistryMutex    sync.RWMutex
 	testNotificationChan chan bool
-	maxConcurrentTests   int
 }
 
 type testDescriptorEntry struct {
@@ -56,7 +55,7 @@ type testDescriptorEntry struct {
 	index      uint64
 }
 
-func NewCoordinator(config *Config, log logrus.FieldLogger, metricsPort, maxConcurrentTests int) *Coordinator {
+func NewCoordinator(config *Config, log logrus.FieldLogger, metricsPort int) *Coordinator {
 	return &Coordinator{
 		log: logger.NewLogger(&logger.ScopeOptions{
 			Parent:      log,
@@ -70,15 +69,14 @@ func NewCoordinator(config *Config, log logrus.FieldLogger, metricsPort, maxConc
 		testQueue:            []types.Test{},
 		testHistory:          []types.Test{},
 		testNotificationChan: make(chan bool, 1),
-		maxConcurrentTests:   maxConcurrentTests,
 	}
 }
 
 // Run executes the coordinator until completion.
 func (c *Coordinator) Run(ctx context.Context) error {
 	defer func() {
 		if err := recover(); err != nil {
-			logrus.WithError(err.(error)).Errorf("uncaught panic in coordinator.Run: %v, stack: %v", err, string(debug.Stack()))
+			c.log.GetLogger().WithError(err.(error)).Errorf("uncaught panic in coordinator.Run: %v, stack: %v", err, string(debug.Stack()))
 		}
 	}()
 
@@ -137,6 +135,9 @@ func (c *Coordinator) Run(ctx context.Context) error {
 	// start test scheduler
 	go c.runTestScheduler(ctx)
 
+	// start test cleanup routine
+	go c.runTestCleanup(ctx)
+
 	// run tests
 	c.runTestExecutionLoop(ctx)
 
@@ -345,7 +346,12 @@ func (c *Coordinator) createTestRun(descriptor types.TestDescriptor, configOverr
 }
 
 func (c *Coordinator) runTestExecutionLoop(ctx context.Context) {
-	semaphore := make(chan bool, c.maxConcurrentTests)
+	concurrencyLimit := c.Config.Coordinator.MaxConcurrentTests
+	if concurrencyLimit < 1 {
+		concurrencyLimit = 1
+	}
+
+	semaphore := make(chan bool, concurrencyLimit)
 
 	for {
 		var nextTest types.Test
@@ -393,6 +399,12 @@ func (c *Coordinator) runTest(ctx context.Context, testRef types.Test) {
 }
 
 func (c *Coordinator) runTestScheduler(ctx context.Context) {
+	defer func() {
+		if err := recover(); err != nil {
+			c.log.GetLogger().WithError(err.(error)).Panicf("uncaught panic in coordinator.runTestScheduler: %v, stack: %v", err, string(debug.Stack()))
+		}
+	}()
+
 	// startup scheduler
 	for _, testDescr := range c.getStartupTests() {
 		_, err := c.ScheduleTest(testDescr, nil, false)
@@ -480,3 +492,49 @@ func (c *Coordinator) getCronTests(cronTime time.Time) []types.TestDescriptor {
 
 	return descriptors
 }
+
+func (c *Coordinator) runTestCleanup(ctx context.Context) {
+	defer func() {
+		if err := recover(); err != nil {
+			c.log.GetLogger().WithError(err.(error)).Panicf("uncaught panic in coordinator.runTestCleanup: %v, stack: %v", err, string(debug.Stack()))
+		}
+	}()
+
+	retentionTime := c.Config.Coordinator.TestRetentionTime.Duration
+	if retentionTime <= 0 {
+		retentionTime = 14 * 24 * time.Hour
+	}
+
+	cleanupInterval := 1 * time.Hour
+	if retentionTime <= 4*time.Hour {
+		cleanupInterval = 10 * time.Minute
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(cleanupInterval):
+		}
+
+		c.cleanupTestHistory(retentionTime)
+	}
+}
+
+func (c *Coordinator) cleanupTestHistory(retentionTime time.Duration) {
+	c.testRegistryMutex.Lock()
+	defer c.testRegistryMutex.Unlock()
+
+	cleanedHistory := []types.Test{}
+
+	for _, test := range c.testHistory {
+		if test.Status() != types.TestStatusPending && test.StartTime().Add(retentionTime).Compare(time.Now()) == -1 {
+			test.Logger().Infof("cleanup test")
+			continue
+		}
+
+		cleanedHistory = append(cleanedHistory, test)
+	}
+
+	c.testHistory = cleanedHistory
+}

pkg/coordinator/web/handlers/index.go

@@ -24,6 +24,7 @@ type IndexPageTestDescriptor struct {
 
 type TestRunData struct {
 	RunID       uint64        `json:"run_id"`
+	TestID      string        `json:"test_id"`
 	Index       uint64        `json:"index"`
 	Name        string        `json:"name"`
 	IsStarted   bool          `json:"started"`
@@ -113,6 +114,7 @@ func (fh *FrontendHandler) getIndexPageData() (*IndexPage, error) {
 func (fh *FrontendHandler) getTestRunData(idx int, test types.Test) *TestRunData {
 	testData := &TestRunData{
 		RunID:      test.RunID(),
+		TestID:     test.TestID(),
 		Index:      uint64(idx),
 		Name:       test.Name(),
 		StartTime:  test.StartTime(),

pkg/coordinator/web/handlers/test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/ethpandaops/assertoor/pkg/coordinator/web"
 	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v3"
 )
 
 type TestPage struct {
@@ -110,12 +109,12 @@ func (fh *FrontendHandler) getTestPageData(testID string) (*TestPage, error) {
 	}
 
 	if testConfig.Config != nil {
-		testCfgYaml, err := yaml.Marshal(testConfig.Config)
+		testCfgJSON, err := json.Marshal(testConfig.Config)
 		if err != nil {
 			return nil, fmt.Errorf("failed marshalling test config: %v", err)
 		}
 
-		pageData.Config = string(testCfgYaml)
+		pageData.Config = string(testCfgJSON)
 	}
 
 	// test runs

pkg/coordinator/web/handlers/test_run.go

@@ -30,7 +30,7 @@ type TestRunPage struct {
 type TestRunTask struct {
 	Index       uint64            `json:"index"`
 	ParentIndex uint64            `json:"parent_index"`
-	IndentPx    uint64            `json:"indent_px"`
+	GraphLevels []uint64          `json:"graph_levels"`
 	Name        string            `json:"name"`
 	Title       string            `json:"title"`
 	IsStarted   bool              `json:"started"`
@@ -157,7 +157,7 @@ func (fh *FrontendHandler) getTestRunPageData(runID int64) (*TestRunPage, error)
 	if taskScheduler != nil && taskScheduler.GetTaskCount() > 0 {
 		indentationMap := map[uint64]int{}
 
-		for _, task := range taskScheduler.GetAllTasks() {
+		for idx, task := range taskScheduler.GetAllTasks() {
 			taskStatus := taskScheduler.GetTaskStatus(task)
 
 			taskData := &TestRunTask{
@@ -171,6 +171,7 @@ func (fh *FrontendHandler) getTestRunPageData(runID int64) (*TestRunPage, error)
 				StopTime:    taskStatus.StopTime,
 				Timeout:     task.Timeout(),
 				HasTimeout:  task.Timeout() > 0,
+				GraphLevels: []uint64{},
 			}
 
 			indentation := 0
@@ -179,7 +180,27 @@ func (fh *FrontendHandler) getTestRunPageData(runID int64) (*TestRunPage, error)
 			}
 
 			indentationMap[taskData.Index] = indentation
-			taskData.IndentPx = uint64(20 * indentation)
+
+			if indentation > 0 {
+				for i := 0; i < indentation; i++ {
+					taskData.GraphLevels = append(taskData.GraphLevels, 0)
+				}
+
+				taskData.GraphLevels[indentation-1] = 3
+
+				for i := idx - 1; i >= 0; i-- {
+					if pageData.Tasks[i].ParentIndex == taskData.ParentIndex {
+						pageData.Tasks[i].GraphLevels[indentation-1] = 2
+						break
+					}
+
+					if pageData.Tasks[i].Index == taskData.ParentIndex {
+						break
+					}
+
+					pageData.Tasks[i].GraphLevels[indentation-1] = 1
+				}
+			}
 
 			switch {
 			case !taskStatus.IsStarted:

pkg/coordinator/web/static/css/layout.css

@@ -1,7 +1,7 @@
 
 body main {
     display: flex;
-    overflow: hidden;
+    /* overflow: hidden; */
     min-height: calc(100vh - 64px)!important;
 }
 
@@ -69,6 +69,10 @@ main .sidebar {
     background-color: rgba(var(--bs-emphasis-color-rgb), 0.03);
 }
 
+main .sidebar .nav-pills {
+    --bs-nav-pills-link-active-bg: rgba(13, 110, 253, 0.7);
+}
+
 main .sidebar .nav-link {
     color: var(--bs-body-color);
 }

pkg/coordinator/web/templates/_layout/layout.html

@@ -24,6 +24,9 @@
       <link rel="preload" as="font" href="/webfonts/fa-brands-400.woff2" crossorigin />
       <link id="app-style" rel="stylesheet" href="/css/layout.css" />
       {{ template "css" .Data }}
+      <script src="/js/jquery.min.js"></script>
+      <script src="/js/bootstrap.bundle.min.js"></script>
+      <script src="/js/color-modes.js"></script>
       {{ if .ShowSidebar }}
       <style>
         .main-navigation {
@@ -51,9 +54,6 @@
         </div>
       </main>
 
-      <script src="/js/jquery.min.js"></script>
-      <script src="/js/bootstrap.bundle.min.js"></script>
-      <script src="/js/color-modes.js"></script>
       <script src="/js/clipboard.min.js"></script>
       <script src="/js/assertoor.js"></script>
       {{ template "js" .Data }}

pkg/coordinator/web/templates/sidebar/sidebar.html

@@ -7,18 +7,18 @@
         </div>
       </div>
       <div class="row">
-        <div class="col-6">
+        <div class="col-5">
           CL: {{ .CLReadyCount }} / {{ .ClientCount }}
         </div>
-        <div class="col-6">
+        <div class="col-7">
           Slot: {{ .CLHeadSlot }}
         </div>
       </div>
       <div class="row">
-        <div class="col-6">
+        <div class="col-5">
           EL: {{ .ELReadyCount }} / {{ .ClientCount }}
         </div>
-        <div class="col-6">
+        <div class="col-7">
           Block: {{ .ELHeadNumber }}
         </div>
       </div>