service cluster: Verify that manager and agent versions match

pkg/cmd/scylla-manager/server.go

@@ -328,6 +328,12 @@ func (s *server) startServices(ctx context.Context) error {
 	if err := s.clusterSvc.Init(ctx); err != nil {
 		return errors.Wrapf(err, "cluster service")
 	}
+
+	// Instead this could be called from the restapi
+	if err := s.clusterSvc.VerifySMAndAgentVersions(ctx); err != nil {
+		return errors.Wrapf(err, "cluster service")
+	}
+
 	if err := s.schedSvc.LoadTasks(ctx); err != nil {
 		return errors.Wrapf(err, "schedule service")
 	}

pkg/service/cluster/service.go

@@ -18,6 +18,7 @@ import (
 	"github.com/scylladb/go-log"
 	"github.com/scylladb/gocqlx/v2"
 	"github.com/scylladb/gocqlx/v2/qb"
+	"github.com/scylladb/scylla-manager/v3/pkg"
 	"github.com/scylladb/scylla-manager/v3/pkg/metrics"
 	"github.com/scylladb/scylla-manager/v3/pkg/schema/table"
 	"github.com/scylladb/scylla-manager/v3/pkg/scyllaclient"
@@ -875,3 +876,34 @@ func GetRPCAddresses(ctx context.Context, client *scyllaclient.Client, hosts []s
 
 	return sessionHosts, combinedError
 }
+
+// VerifySMAndAgentVersions checks if the versions of all agents and scylla manager are the same
+func (s *Service) VerifySMAndAgentVersions(ctx context.Context) error {
+	smVersion := pkg.Version()
+	var combinedError error
+
+	clusters, err := s.ListClusters(ctx, &Filter{})
+	if err != nil {
+		return errors.New("unable to get list of clusters")
+	}
+
+	// Possible improvement could be to process each cluster in parallel
+	for _, cluster := range clusters {
+		client, err := s.CreateClientNoCache(ctx, cluster.ID)
+		if err != nil {
+			return fmt.Errorf("unable to create client for cluster %s", cluster.ID.String())
+		}
+		for _, h := range client.Config().Hosts {
+			ni, err := client.NodeInfo(ctx, h)
+			if err != nil {
+				combinedError = multierr.Append(combinedError, err)
+				continue
+			}
+			if ni.AgentVersion != smVersion {
+				combinedError = multierr.Append(fmt.Errorf("agent version %s on host %s does not match scylla manager version %s", h, ni.AgentVersion, smVersion), combinedError)
+			}
+		}
+	}
+
+	return combinedError
+}

pkg/service/cluster/service_integration_test.go

@@ -812,6 +812,23 @@ func TestServiceStorageIntegration(t *testing.T) {
 	})
 }
 
+func TestVerifySMAndAgentVersionsIntegration(t *testing.T) {
+	session := CreateScyllaManagerDBSession(t)
+	secretsStore := store.NewTableStore(session, table.Secrets)
+	s, err := cluster.NewService(session, metrics.NewClusterMetrics(), secretsStore, scyllaclient.DefaultTimeoutConfig(),
+		server.DefaultConfig().ClientCacheTimeout, log.NewDevelopment())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("verify scylla manager and agent versions", func(t *testing.T) {
+		err = s.VerifySMAndAgentVersions(context.Background())
+		if err != nil {
+			t.Fatalf("scylla manager and agent have different versions %v", err)
+		}
+	})
+}
+
 func validCluster() *cluster.Cluster {
 	return &cluster.Cluster{
 		ID:        uuid.MustRandom(),