Add support of names and digest ScyllaDB image reference

Author: grzywin

assets/config/config.yaml

@@ -1,5 +1,5 @@
 operator:
-  scyllaDBVersion: "6.2.0"
+  scyllaDBVersion: "sha256:5b53a7c60d9f9555bb87791ff29b2e633c6f472aec00de7afaf4db1addc6d594"
   # scyllaDBEnterpriseVersionNeedingConsistentClusterManagementOverride sets enterprise version
   # that requires consistent_cluster_management workaround for restore.
   # In the future, enterprise versions should be run as a different config instance in its own run.
@@ -13,6 +13,6 @@ operator:
   prometheusVersion: "v2.54.1"
 operatorTests:
   scyllaDBVersions:
-    updateFrom: "6.2.0-rc2"
-    upgradeFrom: "6.1.2"
+    updateFrom: "sha256:cf00b8bfcce4d27e8527cad913e83db89af842e6cfb64ff4736bce83594b33d1"
+    upgradeFrom: "sha256:0c1c6b49916230f62bdb0c2539b811b1a99d2c43d06ed810fad6f51015dea3ae"
   nodeSetupImage: "quay.io/scylladb/scylla-operator-images:node-setup-v0.0.3@sha256:c6b3de240cc5c884d5c617485bae35c51572cdfd39b6431d2e1f759c7d7feea1"

pkg/cmd/tests/options.go

@@ -7,11 +7,13 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/blang/semver"
 	"github.com/onsi/ginkgo/v2"
 	configassets "github.com/scylladb/scylla-operator/assets/config"
 	scyllav1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1"
 	"github.com/scylladb/scylla-operator/pkg/genericclioptions"
 	"github.com/scylladb/scylla-operator/pkg/helpers/slices"
+	scyllasemver "github.com/scylladb/scylla-operator/pkg/semver"
 	"github.com/scylladb/scylla-operator/test/e2e/framework"
 	"github.com/spf13/cobra"
 	apierrors "k8s.io/apimachinery/pkg/util/errors"
@@ -27,7 +29,7 @@ const (
 	digestRegexp = `[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][[:xdigit:]]{32,}`
 )
 
-var tagWithOptionalDigestRegexp = regexp.MustCompile("^" + referenceTagRegexp + "(?:@" + digestRegexp + ")?$")
+var tagOrDigestRegexp = regexp.MustCompile("^(?:" + referenceTagRegexp + "(?:@" + digestRegexp + ")?|" + digestRegexp + ")$")
 
 type IngressControllerOptions struct {
 	Address           string
@@ -191,37 +193,64 @@ func (o *TestFrameworkOptions) Validate(args []string) error {
 		errors = append(errors, fmt.Errorf("gcs-service-account-key-path and s3-credentials-file-path can't be set simultanously"))
 	}
 
-	if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBVersion) {
+	if !tagOrDigestRegexp.MatchString(o.ScyllaDBVersion) {
 		errors = append(errors, fmt.Errorf(
-			"invalid scylladb-version format: %q. Expected format: <tag>[@<digest>]",
+			"invalid scylladb-version format: %q. Expected format: <tag>, <tag>@<digest>, or <digest>",
 			o.ScyllaDBVersion,
 		))
 	}
 
-	if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBUpdateFrom) {
+	if _, err := semver.Parse(o.ScyllaDBVersion); err != nil {
+		semScyllaDBVersion, fullScyllaDBVersion, err := scyllasemver.GetImageVersionAndDigest("scylla", o.ScyllaDBVersion)
+		if err != nil {
+			return fmt.Errorf("failed to resolve ScyllaDB version: %w", err)
+		}
+		fmt.Println("Extracted ScyllaDB version:", fullScyllaDBVersion)
+		o.ScyllaDBVersion = semScyllaDBVersion
+	}
+
+	if !tagOrDigestRegexp.MatchString(o.ScyllaDBUpdateFrom) {
 		errors = append(errors, fmt.Errorf(
-			"invalid scylladb-update-from-version format: %q. Expected format: <tag>[@<digest>]",
+			"invalid scylladb-update-from-version format: %q. Expected format: <tag>, <tag>@<digest>, or <digest>",
 			o.ScyllaDBUpdateFrom,
 		))
 	}
 
-	if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBUpgradeFrom) {
+	if _, err := semver.Parse(o.ScyllaDBUpdateFrom); err != nil {
+		semScyllaDBUpdateFromVersion, fullScyllaDBUpdateFromVersion, err := scyllasemver.GetImageVersionAndDigest("scylla", o.ScyllaDBUpdateFrom)
+		if err != nil {
+			return fmt.Errorf("failed to resolve ScyllaDB update from version: %w", err)
+		}
+		fmt.Println("Extracted ScyllaDB update from version:", fullScyllaDBUpdateFromVersion)
+		o.ScyllaDBUpdateFrom = semScyllaDBUpdateFromVersion
+	}
+
+	if !tagOrDigestRegexp.MatchString(o.ScyllaDBUpgradeFrom) {
 		errors = append(errors, fmt.Errorf(
-			"invalid scylladb-upgrade-from-version format: %q. Expected format: <tag>[@<digest>]",
+			"invalid scylladb-upgrade-from-version format: %q. Expected format: <tag>, <tag>@<digest>, or <digest>",
 			o.ScyllaDBUpgradeFrom,
 		))
 	}
 
-	if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBManagerVersion) {
+	if _, err := semver.Parse(o.ScyllaDBUpgradeFrom); err != nil {
+		semScyllaDBUpgradeFromVersion, fullScyllaDBUpgradeFromVersion, err := scyllasemver.GetImageVersionAndDigest("scylla", o.ScyllaDBUpgradeFrom)
+		if err != nil {
+			return fmt.Errorf("failed to resolve ScyllaDB upgrade from version: %w", err)
+		}
+		fmt.Println("Extracted ScyllaDB upgrade from version:", fullScyllaDBUpgradeFromVersion)
+		o.ScyllaDBUpgradeFrom = semScyllaDBUpgradeFromVersion
+	}
+
+	if !tagOrDigestRegexp.MatchString(o.ScyllaDBManagerVersion) {
 		errors = append(errors, fmt.Errorf(
-			"invalid scylladb-manager-version format: %q. Expected format: <tag>[@<digest>]",
+			"invalid scylladb-manager-version format: %q. Expected format: <tag>, <tag>@<digest>, or <digest>",
 			o.ScyllaDBManagerVersion,
 		))
 	}
 
-	if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBManagerAgentVersion) {
+	if !tagOrDigestRegexp.MatchString(o.ScyllaDBManagerAgentVersion) {
 		errors = append(errors, fmt.Errorf(
-			"invalid scylladb-manager-agent-version format: %q. Expected format: <tag>[@<digest>]",
+			"invalid scylladb-manager-agent-version format: %q. Expected format: <tag>, <tag>@<digest>, or <digest>",
 			o.ScyllaDBManagerAgentVersion,
 		))
 	}

pkg/controller/scylladbdatacenter/sync_statefulsets.go

@@ -17,6 +17,7 @@ import (
 	"github.com/scylladb/scylla-operator/pkg/pointer"
 	"github.com/scylladb/scylla-operator/pkg/resourceapply"
 	"github.com/scylladb/scylla-operator/pkg/scyllaclient"
+	scyllasemver "github.com/scylladb/scylla-operator/pkg/semver"
 	"github.com/scylladb/scylla-operator/pkg/util/hash"
 	"github.com/scylladb/scylla-operator/pkg/util/parallel"
 	appsv1 "k8s.io/api/apps/v1"
@@ -976,15 +977,36 @@ func (sdcc *Controller) syncStatefulSets(
 			existingVersionString, existingVersionLabelPresent := existing.Labels[naming.ScyllaVersionLabel]
 
 			if requiredVersionLabelPresent && existingVersionLabelPresent {
-				requiredVersion, err := semver.Parse(requiredVersionString)
-				if err != nil {
-					return progressingConditions, err
+				parsedSemRequiredVersion, parseErr := semver.Parse(requiredVersionString)
+				if parseErr != nil {
+					semRequiredVersion, _, getVersionErr := scyllasemver.GetImageVersionAndDigest("scylla", requiredVersionString)
+					if getVersionErr != nil {
+						return progressingConditions, getVersionErr
+					}
+
+					parsedSemRequiredVersion, err = semver.Parse(semRequiredVersion)
+					if err != nil {
+						return progressingConditions, err
+					}
 				}
-				existingVersion, err := semver.Parse(existingVersionString)
-				if err != nil {
-					return progressingConditions, err
+
+				requiredVersion := parsedSemRequiredVersion
+
+				parsedSemExistingVersion, parseErr := semver.Parse(existingVersionString)
+				if parseErr != nil {
+					semExistingVersion, _, getVersionErr := scyllasemver.GetImageVersionAndDigest("scylla", existingVersionString)
+					if getVersionErr != nil {
+						return progressingConditions, getVersionErr
+					}
+
+					parsedSemExistingVersion, err = semver.Parse(semExistingVersion)
+					if err != nil {
+						return progressingConditions, err
+					}
 				}
 
+				existingVersion := parsedSemExistingVersion
+
 				if requiredVersion.Major != existingVersion.Major ||
 					requiredVersion.Minor != existingVersion.Minor {
 					// We need to run hooks for version upgrades.

pkg/semver/version.go

@@ -17,14 +17,33 @@ limitations under the License.
 package semver
 
 import (
+	"fmt"
+	"regexp"
+	"strings"
+	"sync"
+
 	"github.com/blang/semver"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"k8s.io/klog/v2"
+)
+
+type ImageVersionInfo struct {
+	SemVersion  string
+	fullVersion string
+}
+
+var (
+	cacheLockScyllaDBImageVersionAndDigest sync.RWMutex
+	cacheScyllaDBImageVersionAndDigest     = make(map[string]ImageVersionInfo)
 )
 
 var (
 	ScyllaVersionThatSupportsDisablingWritebackCache = semver.MustParse("2021.0.0")
 )
 
-// ScyllaVersion contains the version of a cluster with unkown version support
+// ScyllaVersion contains the version of a cluster with unknown version support
 type ScyllaVersion struct {
 	version semver.Version
 	unknown bool
@@ -47,3 +66,91 @@ func (sv ScyllaVersion) SupportFeatureUnsafe(featureVersion semver.Version) bool
 func (sv ScyllaVersion) SupportFeatureSafe(featureVersion semver.Version) bool {
 	return !sv.unknown && sv.version.GTE(featureVersion)
 }
+
+func GetImageVersionAndDigest(imageName, version string) (string, string, error) {
+	// If the version is already in <tag>@<digest> format, extract the semantic version
+	if strings.Contains(version, "@") {
+		semVersion, err := computeSemVersion(version)
+		if err != nil {
+			return "", "", err
+		}
+		return semVersion, version, nil
+	}
+
+	imageReference := fmt.Sprintf("scylladb/%s", imageName)
+	if strings.Contains(version, ":") {
+		imageReference = fmt.Sprintf("%s@%s", imageReference, version)
+	} else {
+		imageReference = fmt.Sprintf("%s:%s", imageReference, version)
+	}
+
+	cacheLockScyllaDBImageVersionAndDigest.RLock()
+	cached, ok := cacheScyllaDBImageVersionAndDigest[imageReference]
+	cacheLockScyllaDBImageVersionAndDigest.RUnlock()
+	if ok {
+		return cached.SemVersion, cached.fullVersion, nil
+	}
+
+	fullVersion, err := computeFullImageVersion(imageReference)
+	if err != nil {
+		return "", "", err
+	}
+
+	semVersion, err := computeSemVersion(fullVersion)
+	if err != nil {
+		return "", "", err
+	}
+
+	cacheLockScyllaDBImageVersionAndDigest.Lock()
+	cacheScyllaDBImageVersionAndDigest[imageReference] = ImageVersionInfo{
+		SemVersion:  semVersion,
+		fullVersion: fullVersion,
+	}
+	cacheLockScyllaDBImageVersionAndDigest.Unlock()
+	return semVersion, fullVersion, nil
+}
+
+func computeFullImageVersion(imageReference string) (string, error) {
+	ref, err := name.ParseReference(imageReference)
+	if err != nil {
+		return "", fmt.Errorf("can't parse image reference: %w", err)
+	}
+
+	img, err := remote.Image(ref, remote.WithAuth(authn.Anonymous))
+	if err != nil {
+		return "", fmt.Errorf("can't fetch image: %w", err)
+	}
+
+	configFile, err := img.ConfigFile()
+	if err != nil {
+		return "", fmt.Errorf("can't get image config: %w", err)
+	}
+
+	versionLabel, ok := configFile.Config.Labels["org.opencontainers.image.version"]
+	if !ok {
+		klog.Warningf("no org.opencontainers.image.version label found for image: %s", imageReference)
+		return "", fmt.Errorf("no version label found")
+	}
+
+	digest, err := img.Digest()
+	if err != nil {
+		return "", fmt.Errorf("can't get image digest: %w", err)
+	}
+	return fmt.Sprintf("%s@%s", versionLabel, digest), nil
+}
+
+func computeSemVersion(fullVersion string) (string, error) {
+	reSemVersion := regexp.MustCompile(`\d+\.\d+\.\d+`)
+	semVersion := reSemVersion.FindString(fullVersion)
+
+	if semVersion == "" {
+		return "", fmt.Errorf("could not extract semantic version from: %s", fullVersion)
+	}
+
+	reSuffix := regexp.MustCompile(`[~-]([a-zA-Z]+\d*)`)
+	suffixMatch := reSuffix.FindStringSubmatch(fullVersion)
+	if len(suffixMatch) > 1 {
+		semVersion = fmt.Sprintf("%s-%s", semVersion, suffixMatch[1])
+	}
+	return semVersion, nil
+}