Support for referencePolicy Source while importing release payload & Update reference-mode=source for creating new payloads

Author: deepsm007

pkg/api/ephemeralcluster/v1/ci.openshift.io_ephemeralclusters.yaml

@@ -96,6 +96,10 @@ spec:
                               description: Namespace is the namespace in which the
                                 integration stream lives.
                               type: string
+                            reference_policy:
+                              description: ReferencePolicy is the policy to use when
+                                resolving references (Local or Source)
+                              type: string
                           required:
                           - name
                           - namespace

pkg/api/types.go

@@ -8,6 +8,8 @@ import (
 	prowv1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1"
 	prowconfig "sigs.k8s.io/prow/pkg/config"
 	"sigs.k8s.io/prow/pkg/repoowners"
+
+	imagev1 "github.com/openshift/api/image/v1"
 )
 
 const (
@@ -335,6 +337,8 @@ type Integration struct {
 	// IncludeBuiltImages determines if the release we assemble will include
 	// images built during the test itself.
 	IncludeBuiltImages bool `json:"include_built_images,omitempty"`
+	// ReferencePolicy is the policy to use when resolving references (Local or Source)
+	ReferencePolicy *imagev1.TagReferencePolicyType `json:"reference_policy,omitempty"`
 }
 
 // ReleaseDescriptor holds common data for different types of release payloads
@@ -511,6 +515,9 @@ type ReleaseTagConfiguration struct {
 	// IncludeBuiltImages determines if the release we assemble will include
 	// images built during the test itself.
 	IncludeBuiltImages bool `json:"include_built_images,omitempty"`
+
+	// ReferencePolicy is the policy to use when resolving references (Local or Source)
+	ReferencePolicy *imagev1.TagReferencePolicyType `json:"reference_policy,omitempty"`
 }
 
 func (config ReleaseTagConfiguration) InputsName() string {

pkg/api/zz_generated.deepcopy.go

@@ -187,6 +187,7 @@ func fromConfig(
 			// factor release steps into something more reusable
 			hasReleaseStep = true
 			var value string
+			var err error
 			var overrideCLIReleaseExtractImage *coreapi.ObjectReference
 			var overrideCLIResolveErr error
 			switch {
@@ -214,12 +215,22 @@ func fromConfig(
 				switch {
 				case resolveConfig.Integration != nil:
 					logrus.Infof("Building release %s from a snapshot of %s/%s", resolveConfig.Name, resolveConfig.Integration.Namespace, resolveConfig.Integration.Name)
-					// this is the one case where we're not importing a payload, we need to get the images and build one
-					snapshot := releasesteps.ReleaseSnapshotStep(resolveConfig.Name, *resolveConfig.Integration, podClient, jobSpec, integratedStreams[fmt.Sprintf("%s/%s", resolveConfig.Integration.Namespace, resolveConfig.Integration.Name)])
+
+					var snapshot api.Step
+					var key string
+
+					if resolveConfig.Integration.ReferencePolicy != nil {
+						key = fmt.Sprintf("%s/%s/%s", resolveConfig.Integration.Namespace, resolveConfig.Integration.Name, *resolveConfig.Integration.ReferencePolicy)
+					} else {
+						key = fmt.Sprintf("%s/%s", resolveConfig.Integration.Namespace, resolveConfig.Integration.Name)
+					}
+
+					snapshot = releasesteps.ReleaseSnapshotStep(resolveConfig.Name, *resolveConfig.Integration, podClient, jobSpec, integratedStreams[key])
 					assemble := releasesteps.AssembleReleaseStep(resolveConfig.Name, nodeName, &api.ReleaseTagConfiguration{
 						Namespace:          resolveConfig.Integration.Namespace,
 						Name:               resolveConfig.Integration.Name,
 						IncludeBuiltImages: resolveConfig.Integration.IncludeBuiltImages,
+						ReferencePolicy:    resolveConfig.Integration.ReferencePolicy,
 					}, config.Resources, podClient, jobSpec)
 					for _, s := range []api.Step{snapshot, assemble} {
 						buildSteps = append(buildSteps, s)
@@ -231,7 +242,13 @@ func fromConfig(
 					source = releasesteps.NewReleaseSourceFromConfig(resolveConfig, httpClient)
 				}
 			}
-			step := releasesteps.ImportReleaseStep(resolveConfig.Name, nodeName, resolveConfig.TargetName(), source, false, config.Resources, podClient, jobSpec, pullSecret, overrideCLIReleaseExtractImage)
+			var referencePolicy imagev1.TagReferencePolicyType
+			if resolveConfig.Integration != nil && resolveConfig.Integration.ReferencePolicy != nil {
+				referencePolicy = *resolveConfig.Integration.ReferencePolicy
+			} else {
+				referencePolicy = imagev1.LocalTagReferencePolicy // Provide a default value or handle appropriately
+			}
+			step := releasesteps.ImportReleaseStep(resolveConfig.Name, nodeName, resolveConfig.TargetName(), referencePolicy, source, false, config.Resources, podClient, jobSpec, pullSecret, overrideCLIReleaseExtractImage)
 			buildSteps = append(buildSteps, step)
 			addProvidesForStep(step, params)
 			continue
@@ -283,6 +300,12 @@ func fromConfig(
 			for _, name := range []string{api.InitialReleaseName, api.LatestReleaseName} {
 				var releaseStep api.Step
 				envVar := utils.ReleaseImageEnv(name)
+				var referencePolicy imagev1.TagReferencePolicyType
+				if rawStep.ReleaseImagesTagStepConfiguration.ReferencePolicy != nil {
+					referencePolicy = *rawStep.ReleaseImagesTagStepConfiguration.ReferencePolicy
+				} else {
+					referencePolicy = imagev1.LocalTagReferencePolicy // Provide a default value or handle appropriately
+				}
 				if params.HasInput(envVar) {
 					pullSpec, err := params.Get(envVar)
 					if err != nil {
@@ -291,10 +314,11 @@ func fromConfig(
 					logrus.Infof("Using explicitly provided pull-spec for release %s (%s)", name, pullSpec)
 					target := rawStep.ReleaseImagesTagStepConfiguration.TargetName(name)
 					source := releasesteps.NewReleaseSourceFromPullSpec(pullSpec)
-					releaseStep = releasesteps.ImportReleaseStep(name, nodeName, target, source, true, config.Resources, podClient, jobSpec, pullSecret, nil)
+					releaseStep = releasesteps.ImportReleaseStep(name, nodeName, target, referencePolicy, source, true, config.Resources, podClient, jobSpec, pullSecret, nil)
 				} else {
 					// for backwards compatibility, users get inclusion for free with tag_spec
 					cfg := *rawStep.ReleaseImagesTagStepConfiguration
+					cfg.ReferencePolicy = &referencePolicy
 					cfg.IncludeBuiltImages = name == api.LatestReleaseName
 					releaseStep = releasesteps.AssembleReleaseStep(name, nodeName, &cfg, config.Resources, podClient, jobSpec)
 				}
@@ -449,8 +473,9 @@ func stepForTest(
 			step = steps.ClusterClaimStep(c.As, c.ClusterClaim, hiveClient, client, jobSpec, step, censor)
 			name := c.ClusterClaim.ClaimRelease(c.As).ReleaseName
 			target := api.ReleaseConfiguration{Name: name}.TargetName()
+			referencePolicy := imagev1.LocalTagReferencePolicy
 			source := releasesteps.NewReleaseSourceFromClusterClaim(c.As, c.ClusterClaim, hiveClient)
-			ret = append(ret, releasesteps.ImportReleaseStep(name, nodeName, target, source, false, config.Resources, podClient, jobSpec, pullSecret, nil))
+			ret = append(ret, releasesteps.ImportReleaseStep(name, nodeName, target, referencePolicy, source, false, config.Resources, podClient, jobSpec, pullSecret, nil))
 		}
 		addProvidesForStep(step, params)
 		ret = append(ret, step)

pkg/defaults/defaults.go

@@ -172,8 +172,14 @@ func (s *assembleReleaseStep) run(ctx context.Context) error {
 	now := time.Now().UTC().Truncate(time.Second)
 	version := fmt.Sprintf("%s-%s-test-%s-%s", prefix, now.Format("2006-01-02-150405"), s.jobSpec.Namespace(), s.name)
 
+	referenceModeArg := ""
+	if s.config.ReferencePolicy != nil && *s.config.ReferencePolicy == imagev1.SourceTagReferencePolicy {
+		referenceModeArg = "--reference-mode='source'"
+	}
+
 	destination := fmt.Sprintf("%s:%s", releaseImageStreamRepo, s.name)
 	logrus.Infof("Creating release image %s.", destination)
+
 	podConfig := steps.PodStepConfiguration{
 		WaitFlags: util.SkipLogs,
 		As:        fmt.Sprintf("release-%s", s.name),
@@ -192,7 +198,7 @@ mkdir -p "${XDG_RUNTIME_DIR}"
 oc registry login
 exit_code="0"
 for ((i=1; i<=5; i++)); do
-	if oc adm release new --max-per-registry=32 -n %q --from-image-stream %q --to-image-base %q --to-image %q --name %q; then
+	if oc adm release new %s --max-per-registry=32 -n %q --from-image-stream %q --to-image-base %q --to-image %q --name %q; then
 		echo "Payload creation success."
 		exit_code="0"
 		break
@@ -226,7 +232,7 @@ done
 if [[ "$exit_code" != "0" ]]; then
 	exit $exit_code
 fi
-`, s.jobSpec.Namespace(), streamName, cvo, destination, version, s.name, destination, s.name),
+`, referenceModeArg, s.jobSpec.Namespace(), streamName, cvo, destination, version, s.name, destination, s.name),
 	}
 
 	// set an explicit default for release-latest resources, but allow customization if necessary

pkg/steps/release/create_release.go

@@ -49,10 +49,11 @@ import (
 // branches of those releases.
 type importReleaseStep struct {
 	// name is the name of the release we're importing, like 'latest'
-	name     string
-	nodeName string
-	target   string
-	source   ReleaseSource
+	name            string
+	nodeName        string
+	target          string
+	referencePolicy imagev1.TagReferencePolicyType
+	source          ReleaseSource
 	// append determines if we wait for other processes to create images first
 	append     bool
 	resources  api.ResourceConfiguration
@@ -229,11 +230,15 @@ oc create configmap release-%s --from-file=%s.yaml=${ARTIFACT_DIR}/%s
 			}
 		}
 
+		referencePolicy := imagev1.LocalTagReferencePolicy
+		if s.referencePolicy == imagev1.SourceTagReferencePolicy {
+			referencePolicy = s.referencePolicy
+		}
 		existing := sets.New[string]()
 		tags := make([]imagev1.TagReference, 0, len(releaseIS.Spec.Tags)+len(stable.Spec.Tags))
 		for _, tag := range releaseIS.Spec.Tags {
 			existing.Insert(tag.Name)
-			tag.ReferencePolicy.Type = imagev1.LocalTagReferencePolicy
+			tag.ReferencePolicy.Type = referencePolicy
 			tag.ImportPolicy.ImportMode = imagev1.ImportModePreserveOriginal
 			tags = append(tags, tag)
 		}
@@ -242,7 +247,7 @@ oc create configmap release-%s --from-file=%s.yaml=${ARTIFACT_DIR}/%s
 				continue
 			}
 			existing.Insert(tag.Name)
-			tag.ReferencePolicy.Type = imagev1.LocalTagReferencePolicy
+			tag.ReferencePolicy.Type = referencePolicy
 			tag.ImportPolicy.ImportMode = imagev1.ImportModePreserveOriginal
 			tags = append(tags, tag)
 		}
@@ -315,6 +320,7 @@ func (s *importReleaseStep) Objects() []ctrlruntimeclient.Object {
 // ImportReleaseStep imports an existing update payload image
 func ImportReleaseStep(
 	name, nodeName, target string,
+	referencePolicy imagev1.TagReferencePolicyType,
 	source ReleaseSource,
 	append bool,
 	resources api.ResourceConfiguration,
@@ -326,6 +332,7 @@ func ImportReleaseStep(
 		name:                           name,
 		nodeName:                       nodeName,
 		target:                         target,
+		referencePolicy:                referencePolicy,
 		source:                         source,
 		append:                         append,
 		resources:                      resources,
@@ -345,18 +352,23 @@ func (s *importReleaseStep) getCLIImage(ctx context.Context, target, streamName
 		// create the imagestream to be able to set it there.
 		if err := s.client.Create(ctx, &imagev1.ImageStream{
 			ObjectMeta: metav1.ObjectMeta{Name: overrideCLIStreamName, Namespace: s.jobSpec.Namespace()},
-			Spec:       imagev1.ImageStreamSpec{LookupPolicy: imagev1.ImageLookupPolicy{Local: true}},
+			Spec:       imagev1.ImageStreamSpec{LookupPolicy: imagev1.ImageLookupPolicy{Local: true}, Tags: []imagev1.TagReference{{ReferencePolicy: imagev1.TagReferencePolicy{Type: s.referencePolicy}}}},
 		}); err != nil && !kerrors.IsAlreadyExists(err) {
 			return nil, fmt.Errorf("failed to create %s imagestream: %w", overrideCLIStreamName, err)
 		}
+
+		referencePolicy := imagev1.LocalTagReferencePolicy
+		if s.referencePolicy == imagev1.SourceTagReferencePolicy {
+			referencePolicy = s.referencePolicy
+		}
 		streamTag := &imagev1.ImageStreamTag{
 			ObjectMeta: meta.ObjectMeta{
 				Namespace: s.jobSpec.Namespace(),
 				Name:      overrideCLIStreamName + ":latest",
 			},
 			Tag: &imagev1.TagReference{
 				ReferencePolicy: imagev1.TagReferencePolicy{
-					Type: imagev1.LocalTagReferencePolicy,
+					Type: referencePolicy,
 				},
 				From: s.overrideCLIReleaseExtractImage,
 			},
@@ -416,14 +428,18 @@ func (s *importReleaseStep) getCLIImage(ctx context.Context, target, streamName
 		return nil, err
 	}
 	// tag the cli image into stable so we use the correct pull secrets from the namespace
+	referencePolicy := imagev1.LocalTagReferencePolicy
+	if s.referencePolicy == imagev1.SourceTagReferencePolicy {
+		referencePolicy = s.referencePolicy
+	}
 	streamTag := &imagev1.ImageStreamTag{
 		ObjectMeta: meta.ObjectMeta{
 			Namespace: s.jobSpec.Namespace(),
 			Name:      fmt.Sprintf("%s:cli", streamName),
 		},
 		Tag: &imagev1.TagReference{
 			ReferencePolicy: imagev1.TagReferencePolicy{
-				Type: imagev1.LocalTagReferencePolicy,
+				Type: referencePolicy,
 			},
 			From: &coreapi.ObjectReference{
 				Kind: "DockerImage",