Support for referencePolicy Source while importing release payload & Update reference-mode=source for creating new payloads

Author: deepsm007

pkg/api/ephemeralcluster/v1/ci.openshift.io_ephemeralclusters.yaml

@@ -96,6 +96,10 @@ spec:
                               description: Namespace is the namespace in which the
                                 integration stream lives.
                               type: string
+                            reference_policy:
+                              description: ReferencePolicy is the policy to use when
+                                resolving references (Local or Source)
+                              type: string
                           required:
                           - name
                           - namespace

pkg/api/types.go

@@ -8,6 +8,8 @@ import (
 	prowv1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1"
 	prowconfig "sigs.k8s.io/prow/pkg/config"
 	"sigs.k8s.io/prow/pkg/repoowners"
+
+	imagev1 "github.com/openshift/api/image/v1"
 )
 
 const (
@@ -335,6 +337,8 @@ type Integration struct {
 	// IncludeBuiltImages determines if the release we assemble will include
 	// images built during the test itself.
 	IncludeBuiltImages bool `json:"include_built_images,omitempty"`
+	// ReferencePolicy is the policy to use when resolving references (Local or Source)
+	ReferencePolicy *imagev1.TagReferencePolicyType `json:"reference_policy,omitempty"`
 }
 
 // ReleaseDescriptor holds common data for different types of release payloads
@@ -511,6 +515,9 @@ type ReleaseTagConfiguration struct {
 	// IncludeBuiltImages determines if the release we assemble will include
 	// images built during the test itself.
 	IncludeBuiltImages bool `json:"include_built_images,omitempty"`
+
+	// ReferencePolicy is the policy to use when resolving references (Local or Source)
+	ReferencePolicy *imagev1.TagReferencePolicyType `json:"reference_policy,omitempty"`
 }
 
 func (config ReleaseTagConfiguration) InputsName() string {

pkg/api/zz_generated.deepcopy.go

@@ -187,6 +187,7 @@ func fromConfig(
 			// factor release steps into something more reusable
 			hasReleaseStep = true
 			var value string
+			var err error
 			var overrideCLIReleaseExtractImage *coreapi.ObjectReference
 			var overrideCLIResolveErr error
 			switch {
@@ -214,12 +215,22 @@ func fromConfig(
 				switch {
 				case resolveConfig.Integration != nil:
 					logrus.Infof("Building release %s from a snapshot of %s/%s", resolveConfig.Name, resolveConfig.Integration.Namespace, resolveConfig.Integration.Name)
-					// this is the one case where we're not importing a payload, we need to get the images and build one
-					snapshot := releasesteps.ReleaseSnapshotStep(resolveConfig.Name, *resolveConfig.Integration, podClient, jobSpec, integratedStreams[fmt.Sprintf("%s/%s", resolveConfig.Integration.Namespace, resolveConfig.Integration.Name)])
+
+					var snapshot api.Step
+					var key string
+
+					if resolveConfig.Integration.ReferencePolicy != nil {
+						key = fmt.Sprintf("%s/%s/%s", resolveConfig.Integration.Namespace, resolveConfig.Integration.Name, *resolveConfig.Integration.ReferencePolicy)
+					} else {
+						key = fmt.Sprintf("%s/%s", resolveConfig.Integration.Namespace, resolveConfig.Integration.Name)
+					}
+
+					snapshot = releasesteps.ReleaseSnapshotStep(resolveConfig.Name, *resolveConfig.Integration, podClient, jobSpec, integratedStreams[key])
 					assemble := releasesteps.AssembleReleaseStep(resolveConfig.Name, nodeName, &api.ReleaseTagConfiguration{
 						Namespace:          resolveConfig.Integration.Namespace,
 						Name:               resolveConfig.Integration.Name,
 						IncludeBuiltImages: resolveConfig.Integration.IncludeBuiltImages,
+						ReferencePolicy:    resolveConfig.Integration.ReferencePolicy,
 					}, config.Resources, podClient, jobSpec)
 					for _, s := range []api.Step{snapshot, assemble} {
 						buildSteps = append(buildSteps, s)
@@ -231,7 +242,13 @@ func fromConfig(
 					source = releasesteps.NewReleaseSourceFromConfig(resolveConfig, httpClient)
 				}
 			}
-			step := releasesteps.ImportReleaseStep(resolveConfig.Name, nodeName, resolveConfig.TargetName(), source, false, config.Resources, podClient, jobSpec, pullSecret, overrideCLIReleaseExtractImage)
+			var referencePolicy imagev1.TagReferencePolicyType
+			if resolveConfig.Integration != nil && resolveConfig.Integration.ReferencePolicy != nil {
+				referencePolicy = *resolveConfig.Integration.ReferencePolicy
+			} else {
+				referencePolicy = imagev1.LocalTagReferencePolicy // Provide a default value or handle appropriately
+			}
+			step := releasesteps.ImportReleaseStep(resolveConfig.Name, nodeName, resolveConfig.TargetName(), referencePolicy, source, false, config.Resources, podClient, jobSpec, pullSecret, overrideCLIReleaseExtractImage)
 			buildSteps = append(buildSteps, step)
 			addProvidesForStep(step, params)
 			continue
@@ -283,6 +300,12 @@ func fromConfig(
 			for _, name := range []string{api.InitialReleaseName, api.LatestReleaseName} {
 				var releaseStep api.Step
 				envVar := utils.ReleaseImageEnv(name)
+				var referencePolicy imagev1.TagReferencePolicyType
+				if rawStep.ReleaseImagesTagStepConfiguration.ReferencePolicy != nil {
+					referencePolicy = *rawStep.ReleaseImagesTagStepConfiguration.ReferencePolicy
+				} else {
+					referencePolicy = imagev1.LocalTagReferencePolicy // Provide a default value or handle appropriately
+				}
 				if params.HasInput(envVar) {
 					pullSpec, err := params.Get(envVar)
 					if err != nil {
@@ -291,10 +314,11 @@ func fromConfig(
 					logrus.Infof("Using explicitly provided pull-spec for release %s (%s)", name, pullSpec)
 					target := rawStep.ReleaseImagesTagStepConfiguration.TargetName(name)
 					source := releasesteps.NewReleaseSourceFromPullSpec(pullSpec)
-					releaseStep = releasesteps.ImportReleaseStep(name, nodeName, target, source, true, config.Resources, podClient, jobSpec, pullSecret, nil)
+					releaseStep = releasesteps.ImportReleaseStep(name, nodeName, target, referencePolicy, source, true, config.Resources, podClient, jobSpec, pullSecret, nil)
 				} else {
 					// for backwards compatibility, users get inclusion for free with tag_spec
 					cfg := *rawStep.ReleaseImagesTagStepConfiguration
+					cfg.ReferencePolicy = &referencePolicy
 					cfg.IncludeBuiltImages = name == api.LatestReleaseName
 					releaseStep = releasesteps.AssembleReleaseStep(name, nodeName, &cfg, config.Resources, podClient, jobSpec)
 				}
@@ -449,8 +473,9 @@ func stepForTest(
 			step = steps.ClusterClaimStep(c.As, c.ClusterClaim, hiveClient, client, jobSpec, step, censor)
 			name := c.ClusterClaim.ClaimRelease(c.As).ReleaseName
 			target := api.ReleaseConfiguration{Name: name}.TargetName()
+			referencePolicy := imagev1.LocalTagReferencePolicy
 			source := releasesteps.NewReleaseSourceFromClusterClaim(c.As, c.ClusterClaim, hiveClient)
-			ret = append(ret, releasesteps.ImportReleaseStep(name, nodeName, target, source, false, config.Resources, podClient, jobSpec, pullSecret, nil))
+			ret = append(ret, releasesteps.ImportReleaseStep(name, nodeName, target, referencePolicy, source, false, config.Resources, podClient, jobSpec, pullSecret, nil))
 		}
 		addProvidesForStep(step, params)
 		ret = append(ret, step)

pkg/defaults/defaults.go

@@ -440,7 +440,7 @@ objects:
           mkdir /tmp/.docker && cp /etc/openshift-installer/pull-secret /tmp/.docker/config.json
           oc registry login --to /tmp/.docker/config.json
           MIRROR_BASE=$( oc get is release -o 'jsonpath={.status.publicDockerImageRepository}' )
-          oc adm release new --from-release ${RELEASE_IMAGE_LATEST} --to-image ${MIRROR_BASE}-scratch:release --mirror ${MIRROR_BASE}-scratch || echo 'ignore: the release could not be reproduced from its inputs'
+          oc adm release new --reference-mode="" --from-release ${RELEASE_IMAGE_LATEST} --to-image ${MIRROR_BASE}-scratch:release --mirror ${MIRROR_BASE}-scratch || echo 'ignore: the release could not be reproduced from its inputs'
           oc adm release mirror --from ${MIRROR_BASE}-scratch:release --to ${MIRROR_BASE} --to-release-image ${MIRROR_BASE}:mirrored
           RELEASE_PAYLOAD_IMAGE_SHA=$(oc get istag ${MIRROR_BASE##*/}:mirrored -o=jsonpath="{.image.metadata.name}")
           oc delete imagestream "$(basename "${MIRROR_BASE}-scratch")"

pkg/steps/clusterinstall/template.go

@@ -3,6 +3,7 @@ package release
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/sirupsen/logrus"
@@ -172,6 +173,12 @@ func (s *assembleReleaseStep) run(ctx context.Context) error {
 	now := time.Now().UTC().Truncate(time.Second)
 	version := fmt.Sprintf("%s-%s-test-%s-%s", prefix, now.Format("2006-01-02-150405"), s.jobSpec.Namespace(), s.name)
 
+	referencePolicy := ""
+	if s.config.ReferencePolicy != nil && *s.config.ReferencePolicy == imagev1.SourceTagReferencePolicy {
+		// we need to set the reference policy to Source
+		referencePolicy = strings.ToLower(string(*s.config.ReferencePolicy))
+	}
+
 	destination := fmt.Sprintf("%s:%s", releaseImageStreamRepo, s.name)
 	logrus.Infof("Creating release image %s.", destination)
 	podConfig := steps.PodStepConfiguration{
@@ -192,7 +199,7 @@ mkdir -p "${XDG_RUNTIME_DIR}"
 oc registry login
 exit_code="0"
 for ((i=1; i<=5; i++)); do
-	if oc adm release new --max-per-registry=32 -n %q --from-image-stream %q --to-image-base %q --to-image %q --name %q; then
+	if oc adm release new --reference-mode=%q --max-per-registry=32 -n %q --from-image-stream %q --to-image-base %q --to-image %q --name %q; then
 		echo "Payload creation success."
 		exit_code="0"
 		break
@@ -226,7 +233,7 @@ done
 if [[ "$exit_code" != "0" ]]; then
 	exit $exit_code
 fi
-`, s.jobSpec.Namespace(), streamName, cvo, destination, version, s.name, destination, s.name),
+`, referencePolicy, s.jobSpec.Namespace(), streamName, cvo, destination, version, s.name, destination, s.name),
 	}
 
 	// set an explicit default for release-latest resources, but allow customization if necessary