Support for referencePolicy Source while importing release payload

Author: deepsm007

pkg/api/types.go

@@ -335,6 +335,8 @@ type Integration struct {
 	// IncludeBuiltImages determines if the release we assemble will include
 	// images built during the test itself.
 	IncludeBuiltImages bool `json:"include_built_images,omitempty"`
+	// ReferencePolicy is the policy to use when resolving references (Local or Source)
+	ReferencePolicy string `json:"reference_policy,omitempty"`
 }
 
 // ReleaseDescriptor holds common data for different types of release payloads
@@ -488,12 +490,23 @@ type ImageStreamTagReference struct {
 
 	// As is an optional string to use as the intermediate name for this reference.
 	As string `json:"as,omitempty"`
+
+	// ReferencePolicy is an optional string to use as the reference policy for image pull.
+	ReferencePolicy string `json:"reference_policy,omitempty"`
 }
 
 func (i *ImageStreamTagReference) ISTagName() string {
 	return fmt.Sprintf("%s/%s:%s", i.Namespace, i.Name, i.Tag)
 }
 
+// // ReferencePolicyTag defines the possible values for the reference policy.
+// type ReferencePolicyTag string
+
+// const (
+// 	ReferencePolicyLocal  ReferencePolicyTag = "Local"
+// 	ReferencePolicySource ReferencePolicyTag = "Source"
+// )
+
 // ReleaseTagConfiguration describes how a release is
 // assembled from release artifacts. A release image stream is a
 // single stream with multiple tags (openshift/origin-v3.9:control-plane),
@@ -511,6 +524,9 @@ type ReleaseTagConfiguration struct {
 	// IncludeBuiltImages determines if the release we assemble will include
 	// images built during the test itself.
 	IncludeBuiltImages bool `json:"include_built_images,omitempty"`
+
+	// ReferencePolicy is the policy to use when resolving references (Local or Source)
+	ReferencePolicy string `json:"reference_policy,omitempty"`
 }
 
 func (config ReleaseTagConfiguration) InputsName() string {

pkg/defaults/defaults.go

@@ -215,11 +215,12 @@ func fromConfig(
 				case resolveConfig.Integration != nil:
 					logrus.Infof("Building release %s from a snapshot of %s/%s", resolveConfig.Name, resolveConfig.Integration.Namespace, resolveConfig.Integration.Name)
 					// this is the one case where we're not importing a payload, we need to get the images and build one
-					snapshot := releasesteps.ReleaseSnapshotStep(resolveConfig.Name, *resolveConfig.Integration, podClient, jobSpec, integratedStreams[fmt.Sprintf("%s/%s", resolveConfig.Integration.Namespace, resolveConfig.Integration.Name)])
+					snapshot := releasesteps.ReleaseSnapshotStep(resolveConfig.Name, *resolveConfig.Integration, podClient, jobSpec, integratedStreams[fmt.Sprintf("%s/%s/%s", resolveConfig.Integration.Namespace, resolveConfig.Integration.Name, resolveConfig.Integration.ReferencePolicy)])
 					assemble := releasesteps.AssembleReleaseStep(resolveConfig.Name, nodeName, &api.ReleaseTagConfiguration{
 						Namespace:          resolveConfig.Integration.Namespace,
 						Name:               resolveConfig.Integration.Name,
 						IncludeBuiltImages: resolveConfig.Integration.IncludeBuiltImages,
+						ReferencePolicy:    resolveConfig.Integration.ReferencePolicy,
 					}, config.Resources, podClient, jobSpec)
 					for _, s := range []api.Step{snapshot, assemble} {
 						buildSteps = append(buildSteps, s)
@@ -231,7 +232,7 @@ func fromConfig(
 					source = releasesteps.NewReleaseSourceFromConfig(resolveConfig, httpClient)
 				}
 			}
-			step := releasesteps.ImportReleaseStep(resolveConfig.Name, nodeName, resolveConfig.TargetName(), source, false, config.Resources, podClient, jobSpec, pullSecret, overrideCLIReleaseExtractImage)
+			step := releasesteps.ImportReleaseStep(resolveConfig.Name, nodeName, resolveConfig.TargetName(), imagev1.TagReferencePolicyType(resolveConfig.Integration.ReferencePolicy), source, false, config.Resources, podClient, jobSpec, pullSecret, overrideCLIReleaseExtractImage)
 			buildSteps = append(buildSteps, step)
 			addProvidesForStep(step, params)
 			continue
@@ -283,6 +284,10 @@ func fromConfig(
 			for _, name := range []string{api.InitialReleaseName, api.LatestReleaseName} {
 				var releaseStep api.Step
 				envVar := utils.ReleaseImageEnv(name)
+				referencePolicy := imagev1.LocalTagReferencePolicy
+				if rawStep.ReleaseImagesTagStepConfiguration.ReferencePolicy != "" && rawStep.ReleaseImagesTagStepConfiguration.ReferencePolicy == "Source" {
+					referencePolicy = imagev1.SourceTagReferencePolicy
+				}
 				if params.HasInput(envVar) {
 					pullSpec, err := params.Get(envVar)
 					if err != nil {
@@ -291,10 +296,11 @@ func fromConfig(
 					logrus.Infof("Using explicitly provided pull-spec for release %s (%s)", name, pullSpec)
 					target := rawStep.ReleaseImagesTagStepConfiguration.TargetName(name)
 					source := releasesteps.NewReleaseSourceFromPullSpec(pullSpec)
-					releaseStep = releasesteps.ImportReleaseStep(name, nodeName, target, source, true, config.Resources, podClient, jobSpec, pullSecret, nil)
+					releaseStep = releasesteps.ImportReleaseStep(name, nodeName, target, referencePolicy, source, true, config.Resources, podClient, jobSpec, pullSecret, nil)
 				} else {
 					// for backwards compatibility, users get inclusion for free with tag_spec
 					cfg := *rawStep.ReleaseImagesTagStepConfiguration
+					cfg.ReferencePolicy = string(referencePolicy)
 					cfg.IncludeBuiltImages = name == api.LatestReleaseName
 					releaseStep = releasesteps.AssembleReleaseStep(name, nodeName, &cfg, config.Resources, podClient, jobSpec)
 				}

pkg/steps/release/import_release.go

@@ -49,10 +49,11 @@ import (
 // branches of those releases.
 type importReleaseStep struct {
 	// name is the name of the release we're importing, like 'latest'
-	name     string
-	nodeName string
-	target   string
-	source   ReleaseSource
+	name            string
+	nodeName        string
+	target          string
+	referencePolicy imagev1.TagReferencePolicyType
+	source          ReleaseSource
 	// append determines if we wait for other processes to create images first
 	append     bool
 	resources  api.ResourceConfiguration
@@ -97,6 +98,20 @@ func (s *importReleaseStep) run(ctx context.Context) error {
 			LookupPolicy: imagev1.ImageLookupPolicy{
 				Local: true,
 			},
+			Tags: []imagev1.TagReference{
+				{
+					ReferencePolicy: imagev1.TagReferencePolicy{
+						Type: func() imagev1.TagReferencePolicyType {
+							if s != nil {
+								if s.referencePolicy == imagev1.SourceTagReferencePolicy {
+									return imagev1.SourceTagReferencePolicy
+								}
+							}
+							return imagev1.LocalTagReferencePolicy
+						}(),
+					},
+				},
+			},
 		},
 	})
 	if err != nil && !kerrors.IsAlreadyExists(err) {
@@ -229,11 +244,12 @@ oc create configmap release-%s --from-file=%s.yaml=${ARTIFACT_DIR}/%s
 			}
 		}
 
+		referencePolicy := imagev1.LocalTagReferencePolicy
 		existing := sets.New[string]()
 		tags := make([]imagev1.TagReference, 0, len(releaseIS.Spec.Tags)+len(stable.Spec.Tags))
 		for _, tag := range releaseIS.Spec.Tags {
 			existing.Insert(tag.Name)
-			tag.ReferencePolicy.Type = imagev1.LocalTagReferencePolicy
+			tag.ReferencePolicy.Type = referencePolicy
 			tag.ImportPolicy.ImportMode = imagev1.ImportModePreserveOriginal
 			tags = append(tags, tag)
 		}
@@ -242,7 +258,7 @@ oc create configmap release-%s --from-file=%s.yaml=${ARTIFACT_DIR}/%s
 				continue
 			}
 			existing.Insert(tag.Name)
-			tag.ReferencePolicy.Type = imagev1.LocalTagReferencePolicy
+			tag.ReferencePolicy.Type = referencePolicy
 			tag.ImportPolicy.ImportMode = imagev1.ImportModePreserveOriginal
 			tags = append(tags, tag)
 		}
@@ -315,6 +331,7 @@ func (s *importReleaseStep) Objects() []ctrlruntimeclient.Object {
 // ImportReleaseStep imports an existing update payload image
 func ImportReleaseStep(
 	name, nodeName, target string,
+	referencePolicy imagev1.TagReferencePolicyType,
 	source ReleaseSource,
 	append bool,
 	resources api.ResourceConfiguration,
@@ -326,6 +343,7 @@ func ImportReleaseStep(
 		name:                           name,
 		nodeName:                       nodeName,
 		target:                         target,
+		referencePolicy:                referencePolicy,
 		source:                         source,
 		append:                         append,
 		resources:                      resources,
@@ -345,18 +363,23 @@ func (s *importReleaseStep) getCLIImage(ctx context.Context, target, streamName
 		// create the imagestream to be able to set it there.
 		if err := s.client.Create(ctx, &imagev1.ImageStream{
 			ObjectMeta: metav1.ObjectMeta{Name: overrideCLIStreamName, Namespace: s.jobSpec.Namespace()},
-			Spec:       imagev1.ImageStreamSpec{LookupPolicy: imagev1.ImageLookupPolicy{Local: true}},
+			Spec:       imagev1.ImageStreamSpec{LookupPolicy: imagev1.ImageLookupPolicy{Local: true}, Tags: []imagev1.TagReference{{ReferencePolicy: imagev1.TagReferencePolicy{Type: s.referencePolicy}}}},
 		}); err != nil && !kerrors.IsAlreadyExists(err) {
 			return nil, fmt.Errorf("failed to create %s imagestream: %w", overrideCLIStreamName, err)
 		}
+
+		referencePolicy := imagev1.LocalTagReferencePolicy
+		if s.referencePolicy == imagev1.SourceTagReferencePolicy {
+			referencePolicy = imagev1.SourceTagReferencePolicy
+		}
 		streamTag := &imagev1.ImageStreamTag{
 			ObjectMeta: meta.ObjectMeta{
 				Namespace: s.jobSpec.Namespace(),
 				Name:      overrideCLIStreamName + ":latest",
 			},
 			Tag: &imagev1.TagReference{
 				ReferencePolicy: imagev1.TagReferencePolicy{
-					Type: imagev1.LocalTagReferencePolicy,
+					Type: referencePolicy,
 				},
 				From: s.overrideCLIReleaseExtractImage,
 			},
@@ -416,14 +439,18 @@ func (s *importReleaseStep) getCLIImage(ctx context.Context, target, streamName
 		return nil, err
 	}
 	// tag the cli image into stable so we use the correct pull secrets from the namespace
+	referencePolicy := imagev1.LocalTagReferencePolicy
+	if s.referencePolicy == "source" {
+		referencePolicy = imagev1.SourceTagReferencePolicy
+	}
 	streamTag := &imagev1.ImageStreamTag{
 		ObjectMeta: meta.ObjectMeta{
 			Namespace: s.jobSpec.Namespace(),
 			Name:      fmt.Sprintf("%s:cli", streamName),
 		},
 		Tag: &imagev1.TagReference{
 			ReferencePolicy: imagev1.TagReferencePolicy{
-				Type: imagev1.LocalTagReferencePolicy,
+				Type: referencePolicy,
 			},
 			From: &coreapi.ObjectReference{
 				Kind: "DockerImage",

pkg/util/imagestreamtagwrapper/testdata/imagestream.yaml

@@ -31,7 +31,7 @@ spec:
     importPolicy: {}
     name: root
     referencePolicy:
-      type: Local
+      type: Source
 status:
   dockerImageRepository: docker-registry.default.svc:5000/ci-op-jpdy23wx/pipeline
   publicDockerImageRepository: registry.svc.ci.openshift.org/ci-op-jpdy23wx/pipeline

pkg/util/imagestreamtagwrapper/testdata/imagestreamtags.yaml

@@ -224,7 +224,7 @@ items:
       apiVersion: "1.0"
       kind: DockerImage
     dockerImageMetadataVersion: "1.0"
-    dockerImageReference: docker-registry.default.svc:5000/ci-op-jpdy23wx/pipeline@sha256:fd41b8b3efb8db1187ac231ca84863a8106880d049123d6eddfe21d7f751c627
+    dockerImageReference: docker.io/openshift/origin-release@sha256:fd41b8b3efb8db1187ac231ca84863a8106880d049123d6eddfe21d7f751c627
     metadata:
       annotations:
         image.openshift.io/dockerLayersOrder: ascending
@@ -252,7 +252,7 @@ items:
     importPolicy: {}
     name: root
     referencePolicy:
-      type: Local
+      type: Source
 - apiVersion: image.openshift.io/v1
   generation: 3
   image:

pkg/validation/release.go

@@ -71,6 +71,12 @@ func validateIntegration(fieldRoot, name string, integration api.Integration) []
 	if integration.IncludeBuiltImages && name != api.LatestReleaseName {
 		validationErrors = append(validationErrors, fmt.Errorf("%s: only the `latest` release can set `include_built_images`", fieldRoot))
 	}
+	if integration.ReferencePolicy == "" {
+		integration.ReferencePolicy = "Local" // Set default value
+	}
+	if integration.ReferencePolicy != "Local" && integration.ReferencePolicy != "Source" {
+		validationErrors = append(validationErrors, fmt.Errorf("%s.reference_policy: must be one of Local or Source or empty defaults to Local", fieldRoot))
+	}
 	return validationErrors
 }
 

pkg/validation/release_test.go

@@ -543,6 +543,49 @@ func TestValidateIntegration(t *testing.T) {
 				IncludeBuiltImages: true,
 			},
 		},
+		{
+			name:      "valid integration with local reference policy",
+			inputName: "latest",
+			input: api.Integration{
+				Name:               "4.8",
+				Namespace:          "ocp",
+				IncludeBuiltImages: true,
+				ReferencePolicy:    "Local",
+			},
+		},
+		{
+			name:      "valid integration with source reference policy",
+			inputName: "latest",
+			input: api.Integration{
+				Name:               "4.8",
+				Namespace:          "ocp",
+				IncludeBuiltImages: true,
+				ReferencePolicy:    "Source",
+			},
+		},
+		{
+			name:      "valid integration with empty reference policy setting to Local as default",
+			inputName: "latest",
+			input: api.Integration{
+				Name:               "4.8",
+				Namespace:          "ocp",
+				IncludeBuiltImages: true,
+				ReferencePolicy:    "",
+			},
+		},
+		{
+			name:      "invalid integration with reference policy",
+			inputName: "latest",
+			input: api.Integration{
+				Name:               "4.8",
+				Namespace:          "ocp",
+				IncludeBuiltImages: true,
+				ReferencePolicy:    "something",
+			},
+			output: []error{
+				errors.New("root.reference_policy: must be one of Local or Source or empty defaults to Local"),
+			},
+		},
 		{
 			name:      "invalid integration missing namespace",
 			inputName: "latest",

pkg/webreg/zz_generated.ci_operator_reference.go

@@ -479,6 +479,8 @@ const ciOperatorReferenceYaml = "# The list of base images describe\n" +
 	"        # all release artifacts not built in the current\n" +
 	"        # job are tagged from.\n" +
 	"        namespace: ' '\n" +
+	"        # ReferencePolicy is the policy to use when resolving references (Local or Source)\n" +
+	"        reference_policy: ' '\n" +
 	"      resolved_release_images_step:\n" +
 	"        # Candidate describes a candidate release payload\n" +
 	"        candidate:\n" +
@@ -500,6 +502,8 @@ const ciOperatorReferenceYaml = "# The list of base images describe\n" +
 	"            name: ' '\n" +
 	"            # Namespace is the namespace in which the integration stream lives.\n" +
 	"            namespace: ' '\n" +
+	"            # ReferencePolicy is the policy to use when resolving references (Local or Source)\n" +
+	"            reference_policy: ' '\n" +
 	"        name: ' '\n" +
 	"        # Prerelease describes a yet-to-be released payload\n" +
 	"        prerelease:\n" +
@@ -1328,6 +1332,8 @@ const ciOperatorReferenceYaml = "# The list of base images describe\n" +
 	"            name: ' '\n" +
 	"            # Namespace is the namespace in which the integration stream lives.\n" +
 	"            namespace: ' '\n" +
+	"            # ReferencePolicy is the policy to use when resolving references (Local or Source)\n" +
+	"            reference_policy: ' '\n" +
 	"        # Prerelease describes a yet-to-be released payload\n" +
 	"        prerelease:\n" +
 	"            # Architecture is the architecture for the product.\n" +
@@ -1399,6 +1405,8 @@ const ciOperatorReferenceYaml = "# The list of base images describe\n" +
 	"    # all release artifacts not built in the current\n" +
 	"    # job are tagged from.\n" +
 	"    namespace: ' '\n" +
+	"    # ReferencePolicy is the policy to use when resolving references (Local or Source)\n" +
+	"    reference_policy: ' '\n" +
 	"# TestBinaryBuildCommands will create a \"test-bin\" image based on \"src\" that\n" +
 	"# contains the output of this command. This allows reuse of binary artifacts\n" +
 	"# across other steps. If empty, no \"test-bin\" image will be created.\n" +