Merge pull request #1121 from ravanelli/pr/manifest_tag

jobs/build-node-image: Add a tag field for the staging repo

Author: ravanelli

jobs/build-node-image.Jenkinsfile

@@ -66,15 +66,19 @@ lock(resource: "build-node-image") {
         def archinfo = arches.collectEntries{[it, [:]]}
         def now = java.time.LocalDateTime.now()
         def timestamp = now.format(java.time.format.DateTimeFormatter.ofPattern("yyyyMMddHHmm"))
-        def (container_registry_staging_repo, container_registry_repo, prod_tags) = pipeutils.get_ocp_node_registry_repo(pipecfg, params.RELEASE, timestamp)
-        def container_registry_staging_manifest_tag = "${params.RELEASE}"
-        def container_registry_staging_image_tag = "${params.RELEASE}"
-        def container_registry_staging_manifest = "${container_registry_staging_repo}:${container_registry_staging_manifest_tag}"
+        def (registry_staging_repo, registry_staging_tags, registry_prod_repo, registry_prod_tags) = pipeutils.get_ocp_node_registry_repo(pipecfg, params.RELEASE, timestamp)
+
+        // `staging_tags` is a list to stay consistent with the `prod` objects,
+        // but we only need a single tag here since it's used solely for storing
+        // intermediary images before they are referenced in a multi-arch manifest.
+        def registry_staging_tag = registry_staging_tags[0]
 
         // add any additional root CA cert before we do anything that fetches
         pipeutils.addOptionalRootCA()
 
         def yumrepos_file
+        def node_image_manifest_digest
+        def extensions_image_manifest_digest
         stage('Init') {
             shwrap("git clone ${stream_info.yumrepo.url} yumrepos")
             for (repo in stream_info.yumrepo.files) {
@@ -85,18 +89,15 @@ lock(resource: "build-node-image") {
             archiveArtifacts 'all.repo'
         }
 
-        if (params.PIPECFG_HOTFIX_REPO || params.PIPECFG_HOTFIX_REF) {
-            container_registry_staging_image_tag += "-hotfix-${pipecfg.hotfix.name}"
-        }
         stage('Build Node Image') {
             withCredentials([file(credentialsId: 'oscontainer-push-registry-secret', variable: 'REGISTRY_AUTH_FILE')]) {
                  def build_from = params.FROM ?: stream_info.from
-                 pipeutils.build_and_push_image(arches: arches,
+                 node_image_manifest_digest = pipeutils.build_and_push_image(arches: arches,
                                                 src_commit: commit,
                                                 src_url: src_config_url,
-                                                staging_repository: container_registry_staging_repo,
-                                                image_tag_staging: container_registry_staging_image_tag,
-                                                manifest_tag_staging: container_registry_staging_manifest_tag,
+                                                staging_repository: registry_staging_repo,
+                                                image_tag_staging: registry_staging_tag,
+                                                manifest_tag_staging: "${registry_staging_tag}",
                                                 secret: "id=yumrepos,src=${yumrepos_file}", // notsecret (for secret scanners)
                                                 from: build_from,
                                                 extra_build_args: ["--security-opt label=disable", "--mount-host-ca-certs", "--force"])
@@ -105,13 +106,13 @@ lock(resource: "build-node-image") {
         stage('Build Extensions Image') {
             withCredentials([file(credentialsId: 'oscontainer-push-registry-secret', variable: 'REGISTRY_AUTH_FILE')]) {
                 // Use the node image as from
-                def build_from = container_registry_staging_manifest
-                pipeutils.build_and_push_image(arches: arches,
+                def build_from = "${registry_staging_repo}@${node_image_manifest_digest}"
+                extensions_image_manifest_digest = pipeutils.build_and_push_image(arches: arches,
                                                src_commit: commit,
                                                src_url: src_config_url,
-                                               staging_repository: container_registry_staging_repo,
-                                               image_tag_staging: "${container_registry_staging_image_tag}-extensions",
-                                               manifest_tag_staging: "${container_registry_staging_manifest_tag}-extensions",
+                                               staging_repository: registry_staging_repo,
+                                               image_tag_staging: "${registry_staging_tag}-extensions",
+                                               manifest_tag_staging: "${registry_staging_tag}-extensions",
                                                secret: "id=yumrepos,src=${yumrepos_file}", // notsecret (for secret scanners)
                                                from: build_from,
                                                extra_build_args: ["--security-opt label=disable", "--mount-host-ca-certs",
@@ -123,16 +124,17 @@ lock(resource: "build-node-image") {
                 // copy the extensions first as the node image existing is a signal
                 // that it's ready for release. So we want all the expected artifacts
                 // to be available when the ART tooling kicks in.
-                for ( tag in prod_tags ) {
-                    pipeutils.copy_image("${container_registry_staging_manifest}-extensions",
-                                     "${container_registry_repo}:${tag}-extensions")
-                }
 
                 // Skopeo does not support pushing multiple tags at the same time
                 // So we just recopy the same image multiple times.
                 // https://github.com/containers/skopeo/issues/513
-                for (tag in prod_tags) {
-                    pipeutils.copy_image(container_registry_staging_manifest, "${container_registry_repo}:${tag}")
+                for (tag in registry_prod_tags) {
+                    pipeutils.copy_image("${registry_staging_repo}@${extensions_image_manifest_digest}",
+                                     "${registry_prod_repo}:${tag}-extensions")
+                }
+                for (tag in registry_prod_tags) {
+                    pipeutils.copy_image("${registry_staging_repo}@${node_image_manifest_digest}",
+                                     "${registry_prod_repo}:${tag}")
                 }
             }
         }

utils.groovy

@@ -551,22 +551,25 @@ def get_registry_repos(pipecfg, stream, version) {
 }
 
 def get_ocp_node_registry_repo(pipecfg, release, timestamp) {
-    def staging_repo = pipecfg.ocp_node_builds.registries.staging
-    def repo = pipecfg.ocp_node_builds.registries.prod.image
-    def tags = pipecfg.ocp_node_builds.registries.prod.tags
-
-    processed_tags = []
-    for (tag in tags) {
-        tag = utils.substituteStr(tag, [RELEASE: release, TIMESTAMP: timestamp])
-        if (pipecfg.hotfix) {
-            // this is a hotfix build; include the hotfix name
-            // in the tag suffix so we don't clobber official
-            // tags
-            tag += "-hotfix-${pipecfg.hotfix.name}"
+    def staging_repo = pipecfg.ocp_node_builds.registries.staging.image
+    def staging_manifest_tags = pipecfg.ocp_node_builds.registries.staging.tags
+    def prod_repo = pipecfg.ocp_node_builds.registries.prod.image
+    def prod_tags = pipecfg.ocp_node_builds.registries.prod.tags
+    def processTags = { tagList ->
+        tagList.collect { tag ->
+            def substituted = utils.substituteStr(tag, [RELEASE: release, TIMESTAMP: timestamp])
+            if (pipecfg.hotfix) {
+                // this is a hotfix build; include the hotfix name
+                // in the tag suffix so we don't clobber official
+                // tags
+                substituted += "-hotfix-${pipecfg.hotfix.name}"
+            }
+            return substituted
         }
-        processed_tags += tag
     }
-    return [staging_repo, repo, processed_tags]
+    def final_staging_manifest_tags = processTags(staging_manifest_tags)
+    def final_prod_tags = processTags(prod_tags)
+    return [staging_repo, final_staging_manifest_tags, prod_repo, final_prod_tags]
 }
 
 // Determine if the config.yaml has a test_architectures entry for
@@ -870,15 +873,22 @@ def push_manifest(digests, repo, manifest_tag) {
     for (digest in digests) {
         images += " --image=docker://${repo}@${digest}"
     }
+    def digest = ""
+    def digest_file = "${manifest_tag}.digestfile"
+    // save the digest to a file named after the tag we are pushing
+    push_args = ["--write-digest-to-file", digest_file]
     // arbitrarily selecting the s390x builder; we don't run this
     // locally because podman wants user namespacing (yes, even just
     // to push a manifest...)
     pipeutils.withPodmanRemoteArchBuilder(arch: "s390x") {
         shwrap("""
         cosa push-container-manifest \
-            --tag ${manifest_tag} --repo ${repo} ${images}
+            --tag ${manifest_tag} --repo ${repo} ${images} ${push_args.join(' ')}
         """)
     }
+    digest = readFile(digest_file)
+    shwrap("rm ${digest_file}")
+    return digest
 }
 
 def copy_image(src_image, dest_image, authfile = "") {
@@ -922,13 +932,15 @@ def build_and_push_image(params = [:]) {
 
     def secret = params.get('secret', "");
     def from = params.get('from', "");
+    def manifest_digest = ""
     def extra_build_args = params.get('extra_build_args', "");
 
     def digests = build_remote_image(params['arches'], params['src_commit'], params['src_url'], params['staging_repository'],
                                      params['image_tag_staging'], secret, from, extra_build_args)
     stage("Push Manifest") {
-        push_manifest(digests, params['staging_repository'], params['manifest_tag_staging'])
+        manifest_digest = push_manifest(digests, params['staging_repository'], params['manifest_tag_staging'])
     }
+    return manifest_digest
 }
 
 return this