Publishing and NSPECT Pipeline Changes

Author: JunAr7112

.common-ci.yml

@@ -26,7 +26,7 @@ stages:
   - pull
   - scan
   - release
-  - sign
+  - ngc-publish
 
 .pipeline-trigger-rules:
   rules:

.nvidia-ci.yml

@@ -163,6 +163,9 @@ push-images-to-staging:
   stage: release
   needs:
     - scan-images
+  artifacts:
+    paths:
+      - build-info/${CI_PIPELINE_ID}-${SAMPLE}.txt
   variables:
     IN_REGISTRY: "${CI_REGISTRY}"
     IN_REGISTRY_USER: "${CI_REGISTRY_USER}"
@@ -175,105 +178,93 @@ push-images-to-staging:
     OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
     OUT_IMAGE_NAME: "${NGC_STAGING_REGISTRY}/cuda-samples"
     OUT_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
+  after_script:
+    - mkdir -p build-info
+    - echo "${IN_IMAGE_NAME}:${SAMPLE}-${IN_IMAGE_TAG} ${NGC_REGISTRY_IMAGE}:${SAMPLE}-${CI_COMMIT_TAG}" > build-info/${CI_PIPELINE_ID}-${SAMPLE}.txt
 
-.release-images:
-  extends:
-    - .copy-images
-  stage: release
+generate-build-info:
+  stage: ngc-publish
+  artifacts:
+    paths:
+      - build-info/full-${CI_PIPELINE_ID}.txt
+  script:
+    - |
+      for file in build-info/${CI_PIPELINE_ID}-*; do
+        cat "$file" >> build-info/full-${CI_PIPELINE_ID}.txt
+      done
+
+.update-nspect:
+  stage: ngc-publish
   needs:
-    - scan-images
-    - push-images-to-staging
+    - job: generate-build-info
+  image: 
+    name: "${CNT_NGC_PUBLISH_IMAGE}"
+    pull_policy: always
   variables:
-    IN_REGISTRY: "${CI_REGISTRY}"
-    IN_REGISTRY_USER: "${CI_REGISTRY_USER}"
-    IN_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
-    IN_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/cuda-samples"
-    IN_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
+    PROJECT_NAME: "k8s-samples"
+    REPO_URL: "https://github.com/NVIDIA/k8s-samples.git"
+  script:
+    - |
+      cnt-ngc-publish nspect --versions-file "build-info/full-${CI_PIPELINE_ID}.txt"
 
-    OUT_REGISTRY: "${NGC_REGISTRY}"
-    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
-    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
-    OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
-    OUT_IMAGE_TAG: "${CI_COMMIT_TAG}"
+  # Update the nspect staging environment to test the nspect publishing logic
+update-nspect-staging:
+  extends:
+    - .update-nspect
+  except:
+    - tags
+  variables:
+    ENV: "stage"
+    RELEASE_VERSION: "test"
+    NSPECT_CLIENT_ID: "${NSPECT_STAGING_CLIENT_ID}" 
+    NSPECT_CLIENT_SECRET: "${NSPECT_STAGING_CLIENT_SECRET}"
 
-release-images-to-ngc:
+# Update the nspect production environment with the new release
+update-nspect:
   extends:
-    - .release-images
+    - .update-nspect
   rules:
     - if: $CI_COMMIT_TAG
-
-release-images-dummy:
-  extends:
-    - .release-images
   variables:
-    REGCTL: "echo [DUMMY] regctl"
-  rules:
-    - if: $CI_COMMIT_TAG == null || $CI_COMMIT_TAG == ""
+    OSRB_BUG_ID: "${OSRB_BUG_ID}"
+    ENV: "prod"
+    RELEASE_VERSION: "${CI_COMMIT_TAG}"
+    NSPECT_CLIENT_ID: "${NSPECT_PROD_CLIENT_ID}" 
+    NSPECT_CLIENT_SECRET: "${NSPECT_PROD_CLIENT_SECRET}"
 
-# .sign-images forms the base of the jobs which sign images in the NGC registry.
-.sign-images:
-  stage: sign
-  image: ubuntu:latest
-  parallel:
-    matrix:
-      - SAMPLE:
-        - devicequery
-        - nbody
-        - nvbandwidth
-        - simplemultigpu
-        - vectoradd
-        - vulkan
+.publish-images:
+  stage: ngc-publish
+  image:
+    name: "${CNT_NGC_PUBLISH_IMAGE}"
+    pull_policy: always
   variables:
-    IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
-    IMAGE_TAG: "${CI_COMMIT_TAG}"
-    NGC_CLI: "ngc-cli/ngc"
-  before_script:
-    - !reference [.ngccli-setup, before_script]
+    GITLAB_ACCESS_TOKEN: "${CNT_GITLAB_TOKEN}"
   script:
-    - |
-      # We ensure that the IMAGE_NAME and IMAGE_TAG is set
-      echo Image Name: ${IMAGE_NAME} && [[ -n "${IMAGE_NAME}" ]] || exit 1
-      echo Image Tag: ${IMAGE_TAG} && [[ -n "${IMAGE_TAG}" ]] || exit 1
-
-      export IMAGE=${IMAGE_NAME}:${SAMPLE}-${IMAGE_TAG}
-      echo "Signing the image ${IMAGE}"
-      ${NGC_CLI} registry image publish --source ${IMAGE} ${IMAGE} --public --discoverable --allow-guest --sign --org nvidia
-
-# Define the external image signing steps for NGC
-# Download the ngc cli binary for use in the sign steps
-.ngccli-setup:
-  before_script:
-    - apt-get update && apt-get install -y curl unzip jq
-    - |
-      if [ -z "${NGCCLI_VERSION}" ]; then
-        NGC_VERSION_URL="https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions"
-        # Extract the latest version from the JSON data using jq
-        export NGCCLI_VERSION=$(curl -s $NGC_VERSION_URL | jq -r '.recipe.latestVersionIdStr')
-      fi
-      echo "NGCCLI_VERSION ${NGCCLI_VERSION}"
-    - curl -sSLo ngccli_linux.zip https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/${NGCCLI_VERSION}/files/ngccli_linux.zip
-    - unzip ngccli_linux.zip
-    - chmod u+x ngc-cli/ngc
+    - cnt-ngc-publish render --project-name "k8s-samples" --versions-file "build-info/full-${CI_PIPELINE_ID}.txt" --output k8s-samples.yaml
+    - cnt-ngc-publish merge-request --files "k8s-samples.yaml"
 
-sign-ngc-images:
+# Raise an MR to publish the image to NGC
+ngc-image-publish:
   extends:
-    - .sign-images
-  needs:
-    - release-images-to-ngc
+    - .publish-images
   rules:
     - if: $CI_COMMIT_TAG
+  needs:
+    - job: update-nspect
+    - job: generate-build-info
+      artifacts: true
   variables:
-    NGC_CLI_API_KEY: "${NGC_REGISTRY_TOKEN}"
-  retry:
-    max: 2
+    NGC_PUBLISHING_PROJECT_PATH: "${NGC_PUBLISHING_PROD_PROJECT_PATH}"
 
-sign-images-dummy:
+# Create a dummy MR that exercises the publishing logic
+mock-image-publish:
   extends:
-    - .sign-images
+    - .publish-images
+  except:
+    - tags
   needs:
-    - release-images-dummy
+    - job: update-nspect-staging
+    - job: generate-build-info
+      artifacts: true
   variables:
-    NGC_CLI: "echo [DUMMY] ngc-cli/ngc"
-    IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
-  rules:
-    - if: $CI_COMMIT_TAG == null || $CI_COMMIT_TAG == ""
+    NGC_PUBLISHING_PROJECT_PATH: "${NGC_PUBLISHING_TEST_PROJECT_PATH}"