Merge pull request #6 from NGWPC/pena-pipeline-updates

updated cicd file

Author: miguelp1986

.github/workflows/ci-cd.yml

@@ -2,11 +2,27 @@ name: CI/CD Pipeline
 
 on:
   pull_request:
-    branches: [ngwpc-candidate, ngwpc-release, main, nwm-main, development, release-candidate]
+    branches:
+      - ngwpc-candidate
+      - ngwpc-release
+      - main
+      - nwm-main
+      - development
+      - release-candidate
   push:
-    branches: [ngwpc-candidate, ngwpc-release, main, nwm-main, development, release-candidate]
-  release:
-    types: [published]
+    branches:
+      - ngwpc-candidate
+      - ngwpc-release
+      - main
+      - nwm-main
+      - development
+      - release-candidate
+  workflow_dispatch:
+    inputs:
+      NGEN_IMAGE_TAG:
+        description: 'NGEN_IMAGE_TAG'
+        required: false
+        type: string
 
 permissions:
   contents: read
@@ -18,47 +34,122 @@ env:
   PYTHON_VERSION: '3.11'
 
 jobs:
+  # set variables for use in other jobs
   setup:
+    name: setup
     runs-on: ubuntu-latest
     outputs:
+      org: ${{ steps.vars.outputs.org }}
       image_base: ${{ steps.vars.outputs.image_base }}
       pr_tag: ${{ steps.vars.outputs.pr_tag }}
       commit_sha: ${{ steps.vars.outputs.commit_sha }}
       commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }}
       test_image_tag: ${{ steps.vars.outputs.test_image_tag }}
+      alias_tag: ${{ steps.vars.outputs.alias_tag }}
+      build_date: ${{ steps.vars.outputs.build_date }}
+      clean_ref: ${{ steps.vars.outputs.clean_ref }}
     steps:
       - name: Compute image vars
         id: vars
         shell: bash
         run: |
           set -euo pipefail
+
+          # set variables to use with Docker images
           ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')"
           REPO="$(basename "${GITHUB_REPOSITORY}")"
           IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}"
-          echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT"
 
-          if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then
-            PR_NUM="${{ github.event.pull_request.number }}"
-            PR_TAG="pr-${PR_NUM}-build"
-            echo "pr_tag=${PR_TAG}" >> "$GITHUB_OUTPUT"
-            echo "test_image_tag=${PR_TAG}" >> "$GITHUB_OUTPUT"
+          # one datetime for all time variables
+          NOW=$(date -u +'%Y-%m-%d %H:%M:%S')
+
+          # for OCI labels
+          BUILD_DATE=$(date -u -d "$NOW" +'%Y-%m-%dT%H:%M:%SZ')
+
+          # for Docker image tags
+          TIMESTAMP=$(date -u -d "$NOW" +'%Y%m%d%H%M%SZ')
+
+          # logic to get the real branch name and commit SHA on pull requests
+          if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+            REAL_REF="${{ github.head_ref }}"
+            REAL_SHA="${{ github.event.pull_request.head.sha }}"
+          else
+            REAL_REF="${{ github.ref_name }}"
+            REAL_SHA="${GITHUB_SHA}"
           fi
 
-          if [ "${GITHUB_EVENT_NAME}" = "push" ]; then
-            COMMIT_SHA="${GITHUB_SHA}"
-            SHORT_SHA="${COMMIT_SHA:0:12}"
-            echo "commit_sha=${COMMIT_SHA}" >> "$GITHUB_OUTPUT"
-            echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
-            echo "test_image_tag=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+          # clean ref name and short commit sha
+          CLEAN_REF=$(echo "$REAL_REF" | tr '[:upper:]' '[:lower:]' | sed 's/\//-/g')
+          SHORT_SHA="${REAL_SHA:0:7}"
+
+          # logic for the tags:
+          # test_image_tag (commit short sha): used for the initial build and test
+          # alias_tag: used for final tagging on successful tests
+
+          # test tag is always commit short sha
+          TEST_TAG="${SHORT_SHA}"
+
+          if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+            # for pull requests, use pr-<pr number>-build
+            ALIAS="pr-${{ github.event.pull_request.number }}-build"
+          elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] && [ "${{ github.ref_type }}" = "tag" ]; then
+            # for manual workflow dispatch on tags, use the git tag
+            ALIAS="${CLEAN_REF}"
+          else
+            # for pushes to branches, use timestamp-branchname
+            ALIAS="${TIMESTAMP}-${CLEAN_REF}"
           fi
 
+          # save outputs
+          echo "org=${ORG}" >> "$GITHUB_OUTPUT"
+          echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT"
+          echo "build_date=${BUILD_DATE}" >> "$GITHUB_OUTPUT"
+          echo "test_image_tag=${TEST_TAG}" >> "$GITHUB_OUTPUT"
+          echo "alias_tag=${ALIAS}" >> "$GITHUB_OUTPUT"
+          echo "commit_sha=${REAL_SHA}" >> "$GITHUB_OUTPUT"
+          echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+          echo "clean_ref=${CLEAN_REF}" >> "$GITHUB_OUTPUT"
+
+  # CodeQL scan
+  codeql-scan:
+    name: codeql-scan
+    if: |
+      (github.event_name == 'pull_request') ||
+      (github.event_name == 'push') ||
+      (github.event_name == 'workflow_dispatch')
+    runs-on: ubuntu-latest
+    needs: setup
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v4
+        with:
+          languages: python
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v4
+
   build:
     name: build
-    if: github.event_name == 'pull_request' || github.event_name == 'push'
+    if: |
+      (github.event_name == 'pull_request') ||
+      (github.event_name == 'push') ||
+      (github.event_name == 'workflow_dispatch')
     runs-on: ubuntu-latest
     needs: setup
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Log in to registry
         uses: docker/login-action@v3
@@ -74,145 +165,106 @@ jobs:
           push: true
           tags: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
           build-args: |
-            NGEN_IMAGE_TAG=${{ env.NGEN_IMAGE_TAG || 'latest' }}
-            CI_COMMIT_REF_NAME=${{ github.ref_name }}
+            ORG=${{ needs.setup.outputs.org }}
+            NGEN_IMAGE_TAG=${{ inputs.NGEN_IMAGE_TAG || 'latest' }}
+            IMAGE_SOURCE=https://github.com/${{ github.repository }}
+            IMAGE_VENDOR=${{ github.repository_owner }}
+            IMAGE_VERSION=${{ needs.setup.outputs.clean_ref }}
+            IMAGE_REVISION=${{ needs.setup.outputs.commit_sha }}
+            IMAGE_CREATED=${{ needs.setup.outputs.build_date }}
+            CI_COMMIT_REF_NAME=${{ needs.setup.outputs.clean_ref }}
 
   unit-test:
     name: unit-test
-    if: github.event_name == 'pull_request' || github.event_name == 'push'
+    if: |
+      (github.event_name == 'pull_request') ||
+      (github.event_name == 'push') ||
+      (github.event_name == 'workflow_dispatch')
     runs-on: ubuntu-latest
-    needs: [setup, build]
+    needs:
+      - setup
+      - build
     container:
       image: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
     steps:
       - name: Run unit tests
         run: |
           echo "TODO: add unit tests here"
 
-  codeql-scan:
-    if: github.event_name == 'pull_request' || github.event_name == 'push'
-    runs-on: ubuntu-latest
-    needs: [setup, build]
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: python
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
-
+# run container security scan using Trivy
   container-scanning:
-    if: github.event_name == 'pull_request' || github.event_name == 'push'
+    if: |
+      (github.event_name == 'pull_request') ||
+      (github.event_name == 'push') ||
+      (github.event_name == 'workflow_dispatch')
     runs-on: ubuntu-latest
-    needs: [setup, build]
+    needs:
+      - setup
+      - build
     steps:
-      - name: Scan container with Trivy
-        uses: aquasecurity/trivy-action@0.20.0
-        with:
-          image-ref: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
-          format: 'template'
-          template: '@/contrib/sarif.tpl'
-          output: 'trivy-results.sarif'
-          severity: 'CRITICAL,HIGH'
-      - name: Upload Trivy SARIF
-        uses: github/codeql-action/upload-sarif@v3
+      - name: Install Trivy
+        uses: aquasecurity/setup-trivy@v0.2.2
         with:
-          sarif_file: 'trivy-results.sarif'
+          cache: true
+          version: v0.68.2
 
-  deploy-latest-on-development:
-    name: deploy-latest-on-development
-    if: github.event_name == 'push' && github.ref_name == 'development'
-    runs-on: ubuntu-latest
-    needs: [setup, build, unit-test, codeql-scan, container-scanning]
-    steps:
-      - name: Tag image with 'latest'
-        shell: bash
+      - name: Trivy scan
+        env:
+          TMPDIR: /mnt/trivy-temp
         run: |
-          set -euo pipefail
-          IMAGE_BASE="${{ needs.setup.outputs.image_base }}"
-          SHORT_SHA="${{ needs.setup.outputs.commit_sha_short }}"
+          sudo mkdir -p $TMPDIR
+          sudo chown -R $USER:$USER $TMPDIR
 
-          # ensure skopeo is available
-          if ! command -v skopeo >/dev/null 2>&1; then
-            sudo apt-get update -y
-            sudo apt-get install -y --no-install-recommends skopeo
-          fi
-
-          skopeo copy \
-            --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
-            --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
-            docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:latest"
+          trivy image \
+            --format sarif \
+            --output trivy-results.sarif \
+            --severity CRITICAL,HIGH \
+            --scanners vuln \
+            --ignore-unfixed \
+            --timeout 45m \
+            ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
 
-  release:
-    name: release
-    if: github.event_name == 'release' && github.event.action == 'published'
+  # promote Docker image tags after successful tests
+  promote-tags:
+    name: Promote Tags
+    if: |
+      (github.event_name == 'pull_request') ||
+      (github.event_name == 'push') ||
+      (github.event_name == 'workflow_dispatch')
     runs-on: ubuntu-latest
-    needs: setup
+    needs:
+      - setup
+      - codeql-scan
+      - build
+      - container-scanning
     steps:
-      - name: Get commit sha for the tag
-        id: rev
-        shell: bash
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          set -euo pipefail
-          TAG="${{ github.event.release.tag_name }}"
-          REPO="${{ github.repository }}"
-
-          # ensure jq is available
-          if ! command -v jq >/dev/null 2>&1; then
-            sudo apt-get update -y
-            sudo apt-get install -y --no-install-recommends jq
-          fi
-
-          # ensure gh cli is available
-          if ! command -v gh >/dev/null 2>&1; then
-            sudo apt-get update -y
-            sudo apt-get install -y --no-install-recommends gh
-          fi
-
-          REF_JSON="$(gh api "repos/${REPO}/git/refs/tags/${TAG}")"
-          OBJ_SHA="$(jq -r '.object.sha' <<<"$REF_JSON")"
-          OBJ_TYPE="$(jq -r '.object.type' <<<"$REF_JSON")"
-
-          if [ "$OBJ_TYPE" = "tag" ]; then
-            TAG_OBJ="$(gh api "repos/${REPO}/git/tags/${OBJ_SHA}")"
-            COMMIT_SHA="$(jq -r '.object.sha' <<<"$TAG_OBJ")"
-          else
-            COMMIT_SHA="$OBJ_SHA"
-          fi
-
-          SHORT_SHA="${COMMIT_SHA:0:12}"
-          echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
-
-      - name: Tag image with release tag
+      - name: Tag image with alias and latest
         shell: bash
         run: |
           set -euo pipefail
-          IMAGE_BASE="${{ needs.setup.outputs.image_base }}"
-          SHORT_SHA="${{ steps.rev.outputs.short_sha }}"
-          RELEASE_TAG="${{ github.event.release.tag_name }}"
 
-          # ensure skopeo is available
+          # ensure skopeo is available for promotion
           if ! command -v skopeo >/dev/null 2>&1; then
             sudo apt-get update -y
             sudo apt-get install -y --no-install-recommends skopeo
           fi
 
+          IMAGE_BASE="${{ needs.setup.outputs.image_base }}"
+          TEST_TAG="${{ needs.setup.outputs.test_image_tag }}"
+          ALIAS_TAG="${{ needs.setup.outputs.alias_tag }}"
+
+          # apply the primary alias (pr tag or timestamp-branch)
           skopeo copy \
             --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
             --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
-            docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:${RELEASE_TAG}"
+            --all \
+            "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:${ALIAS_TAG}"
+
+          # tag with 'latest' on development branch push
+          if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF_NAME" = "development" ]; then
+            skopeo copy \
+              --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
+              --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
+              --all \
+              "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:latest"
+          fi