CI/CD Pipeline #341
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD Pipeline | |
| on: | |
| pull_request: | |
| branches: | |
| - ngwpc-candidate | |
| - ngwpc-release | |
| - main | |
| - nwm-main | |
| - development | |
| - development-pw | |
| - release-candidate | |
| push: | |
| branches: | |
| - ngwpc-candidate | |
| - ngwpc-release | |
| - main | |
| - nwm-main | |
| - development | |
| - development-pw | |
| - release-candidate | |
| workflow_dispatch: | |
| inputs: | |
| GHCR_ORG: | |
| description: 'GHCR_ORG' | |
| required: false | |
| type: string | |
| NGEN_IMAGE_TAG: | |
| description: 'NGEN_IMAGE_TAG' | |
| required: false | |
| type: string | |
| EWTS_ORG: | |
| description: 'EWTS_ORG' | |
| required: false | |
| type: string | |
| EWTS_REF: | |
| description: 'EWTS_REF' | |
| required: false | |
| type: string | |
| MSW_MGR_ORG: | |
| description: 'MSW_MGR_ORG' | |
| required: false | |
| type: string | |
| MSW_MGR_REF: | |
| description: 'MSW_MGR_REF' | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| packages: write | |
| security-events: write | |
| env: | |
| REGISTRY: ghcr.io | |
| PYTHON_VERSION: '3.11' | |
| jobs: | |
| # set variables for use in other jobs | |
| setup: | |
| name: setup | |
| runs-on: ubuntu-latest | |
| outputs: | |
| org: ${{ steps.vars.outputs.org }} | |
| image_base: ${{ steps.vars.outputs.image_base }} | |
| pr_tag: ${{ steps.vars.outputs.pr_tag }} | |
| commit_sha: ${{ steps.vars.outputs.commit_sha }} | |
| commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }} | |
| test_image_tag: ${{ steps.vars.outputs.test_image_tag }} | |
| alias_tag: ${{ steps.vars.outputs.alias_tag }} | |
| clean_ref: ${{ steps.vars.outputs.clean_ref }} | |
| default_ref: ${{ steps.vars.outputs.default_ref }} | |
| default_base_tag: ${{ steps.vars.outputs.default_base_tag }} | |
| ngen_image_digest: ${{ steps.vars.outputs.ngen_image_digest }} | |
| ngen_image_revision: ${{ steps.vars.outputs.ngen_image_revision }} | |
| ewts_revision: ${{ steps.vars.outputs.ewts_revision }} | |
| msw_mgr_revision: ${{ steps.vars.outputs.msw_mgr_revision }} | |
| steps: | |
| - name: Compute image vars | |
| id: vars | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # set variables to use with Docker images | |
| ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" | |
| REPO="$(basename "${GITHUB_REPOSITORY}")" | |
| IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}" | |
| # one datetime for all time variables | |
| NOW=$(date -u +'%Y-%m-%d %H:%M:%S') | |
| # for Docker image tags | |
| TIMESTAMP=$(date -u -d "$NOW" +'%Y%m%d%H%M%SZ') | |
| # logic to get the real branch name and commit SHA on pull requests | |
| if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then | |
| REAL_REF="${{ github.head_ref }}" | |
| REAL_SHA="${{ github.event.pull_request.head.sha }}" | |
| else | |
| REAL_REF="${{ github.ref_name }}" | |
| REAL_SHA="${GITHUB_SHA}" | |
| fi | |
| # clean ref name and short commit sha | |
| CLEAN_REF=$(echo "$REAL_REF" | tr '[:upper:]' '[:lower:]' | sed 's/\//-/g') | |
| SHORT_SHA="${REAL_SHA:0:7}" | |
| # default source-repo ref: follow the release-line branch we're building from | |
| # (development / development-pw / ngwpc-candidate / ngwpc-release); tags and | |
| # other branches fall back to development. Override per source repo via the | |
| # *_REF dispatch inputs. | |
| case "${GITHUB_REF_TYPE}:${GITHUB_REF_NAME}" in | |
| branch:development|branch:development-pw|branch:ngwpc-candidate|branch:ngwpc-release) | |
| DEFAULT_REF="$GITHUB_REF_NAME" ;; | |
| *) | |
| DEFAULT_REF="development" ;; | |
| esac | |
| # use an explicit *_REF input if provided, else DEFAULT_REF | |
| ref_or_default() { [ -n "$1" ] && echo "$1" || echo "$DEFAULT_REF"; } | |
| # base-image alias follows the lane: development-pw (or a PR into it) pulls | |
| # the rocky 'latest-pw' alias, every other ref the bookworm 'latest'. An | |
| # explicit NGEN_IMAGE_TAG input always wins over this default. | |
| LANE_REF="${GITHUB_BASE_REF:-$GITHUB_REF_NAME}" | |
| if [ "$LANE_REF" = "development-pw" ]; then | |
| DEFAULT_BASE_TAG="latest-pw" | |
| else | |
| DEFAULT_BASE_TAG="latest" | |
| fi | |
| # logic for the tags: | |
| # test_image_tag (commit short sha): used for the initial build and test | |
| # alias_tag: used for final tagging on successful tests | |
| # test tag is always commit short sha | |
| TEST_TAG="${SHORT_SHA}" | |
| if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then | |
| # for pull requests, use pr-<pr number>-build | |
| ALIAS="pr-${{ github.event.pull_request.number }}-build" | |
| elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] && [ "${{ github.ref_type }}" = "tag" ]; then | |
| # for manual workflow dispatch on tags, use the git tag | |
| ALIAS="${CLEAN_REF}" | |
| else | |
| # for pushes to branches, use timestamp-branchname | |
| ALIAS="${TIMESTAMP}-${CLEAN_REF}" | |
| fi | |
| # skopeo is needed to inspect the base image | |
| if ! command -v skopeo >/dev/null 2>&1; then | |
| sudo apt-get update -y | |
| sudo apt-get install -y --no-install-recommends skopeo | |
| fi | |
| # base image (ngen) metadata for Dockerfile labels | |
| NGEN_IMAGE_TAG="${{ inputs.NGEN_IMAGE_TAG }}" | |
| NGEN_IMAGE_TAG="${NGEN_IMAGE_TAG:-$DEFAULT_BASE_TAG}" | |
| NGEN_IMAGE="ghcr.io/${ORG}/ngen:${NGEN_IMAGE_TAG}" | |
| NGEN_INSPECT=$(skopeo inspect --override-os linux --override-arch amd64 "docker://${NGEN_IMAGE}" 2>/dev/null || echo '{}') | |
| NGEN_IMAGE_DIGEST=$(echo "$NGEN_INSPECT" | jq -r '.Digest // "unknown"') | |
| NGEN_IMAGE_REVISION=$(echo "$NGEN_INSPECT" | jq -r '.Labels["org.opencontainers.image.revision"] // "unknown"') | |
| # resolve each source repo's ref (branch/tag/SHA) to its commit SHA for revision labels | |
| resolve_sha() { | |
| local url="$1" ref="$2" out sha | |
| # a full 40-char SHA can't be looked up via ls-remote; use it directly | |
| if [[ "$ref" =~ ^[0-9a-f]{40}$ ]]; then echo "$ref"; return; fi | |
| out=$(git ls-remote "$url" "$ref" "refs/tags/${ref}^{}" 2>/dev/null) | |
| # prefer the dereferenced commit (^{}) for annotated tags; else first match | |
| sha=$(echo "$out" | grep '\^{}$' | head -n1 | cut -f1) | |
| [ -z "$sha" ] && sha=$(echo "$out" | head -n1 | cut -f1) | |
| echo "${sha:-unknown}" | |
| } | |
| # Each *_REVISION is the dependency's resolved commit SHA: it labels the | |
| # image AND doubles as the per-dependency cache-bust build-arg, so a new | |
| # commit on the requested ref reinstalls that dep instead of a stale cache. | |
| EWTS_REVISION=$(resolve_sha "https://github.com/${{ inputs.EWTS_ORG || github.repository_owner }}/nwm-ewts.git" "$(ref_or_default "${{ inputs.EWTS_REF }}")") | |
| MSW_MGR_REVISION=$(resolve_sha "https://github.com/${{ inputs.MSW_MGR_ORG || github.repository_owner }}/nwm-msw-mgr.git" "$(ref_or_default "${{ inputs.MSW_MGR_REF }}")") | |
| # save outputs | |
| cat >> "$GITHUB_OUTPUT" <<EOF | |
| org=${ORG} | |
| image_base=${IMAGE_BASE} | |
| test_image_tag=${TEST_TAG} | |
| alias_tag=${ALIAS} | |
| commit_sha=${REAL_SHA} | |
| commit_sha_short=${SHORT_SHA} | |
| clean_ref=${CLEAN_REF} | |
| default_ref=${DEFAULT_REF} | |
| default_base_tag=${DEFAULT_BASE_TAG} | |
| ngen_image_digest=${NGEN_IMAGE_DIGEST} | |
| ngen_image_revision=${NGEN_IMAGE_REVISION} | |
| ewts_revision=${EWTS_REVISION} | |
| msw_mgr_revision=${MSW_MGR_REVISION} | |
| EOF | |
| # CodeQL scan | |
| codeql-scan: | |
| name: codeql-scan | |
| if: | | |
| (github.event_name == 'pull_request') || | |
| (github.event_name == 'push') || | |
| (github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Initialize CodeQL | |
| uses: github/codeql-action/init@v4 | |
| with: | |
| languages: python | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@v4 | |
| build: | |
| name: build | |
| if: | | |
| (github.event_name == 'pull_request') || | |
| (github.event_name == 'push') || | |
| (github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Log in to registry | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build & push image | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} | |
| build-args: | | |
| GHCR_ORG=${{ inputs.GHCR_ORG || needs.setup.outputs.org }} | |
| NGEN_IMAGE=ghcr.io/${{ needs.setup.outputs.org }}/ngen:${{ inputs.NGEN_IMAGE_TAG || needs.setup.outputs.default_base_tag }} | |
| NGEN_IMAGE_DIGEST=${{ needs.setup.outputs.ngen_image_digest }} | |
| NGEN_IMAGE_REVISION=${{ needs.setup.outputs.ngen_image_revision }} | |
| EWTS_ORG=${{ inputs.EWTS_ORG || github.repository_owner }} | |
| EWTS_REF=${{ inputs.EWTS_REF || needs.setup.outputs.default_ref }} | |
| EWTS_REVISION=${{ needs.setup.outputs.ewts_revision }} | |
| EWTS_CACHE_BUST=${{ needs.setup.outputs.ewts_revision }} | |
| MSW_MGR_ORG=${{ inputs.MSW_MGR_ORG || github.repository_owner }} | |
| MSW_MGR_REF=${{ inputs.MSW_MGR_REF || needs.setup.outputs.default_ref }} | |
| MSW_MGR_REVISION=${{ needs.setup.outputs.msw_mgr_revision }} | |
| MSW_MGR_CACHE_BUST=${{ needs.setup.outputs.msw_mgr_revision }} | |
| IMAGE_SOURCE=https://github.com/${{ github.repository }} | |
| IMAGE_VENDOR=${{ github.repository_owner }} | |
| IMAGE_VERSION=${{ needs.setup.outputs.clean_ref }} | |
| IMAGE_REVISION=${{ needs.setup.outputs.commit_sha }} | |
| CI_COMMIT_REF_NAME=${{ needs.setup.outputs.clean_ref }} | |
| # run unit tests inside the built Docker image | |
| unit-test: | |
| name: unit-test | |
| if: | | |
| (github.event_name == 'pull_request') || | |
| (github.event_name == 'push') || | |
| (github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-latest | |
| needs: | |
| - setup | |
| - build | |
| container: | |
| image: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} | |
| steps: | |
| - name: Run unit tests | |
| run: | | |
| echo "TODO: add unit tests here" | |
| # run container security scan using Trivy | |
| container-scanning: | |
| if: | | |
| (github.event_name == 'pull_request') || | |
| (github.event_name == 'push') || | |
| (github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-latest | |
| needs: | |
| - setup | |
| - build | |
| steps: | |
| - name: Install Trivy | |
| uses: aquasecurity/setup-trivy@v0.2.6 | |
| with: | |
| cache: true | |
| version: v0.69.3 | |
| - name: Trivy scan | |
| env: | |
| TMPDIR: /mnt/trivy-temp | |
| run: | | |
| sudo mkdir -p $TMPDIR | |
| sudo chown -R $USER:$USER $TMPDIR | |
| trivy image \ | |
| --format sarif \ | |
| --output trivy-results.sarif \ | |
| --severity CRITICAL,HIGH \ | |
| --scanners vuln \ | |
| --ignore-unfixed \ | |
| --timeout 45m \ | |
| ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} | |
| # promote Docker image tags after successful tests | |
| promote-tags: | |
| name: Promote Tags | |
| if: | | |
| (github.event_name == 'pull_request') || | |
| (github.event_name == 'push') || | |
| (github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-latest | |
| needs: | |
| - setup | |
| - codeql-scan | |
| - build | |
| - unit-test | |
| - container-scanning | |
| steps: | |
| - name: Tag image with alias and latest | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # ensure skopeo is available for promotion | |
| if ! command -v skopeo >/dev/null 2>&1; then | |
| sudo apt-get update -y | |
| sudo apt-get install -y --no-install-recommends skopeo | |
| fi | |
| IMAGE_BASE="${{ needs.setup.outputs.image_base }}" | |
| TEST_TAG="${{ needs.setup.outputs.test_image_tag }}" | |
| ALIAS_TAG="${{ needs.setup.outputs.alias_tag }}" | |
| # apply the primary alias (pr tag or timestamp-branch) | |
| skopeo copy \ | |
| --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --all \ | |
| "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:${ALIAS_TAG}" | |
| # Branch-keyed promotion: development promotes 'latest' (bookworm/AWS | |
| # lane); development-pw promotes 'latest-pw' (rocky/PW lane). Do not | |
| # manually dispatch a cross-lane build from either branch: promotion | |
| # keys off the branch name, so the tested image would promote under | |
| # the wrong alias. | |
| if [ "$GITHUB_REF_NAME" = "development" ]; then | |
| skopeo copy \ | |
| --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --all \ | |
| "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:latest" | |
| elif [ "$GITHUB_REF_NAME" = "development-pw" ]; then | |
| skopeo copy \ | |
| --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --all \ | |
| "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:latest-pw" | |
| fi | |
| # Build the Apptainer .sif from the tested + promoted image and publish it to | |
| # GHCR as an OCI artifact (oras://). AWS PCS compute nodes pull this onto the | |
| # shared EFS and run it with `apptainer run`. SIF images are NOT baked into the | |
| # compute AMI. The AMI carries only the Apptainer runtime; the images ship here | |
| # and are staged on EFS. | |
| build-sif: | |
| name: build-sif | |
| # Only on real branch builds (push) or manual dispatch, not PRs (a multi-GB | |
| # SIF build on every PR is wasteful). Runs after the image is tested + promoted. | |
| if: | | |
| (github.event_name == 'push') || | |
| (github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-latest | |
| needs: | |
| - setup | |
| - promote-tags | |
| env: | |
| # Keep Apptainer's build temp + layer cache off the small root volume; /mnt | |
| # is the roomy ephemeral disk on GitHub-hosted runners (a SIF needs room for | |
| # the uncompressed image during mksquashfs). | |
| APPTAINER_TMPDIR: /mnt/apptainer-tmp | |
| APPTAINER_CACHEDIR: /mnt/apptainer-cache | |
| steps: | |
| - name: Install Apptainer | |
| uses: eWaterCycle/setup-apptainer@v2 | |
| with: | |
| apptainer-version: 1.3.6 | |
| - name: Log in to GHCR | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build .sif from the promoted image and push to GHCR (oras) | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| sudo mkdir -p "$APPTAINER_TMPDIR" "$APPTAINER_CACHEDIR" | |
| sudo chown -R "$USER":"$USER" "$APPTAINER_TMPDIR" "$APPTAINER_CACHEDIR" | |
| IMAGE_BASE="${{ needs.setup.outputs.image_base }}" # ghcr.io/<org>/nwm-fcst-mgr | |
| ALIAS_TAG="${{ needs.setup.outputs.alias_tag }}" # the tested + promoted tag | |
| SIF_REPO="${IMAGE_BASE}-sif" # ghcr.io/<org>/nwm-fcst-mgr-sif | |
| # Convert the promoted OCI image to a SIF. docker/login-action wrote | |
| # ~/.docker/config.json, which Apptainer reads for the docker:// pull and | |
| # the oras:// push. Unprivileged build uses --fakeroot automatically | |
| # (GitHub-hosted runners have unprivileged user namespaces enabled). | |
| apptainer build --force nwm-fcst-mgr.sif "docker://${IMAGE_BASE}:${ALIAS_TAG}" | |
| # Publish as an OCI artifact. A SIF pushed via oras:// MUST be pulled via | |
| # oras:// (not docker://). The in-VPC staging step pulls this onto EFS. | |
| apptainer push nwm-fcst-mgr.sif "oras://${SIF_REPO}:${ALIAS_TAG}" | |
| echo "Published oras://${SIF_REPO}:${ALIAS_TAG}" | |
| # Branch-keyed SIF promotion, mirroring the image promotion above: | |
| # development also publishes ':latest', development-pw ':latest-pw'. | |
| # The registry already holds the SIF blob from the push above, so this | |
| # re-push uploads only the manifest and tag, not the multi-GB SIF. Do | |
| # not manually dispatch a cross-lane build from either branch: | |
| # promotion keys off the branch name, so the SIF would promote under | |
| # the wrong alias. | |
| if [ "$GITHUB_REF_NAME" = "development" ]; then | |
| apptainer push nwm-fcst-mgr.sif "oras://${SIF_REPO}:latest" | |
| echo "Promoted oras://${SIF_REPO}:latest" | |
| elif [ "$GITHUB_REF_NAME" = "development-pw" ]; then | |
| apptainer push nwm-fcst-mgr.sif "oras://${SIF_REPO}:latest-pw" | |
| echo "Promoted oras://${SIF_REPO}:latest-pw" | |
| fi |