Merge pull request #6 from NGWPC/pena-pipeline-updates #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD Pipeline | |
| on: | |
| pull_request: | |
| branches: | |
| - ngwpc-candidate | |
| - ngwpc-release | |
| - main | |
| - nwm-main | |
| - development | |
| - release-candidate | |
| push: | |
| branches: | |
| - ngwpc-candidate | |
| - ngwpc-release | |
| - main | |
| - nwm-main | |
| - development | |
| - release-candidate | |
| workflow_dispatch: | |
| inputs: | |
| NGEN_IMAGE_TAG: | |
| description: 'NGEN_IMAGE_TAG' | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| packages: write | |
| security-events: write | |
| env: | |
| REGISTRY: ghcr.io | |
| PYTHON_VERSION: '3.11' | |
| jobs: | |
| # set variables for use in other jobs | |
| setup: | |
| name: setup | |
| runs-on: ubuntu-latest | |
| outputs: | |
| org: ${{ steps.vars.outputs.org }} | |
| image_base: ${{ steps.vars.outputs.image_base }} | |
| pr_tag: ${{ steps.vars.outputs.pr_tag }} | |
| commit_sha: ${{ steps.vars.outputs.commit_sha }} | |
| commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }} | |
| test_image_tag: ${{ steps.vars.outputs.test_image_tag }} | |
| alias_tag: ${{ steps.vars.outputs.alias_tag }} | |
| build_date: ${{ steps.vars.outputs.build_date }} | |
| clean_ref: ${{ steps.vars.outputs.clean_ref }} | |
| steps: | |
| - name: Compute image vars | |
| id: vars | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # set variables to use with Docker images | |
| ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" | |
| REPO="$(basename "${GITHUB_REPOSITORY}")" | |
| IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}" | |
| # one datetime for all time variables | |
| NOW=$(date -u +'%Y-%m-%d %H:%M:%S') | |
| # for OCI labels | |
| BUILD_DATE=$(date -u -d "$NOW" +'%Y-%m-%dT%H:%M:%SZ') | |
| # for Docker image tags | |
| TIMESTAMP=$(date -u -d "$NOW" +'%Y%m%d%H%M%SZ') | |
| # logic to get the real branch name and commit SHA on pull requests | |
| if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then | |
| REAL_REF="${{ github.head_ref }}" | |
| REAL_SHA="${{ github.event.pull_request.head.sha }}" | |
| else | |
| REAL_REF="${{ github.ref_name }}" | |
| REAL_SHA="${GITHUB_SHA}" | |
| fi | |
| # clean ref name and short commit sha | |
| CLEAN_REF=$(echo "$REAL_REF" | tr '[:upper:]' '[:lower:]' | sed 's/\//-/g') | |
| SHORT_SHA="${REAL_SHA:0:7}" | |
| # logic for the tags: | |
| # test_image_tag (commit short sha): used for the initial build and test | |
| # alias_tag: used for final tagging on successful tests | |
| # test tag is always commit short sha | |
| TEST_TAG="${SHORT_SHA}" | |
| if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then | |
| # for pull requests, use pr-<pr number>-build | |
| ALIAS="pr-${{ github.event.pull_request.number }}-build" | |
| elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] && [ "${{ github.ref_type }}" = "tag" ]; then | |
| # for manual workflow dispatch on tags, use the git tag | |
| ALIAS="${CLEAN_REF}" | |
| else | |
| # for pushes to branches, use timestamp-branchname | |
| ALIAS="${TIMESTAMP}-${CLEAN_REF}" | |
| fi | |
| # save outputs | |
| echo "org=${ORG}" >> "$GITHUB_OUTPUT" | |
| echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT" | |
| echo "build_date=${BUILD_DATE}" >> "$GITHUB_OUTPUT" | |
| echo "test_image_tag=${TEST_TAG}" >> "$GITHUB_OUTPUT" | |
| echo "alias_tag=${ALIAS}" >> "$GITHUB_OUTPUT" | |
| echo "commit_sha=${REAL_SHA}" >> "$GITHUB_OUTPUT" | |
| echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT" | |
| echo "clean_ref=${CLEAN_REF}" >> "$GITHUB_OUTPUT" | |
| # CodeQL scan | |
| codeql-scan: | |
| name: codeql-scan | |
| if: | | |
| (github.event_name == 'pull_request') || | |
| (github.event_name == 'push') || | |
| (github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Initialize CodeQL | |
| uses: github/codeql-action/init@v4 | |
| with: | |
| languages: python | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@v4 | |
| build: | |
| name: build | |
| if: | | |
| (github.event_name == 'pull_request') || | |
| (github.event_name == 'push') || | |
| (github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Log in to registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build & push image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} | |
| build-args: | | |
| ORG=${{ needs.setup.outputs.org }} | |
| NGEN_IMAGE_TAG=${{ inputs.NGEN_IMAGE_TAG || 'latest' }} | |
| IMAGE_SOURCE=https://github.com/${{ github.repository }} | |
| IMAGE_VENDOR=${{ github.repository_owner }} | |
| IMAGE_VERSION=${{ needs.setup.outputs.clean_ref }} | |
| IMAGE_REVISION=${{ needs.setup.outputs.commit_sha }} | |
| IMAGE_CREATED=${{ needs.setup.outputs.build_date }} | |
| CI_COMMIT_REF_NAME=${{ needs.setup.outputs.clean_ref }} | |
| unit-test: | |
| name: unit-test | |
| if: | | |
| (github.event_name == 'pull_request') || | |
| (github.event_name == 'push') || | |
| (github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-latest | |
| needs: | |
| - setup | |
| - build | |
| container: | |
| image: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} | |
| steps: | |
| - name: Run unit tests | |
| run: | | |
| echo "TODO: add unit tests here" | |
| # run container security scan using Trivy | |
| container-scanning: | |
| if: | | |
| (github.event_name == 'pull_request') || | |
| (github.event_name == 'push') || | |
| (github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-latest | |
| needs: | |
| - setup | |
| - build | |
| steps: | |
| - name: Install Trivy | |
| uses: aquasecurity/setup-trivy@v0.2.2 | |
| with: | |
| cache: true | |
| version: v0.68.2 | |
| - name: Trivy scan | |
| env: | |
| TMPDIR: /mnt/trivy-temp | |
| run: | | |
| sudo mkdir -p $TMPDIR | |
| sudo chown -R $USER:$USER $TMPDIR | |
| trivy image \ | |
| --format sarif \ | |
| --output trivy-results.sarif \ | |
| --severity CRITICAL,HIGH \ | |
| --scanners vuln \ | |
| --ignore-unfixed \ | |
| --timeout 45m \ | |
| ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} | |
| # promote Docker image tags after successful tests | |
| promote-tags: | |
| name: Promote Tags | |
| if: | | |
| (github.event_name == 'pull_request') || | |
| (github.event_name == 'push') || | |
| (github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-latest | |
| needs: | |
| - setup | |
| - codeql-scan | |
| - build | |
| - container-scanning | |
| steps: | |
| - name: Tag image with alias and latest | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # ensure skopeo is available for promotion | |
| if ! command -v skopeo >/dev/null 2>&1; then | |
| sudo apt-get update -y | |
| sudo apt-get install -y --no-install-recommends skopeo | |
| fi | |
| IMAGE_BASE="${{ needs.setup.outputs.image_base }}" | |
| TEST_TAG="${{ needs.setup.outputs.test_image_tag }}" | |
| ALIAS_TAG="${{ needs.setup.outputs.alias_tag }}" | |
| # apply the primary alias (pr tag or timestamp-branch) | |
| skopeo copy \ | |
| --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --all \ | |
| "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:${ALIAS_TAG}" | |
| # tag with 'latest' on development branch push | |
| if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF_NAME" = "development" ]; then | |
| skopeo copy \ | |
| --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --all \ | |
| "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:latest" | |
| fi |