diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index cc609ed..cbb0ec4 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -2,11 +2,27 @@ name: CI/CD Pipeline on: pull_request: - branches: [ngwpc-candidate, ngwpc-release, main, nwm-main, development, release-candidate] + branches: + - ngwpc-candidate + - ngwpc-release + - main + - nwm-main + - development + - release-candidate push: - branches: [ngwpc-candidate, ngwpc-release, main, nwm-main, development, release-candidate] - release: - types: [published] + branches: + - ngwpc-candidate + - ngwpc-release + - main + - nwm-main + - development + - release-candidate + workflow_dispatch: + inputs: + NGEN_IMAGE_TAG: + description: 'NGEN_IMAGE_TAG' + required: false + type: string permissions: contents: read @@ -18,47 +34,122 @@ env: PYTHON_VERSION: '3.11' jobs: + # set variables for use in other jobs setup: + name: setup runs-on: ubuntu-latest outputs: + org: ${{ steps.vars.outputs.org }} image_base: ${{ steps.vars.outputs.image_base }} pr_tag: ${{ steps.vars.outputs.pr_tag }} commit_sha: ${{ steps.vars.outputs.commit_sha }} commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }} test_image_tag: ${{ steps.vars.outputs.test_image_tag }} + alias_tag: ${{ steps.vars.outputs.alias_tag }} + build_date: ${{ steps.vars.outputs.build_date }} + clean_ref: ${{ steps.vars.outputs.clean_ref }} steps: - name: Compute image vars id: vars shell: bash run: | set -euo pipefail + + # set variables to use with Docker images ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" REPO="$(basename "${GITHUB_REPOSITORY}")" IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}" - echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT" - if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then - PR_NUM="${{ github.event.pull_request.number }}" - PR_TAG="pr-${PR_NUM}-build" - echo "pr_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" - echo "test_image_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" + # one datetime for all time variables + NOW=$(date -u +'%Y-%m-%d %H:%M:%S') + + # for OCI labels + BUILD_DATE=$(date -u -d "$NOW" +'%Y-%m-%dT%H:%M:%SZ') + + # for Docker image tags + TIMESTAMP=$(date -u -d "$NOW" +'%Y%m%d%H%M%SZ') + + # logic to get the real branch name and commit SHA on pull requests + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + REAL_REF="${{ github.head_ref }}" + REAL_SHA="${{ github.event.pull_request.head.sha }}" + else + REAL_REF="${{ github.ref_name }}" + REAL_SHA="${GITHUB_SHA}" fi - if [ "${GITHUB_EVENT_NAME}" = "push" ]; then - COMMIT_SHA="${GITHUB_SHA}" - SHORT_SHA="${COMMIT_SHA:0:12}" - echo "commit_sha=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" - echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT" - echo "test_image_tag=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + # clean ref name and short commit sha + CLEAN_REF=$(echo "$REAL_REF" | tr '[:upper:]' '[:lower:]' | sed 's/\//-/g') + SHORT_SHA="${REAL_SHA:0:7}" + + # logic for the tags: + # test_image_tag (commit short sha): used for the initial build and test + # alias_tag: used for final tagging on successful tests + + # test tag is always commit short sha + TEST_TAG="${SHORT_SHA}" + + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + # for pull requests, use pr--build + ALIAS="pr-${{ github.event.pull_request.number }}-build" + elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] && [ "${{ github.ref_type }}" = "tag" ]; then + # for manual workflow dispatch on tags, use the git tag + ALIAS="${CLEAN_REF}" + else + # for pushes to branches, use timestamp-branchname + ALIAS="${TIMESTAMP}-${CLEAN_REF}" fi + # save outputs + echo "org=${ORG}" >> "$GITHUB_OUTPUT" + echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT" + echo "build_date=${BUILD_DATE}" >> "$GITHUB_OUTPUT" + echo "test_image_tag=${TEST_TAG}" >> "$GITHUB_OUTPUT" + echo "alias_tag=${ALIAS}" >> "$GITHUB_OUTPUT" + echo "commit_sha=${REAL_SHA}" >> "$GITHUB_OUTPUT" + echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + echo "clean_ref=${CLEAN_REF}" >> "$GITHUB_OUTPUT" + + # CodeQL scan + codeql-scan: + name: codeql-scan + if: | + (github.event_name == 'pull_request') || + (github.event_name == 'push') || + (github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + needs: setup + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: actions/checkout@v6 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: python + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + build: name: build - if: github.event_name == 'pull_request' || github.event_name == 'push' + if: | + (github.event_name == 'pull_request') || + (github.event_name == 'push') || + (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest needs: setup steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Log in to registry uses: docker/login-action@v3 @@ -74,14 +165,25 @@ jobs: push: true tags: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} build-args: | - NGEN_IMAGE_TAG=${{ env.NGEN_IMAGE_TAG || 'latest' }} - CI_COMMIT_REF_NAME=${{ github.ref_name }} + ORG=${{ needs.setup.outputs.org }} + NGEN_IMAGE_TAG=${{ inputs.NGEN_IMAGE_TAG || 'latest' }} + IMAGE_SOURCE=https://github.com/${{ github.repository }} + IMAGE_VENDOR=${{ github.repository_owner }} + IMAGE_VERSION=${{ needs.setup.outputs.clean_ref }} + IMAGE_REVISION=${{ needs.setup.outputs.commit_sha }} + IMAGE_CREATED=${{ needs.setup.outputs.build_date }} + CI_COMMIT_REF_NAME=${{ needs.setup.outputs.clean_ref }} unit-test: name: unit-test - if: github.event_name == 'pull_request' || github.event_name == 'push' + if: | + (github.event_name == 'pull_request') || + (github.event_name == 'push') || + (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest - needs: [setup, build] + needs: + - setup + - build container: image: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} steps: @@ -89,130 +191,80 @@ jobs: run: | echo "TODO: add unit tests here" - codeql-scan: - if: github.event_name == 'pull_request' || github.event_name == 'push' - runs-on: ubuntu-latest - needs: [setup, build] - permissions: - actions: read - contents: read - security-events: write - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: python - - name: Install dependencies - run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 - +# run container security scan using Trivy container-scanning: - if: github.event_name == 'pull_request' || github.event_name == 'push' + if: | + (github.event_name == 'pull_request') || + (github.event_name == 'push') || + (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest - needs: [setup, build] + needs: + - setup + - build steps: - - name: Scan container with Trivy - uses: aquasecurity/trivy-action@0.20.0 - with: - image-ref: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} - format: 'template' - template: '@/contrib/sarif.tpl' - output: 'trivy-results.sarif' - severity: 'CRITICAL,HIGH' - - name: Upload Trivy SARIF - uses: github/codeql-action/upload-sarif@v3 + - name: Install Trivy + uses: aquasecurity/setup-trivy@v0.2.2 with: - sarif_file: 'trivy-results.sarif' + cache: true + version: v0.68.2 - deploy-latest-on-development: - name: deploy-latest-on-development - if: github.event_name == 'push' && github.ref_name == 'development' - runs-on: ubuntu-latest - needs: [setup, build, unit-test, codeql-scan, container-scanning] - steps: - - name: Tag image with 'latest' - shell: bash + - name: Trivy scan + env: + TMPDIR: /mnt/trivy-temp run: | - set -euo pipefail - IMAGE_BASE="${{ needs.setup.outputs.image_base }}" - SHORT_SHA="${{ needs.setup.outputs.commit_sha_short }}" + sudo mkdir -p $TMPDIR + sudo chown -R $USER:$USER $TMPDIR - # ensure skopeo is available - if ! command -v skopeo >/dev/null 2>&1; then - sudo apt-get update -y - sudo apt-get install -y --no-install-recommends skopeo - fi - - skopeo copy \ - --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ - --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ - docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:latest" + trivy image \ + --format sarif \ + --output trivy-results.sarif \ + --severity CRITICAL,HIGH \ + --scanners vuln \ + --ignore-unfixed \ + --timeout 45m \ + ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} - release: - name: release - if: github.event_name == 'release' && github.event.action == 'published' + # promote Docker image tags after successful tests + promote-tags: + name: Promote Tags + if: | + (github.event_name == 'pull_request') || + (github.event_name == 'push') || + (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest - needs: setup + needs: + - setup + - codeql-scan + - build + - container-scanning steps: - - name: Get commit sha for the tag - id: rev - shell: bash - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - TAG="${{ github.event.release.tag_name }}" - REPO="${{ github.repository }}" - - # ensure jq is available - if ! command -v jq >/dev/null 2>&1; then - sudo apt-get update -y - sudo apt-get install -y --no-install-recommends jq - fi - - # ensure gh cli is available - if ! command -v gh >/dev/null 2>&1; then - sudo apt-get update -y - sudo apt-get install -y --no-install-recommends gh - fi - - REF_JSON="$(gh api "repos/${REPO}/git/refs/tags/${TAG}")" - OBJ_SHA="$(jq -r '.object.sha' <<<"$REF_JSON")" - OBJ_TYPE="$(jq -r '.object.type' <<<"$REF_JSON")" - - if [ "$OBJ_TYPE" = "tag" ]; then - TAG_OBJ="$(gh api "repos/${REPO}/git/tags/${OBJ_SHA}")" - COMMIT_SHA="$(jq -r '.object.sha' <<<"$TAG_OBJ")" - else - COMMIT_SHA="$OBJ_SHA" - fi - - SHORT_SHA="${COMMIT_SHA:0:12}" - echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT" - - - name: Tag image with release tag + - name: Tag image with alias and latest shell: bash run: | set -euo pipefail - IMAGE_BASE="${{ needs.setup.outputs.image_base }}" - SHORT_SHA="${{ steps.rev.outputs.short_sha }}" - RELEASE_TAG="${{ github.event.release.tag_name }}" - # ensure skopeo is available + # ensure skopeo is available for promotion if ! command -v skopeo >/dev/null 2>&1; then sudo apt-get update -y sudo apt-get install -y --no-install-recommends skopeo fi + IMAGE_BASE="${{ needs.setup.outputs.image_base }}" + TEST_TAG="${{ needs.setup.outputs.test_image_tag }}" + ALIAS_TAG="${{ needs.setup.outputs.alias_tag }}" + + # apply the primary alias (pr tag or timestamp-branch) skopeo copy \ --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ - docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:${RELEASE_TAG}" + --all \ + "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:${ALIAS_TAG}" + + # tag with 'latest' on development branch push + if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF_NAME" = "development" ]; then + skopeo copy \ + --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ + --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ + --all \ + "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:latest" + fi diff --git a/Dockerfile b/Dockerfile index b224149..21553b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,33 @@ # syntax=docker/dockerfile:1.4 -ARG NGEN_IMAGE_TAG=latest -FROM ghcr.io/ngwpc/ngen:${NGEN_IMAGE_TAG} +ARG ORG=ngwpc +ARG NGEN_IMAGE_TAG=latest +ARG NGEN_IMAGE=ghcr.io/ngwpc/ngen:${NGEN_IMAGE_TAG} +FROM ${NGEN_IMAGE} # Uncomment when building ngen locally or if ngen-int image is available locally # modify to use image tag for local ngen image if needed #FROM ngen +# OCI Metadata Arguments +ARG NGEN_IMAGE +ARG BASE_IMAGE_DIGEST="unknown" +ARG BASE_IMAGE_REVISION="unknown" +ARG IMAGE_SOURCE="unknown" +ARG IMAGE_VENDOR="unknown" +ARG IMAGE_VERSION="unknown" +ARG IMAGE_REVISION="unknown" +ARG IMAGE_CREATED="unknown" + +# OCI Standard Labels +LABEL org.opencontainers.image.base.name="${NGEN_IMAGE}" \ + org.opencontainers.image.base.digest="${BASE_IMAGE_DIGEST}" \ + io.ngwpc.image.base.revision="${BASE_IMAGE_REVISION}" \ + org.opencontainers.image.source="${IMAGE_SOURCE}" \ + org.opencontainers.image.vendor="${IMAGE_VENDOR}" \ + org.opencontainers.image.version="${IMAGE_VERSION}" \ + org.opencontainers.image.revision="${IMAGE_REVISION}" \ + org.opencontainers.image.created="${IMAGE_CREATED}" + # Activate the existing virtual environment ENV PATH="/ngen-app/ngen-python/bin:${PATH}"