Add hindcast orchestration logger to hindcasting workflow #219

Workflow file for this run

	name: CI/CD Pipeline

	on:
	pull_request:
	branches:
	- ngwpc-candidate
	- ngwpc-release
	- main
	- nwm-main
	- development
	- release-candidate
	push:
	branches:
	- ngwpc-candidate
	- ngwpc-release
	- main
	- nwm-main
	- development
	- release-candidate
	workflow_dispatch:
	inputs:
	GHCR_ORG:
	description: 'GHCR_ORG'
	required: false
	type: string
	NGEN_IMAGE_TAG:
	description: 'NGEN_IMAGE_TAG'
	required: false
	type: string
	EWTS_ORG:
	description: 'EWTS_ORG'
	required: false
	type: string
	EWTS_REF:
	description: 'EWTS_REF'
	required: false
	type: string
	MSW_MGR_ORG:
	description: 'MSW_MGR_ORG'
	required: false
	type: string
	MSW_MGR_REF:
	description: 'MSW_MGR_REF'
	required: false
	type: string

	permissions:
	contents: read
	packages: write
	security-events: write

	env:
	REGISTRY: ghcr.io
	PYTHON_VERSION: '3.11'

	jobs:
	# set variables for use in other jobs
	setup:
	name: setup
	runs-on: ubuntu-latest
	outputs:
	org: ${{ steps.vars.outputs.org }}
	image_base: ${{ steps.vars.outputs.image_base }}
	pr_tag: ${{ steps.vars.outputs.pr_tag }}
	commit_sha: ${{ steps.vars.outputs.commit_sha }}
	commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }}
	test_image_tag: ${{ steps.vars.outputs.test_image_tag }}
	alias_tag: ${{ steps.vars.outputs.alias_tag }}
	clean_ref: ${{ steps.vars.outputs.clean_ref }}
	default_ref: ${{ steps.vars.outputs.default_ref }}
	ngen_image_digest: ${{ steps.vars.outputs.ngen_image_digest }}
	ngen_image_revision: ${{ steps.vars.outputs.ngen_image_revision }}
	ewts_revision: ${{ steps.vars.outputs.ewts_revision }}
	msw_mgr_revision: ${{ steps.vars.outputs.msw_mgr_revision }}
	steps:
	- name: Compute image vars
	id: vars
	shell: bash
	run: \|
	set -euo pipefail

	# set variables to use with Docker images
	ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" \| tr '[:upper:]' '[:lower:]')"
	REPO="$(basename "${GITHUB_REPOSITORY}")"
	IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}"

	# one datetime for all time variables
	NOW=$(date -u +'%Y-%m-%d %H:%M:%S')

	# for Docker image tags
	TIMESTAMP=$(date -u -d "$NOW" +'%Y%m%d%H%M%SZ')

	# logic to get the real branch name and commit SHA on pull requests
	if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
	REAL_REF="${{ github.head_ref }}"
	REAL_SHA="${{ github.event.pull_request.head.sha }}"
	else
	REAL_REF="${{ github.ref_name }}"
	REAL_SHA="${GITHUB_SHA}"
	fi

	# clean ref name and short commit sha
	CLEAN_REF=$(echo "$REAL_REF" \| tr '[:upper:]' '[:lower:]' \| sed 's/\//-/g')
	SHORT_SHA="${REAL_SHA:0:7}"

	# default source-repo ref: follow the release-line branch we're building from
	# (development / ngwpc-candidate / ngwpc-release); tags and other branches fall
	# back to development. Override per source repo via the *_REF dispatch inputs.
	case "${GITHUB_REF_TYPE}:${GITHUB_REF_NAME}" in
	branch:development\|branch:ngwpc-candidate\|branch:ngwpc-release)
	DEFAULT_REF="$GITHUB_REF_NAME" ;;
	*)
	DEFAULT_REF="development" ;;
	esac
	# use an explicit *_REF input if provided, else DEFAULT_REF
	ref_or_default() { [ -n "$1" ] && echo "$1" \|\| echo "$DEFAULT_REF"; }

	# logic for the tags:
	# test_image_tag (commit short sha): used for the initial build and test
	# alias_tag: used for final tagging on successful tests

	# test tag is always commit short sha
	TEST_TAG="${SHORT_SHA}"

	if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
	# for pull requests, use pr-<pr number>-build
	ALIAS="pr-${{ github.event.pull_request.number }}-build"
	elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] && [ "${{ github.ref_type }}" = "tag" ]; then
	# for manual workflow dispatch on tags, use the git tag
	ALIAS="${CLEAN_REF}"
	else
	# for pushes to branches, use timestamp-branchname
	ALIAS="${TIMESTAMP}-${CLEAN_REF}"
	fi

	# skopeo is needed to inspect the base image
	if ! command -v skopeo >/dev/null 2>&1; then
	sudo apt-get update -y
	sudo apt-get install -y --no-install-recommends skopeo
	fi

	# base image (ngen) metadata for Dockerfile labels
	NGEN_IMAGE_TAG="${{ inputs.NGEN_IMAGE_TAG \|\| 'latest' }}"
	NGEN_IMAGE="ghcr.io/${ORG}/ngen:${NGEN_IMAGE_TAG}"
	NGEN_INSPECT=$(skopeo inspect --override-os linux --override-arch amd64 "docker://${NGEN_IMAGE}" 2>/dev/null \|\| echo '{}')
	NGEN_IMAGE_DIGEST=$(echo "$NGEN_INSPECT" \| jq -r '.Digest // "unknown"')
	NGEN_IMAGE_REVISION=$(echo "$NGEN_INSPECT" \| jq -r '.Labels["org.opencontainers.image.revision"] // "unknown"')

	# resolve each source repo's ref (branch/tag/SHA) to its commit SHA for revision labels
	resolve_sha() {
	local url="$1" ref="$2" out sha
	# a full 40-char SHA can't be looked up via ls-remote; use it directly
	if [[ "$ref" =~ ^[0-9a-f]{40}$ ]]; then echo "$ref"; return; fi
	out=$(git ls-remote "$url" "$ref" "refs/tags/${ref}^{}" 2>/dev/null)
	# prefer the dereferenced commit (^{}) for annotated tags; else first match
	sha=$(echo "$out" \| grep '\^{}$' \| head -n1 \| cut -f1)
	[ -z "$sha" ] && sha=$(echo "$out" \| head -n1 \| cut -f1)
	echo "${sha:-unknown}"
	}

	EWTS_REVISION=$(resolve_sha "https://github.com/${{ inputs.EWTS_ORG \|\| github.repository_owner }}/nwm-ewts.git" "$(ref_or_default "${{ inputs.EWTS_REF }}")")
	MSW_MGR_REVISION=$(resolve_sha "https://github.com/${{ inputs.MSW_MGR_ORG \|\| github.repository_owner }}/nwm-msw-mgr.git" "$(ref_or_default "${{ inputs.MSW_MGR_REF }}")")

	# save outputs
	cat >> "$GITHUB_OUTPUT" <<EOF
	org=${ORG}
	image_base=${IMAGE_BASE}
	test_image_tag=${TEST_TAG}
	alias_tag=${ALIAS}
	commit_sha=${REAL_SHA}
	commit_sha_short=${SHORT_SHA}
	clean_ref=${CLEAN_REF}
	default_ref=${DEFAULT_REF}
	ngen_image_digest=${NGEN_IMAGE_DIGEST}
	ngen_image_revision=${NGEN_IMAGE_REVISION}
	ewts_revision=${EWTS_REVISION}
	msw_mgr_revision=${MSW_MGR_REVISION}
	EOF

	# CodeQL scan
	codeql-scan:
	name: codeql-scan
	if: \|
	(github.event_name == 'pull_request') \|\|
	(github.event_name == 'push') \|\|
	(github.event_name == 'workflow_dispatch')
	runs-on: ubuntu-latest
	needs: setup
	permissions:
	actions: read
	contents: read
	security-events: write
	steps:
	- uses: actions/checkout@v6
	- name: Set up Python
	uses: actions/setup-python@v6
	with:
	python-version: ${{ env.PYTHON_VERSION }}
	- name: Initialize CodeQL
	uses: github/codeql-action/init@v4
	with:
	languages: python
	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
	- name: Perform CodeQL Analysis
	uses: github/codeql-action/analyze@v4

	build:
	name: build
	if: \|
	(github.event_name == 'pull_request') \|\|
	(github.event_name == 'push') \|\|
	(github.event_name == 'workflow_dispatch')
	runs-on: ubuntu-latest
	needs: setup
	steps:
	- uses: actions/checkout@v6

	- name: Log in to registry
	uses: docker/login-action@v4
	with:
	registry: ${{ env.REGISTRY }}
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Build & push image
	uses: docker/build-push-action@v7
	with:
	context: .
	push: true
	tags: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
	build-args: \|
	GHCR_ORG=${{ inputs.GHCR_ORG \|\| needs.setup.outputs.org }}
	NGEN_IMAGE_TAG=${{ inputs.NGEN_IMAGE_TAG \|\| 'latest' }}
	BASE_IMAGE_DIGEST=${{ needs.setup.outputs.ngen_image_digest }}
	BASE_IMAGE_REVISION=${{ needs.setup.outputs.ngen_image_revision }}
	EWTS_ORG=${{ inputs.EWTS_ORG \|\| github.repository_owner }}
	EWTS_REF=${{ inputs.EWTS_REF \|\| needs.setup.outputs.default_ref }}
	EWTS_REVISION=${{ needs.setup.outputs.ewts_revision }}
	EWTS_CACHE_BUST=${{ needs.setup.outputs.ewts_revision }}
	MSW_MGR_ORG=${{ inputs.MSW_MGR_ORG \|\| github.repository_owner }}
	MSW_MGR_REF=${{ inputs.MSW_MGR_REF \|\| needs.setup.outputs.default_ref }}
	MSW_MGR_REVISION=${{ needs.setup.outputs.msw_mgr_revision }}
	MSW_MGR_CACHE_BUST=${{ needs.setup.outputs.msw_mgr_revision }}
	IMAGE_SOURCE=https://github.com/${{ github.repository }}
	IMAGE_VENDOR=${{ github.repository_owner }}
	IMAGE_VERSION=${{ needs.setup.outputs.clean_ref }}
	IMAGE_REVISION=${{ needs.setup.outputs.commit_sha }}
	CI_COMMIT_REF_NAME=${{ needs.setup.outputs.clean_ref }}

	# run unit tests inside the built Docker image
	unit-test:
	name: unit-test
	if: \|
	(github.event_name == 'pull_request') \|\|
	(github.event_name == 'push') \|\|
	(github.event_name == 'workflow_dispatch')
	runs-on: ubuntu-latest
	needs:
	- setup
	- build
	container:
	image: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
	steps:
	- name: Run unit tests
	run: \|
	echo "TODO: add unit tests here"

	# run container security scan using Trivy
	container-scanning:
	if: \|
	(github.event_name == 'pull_request') \|\|
	(github.event_name == 'push') \|\|
	(github.event_name == 'workflow_dispatch')
	runs-on: ubuntu-latest
	needs:
	- setup
	- build
	steps:
	- name: Install Trivy
	uses: aquasecurity/setup-trivy@v0.2.6
	with:
	cache: true
	version: v0.69.3

	- name: Trivy scan
	env:
	TMPDIR: /mnt/trivy-temp
	run: \|
	sudo mkdir -p $TMPDIR
	sudo chown -R $USER:$USER $TMPDIR

	trivy image \
	--format sarif \
	--output trivy-results.sarif \
	--severity CRITICAL,HIGH \
	--scanners vuln \
	--ignore-unfixed \
	--timeout 45m \
	${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}

	# promote Docker image tags after successful tests
	promote-tags:
	name: Promote Tags
	if: \|
	(github.event_name == 'pull_request') \|\|
	(github.event_name == 'push') \|\|
	(github.event_name == 'workflow_dispatch')
	runs-on: ubuntu-latest
	needs:
	- setup
	- codeql-scan
	- build
	- unit-test
	- container-scanning
	steps:
	- name: Tag image with alias and latest
	shell: bash
	run: \|
	set -euo pipefail

	# ensure skopeo is available for promotion
	if ! command -v skopeo >/dev/null 2>&1; then
	sudo apt-get update -y
	sudo apt-get install -y --no-install-recommends skopeo
	fi

	IMAGE_BASE="${{ needs.setup.outputs.image_base }}"
	TEST_TAG="${{ needs.setup.outputs.test_image_tag }}"
	ALIAS_TAG="${{ needs.setup.outputs.alias_tag }}"

	# apply the primary alias (pr tag or timestamp-branch)
	skopeo copy \
	--src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
	--dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
	--all \
	"docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:${ALIAS_TAG}"

	# tag with 'latest' on development branch
	if [ "$GITHUB_REF_NAME" = "development" ]; then
	skopeo copy \
	--src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
	--dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
	--all \
	"docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:latest"
	fi

	# Build the Apptainer .sif from the tested + promoted image and publish it to
	# GHCR as an OCI artifact (oras://). AWS PCS compute nodes pull this onto the
	# shared EFS and run it with `apptainer run`. SIF images are NOT baked into the
	# compute AMI — the AMI carries only the Apptainer runtime; the images ship here
	# and are staged on EFS. See nwm-ngencerf-infra-learning/docs/design-decisions.md.
	build-sif:
	name: build-sif
	# Only on real branch builds (push) or manual dispatch — not PRs (a multi-GB
	# SIF build on every PR is wasteful). Runs after the image is tested + promoted.
	if: \|
	(github.event_name == 'push') \|\|
	(github.event_name == 'workflow_dispatch')
	runs-on: ubuntu-latest
	needs:
	- setup
	- promote-tags
	env:
	# Keep Apptainer's build temp + layer cache off the small root volume; /mnt
	# is the roomy ephemeral disk on GitHub-hosted runners (a SIF needs room for
	# the uncompressed image during mksquashfs).
	APPTAINER_TMPDIR: /mnt/apptainer-tmp
	APPTAINER_CACHEDIR: /mnt/apptainer-cache
	steps:
	- name: Install Apptainer
	uses: eWaterCycle/setup-apptainer@v2
	with:
	apptainer-version: 1.3.6

	- name: Log in to GHCR
	uses: docker/login-action@v4
	with:
	registry: ${{ env.REGISTRY }}
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Build .sif from the promoted image and push to GHCR (oras)
	shell: bash
	run: \|
	set -euo pipefail
	sudo mkdir -p "$APPTAINER_TMPDIR" "$APPTAINER_CACHEDIR"
	sudo chown -R "$USER":"$USER" "$APPTAINER_TMPDIR" "$APPTAINER_CACHEDIR"

	IMAGE_BASE="${{ needs.setup.outputs.image_base }}" # ghcr.io/<org>/nwm-fcst-mgr
	ALIAS_TAG="${{ needs.setup.outputs.alias_tag }}" # the tested + promoted tag
	SIF_REPO="${IMAGE_BASE}-sif" # ghcr.io/<org>/nwm-fcst-mgr-sif

	# Convert the promoted OCI image to a SIF. docker/login-action wrote
	# ~/.docker/config.json, which Apptainer reads for the docker:// pull and
	# the oras:// push. Unprivileged build uses --fakeroot automatically
	# (GitHub-hosted runners have unprivileged user namespaces enabled).
	apptainer build --force nwm-fcst-mgr.sif "docker://${IMAGE_BASE}:${ALIAS_TAG}"

	# Publish as an OCI artifact. A SIF pushed via oras:// MUST be pulled via
	# oras:// (not docker://). The in-VPC staging step pulls this onto EFS.
	apptainer push nwm-fcst-mgr.sif "oras://${SIF_REPO}:${ALIAS_TAG}"
	echo "Published oras://${SIF_REPO}:${ALIAS_TAG}"

Navigation Menu

Uh oh!

Add hindcast orchestration logger to hindcasting workflow #219

Workflow file

Add hindcast orchestration logger to hindcasting workflow #219

Uh oh!

Workflow file for this run