Add async ngen execution via new context manager class ForecastExecutionManager
#3
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD Pipeline | |
| on: | |
| pull_request: | |
| branches: [main, nwm-main, development, release-candidate] | |
| push: | |
| branches: [main, nwm-main, development, release-candidate] | |
| release: | |
| types: [published] | |
| permissions: | |
| contents: read | |
| packages: write | |
| security-events: write | |
| env: | |
| REGISTRY: ghcr.io | |
| PYTHON_VERSION: '3.11' | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| image_base: ${{ steps.vars.outputs.image_base }} | |
| pr_tag: ${{ steps.vars.outputs.pr_tag }} | |
| commit_sha: ${{ steps.vars.outputs.commit_sha }} | |
| commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }} | |
| test_image_tag: ${{ steps.vars.outputs.test_image_tag }} | |
| steps: | |
| - name: Compute image vars | |
| id: vars | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" | |
| REPO="$(basename "${GITHUB_REPOSITORY}")" | |
| IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}" | |
| echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT" | |
| if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then | |
| PR_NUM="${{ github.event.pull_request.number }}" | |
| PR_TAG="pr-${PR_NUM}-build" | |
| echo "pr_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" | |
| echo "test_image_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" | |
| fi | |
| if [ "${GITHUB_EVENT_NAME}" = "push" ]; then | |
| COMMIT_SHA="${GITHUB_SHA}" | |
| SHORT_SHA="${COMMIT_SHA:0:12}" | |
| echo "commit_sha=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" | |
| echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT" | |
| echo "test_image_tag=${SHORT_SHA}" >> "$GITHUB_OUTPUT" | |
| fi | |
| build: | |
| name: build | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Log in to registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build & push image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} | |
| build-args: | | |
| NGEN_IMAGE_TAG=${{ env.NGEN_IMAGE_TAG || 'latest' }} | |
| CI_COMMIT_REF_NAME=${{ github.ref_name }} | |
| unit-test: | |
| name: unit-test | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| needs: [setup, build] | |
| container: | |
| image: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} | |
| steps: | |
| - name: Run unit tests | |
| run: | | |
| echo "TODO: add unit tests here" | |
| codeql-scan: | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| needs: [setup, build] | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Initialize CodeQL | |
| uses: github/codeql-action/init@v3 | |
| with: | |
| languages: python | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@v3 | |
| container-scanning: | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| needs: [setup, build] | |
| steps: | |
| - name: Scan container with Trivy | |
| uses: aquasecurity/trivy-action@0.20.0 | |
| with: | |
| image-ref: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} | |
| format: 'template' | |
| template: '@/contrib/sarif.tpl' | |
| output: 'trivy-results.sarif' | |
| severity: 'CRITICAL,HIGH' | |
| - name: Upload Trivy SARIF | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: 'trivy-results.sarif' | |
| deploy-latest-on-development: | |
| name: deploy-latest-on-development | |
| if: github.event_name == 'push' && github.ref_name == 'development' | |
| runs-on: ubuntu-latest | |
| needs: [setup, build, unit-test, codeql-scan, container-scanning] | |
| steps: | |
| - name: Tag image with 'latest' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| IMAGE_BASE="${{ needs.setup.outputs.image_base }}" | |
| SHORT_SHA="${{ needs.setup.outputs.commit_sha_short }}" | |
| # ensure skopeo is available | |
| if ! command -v skopeo >/dev/null 2>&1; then | |
| sudo apt-get update -y | |
| sudo apt-get install -y --no-install-recommends skopeo | |
| fi | |
| skopeo copy \ | |
| --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:latest" | |
| release: | |
| name: release | |
| if: github.event_name == 'release' && github.event.action == 'published' | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - name: Get commit sha for the tag | |
| id: rev | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| TAG="${{ github.event.release.tag_name }}" | |
| REPO="${{ github.repository }}" | |
| # ensure jq is available | |
| if ! command -v jq >/dev/null 2>&1; then | |
| sudo apt-get update -y | |
| sudo apt-get install -y --no-install-recommends jq | |
| fi | |
| # ensure gh cli is available | |
| if ! command -v gh >/dev/null 2>&1; then | |
| sudo apt-get update -y | |
| sudo apt-get install -y --no-install-recommends gh | |
| fi | |
| REF_JSON="$(gh api "repos/${REPO}/git/refs/tags/${TAG}")" | |
| OBJ_SHA="$(jq -r '.object.sha' <<<"$REF_JSON")" | |
| OBJ_TYPE="$(jq -r '.object.type' <<<"$REF_JSON")" | |
| if [ "$OBJ_TYPE" = "tag" ]; then | |
| TAG_OBJ="$(gh api "repos/${REPO}/git/tags/${OBJ_SHA}")" | |
| COMMIT_SHA="$(jq -r '.object.sha' <<<"$TAG_OBJ")" | |
| else | |
| COMMIT_SHA="$OBJ_SHA" | |
| fi | |
| SHORT_SHA="${COMMIT_SHA:0:12}" | |
| echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT" | |
| - name: Tag image with release tag | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| IMAGE_BASE="${{ needs.setup.outputs.image_base }}" | |
| SHORT_SHA="${{ steps.rev.outputs.short_sha }}" | |
| RELEASE_TAG="${{ github.event.release.tag_name }}" | |
| # ensure skopeo is available | |
| if ! command -v skopeo >/dev/null 2>&1; then | |
| sudo apt-get update -y | |
| sudo apt-get install -y --no-install-recommends skopeo | |
| fi | |
| skopeo copy \ | |
| --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:${RELEASE_TAG}" |