fix: build cleanup and terminate on failure #62
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Pipeline | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| types: [opened, synchronize, reopened] | |
| push: | |
| branches: [main] | |
| schedule: | |
| - cron: "0 12 * * 1" | |
| permissions: | |
| id-token: write | |
| contents: read | |
| packages: write | |
| env: | |
| AWS_REGION: us-east-1 | |
| DEFAULT_INSTANCE_TYPE: g5.xlarge | |
| DOCKER_REPOSITORY: ghcr.io/omsf-eco-infra/omsf | |
| jobs: | |
| # ── Step 1: Prepare shared build inputs ──────────────────────────── | |
| lock-environments: | |
| name: Prepare build context | |
| runs-on: ubuntu-latest | |
| outputs: | |
| environment_names: ${{ steps.context.outputs.environment_names }} | |
| selected_environment_names: ${{ steps.context.outputs.selected_environment_names }} | |
| environment_matrix: ${{ steps.context.outputs.environment_matrix }} | |
| packer_environments: ${{ steps.context.outputs.packer_environments }} | |
| image_name: ${{ steps.context.outputs.image_name }} | |
| default_environment: ${{ steps.context.outputs.default_environment }} | |
| build_kind: ${{ steps.context.outputs.build_kind }} | |
| ami_name_suffix: ${{ steps.context.outputs.ami_name_suffix }} | |
| build_timestamp: ${{ steps.context.outputs.build_timestamp }} | |
| published_date: ${{ steps.context.outputs.published_date }} | |
| published_timestamp: ${{ steps.context.outputs.published_timestamp }} | |
| published_name: ${{ steps.context.outputs.published_name }} | |
| additional_tags: ${{ steps.context.outputs.additional_tags }} | |
| aws_region: ${{ steps.context.outputs.aws_region }} | |
| default_instance_type: ${{ steps.context.outputs.default_instance_type }} | |
| docker_repository: ${{ steps.context.outputs.docker_repository }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Set up Pixi | |
| uses: prefix-dev/setup-pixi@v0.9.3 | |
| with: | |
| pixi-version: v0.63.0 | |
| cache: false | |
| - name: Generate lockfile | |
| run: | | |
| set -euo pipefail | |
| rm -f environments/pixi.lock | |
| pixi lock --manifest-path environments/pixi.toml | |
| - name: Upload lockfile | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pixi-lock | |
| path: environments/pixi.lock | |
| retention-days: 7 | |
| - name: Compute shared build context | |
| id: context | |
| shell: python | |
| run: | | |
| import datetime | |
| import importlib.util | |
| import json | |
| import os | |
| import sys | |
| import tomllib | |
| from pathlib import Path | |
| helper_path = Path("environments/pixi-environment-metadata.py") | |
| toml_path = Path("environments/pixi.toml") | |
| spec = importlib.util.spec_from_file_location("pixi_environment_metadata", helper_path) | |
| if spec is None or spec.loader is None: | |
| raise SystemExit(f"Failed to load metadata helper from {helper_path}") | |
| metadata_helper = importlib.util.module_from_spec(spec) | |
| sys.modules[spec.name] = metadata_helper | |
| spec.loader.exec_module(metadata_helper) | |
| metadata = metadata_helper.load_metadata(toml_path) | |
| env_names = metadata["selected_environments"] | |
| if not env_names: | |
| raise SystemExit("No environments found in pixi.toml") | |
| with toml_path.open("rb") as fp: | |
| manifest = tomllib.load(fp) | |
| workspace = manifest.get("workspace") | |
| if not isinstance(workspace, dict): | |
| raise SystemExit("Workspace metadata is missing from pixi.toml") | |
| platforms = workspace.get("platforms") | |
| if not isinstance(platforms, list) or not platforms: | |
| raise SystemExit("No workspace platforms found in pixi.toml") | |
| selected_site_environments = [] | |
| for env_name in env_names: | |
| selected_site_environments.extend((env_name, f"{env_name}-test")) | |
| packer_environments = json.dumps(env_names, separators=(",", ":")) | |
| environment_matrix = json.dumps(metadata["environment_matrix"], separators=(",", ":")) | |
| event_name = os.environ["GITHUB_EVENT_NAME"] | |
| with open(os.environ["GITHUB_EVENT_PATH"], encoding="utf-8") as fp: | |
| event = json.load(fp) | |
| now = datetime.datetime.now(datetime.UTC) | |
| build_timestamp = str(int(now.timestamp())) | |
| build_time = datetime.datetime.fromtimestamp(int(build_timestamp), datetime.UTC) | |
| today = build_time.strftime("%Y%m%d") | |
| published_date = today | |
| published_timestamp = build_time.strftime("%Y-%m-%dT%H:%M:%SZ") | |
| published_name = f"{metadata['image_name']}-{published_date}" | |
| full_sha = os.environ["GITHUB_SHA"] | |
| short_sha = full_sha[:7] | |
| suffix = f"{today}-{short_sha}" | |
| extra_tags = {} | |
| delete_after_date = now + datetime.timedelta(days=7) | |
| if event_name == "pull_request": | |
| build_kind = "pr" | |
| pr_number = event.get("number") | |
| head_sha = event.get("pull_request", {}).get("head", {}).get("sha", full_sha) | |
| short_sha = head_sha[:7] | |
| suffix = f"pr{pr_number}-{today}-{short_sha}" | |
| extra_tags["pr"] = str(pr_number) if pr_number is not None else "unknown" | |
| elif event_name in {"schedule", "workflow_dispatch", "push"}: | |
| build_kind = { | |
| "schedule": "scheduled", | |
| "workflow_dispatch": "manual", | |
| "push": "push", | |
| }[event_name] | |
| else: | |
| raise SystemExit(f"Unsupported event type for build metadata: {event_name}") | |
| tags = { | |
| "omsf-ami-type": build_kind, | |
| "commit": short_sha, | |
| "delete-after": delete_after_date.strftime("%Y-%m-%d"), | |
| } | |
| tags.update(extra_tags) | |
| print(f"Image name: {metadata['image_name']}") | |
| print(f"Default environment: {metadata['default_environment']}") | |
| print(f"Selected environments: {env_names}") | |
| print(f"Build kind: {build_kind}") | |
| print(f"Image suffix: {suffix}") | |
| print(f"Build timestamp: {build_timestamp}") | |
| site_export_matrix = [ | |
| { | |
| "environment": environment_name, | |
| "platform": platform, | |
| "slug": f"{environment_name}-{platform}", | |
| "conda_explicit_spec_path": ( | |
| f"public/artifacts/{published_name}/{environment_name}-{platform}/" | |
| f"{environment_name}-{platform}.conda-spec.txt" | |
| ), | |
| } | |
| for environment_name in selected_site_environments | |
| for platform in platforms | |
| ] | |
| build_context = { | |
| "image_name": metadata["image_name"], | |
| "default_environment": metadata["default_environment"], | |
| "selected_runtime_environments": env_names, | |
| "selected_site_environments": selected_site_environments, | |
| "platforms": platforms, | |
| "build_timestamp": build_timestamp, | |
| "published_date": published_date, | |
| "published_timestamp": published_timestamp, | |
| "published_name": published_name, | |
| "docker_repository": os.environ["DOCKER_REPOSITORY"], | |
| "site_export_matrix": site_export_matrix, | |
| } | |
| Path("build-context.json").write_text( | |
| json.dumps(build_context, indent=2) + "\n", | |
| encoding="utf-8", | |
| ) | |
| output_path = Path(os.environ["GITHUB_OUTPUT"]) | |
| with output_path.open("a", encoding="utf-8") as fh: | |
| fh.write(f"environment_names={packer_environments}\n") | |
| fh.write(f"selected_environment_names={packer_environments}\n") | |
| fh.write(f"environment_matrix={environment_matrix}\n") | |
| fh.write(f"packer_environments={packer_environments}\n") | |
| fh.write(f"image_name={metadata['image_name']}\n") | |
| fh.write(f"default_environment={metadata['default_environment']}\n") | |
| fh.write(f"build_kind={build_kind}\n") | |
| fh.write(f"ami_name_suffix={suffix}\n") | |
| fh.write(f"build_timestamp={build_timestamp}\n") | |
| fh.write(f"published_date={published_date}\n") | |
| fh.write(f"published_timestamp={published_timestamp}\n") | |
| fh.write(f"published_name={published_name}\n") | |
| fh.write(f"additional_tags={json.dumps(tags)}\n") | |
| fh.write(f"aws_region={os.environ['AWS_REGION']}\n") | |
| fh.write(f"default_instance_type={os.environ['DEFAULT_INSTANCE_TYPE']}\n") | |
| fh.write(f"docker_repository={os.environ['DOCKER_REPOSITORY']}\n") | |
| - name: Upload build context | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-context | |
| path: build-context.json | |
| retention-days: 7 | |
| # ── Step 2a: Build AMI ───────────────────────────────────────────── | |
| build-ami: | |
| name: Build AMI | |
| needs: lock-environments | |
| runs-on: ubuntu-latest | |
| outputs: | |
| ami_id: ${{ steps.collect.outputs.ami_id }} | |
| ami_name: ${{ steps.collect.outputs.ami_name }} | |
| aws_region: ${{ needs.lock-environments.outputs.aws_region }} | |
| default_instance_type: ${{ needs.lock-environments.outputs.default_instance_type }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Download lockfile | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: pixi-lock | |
| path: environments | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_ARN }} | |
| role-session-name: gha-ami-builder | |
| aws-region: ${{ needs.lock-environments.outputs.aws_region }} | |
| - name: Set up Packer | |
| uses: hashicorp/setup-packer@v3 | |
| with: | |
| version: latest | |
| - name: Build AMI | |
| env: | |
| IMAGE_NAME: ${{ needs.lock-environments.outputs.image_name }} | |
| DEFAULT_ENVIRONMENT: ${{ needs.lock-environments.outputs.default_environment }} | |
| PACKER_ENVIRONMENTS: ${{ needs.lock-environments.outputs.packer_environments }} | |
| AMI_NAME_SUFFIX: ${{ needs.lock-environments.outputs.ami_name_suffix }} | |
| ADDITIONAL_TAGS: ${{ needs.lock-environments.outputs.additional_tags }} | |
| run: | | |
| set -euo pipefail | |
| packer init build-ami.pkr.hcl | |
| packer build \ | |
| -var "ami_name=${IMAGE_NAME}" \ | |
| -var "default_environment=${DEFAULT_ENVIRONMENT}" \ | |
| -var "environments=${PACKER_ENVIRONMENTS}" \ | |
| -var "ami_name_suffix=${AMI_NAME_SUFFIX}" \ | |
| -var "additional_tags=${ADDITIONAL_TAGS}" \ | |
| build-ami.pkr.hcl | |
| shell: bash | |
| - name: Collect packer manifest outputs | |
| id: collect | |
| uses: ./.github/actions/collect-packer-metadata | |
| with: | |
| manifest_path: packer-manifest.json | |
| kind: ami | |
| # ── Step 2b: Build Docker image ──────────────────────────────────── | |
| build-docker: | |
| name: Build Docker Image | |
| needs: lock-environments | |
| runs-on: ubuntu-latest | |
| outputs: | |
| image_repo: ${{ steps.collect.outputs.image_repo }} | |
| image_tag: ${{ steps.collect.outputs.image_tag }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Download lockfile | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: pixi-lock | |
| path: environments | |
| - name: Set up Packer | |
| uses: hashicorp/setup-packer@v3 | |
| with: | |
| version: latest | |
| - name: Log in to GHCR | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Build Docker image | |
| env: | |
| IMAGE_NAME: ${{ needs.lock-environments.outputs.image_name }} | |
| DEFAULT_ENVIRONMENT: ${{ needs.lock-environments.outputs.default_environment }} | |
| PACKER_ENVIRONMENTS: ${{ needs.lock-environments.outputs.packer_environments }} | |
| AMI_NAME_SUFFIX: ${{ needs.lock-environments.outputs.ami_name_suffix }} | |
| BUILD_TIMESTAMP: ${{ needs.lock-environments.outputs.build_timestamp }} | |
| ADDITIONAL_TAGS: ${{ needs.lock-environments.outputs.additional_tags }} | |
| DOCKER_REPO: ${{ needs.lock-environments.outputs.docker_repository }} | |
| run: | | |
| set -euo pipefail | |
| packer init build-docker.pkr.hcl | |
| packer build \ | |
| -var "ami_name=${IMAGE_NAME}" \ | |
| -var "default_environment=${DEFAULT_ENVIRONMENT}" \ | |
| -var "environments=${PACKER_ENVIRONMENTS}" \ | |
| -var "ami_name_suffix=${AMI_NAME_SUFFIX}" \ | |
| -var "build_timestamp=${BUILD_TIMESTAMP}" \ | |
| -var "additional_tags=${ADDITIONAL_TAGS}" \ | |
| -var "docker_repository=${DOCKER_REPO}" \ | |
| build-docker.pkr.hcl | |
| shell: bash | |
| - name: Push Docker image | |
| run: | | |
| set -euo pipefail | |
| docker push --all-tags "${{ needs.lock-environments.outputs.docker_repository }}" | |
| - name: Collect packer manifest outputs | |
| id: collect | |
| uses: ./.github/actions/collect-packer-metadata | |
| with: | |
| manifest_path: packer-docker-manifest.json | |
| kind: docker | |
| image_repo: ${{ needs.lock-environments.outputs.docker_repository }} | |
| # ── Step 3a: Test AMI environments ───────────────────────────────── | |
| # Intentionally dormant while AMI test hangs are under investigation. | |
| # The pipeline still computes `environment_matrix`, and the reusable workflow | |
| # below can fan out one isolated EC2 test instance per environment when AMI | |
| # testing is re-enabled. When restoring this job, also decide whether AMI test | |
| # success should gate the `promote` job below. | |
| # | |
| # test-ami: | |
| # name: Test AMI (${{ matrix.environment.name }}) | |
| # needs: [lock-environments, build-ami] | |
| # if: ${{ needs.build-ami.outputs.ami_id != '' }} | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # environment: ${{ fromJson(needs.lock-environments.outputs.environment_matrix) }} | |
| # uses: ./.github/workflows/test-ami-reusable.yml | |
| # with: | |
| # ami_id: ${{ needs.build-ami.outputs.ami_id }} | |
| # ami_name: ${{ needs.build-ami.outputs.ami_name }} | |
| # environment_name: ${{ matrix.environment.name }} | |
| # full_test_script: ${{ matrix.environment.full_script }} | |
| # full_test_timeout: 21000 | |
| # instance_type: ${{ needs.build-ami.outputs.default_instance_type }} | |
| # region: ${{ needs.build-ami.outputs.aws_region }} | |
| # secrets: | |
| # test_assume_role_arn: ${{ secrets.AWS_TEST_ASSUME_ROLE_ARN }} | |
| # test_security_group_id: ${{ secrets.TEST_SECURITY_GROUP_ID }} | |
| # test_instance_profile_name: ${{ secrets.TEST_INSTANCE_PROFILE_NAME }} | |
| # test_log_group_name: ${{ secrets.TEST_LOG_GROUP_NAME }} | |
| # ── Step 3b: Test Docker environments ────────────────────────────── | |
| test-docker: | |
| name: Test Docker (${{ matrix.environment.name }}) | |
| needs: [lock-environments, build-docker] | |
| if: ${{ needs.build-docker.outputs.image_repo != '' }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| packages: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| environment: ${{ fromJson(needs.lock-environments.outputs.environment_matrix) }} | |
| uses: ./.github/workflows/test-docker-reusable.yml | |
| with: | |
| image_repo: ${{ needs.build-docker.outputs.image_repo }} | |
| image_tag: ${{ needs.build-docker.outputs.image_tag }} | |
| environment_name: ${{ matrix.environment.name }} | |
| full_test_script: ${{ matrix.environment.full_script }} | |
| instance_type: ${{ vars.AWS_DOCKER_GHA_RUNNER_INSTANCE_TYPE || needs.lock-environments.outputs.default_instance_type }} | |
| region: ${{ needs.lock-environments.outputs.aws_region }} | |
| secrets: | |
| docker_test_assume_role_arn: ${{ secrets.AWS_DOCKER_GHA_RUNNER_ASSUME_ROLE_ARN }} | |
| gh_pat: ${{ secrets.GH_PAT }} | |
| # ── Step 4: Promote artifacts ────────────────────────────────────── | |
| promote: | |
| name: Promote artifacts | |
| # AMI testing is currently disabled above; add `test-ami` back here when that | |
| # job is re-enabled. | |
| # Restore `needs.test-ami.result == 'success'` in the `if` expression below | |
| # when `test-ami` is re-enabled. | |
| needs: [lock-environments, build-ami, build-docker, test-docker] | |
| if: >- | |
| ${{ always() && | |
| github.event_name != 'pull_request' && | |
| needs.build-ami.outputs.ami_id != '' && | |
| needs.build-docker.outputs.image_repo != '' && | |
| needs.test-docker.result == 'success' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Download lockfile | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: pixi-lock | |
| path: environments | |
| - name: Download build context | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: build-context | |
| path: workflow-metadata | |
| # ── AMI promotion ────────────────────────────────────────────── | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Set up Pixi | |
| uses: prefix-dev/setup-pixi@v0.9.3 | |
| with: | |
| pixi-version: v0.63.0 | |
| cache: false | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade pip boto3 click pyyaml | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_ARN }} | |
| role-session-name: gha-ami-publish | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Recommend AMI state | |
| id: recommend | |
| run: | | |
| set -euo pipefail | |
| state=$(python cli/ami_cli.py recommend-state) | |
| echo "state=${state}" | tee -a "$GITHUB_OUTPUT" | |
| shell: bash | |
| - name: Publish or bless AMI | |
| env: | |
| AMI_ID: ${{ needs.build-ami.outputs.ami_id }} | |
| RECOMMENDED_STATE: ${{ steps.recommend.outputs.state }} | |
| run: | | |
| set -euo pipefail | |
| case "$RECOMMENDED_STATE" in | |
| bless|publish) | |
| python cli/ami_cli.py "$RECOMMENDED_STATE" "$AMI_ID" | |
| ;; | |
| *) | |
| echo "Recommended state is '$RECOMMENDED_STATE'; no publish/bless action." | |
| ;; | |
| esac | |
| shell: bash | |
| # ── Docker promotion ─────────────────────────────────────────── | |
| - name: Log in to GHCR | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Promote Docker image | |
| env: | |
| IMAGE_REPO: ${{ needs.build-docker.outputs.image_repo }} | |
| IMAGE_TAG: ${{ needs.build-docker.outputs.image_tag }} | |
| PUBLISHED_DATE: ${{ needs.lock-environments.outputs.published_date }} | |
| run: | | |
| set -euo pipefail | |
| echo "Pulling build image ${IMAGE_REPO}:${IMAGE_TAG}" | |
| docker pull "${IMAGE_REPO}:${IMAGE_TAG}" | |
| echo "Tagging as latest and ${PUBLISHED_DATE}" | |
| docker tag "${IMAGE_REPO}:${IMAGE_TAG}" "${IMAGE_REPO}:latest" | |
| docker tag "${IMAGE_REPO}:${IMAGE_TAG}" "${IMAGE_REPO}:${PUBLISHED_DATE}" | |
| echo "Pushing promoted tags" | |
| docker push "${IMAGE_REPO}:latest" | |
| docker push "${IMAGE_REPO}:${PUBLISHED_DATE}" | |
| shell: bash | |
| # ── Site metadata export ─────────────────────────────────────── | |
| - name: Generate promoted site metadata | |
| env: | |
| AMI_ID: ${{ needs.build-ami.outputs.ami_id }} | |
| BUILD_CONTEXT_PATH: workflow-metadata/build-context.json | |
| IMAGE_REPO: ${{ needs.build-docker.outputs.image_repo }} | |
| PUBLISHED_DATE: ${{ needs.lock-environments.outputs.published_date }} | |
| run: | | |
| set -euo pipefail | |
| published_name="$(jq -r '.published_name' "${BUILD_CONTEXT_PATH}")" | |
| published_timestamp="$(jq -r '.published_timestamp' "${BUILD_CONTEXT_PATH}")" | |
| published_docker_image="${IMAGE_REPO}:${PUBLISHED_DATE}" | |
| mkdir -p public/lockfiles | |
| cp environments/pixi.lock "public/lockfiles/${published_name}.pixi.lock" | |
| while IFS=$'\t' read -r environment_name platform output_path; do | |
| mkdir -p "$(dirname "${output_path}")" | |
| pixi workspace export conda-explicit-spec \ | |
| --manifest-path environments/pixi.toml \ | |
| --frozen \ | |
| --environment "${environment_name}" \ | |
| --platform "${platform}" \ | |
| "$(dirname "${output_path}")" | |
| generated_path="$(dirname "${output_path}")/${environment_name}_${platform}_conda_spec.txt" | |
| if [[ "${generated_path}" != "${output_path}" ]]; then | |
| mv "${generated_path}" "${output_path}" | |
| fi | |
| done < <( | |
| jq -r ' | |
| .site_export_matrix[] | |
| | [.environment, .platform, .conda_explicit_spec_path] | |
| | @tsv | |
| ' "${BUILD_CONTEXT_PATH}" | |
| ) | |
| jsonlockfile_command=( | |
| python cli/jsonlockfile.py | |
| --lockfile environments/pixi.lock | |
| --manifest environments/pixi.toml | |
| --output-root public | |
| --name "${published_name}" | |
| --timestamp "${published_timestamp}" | |
| --ami-id "${AMI_ID}" | |
| --docker-image "${published_docker_image}" | |
| ) | |
| # Keep these workflow-local for now; move them to workflow inputs once | |
| # the promoted metadata contract is finalized. | |
| jsonlockfile_command+=( | |
| --summary-package "OpenFE=openfe" | |
| --summary-package "OpenFF-Toolkit=openff-toolkit" | |
| --summary-package "OpenFold=openfold3" | |
| ) | |
| "${jsonlockfile_command[@]}" | |
| shell: bash | |
| - name: Upload promoted site artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: promoted-site-artifacts | |
| path: public | |
| retention-days: 7 |