First pass at adding apptainer image building

.github/workflows/build-pipeline.yaml

@@ -18,6 +18,7 @@ env:
   AWS_REGION: us-east-1
   DEFAULT_INSTANCE_TYPE: g5.xlarge
   DOCKER_REPOSITORY: ghcr.io/omsf-eco-infra/omsf
+  SIF_REPOSITORY: ghcr.io/omsf-eco-infra/omsf-sif
 
 jobs:
   # ── Step 1: Prepare shared build inputs ────────────────────────────
@@ -241,6 +242,7 @@ jobs:
           role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_ARN }}
           role-session-name: gha-ami-builder
           aws-region: ${{ needs.lock-environments.outputs.aws_region }}
+          role-duration-seconds: 7200
 
       - name: Set up Packer
         uses: hashicorp/setup-packer@v3
@@ -335,6 +337,50 @@ jobs:
           kind: docker
           image_repo: ${{ needs.lock-environments.outputs.docker_repository }}
 
+  # ── Step 2c: Build and publish SIF image ────────────────────────────
+  build-sif:
+    name: Build SIF Image
+    needs: build-docker
+    if: ${{ needs.build-docker.outputs.image_repo != '' && needs.build-docker.outputs.image_tag != '' }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    outputs:
+      image_repo: ${{ steps.metadata.outputs.image_repo }}
+      image_tag: ${{ steps.metadata.outputs.image_tag }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install Apptainer
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y software-properties-common
+          sudo add-apt-repository -y ppa:apptainer/ppa
+          sudo apt-get update
+          sudo apt-get install -y apptainer
+          apptainer version
+
+      - name: Build and publish SIF image
+        run: build-scripts/build-singularity-image.sh
+        env:
+          APPTAINER_DOCKER_USERNAME: ${{ github.actor }}
+          APPTAINER_DOCKER_PASSWORD: ${{ github.token }}
+          APPTAINER_TMPDIR: ${{ runner.temp }}/apptainer-tmp
+          DOCKER_IMAGE: ${{ needs.build-docker.outputs.image_repo }}:${{ needs.build-docker.outputs.image_tag }}
+          SIF_IMAGE: ${{ env.SIF_REPOSITORY }}:${{ needs.build-docker.outputs.image_tag }}
+          SIF_OUTPUT: ${{ runner.temp }}/${{ needs.build-docker.outputs.image_tag }}.sif
+
+      - name: Record SIF image metadata
+        id: metadata
+        run: |
+          set -euo pipefail
+          echo "image_repo=${SIF_REPOSITORY}" >> "${GITHUB_OUTPUT}"
+          echo "image_tag=${IMAGE_TAG}" >> "${GITHUB_OUTPUT}"
+        env:
+          IMAGE_TAG: ${{ needs.build-docker.outputs.image_tag }}
+
   # ── Step 3a: Test AMI environments ─────────────────────────────────
   # Intentionally dormant while AMI test hangs are under investigation.
   # The pipeline still computes `environment_matrix`, and the reusable workflow
@@ -380,6 +426,7 @@ jobs:
         environment: ${{ fromJson(needs.lock-environments.outputs.environment_matrix) }}
     uses: ./.github/workflows/test-docker-reusable.yml
     with:
+      container_runtime: docker
       image_repo: ${{ needs.build-docker.outputs.image_repo }}
       image_tag: ${{ needs.build-docker.outputs.image_tag }}
       environment_name: ${{ matrix.environment.name }}
@@ -390,20 +437,48 @@ jobs:
       docker_test_assume_role_arn: ${{ secrets.AWS_DOCKER_GHA_RUNNER_ASSUME_ROLE_ARN }}
       gh_pat: ${{ secrets.GH_PAT }}
 
+  # ── Step 3c: Test SIF environments ─────────────────────────────────
+  test-sif:
+    name: Test SIF (${{ matrix.environment.name }})
+    needs: [lock-environments, build-sif]
+    if: ${{ needs.build-sif.outputs.image_repo != '' }}
+    permissions:
+      id-token: write
+      contents: read
+      packages: read
+    strategy:
+      fail-fast: false
+      matrix:
+        environment: ${{ fromJson(needs.lock-environments.outputs.environment_matrix) }}
+    uses: ./.github/workflows/test-docker-reusable.yml
+    with:
+      container_runtime: apptainer
+      image_repo: ${{ needs.build-sif.outputs.image_repo }}
+      image_tag: ${{ needs.build-sif.outputs.image_tag }}
+      environment_name: ${{ matrix.environment.name }}
+      full_test_script: ${{ matrix.environment.full_script }}
+      instance_type: ${{ vars.AWS_DOCKER_GHA_RUNNER_INSTANCE_TYPE || needs.lock-environments.outputs.default_instance_type }}
+      region: ${{ needs.lock-environments.outputs.aws_region }}
+    secrets:
+      docker_test_assume_role_arn: ${{ secrets.AWS_DOCKER_GHA_RUNNER_ASSUME_ROLE_ARN }}
+      gh_pat: ${{ secrets.GH_PAT }}
+
   # ── Step 4: Promote artifacts ──────────────────────────────────────
   promote:
     name: Promote artifacts
     # AMI testing is currently disabled above; add `test-ami` back here when that
     # job is re-enabled.
     # Restore `needs.test-ami.result == 'success'` in the `if` expression below
     # when `test-ami` is re-enabled.
-    needs: [lock-environments, build-ami, build-docker, test-docker]
+    needs: [lock-environments, build-ami, build-docker, build-sif, test-docker, test-sif]
     if: >-
       ${{ always() &&
           github.event_name != 'pull_request' &&
           needs.build-ami.outputs.ami_id != '' &&
           needs.build-docker.outputs.image_repo != '' &&
-          needs.test-docker.result == 'success' }}
+          needs.build-sif.outputs.image_repo != '' &&
+          needs.test-docker.result == 'success' &&
+          needs.test-sif.result == 'success' }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -492,6 +567,26 @@ jobs:
           docker push "${IMAGE_REPO}:${PUBLISHED_DATE}"
         shell: bash
 
+      # ── SIF promotion ──────────────────────────────────────────────
+      - name: Set up ORAS
+        uses: oras-project/setup-oras@v1
+
+      - name: Log in to GHCR with ORAS
+        run: echo "${GH_TOKEN}" | oras login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Promote SIF image
+        env:
+          IMAGE_REPO: ${{ needs.build-sif.outputs.image_repo }}
+          IMAGE_TAG: ${{ needs.build-sif.outputs.image_tag }}
+          PUBLISHED_DATE: ${{ needs.lock-environments.outputs.published_date }}
+        run: |
+          set -euo pipefail
+          echo "Tagging ${IMAGE_REPO}:${IMAGE_TAG} as latest and ${PUBLISHED_DATE}"
+          oras tag "${IMAGE_REPO}:${IMAGE_TAG}" latest "${PUBLISHED_DATE}"
+        shell: bash
+
       # ── Site metadata export ───────────────────────────────────────
       - name: Generate promoted site metadata
         env:

.github/workflows/test-docker-reusable.yml

@@ -1,14 +1,18 @@
-name: Reusable Docker environment test
+name: Reusable container environment test
 
 on:
   workflow_call:
     inputs:
+      container_runtime:
+        description: Container runtime to test (docker or apptainer).
+        required: true
+        type: string
       image_repo:
-        description: Docker repository containing the image to test.
+        description: OCI repository containing the image to test.
         required: true
         type: string
       image_tag:
-        description: Docker image tag to test.
+        description: Image tag to test.
         required: true
         type: string
       environment_name:
@@ -20,7 +24,7 @@ on:
         required: true
         type: string
       instance_type:
-        description: EC2 instance type to use for the self-hosted Docker test runner.
+        description: EC2 instance type to use for the self-hosted container test runner.
         required: false
         type: string
         default: g5.xlarge
@@ -46,7 +50,7 @@ env:
 
 jobs:
   start-runner:
-    name: Start Docker test runner
+    name: Start ${{ inputs.container_runtime }} test runner
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -55,21 +59,29 @@ jobs:
       mapping: ${{ steps.aws-start.outputs.mapping }}
       instances: ${{ steps.aws-start.outputs.instances }}
     steps:
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: ${{ secrets.docker_test_assume_role_arn }}
-          role-session-name: gha-start-docker-test-runner
-          aws-region: ${{ inputs.region }}
-
       - name: Validate runner configuration
         run: |
           set -euo pipefail
+          case "${CONTAINER_RUNTIME}" in
+            docker|apptainer) ;;
+            *)
+              echo "container_runtime must be docker or apptainer, got: ${CONTAINER_RUNTIME}" >&2
+              exit 2
+              ;;
+          esac
           : "${AWS_GHA_RUNNER_IMAGE_ID:?Repository variable AWS_DOCKER_GHA_RUNNER_IMAGE_ID is required}"
           : "${RUNNER_INSTANCE_TYPE:?Runner instance type must be set}"
         env:
+          CONTAINER_RUNTIME: ${{ inputs.container_runtime }}
           RUNNER_INSTANCE_TYPE: ${{ inputs.instance_type }}
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.docker_test_assume_role_arn }}
+          role-session-name: gha-start-container-test-runner
+          aws-region: ${{ inputs.region }}
+
       - name: Start runner
         id: aws-start
         uses: omsf/start-aws-gha-runner@v1.0.0
@@ -83,45 +95,77 @@ jobs:
           GH_PAT: ${{ secrets.gh_pat }}
 
   test:
-    name: Run Docker test
+    name: Run ${{ inputs.container_runtime }} test
     needs: start-runner
     if: ${{ inputs.image_repo != '' && inputs.image_tag != '' && needs.start-runner.outputs.instances != '' }}
     runs-on: ${{ fromJson(needs.start-runner.outputs.instances) }}
     permissions:
       contents: read
       packages: read
     steps:
-      - name: Ensure runner dependencies (Ubuntu)
+      - name: Ensure Docker is installed
+        if: ${{ inputs.container_runtime == 'docker' }}
         run: |
           set -euo pipefail
-          if command -v docker >/dev/null 2>&1; then
-            exit 0
+          if ! command -v docker >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y docker.io
           fi
-          sudo apt-get update
-          sudo apt-get install -y docker.io
+          docker --version
 
-      - name: Log in to GHCR
+      - name: Install Apptainer
+        if: ${{ inputs.container_runtime == 'apptainer' }}
         run: |
-          echo "${{ github.token }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y software-properties-common
+          sudo add-apt-repository -y ppa:apptainer/ppa
+          sudo apt-get update
+          sudo apt-get install -y apptainer
+          apptainer version
+
+      - name: Log in to GHCR with Docker
+        if: ${{ inputs.container_runtime == 'docker' }}
+        run: echo "${GH_TOKEN}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Pull Docker image
+        if: ${{ inputs.container_runtime == 'docker' }}
+        run: docker pull "${IMAGE_REPO}:${IMAGE_TAG}"
+        env:
+          IMAGE_REPO: ${{ inputs.image_repo }}
+          IMAGE_TAG: ${{ inputs.image_tag }}
 
-      - name: Pull image
+      - name: Pull SIF image
+        if: ${{ inputs.container_runtime == 'apptainer' }}
         run: |
           set -euo pipefail
-          docker pull "${IMAGE_REPO}:${IMAGE_TAG}"
+          apptainer pull \
+            --disable-cache \
+            --force \
+            "${SIF_PATH}" \
+            "oras://${IMAGE_REPO}:${IMAGE_TAG}"
         env:
+          APPTAINER_DOCKER_USERNAME: ${{ github.actor }}
+          APPTAINER_DOCKER_PASSWORD: ${{ github.token }}
           IMAGE_REPO: ${{ inputs.image_repo }}
           IMAGE_TAG: ${{ inputs.image_tag }}
+          SIF_PATH: ${{ runner.temp }}/omsf.sif
 
-      - name: Verify GPU runtime
+      - name: Verify host GPU runtime
         run: |
           set -euo pipefail
-
           if ! command -v nvidia-smi >/dev/null 2>&1; then
-            echo "[test] nvidia-smi is not available on the Docker test runner" >&2
+            echo "[test] nvidia-smi is not available on the container test runner" >&2
             exit 1
           fi
           nvidia-smi -L
 
+      - name: Verify Docker GPU runtime
+        if: ${{ inputs.container_runtime == 'docker' }}
+        run: |
+          set -euo pipefail
           docker run --rm --gpus all \
             -e "NVIDIA_VISIBLE_DEVICES=all" \
             -e "NVIDIA_DRIVER_CAPABILITIES=compute,utility" \
@@ -131,31 +175,62 @@ jobs:
           IMAGE_REPO: ${{ inputs.image_repo }}
           IMAGE_TAG: ${{ inputs.image_tag }}
 
-      - name: Run full tests
+      - name: Verify SIF GPU runtime
+        if: ${{ inputs.container_runtime == 'apptainer' }}
+        run: apptainer exec --cleanenv --nv "${SIF_PATH}" nvidia-smi -L
+        env:
+          SIF_PATH: ${{ runner.temp }}/omsf.sif
+
+      - name: Run Docker full tests
+        if: ${{ inputs.container_runtime == 'docker' }}
         run: |
           set -euo pipefail
-
           if [[ ! "${ENVIRONMENT_NAME}" =~ ^[A-Za-z0-9_.-]+$ ]]; then
             echo "[test] Invalid environment name: ${ENVIRONMENT_NAME}" >&2
             exit 1
           fi
 
           script_path="${FULL_TEST_SCRIPT#/}"
           docker run --rm --gpus all \
-            -e "PIXI_DEFAULT_ENVIRONMENT=${ENVIRONMENT_NAME}-test" \
+            -e "PIXI_DEFAULT_ENVIRONMENT=${ENVIRONMENT_NAME}" \
             -e "CONDA_OVERRIDE_CUDA=12" \
             -e "NVIDIA_VISIBLE_DEVICES=all" \
             -e "NVIDIA_DRIVER_CAPABILITIES=compute,utility" \
             "${IMAGE_REPO}:${IMAGE_TAG}" \
-            bash "/tmp/${script_path}"
+            bash "/opt/omsf/${script_path}"
         env:
           IMAGE_REPO: ${{ inputs.image_repo }}
           IMAGE_TAG: ${{ inputs.image_tag }}
           ENVIRONMENT_NAME: ${{ inputs.environment_name }}
           FULL_TEST_SCRIPT: ${{ inputs.full_test_script }}
 
+      - name: Run SIF full tests
+        if: ${{ inputs.container_runtime == 'apptainer' }}
+        run: |
+          set -euo pipefail
+          if [[ ! "${ENVIRONMENT_NAME}" =~ ^[A-Za-z0-9_.-]+$ ]]; then
+            echo "[test] Invalid environment name: ${ENVIRONMENT_NAME}" >&2
+            exit 1
+          fi
+
+          script_path="${FULL_TEST_SCRIPT#/}"
+          apptainer exec \
+            --cleanenv \
+            --nv \
+            --env "OMSF_PIXI_WORKSPACE=/opt/omsf/workspace" \
+            --env "PIXI_HOME=/opt/omsf/pixi-home" \
+            --env "PIXI_DEFAULT_ENVIRONMENT=${ENVIRONMENT_NAME}" \
+            --env "CONDA_OVERRIDE_CUDA=12" \
+            "${SIF_PATH}" \
+            /usr/local/bin/omsf-entrypoint.sh \
+            bash "/opt/omsf/${script_path}"
+        env:
+          ENVIRONMENT_NAME: ${{ inputs.environment_name }}
+          FULL_TEST_SCRIPT: ${{ inputs.full_test_script }}
+          SIF_PATH: ${{ runner.temp }}/omsf.sif
+
   stop-runner:
-    name: Stop Docker test runner
+    name: Stop ${{ inputs.container_runtime }} test runner
     needs: [start-runner, test]
     if: ${{ always() && needs.start-runner.outputs.mapping != '' }}
     runs-on: ubuntu-latest
@@ -167,7 +242,7 @@ jobs:
         uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.docker_test_assume_role_arn }}
-          role-session-name: gha-stop-docker-test-runner
+          role-session-name: gha-stop-container-test-runner
           aws-region: ${{ inputs.region }}
 
       - name: Stop runner