Single docker file + Apptainer Support

.github/scripts/util_free_space.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eux
+
+df -h
+echo "::group::/usr/local/*"
+du -hsc /usr/local/*
+echo "::endgroup::"
+# ~1GB
+sudo rm -rf \
+  /usr/local/aws-sam-cil \
+  /usr/local/julia* || :
+echo "::group::/usr/local/bin/*"
+du -hsc /usr/local/bin/*
+echo "::endgroup::"
+# ~1GB (From 1.2GB to 214MB)
+sudo rm -rf \
+  /usr/local/bin/aliyun \
+  /usr/local/bin/azcopy \
+  /usr/local/bin/bicep \
+  /usr/local/bin/cmake-gui \
+  /usr/local/bin/cpack \
+  /usr/local/bin/helm \
+  /usr/local/bin/hub \
+  /usr/local/bin/kubectl \
+  /usr/local/bin/minikube \
+  /usr/local/bin/node \
+  /usr/local/bin/packer \
+  /usr/local/bin/pulumi* \
+  /usr/local/bin/sam \
+  /usr/local/bin/stack \
+  /usr/local/bin/terraform || :
+# 142M
+sudo rm -rf /usr/local/bin/oc || : \
+echo "::group::/usr/local/share/*"
+du -hsc /usr/local/share/*
+echo "::endgroup::"
+# 506MB
+sudo rm -rf /usr/local/share/chromium || :
+# 1.3GB
+sudo rm -rf /usr/local/share/powershell || :
+echo "::group::/usr/local/lib/*"
+du -hsc /usr/local/lib/*
+echo "::endgroup::"
+# 15GB
+sudo rm -rf /usr/local/lib/android || :
+# 341MB
+sudo rm -rf /usr/local/lib/heroku || :
+# 1.2GB
+sudo rm -rf /usr/local/lib/node_modules || :
+echo "::group::/opt/*"
+du -hsc /opt/*
+echo "::endgroup::"
+# 679MB
+sudo rm -rf /opt/az || :
+echo "::group::/opt/microsoft/*"
+du -hsc /opt/microsoft/*
+echo "::endgroup::"
+# 197MB
+sudo rm -rf /opt/microsoft/powershell || :
+echo "::group::/opt/hostedtoolcache/*"
+du -hsc /opt/hostedtoolcache/*
+echo "::endgroup::"
+# 5.3GB
+sudo rm -rf /opt/hostedtoolcache/CodeQL || :
+# 1.4GB
+sudo rm -rf /opt/hostedtoolcache/go || :
+# 489MB
+sudo rm -rf /opt/hostedtoolcache/PyPy || :
+# 376MB
+sudo rm -rf /opt/hostedtoolcache/node || :
+# Remove Web browser packages
+#sudo apt purge -y \
+#  firefox \
+#  google-chrome-stable \
+# microsoft-edge-stable
+df -h

.github/workflows/apptainer-build-local.yml

@@ -0,0 +1,138 @@
+name: Test Apptainer Build Local
+
+on:
+  pull_request:
+    branches: [ master ]
+  workflow_run:
+    workflows: ["Docker Build (Local)"]
+    types:
+      - completed
+  workflow_dispatch:  # Allow manual triggering
+
+jobs:
+  Test-Apptainer-Local-Uploads-and-Build:
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'pull_request' || github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
+
+    env:
+      COMPOSE_FILE: ./docker-compose.yml
+      REGISTRY_PREFIX: docker.io/library
+      SERVER_NAME: localhost
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true  # Ensure we get the submodule info
+
+      - name: Set repository name
+        run: echo "REPO_NAME=$(basename $GITHUB_WORKSPACE)" >> $GITHUB_ENV
+
+      - name: Cache test data
+        uses: actions/cache@v4
+        id: test-data-cache
+        with:
+          path: test/test_data
+          key: test-data-${{ hashFiles('.gitmodules') }}
+          restore-keys: |
+            test-data-
+
+      - name: Get test data if not cached
+        if: steps.test-data-cache.outputs.cache-hit != 'true'
+        run: |
+          make get-test-data
+
+      - name: Cache Docker images
+        uses: actions/cache@v4
+        id: docker-cache
+        with:
+          path: |
+            mongo.tar
+            api.tar
+            handler.tar
+            ui.tar
+          key: docker-local-images-${{ github.event.workflow_run.head_sha || github.sha }}
+          restore-keys: |
+            docker-local-images-
+
+      - name: Setup environment
+        run: |
+          # copy the example.env to .env
+          cp example.env .env
+
+          # Update .env to use the hostname
+          sed -i "s/SERVER_NAME=.*/SERVER_NAME=localhost/" .env
+
+      - name: Build or load images
+        run: |
+          if [ -f mongo.tar ] && [ -f api.tar ] && [ -f handler.tar ] && [ -f ui.tar ]; then
+            echo "Loading cached images..."
+            if ! docker load < mongo.tar || ! docker load < api.tar || ! docker load < handler.tar || ! docker load < ui.tar; then
+              echo "Failed to load cached images, building instead..."
+              docker compose -f ${{ env.COMPOSE_FILE }} build
+            else
+              rm mongo.tar api.tar handler.tar ui.tar
+            fi
+          else
+            echo "Building images..."
+            docker compose -f ${{ env.COMPOSE_FILE }} build
+          fi
+
+          # Tag the images to match docker-compose
+          docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-api:latest api:latest
+          docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-handler:latest handler:latest
+          docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-ui:latest ui:latest
+
+      - name: Clean Up some More Space
+        run: bash .github/scripts/util_free_space.sh
+
+      - name: Setup Apptainer
+        run: |
+          sudo apt update
+          sudo apt install -y software-properties-common
+          sudo add-apt-repository -y ppa:apptainer/ppa
+          sudo apt update
+          sudo apt install -y apptainer
+
+          apptainer --version
+
+      - name: Build Single Docker Image
+        run: |
+          docker build -f EverythingDockerfile -t ezbids-everything .
+
+      - name: Build Apptainer Image
+        run: |
+          apptainer build ezbids-everything.sif docker-daemon://ezbids-everything:latest
+
+      - name: Run Apptainer Image
+        run: |
+          apptainer instance start --fakeroot --writable-tmpfs --cleanenv --no-home ezbids-everything.sif ezbids-everything
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Verify uv installation
+        run: uv --version
+
+      - name: Verify Apptainer image is running
+        run: |
+          apptainer instance list | grep ezbids-everything
+          if [ $? -ne 0 ]; then
+            echo "Apptainer image is not running"
+            exit 1
+          fi
+
+      - name: Run upload test
+        run: |
+          make test-upload
+
+      - name: Verify Apptainer image is running after upload 
+        run: |
+          apptainer instance list | grep ezbids-everything
+          if [ $? -ne 0 ]; then
+            echo "Apptainer image is not running"
+            exit 1
+          fi
+
+

.github/workflows/docker-build-everything-image.yaml

@@ -0,0 +1,105 @@
+name: Test Docker Build Everything Image
+
+on:
+  pull_request:
+    branches: [ master ]
+  workflow_run:
+    workflows: ["Docker Build (Local)"]
+    types:
+      - completed
+  workflow_dispatch:  # Allow manual triggering
+
+jobs:
+  Test-Docker-Build-Everything-Image:
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'pull_request' || github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
+
+    env:
+      COMPOSE_FILE: ./docker-compose.yml
+      REGISTRY_PREFIX: docker.io/library
+      SERVER_NAME: localhost
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true  # Ensure we get the submodule info
+
+      - name: Set repository name
+        run: echo "REPO_NAME=$(basename $GITHUB_WORKSPACE)" >> $GITHUB_ENV
+
+      - name: Cache test data
+        uses: actions/cache@v4
+        id: test-data-cache
+        with:
+          path: test/test_data
+          key: test-data-${{ hashFiles('.gitmodules') }}
+          restore-keys: |
+            test-data-
+
+      - name: Get test data if not cached
+        if: steps.test-data-cache.outputs.cache-hit != 'true'
+        run: |
+          make get-test-data
+
+      - name: Cache Docker images
+        uses: actions/cache@v4
+        id: docker-cache
+        with:
+          path: |
+            mongo.tar
+            api.tar
+            handler.tar
+            ui.tar
+          key: docker-local-images-${{ github.event.workflow_run.head_sha || github.sha }}
+          restore-keys: |
+            docker-local-images-
+
+      - name: Setup environment
+        run: |
+          # copy the example.env to .env
+          cp example.env .env
+
+          # Update .env to use the hostname
+          sed -i "s/SERVER_NAME=.*/SERVER_NAME=localhost/" .env
+
+      - name: Build or load images
+        run: |
+          if [ -f mongo.tar ] && [ -f api.tar ] && [ -f handler.tar ] && [ -f ui.tar ]; then
+            echo "Loading cached images..."
+            if ! docker load < mongo.tar || ! docker load < api.tar || ! docker load < handler.tar || ! docker load < ui.tar; then
+              echo "Failed to load cached images, building instead..."
+              docker compose -f ${{ env.COMPOSE_FILE }} build
+            else
+              rm mongo.tar api.tar handler.tar ui.tar
+            fi
+          else
+            echo "Building images..."
+            docker compose -f ${{ env.COMPOSE_FILE }} build
+          fi
+
+          # Tag the images to match docker-compose
+          docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-api:latest api:latest
+          docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-handler:latest handler:latest
+          docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-ui:latest ui:latest
+
+      - name: Build Single Docker Image
+        run: |
+          docker build -f EverythingDockerfile -t ezbids-everything .
+
+      - name: Start Services
+        run: |
+          docker run -d -p 27017:27017 -p 8082:8082 -p 3000:3000 -p 8000:8000 -v /tmp/ezbids-workdir:/tmp ezbids-everything       
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Verify uv installation
+        run: uv --version
+
+      - name: Run upload test
+        run: |
+          make test-upload
+
+
+