diff --git a/.github/scripts/util_free_space.sh b/.github/scripts/util_free_space.sh new file mode 100644 index 00000000..3f1fc713 --- /dev/null +++ b/.github/scripts/util_free_space.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eux + +df -h +echo "::group::/usr/local/*" +du -hsc /usr/local/* +echo "::endgroup::" +# ~1GB +sudo rm -rf \ + /usr/local/aws-sam-cil \ + /usr/local/julia* || : +echo "::group::/usr/local/bin/*" +du -hsc /usr/local/bin/* +echo "::endgroup::" +# ~1GB (From 1.2GB to 214MB) +sudo rm -rf \ + /usr/local/bin/aliyun \ + /usr/local/bin/azcopy \ + /usr/local/bin/bicep \ + /usr/local/bin/cmake-gui \ + /usr/local/bin/cpack \ + /usr/local/bin/helm \ + /usr/local/bin/hub \ + /usr/local/bin/kubectl \ + /usr/local/bin/minikube \ + /usr/local/bin/node \ + /usr/local/bin/packer \ + /usr/local/bin/pulumi* \ + /usr/local/bin/sam \ + /usr/local/bin/stack \ + /usr/local/bin/terraform || : +# 142M +sudo rm -rf /usr/local/bin/oc || : \ +echo "::group::/usr/local/share/*" +du -hsc /usr/local/share/* +echo "::endgroup::" +# 506MB +sudo rm -rf /usr/local/share/chromium || : +# 1.3GB +sudo rm -rf /usr/local/share/powershell || : +echo "::group::/usr/local/lib/*" +du -hsc /usr/local/lib/* +echo "::endgroup::" +# 15GB +sudo rm -rf /usr/local/lib/android || : +# 341MB +sudo rm -rf /usr/local/lib/heroku || : +# 1.2GB +sudo rm -rf /usr/local/lib/node_modules || : +echo "::group::/opt/*" +du -hsc /opt/* +echo "::endgroup::" +# 679MB +sudo rm -rf /opt/az || : +echo "::group::/opt/microsoft/*" +du -hsc /opt/microsoft/* +echo "::endgroup::" +# 197MB +sudo rm -rf /opt/microsoft/powershell || : +echo "::group::/opt/hostedtoolcache/*" +du -hsc /opt/hostedtoolcache/* +echo "::endgroup::" +# 5.3GB +sudo rm -rf /opt/hostedtoolcache/CodeQL || : +# 1.4GB +sudo rm -rf /opt/hostedtoolcache/go || : +# 489MB +sudo rm -rf /opt/hostedtoolcache/PyPy || : +# 376MB +sudo rm -rf /opt/hostedtoolcache/node || : +# Remove Web browser packages +#sudo apt purge -y \ +# firefox \ +# google-chrome-stable \ +# microsoft-edge-stable +df -h \ No newline at end of file diff --git a/.github/workflows/apptainer-build-local.yml b/.github/workflows/apptainer-build-local.yml new file mode 100644 index 00000000..09705b56 --- /dev/null +++ b/.github/workflows/apptainer-build-local.yml @@ -0,0 +1,138 @@ +name: Test Apptainer Build Local + +on: + pull_request: + branches: [ master ] + workflow_run: + workflows: ["Docker Build (Local)"] + types: + - completed + workflow_dispatch: # Allow manual triggering + +jobs: + Test-Apptainer-Local-Uploads-and-Build: + runs-on: ubuntu-latest + if: ${{ github.event_name == 'pull_request' || github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} + + env: + COMPOSE_FILE: ./docker-compose.yml + REGISTRY_PREFIX: docker.io/library + SERVER_NAME: localhost + + steps: + - uses: actions/checkout@v4 + with: + submodules: true # Ensure we get the submodule info + + - name: Set repository name + run: echo "REPO_NAME=$(basename $GITHUB_WORKSPACE)" >> $GITHUB_ENV + + - name: Cache test data + uses: actions/cache@v4 + id: test-data-cache + with: + path: test/test_data + key: test-data-${{ hashFiles('.gitmodules') }} + restore-keys: | + test-data- + + - name: Get test data if not cached + if: steps.test-data-cache.outputs.cache-hit != 'true' + run: | + make get-test-data + + - name: Cache Docker images + uses: actions/cache@v4 + id: docker-cache + with: + path: | + mongo.tar + api.tar + handler.tar + ui.tar + key: docker-local-images-${{ github.event.workflow_run.head_sha || github.sha }} + restore-keys: | + docker-local-images- + + - name: Setup environment + run: | + # copy the example.env to .env + cp example.env .env + + # Update .env to use the hostname + sed -i "s/SERVER_NAME=.*/SERVER_NAME=localhost/" .env + + - name: Build or load images + run: | + if [ -f mongo.tar ] && [ -f api.tar ] && [ -f handler.tar ] && [ -f ui.tar ]; then + echo "Loading cached images..." + if ! docker load < mongo.tar || ! docker load < api.tar || ! docker load < handler.tar || ! docker load < ui.tar; then + echo "Failed to load cached images, building instead..." + docker compose -f ${{ env.COMPOSE_FILE }} build + else + rm mongo.tar api.tar handler.tar ui.tar + fi + else + echo "Building images..." + docker compose -f ${{ env.COMPOSE_FILE }} build + fi + + # Tag the images to match docker-compose + docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-api:latest api:latest + docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-handler:latest handler:latest + docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-ui:latest ui:latest + + - name: Clean Up some More Space + run: bash .github/scripts/util_free_space.sh + + - name: Setup Apptainer + run: | + sudo apt update + sudo apt install -y software-properties-common + sudo add-apt-repository -y ppa:apptainer/ppa + sudo apt update + sudo apt install -y apptainer + + apptainer --version + + - name: Build Single Docker Image + run: | + docker build -f EverythingDockerfile -t ezbids-everything . + + - name: Build Apptainer Image + run: | + apptainer build ezbids-everything.sif docker-daemon://ezbids-everything:latest + + - name: Run Apptainer Image + run: | + apptainer instance start --fakeroot --writable-tmpfs --cleanenv --no-home ezbids-everything.sif ezbids-everything + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Verify uv installation + run: uv --version + + - name: Verify Apptainer image is running + run: | + apptainer instance list | grep ezbids-everything + if [ $? -ne 0 ]; then + echo "Apptainer image is not running" + exit 1 + fi + + - name: Run upload test + run: | + make test-upload + + - name: Verify Apptainer image is running after upload + run: | + apptainer instance list | grep ezbids-everything + if [ $? -ne 0 ]; then + echo "Apptainer image is not running" + exit 1 + fi + + diff --git a/.github/workflows/docker-build-everything-image.yaml b/.github/workflows/docker-build-everything-image.yaml new file mode 100644 index 00000000..18a45d58 --- /dev/null +++ b/.github/workflows/docker-build-everything-image.yaml @@ -0,0 +1,105 @@ +name: Test Docker Build Everything Image + +on: + pull_request: + branches: [ master ] + workflow_run: + workflows: ["Docker Build (Local)"] + types: + - completed + workflow_dispatch: # Allow manual triggering + +jobs: + Test-Docker-Build-Everything-Image: + runs-on: ubuntu-latest + if: ${{ github.event_name == 'pull_request' || github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} + + env: + COMPOSE_FILE: ./docker-compose.yml + REGISTRY_PREFIX: docker.io/library + SERVER_NAME: localhost + + steps: + - uses: actions/checkout@v4 + with: + submodules: true # Ensure we get the submodule info + + - name: Set repository name + run: echo "REPO_NAME=$(basename $GITHUB_WORKSPACE)" >> $GITHUB_ENV + + - name: Cache test data + uses: actions/cache@v4 + id: test-data-cache + with: + path: test/test_data + key: test-data-${{ hashFiles('.gitmodules') }} + restore-keys: | + test-data- + + - name: Get test data if not cached + if: steps.test-data-cache.outputs.cache-hit != 'true' + run: | + make get-test-data + + - name: Cache Docker images + uses: actions/cache@v4 + id: docker-cache + with: + path: | + mongo.tar + api.tar + handler.tar + ui.tar + key: docker-local-images-${{ github.event.workflow_run.head_sha || github.sha }} + restore-keys: | + docker-local-images- + + - name: Setup environment + run: | + # copy the example.env to .env + cp example.env .env + + # Update .env to use the hostname + sed -i "s/SERVER_NAME=.*/SERVER_NAME=localhost/" .env + + - name: Build or load images + run: | + if [ -f mongo.tar ] && [ -f api.tar ] && [ -f handler.tar ] && [ -f ui.tar ]; then + echo "Loading cached images..." + if ! docker load < mongo.tar || ! docker load < api.tar || ! docker load < handler.tar || ! docker load < ui.tar; then + echo "Failed to load cached images, building instead..." + docker compose -f ${{ env.COMPOSE_FILE }} build + else + rm mongo.tar api.tar handler.tar ui.tar + fi + else + echo "Building images..." + docker compose -f ${{ env.COMPOSE_FILE }} build + fi + + # Tag the images to match docker-compose + docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-api:latest api:latest + docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-handler:latest handler:latest + docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-ui:latest ui:latest + + - name: Build Single Docker Image + run: | + docker build -f EverythingDockerfile -t ezbids-everything . + + - name: Start Services + run: | + docker run -d -p 27017:27017 -p 8082:8082 -p 3000:3000 -p 8000:8000 -v /tmp/ezbids-workdir:/tmp ezbids-everything + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Verify uv installation + run: uv --version + + - name: Run upload test + run: | + make test-upload + + + diff --git a/EverythingDockerfile b/EverythingDockerfile new file mode 100644 index 00000000..da28e9f0 --- /dev/null +++ b/EverythingDockerfile @@ -0,0 +1,247 @@ +# EverythingDockerfile - Multistage build combining all services +# This combines mongodb, api, handler, ui, and telemetry into a single container + +# Stage 1: MongoDB base +FROM mongo:4.4.15 AS mongodb-stage +RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* + +# Stage 2: Neuroimaging tools and handler dependencies +FROM neurodebian:nd20.04-non-free AS neuroimaging-stage +SHELL ["/bin/bash", "-c"] +ENV DEBIAN_FRONTEND noninteractive + +RUN apt update && \ + apt-get update && apt-get upgrade -y + +RUN apt update && apt install -y parallel python3 python3-pip tree curl unzip git jq python libgl-dev python-numpy bc + +RUN pip3 install numpy==1.23.0 nibabel==4.0.0 pandas matplotlib pyyaml==5.4.1 pydicom==2.3.1 natsort pydeface && \ + pip3 install quickshear mne mne-bids pypet2bids==1.4.1 + +RUN apt-get install -y build-essential pkg-config cmake git pigz rename zstd libopenjp2-7 libgdcm-tools wget libopenblas-dev && \ + apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y + +RUN touch /.pet2bidsconfig && chown 1001:1001 /.pet2bidsconfig +RUN echo "DEFAULT_METADATA_JSON=/usr/local/lib/python3.8/dist-packages/pypet2bids/template_json.json" > /.pet2bidsconfig + +RUN mkdir -p /usr/local/fsl && \ + git clone https://github.com/dlevitas/FSL_binaries /usr/local/fsl && \ + rm -rf /usr/local/fsl/README.md && \ + mkdir -p /usr/local/fsl/data/standard && \ + mv /usr/local/fsl/bin/MNI152_T1_2mm_brain.nii.gz /usr/local/fsl/data/standard + +ENV FSLDIR=/usr/local/fsl +ENV PATH=$PATH:$FSLDIR/bin +ENV FSLOUTPUTTYPE=NIFTI_GZ + +RUN apt-get update \ + && apt-get install -y ca-certificates curl gnupg \ + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg + +ARG NODE_MAJOR=20 +RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list + +RUN apt-get update \ + && apt-get install nodejs -y + +RUN cd /tmp && curl -fLO https://github.com/rordenlab/dcm2niix/releases/latest/download/dcm2niix_lnx.zip \ + && unzip /tmp/dcm2niix_lnx.zip \ + && mv dcm2niix /usr/local/bin + +RUN mkdir -p /app + +# Get bids-specification from github +RUN cd /app && git clone https://github.com/bids-standard/bids-specification && \ + cd bids-specification && git checkout 3537e9edbc81545614d3ee605c398361099b6977 + +#install ROBEX +ADD https://www.nitrc.org/frs/download.php/5994/ROBEXv12.linux64.tar.gz//?i_agree=1&download_now=1 / +RUN tar -xzf /ROBEXv12.linux64.tar.gz +ENV PATH /ROBEX:$PATH + +ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/v$NODE_VERSION/bin:$PATH + +#install bids-validator +RUN npm install -g npm@9.5.1 +RUN npm install -g bids-validator@1.14.8 +RUN git clone https://github.com/bids-standard/bids-validator + +# Stage 3: Node.js base for API and UI +FROM node:20 AS nodejs-stage +RUN npm install -g npm@9.5.1 pm2 typescript tsc-watch + +# Stage 4: Telemetry dependencies +FROM neurodebian:nd20.04-non-free AS telemetry-stage +SHELL ["/bin/bash", "-c"] +ENV DEBIAN_FRONTEND noninteractive + +RUN apt update && \ + apt-get update && apt-get upgrade -y + +RUN apt install -y parallel python3 python3-pip tree curl unzip git jq python libgl-dev python-numpy +RUN pip3 install --upgrade pip +RUN pip3 install conversiontelemetry + +# Stage 5: Final combined stage +FROM neurodebian:nd20.04-non-free AS everything +SHELL ["/bin/bash", "-c"] +ENV DEBIAN_FRONTEND noninteractive + +# Install system dependencies +RUN apt update && \ + apt-get update && apt-get upgrade -y && \ + apt install -y parallel python3 python3-pip tree curl unzip git jq python libgl-dev python-numpy bc \ + build-essential pkg-config cmake git pigz rename zstd libopenjp2-7 libgdcm-tools wget libopenblas-dev \ + ca-certificates gnupg supervisor && \ + apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y + +# Install Node.js 20 +RUN mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg + +ARG NODE_MAJOR=20 +RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list + +RUN apt-get update && apt-get install nodejs -y + +# Install Python packages +RUN pip3 install numpy==1.23.0 nibabel==4.0.0 pandas matplotlib pyyaml==5.4.1 pydicom==2.3.1 natsort pydeface && \ + pip3 install quickshear mne mne-bids pypet2bids==1.4.1 conversiontelemetry + +# Install MongoDB +RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - && \ + echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list && \ + apt-get update && \ + apt-get install -y mongodb-org=4.4.15 mongodb-org-server=4.4.15 mongodb-org-shell=4.4.15 mongodb-org-mongos=4.4.15 mongodb-org-tools=4.4.15 + +# Install neuroimaging tools +RUN touch /.pet2bidsconfig && chown 1001:1001 /.pet2bidsconfig +RUN echo "DEFAULT_METADATA_JSON=/usr/local/lib/python3.8/dist-packages/pypet2bids/template_json.json" > /.pet2bidsconfig + +RUN mkdir -p /usr/local/fsl && \ + git clone https://github.com/dlevitas/FSL_binaries /usr/local/fsl && \ + rm -rf /usr/local/fsl/README.md && \ + mkdir -p /usr/local/fsl/data/standard && \ + mv /usr/local/fsl/bin/MNI152_T1_2mm_brain.nii.gz /usr/local/fsl/data/standard + +ENV FSLDIR=/usr/local/fsl +ENV PATH=$PATH:$FSLDIR/bin +ENV FSLOUTPUTTYPE=NIFTI_GZ + +# Install dcm2niix +RUN cd /tmp && curl -fLO https://github.com/rordenlab/dcm2niix/releases/latest/download/dcm2niix_lnx.zip \ + && unzip /tmp/dcm2niix_lnx.zip \ + && mv dcm2niix /usr/local/bin + +# Install ROBEX +ADD https://www.nitrc.org/frs/download.php/5994/ROBEXv12.linux64.tar.gz//?i_agree=1&download_now=1 / +RUN tar -xzf /ROBEXv12.linux64.tar.gz +ENV PATH /ROBEX:$PATH + +# Install Node.js global packages +RUN npm install -g npm@9.5.1 pm2 typescript tsc-watch bids-validator@1.14.8 + +# Get bids-specification from github +RUN mkdir -p /app && \ + cd /app && git clone https://github.com/bids-standard/bids-specification && \ + cd bids-specification && git checkout 3537e9edbc81545614d3ee605c398361099b6977 + +# Clone bids-validator +RUN git clone https://github.com/bids-standard/bids-validator /app/bids-validator + +# Copy application code +COPY . /app + +# Generate keys for API +WORKDIR /app +RUN ./generate_keys.sh + +# Install API dependencies and build +WORKDIR /app/api +RUN cp /app/package.json /app/api/ && npm install && npx tsc + +# Install handler dependencies and build +WORKDIR /app/handler +RUN npm install && npx tsc + +# Install UI dependencies +WORKDIR /app/ui +RUN npm install + +# Copy UI entrypoint +COPY ui/entrypoint.sh /entrypoint-ui.sh +RUN chmod +x /entrypoint-ui.sh + +# Copy telemetry environment +COPY telemetry/telemetry.env /root/.telemetry.env + +# Create supervisor configuration +RUN mkdir -p /etc/supervisor/conf.d +COPY <