Merge pull request #3 from prehensilecode/update-for-2.2.2

Update for 2.2.2

Author: prehensilecode

Singularity.def

@@ -1,5 +1,5 @@
 Bootstrap: docker
-From: nvidia/cuda:11.0-cudnn8-runtime-ubuntu18.04
+From: nvidia/cuda:11.1.1-cudnn8-runtime-ubuntu18.04
 Stage: spython-base
 
 %files
@@ -25,7 +25,7 @@ Stage: spython-base
 apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 build-essential \
 cmake \
-cuda-command-line-tools-11-0 \
+cuda-command-line-tools-11-1 \
 git \
 hmmer \
 kalign \
@@ -55,29 +55,37 @@ PATH="/opt/conda/bin:$PATH"
 conda update -qy conda \
 && conda install -y -c conda-forge \
 openmm=7.5.1 \
-cudatoolkit==11.0.3 \
+cudatoolkit==11.1.1 \
 pdbfixer \
 pip \
 python=3.7
 
+### /bin/cp -r . /app/alphafold
+
 wget -q -P /app/alphafold/alphafold/common/ \
 https://git.scicore.unibas.ch/schwede/openstructure/-/raw/7102c63615b64735c4941278d92b554ec94415f8/modules/mol/alg/src/stereo_chemical_props.txt
 
 # Install pip packages.
+# N.B. The URL specifies the list of jaxlib releases. The URL in alphafold release 2.2.2 is incorrect
+#      as it specifies the non-CUDA releases.
 pip3 install --upgrade pip \
 && pip3 install -r /app/alphafold/requirements.txt \
-&& pip3 install --upgrade jax jaxlib==0.1.69+cuda110 -f \
-https://storage.googleapis.com/jax-releases/jax_releases.html
+&& pip3 install --upgrade jax==0.2.14 jaxlib==0.1.69+cuda111 -f \
+https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
 
 # Apply OpenMM patch.
 cd /opt/conda/lib/python3.7/site-packages
 patch -p0 < /app/alphafold/docker/openmm.patch
 
+# Add SETUID bit to the ldconfig binary so that non-root users can run it.
+chmod u+s /sbin/ldconfig.real
+
 %environment
 export PATH="/opt/conda/bin:$PATH"
 %runscript
 cd /app/alphafold
+ldconfig
 exec python /app/alphafold/run_alphafold.py "$@"
-%startscript
-cd /app/alphafold
-exec python /app/alphafold/run_alphafold.py "$@"
+# %startscript
+# cd /app/alphafold
+# exec python /app/alphafold/run_alphafold.py "$@"

example_slurm_job.sh

@@ -1,32 +1,45 @@
 #!/bin/bash
-#SBATCH --partition=gpu
-#SBATCH --time=2:00:00
-#SBATCH --gres=gpu:1
+#SBATCH -p gpu
+#SBATCH --time=18:00:00
+#SBATCH --gpus=4
 #SBATCH --cpus-per-gpu=12
-#SBATCH --mem-per-gpu=30G
+#SBATCH --mem=140G
 
-module load alphafold
+module load alphafold/2.2.2
+module load python/gcc/3.10
 
-# the alphafold modulefile should define:
-# * ALPHAFOLD_DIR -- install location AlphaFold
-# * ALPHAFOLD_DATADIR -- the DOWLOAD_DIR set in scripts/download_all_data.sh
+### Check values of some environment variables
+echo SLURM_JOB_GPUS=$SLURM_JOB_GPUS
+echo ALPHAFOLD_DIR=$ALPHAFOLD_DIR
+echo ALPHAFOLD_DATADIR=$ALPHAFOLD_DATADIR
 
-# Run AlphaFold; default is to use GPUs, i.e. "--use_gpu"
-python3 ${ALPHAFOLD_DIR}/singularity/run_singularity.py \
-  --fasta_paths=T1050.fasta \
-  --max_template_date=2020-05-14 \
-  --preset=reduced_dbs
+###
+### README This runs AlphaFold 2.2.2 on the T1050.fasta file
+###
 
-# AlphaFold should use all GPU devices available to the job.
+# AlphaFold should use all GPU devices available to the job by default.
 # To explicitly specify use of GPUs, and the GPU devices to use, add
 #   --use_gpu --gpu_devices=${SLURM_JOB_GPUS}
-
+#
 # To run the CASP14 evaluation, use:
-#   --preset=casp14
-
-# To benchmark, running multiple JAX model evaluations:
+#   --model_preset=monomer_casp14
+#
+# To benchmark, running multiple JAX model evaluations (NB this 
+# significantly increases run time):
 #   --benchmark
 
-# Copy all output from AlphaFold back to directory where "sbatch" command was issued
-cp -R $TMPDIR $SLURM_SUBMIT_DIR
+# Run AlphaFold; default is to use GPUs, i.e. "--use_gpu" can be omitted.
+python3 ${ALPHAFOLD_DIR}/singularity/run_singularity.py \
+    --use_gpu --gpu_devices=${SLURM_JOB_GPUS} \
+    --data_dir=${ALPHAFOLD_DATADIR} \
+    --fasta_paths=T1050.fasta \
+    --max_template_date=2020-05-14 \
+    --model_preset=monomer_casp14 \
+    --benchmark
+
+echo INFO: AlphaFold returned $?
+
+### Copy Alphafold output back to directory where "sbatch" command was issued.
+mkdir $SLURM_SUBMIT_DIR/Output-$SLURM_JOB_ID
+cp -R $TMPDIR $SLURM_SUBMIT_DIR/Output-$SLURM_JOB_ID