Adjusting the script again for LUMI

Author: kinow

cwl-conformance-tests/cwl_conformance_tests.sh

@@ -72,8 +72,10 @@ fi
 CWL_VERSIONS=("v1.0")
 
 # How many tests we will run in parallel
-# N_TESTS_IN_PARALLEL=16
-N_TESTS_IN_PARALLEL=4
+# You may find that 1 is too slow, and 16 is fast. But this number is applied
+# to all tests; login, Slurm, batch. If you use 16 on Slurm or batch, you will
+# very likely be penalised by the Slurm algorithm. 4 or 8 may work better.
+N_TESTS_IN_PARALLEL=8
 
 # CWL repo URLs
 declare -A CWL_REPO
@@ -101,11 +103,6 @@ declare -A TOOL_BIN
 TOOL_BIN[cwltool]="cwltool"
 TOOL_BIN[toil]="toil-cwl-runner"
 
-# Extra args passed to the binaries
-declare -A TOOL_ARGS
-TOOL_ARGS[cwltool]="--singularity --enable-dev --tmpdir-prefix=$HPC_SCRATCH_DIR"
-TOOL_ARGS[toil]="--singularity --disableCaching --disableProgress --defaultMemory=2G --maxMemory=2G --cwl-min-ram=2G --enable-dev --clean=always"
-
 # Base working directory
 BASE_DIR=$(pwd)/runs
 mkdir -p "$BASE_DIR"
@@ -140,7 +137,7 @@ setup_python_env() {
         "$ENV_DIR/bin/pip" install cwltool==3.2.20260413085819
         ;;
     toil)
-        "$ENV_DIR/bin/pip" install toil[cwl]==9.5.0
+        "$ENV_DIR/bin/pip" install toil[cwl]==9.4.1
         ;;
     esac
 }
@@ -224,30 +221,46 @@ run_tests() {
         # shellcheck disable=SC1090
         source "$BASE_DIR/venv-$TOOL/bin/activate"
 
+        # ------------ DIRS
+
+        RUN_ROOT_DIR="$HPC_SCRATCH_DIR/$(uuidgen)"
+        mkdir -p "$RUN_ROOT_DIR"
+
+        # Directories used by Toil
+        declare -A RUN_DIRS=(
+            [outdir]="$RUN_ROOT_DIR/outdir"
+            [logdir]="$RUN_ROOT_DIR/logdir"
+            [tmpdir]="$RUN_ROOT_DIR/tmpdir"
+            [tmpoutdir]="$RUN_ROOT_DIR/tmpoutdir"
+            [workdir]="$RUN_ROOT_DIR/workdir"
+            [coordination]="$RUN_ROOT_DIR/coordination"
+        )
+
+        # Extra args passed to the binaries
+        declare -A TOOL_ARGS
+        TOOL_ARGS[cwltool]="--singularity --enable-dev --tmpdir-prefix=${RUN_DIRS[tmpdir]}"
+
+        # JOB_STORE="${RUN_ROOT_DIR}/$(uuidgen)"
+
+        TOOL_ARGS[toil]="--singularity \
+        --disableCaching \
+        --disableProgress \
+        --defaultMemory=2G \
+        --maxMemory=2G \
+        --cwl-min-ram=2G \
+        --enable-dev \
+        --clean=always \
+        --outdir=${RUN_DIRS[outdir]} \
+        --log-dir=${RUN_DIRS[logdir]} \
+        --tmpdir-prefix=${RUN_DIRS[tmpdir]} \
+        --tmp-outdir-prefix=${RUN_DIRS[tmpoutdir]} \
+        --workDir=${RUN_DIRS[workdir]} \
+        --coordinationDir=${RUN_DIRS[coordination]}"
+        # --jobStore=${JOB_STORE}"
+
         RUNNER="${TOOL_BIN[$TOOL]}"
         EXTRA="${TOOL_ARGS[$TOOL]}"
 
-        RUN_ROOT_DIR=""
-        JOB_STORE=""
-
-        if [ "$TOOL" = "toil" ]; then
-            # One dedicated root directory for this complete tool/version/mode run.
-            RUN_ROOT_DIR="${HPC_SCRATCH_DIR}/$(uuidgen)"
-            mkdir -p "$RUN_ROOT_DIR"
-
-            # Toil job stores cannot be reused, so create a fresh one for each run.
-            JOB_STORE="${RUN_ROOT_DIR}/$(uuidgen)"
-
-            # Keep all Toil working directories under the same dedicated root.
-            EXTRA="${EXTRA} --outdir=${RUN_ROOT_DIR}"
-            EXTRA="${EXTRA} --log-dir=${RUN_ROOT_DIR}"
-            EXTRA="${EXTRA} --tmpdir-prefix=${RUN_ROOT_DIR}"
-            EXTRA="${EXTRA} --tmp-outdir-prefix=${RUN_ROOT_DIR}"
-            EXTRA="${EXTRA} --workDir=${RUN_ROOT_DIR}"
-            EXTRA="${EXTRA} --coordinationDir=${RUN_ROOT_DIR}"
-            EXTRA="${EXTRA} --jobStore=${JOB_STORE}"
-        fi
-
         # For toil + batch mode, we want to use the --batchSystem=slurm option.
         # And the --bypass-file-store is used to assume the data is available
         # everywhere (e.g., GPFS, Lustre).
@@ -270,39 +283,30 @@ run_tests() {
             TEST_FILE="conformance_test_v1.0.yaml"
         fi
 
-        # cwltest parallelism.
-        # Toil needs one test at a time because each cwltest invocation uses
-        # the same Toil job store.
-        TESTS_IN_PARALLEL="$N_TESTS_IN_PARALLEL"
-        if [ "$TOOL" = "toil" ] && [ "$MODE" = "batch" ]; then
-            TESTS_IN_PARALLEL=1
+        CWLTEST_OUTDIR="$RUN_ROOT_DIR/$(uuidgen)"
+
+        # Pre-create all directories (cwltest only uses tmpdir, and it handles directory creation).
+        if [ "$TOOL" = "toil" ] ; then
+            mkdir -p "${RUN_DIRS[@]}"
+            mkdir -p "$CWLTEST_OUTDIR"
         fi
 
-        # Build base args
+        # Build base args passed to cwltest.
         ARGS=(
             "--verbose"
             "--test=${TEST_FILE}"
+            "--outdir=${CWLTEST_OUTDIR}"
             "--junit-verbose"
             "--junit-xml=${TOOL_DIR}/junit.xml"
-            "--timeout=600"
-            "-j${TESTS_IN_PARALLEL}"
+            "-j${N_TESTS_IN_PARALLEL}"
             "--tool=$RUNNER"
-            "--"
+            "--timeout=3600"
         )
-
-        if [ "$TOOL" = "toil" ]; then
-            ARGS=(
-                "--verbose"
-                "--test=${TEST_FILE}"
-                "--outdir=${RUN_ROOT_DIR}"
-                "--junit-verbose"
-                "--junit-xml=${TOOL_DIR}/junit.xml"
-                "--timeout=600"
-                "-j${TESTS_IN_PARALLEL}"
-                "--tool=$RUNNER"
-                "--"
-            )
-        fi
+        # NOTE: On LUMI, the queues for the project I was using were a bit bad,
+        #       even though it was Sunday, and not many jobs were running
+        #       (fair-share?). Be careful with the timeout or you will have
+        #       several Timed-out errors.
+        # "--timeout=600"
 
         # split EXTRA into words safely
         read -r -a EXTRA_ARR <<<"$EXTRA"
@@ -321,16 +325,17 @@ run_tests() {
             echo "VERSION=$VERSION"
             echo "MODE=$MODE"
             echo "RUNNER=${TOOL_BIN[$TOOL]}"
-            echo "RUN_ROOT_DIR=${RUN_ROOT_DIR}"
-            echo "JOB_STORE=${JOB_STORE}"
+            echo "HPC_SCRATCH_DIR=$HPC_SCRATCH_DIR"
+            echo "RUN_ROOT_DIR=$RUN_ROOT_DIR"
+            echo "CWLTEST_OUTDIR=$CWLTEST_OUTDIR"
             echo "EXTRA=${EXTRA}"
             echo "================="
             echo ""
 
             "$RUNNER" --version || true
             echo ""
             set -exv
-            "$TEST_SCRIPT" "${ARGS[@]}" "${EXTRA_ARR[@]}"
+            "$TEST_SCRIPT" "${ARGS[@]}" -- "${EXTRA_ARR[@]}"
             set +exv
         }