[batch] Use earlyoom to terminate a process using too much memory

deploy/Dockerfile

@@ -1,23 +1,17 @@
 FROM python:3.11-slim-bookworm
 
-# Setup SSH
-RUN mkdir /root/.ssh
-COPY id_ed25519 /root/.ssh/id_ed25519
-COPY known_hosts /root/.ssh/known_hosts
-RUN chmod -R 0600 /root/.ssh
-
 # Install Git, Git LFS.  How to silence apt-get commands:
 # https://peteris.rocks/blog/quiet-and-unattended-installation-with-apt-get/
 RUN apt-get update -qq
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -qq -y git git-lfs < /dev/null > /dev/null
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -qq -y git git-lfs earlyoom < /dev/null > /dev/null
 # Cleanup apt cache
 RUN rm -rf /var/lib/apt/lists/*
 # Configure git lfs
 RUN git lfs install
 
 # Clone the TLOModel repository and move in it
 ARG BRANCH_NAME="master"
-RUN git clone --branch "${BRANCH_NAME}" git@github.com:UCL/TLOmodel.git /TLOmodel
+RUN git clone --branch "${BRANCH_NAME}" https://github.com/UCL/TLOmodel.git /TLOmodel
 WORKDIR /TLOmodel
 RUN git gc --aggressive
 

src/tlo/cli.py

@@ -234,6 +234,25 @@ def batch_submit(ctx, scenario_file, asserts_on, more_memory, keep_pool_alive, i
     task_dir = "${{AZ_BATCH_TASK_DIR}}"
     gzip_pattern_match = "{{txt,log}}"
     command = f"""
+    # Logfile for `earlyoom`
+    EARLYOOM_LOG=/tmp/earlyoom.log
+
+    function terminate_and_log() {{{{
+        # Terminate the `earlyoom` process
+        kill "${{{{EARLYOOM_PID}}}}"
+        # Print line(s) in the log file referencing terminated processes, if any.
+        grep "^sending SIGTERM to process" "${{{{EARLYOOM_LOG}}}}"
+    }}}}
+
+    # Add a trap at exit to terminate `earlyoom` and print relevant log lines.
+    trap terminate_and_log EXIT
+
+    # Delete existing log files, if any.
+    rm -f "${{{{EARLYOOM_LOG}}}}"
+    # Start the `earlyoom` daemon.
+    nohup earlyoom -m 95 -s 100 &> "${{{{EARLYOOM_LOG}}}}" &
+    EARLYOOM_PID="${{{{!}}}}"
+
     git fetch origin {commit.hexsha}
     git checkout {commit.hexsha}
     pip install -r requirements/base.txt