From c01e119883f51489fa49957205aafe0bbcbb924e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Wed, 4 Mar 2026 09:32:05 +0100 Subject: [PATCH 1/3] Consolidate GPU destinations Replace `condor_docker_gpu_pxe` and `condor_singularity_gpu_pxe` with a single containerized GPU destination `condor_container_gpu` that inherits from `condor_container`. In addition, make `condor_docker_gpu_pxe_divide4` inherit from `condor_container`. Simplifies the TPV configuration for GPU jobs (and fixes problems with a few edge cases), closing https://github.com/usegalaxy-eu/issues/issues/928. --- files/galaxy/tpv/destinations.yml.j2 | 81 +++++++++++----------------- 1 file changed, 31 insertions(+), 50 deletions(-) diff --git a/files/galaxy/tpv/destinations.yml.j2 b/files/galaxy/tpv/destinations.yml.j2 index 4efc3ebc9..fdbd91e3b 100644 --- a/files/galaxy/tpv/destinations.yml.j2 +++ b/files/galaxy/tpv/destinations.yml.j2 @@ -457,6 +457,8 @@ destinations: - name: APPTAINER_CACHEDIR value: /scratch/singularity/ params: + request_gpus: "{gpus or 0}" + # we could set this to false, and usually, afaik, this would then fallback to conda-resolution, however, I put in here a `container:` fallback, so maybe we can leave that to true require_container: true # @@ -502,6 +504,35 @@ destinations: - docker - singularity + condor_container_gpu: + inherits: condor_container + max_accepted_cores: 128 + max_accepted_mem: 500 + min_accepted_gpus: 1 + max_accepted_gpus: 4 + params: + docker_sudo: false + docker_net: bridge + docker_auto_rm: true + outputs_to_working_directory: false + container_monitor_result: callback + + requirements: 'GalaxyGroup == "pxe-gpu"' + docker_run_extra_arguments: "{entity.params.get('docker_run_extra_arguments') or ''} --gpus all --env CUDA_VISIBLE_DEVICES=$_CONDOR_AssignedGPUs --env NVIDIA_VISIBLE_DEVICES=$_CONDOR_AssignedGPUs" + singularity_run_extra_arguments: "{entity.params.get('singularity_run_extra_arguments') or ''} --nv --env CUDA_VISIBLE_DEVICES=$_CONDOR_AssignedGPUs" + env: + GPU_AVAILABLE: 1 + + # This means a GPU can be shared by max 4 jobs at the same time + condor_container_gpu_divide4: + inherits: condor_container_gpu + max_accepted_gpus: 1 + params: + requirements: 'GalaxyGroup == "pxe-gpu-div4"' + scheduling: + require: + - gpu-divided + condor_singularity_with_conda: inherits: basic_singularity_destination runner: condor @@ -536,53 +567,3 @@ destinations: scheduling: require: - internal - - condor_docker_gpu_pxe: - inherits: basic_docker_destination - # shorter than inheriting from condor_gpu - runner: condor - max_accepted_cores: 128 - max_accepted_mem: 500 - min_accepted_gpus: 1 - max_accepted_gpus: 4 - scheduling: - # Try to make this the default GPU destination if a tool is untagged (e.g. from upstream) - accept: - - docker - env: - GPU_AVAILABLE: 1 - params: - requirements: 'GalaxyGroup == "pxe-gpu"' - request_gpus: "{gpus or 0}" - docker_run_extra_arguments: "{entity.params.get('docker_run_extra_arguments') or ''} --gpus all --env CUDA_VISIBLE_DEVICES=$_CONDOR_AssignedGPUs --env NVIDIA_VISIBLE_DEVICES=$_CONDOR_AssignedGPUs" - - # This means a GPU can be shared by max 4 jobs at the same time - condor_docker_gpu_pxe_divide4: - inherits: condor_docker_gpu_pxe - max_accepted_gpus: 1 - params: - requirements: 'GalaxyGroup == "pxe-gpu-div4"' - scheduling: - require: - - gpu-divided - - condor_singularity_gpu_pxe: - inherits: basic_singularity_destination - # shorter than inheriting from condor_gpu - runner: condor - max_accepted_cores: 128 - max_accepted_mem: 500 - min_accepted_gpus: 1 - max_accepted_gpus: 1 - env: - - name: SINGULARITY_CACHEDIR - value: /scratch/singularity/ - - name: APPTAINER_CACHEDIR - value: /scratch/singularity/ - scheduling: - require: - - singularity - params: - requirements: 'GalaxyGroup == "pxe-gpu"' - request_gpus: "{gpus or 0}" - singularity_run_extra_arguments: "{entity.params.get('singularity_run_extra_arguments') or ''} --nv --env CUDA_VISIBLE_DEVICES=$_CONDOR_AssignedGPUs" From 6a1f42d2c27b34fd312669b94f2df8b12b12337c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Wed, 4 Mar 2026 14:22:00 +0100 Subject: [PATCH 2/3] Remove various parameters from `condor_container_gpu` destination Remove `docker_sudo`, `docker_net`, `docker_auto_rm`, `outputs_to_working_directory`, `container_monitor_result`. Some of them were copied over from embedded Pulsar in the past. The defaults should be good for regular jobs that do not run in embedded Pulsar. --- files/galaxy/tpv/destinations.yml.j2 | 6 ------ 1 file changed, 6 deletions(-) diff --git a/files/galaxy/tpv/destinations.yml.j2 b/files/galaxy/tpv/destinations.yml.j2 index fdbd91e3b..77b8a53e5 100644 --- a/files/galaxy/tpv/destinations.yml.j2 +++ b/files/galaxy/tpv/destinations.yml.j2 @@ -511,12 +511,6 @@ destinations: min_accepted_gpus: 1 max_accepted_gpus: 4 params: - docker_sudo: false - docker_net: bridge - docker_auto_rm: true - outputs_to_working_directory: false - container_monitor_result: callback - requirements: 'GalaxyGroup == "pxe-gpu"' docker_run_extra_arguments: "{entity.params.get('docker_run_extra_arguments') or ''} --gpus all --env CUDA_VISIBLE_DEVICES=$_CONDOR_AssignedGPUs --env NVIDIA_VISIBLE_DEVICES=$_CONDOR_AssignedGPUs" singularity_run_extra_arguments: "{entity.params.get('singularity_run_extra_arguments') or ''} --nv --env CUDA_VISIBLE_DEVICES=$_CONDOR_AssignedGPUs" From 7da29be7b065fcd8952894781e8ffb3567a8aedb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Wed, 4 Mar 2026 14:27:08 +0100 Subject: [PATCH 3/3] Do not run Singularity containers in `condor_container_gpu_divide4` Sharing GPUs has not been tested with Singularity. --- files/galaxy/tpv/destinations.yml.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/files/galaxy/tpv/destinations.yml.j2 b/files/galaxy/tpv/destinations.yml.j2 index 77b8a53e5..9a6358a66 100644 --- a/files/galaxy/tpv/destinations.yml.j2 +++ b/files/galaxy/tpv/destinations.yml.j2 @@ -522,10 +522,13 @@ destinations: inherits: condor_container_gpu max_accepted_gpus: 1 params: + singularity_enabled: false # sharing GPUs has not been tested with Singularity requirements: 'GalaxyGroup == "pxe-gpu-div4"' scheduling: require: - gpu-divided + reject: + - singularity # sharing GPUs has not been tested with Singularity condor_singularity_with_conda: inherits: basic_singularity_destination