DQM automation (#192)

* Adapt directory paths to new local ones at CEA and new ones on DIRAC

* Jobs running on GRIF with 16 GB RAM max

* Updated path on DIRAC for DQM result parser on OpenStack

* Updated DIRAC group (from cta_nectarcam to ctao_nectarcam) for proxy initialization

* Fix path pattern on DIRAC

* Don't set a requirement on minimal amount of RAM per job

Author: jlenain

src/nectarchain/user_scripts/jlenain/cronjob_parse_dqm_fits_file.sh

@@ -12,12 +12,12 @@ exec 1>"$LOGFILE" 2>&1
 conda activate nectar-dev
 
 # Initialize DIRAC proxy from user certificate:
-if ! dirac-proxy-init -M -g cta_nectarcam --pwstdin < ~/.dirac.pwd; then
+if ! dirac-proxy-init -M -g ctao_nectarcam --pwstdin < ~/.dirac.pwd; then
     echo "DIRAC proxy initialization failed..."
     exit 1
 fi
 
-remoteParentDir="/vo.cta.in2p3.fr/user/j/jlenain/nectarcam/dqm"
+remoteParentDir="/ctao/user/j/jlenain/nectarcam/dqm"
 nectarchainScriptDir="/opt/cta/nectarchain/src/nectarchain/user_scripts/jlenain"
 
-python ${nectarchainScriptDir}/parse_dqm_fits_file.py -r $(dls ${remoteParentDir} | grep -ve "/vo.cta" | awk -F. '{print $1}' | awk -Fn '{print $2}' | tr '\n' ' ')
+python ${nectarchainScriptDir}/parse_dqm_fits_file.py -r $(dls ${remoteParentDir} | grep -ve "/ctao" | awk -F. '{print $1}' | awk -Fn '{print $2}' | tr '\n' ' ')

src/nectarchain/user_scripts/jlenain/dqm_job_submitter/cronjob_launchDQM.sh

@@ -13,8 +13,9 @@ exec 1>"$LOGFILE" 2>&1
 source /opt/cta/mambaforge/etc/profile.d/conda.sh
 conda activate ctadirac
 
-localParentDir="/data/nvme/ZFITS"
+localParentDir="/data/nvme/NectarCAMQM/ZFITS"
 remoteParentDir="/vo.cta.in2p3.fr/nectarcam"
+remoteParentDir="/ctao/nectarcam"
 nectarchainScriptDir="$HOME/local/src/python/cta-observatory/nectarchain/src/nectarchain/user_scripts/jlenain/dqm_job_submitter"
 
 cd $nectarchainScriptDir || (echo "Failed to cd into ${nectarchainScriptDir}, exiting..."; exit 1)
@@ -30,7 +31,7 @@ for run in $(find ${localParentDir} -type f -name "NectarCAM*.fits.fz" | awk -F.
         echo "  Run $run: number of local and remote files matching, will attempt to submit a DQM job"
         # Has this DQM run already been submitted ?
         if [ $(dstat | grep --count -e "NectarCAM DQM run ${run}") -eq 0 ]; then
-            yyyymmdd=$(find ${localParentDir} -type f -name "NectarCAM.Run${run}.????.fits.fz" | head -n 1 | awk -F/ '{print $6}')
+            yyyymmdd=$(find ${localParentDir} -type f -name "NectarCAM.Run${run}.????.fits.fz" | head -n 1 | awk -F/ '{print $7}')
             yyyy=${yyyymmdd:0:4}
             mm=${yyyymmdd:4:2}
             dd=${yyyymmdd:6:2}

src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh

@@ -45,12 +45,12 @@ WRAPPER="singularity_wrapper.sh"
 # We can instantiate a particular container version:
 # CONTAINER="oras://ghcr.io/cta-observatory/nectarchain@sha256:cf5a812bdb1113d027facceec30009759535a8579eb4caf38f56143e65cb35e7"
 # or just use the latest available:
-# CONTAINER="oras://ghcr.io/cta-observatory/nectarchain:latest"
+CONTAINER="oras://ghcr.io/cta-observatory/nectarchain:latest"
 # of from CVMFS:
 # CONTAINER="/cvmfs/sw.cta-observatory.org/software/containers/nectarchain-latest"
-CONTAINER="/cvmfs/sw.hess-experiment.eu/software/containers/nectarchain_w_2025_03"
+# CONTAINER="/cvmfs/sw.hess-experiment.eu/software/containers/nectarchain_w_2025_03"
 OUTDIR=NectarCAM_DQM_Run${runnb}
-DIRAC_OUTDIR=/vo.cta.in2p3.fr/user/j/jlenain/nectarcam/dqm
+DIRAC_OUTDIR=/ctao/user/j/jlenain/nectarcam/dqm
 
 function exit_script() {
     return_code=$1

src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py

@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-#
-# Time-stamp: "2023-05-30 13:09:04 jlenain"
 
 import argparse
 import logging
@@ -173,7 +171,8 @@
     # j.setExecutable(f'{executable_wrapper}', '<SOME POSSIBLE ARGUMENTS such as run number>')
     j.setExecutable(f"{executable_wrapper}", f"-r {run}")
     # Force job to be run from a given Computing Element:
-    j.setDestination(["LCG.GRIF.fr", "ARC.CEA.fr"])
+    # j.setDestination(["LCG.GRIF.fr", "ARC.CEA.fr"])
+    j.setDestination(["LCG.GRIF.fr"])
     # j.setTag(["16GBMemory"])
     j.setName(f"NectarCAM DQM run {run}")
     j.setJobGroup("NectarCAM DQM")