Merge branch 'master' into fix_validator_flagging_runtime_outliers

Author: amereghe

boinc_software/cronjobs/acrontab_jobs_sixtadm.txt

@@ -4,7 +4,7 @@
 # Puppet Name: worksubmit sixtrack 2
 7,17,27,37,47,57 * * * * boincai11.cern.ch /usr/local/boinc/project/sixtrack/bin/cron.submit-sixtracktest-simo3 -m 3000 > /dev/null 2>&1
 # Puppet Name: worksubmit sixtrack
-1,11,21,31,41,51 * * * * boincai11.cern.ch /usr/local/boinc/project/sixtrack/bin/cron.submit-sixtrack-simo3 -m 3000 > /dev/null 2>&1
+1 * * * * boincai11.cern.ch /usr/local/boinc/project/sixtrack/bin/cron.submit-sixtrack-simo3 -m 3000 > /dev/null 2>&1
 # zip all WUs which were not given back to user
 # crontab every 3h
 30 */3 * * * boincai11.cern.ch cd /share/sixtrack/assimilation ; /usr/local/boinc/project/sixtrack/bin/zip-trashed-WUs.sh > /dev/null 2>&1
@@ -32,6 +32,9 @@
 0 4 21 * * lxplus.cern.ch cd /afs/cern.ch/work/b/boinc/boinc ; /afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/cronjobs/listDeleteStudies.sh results >> /afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/cronjobs/deleteStudies.log 2>&1
 0 5 21 * * lxplus.cern.ch cd /afs/cern.ch/work/b/boinc/boinctest ; /afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/cronjobs/listDeleteStudies.sh results >> /afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/cronjobs/deleteStudies.log 2>&1
 #
+# tar.gz new job and spool it to EOS
+4,14,24,34,44,54 * * * * lxplus.cern.ch /afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/cronjobs/mv2EOS.sh > /dev/null 2>&1
+#
 #
 # new scripts for monitoring BOINC server:
 # - server status and plotting;

boinc_software/cronjobs/cron.submit-simo3

@@ -7,6 +7,7 @@
 # Updated 06.10.2016 A.Mereghetti, added megaZip functionality
 # Updated 25.03.2017 A.Mereghetti, SixOut.zip + appname (from .desc)
 # Updated 01.05.2019 A.Mereghetti, appnum + default appNum
+# Updated 13.09.2019 A.Mereghetti, EOS spooling
 set -e
 
 #Set Variables
@@ -37,6 +38,13 @@ WU_template=templates/${applicationDef}_wu_template.xml
 # this template will bring back also the Sixout.zip results
 result_template_SixOutZip=templates/${applicationDef}_res_template.xml
 
+export EOS_MGM_URL=root://eosuser.cern.ch 
+pathInEos=/eos/user/s/sixtadm/spooldirs/uploads/boinc
+tmpDirBase=/tmp/sixtadm/`basename $0`/boinc
+nMaxRetrial=10
+
+TLpath="/afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/monitor-boinc-server/general-activity/SixTrack_status_????-??-??.dat"
+
 #Define finctions
 abort(){
     exitcode=1
@@ -112,7 +120,8 @@ run_spool_megazip(){
 
     megaZip=1
 
-    find "$spooldirUpload" -mmin +5 -name "*.zip" | (
+    # take into account arrival time
+    find "$spooldirUpload" -mmin +5 -name "*.zip" -printf "%T+\t%p\n" | sort | awk '{print ($2)}' | (
 	while read workBatch ; do
 	    unzip -t $workBatch  >/dev/null 2>&1
 	    if [ $? -ne 0 ] ; then
@@ -123,22 +132,25 @@ run_spool_megazip(){
 	    origPath=`mktemp -d -p /tmp/`
 	    warn "$workBatch being processed in $origPath ..."
 	    cp $workBatch $origPath
-	    # remember that you must run in $boincdir
-	    unzip $workBatch -d $origPath >/dev/null 2>&1
-	    # check for unzipped .desc files
-	    find $origPath -maxdepth 1 -type f -name '*.desc' | (
-	        while read descfile ; do
-		    # process the desc files
-		    if submit_descfile "$descfile" ; then
-			logstudy "Submitted $WUname"
-		    else
-			logstudy "Problem submitting $WUname"
-		    fi
-	        done
-	    )
-	    warn " ...going on with next MegaZip."
-	    rm -rf $origPath
-	    rm $workBatch
+	    nForeseen=`unzip -l ${workBatch} | wc -l`
+	    if traffic_light ; then
+	        # remember that you must run in $boincdir
+	        unzip $workBatch -d $origPath >/dev/null 2>&1
+	        # check for unzipped .desc files
+	        find $origPath -maxdepth 1 -type f -name '*.desc' | (
+	            while read descfile ; do
+	        	    # process the desc files
+	        	    if submit_descfile "$descfile" ; then
+	        		logstudy "Submitted $WUname"
+	        	    else
+	        		logstudy "Problem submitting $WUname"
+	        	    fi
+	            done
+	        )
+	        warn " ...going on with next MegaZip."
+	        rm -rf $origPath
+	        rm $workBatch
+	    fi
 	done
     )
 }
@@ -158,7 +170,8 @@ run_spool(){ # max_jobs_to_submit, max_jobs_perStudy, specific_study
 
     if [ -z "$3" ] ; then
         #find the work dirs 2 levels down
-	local allWorkDirs=`find "$spooldir" -maxdepth 2 -type d -name "work"`
+	# take into account arrival time
+	local allWorkDirs=`find "$spooldir" -maxdepth 2 -type d -name "work" -printf "%T+\t%p\n" | sort | awk '{print ($2)}'`
     else
 	#target a specific study
 	local allWorkDirs=$spooldir/$3/work
@@ -168,39 +181,172 @@ run_spool(){ # max_jobs_to_submit, max_jobs_perStudy, specific_study
     for workdir in ${allWorkDirs} ; do
         ! ${lMaxJobsPerStudy} || local __StudyComplete=0
 	#check for desc files in the current work dir, and subfolders
+	# take into account arrival time
 	if ${lMaxJobsPerStudy} ; then
-	    local allDescs=`find "$workdir" -maxdepth 2 -type f -name '*.desc' | head -n ${max_jobs_perStudy}`
+	    local allDescs=`find "$workdir" -maxdepth 2 -type f -name '*.desc' -printf "%T+\t%p\n" | sort | awk '{print ($2)}' | head -n ${max_jobs_perStudy}`
+	else
+	    local allDescs=`find "$workdir" -maxdepth 2 -type f -name '*.desc' -printf "%T+\t%p\n" | sort | awk '{print ($2)}'`
+	fi
+	nForeseen=`echo "${allDescs}" | wc -l`
+	let nForeseen=${nForeseen}*2
+	if traffic_light ; then
+	    for descfile in ${allDescs} ; do
+	        #process the desc files
+	        origPath=`dirname ${descfile}`
+                ! ${lMaxJobsPerStudy} || let __StudyComplete+=1
+	        if submit_descfile "$descfile" ; then
+	    	logstudy "Submitted $WUname"
+	    	#stop after max_jobs (0=unlimited)
+	    	if ${lMaxJobs} ; then
+	    	    complete=$(( $complete + 1 ))
+	    	    if [ $complete -ge $max_jobs ] ; then
+                            log "reached ${max_jobs} in total"
+	    		break 2
+	    	    fi
+	    	fi
+	        else
+	    	logstudy "Problem submitting $WUname"
+	        fi
+	    done
+	    if [ -n "${allDescs}" ] ; then
+	        if ${lMaxJobsPerStudy} ; then
+                    if [ ${__StudyComplete} -eq ${max_jobs_perStudy} ] ; then
+	    	    log "limit to ${max_jobs_perStudy} reached for ${workdir}"
+	            fi
+                fi
+	    fi
+	fi
+    done
+}
+
+run_spool_EOS(){
+
+    # get .tar.gz files from ${pathInEos} (path in EOS) and 
+    #     ${spooldirUpload} (back-up in AFS in case of problems with EOS)
+
+    megaZip=0
+    origPath=${tmpDirBase}
+
+    for __tmpSourceDir in ${pathInEos} ${spooldirUpload} ; do
+
+	if [[ "${__tmpSourceDir}" == "/eos"* ]] ; then
+	    gzFiles=`eos find -name "*.tar.gz" ${__tmpSourceDir}`
 	else
-	    local allDescs=`find "$workdir" -maxdepth 2 -type f -name '*.desc'`
+	    gzFiles=`find ${__tmpSourceDir} -mmin +5 -name "*.tar.gz"`
 	fi
-	for descfile in ${allDescs} ; do
-	    #process the desc files
-	    origPath=`dirname ${descfile}`
-            ! ${lMaxJobsPerStudy} || let __StudyComplete+=1
-	    if submit_descfile "$descfile" ; then
-		logstudy "Submitted $WUname"
-		#stop after max_jobs (0=unlimited)
-		if ${lMaxJobs} ; then
-		    complete=$(( $complete + 1 ))
-		    if [ $complete -ge $max_jobs ] ; then
-                        log "reached ${max_jobs} in total"
-			break 2
-		    fi
+	
+	log ".tar.gz files in ${__tmpSourceDir}:"
+	echo "${gzFiles}" | log
+	
+        for gzFile in ${gzFiles} ; do
+        
+            gzFileName=`basename ${gzFile}`
+
+	    if [[ "${__tmpSourceDir}" == "/eos"* ]] ; then
+		myCommand="xrdcp --cksum adler32 ${EOS_MGM_URL}/${pathInEos}/${gzFileName} ${tmpDirBase}"
+		loopMe
+		if [ $? -ne 0 ] ; then
+        	    log "unable to download ${gzFileName} from EOS ${EOS_MGM_URL}/${pathInEos}"
+        	    continue
 		fi
 	    else
-		logstudy "Problem submitting $WUname"
+		myCommand="cp ${gzFile} ${tmpDirBase}"
+		loopMe
+		if [ $? -ne 0 ] ; then
+		    log "unable to download ${gzFileName} from AFS ${spooldirUpload}"
+		    continue
+		fi
 	    fi
-	done
-	if [ -n "${allDescs}" ] ; then
-	    if ${lMaxJobsPerStudy} ; then
-                if [ ${__StudyComplete} -eq ${max_jobs_perStudy} ] ; then
-		    log "limit to ${max_jobs_perStudy} reached for ${workdir}"
-	        fi
-            fi
+
+	    spool_EOS_gunzip || continue
+		
+	    tarFile=${tmpDirBase}/${gzFileName%.gz}
+	    spool_EOS_untar || continue
+	    
+	    spool_EOS_descfiles
+		
+	    # clean
+	    if [[ "${__tmpSourceDir}" == "/eos"* ]] ; then
+		eos rm ${pathInEos}/${gzFileName}
+	    else
+		rm ${gzFile}
+	    fi
+	    rm ${tarFile}
+        done
+
+    done
+}
+
+function spool_EOS_gunzip(){
+    myCommand="gunzip ${tmpDirBase}/${gzFileName}"
+    if loopMe ; then
+	return 0
+    else
+        log "unable to gunzip ${gzFileName}"
+	rm ${tmpDirBase}/${gzFileName}
+        return 1
+    fi
+}
+
+function spool_EOS_untar(){
+    nForeseen=`tar -tvf ${tarFile} | wc -l`
+    # tar -tvf always returns a '.'
+    nForeseen=$(( $nForeseen -1 ))
+    if traffic_light ; then
+	myCommand="tar -xvf ${tarFile} -C ${tmpDirBase}"
+	loopMe
+	if [ $? -ne 0 ] ; then
+            log "unable to untar ${tarFile}"
+	    rm ${tarFile}
+            return 1
 	fi
+    else
+	rm ${tarFile}
+        return 2
+    fi
+    return 0
+}
+
+spool_EOS_descfiles(){
+    # remember that you must run in $boincdir
+    for descfile in `find "${tmpDirBase}" -type f -name '*.desc'` ; do
+        #process the desc files
+        if submit_descfile "$descfile" ; then
+            logstudy "Submitted $WUname"
+        else
+            logstudy "Problem submitting $WUname"
+        fi
     done
 }
 
+function traffic_light(){
+    # nPresent: how many tasks presently in queue
+    # nTreated: how many tasks (factor 2 only wrt WUs) will be submitted now
+    # 0: green
+    # 1: red
+    local __nPresentTemp=`tail -1 ${TLpath} 2> /dev/null | awk '{print ($3)}'`
+    if [ -z "${__nPresentTemp}" ] ; then
+	if [ -z "${nPresent}" ] ; then
+	    log "unable to get number of queued ${applicationDef} tasks from ${TLpath}"
+	    return 2
+	else
+	    log "unable to get number of queued ${applicationDef} tasks from ${TLpath} - going on with ${nPresent}"
+	fi
+    else
+	nPresent=${__nPresentTemp}
+    fi
+    local __nTemp=$(( ${nTreated} +${nPresent} +${nForeseen} ))
+    if [ ${__nTemp} -ge ${currentLimit} ] ; then
+	log "traffic light is red (treated+present+foreseen>limit): ${nTreated}+${nPresent}+${nForeseen}>${currentLimit}"
+	nResidual=0
+	return 1
+    else
+	log "traffic light is green (treated+present+foreseen>limit): ${nTreated}+${nPresent}+${nForeseen}<${currentLimit}"
+	let nResidual=${currentLimit}+${__nTemp}
+	return 0
+    fi
+}
+
 submit_descfile(){ 
         # variables from outer function:
         # - megaZip: flag for megaZip;
@@ -309,6 +455,7 @@ create_cmd="$boincdir/bin/create_work 		   \
 	fi
         # in case of megazip, zipfile will be deleted since the
 	#  tmp dir will be deleted
+	nTreated=$(( $nTreated +2 ))
 
 	return 0
 }
@@ -406,6 +553,22 @@ clearWUvars(){
 	unset WUapplication
 }
 
+function loopMe(){
+    # myCommand should be defined before the call
+    local __reply=1
+    local __iTrials=0
+    while [ ${__reply} -ne 0 ] && [ ${__iTrials} -lt ${nMaxRetrial} ] ; do
+        let __iTrials+=1
+        log "command: ${myCommand} - run at `date` - trial ${__iTrials}"
+        ${myCommand}
+        __reply=$?
+    done
+    if [ ${__reply} -ne 0 ] ; then
+        log " ...giving up on command."
+    fi
+    return ${__reply}
+}
+
 printhelp(){
 cat <<EOF
 Usage: $(basename $0) [options]
@@ -457,14 +620,58 @@ cd $boincdir
 getlock
 #Klog
 
+megaZip=0
+nResidual=0
+
 # preliminary: remove temp dirs in work older than 1d
-log find ${spooldir} -mindepth 3 -maxdepth 3 -mtime +1 -type d -empty -delete -print
-find ${spooldir} -mindepth 3 -maxdepth 3 -mtime +1 -type d -empty -delete -print | log
+# log find ${spooldir} -mindepth 3 -maxdepth 3 -mtime +1 -type d -empty -delete -print
+# find ${spooldir} -mindepth 3 -maxdepth 3 -mtime +1 -type d -empty -delete -print | log
 
-log run_spool $maxjobs $maxjobs_perStudy $studyName
-run_spool $maxjobs $maxjobs_perStudy $studyName
+# traffic light
+currentLimit=`grep -v '#' ${boincdir}/queue_thresholds.txt | awk -v "appName=${applicationDef}" '{if ($1==appName) {print ($2)}}' | tail -1`
+if [ -z "${currentLimit}" ] ; then
+    abort 1 "unable to get threshold of queued ${applicationDef} tasks from ${boincdir}/queue_thresholds.txt"
+fi
+nTreated=0
+nForeseen=0
+traffic_light || abort 1 "traffic light: red"
+
+if ! [ -d ${tmpDirBase} ] ; then
+    mkdir -p ${tmpDirBase}
+    if [ $? -ne 0 ] ; then
+	abort 1 "problems in creating ${tmpDirBase} - cannot proceed"
+    fi
+fi
+origPath=${tmpDirBase}
+log "remaining .tar.gz files in ${tmpDirBase} from previous run"
+for gzFile in `find ${tmpDirBase} -name "*.tar.gz"` ; do
+    gzFileName=`basename ${gzFile}`
+    spool_EOS_gunzip || continue
+    tarFile=${tmpDirBase}/${gzFileName%.gz}
+    spool_EOS_untar || continue
+    spool_EOS_descfiles
+    rm ${tarFile}
+done
+log "remaining .tar files in ${tmpDirBase} from previous run"
+for tarFile in `find ${tmpDirBase} -name "*.tar"` ; do
+    spool_EOS_untar || continue
+    spool_EOS_descfiles
+    rm ${tarFile}
+done
+log "remaining .desc files in ${tmpDirBase} from previous run"
+spool_EOS_descfiles
+
+log run_spool_EOS
+run_spool_EOS
+
+if [ ${nResidual} -gt 0 ] && [ ${maxjobs} -gt ${nResidual} ] ; then
+    log "updating maxjobs from ${maxjobs} to ${nResidual} following limit on max number or queued tasks at ${currentLimit}"
+    maxjobs=${nResidual}
+fi
+# log run_spool $maxjobs $maxjobs_perStudy $studyName
+# run_spool $maxjobs $maxjobs_perStudy $studyName
 
-log run_spool_megazip
-run_spool_megazip
+# log run_spool_megazip
+# run_spool_megazip
 
 log FINISHING