Merge remote-tracking branch 'origin/dev' into pro

Author: amereghe

boinc_software/cronjobs/cron.submit-sixtrack-simo3

@@ -106,17 +106,10 @@ getlock(){
 }
 
 
-run_spool_megazip(){ # max_jobs_to_submit
+run_spool_megazip(){
 
     megaZip=1
 
-    if [ -z "$1" -o "$1" = "0" ] ; then
-	local max_jobs=""
-    else
-	local max_jobs="$1"	
-	local complete=0
-    fi
-
     find "$spooldirUpload" -mmin +5 -name "*.zip" | (
 	while read workBatch ; do
 	    unzip -t $workBatch  >/dev/null 2>&1
@@ -136,13 +129,6 @@ run_spool_megazip(){ # max_jobs_to_submit
 		    # process the desc files
 		    if submit_descfile "$descfile" ; then
 			logstudy "Submitted $WUname"
-			#stop after max_jobs (0=unlimited)
-			if [ -n "$max_jobs" ] ; then
-			    complete=$(( $complete + 1 ))
-			    if [ $complete -ge $max_jobs ] ; then
-				break 2
-			    fi
-			fi
 		    else
 			logstudy "Problem submitting $WUname"
 		    fi
@@ -155,39 +141,61 @@ run_spool_megazip(){ # max_jobs_to_submit
     )
 }
 
-run_spool(){ # max_jobs_to_submit 
+run_spool(){ # max_jobs_to_submit, max_jobs_perStudy, specific_study
 
     megaZip=0
 
-    if [ -z "$1" -o "$1" = "0" ] ; then
-	local max_jobs=""
+    local max_jobs="$1"	
+    local complete=0
+    local lMaxJobs=false
+    [ -z "${max_jobs}" -o "${max_jobs}" = "0" ] || lMaxJobs=true
+
+    local max_jobs_perStudy="$2"	
+    local lMaxJobsPerStudy=false
+    [ -z "${max_jobs_perStudy}" -o "${max_jobs_perStudy}" = "0" ] || lMaxJobsPerStudy=true
+
+    if [ -z "$3" ] ; then
+        #find the work dirs 2 levels down
+	local allWorkDirs=`find "$spooldir" -maxdepth 2 -type d -name "work"`
     else
-	local max_jobs="$1"	
-	local complete=0
+	#target a specific study
+	local allWorkDirs=$spooldir/$3/work
     fi
 
-    #find the work dirs 2 levels down
-    find "$spooldir" -maxdepth 2 -type d -name "work" | (
+    # main loop
+    echo "${allWorkDirs}" | (
 	while read workdir ; do
-	    origPath=$workdir
-	    #check for desc files in the work dirs
-	    find "$workdir" -maxdepth 1 -type f -name '*.desc' | (
+	    local complete_perStudy=0
+	    #check for desc files in the current work dir, and subfolders
+	    find "$workdir" -maxdepth 2 -type f -name '*.desc' | (
 	        while read descfile ; do
 		    #process the desc files
+	            origPath=`dirname ${descfile}`
 		    if submit_descfile "$descfile" ; then
 			logstudy "Submitted $WUname"
 			#stop after max_jobs (0=unlimited)
-			if [ -n "$max_jobs" ] ; then
+			if ${lMaxJobs} ; then
 			    complete=$(( $complete + 1 ))
 			    if [ $complete -ge $max_jobs ] ; then
+                                log "reached ${max_jobs} in total"
 				break 2
 			    fi
 			fi
+			if ${lMaxJobsPerStudy} ; then
+			    complete_perStudy=$(( ${complete_perStudy} + 1 ))
+			    if [ ${complete_perStudy} -ge ${max_jobs_perStudy} ] ; then
+                                # continue with next study
+                                log "reached ${max_jobs_perStudy} in ${workdir}"
+				break 1
+			    fi
+			fi
 		    else
 			logstudy "Problem submitting $WUname"
 		    fi
 	        done
-	    ) 
+	    )
+            # remove temp dirs in work
+            find ${workdir} -mindepth 1 -maxdepth 1 -type d -empty -delete -print | log
 	done
     )
 }
@@ -411,15 +419,19 @@ if [ -x /usr/bin/ionice ] ; then
 fi
 
 maxjobs=0
+maxjobs_perStudy=0
 keepzip=0
+studyName=""
 
-while getopts ":hn:k"  OPT
+while getopts ":hn:m:d:k"  OPT
 do
   #Debug
   #echo "OPT is $OPT. OPTIND is $OPTIND. OPTARG is $OPTARG."
 case "$OPT" in
 h) printhelp ; exit 0 ;;
+m) maxjobs_perStudy="$OPTARG" ;;
 n) maxjobs="$OPTARG" ;;
+d) studyName="$OPTARG" ;;
 k) keepzip=1 ;;
 :|?) error 'Invalid Argument(s)' ; printhelp; exit 0 ;;
 esac
@@ -433,10 +445,10 @@ cd $boincdir
 
 getlock
 #Klog
-log run_spool $maxjobs
-run_spool $maxjobs
+log run_spool $maxjobs $maxjobs_perStudy $studyName
+run_spool $maxjobs $maxjobs_perStudy $studyName
 
-log run_spool_megazip $maxjobs
-run_spool_megazip $maxjobs
+log run_spool_megazip
+run_spool_megazip
 
 log FINISHING

boinc_software/cronjobs/crontab_jobs_boincai08_sixtadm.txt

@@ -15,7 +15,7 @@
 # zip all WUs which were not given back to user
 # crontab every 3h
 # Puppet Name: assimilation zip
-30 */3 * * * cd /share/sixtrack/assimilation ; ./zip-trashed-WUs/zip-trashed-WUs.sh >> zip-trashed-WUs/zip-trashed-WUs.log 2>&1
+30 */3 * * * cd /share/sixtrack/assimilation ; /data/boinc/project/sixtrack/bin/zip-trashed-WUs.sh > /dev/null 2>&1
 # Antique file deleter:
 13 13 * * 3 cd /share/boinc/project/sixtrack ; ./bin/antique_file_deleter  > /dev/null 2>&1
 # Delete spam:

boinc_software/cronjobs/crontab_jobs_lxplus_sixtadm.txt

@@ -7,8 +7,8 @@
 40 3 * * * lxplus.cern.ch cd /afs/cern.ch/work/b/boinc/boinctest ; /afs/cern.ch/user/s/sixtadm/boinc_soft/deleteStudies.sh >> /afs/cern.ch/user/s/sixtadm/boinc_soft/deleteStudies.log 2>&1
 #
 # list studies in spooldirs that could be deleted and notify users (based on dir itself):
-0 4 1 * * lxplus.cern.ch cd /afs/cern.ch/work/b/boinc/boinc ; /afs/cern.ch/user/s/sixtadm/boinc_soft/listDeleteStudies.sh studyName >> /afs/cern.ch/user/s/sixtadm/boinc_soft/deleteStudies.log 2>&1
-0 5 1 * * lxplus.cern.ch cd /afs/cern.ch/work/b/boinc/boinctest ; /afs/cern.ch/user/s/sixtadm/boinc_soft/listDeleteStudies.sh studyName >> /afs/cern.ch/user/s/sixtadm/boinc_soft/deleteStudies.log 2>&1
+0 4 1 * * lxplus.cern.ch cd /afs/cern.ch/work/b/boinc/boinc ; /afs/cern.ch/user/s/sixtadm/boinc_soft/listDeleteStudies.sh >> /afs/cern.ch/user/s/sixtadm/boinc_soft/deleteStudies.log 2>&1
+0 5 1 * * lxplus.cern.ch cd /afs/cern.ch/work/b/boinc/boinctest ; /afs/cern.ch/user/s/sixtadm/boinc_soft/listDeleteStudies.sh >> /afs/cern.ch/user/s/sixtadm/boinc_soft/deleteStudies.log 2>&1
 #
 # list studies in spooldirs that could be deleted and notify users (based on <workspace>_<study>/work dir):
 0 4 15 * * lxplus.cern.ch cd /afs/cern.ch/work/b/boinc/boinc ; /afs/cern.ch/user/s/sixtadm/boinc_soft/listDeleteStudies.sh >> /afs/cern.ch/user/s/sixtadm/boinc_soft/deleteStudies.log 2>&1

boinc_software/cronjobs/listDeleteStudies.sh

@@ -5,6 +5,7 @@
 # to be run in:
 #     /afs/cern.ch/work/b/boinc/boinc
 oldN=30 # days
+threshOccupancy=0.5 # [GB]
 BOINCspoolDir=$PWD
 SCRIPTDIR=`dirname $0`
 SCRIPTDIR="`cd ${SCRIPTDIR} ; pwd`"
@@ -17,11 +18,11 @@ function treatSingleDir(){
 	local __ownerFromFile="-"
     fi
     local __ownerFromUnix=`\ls -ld ${__tmpStudy} | awk '{print ($3)}'`
-    echo "${__tmpStudy}" >> /tmp/delete_${__ownerFromUnix}_${now}.txt
+    echo "${__tmpStudy}" >> /tmp/${LOGNAME}/delete_${__ownerFromUnix}_${now}.txt
     if [ "${__ownerFromFile}" == "-" ] ; then
-	echo "${__tmpStudy}" >> /tmp/no_owner_${now}.txt
+	echo "${__tmpStudy}" >> /tmp/${LOGNAME}/no_owner_${now}.txt
     elif [ "${__ownerFromFile}" != "${__ownerFromUnix}" ] ; then
-	echo "${__tmpStudy}" >> /tmp/mismatched_owners_${now}.txt
+	echo "${__tmpStudy}" >> /tmp/${LOGNAME}/mismatched_owners_${now}.txt
     fi
     echo "study ${__tmpStudy} belongs to ${__ownerFromFile}/${__ownerFromUnix} (owner file / unix)"
 }
@@ -36,6 +37,9 @@ fi
 # prepare delete dir
 [ -d delete ] || mkdir delete
 
+# convert threshold in kB
+threshOccupancyKB=`echo ${threshOccupancy} | awk '{print ($1*1024**2)}'`
+
 echo " checking BOINC spooldir ${BOINCspoolDir} ..."
 
 # list spooldirs that can be labelled for deletion and
@@ -56,21 +60,26 @@ else
     done
 fi
 
-allUsers=`ls -1 /tmp/delete_*txt | cut -d\_ -f2`
+allUsers=`ls -1 /tmp/${LOGNAME}/delete_*txt | cut -d\_ -f2`
 for tmpUserName in ${allUsers} ; do
-    echo "sending notification to ${tmpUserName} ..."
     fileList=delete_${tmpUserName}_${now}.txt
-    allStudies=`cat /tmp/${fileList}`
-    echo -e "`cat ${SCRIPTDIR}/mail.txt`\n`\du -ch --summarize ${allStudies}`" | sed -e "s/<SixDeskUser>/${tmpUserName}/g" -e "s#<spooldir>#${BOINCspoolDir}#g" -e "s/<xxx>/${oldN}/g" -e "s#<fileList>#${fileList}#g" | mail -a /tmp/${fileList} -c [email protected] -s "old studies in BOINC spooldir ${BOINCspoolDir}" ${tmpUserName}@cern.ch
-    rm /tmp/${fileList}
+    allStudies=`cat /tmp/${LOGNAME}/${fileList}`
+    # send notification email only if dimension is larger than
+    tmpTotOccupancyKB=`\du -c --summarize ${allStudies} | tail -1 | awk '{print ($1)}'`
+    if [ ${tmpTotOccupancyKB} -gt ${threshOccupancyKB} ] ; then
+	tmpTotOccupancy=`echo ${tmpTotOccupancyKB} | awk '{print ($1/1024**2)}'`
+	echo "sending notification to ${tmpUserName} ..."
+	echo -e "`cat ${SCRIPTDIR}/mail.txt`\n`\du -ch --summarize ${allStudies}`" | sed -e "s/<SixDeskUser>/${tmpUserName}/g" -e "s#<spooldir>#${BOINCspoolDir}#g" -e "s/<xxx>/${oldN}/g" -e "s#<fileList>#${fileList}#g" -e "s/<diskSpace>/${tmpTotOccupancy}/g" | mail -a /tmp/${LOGNAME}/${fileList} -c [email protected] -s "old studies in BOINC spooldir ${BOINCspoolDir}" ${tmpUserName}@cern.ch
+    fi
+    rm /tmp/${LOGNAME}/${fileList}
 done
 
 errorFiles=""
-[ ! -e /tmp/no_owner_${now}.txt ] || errorFiles="${errorFiles} /tmp/no_owner_${now}.txt"
-[ ! -e /tmp/mismatched_owners_${now}.txt ] || errorFiles="${errorFiles} /tmp/mismatched_owners_${now}.txt"
+[ ! -e /tmp/${LOGNAME}/no_owner_${now}.txt ] || errorFiles="${errorFiles} /tmp/${LOGNAME}/no_owner_${now}.txt"
+[ ! -e /tmp/${LOGNAME}/mismatched_owners_${now}.txt ] || errorFiles="${errorFiles} /tmp/${LOGNAME}/mismatched_owners_${now}.txt"
 if [ -n "${errorFiles}" ] ; then
     echo "sending error notification to amereghe ..."
-    echo "errors!" | mail -a /tmp/no_owner_${now}.txt -s "old studies in BOINC spooldir ${BOINCspoolDir} - errors!" [email protected]
-    rm -f /tmp/no_owner_${now}.txt
-    rm -f /tmp/mismatched_owners_${now}.txt
+    echo "errors!" | mail -a /tmp/${LOGNAME}/no_owner_${now}.txt -s "old studies in BOINC spooldir ${BOINCspoolDir} - errors!" [email protected]
+    rm -f /tmp/${LOGNAME}/no_owner_${now}.txt
+    rm -f /tmp/${LOGNAME}/mismatched_owners_${now}.txt
 fi

boinc_software/cronjobs/mail.txt

@@ -2,14 +2,15 @@ Dear SixDesk user <SixDeskUser>,
 
 Please find in attachment a list of buffer directories in the BOINC spooldir
    <spooldir>
-older than <xxx> days which belong to you. These directories have been
-used ONLY as BOINC buffers, i.e. to submit work units to BOINC and collect
+older than <xxx> days which belong to you. These directories are located
+on the work.boinc volume on AFS and have been used ONLY as buffers,
+i.e. to submit workunits (WUs) to BOINC and collect
 results as they are produced by BOINC volunteers.
 
 If you think that you will no longer need these directories (e.g. because
-the study is complete), then I could remove them, freeing disk space and
-improving the performance of the system. To do so, after you had a look at
-the list in attachment, please upload it to
+the study is complete), then I could remove them, freeing <diskSpace> GB of
+disk space and improving the performance of the system. To do so, after
+ you had a look at the list in attachment, please upload it to
    <spooldir>/delete
 and I will take care of cleaning. To upload the list, you can use scp, i.e.:
 scp <fileList> <SixDeskUser>@lxplus.cern.ch:<spooldir>/delete
@@ -20,8 +21,15 @@ results), then please remove them from the list in the file, either heading
 a '#' char to the name of the directory or deleting the corresponding line
 in the file.
 
-I remain at your disposal for any questions.
+Please bear in mind that keeping the work.boinc volume clean and light
+in space is extremely important, in order to ease submission of jobs
+of other people, and ultimately your future jobs. Moreover, please keep
+in mind that when you retrieve results with run_results, the downloaded
+results are not removed from this workspace but stored in the processed/
+subfolder, to keep results in case of problems with the jobs database.
+Hence, it is important to keep the work.boinc volume clean and light.
 
+I remain at your disposal for any questions.
 Thanks for your cooperation,
 Alessio (via a crontab job)
 
@@ -30,9 +38,13 @@ Once you upload the list, it may take up to 24h before deleting actually
 takes place.
 
 NB: please be reminded that the TOTAL disk space available on the AFS
-volume work.boinc (where the BOINC spooldir is located) is 100GB.
-This value is a total one; hence, the occupancy by all users is summed up.
+volume work.boinc is 100GB. This value is a total one; hence, the occupancy
+by all users is summed up.
+
+NB: the scp command may raise an error message about permissions. This
+is due to ssh and acl groups not fully talking to each other. In general,
+it not real. If you want to be sure, just log in to lxplus and
+check that the scp command went fine.
 
 NB: Please find below the breakdown of the space taken by your directories in
 the quoted BOINC spooldir:
-