Skip to content

Commit 3eaa038

Browse files
ameregheamereghe
authored
Merge pull request #67 from sixtadm/master
Updates in some admin stuff
2 parents 31a830d + 099a217 commit 3eaa038

File tree

12 files changed

+157
-8
lines changed

12 files changed

+157
-8
lines changed

‎boinc_software/cronjobs/acrontab_jobs_sixtadm.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,6 @@
5454
#
5555
# query the BOINC production DB and get current status of studies
5656
24 * * * * lxplus.cern.ch cd /afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/monitor-boinc-server/general-activity ; ./queryStudies.sh >> queryStudies.sh.log 2>&1
57+
#
58+
# monitor BOINC project space on AFS and EOS
59+
17 1,7,13,19 * * * lxplus.cern.ch cd /afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/cronjobs ; ./checkOccupancy.sh >> checkOccupancy.sh.log 2>&1

‎boinc_software/cronjobs/checkOccupancy.sh‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#!/bin/bash
2+
3+
amountWarn=50 # [%]
4+
amountAlarm=98 # [%]
5+
6+
echo ""
7+
echo "starting `basename $0` at `date` on `hostname`..."
8+
echo "warning level at ${amountWarn} %"
9+
echo "alarm level at ${amountAlarm} %"
10+
11+
# check AFS workspace
12+
13+
# get current quota:
14+
# NB: example reply of fs listquota:
15+
# Volume Name Quota Used %Used Partition
16+
# work.boinc 104857600 19303795 18% 0%
17+
18+
fsListquota=`fs listquota /afs/cern.ch/work/b/boinc/boinc`
19+
currAmount=`echo "${fsListquota}" | grep work.boinc | awk '{print ($4)}' | cut -d\% -f1`
20+
21+
echo "output of fs listquota /afs/cern.ch/work/b/boinc/boinc:"
22+
echo "${fsListquota}"
23+
if [ ${currAmount} -gt ${amountAlarm} ] ; then
24+
echo "...sending alarm to [email protected]"
25+
echo "${fsListquota}" | mail -s "boinc quota on AFS above alarm level!" [email protected]
26+
elif [ ${currAmount} -gt ${amountWarn} ] ; then
27+
echo "...sending warning to [email protected]"
28+
echo "${fsListquota}" | mail -s "boinc quota on AFS above warning level..." [email protected]
29+
else
30+
echo "...AFS quota is fine."
31+
fi
32+
33+
# check EOS workspace
34+
35+
# get current quota:
36+
# NB: example reply of eos quota:
37+
38+
# pre-configuring default route to /eos/user/s/sixtadm/
39+
# -use $EOSHOME variable to override
40+
#
41+
# By user:
42+
# ┏━> Quota Node: /eos/user/
43+
# ┌──────────┬──────────┬──────────┬──────────┬──────────┬──────────┬──────────┬──────────┬──────────┬──────────┐
44+
# │user │used bytes│logi bytes│used files│aval bytes│aval logib│aval files│ filled[%]│vol-status│ino-status│
45+
# └──────────┴──────────┴──────────┴──────────┴──────────┴──────────┴──────────┴──────────┴──────────┴──────────┘
46+
# sixtadm 0 B 0 B 0 2.00 TB 1.00 TB 1.00 M 0.00 % ok ok
47+
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
48+
#
49+
# By group:
50+
# ┏━> Quota Node: /eos/user/
51+
# ┌──────────┬──────────┬──────────┬──────────┬──────────┬──────────┬──────────┬──────────┬──────────┬──────────┐
52+
# │group │used bytes│logi bytes│used files│aval bytes│aval logib│aval files│ filled[%]│vol-status│ino-status│
53+
# └──────────┴──────────┴──────────┴──────────┴──────────┴──────────┴──────────┴──────────┴──────────┴──────────┘
54+
# def-cg 83.42 TB 41.71 TB 16.36 M 0 B 0 B 0 100.00 % ignored ignored
55+
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
56+
57+
export EOS_MGM_URL=root://eosuser.cern.ch
58+
eosQuota=`eos quota`
59+
currAmount=`echo "${eosQuota}" | grep sixtadm | awk '{printf ("%.0f",$13)}'`
60+
61+
echo "output of eos quota:"
62+
echo "${eosQuota}"
63+
if [ ${currAmount} -gt ${amountAlarm} ] ; then
64+
echo "...sending alarm to [email protected]"
65+
echo "${eosQuota}" | mail -s "boinc quota on EOS above alarm level!" [email protected]
66+
elif [ ${currAmount} -gt ${amountWarn} ] ; then
67+
echo "...sending warning to [email protected]"
68+
echo "${eosQuota}" | mail -s "boinc quota on EOS above warning level..." [email protected]
69+
else
70+
echo "...EOS quota is fine."
71+
fi
72+
73+
echo "...done by `date`."

‎boinc_software/cronjobs/listDeleteStudies.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ for tmpUserName in ${allUsers} ; do
7777
if [ ${tmpTotOccupancyKB} -gt ${threshOccupancyKB} ] ; then
7878
tmpTotOccupancy=`echo ${tmpTotOccupancyKB} | awk '{print ($1/1024**2)}'`
7979
echo "sending notification to ${tmpUserName} ..."
80-
echo -e "`cat ${SCRIPTDIR}/mail.txt`\n`\du -ch --summarize ${allStudies}`" | sed -e "s/<SixDeskUser>/${tmpUserName}/g" -e "s#<spooldir>#${BOINCspoolDir}#g" -e "s/<xxx>/${oldN}/g" -e "s#<fileList>#${fileList}#g" -e "s/<diskSpace>/${tmpTotOccupancy}/g" | mail -a /tmp/${LOGNAME}/${fileList} -c amereghe@cern.ch -s "old studies in BOINC spooldir ${BOINCspoolDir}" ${tmpUserName}@cern.ch
80+
echo -e "`cat ${SCRIPTDIR}/mail.txt`\n`\du -ch --summarize ${allStudies}`" | sed -e "s/<SixDeskUser>/${tmpUserName}/g" -e "s#<spooldir>#${BOINCspoolDir}#g" -e "s/<xxx>/${oldN}/g" -e "s#<fileList>#${fileList}#g" -e "s/<diskSpace>/${tmpTotOccupancy}/g" | mail -a /tmp/${LOGNAME}/${fileList} -c sixtadm@cern.ch -s "old studies in BOINC spooldir ${BOINCspoolDir}" ${tmpUserName}@cern.ch
8181
fi
8282
rm /tmp/${LOGNAME}/${fileList}
8383
done
@@ -86,8 +86,8 @@ errorFiles=""
8686
[ ! -e /tmp/${LOGNAME}/no_owner_${now}.txt ] || errorFiles="${errorFiles} /tmp/${LOGNAME}/no_owner_${now}.txt"
8787
[ ! -e /tmp/${LOGNAME}/mismatched_owners_${now}.txt ] || errorFiles="${errorFiles} /tmp/${LOGNAME}/mismatched_owners_${now}.txt"
8888
if [ -n "${errorFiles}" ] ; then
89-
echo "sending error notification to amereghe ..."
90-
echo "errors!" | mail -a /tmp/${LOGNAME}/no_owner_${now}.txt -s "old studies in BOINC spooldir ${BOINCspoolDir} - errors!" amereghe@cern.ch
89+
echo "sending error notification to sixtadm ..."
90+
echo "errors!" | mail -a /tmp/${LOGNAME}/no_owner_${now}.txt -s "old studies in BOINC spooldir ${BOINCspoolDir} - errors!" sixtadm@cern.ch
9191
rm -f /tmp/${LOGNAME}/no_owner_${now}.txt
9292
rm -f /tmp/${LOGNAME}/mismatched_owners_${now}.txt
9393
fi

‎boinc_software/cronjobs/mv2EOS.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ EOF
184184

185185
maxjobs=0 # =0: no limits
186186
maxjobs_perStudy=1000
187-
maxjobs_perTar=10000 # WUs
187+
maxjobs_perTar=4000 # WUs - keep number 'low', to avoid 'argument list too long' errors
188188
studyName=""
189189

190190
while getopts ":hd:m:n:N:" OPT

‎boinc_software/maintain_AFS_spooldir/getSpace.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
3+
# get data
4+
du -cS * | sort -g -k1 > spaceOccupancy.txt
5+
6+
# all the processed dirs
7+
grep processed spaceOccupancy.txt | awk '{tot+=$1}END{print tot}'
8+
9+
# 10 most problematic workspaces
10+
tail spaceOccupancy.txt

‎boinc_software/maintain_AFS_spooldir/mvStudies.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
3+
boincSpoolDirPath=/afs/cern.ch/work/b/boinc/boinc
4+
5+
for tmpStudy in `ls -1d */ | grep -v -e private -e public` ; do
6+
tmpStudy=${tmpStudy//\//}
7+
if [ -d ${boincSpoolDirPath}/${tmpStudy} ] ; then
8+
mv ${tmpStudy}/work/* ${boincSpoolDirPath}/${tmpStudy}/work
9+
else
10+
mv ${tmpStudy} ${boincSpoolDirPath}
11+
fi
12+
done

‎boinc_software/maintain_AFS_spooldir/unzipList.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
studyName="wfcchtc_job_arctripdipoctu_geb3_inj"
4+
boincPath="/afs/cern.ch/work/b/boinc/boinc"
5+
6+
echo " analysing status of results of study ${studyName} in path ${boincPath} ..."
7+
8+
# unprocessed results
9+
nFiles=0
10+
totNfiles=0
11+
for zipFile in `ls -1 ${boincPath}/${studyName}/results/*.zip` ; do
12+
let nFiles=${nFiles}+1
13+
currNFiles=`unzip -l ${zipFile} | grep "_0$" | wc -l`
14+
let totNfiles=${totNfiles}+${currNFiles}
15+
done
16+
echo " ...found ${nFiles} .zip archives for ${totNfiles} un-processed results!"
17+
18+
# processed results
19+
nFiles=0
20+
totNfiles=0
21+
for zipFile in `ls -1 ${boincPath}/${studyName}/results/processed/*.zip` ; do
22+
let nFiles=${nFiles}+1
23+
currNFiles=`unzip -l ${zipFile} | grep "_0$" | wc -l`
24+
let totNfiles=${totNfiles}+${currNFiles}
25+
done
26+
echo " ...found ${nFiles} .zip archives for ${totNfiles} processed results!"

‎boinc_software/monitor-boinc-server/general-activity/archive/correctMe.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
3+
for iFile in `find ../ -name "server_status*.dat"` ; do
4+
echo file "${iFile}"
5+
awk '{if (NF!=16 && $1!="#") {print ("#",$0)}else{print ($0)}}' ${iFile} > temp.dat
6+
tmpDiff=`diff ${iFile} temp.dat`
7+
if [ -n "${tmpDiff}" ] ; then
8+
echo "${tmpDiff}" | colordiff
9+
mv temp.dat ${iFile}
10+
fi
11+
sed -i 's/# #/#/g' ${iFile}
12+
done
13+
rm temp.dat
14+
15+
16+
17+

‎boinc_software/monitor-boinc-server/general-activity/archive/plotData_period.gnu‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
period='2020-01'
1+
period='2020-08'
22
iFileName='server_status_'.period
33

44
# changes in status page:

‎boinc_software/monitor-boinc-server/general-activity/archive/plotPeriod.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
period='2020-01'
3+
period='2020-08'
44
toolsdir=`dirname $0`
55

66
# create temporary period files

0 commit comments

Comments
 (0)