diff --git a/bin/qtl-daqconfig b/bin/qtl-daqconfig new file mode 100755 index 000000000..1cb5494b5 --- /dev/null +++ b/bin/qtl-daqconfig @@ -0,0 +1,494 @@ +#!/usr/bin/env bash + +set -e +set -u +source $(dirname $0)/../libexec/environ.sh + +################################################################################## + +# timeline webserver directory +WEBURL="https://clas12mon.jlab.org" +WEBDIR=/group/clas/www/clas12mon/html/hipo + +################################################################################## + +# default options +match="^" +inputDir="" +publishDir="" +publishNote="" +metaSettings="" # FIXME: generalize `metaSettings` to run-group-specific settings (e.g., exclude irrelevant timelines for RG-L) +dataset="" +outputDir="" +numThreads=8 +singleTimeline="" +# modes, set by CLI long opts (--$key) +declare -A modes +for key in list just-pub just-ana skip-mya debug overwrite clobber custom-pub help; do + modes[$key]=false +done + +################################################################################## + +# usage +sep="================================================================" +usageTerse() { + echo """ + $sep + USAGE: qtl daqconfig [OPTIONS]... + $sep + Analyzes histograms and produces timelines. + + CHEF OPTIONS: most of these are required for chef's production + + -i [INPUT_DIR] input directory; use the output workflow + directory, which has subdirectories for each run + + -p [PUBLISH_DIR] publish timeline results to URL + $WEBURL/[PUBLISH_DIR]/tlsummary + (i.e., $WEBDIR/[PUBLISH_DIR]) + + -n [NOTE] additional note that will be shown on the clas12mon + webpage; surround it in quotes + - default: '' + + -s [SETTINGS] apply run-group or dataset specific settings; one of: +$(find $TIMELINESRC/data/metadata -name "*.json" -exec basename {} .json \; | sed 's;^; ;') + - default: no custom settings + """>&2 +} +usageVerbose() { + echo """ DEVELOPER OPTIONS: these are generally for developers, not chefs + + -d [DATASET_NAME] unique dataset name, defined by the user + - you may have used it in 'qtl histogram' + - default: directory basename of [PUBLISH_DIR] + + -i [INPUT_DIR] input directory + - the default is such that users only ever have + to use '-d [DATASET]' to refer to a local dataset + - default: ./outfiles/[DATASET_NAME] + + -o [OUTPUT_DIR] output directory for the analyzed timelines + - default = ./outfiles/[DATASET_NAME] + + -j [NUM_THREADS] number of parallel threads to run + - default = $numThreads + + -t [TIMELINE] produce only the single detector timeline [TIMELINE]; useful for debugging + - use --list to dump the list of timelines + - default: run all + + -m [MATCH] only produce timelines matching [MATCH] + + --list dump the list of timelines and exit + + --skip-mya skip timelines which require MYA (needed if running offsite or on CI) + + --debug enable debug mode: run a single timeline with stderr and stdout printed to screen; + use this with the '-t' option to debug specific timeline issues + + PUBLISHING OPTIONS + + --just-ana just do the daqconfig and do not attempt to publish + - follow up with '--just-pub' and '-p' to publish + + --just-pub just publish analyzed timelines from [OUTPUT_DIR] + + --overwrite remove the target publishing directory, then recreate it + + --clobber clobber the target publishing directory + + --custom-pub interpret [PUBLISH_DIR] as a fully custom directory, + rather than as a subdirectory of the web server + """ >&2 +} +if [ $# -eq 0 ]; then + usageTerse + echo " For more options, run with '--help'" >&2 + exit 101 +fi + +# parse options +while getopts "d:i:p:n:s:o:j:t:m:h-:" opt; do + case $opt in + d) + echo $OPTARG | grep -q "/" && printError "dataset name must not contain '/' " && exit 100 + dataset=$OPTARG + ;; + i) inputDir=$OPTARG ;; + p) publishDir=$OPTARG ;; + n) publishNote=$OPTARG ;; + s) metaSettings=$OPTARG ;; + o) outputDir=$OPTARG ;; + j) numThreads=$OPTARG ;; + t) singleTimeline=$OPTARG ;; + m) match=$OPTARG ;; + h) modes['help']=true ;; + -) + for key in "${!modes[@]}"; do + [ "$key" == "$OPTARG" ] && modes[$OPTARG]=true && break + done + [ -z "${modes[$OPTARG]-}" ] && printError "unknown option --$OPTARG" && exit 100 + ;; + *) exit 100;; + esac +done +if ${modes['help']}; then + usageTerse + usageVerbose + exit 101 +fi + +################################################################################## + +# main executable for detector timelines +run_daqconfig_script="org.jlab.clas.timeline.daqconfig.run_daqconfig" + +# grep which exits zero when no match is found +grep_zero() { grep $1 || true; } + +# build list of timelines +if ${modes['skip-mya']}; then + timelineList=($(java $TIMELINE_JAVA_OPTS $run_daqconfig_script --timelines | grep -vE '^epics_' | sort | grep_zero $match)) +else + timelineList=($(java $TIMELINE_JAVA_OPTS $run_daqconfig_script --timelines | sort | grep_zero $match)) +fi +if [ ${#timelineList[@]} -eq 0 ]; then + printError "empty 'timelineList'; most likely your argument '-m $match' does not match any timelines; try '--list' to see the list of all available timelines" + exit 100 +fi + +# list detector timelines, if requested +if ${modes['list']}; then + echo $sep + echo "LIST OF TIMELINES" + echo $sep + echo ${timelineList[@]} | sed 's; ;\n;g' + exit $? +fi + +################################################################################## + +# set flow control booleans +## handle `--just-ana` and `--just-pub` +enableAna=false +enablePub=false +if ${modes['just-ana']} && ! ${modes['just-pub']}; then # --just-ana + enableAna=true + enablePub=false +elif ! ${modes['just-ana']} && ${modes['just-pub']}; then # --just-pub + enableAna=false + enablePub=true +elif ${modes['just-ana']} && ${modes['just-pub']}; then # --just-ana AND --just-pub + printWarning "both '--just-ana' and '--just-pub' means that we will do both daqconfig and publishing, i.e., the default behavior" + enableAna=true + enablePub=true +else # default behavior (neither --just-ana NOR --just-pub) + enableAna=true + enablePub=true +fi +## disable publishing when `--debug` +if ${modes['debug']}; then + printWarning "DEBUG mode used; timelines will NOT be published" + enableAna=true + enablePub=false + if [ -z "$singleTimeline" ]; then + printError "DEBUG mode used, but you need to also choose a timeline with '-t'" + exit 100 + fi +fi + +################################################################################## + +# check publishing options +publishUrl="" +if $enablePub; then + # check publishDir arg + if [ -z "$publishDir" ]; then + printError "option '-p' is required, to specify a publishing directory" + printError "(otherwise, use '--just-ana' if you really do not want to publish)" + exit 100 + fi + # set full path to publish dir (unless --custom-pub) + if ! ${modes['custom-pub']}; then + publishUrl=$WEBURL/$publishDir/tlsummary + [ "$publishDir" = "." ] && printError "publishing directory argument cannot be '.'" && exit 100 + [[ "$publishDir" =~ '..' ]] && printError "publishing directory cannot contain '..'" && exit 100 + publishDir=$WEBDIR/$publishDir + else + publishUrl="option '--custom-pub' was used, therefore timeline URL is UNKNOWN" + fi + echo "publishing directory: $publishDir" + # check if the publishing directory already exists + if [ -d $publishDir ]; then + if ${modes['overwrite']}; then + printWarning "Publishing directory already exists! Timelines will be overwritten!" + printWarning "Press 'Ctrl-C' NOW if this is not what you want! Otherwise, just wait..." + sleep 5 + rm -rv $publishDir + elif ${modes['clobber']}; then + printWarning "Publishing directory already exists! Timelines will be clobbered!" + printWarning "Press 'Ctrl-C' NOW if this is not what you want! Otherwise, just wait..." + sleep 5 + else + printError "Publishing directory already exists! Either choose another directory, or re-run with the '--overwrite' or '--clobber' option to overwrite (be careful...); use '--help' for more guidance" + exit 100 + fi + fi + # make the publishing directory (which also verifies write permission) + mkdir -pv $publishDir + # path to filter file + if [ -n "$metaSettings" ]; then + metaSettings=$TIMELINESRC/data/metadata/$metaSettings.json + if [ -f "$metaSettings" ]; then + echo "using metadata file '$metaSettings'" + else + printError "metadata file '$metaSettings' does not exist (bad argument for option '-s')" + exit 100 + fi + fi + # add info to the note + version=$($TIMELINESRC/bin/qtl --version) + timestamp=$(date '+%a %D') + [ -n "$publishNote" ] && publishNote="; $publishNote" + publishNote="${timestamp}${publishNote}; timeline-code v$version" +fi + +# set the dataset name, if not already set +if [ -z "$dataset" ]; then + if [ -n "$publishDir" ]; then + dataset=$(basename $publishDir) + echo "setting dataset name from the publishing directory to '$dataset'" + else + printError "neither dataset name (-d) nor publishing directory (-p) are set; need at least one..." + exit 100 + fi +fi + +# set input and output directories +[ -z "$inputDir" ] && inputDir=$(pwd -P)/outfiles/$dataset/timeline_daqconfig # set default `inputDir` +[ ! -d $inputDir ] && printError "input directory $inputDir does not exist" && exit 100 +inputDir=$(realpath $inputDir) # sanitize CLI argument's `intputDir` +[ -z "$outputDir" ] && outputDir=$(pwd -P)/outfiles/$dataset # set default `outputDir` +mkdir -p $outputDir +outputDir=$(realpath $outputDir) # sanitize CLI argument's `outputDir` + +# set subdirectories +finalDir=$outputDir/timeline_web +logDir=$outputDir/log + +# print settings +if $enableAna; then + echo """ +Settings: +$sep +INPUT_DIR = $inputDir +DATASET_NAME = $dataset +OUTPUT_DIR = $outputDir +FINAL_DIR = $finalDir +LOG_DIR = $logDir +PUBLISH_DIR = $publishDir +NUM_THREADS = $numThreads +OPTIONS = {""" + for key in "${!modes[@]}"; do printf "%20s => %s,\n" $key ${modes[$key]}; done + echo "}" +fi + + +# output detector subdirectories +detDirs=( + alert +) + +# cleanup output directories +if $enableAna; then + if [ -d $finalDir ]; then + printWarning "removing output directory $finalDir" + rm -r $finalDir + fi + if [ -d $logDir ]; then + printWarning "removing log directory $logDir" + rm -r $logDir + fi +fi + +# make output directories +mkdir -p $logDir $finalDir + + +################################################################################## +################################################################################## +################################################################################## + + +###################################### +# produce detector timelines +###################################### +if $enableAna; then + + function wait_for_jobs() { + stat=10 + while [ "${#job_ids[@]}" -gt $1 ]; do + for i in "${!job_ids[@]}"; do + if [ "$1" -eq 0 ]; then + if [ "${#job_ids[@]}" -lt $stat ]; then + echo ">>> $(date) >>> waiting on ${#job_ids[@]} jobs:" + let stat=${#job_ids[@]} + #let stat=stat-1 + printf '>>> %s\n' "${job_names[@]}" + fi + fi + set +e + ps ${job_ids[$i]} >& /dev/null + if [ "$?" -ne 0 ]; then + echo ">>> ${job_names[$i]} finished." + unset job_ids[$i] + unset job_names[$i] + fi + set -e + done + sleep 1 + done + } + + # change working directory to output directory + pushd $finalDir + + # make detector subdirectories + for detDir in ${detDirs[@]}; do + mkdir -p $detDir + done + + # produce timelines, multithreaded + job_ids=() + job_names=() + for timelineObj in ${timelineList[@]}; do + logFile=$logDir/$timelineObj + [ -n "$singleTimeline" -a "$timelineObj" != "$singleTimeline" ] && continue + echo ">>> producing timeline '$timelineObj' ..." + if ${modes['debug']}; then + java $TIMELINE_JAVA_OPTS $run_daqconfig_script $timelineObj $inputDir + echo "PREMATURE EXIT, since --debug option was used" + exit + else + java $TIMELINE_JAVA_OPTS $run_daqconfig_script $timelineObj $inputDir > $logFile.out 2> $logFile.err || touch $logFile.fail & + job_ids+=($!) + job_names+=($timelineObj) + fi + wait_for_jobs $numThreads + done + + wait_for_jobs 0 + + # organize output timelines + echo ">>> organizing output timelines..." + timelineFiles=$(find -name "*.hipo") + [ -z "$timelineFiles" ] && printError "no timelines were produced; check error logs in $logDir/" && exit 100 + for timelineFile in $timelineFiles; do + det=$(basename $timelineFile .hipo | sed 's;_.*;;g') + case $det in + bmt) mv $timelineFile bmtbst/ ;; + bst) mv $timelineFile bmtbst/ ;; + cen) mv $timelineFile central/ ;; + ftc) mv $timelineFile ft/ ;; + fth) mv $timelineFile ft/ ;; + rat) mv $timelineFile trigger/ ;; + rftime) mv $timelineFile rf/ ;; + # ctof|ftof) + # [[ "$timelineFile" =~ _m2_ ]] && mv $timelineFile m2_ctof_ftof/ || mv $timelineFile $det/ + # ;; + *) + if [ -d $det ]; then + mv $timelineFile $det/ + else + printError "not sure where to put timeline '$timelineFile' for detector '$det'; please update $0 to fix this" && exit 100 + fi + ;; + esac + done + + # check timelines; remove and complain about any bad ones + echo ">>> running hipo-check on timeline HIPO files..." + outputFiles=$(find . -name "*.hipo") + if [ -n "$outputFiles" ]; then + logFile=$logDir/hipo-check + $TIMELINESRC/libexec/hipo-check.sh --rm-bad $outputFiles > $logFile.out 2> $logFile.err || touch $logFile.fail + fi + + # remove any empty directories + echo ">>> removing any empty directories..." + for detDir in ${detDirs[@]}; do + [ -z "$(find $detDir -name '*.hipo')" ] && rm -rv $detDir + done + + echo ">>> done producing timelines..." + popd +fi + + +###################################### +# error checking +###################################### + +somethingFailed=false +if $enableAna; then + + # print log file info + echo """ + $sep + OUTPUT AND ERROR LOGS: + $logDir/*.out + $logDir/*.err + """ + + # exit nonzero if any jobs exitted nonzero + failedJobs=($(find $logDir -name "*.fail" | xargs -I{} basename {} .fail)) + if [ ${#failedJobs[@]} -gt 0 ]; then + for failedJob in ${failedJobs[@]}; do + echo $sep + printError "job '$failedJob' returned non-zero exit code; error log dump:" + cat $logDir/$failedJob.err + if [ "$failedJob" = "hipo-check" ]; then + printWarning "These HIPO files are TIMELINE files; they will NOT be published" + fi + done + somethingFailed=true + else + echo "All jobs exitted normally" + fi + + # grep for suspicious things in error logs + errPattern="error:|exception:|warning" + echo "Now scanning for any quieter errors, by running \`grep -iE '$errPattern'\` on *.err files:" + echo $sep + grep -iE --color "$errPattern" $logDir/*.err && printWarning "errors/warnings found in log files!" || echo "Good news: grep found no errors, but you still may want to take a look yourself..." + echo $sep + +fi + + +###################################### +# publishing +###################################### + +if $enablePub; then + echo "Publishing..." + cp -r $finalDir/* $publishDir/ + [ -n "$metaSettings" ] && cp -v $metaSettings $publishDir/metadata.json + [ -n "$publishNote" ] && echo "$publishNote" > $publishDir/README + $TIMELINESRC/libexec/run-groovy-timeline.sh $TIMELINESRC/libexec/index-webpage.groovy $publishDir + echo """ +Done! Timeline files published to: $publishDir +___________________________________________________________ +TIMELINE URL: + $publishUrl +___________________________________________________________ + """ +fi + +# exit nonzero if something failed +if $somethingFailed; then + printWarning "At least one job had issues; look above or in the log files to see what's wrong." + exit 100 +fi diff --git a/src/main/java/org/jlab/clas/timeline/daqconfig/alert/alert_daqconfig.groovy b/src/main/java/org/jlab/clas/timeline/daqconfig/alert/alert_daqconfig.groovy new file mode 100644 index 000000000..88976eaf6 --- /dev/null +++ b/src/main/java/org/jlab/clas/timeline/daqconfig/alert/alert_daqconfig.groovy @@ -0,0 +1,58 @@ +package org.jlab.clas.timeline.daqconfig +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicBoolean +import org.jlab.groot.data.TDirectory +import org.jlab.groot.data.GraphErrors +import org.jlab.clas.timeline.fitter.ALERTFitter + +class alert_daqconfig { + +def data = new ConcurrentHashMap() +def has_data = new AtomicBoolean(false) + + + def processRun(fname, run) { + + File file = new File(fname) + def lines = file.readLines() + def lineCount = lines.size() + + data[run] = [run:run] + if (lineCount>=10){ + (0..14).collect{slot-> + def name = String.format('atof_threshold_%d', slot) + def string_to_be_parsed = lines[lineCount -1 - (14-slot)*2] + def value = string_to_be_parsed.split(/\s+/)[1].toFloat() + data[run].put(name, value) + } + } + } + + + + def write() { + + TDirectory out = new TDirectory() + out.mkdir('/timelines') + (0..14).collect{slot-> + def name = String.format('atof_threshold_%d', slot) + def gr = new GraphErrors(name) + gr.setTitle( String.format("ATOF Threshold") ) + gr.setTitleY( String.format("ATOF Threshold") ) + gr.setTitleX("run number") + data.sort{it.key}.each{run,it-> + out.mkdir('/'+it.run) + out.cd('/'+it.run) + if (it.containsKey(name)){ + // out.addDataSet(it[name]) + // out.addDataSet(it['fit_'+name]) + gr.addPoint(it.run, it[name], 0, 0) + } + else println(String.format("run %d: %s does not contain the desired information.", it.run, name)) + } + out.cd('/timelines') + out.addDataSet(gr) + } + out.writeFile(String.format('alert_daqconfig.hipo')) + } +} diff --git a/src/main/java/org/jlab/clas/timeline/run_daqconfig.groovy b/src/main/java/org/jlab/clas/timeline/run_daqconfig.groovy new file mode 100644 index 000000000..aad13c0fd --- /dev/null +++ b/src/main/java/org/jlab/clas/timeline/run_daqconfig.groovy @@ -0,0 +1,93 @@ +package org.jlab.clas.timeline.daqconfig +import org.jlab.clas.timeline.util.RunDependentCut + +import org.jlab.groot.data.TDirectory + +// define timeline engines +def engines = [ + out_ALERT: [new alert_daqconfig(), + ] +] + + +// parse arguments +if(args.any{it=="--timelines"}) { + engines.values().flatten().each{ + println(it.getClass().getSimpleName()) + } + System.exit(0) +} +if(args.length != 2) { + System.err.println "ARGUMENTS: [timeline] [input_dir]" + System.err.println "use --timelines for a list of available timelines" + System.exit(101) +} +def (timelineArg, inputDirArg) = args + +// check the timeline argument +def eng = engines.collectMany{key,engs->engs.collect{[key,it]}} + .find{name,eng->eng.getClass().getSimpleName()==timelineArg} +if(eng == null) { + System.err.println("error: timeline '$timelineArg' is not defined") + System.exit(100) +} + +// get list of input HIPO histogram files +def (name,engine) = eng +def inputDir = new File(inputDirArg) +println([name,timelineArg,engine.getClass().getSimpleName(),inputDir]) +def fnames = [] +inputDir.traverse { + if(it.name.endsWith('.daq.config') && it.name.contains(name)) + fnames.add(it.absolutePath) +} + +// loop over input HIPO histogram files +def allow_timeline = false +fnames.sort().each{ fname -> + try{ + println("debug: "+engine.getClass().getSimpleName()+" started $fname") + + // get run number from directory name + def dname = fname.split('/')[-2] + def m = dname =~ /\d+/ + def run = m[0].toInteger() + + // exclude certain run ranges from certain timelines + def allow_run = true + def dataset = RunDependentCut.findDataset(run) + if(dataset == 'rgl') { + if( timelineArg ==~ /^bmt.*/ || + timelineArg ==~ /^bst.*/ || + timelineArg ==~ /^cen.*/ || + timelineArg ==~ /^cvt.*/ ) { allow_run = false } + } + else { // not RG-L + if(timelineArg ==~ /^alert.*/) { allow_run = false } + } + + // run the daqconfig for this run + if(allow_run) { + allow_timeline = true // allow the timeline if at least one run is allowed + engine.processRun(fname, run) + println("debug: "+engine.getClass().getSimpleName()+" finished $fname") + } + else { + println("debug: "+engine.getClass().getSimpleName()+" excludes run $run") + } + + } catch(Exception ex) { + System.err.println("error: "+engine.getClass().getSimpleName()+" didn't process $fname, due to exception:") + ex.printStackTrace() + System.exit(100) + } +} + +// write the timeline HIPO file +if(allow_timeline) { + engine.write() + println("debug: "+engine.getClass().getSimpleName()+" ended") +} +else { + println("debug: "+engine.getClass().getSimpleName()+" was not produced, since all runs were excluded") +}