Skip to content

refactor: clean up recently added build options for data-file retrieval #978

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: development
Choose a base branch
from
+185 −108
Open

refactor: clean up recently added build options for data-file retrieval #978

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9fa638c
fix: clean up recently added build options
c-dilks Dec 3, 2025
e8c5843
refactor: use booleans rather than yes/no strings
c-dilks Dec 3, 2025
6940ddb
refactor: put cleaning and depana before downloading stuff
c-dilks Dec 3, 2025
0ac247c
feat: notifiers
c-dilks Dec 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
[submodule "etc/data/magfield"]
path = etc/data/magfield
url = https://code.jlab.org/hallb/clas12/magfield
url = https://code.jlab.org/hallb/clas12/magfield.git
[submodule "validation/advanced-tests/data"]
path = validation/advanced-tests/data
url = https://code.jlab.org/hallb/clas12/coatjava/advanced-tests
url = https://code.jlab.org/hallb/clas12/coatjava/advanced-tests.git
[submodule "etc/nnet"]
path = etc/data/nnet
url = https://code.jlab.org/hallb/clas12/coatjava/networks
url = https://code.jlab.org/hallb/clas12/coatjava/networks.git
287 changes: 182 additions & 105 deletions build-coatjava.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,37 +5,44 @@ set -e
set -u
set -o pipefail

usage='''build-coatjava.sh [OPTIONS]... [MAVEN_OPTIONS]...

OPTIONS
--clara install clara too
--clean clean up built objects and exit (does not compile)
--quiet run more quietly
--no-progress no download progress printouts
--help show this message

OPTIONS FOR MAGNETIC FIELD MAPS
--lfs use git-lfs for field maps and test data
--cvmfs use cvmfs to download field maps
--xrootd use xrootd to download field maps
--nomaps do not download field maps

OPTIONS FOR TESTING
--spotbugs also run spotbugs plugin
--unittests also run unit tests
--depana run dependency analysis (only)
--data download test data (requires lfs)

MAVEN_OPTIONS
all other arguments will be passed to `mvn`; for example,
-T4 will build with 4 parallel threads
usage='''build-coatjava.sh [OPTIONS]...

GENERAL OPTIONS
--clara install clara too
--clean clean up built objects and exit (does not compile)
--quiet run more quietly
--no-progress no download progress printouts
--help show this message

DATA RETRIEVAL OPTIONS
How to retrieve magnetic field maps, neural network models, etc.;
choose only one:
--lfs use Git Large File Storage (requires `git-lfs`)
--cvmfs use CernVM-FS (requires `/cvfms`)
--xrootd use XRootD (requires `xrootd`)
--nomaps do not download field maps

TESTING OPTIONS
--spotbugs also run spotbugs plugin
--unittests also run unit tests
--depana run dependency analysis (only)
--data download test data (requires option `--lfs`)

MAVEN OPTIONS
all other arguments will be passed to `mvn`; for example,
-T4 will build with 4 parallel threads
'''

cleanBuild="no"
anaDepends="no"
runSpotBugs="no"
downloadMaps="yes"
runUnitTests="no"

################################################################################
# parse arguments
################################################################################

cleanBuild=false
anaDepends=false
runSpotBugs=false
downloadMaps=true
runUnitTests=false
useXrootd=false
useCvmfs=false
useLfs=false
Expand All @@ -46,12 +53,12 @@ wgetArgs=()
for xx in $@
do
case $xx in
--spotbugs) runSpotBugs="yes" ;;
-n) runSpotBugs="no" ;;
--nomaps) downloadMaps="no" ;;
--unittests) runUnitTests="yes" ;;
--clean) cleanBuild="yes" ;;
--depana) anaDepends="yes" ;;
--spotbugs) runSpotBugs=true ;;
-n) runSpotBugs=false ;;
--nomaps) downloadMaps=false ;;
--unittests) runUnitTests=true ;;
--clean) cleanBuild=true ;;
--depana) anaDepends=true ;;
--quiet)
mvnArgs+=(--quiet --batch-mode)
wgetArgs+=(--quiet)
Expand All @@ -60,9 +67,9 @@ do
mvnArgs+=(--no-transfer-progress)
wgetArgs+=(--no-verbose)
;;
--xrootd) useXrootd=true ;;
--cvmfs) useCvmfs=true ;;
--lfs) useLfs=true ;;
--xrootd) useXrootd=true ;;
--cvmfs) useCvmfs=true ;;
--lfs) useLfs=true ;;
--clara) installClara=true ;;
--data) downloadData=true ;;
-h|--help)
Expand All @@ -73,21 +80,22 @@ do
esac
done

if $downloadData && ! $useLfs; then
echo "$usage"
echo "ERROR::::::::::: --data requires --lfs" >&2
exit 2
fi

# Currently only git-lfs works from offsite:
if ! [[ $(hostname) == *.jlab.org ]]; then
echo "INFO: using --lfs for offsite usage"
useLfs=true
fi


################################################################################
# setup
################################################################################

# directories
src_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"
prefix_dir=$src_dir/coatjava
clara_home=$src_dir/clara
magfield_dir=$src_dir/etc/data/magfield

# working directory should be the source code directory
cd $src_dir
Expand All @@ -97,33 +105,96 @@ wgetArgs+=(--timestamping --no-check-certificate) # `--timestamping` only redown
mvn="mvn ${mvnArgs[@]:-}"
wget="wget ${wgetArgs[@]:-}"

# environment
source libexec/env.sh --no-classpath

# install LFS
if $useLfs; then
git lfs install
fi

################################################################################
# cleaning, dependency analysis, etc.
################################################################################

# function to clean installation prefixes
clean_prefixes() {
rm -rf $prefix_dir $clara_home
}

# clean up any cache copies
if $cleanBuild; then
clean_prefixes
$mvn clean
for target_dir in $(find $src_dir -type d -name target); do
echo "WARNING: target directory '$target_dir' was not removed! JAR files within may be accidentally installed!" >&2
done
echo """DONE CLEANING.
NOTE:
- to remove local magnetic field maps:
rm $magfield_dir/*.dat
- to clear all LFS git submodules:
git submodule deinit --all

Now re-run without \`--clean\` to build."""
exit
fi

# run dependency analysis and exit
if $anaDepends; then
libexec/dependency-analysis.sh
libexec/dependency-tree.sh
exit 0
fi


################################################################################
# download field maps, NN models, etc.
################################################################################

# check if a command exists
command_exists () {
type "$1" &> /dev/null
type "$1" &> /dev/null
}
download () {

# print retrieval notice
notify_retrieval() {
echo "Retrieving $1 from $2 ..."
}

# update an LFS submodule
download_lfs() {
if ! $useLfs; then
echo 'ERROR: attempted to use LFS, but option `--lfs` not set' >&2
exit 1
fi
if command_exists git-lfs ; then
cd $src_dir > /dev/null
git submodule update --init $1
cd - > /dev/null
else
echo 'ERROR: `git-lfs` not found; please install it, or use a different option other than `--lfs`' >&2
exit 1
fi
}

# download a magnetic field map
download_map () {
ret=0
if $useXrootd; then
notify_retrieval 'field map' 'xrootd'
xrdcp $1 ./
ret=$?
elif $useLfs; then
if command_exists git-lfs ; then
cd $src_dir > /dev/null
git lfs install
git submodule update --init etc/data/magfield
git submodule update --init etc/data/nnet
if $downloadData; then git submodule update --init validation/advanced-tests/data; fi
cd - > /dev/null
else
echo 'ERROR: `git-lfs` not found; please install it, or use a different option other than `--lfs`' >&2
ret=1
fi
elif $useCvmfs; then
cp -v $1 ./
notify_retrieval 'field map' 'cvmfs'
cp $1 ./
ret=$?
elif command_exists wget ; then
notify_retrieval 'field map' 'clasweb via wget'
$wget $1
ret=$?
elif command_exists curl ; then
notify_retrieval 'field map' 'clasweb via curl'
if ! [ -e ${1##*/} ]; then
curl $1 -o ${1##*/}
ret=$?
Expand All @@ -137,55 +208,55 @@ download () {

# download the default field maps, as defined in libexec/env.sh:
# (and duplicated in etc/services/reconstruction.yaml):
source libexec/env.sh --no-classpath
magfield_dir=$src_dir/etc/data/magfield
if [ $cleanBuild == "no" ] && [ $downloadMaps == "yes" ]; then
echo 'Retrieving field maps ...'
webDir=https://clasweb.jlab.org/clas12offline/magfield
if $useLfs; then webDir=${magfield_dir##$src_dir}; fi
if $useXrootd; then webDir=xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/coatjava/magfield; fi
if $useCvmfs; then webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield; fi
mkdir -p $magfield_dir
cd $magfield_dir
for map in $COAT_MAGFIELD_SOLENOIDMAP $COAT_MAGFIELD_TORUSMAP $COAT_MAGFIELD_TORUSSECONDARYMAP
do
download $webDir/$map
if [ $? -ne 0 ]; then
echo "ERROR::::::::::: Could not download field map:" >&2
echo "$webDir/$map" >&2
echo "One option is to download manually into etc/data/magfield and then run this build script with --nomaps" >&2
exit 1
fi
$useLfs && break
done
cd -
if $downloadMaps; then
if $useLfs; then
notify_retrieval 'field maps' 'lfs'
download_lfs etc/data/magfield
else
webDir=https://clasweb.jlab.org/clas12offline/magfield
if $useXrootd; then webDir=xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/coatjava/magfield; fi
if $useCvmfs; then webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield; fi
mkdir -p $magfield_dir
cd $magfield_dir
for map in $COAT_MAGFIELD_SOLENOIDMAP $COAT_MAGFIELD_TORUSMAP $COAT_MAGFIELD_TORUSSECONDARYMAP
do
download_map $webDir/$map
if [ $? -ne 0 ]; then
echo "ERROR::::::::::: Could not download field map:" >&2
echo "$webDir/$map" >&2
echo "One option is to download manually into etc/data/magfield and then run this build script with --nomaps" >&2
exit 1
fi
done
cd -
fi
fi

# always clean the installation prefix
rm -rf $prefix_dir $clara_home

# clean up any cache copies
if [ $cleanBuild == "yes" ]; then
$mvn clean
for target_dir in $(find $src_dir -type d -name target); do
echo "WARNING: target directory '$target_dir' was not removed! JAR files within may be accidentally installed!" >&2
done
echo """DONE CLEANING.
NOTE: if you want to remove locally downloaded magnetic field maps, run:
rm $magfield_dir/*.dat

Now re-run without \`--clean\` to build."""
exit
# download neural networks
if $useLfs; then
notify_retrieval 'neural networks' 'lfs'
download_lfs etc/data/nnet
elif $useCvmfs; then
notify_retrieval 'neural networks' 'cvmfs'
cp -r /cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/networks/* etc/data/nnet/
else
echo 'WARNING: neural networks not downloaded; run with `--help` for guidance' >&2
sleep 1
fi

# run dependency analysis and exit
if [ $anaDepends == "yes" ]; then
libexec/dependency-analysis.sh
libexec/dependency-tree.sh
exit 0
# download validation data
if $downloadData; then
notify_retrieval 'validation data' 'lfs'
download_lfs validation/advanced-tests/data
fi


################################################################################
# build
################################################################################

# start new installation tree
clean_prefixes # always clean the installation prefix
mkdir -p $prefix_dir
cp -r bin $prefix_dir/
cp -r etc $prefix_dir/
Expand All @@ -199,23 +270,28 @@ $python etc/bankdefs/util/bankSplit.py $prefix_dir/etc/bankdefs/hipo4 || exit 1
mkdir -p $prefix_dir/lib/utils
cp external-dependencies/jclara-4.3-SNAPSHOT.jar $prefix_dir/lib/utils

# spotbugs, unit tests
# build (and test)
unset CLAS12DIR
if [ $runUnitTests == "yes" ]; then
if $runUnitTests; then
$mvn install # also runs unit tests
else
$mvn install -DskipTests
fi

if [ $runSpotBugs == "yes" ]; then
# run spotbugs
if $runSpotBugs; then
# mvn com.github.spotbugs:spotbugs-maven-plugin:spotbugs # spotbugs goal produces a report target/spotbugsXml.xml for each module
$mvn com.github.spotbugs:spotbugs-maven-plugin:check # check goal produces a report and produces build failed if bugs
# the spotbugsXml.xml file is easiest read in a web browser
# see http://spotbugs.readthedocs.io/en/latest/maven.html and https://spotbugs.github.io/spotbugs-maven-plugin/index.html for more info
if [ $? != 0 ] ; then echo "spotbugs failure" >&2 ; exit 1 ; fi
fi

# installation

################################################################################
# install
################################################################################

# NOTE: a maven plugin, such as `maven-assembly-plugin`, would be better, but it seems that they:
# - require significantly more repetition of the module names and/or generation of additional XML file(s)
# - seem to break thread safety of `mvn install`, i.e., we'd need to run `mvn package` first, then `mvn install`
Expand Down Expand Up @@ -243,6 +319,7 @@ for pom in $(find common-tools -name pom.xml); do
done
echo "installed coatjava to: $prefix_dir"

# install clara
if $installClara; then ./install-clara -c $prefix_dir $clara_home; fi

echo "COATJAVA SUCCESSFULLY BUILT !"