diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 4c102408..8f8d28cb 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -71,6 +71,7 @@ jobs: PLATFORM: linux/arm64 runner: ubuntu-24.04-arm arch: arm64 + fail-fast: false steps: - name: Checkout uses: actions/checkout@v4 @@ -297,6 +298,7 @@ jobs: - arch: arm64 runner: ubuntu-24.04-arm PLATFORM: linux/arm64 + fail-fast: false steps: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@v1.3.1 diff --git a/containers/debian/Dockerfile b/containers/debian/Dockerfile index 4546e083..7a2bdbad 100644 --- a/containers/debian/Dockerfile +++ b/containers/debian/Dockerfile @@ -239,6 +239,7 @@ target=${target[${TARGETPLATFORM}]} spack config --scope site add "packages:all:require:[target=${target}]" spack config --scope site add "packages:all:target:[${target}]" spack config blame packages +mkdir -p $HOME/.spack/ # workaround for Spack not creating config directory automatically in some versions (see https://github.com/spack/spack/issues/51564) spack config --scope user add "config:suppress_gpg_warnings:true" spack config --scope user add "config:build_jobs:${jobs}" spack config --scope user add "config:db_lock_timeout:${jobs}00" diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile index 833e3040..fc47722b 100644 --- a/containers/eic/Dockerfile +++ b/containers/eic/Dockerfile @@ -46,6 +46,14 @@ ARG ENV=xl ENV SPACK_ENV=/opt/spack-environment/${ENV} ARG SPACK_FLAGS="--backtrace" ARG SPACK_INSTALL_FLAGS="--no-check-signature --show-log-on-error --yes-to-all" +# The SPACK_DUPLICATE_ALLOWLIST specifies packages that are allowed to have duplicate concretizations. +# This is necessary for certain packages (e.g., epic, llvm, py-setuptools, py-urllib3, py-dask) due to +# their complex dependency trees or compatibility requirements. The new additions (py-dask-awkward, +# py-dask-histogram, py-distributed, py-requests) are included because they may be required in multiple +# versions or configurations by different packages in the environment, and allowing duplicates prevents +# build failures or conflicts during concretization. This list should be reviewed periodically to ensure +# only necessary packages are included. +ARG SPACK_DUPLICATE_ALLOWLIST="epic|llvm|py-setuptools|py-urllib3|py-dask|py-dask-awkward|py-dask-histogram|py-distributed|py-requests" ENV SPACK_COLOR="always" ENV GIT_TERMINAL_PROMPT=0 @@ -57,7 +65,7 @@ spack external find --not-buildable --scope env:${SPACK_ENV} --path /usr/local/c spack external find --scope env:${SPACK_ENV} llvm spack concretize --force spack --color=never find --long --no-groups --show-concretized --format "{name}" \ -| uniq -D -f2 | grep -v -w -e "\(epic\|llvm\|py-setuptools\|py-urllib3\)" \ +| uniq -D -f2 | grep -Evw "(${SPACK_DUPLICATE_ALLOWLIST})" \ | tee /tmp/duplicates.txt if [ -s /tmp/duplicates.txt ] ; then echo "Duplicate packages found" @@ -98,6 +106,7 @@ export CCACHE_DIR=/ccache mkdir -p /var/cache/spack/blobs/sha256/ find /var/cache/spack/blobs/sha256/ -ignore_readdir_race -atime +7 -delete spack ${SPACK_FLAGS} install ${SPACK_INSTALL_FLAGS} +spack clean --downloads --stage ccache --show-stats ccache --zero-stats EOF @@ -193,7 +202,7 @@ if [ -n "${JUGGLER_SHA}" ] ; then fi spack concretize --force spack --color=never find --long --no-groups --show-concretized --format "{name}" \ -| uniq -D -f2 | grep -v -w -e "\(epic\|llvm\|py-setuptools\|py-urllib3\)" \ +| uniq -D -f2 | grep -Evw "(${SPACK_DUPLICATE_ALLOWLIST})" \ | tee /tmp/duplicates.txt if [ -s /tmp/duplicates.txt ] ; then echo "Duplicate packages found" @@ -232,6 +241,7 @@ RUN --mount=type=cache,target=/ccache,id=ccache-${TARGETPLATFORM} \ set -e export CCACHE_DIR=/ccache spack ${SPACK_FLAGS} install ${SPACK_INSTALL_FLAGS} +spack clean --downloads --stage spack gc --yes-to-all go go-bootstrap rust rust-bootstrap py-setuptools-rust py-maturin ccache --show-stats ccache --zero-stats diff --git a/spack-environment/ci/spack.yaml b/spack-environment/ci/spack.yaml index eeb95abf..33faa091 100644 --- a/spack-environment/ci/spack.yaml +++ b/spack-environment/ci/spack.yaml @@ -2,6 +2,7 @@ spack: include: - ../concretizer.yaml - ../packages.yaml + - ../packages_root_without_opengl.yaml - ../view.yaml specs: - acts @@ -9,7 +10,7 @@ spack: - afterburner - cmake - covfie - - dd4hep -ddeve + - dd4hep - edm4hep - eigen - fastjet @@ -66,7 +67,7 @@ spack: - py-toml - py-uproot - py-vector - - root -opengl -webgui + - root - snakemake - spdlog - xrootd diff --git a/spack-environment/packages.yaml b/spack-environment/packages.yaml index 7b5be222..b8d53faa 100644 --- a/spack-environment/packages.yaml +++ b/spack-environment/packages.yaml @@ -251,9 +251,6 @@ packages: require: - '@6.5.5' - +python - llvm: - require: - - ~gold madx: require: - '@5.08.01:' @@ -330,7 +327,10 @@ packages: pyrobird: require: - '@0.1.23:' - - +batch +xrootd + - +xrootd + - spec: ~batch + when: '@:0.2.6' + message: "through at least 0.2.6, +batch requires py-pyppeteer which requires py-urllib3@1" pythia8: require: - '@8.315' @@ -356,6 +356,9 @@ packages: py-epic-capybara: require: - '@git.ef2a8790312b9b697655b28a07b353e601bfa3b8' + py-flatbuffers: + require: + - '@25.9.23:' py-hepunits: require: - '@2.3.2:' diff --git a/spack-environment/packages_root_with_opengl.yaml b/spack-environment/packages_root_with_opengl.yaml new file mode 100644 index 00000000..ced3cc10 --- /dev/null +++ b/spack-environment/packages_root_with_opengl.yaml @@ -0,0 +1,4 @@ +packages: + root: + require: + - +opengl +webgui diff --git a/spack-environment/packages_root_without_opengl.yaml b/spack-environment/packages_root_without_opengl.yaml new file mode 100644 index 00000000..e57a100c --- /dev/null +++ b/spack-environment/packages_root_without_opengl.yaml @@ -0,0 +1,4 @@ +packages: + root: + require: + - ~opengl ~webgui diff --git a/spack-environment/xl/spack.yaml b/spack-environment/xl/spack.yaml index 2384ec06..b6873026 100644 --- a/spack-environment/xl/spack.yaml +++ b/spack-environment/xl/spack.yaml @@ -2,9 +2,8 @@ spack: include: - ../concretizer.yaml - ../packages.yaml + - ../packages_root_with_opengl.yaml - ../view.yaml - concretizer: - unify: when_possible # multiple epic versions specs: - acts - actsvg @@ -23,7 +22,7 @@ spack: - cppcoro - dawn - dawncut - - dd4hep +ddeve + - dd4hep - east - edm4hep - eic-smear @@ -116,7 +115,7 @@ spack: - py-wurlitzer - py-yapf - rivet - - root +opengl +webgui +x + - root - sherpa - simsipm - slurm sysconfdir=/etc/slurm diff --git a/spack-packages.sh b/spack-packages.sh index cab6991c..a465de68 100644 --- a/spack-packages.sh +++ b/spack-packages.sh @@ -45,6 +45,7 @@ c4d983b764d7b1ee2b63da79f5f25365ac61ce7a 44da889cc86bb8a5315c729a7c79f2c002c9c951 5c37f836753e8f9683fdc547f3661c5045abcbd1 c75e10845431600b163c597545bd099e427c62f5 +2ba80e697faf80613b038615b2345b7a777cc438 --- ## Optional hash table with comma-separated file list read -r -d '' SPACKPACKAGES_CHERRYPICKS_FILES <<- \ @@ -90,4 +91,5 @@ read -r -d '' SPACKPACKAGES_CHERRYPICKS_FILES <<- \ ## c4d983b764d7b1ee2b63da79f5f25365ac61ce7a: Julia: add v1.11.6 ## 44da889cc86bb8a5315c729a7c79f2c002c9c951: Julia: add v1.11.7 ## 5c37f836753e8f9683fdc547f3661c5045abcbd1: root: require openblas ~ilp64 symbol_suffix=none when ^openblas -## c75e10845431600b163c597545bd099e427c62f5: dd4hep: v1.33 \ No newline at end of file +## c75e10845431600b163c597545bd099e427c62f5: dd4hep: v1.33 +## 2ba80e697faf80613b038615b2345b7a777cc438: py-flatbuffers: add v25.9.23 diff --git a/spack.sh b/spack.sh index 45e5a83f..5aedca8e 100644 --- a/spack.sh +++ b/spack.sh @@ -3,7 +3,7 @@ SPACK_ORGREPO="spack/spack" ## Spack github version, e.g. v0.18.1 or commit hash ## note: nightly builds will use e.g. releases/v1.0 -SPACK_VERSION="v1.0.2" +SPACK_VERSION="v1.1.0" ## Space-separated list of spack cherry-picks read -r -d '' SPACK_CHERRYPICKS <<- \