Skip to content
nightly

nightly #37

Sign in to view logs

nightly

nightly #37

Workflow file for this run

.github/workflows/nightly.yaml at 59ad3a4
name: nightly
# Everything too expensive, network-dependent or non-gating for the PR path:
# foundation-model tests (downloads), the full-matrix e2e training suite,
# external-dep refresh (polar/les install from git and can break on their
# own), CPU benchmarks, the real-tier LAMMPS integration, and the durations
# refresh. GPU nightly coverage runs via ci-gpu-mpcdf.yaml's own schedule.
on:
schedule:
- cron: '47 2 * * *'
workflow_dispatch:
permissions:
contents: read
concurrency:
group: nightly
cancel-in-progress: true
jobs:
foundations:
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: ./.github/actions/setup-mace
with:
python-version: "3.12"
extras: dev
# graph_longrange so the polar foundation loaders run too.
pip-packages: -r requirements/polar.txt
- name: Cache foundation model downloads
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/.cache/mace
# URLs live in foundations_models.py: new URLs -> new cache.
key: mace-foundations-${{ hashFiles('mace/calculators/foundations_models.py') }}
# Downloads are allowed and REQUIRED here: a broken download host
# must turn this job red, never a PR.
#
# tests/golden joins the selection for its `network`-marked foundation
# goldens: they are the only place a re-uploaded or re-pointed
# published artifact is caught, and with require-caps below they fail
# here rather than skipping. The unmarked goldens in that directory run
# again here, which is cheap and means a nightly is a complete
# statement about the golden set.
- uses: ./.github/actions/run-tests
with:
tests: tests/foundations tests/golden
markers: not gpu
require-caps: network,polar
allow-network: "true"
timeout: "1200"
workflows-full:
runs-on: ubuntu-latest
name: workflows-full (py${{ matrix.python-version }}, group ${{ matrix.split-group }})
timeout-minutes: 150
strategy:
fail-fast: false
matrix:
python-version: [ '3.10', '3.11', '3.12', '3.13' ]
split-group: [ 1, 2 ]
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: ./.github/actions/setup-mace
with:
python-version: ${{ matrix.python-version }}
# wandb for the same reason as the PR-time workflows job: the flags
# are only tested where the client is installed.
extras: dev, wandb
- name: Free up disk space
run: |
sudo rm -rf /usr/lib/jvm /usr/share/dotnet /usr/share/swift \
/usr/local/.ghcup /usr/local/lib/android \
/usr/local/share/chromium /opt/microsoft /opt/google
# The e2e suite INCLUDING its network-marked tests (the PR path
# runs this suite without network).
- uses: ./.github/actions/run-tests
with:
tests: tests/workflows
require-caps: wandb
splits: "2"
group: ${{ matrix.split-group }}
allow-network: "true"
timeout: "1200"
extensions-refresh:
# Re-runs the extension jobs even when nobody touched the repo: polar/les
# install from external git and can break on their own.
uses: ./.github/workflows/ci-extensions.yaml
permissions:
contents: read
benchmarks:
# The performance baseline the rewrite will be judged against, recorded
# while legacy is still the live stack: "not slower" is only falsifiable
# against an old number, and after legacy retires the comparison runs
# against the frozen one, so these have to be reproducible cases with
# recorded metadata rather than one-off timings.
#
# This job used to be continue-on-error and published nothing at all: the
# only test in tests/benchmarks was marked `gpu` AND `network`, so all
# sixteen parametrizations skipped on a CPU runner, the job went green,
# and benchmark.json was reproducibly a 0-byte file. tests/benchmarks/
# test_inference_cpu.py adds the CPU cases, and since a job that cannot
# fail is not a baseline, continue-on-error is gone and the artifact is
# checked for contents below -- not merely for existence, which it always
# had.
#
# numprocs 0 is load-bearing: pytest-benchmark disables itself under
# xdist, so a parallel run of this job would collect timings from nothing.
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: ./.github/actions/setup-mace
with:
python-version: "3.12"
extras: dev
- name: Cache foundation model downloads
# The MP-small cases are network-gated; without the cache every night
# re-downloads the same checkpoint.
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/.cache/mace
key: mace-foundations-${{ hashFiles('mace/calculators/foundations_models.py') }}
- uses: ./.github/actions/run-tests
with:
tests: tests/benchmarks
# Deliberately no marker expression: every narrowing applied here so
# far has ended with an empty artifact. The gpu-marked cases skip on
# this runner and say so.
allow-network: "true"
# The MP-small half of the baseline is downloaded, so a lost network
# must turn this job red rather than quietly halve the artifact.
require-caps: network
numprocs: "0"
timeout: "3600"
extra-args: --benchmark-json=benchmark.json
# THE check that this job stopped being green-while-measuring-nothing.
#
# `if-no-files-found: error` on the upload below cannot catch it:
# pytest-benchmark creates the --benchmark-json path regardless of what
# ran, and a session in which every case skipped leaves a **0-byte
# file** behind (reproduced locally: `pytest tests/benchmarks/
# test_benchmark.py --benchmark-json=...` -> 16 skipped, 0-byte json).
# So the file is always present, the upload always succeeds, and the
# only thing that distinguishes a baseline from an empty artifact is its
# contents. Hence a content assertion rather than a presence one.
#
# It also checks one case per declared size, because the hole this job
# had was size-shaped: skipping is per-parametrization, so a single
# surviving case would satisfy "non-empty" while the size spread the
# downstream comparison needs had quietly gone.
- name: The artifact must actually contain a baseline
run: |
python - <<'PY'
import json, pathlib, sys
raw = pathlib.Path("benchmark.json").read_text() or "{}"
runs = json.loads(raw).get("benchmarks", [])
if not runs:
sys.exit(
"benchmark.json carries no benchmarks: every case skipped "
"and this job would have published an empty baseline."
)
sizes = {b.get("extra_info", {}).get("regime") for b in runs}
missing = {"subdomain216", "subdomain512", "kernel1728"} - sizes
if missing:
sys.exit(f"no benchmark ran at size(s) {sorted(missing)}")
for b in runs:
info = b.get("extra_info", {})
absent = [
k for k in ("dtype", "device", "torch_version", "backend")
if not info.get(k)
]
if absent:
sys.exit(f"{b['name']} records no {absent}: not reproducible")
print(f"{len(runs)} benchmarks across sizes {sorted(sizes)}")
PY
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
# always(), so the artifact is there to diagnose a failed content check
# rather than only on the nights that pass it.
if: always()
with:
name: benchmark-json
path: benchmark.json
retention-days: 90
if-no-files-found: error
lammps-real:
# Real-tier LAMMPS via conda-forge (ML-IAP unified path). Runs a
# single-layer model; tests/integrations/README.md says why.
runs-on: ubuntu-latest
timeout-minutes: 90
defaults:
run:
shell: bash -leo pipefail {0}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Set up conda env with LAMMPS (ML-IAP + python)
uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1
with:
miniforge-version: latest
python-version: "3.11"
activate-environment: lammps-mace
- name: Install LAMMPS and mace
run: |
conda install -y -c conda-forge "lammps>=2024.06" python-devtools
python -m pip install -U pip
python -m pip install ".[dev, cueq]"
python -c "import lammps; print('lammps import OK')"
# The conda env activates only in a login shell, and this job's
# `defaults.run.shell` does NOT reach steps inside a composite action --
# run-tests hardcodes `shell: bash`, so pytest was simply not found
# (exit 127) and the real tier never ran a single test. Putting the env's
# bin on PATH is what makes it visible to the action.
- name: Expose the conda env to non-login shells
run: |
test -n "$CONDA_PREFIX" || { echo "conda env is not active"; exit 1; }
test -x "$CONDA_PREFIX/bin/pytest" || { echo "no pytest in $CONDA_PREFIX"; exit 1; }
echo "$CONDA_PREFIX/bin" >> "$GITHUB_PATH"
- uses: ./.github/actions/run-tests
with:
tests: tests/integrations
markers: bin_lammps
require-caps: bin_lammps,cueq
numprocs: "0"
timeout: "1200"
coverage-full:
# THE project coverage number: full fixed CPU scope (network, cueq, polar,
# les, torchsim, schedulefree), comparable night to night. The per-PR
# coverage job measures only the deterministic PR slice by design.
#
# Sharded, one xdist worker per shard, because this is the ONLY job that
# installs every extra at once: extensions-refresh runs polar, les,
# torchsim and backends-cpu in separate jobs, and durations-refresh runs
# the same paths with `dev` alone, so they all skip the heavy ones. On
# 2026-08-06 the single job died at 87% with the runner itself killed
# ("the runner has received a shutdown signal") while one worker was in a
# cueq CPU-fallback polar case and the other was loading the omol
# foundation model. Shards bound both the concurrent peak and how much a
# single process accumulates; `[METRICS] mem/peak rss` in the log says
# which of the two it was, should it come back.
name: coverage-full (group ${{ matrix.split-group }})
runs-on: ubuntu-latest
timeout-minutes: 150
strategy:
fail-fast: false
matrix:
split-group: [ 1, 2, 3 ]
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: ./.github/actions/setup-mace
with:
python-version: "3.12"
extras: dev, cueq, torchsim, schedulefree, magnetic, wandb
pip-packages: -r requirements/polar.txt -r requirements/les.txt
- name: Cache foundation model downloads
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/.cache/mace
key: mace-foundations-${{ hashFiles('mace/calculators/foundations_models.py') }}
- name: Free up disk space
run: |
sudo rm -rf /usr/lib/jvm /usr/share/dotnet /usr/share/swift \
/usr/local/.ghcup /usr/local/lib/android \
/usr/local/share/chromium /opt/microsoft /opt/google
- uses: ./.github/actions/run-tests
with:
tests: tests
markers: not gpu and not benchmark
require-caps: network,cueq,polar,les,torchsim,schedulefree,magnetic,wandb
allow-network: "true"
coverage: "data"
splits: "3"
group: ${{ matrix.split-group }}
numprocs: "1"
timeout: "1200"
- name: Name this shard's coverage data
run: mv .coverage ".coverage.${{ matrix.split-group }}"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: coverage-data-${{ matrix.split-group }}
path: .coverage.${{ matrix.split-group }}
retention-days: 7
# A dotfile, and upload-artifact skips hidden files by default --
# the same trap that silently emptied the durations upload.
include-hidden-files: true
coverage-report:
# Each shard measures a third of the suite: this is where the nightly
# coverage number is actually produced. Deliberately NOT `if: always()` --
# a report combined from a subset of the shards would read as the project
# number while missing whatever the failed shard covered.
needs: coverage-full
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
# `coverage` alone, not the whole dev stack: combining and reporting
# needs the data files and the sources, not an importable mace.
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.12"
- run: python -m pip install "coverage[toml]"
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: coverage-data-*
merge-multiple: true
- name: Combine the shards
run: |
coverage combine
coverage xml
coverage html
- name: Coverage summary
run: |
{
echo '## Full-scope CPU coverage (nightly)'
echo '```'
coverage report --skip-covered --sort=cover | tail -40
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload coverage to Coveralls
uses: coverallsapp/github-action@v2
with:
github-token: ${{ github.token }}
file: coverage.xml
format: cobertura
fail-on-error: false
# THE coverage gate. The overall percentage above stays informative in
# both this job and the per-PR one; only the files listed below are
# enforced.
#
# WHY ONLY THESE FILES. The rewrite's test-prerequisite bar is golden
# characterization + end-to-end contracts + line-coverage floors on the
# modules whose *behaviour* ports. There is deliberately no global
# target: everything else in mace/ is pinned by the goldens in
# tests/golden and the CLI/calculator contracts in tests/workflows, and
# a global number would be a second, weaker statement about the same
# code. These six are the pure-math and physics-glue modules that have
# no golden standing in for them.
#
# MIGRATION INVARIANT. A floor protects the behaviour, not the legacy
# file. When a capability moves to the new stack, its floor moves with
# it to the corresponding mace_core/mace_torch module in the same
# commit. Forgetting to is not silent: `--include` on a path that no
# longer exists selects nothing and `coverage report` exits 1 with
# "No data to report" (and tests/unit/test_ci_gates.py fails on the PR
# that renames the file, a night earlier).
#
# WHY HERE AND NOWHERE ELSE. Two other places look plausible and are
# both wrong. Inside coverage-full the check would run in a shard
# holding one third of the suite (splits: 3), so a module exercised
# entirely by tests that landed in another group would fail its own
# floor. In ci-core's coverage job the denominator is different: it
# measures tests/unit + tests/workflows only -- no extensions, backends,
# integrations or network -- so the same percentage means a different
# thing.
#
# That second one is not a theoretical objection, and the file that
# settles it is mace/modules/utils.py. Measured on one host over the
# whole selection: 76% on the PR slice, 87% on the full scope. Its floor
# is 85. The same number against the same file therefore PASSES here and
# FAILS in the PR job. The gap is 36 statements and it is not diffuse:
# compute_forces_virials_magforces, compute_forces_magforces, the
# get_outputs branches that dispatch to them, and
# compute_total_charge_dipole_permuted -- reached from tests/extensions,
# which the PR slice does not run. A floor is meaningless without naming
# the selection, so:
#
# THE SELECTION THESE FLOORS ARE MEASURED UNDER is exactly
# coverage-full's, combined over all three shards:
# pytest tests -m "not gpu and not benchmark"
# with extras dev,cueq,torchsim,schedulefree,magnetic plus
# requirements/polar.txt and requirements/les.txt, and
# MACE_CI_ALLOW_NETWORK=1. The magnetic extra is not optional here: the
# modules/utils.py floor counts compute_forces_virials_magforces and
# compute_forces_magforces, which only tests/extensions/magnetic reach,
# and they are ~8 points of that file against 2 points of headroom.
#
# HEADROOM, as measured under that selection when the floors went in:
# loss.py 100 (floor 90) data/utils.py 96 (85)
# radial.py 100 (floor 90) neighborhood.py 100 (95)
# modules/utils.py 87 (85) atomic_data.py 96 (85)
# Two of those are ratchets rather than newly-cleared bars --
# neighborhood.py and atomic_data.py already sat above their floors
# before the characterization tests landed, so they gate against
# regression and nothing else. modules/utils.py is the tight one at two
# points; the honest reading is that its floor is the one that will bite
# first, which is the point of putting it here rather than rounding it
# down to something comfortable.
#
# This step runs after `coverage combine`, which is also why
# coverage-report is deliberately not `if: always()`: a gate must never
# be computed from a subset of the shards.
- name: Per-file coverage floors
run: |
status=0
echo '## Per-file coverage floors' >> "$GITHUB_STEP_SUMMARY"
echo '```' >> "$GITHUB_STEP_SUMMARY"
while read -r file floor; do
case "$file" in ''|'#'*) continue ;; esac
if report=$(coverage report --include="$file" --fail-under="$floor" 2>&1); then
verdict='ok'
else
verdict='BELOW FLOOR'
status=1
fi
printf '%-34s >= %3s%% %s\n' "$file" "$floor" "$verdict" \
>> "$GITHUB_STEP_SUMMARY"
printf '%s\n%s\n' "--- $file (floor ${floor}%): $verdict" "$report"
done <<'FLOORS'
mace/modules/loss.py 90
mace/modules/radial.py 90
mace/modules/utils.py 85
mace/data/utils.py 85
mace/data/neighborhood.py 95
mace/data/atomic_data.py 85
FLOORS
echo '```' >> "$GITHUB_STEP_SUMMARY"
exit "$status"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
# always(), so the html report that explains a floor failure is
# available on the run that failed.
if: always()
with:
name: coverage-full
path: |
coverage.xml
htmlcov/
retention-days: 90
durations-refresh:
runs-on: ubuntu-latest
timeout-minutes: 180
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: ./.github/actions/setup-mace
with:
python-version: "3.11"
extras: dev
- name: Free up disk space
run: |
sudo rm -rf /usr/lib/jvm /usr/share/dotnet /usr/share/swift \
/usr/local/.ghcup /usr/local/lib/android \
/usr/local/share/chromium /opt/microsoft /opt/google
- uses: ./.github/actions/run-tests
with:
tests: tests
allow-network: "true"
store-durations: "true"
timeout: "1200"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: test-durations
path: tests/.test_durations
retention-days: 30
# The path is a dotfile and upload-artifact skips hidden files by
# default: without this the job stayed green while uploading nothing,
# so the committed durations silently went stale.
include-hidden-files: true