Merge branch 'master' into hines-grok/ctest-wheels

Author: nrnhines

.github/workflows/neuron-ci.yml

@@ -217,16 +217,17 @@ jobs:
         # A automake >= 1.16.5 is needed for python 3.12 because it generates a python script
         # called py-compile and the original one is not supporting this version of python
         # Once ubuntu got a newer version of automake we can remove this part.
+        # Tarball from ci-deps-v1 via ci/deps (see MANIFEST id automake-1.16.5-src).
         if: matrix.config.music == 'ON' && startsWith(matrix.os, 'ubuntu')
         run: |
-          curl -L -o automake.tar.xz https://ftpmirror.gnu.org/gnu/automake/automake-1.16.5.tar.xz
-          tar -xf automake.tar.xz
-          cd automake-1.16.5/
+          export NRN_CI_DEPS_SOURCE=release
+          bash "${{ github.workspace }}/ci/deps/fetch.sh" automake-1.16.5-src "${{ runner.temp }}"
+          tar -xf "${{ runner.temp }}/automake-1.16.5.tar.xz" -C "${{ runner.temp }}"
+          cd "${{ runner.temp }}/automake-1.16.5/"
           ./configure --prefix=/usr/
           make -j
           sudo make -j install
           automake --version
-        working-directory: ${{runner.temp}}
 
       - name: Setup MUSIC@${{ env.MUSIC_VERSION }}
         if: matrix.config.music == 'ON'

.github/workflows/wheels-template.yml

@@ -151,18 +151,9 @@ jobs:
         run: |
           sudo apt update
           sudo apt install -y mpich openmpi-bin libopenmpi-dev libmpich-dev ninja-build
-
-      - name: "Apply workaround for MPICH on Ubuntu 24.04"
-        if: inputs.platform == 'ubuntu-24.04'
-        run: |
-          # mpich is fundamentally broken on Ubuntu 24.04 LTS, for more details see:
-          # https://bugs.launchpad.net/ubuntu/+source/mpich/+bug/2072338
-          # This is fixed in version 4.2.0-5.1, but this version has not been
-          # backported, so as a workaround, we install that version manually
-          sudo apt install -y mpich
-          wget 'https://launchpad.net/ubuntu/+source/mpich/4.2.0-5.1/+build/28285882/+files/mpich_4.2.0-5.1_amd64.deb'
-          wget 'https://launchpad.net/ubuntu/+source/mpich/4.2.0-5.1/+build/28285882/+files/libmpich12_4.2.0-5.1_amd64.deb'
-          sudo dpkg --install mpich_4.2.0-5.1_amd64.deb libmpich12_4.2.0-5.1_amd64.deb
+          # Ubuntu 24.04: pin working mpich from ci/deps (no Launchpad download).
+          # See ci/deps/README.md and LP#2072338.
+          bash ci/deps/install_mpich_noble.sh
 
       - name: Set env vars for testing (Linux)
         if: startsWith(inputs.platform, 'ubuntu')

.gitignore

@@ -22,6 +22,8 @@ virtualenv
 *.lo
 docs/_build
 docs/_generated
+# CI managed-download scratch (publish.sh); blobs live on GitHub Releases
+ci/deps/assets/
 .vscode
 .coverage*
 coverage*.xml

azure-pipelines.yml

@@ -81,21 +81,9 @@ stages:
       - script: |
           sudo apt update
           sudo apt install -y mpich openmpi-bin libopenmpi-dev libmpich-dev
-          # mpich is fundamentally broken on Ubuntu 24.04 LTS, for more details see:
-          # https://bugs.launchpad.net/ubuntu/+source/mpich/+bug/2072338
-          # This is fixed in version 4.2.0-5.1, but this version has not been
-          # backported, so as a workaround, we install that version manually
-          source /etc/os-release
-          if [[ "$ID" == "ubuntu" && "$VERSION_ID" == "24.04" ]]; then
-            # mpich is fundamentally broken on Ubuntu 24.04 LTS, for more details see:
-            # https://bugs.launchpad.net/ubuntu/+source/mpich/+bug/2072338
-            # This is fixed in version 4.2.0-5.1, but this version has not been
-            # backported, so as a workaround, we install that version manually
-            sudo apt install -y mpich
-            wget 'https://launchpad.net/ubuntu/+source/mpich/4.2.0-5.1/+build/28285882/+files/mpich_4.2.0-5.1_amd64.deb'
-            wget 'https://launchpad.net/ubuntu/+source/mpich/4.2.0-5.1/+build/28285882/+files/libmpich12_4.2.0-5.1_amd64.deb'
-            sudo dpkg --install mpich_4.2.0-5.1_amd64.deb libmpich12_4.2.0-5.1_amd64.deb
-          fi
+          # Ubuntu 24.04: pin working mpich from ci/deps (no Launchpad download).
+          # See ci/deps/README.md and LP#2072338.
+          ci/deps/install_mpich_noble.sh
         displayName: 'Install Test System Dependencies'
 
       - template: ci/azure-wheel-test-upload.yml

ci/deps/MANIFEST.yml

@@ -0,0 +1,176 @@
+# NEURON CI dependency archive (catalog)
+#
+# Purpose: pin external CI downloads on a GitHub Release, prefer those over flaky
+# third-party hosts, and make upgrades explicit (update this file; re-publish;
+# run check-upstream.sh). Binary blobs are NOT committed under ci/deps/assets/
+# (that directory is gitignored scratch for publish.sh).
+#
+# Fetch order used by fetch.sh (unless NRN_CI_DEPS_SOURCE is set):
+#   1. local   — ci/deps/assets/<file>  (optional local scratch; not in git)
+#   2. release — ${NRN_CI_DEPS_BASE_URL}/<file> or default_release_base_url
+#   3. upstream — upstream_url (last resort)
+#
+# managed:
+#   true  — pin hosted on our GitHub Release (ci-deps-vN); CI should use fetch.sh
+#   false — catalog row only; not yet switched off upstream
+#
+# See ci/deps/README.md
+
+version: 1
+
+# GitHub Release assets for managed pins (override with NRN_CI_DEPS_BASE_URL).
+# Host repo is neuronsimulator/nrn-ci-deps (not product releases on nrn).
+# Navigate: https://github.com/neuronsimulator/nrn-ci-deps/releases → tag ci-deps-v1
+default_release_base_url: "https://github.com/neuronsimulator/nrn-ci-deps/releases/download/ci-deps-v1"
+
+assets:
+  # --- P0: wheel-test MPICH workaround (Ubuntu 24.04 / noble) ---
+  - id: mpich-noble-4.2.0-5.1
+    file: mpich_4.2.0-5.1_amd64.deb
+    sha256: c696dff0975f1857faaae0e450ce6262d13693a7894e523741e6b7581beee0ad
+    upstream_url: https://launchpad.net/ubuntu/+source/mpich/4.2.0-5.1/+build/28285882/+files/mpich_4.2.0-5.1_amd64.deb
+    managed: true
+    consumers:
+      - azure-pipelines.yml
+      - .github/workflows/wheels-template.yml
+      - ci/deps/install_mpich_noble.sh
+    notes: >
+      Ubuntu 24.04 LP#2072338 workaround. Pair with libmpich12-noble-4.2.0-5.1.
+      Only needed while distro mpich remains broken for our tests.
+
+  - id: libmpich12-noble-4.2.0-5.1
+    file: libmpich12_4.2.0-5.1_amd64.deb
+    sha256: 13a233b7dd6d8a51eccf58d981cdb1ae110b673e4dfdeb607fe439918340238f
+    upstream_url: https://launchpad.net/ubuntu/+source/mpich/4.2.0-5.1/+build/28285882/+files/libmpich12_4.2.0-5.1_amd64.deb
+    managed: true
+    consumers:
+      - azure-pipelines.yml
+      - .github/workflows/wheels-template.yml
+      - ci/deps/install_mpich_noble.sh
+    notes: Runtime library package for mpich-noble-4.2.0-5.1.
+
+  # --- P0/P1: static readline/ncurses sources (GNU mirrors flake) ---
+  - id: ncurses-6.4-src
+    file: ncurses-6.4.tar.gz
+    sha256: 6931283d9ac87c5073f30b6290c4c75f21632bb4fc3603ac8100812bed248159
+    upstream_url: https://ftp.gnu.org/gnu/ncurses/ncurses-6.4.tar.gz
+    managed: true
+    consumers:
+      - packaging/python/build_static_readline_osx.bash
+      - packaging/python/Dockerfile
+    notes: Hosted on ci-deps-v1; consumers must not use ftpmirror.gnu.org.
+
+  - id: readline-7.0-src
+    file: readline-7.0.tar.gz
+    sha256: 750d437185286f40a369e1e4f4764eda932b9459b5ec9a731628393dd3d32334
+    upstream_url: https://ftp.gnu.org/gnu/readline/readline-7.0.tar.gz
+    managed: true
+    consumers:
+      - packaging/python/Dockerfile
+    notes: >
+      Linux manylinux Dockerfile uses 7.0. Azure Mac wheels use prebuilt secure
+      file readline7.0-ncurses6.4.tar.gz (not in this manifest yet).
+
+  - id: readline-8.3-src
+    file: readline-8.3.tar.gz
+    sha256: fe5383204467828cd495ee8d1d3c037a7eba1389c22bc6a041f627976f9061cc
+    upstream_url: https://ftp.gnu.org/gnu/readline/readline-8.3.tar.gz
+    managed: true
+    consumers:
+      - packaging/python/build_static_readline_osx.bash
+    notes: >
+      GH Actions Mac wheels rebuild from 8.3 every job — version skew vs Azure 7.0
+      prebuilt. Unify in a later PR.
+
+  # --- P1: Windows installer toolchain ---
+  - id: nsis-3.05
+    file: nsis-3.05-setup.exe
+    sha256: ""  # fill when vendoring; SourceForge redirects make automated HEAD unreliable
+    upstream_url: http://prdownloads.sourceforge.net/nsis/nsis-3.05-setup.exe
+    managed: false
+    consumers:
+      - ci/win_download_deps.cmd
+    notes: SourceForge historically flaky.
+
+  - id: envar-plugin-0.3.1
+    file: EnVar-Plugin.zip
+    sha256: e5b337fcad68252d18282f7259a0306053626e41b9480fa09df3fab012b85e00
+    upstream_url: https://github.com/GsNSIS/EnVar/releases/download/v0.3.1/EnVar-Plugin.zip
+    managed: false
+    consumers:
+      - ci/win_download_deps.cmd
+
+  - id: msmpi-runtime
+    file: msmpisetup.exe
+    sha256: ""
+    upstream_url: https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe
+    managed: false
+    consumers:
+      - ci/win_download_deps.cmd
+    notes: Opaque Microsoft CDN path; pin by URL + hash when vendoring.
+
+  - id: msmpi-sdk
+    file: msmpisdk.msi
+    sha256: ""
+    upstream_url: https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisdk.msi
+    managed: false
+    consumers:
+      - ci/win_download_deps.cmd
+
+  # --- P1/P2: occasional CI tools ---
+  - id: ccache-4.11.3-linux-x86_64
+    file: ccache-4.11.3-linux-x86_64.tar.xz
+    sha256: 7766991b91b3a5a177ab33fa043fe09e72c68586d5a86d20a563a05b74f119c0
+    upstream_url: https://github.com/ccache/ccache/releases/download/v4.11.3/ccache-4.11.3-linux-x86_64.tar.xz
+    managed: false
+    consumers:
+      - .github/workflows/coverage.yml
+    notes: Already on GitHub releases; low flake priority.
+
+  - id: automake-1.16.5-src
+    file: automake-1.16.5.tar.xz
+    sha256: f01d58cd6d9d77fbdca9eb4bbd5ead1988228fdb73d6f7a201f5f8d6b118b469
+    upstream_url: https://ftp.gnu.org/gnu/automake/automake-1.16.5.tar.xz
+    managed: true
+    consumers:
+      - .github/workflows/neuron-ci.yml
+    notes: Ubuntu MUSIC job path (neuron-ci); needs automake >= 1.16.5 for Python 3.12.
+
+  # --- Azure-only secure files (document; not managed by fetch.sh yet) ---
+  - id: azure-secure-readline7.0-ncurses6.4
+    file: readline7.0-ncurses6.4.tar.gz
+    sha256: ""
+    upstream_url: ""
+    managed: false
+    consumers:
+      - azure-pipelines.yml
+    notes: Azure Pipelines secure file (prebuilt Mac static libs). Prefer migrating to this archive.
+
+  - id: azure-secure-mpt-headers-2.21
+    file: mpt_headears.2.21.tar.gz
+    sha256: ""
+    upstream_url: ""
+    managed: false
+    consumers:
+      - azure-pipelines.yml
+    notes: Azure secure file for SGI MPT headers (dynamic MPI in Linux wheels). Typo "headears" is the uploaded name.
+
+  # --- Tier B (large; usually reliable; managed: false for now) ---
+  - id: python-3.10.0-amd64-exe
+    file: python-3.10.0-amd64.exe
+    sha256: ""
+    upstream_url: https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe
+    managed: false
+    consumers:
+      - ci/win_download_deps.cmd
+    notes: ~28 MB; self-host only if python.org flakes.
+
+  - id: python-3.14.0-macos11-pkg
+    file: python-3.14.0-macos11.pkg
+    sha256: ""
+    upstream_url: https://www.python.org/ftp/python/3.14.0/python-3.14.0-macos11.pkg
+    managed: false
+    consumers:
+      - azure-pipelines.yml
+      - .github/workflows/wheels-template.yml
+    notes: Representative Mac python.org installer; matrix has one pkg per Python version.

ci/deps/README.md

@@ -0,0 +1,99 @@
+# CI dependency archive (`ci/deps`)
+
+Pinned external downloads used by NEURON CI. Goal: **stop relying on flaky third-party hosts** (Launchpad, GNU mirrors, SourceForge) for known-good blobs, while keeping an explicit upgrade path.
+
+**Managed binaries live on a dedicated GitHub repo’s Releases**, not in the `nrn` git tree and **not** on NEURON product Releases:
+
+**https://github.com/neuronsimulator/nrn-ci-deps/releases** → tag **`ci-deps-v1`** (or later `ci-deps-vN`)
+
+Sphinx / website entry point: [docs/install/ci_deps.md](../../docs/install/ci_deps.md).
+
+## Layout
+
+| Path | Role |
+|------|------|
+| `MANIFEST.yml` | Catalog: id, file, sha256, upstream URL, consumers, `managed`, release base URL |
+| `assets/` | **Not in git** (gitignored). Optional local scratch for `publish.sh` |
+| `fetch.sh` | Resolve one asset (local → release → upstream) + verify sha256 |
+| `check-upstream.sh` | Report when upstream no longer matches the pin |
+| `publish.sh` | Upload local scratch files to **nrn-ci-deps** Releases |
+| `install_mpich_noble.sh` | Ubuntu 24.04 wheel-test helper (first consumer) |
+
+## `managed: true|false`
+
+| Value | Meaning |
+|-------|---------|
+| **`true`** | We host the pin on **nrn-ci-deps** Releases. CI should use `fetch.sh` / install helpers. |
+| **`false`** | Catalog only: known URL/sha and consumers; not yet switched off upstream. |
+
+`managed` answers “do **we** manage this pin’s content?” — not “is this from the original software vendor.”
+
+## Fetch order
+
+```text
+1. local:   ci/deps/assets/<file>     # optional gitignored scratch
+2. release: $NRN_CI_DEPS_BASE_URL/…   # or MANIFEST default_release_base_url (nrn-ci-deps)
+3. upstream: upstream_url             # last resort
+```
+
+Override with `NRN_CI_DEPS_SOURCE=local|release|upstream`.
+
+Wheel install helpers default to **`release`** so a missing Release asset fails loudly instead of silently re-hitting Launchpad.
+
+## Adding / promoting the next blocker
+
+1. Reproduce the failing URL; download once; `sha256sum` it.
+2. Add or update a row in `MANIFEST.yml` (`sha256`, `upstream_url`, consumers).
+3. Stage files only on disk (not for commit):
+
+   ```bash
+   mkdir -p ci/deps/assets
+   # copy or: NRN_CI_DEPS_SOURCE=upstream ci/deps/fetch.sh <id> ci/deps/assets
+   ci/deps/publish.sh --tag ci-deps-v1   # uploads to neuronsimulator/nrn-ci-deps
+   rm -f ci/deps/assets/*                # optional
+   ```
+
+4. Set `managed: true`. Ensure `default_release_base_url` matches the release tag/repo.
+5. Point the pipeline at `ci/deps/fetch.sh <id>` or an install helper.
+6. Open a PR on **nrn** with **text only** (MANIFEST / scripts / docs).
+
+## Upgrading a pin
+
+```bash
+ci/deps/check-upstream.sh mpich-noble-4.2.0-5.1
+
+# If STALE and you intend to move:
+#   1. download new upstream artifacts into a temp dir / assets/
+#   2. update sha256 (and file names if needed) in MANIFEST.yml
+#   3. publish.sh --tag ci-deps-v1 (or a new tag + update default_release_base_url)
+#   4. re-run check-upstream.sh
+#   5. PR the MANIFEST (and consumer) changes only
+```
+
+There is no automatic upgrade. `check-upstream.sh` is awareness only.
+
+## Currently managed assets (`managed: true`)
+
+Hosted on **nrn-ci-deps** release **`ci-deps-v1`**:
+
+- `mpich_4.2.0-5.1_amd64.deb` + `libmpich12_4.2.0-5.1_amd64.deb` — Ubuntu 24.04 wheel tests (LP#2072338)
+- `ncurses-6.4.tar.gz`, `readline-8.3.tar.gz` — Mac static readline (`build_static_readline_osx.bash`)
+- `readline-7.0.tar.gz`, `ncurses-6.4.tar.gz` — manylinux wheel image (`packaging/python/Dockerfile`)
+- `automake-1.16.5.tar.xz` — Ubuntu MUSIC path in `neuron-ci.yml`
+
+## Related but not yet managed (`managed: false`)
+
+- Azure secure files: `readline7.0-ncurses6.4.tar.gz`, `mpt_headears.2.21.tar.gz`
+- python.org installers (large)
+- Windows NSIS / MS-MPI installers
+- apt / brew / pip (package managers — different strategy)
+
+## MPI coverage note
+
+Wheel testing (`packaging/python/test_wheels.sh`) runs **serial** tests with MPI packages present, then **parallel** tests under both MPICH and OpenMPI. It does **not** currently isolate “MPI not installed.” Source CI mixes `NRN_ENABLE_MPI=OFF` with default MPI-linked builds; dynamic multi-MPI is primarily a **wheel** concern.
+
+## Requirements
+
+- `bash`, `curl` (or `wget`), `sha256sum` / `shasum`
+- `python3` (stdlib only; `ci/deps/_manifest.py` parses the restricted MANIFEST subset without PyYAML)
+- `gh` only for `publish.sh` (needs write access to `neuronsimulator/nrn-ci-deps`)