Merge branch 'master' into jelic/ci_docker_base_image

Author: Goran Jelic-Cizmek

.bbp-project.yaml

@@ -9,6 +9,13 @@ tools:
   ClangFormat:
     enable: True
     version: == 12.0.1
+    exclude:
+      match:
+      - src/nmodl/language/templates/*
   CMakeFormat:
     enable: True
     version: == 0.6.13
+    exclude:
+      match:
+      - src/nmodl/language/templates/*
+      - test/nmodl/usecases/*/*

.circleci/config.yml

@@ -10,35 +10,22 @@ assignees: ''
 Action items
 ============
 
-Pre-release
----
-- [ ] Make sur to look out for ModelDB regressions by launching analyzing [nrn-modeldb-ci last version vs nightly reports](https://github.com/neuronsimulator/nrn-modeldb-ci/actions/workflows/nrn-modeldb-ci.yaml?query=event%3Aschedule++)
-
 Sanity checks
 ---
-- [ ] Create `release/x.y` branch and make sure GitHub, Azure and CircleCI builds pass
-- [ ] Run [nrn-build-ci](https://github.com/neuronsimulator/nrn-build-ci/actions/workflows/build-neuron.yml) for the respective Azure build; see [Azure drop guide](https://github.com/neuronsimulator/nrn-build-ci#azure-wheels-testing---manual-workflow)
-- [ ] Activate ReadTheDocs build for `release/x.y` & make it hidden. Check docs are fine after build is done.
-- [ ] Run a test wheel build WITHOUT upload for `release/x.y` to ensure all the wheels build ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure))
-- [ ] Run BBP Simulation Stack & other relevant tests
+- [ ] Create `release/x.y` branch and make sure GitHub and Azure builds pass
+- [ ] Run any tests not contained in `nrn-build-ci` and `nrn-modeldb-ci`
 
 
 Releasing
 ---
 - [ ] Update semantic version in `CMakeLists.txt`
 - [ ] Update changelog below and agree on it with everyone; then commit it to `docs/changelog` (copy structure as-is)
 - [ ] Update `docs/index.rst` accordingly with the new `.pkg` and `.exe` links for `PKG installer` and `Windows Installer`
-- [ ] Run the ReadTheDocs build again for `release-x.y`, make sure the build passes and inspect the Changelog page.
-- [ ] Create new release+tag on GitHub via [release workflow](https://github.com/neuronsimulator/nrn/actions/workflows/release.yml?query=workflow%3A%22NEURON+Release%22). Note that the GitHub release will be marked as pre-release and  will contain the full-src-package and the Windows installer at the end of the release workflow.
-- [ ] Build release wheels but WITHOUT upload ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure))
+- [ ] Run the ReadTheDocs build again for `release/x.y`, make sure the build passes and inspect the Changelog page.
 - [ ] Create, test and upload manual artifacts
   - [ ] MacOS package installer (manual task, ask Michael)
-  - [ ] arm64 wheels (manual task, check with Erik, Goran or Pramod)
-  - [ ] aarch64 wheels (create a `release/x.y-aarch64` branch for this, see [guide](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-circleci))
-- [ ] Publish the `x.y.z` wheels on Pypi; see [wheel publishing instructions](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure)
+- [ ] Create new release+tag on GitHub via [release workflow](https://github.com/neuronsimulator/nrn/actions/workflows/release.yml?query=workflow%3A%22NEURON+Release%22). Note that the GitHub release will be marked as pre-release and will contain the full-src-package and the Windows installer at the end of the release workflow.
 - [ ] Once wheels are published, activate the `x.y.z` tag on ReadTheDocs
-- [ ] Rename the Windows installer in the GitHub release to match the new version and the supported python versions (i.e. `nrn-8.2.2.w64-mingw-py-39-310-311-312-setup.exe`
-)
 - [ ] Publish release on GitHub (edit https://github.com/neuronsimulator/nrn/releases/tag/x.y.z and un-tick the pre-release checkbox)
 
 

.github/ISSUE_TEMPLATE/release.md

@@ -25,6 +25,11 @@ env:
   PY_MIN_VERSION: '3.9'
   PY_MID_VERSION: '3.10'
   PY_MAX_VERSION: '3.13'
+  # The Linux runners have 4 cores:
+  # https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
+  CMAKE_BUILD_PARALLEL_LEVEL: '4'
+  CTEST_PARALLEL_LEVEL: '4'
+  CCACHE_DIR: ${{ github.workspace }}/ccache
 
 jobs:
   coverage:
@@ -47,27 +52,25 @@ jobs:
           sudo apt-get install xfonts-100dpi build-essential doxygen lcov libboost-all-dev libopenmpi-dev libmpich-dev libx11-dev libxcomposite-dev mpich openmpi-bin gpg ninja-build flex bison libfl-dev
         shell: bash
 
-      - name: Setup Caliper profiler
+      - name: Install a new ccache
         run: |
-          git clone https://github.com/LLNL/Caliper.git
-          cd Caliper
-          mkdir build && cd build
-          cmake ..
-          make && sudo make install
+          # versions older than 4.10 do not support -fprofile-update=atomic
+          wget 'https://github.com/ccache/ccache/releases/download/v4.11.3/ccache-4.11.3-linux-x86_64.tar.xz'
+          tar xf ccache-4.11.3-linux-x86_64.tar.xz
+          sudo cp -a ccache-4.11.3-linux-x86_64/ccache /usr/bin/ccache
+        working-directory: ${{runner.temp}}
 
-      - name: Setup MUSIC@${{ env.MUSIC_VERSION }}
+      - name: Setup cmake
+        uses: jwlawson/actions-setup-cmake@v2
+        with:
+          cmake-version : '3.18'
+
+      - name: Setup Caliper profiler
         run: |
-          python3 -m venv music-venv
-          source music-venv/bin/activate
-          python3 -m pip install mpi4py cython numpy setuptools
-          sudo mkdir -p $MUSIC_INSTALL_DIR
-          sudo chown -R $USER $MUSIC_INSTALL_DIR
-          curl -L -o MUSIC.zip https://github.com/INCF/MUSIC/archive/${MUSIC_VERSION}.zip
-          unzip MUSIC.zip && mv MUSIC-* MUSIC && cd MUSIC
-          ./autogen.sh
-          ./configure --with-python-sys-prefix --prefix=$MUSIC_INSTALL_DIR --disable-anysource
-          make -j install
-          deactivate
+          git clone --branch v2.12.1 --depth 1 --recurse-submodules --shallow-submodules https://github.com/LLNL/Caliper.git
+          cd Caliper
+          cmake -B build -G Ninja .
+          sudo cmake --build build --target install
         working-directory: ${{runner.temp}}
 
       - name: Setup Xvfb
@@ -79,10 +82,13 @@ jobs:
         with:
           fetch-depth: 2
 
-      - name: Clone nmodl
-        working-directory: ${{runner.workspace}}/nrn
-        run: |
-            git submodule update --init --recursive --force --depth 1 -- external/nmodl
+      # Restore (and later save) dependencies for min version of Python
+      - name: Restore Python@${{ env.PY_MIN_VERSION }} dependencies
+        id: cache-python-min-packages-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ github.workspace }}/pip_cache${{ env.PY_MIN_VERSION }}
+          key: cache-${{ env.PY_MIN_VERSION }}-${{ hashFiles('ci/requirements.txt') }}
 
       - name: Set up Python@${{ env.PY_MIN_VERSION }}
         uses: actions/setup-python@v5
@@ -92,15 +98,52 @@ jobs:
       - name: Install Python@${{ env.PY_MIN_VERSION }} dependencies
         working-directory: ${{runner.workspace}}/nrn
         run: |
-          python -m pip install --upgrade -r external/nmodl/requirements.txt
-          python -m pip install --upgrade -r ci_requirements.txt
-          python -m pip install --upgrade pip -r nrn_requirements.txt
+          python -m pip install -r ci/uv_requirements.txt
+          python -m uv pip install -r ci/requirements.txt --cache-dir ${{ github.workspace }}/pip_cache${{ env.PY_MIN_VERSION }}
+
+      - name: Save Python@${{ env.PY_MIN_VERSION }} dependencies
+        id: cache-python-min-packages-save
+        if: always() && steps.cache-python-min-packages-restore.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          key: ${{ steps.cache-python-min-packages-restore.outputs.cache-primary-key }}
+          path: ${{ github.workspace }}/pip_cache${{ env.PY_MIN_VERSION }}
+
+      # Restore (and later save) dependencies for mid version of Python
+      - name: Restore Python@${{ env.PY_MID_VERSION }} dependencies
+        id: cache-python-mid-packages-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ github.workspace }}/pip_cache${{ env.PY_MID_VERSION }}
+          key: cache-${{ env.PY_MID_VERSION }}-${{ hashFiles('ci/requirements.txt') }}
 
       - name: Set up Python@${{ env.PY_MID_VERSION }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PY_MID_VERSION }}
 
+      - name: Install Python@${{ env.PY_MID_VERSION }} dependencies
+        working-directory: ${{runner.workspace}}/nrn
+        run: |
+          python -m pip install -r ci/uv_requirements.txt
+          python -m uv pip install -r ci/requirements.txt --cache-dir ${{ github.workspace }}/pip_cache${{ env.PY_MID_VERSION }}
+
+      - name: Save Python@${{ env.PY_MID_VERSION }} dependencies
+        id: cache-python-mid-packages-save
+        if: always() && steps.cache-python-mid-packages-restore.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          key: ${{ steps.cache-python-mid-packages-restore.outputs.cache-primary-key }}
+          path: ${{ github.workspace }}/pip_cache${{ env.PY_MID_VERSION }}
+
+      # Restore (and later save) dependencies for max version of Python
+      - name: Restore Python@${{ env.PY_MAX_VERSION }} dependencies
+        id: cache-python-max-packages-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ github.workspace }}/pip_cache${{ env.PY_MAX_VERSION }}
+          key: cache-${{ env.PY_MAX_VERSION }}-${{ hashFiles('ci/requirements.txt') }}
+
       - name: Set up Python@${{ env.PY_MAX_VERSION }}
         uses: actions/setup-python@v5
         with:
@@ -109,10 +152,42 @@ jobs:
       - name: Install Python@${{ env.PY_MAX_VERSION }} dependencies
         working-directory: ${{runner.workspace}}/nrn
         run: |
-          python -m pip install --upgrade -r external/nmodl/requirements.txt
-          python -m pip install --upgrade -r ci_requirements.txt
-          python -m pip install --upgrade pip -r nrn_requirements.txt
+          python -m pip install -r ci/uv_requirements.txt
+          python -m uv pip install -r ci/requirements.txt --cache-dir ${{ github.workspace }}/pip_cache${{ env.PY_MAX_VERSION }}
+
+      - name: Save Python@${{ env.PY_MAX_VERSION }} dependencies
+        id: cache-python-max-packages-save
+        if: always() && steps.cache-python-max-packages-restore.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          key: ${{ steps.cache-python-max-packages-restore.outputs.cache-primary-key }}
+          path: ${{ github.workspace }}/pip_cache${{ env.PY_MAX_VERSION }}
+
+      - name: Setup MUSIC@${{ env.MUSIC_VERSION }}
+        run: |
+          python -m venv music-venv
+          source music-venv/bin/activate
+          python -m pip install -r ${{ github.workspace }}/ci/uv_requirements.txt
+          python -m uv pip install -r ${{ github.workspace }}/ci/requirements.txt --cache-dir ${{ github.workspace }}/pip_cache${{ env.PY_MAX_VERSION }}
+          sudo mkdir -p $MUSIC_INSTALL_DIR
+          sudo chown -R $USER $MUSIC_INSTALL_DIR
+          curl -L -o MUSIC.zip https://github.com/INCF/MUSIC/archive/${MUSIC_VERSION}.zip
+          unzip MUSIC.zip && mv MUSIC-* MUSIC && cd MUSIC
+          ./autogen.sh
+          ./configure --with-python-sys-prefix --prefix=$MUSIC_INSTALL_DIR --disable-anysource
+          make -j install
+          deactivate
+        working-directory: ${{runner.temp}}
 
+      # Restore (and later save) ccache
+      - name: Restore compiler cache
+        id: restore-compiler-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ github.workspace }}/ccache
+          key: ccache-coverage-${{runner.os}}-${{github.ref_name}}
+          restore-keys: |
+            ccache-coverage-${{runner.os}}-master
 
       - name: Build & Test
         id: build-test
@@ -130,13 +205,18 @@ jobs:
           export PYTHON_MID=$(which $PYTHON_MID_NAME);
           export PYTHON_MAX=$(which $PYTHON_MAX_NAME);
 
+          # Update all submodules with depth 1
+          git submodule update --init --recursive --depth 1
+
           mkdir build && cd build;
 
           # CMake options & flags
           cmake_args=(-G Ninja \
                       -DCMAKE_BUILD_TYPE=Debug \
                       -DCMAKE_C_COMPILER="$CC" \
                       -DCMAKE_CXX_COMPILER="$CXX" \
+                      -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+                      -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                       -DNRN_ENABLE_BACKTRACE=ON \
                       -DNRN_ENABLE_CORENEURON=ON \
                       -DNRN_ENABLE_COVERAGE=ON \
@@ -154,12 +234,12 @@ jobs:
                       -DMUSIC_ROOT="${MUSIC_INSTALL_DIR}")
           cmake .. "${cmake_args[@]}"
           # Coverage
-          # The Linux runners apparently have 2 cores, but jobs were being killed when we did not specify this explicitly.
-          # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
-          # By default we get a modern version of CMake that understands --parallel.
-          cmake --build . --parallel 2
+          ccache -z
+          ccache -s -v || ccache -s
+          cmake --build .
+          ccache -s -v || ccache -s
           cmake --build . --target cover_baseline
-          xvfb-run ctest --rerun-failed --output-on-failure;
+          xvfb-run ctest --rerun-failed --output-on-failure
           for python in "${PYTHON_MIN}" "${PYTHON_MAX}"
           do
             echo "Using ${python}"
@@ -168,7 +248,7 @@ jobs:
               PATH="${PWD}/bin:${PATH}" \
               LD_LIBRARY_PATH="${PWD}/lib:${LD_LIBRARY_PATH}" \
               DYLD_LIBRARY_PATH="${PWD}/lib:${DYLD_LIBRARY_PATH}" \
-              "${python}" -c "from neuron import h; import neuron; neuron.test();neuron.test_rxd();"
+              "${python}" -c "from neuron import h; import neuron; neuron.test();neuron.test_rxd(); from neuron.tests import test_nmodl; test_nmodl.test_nmodl()"
           done
           cmake --build . --target cover_collect
           cmake --build . --target cover_combine
@@ -178,6 +258,24 @@ jobs:
           PYTHON_MID_NAME: "python${{ env.PY_MID_VERSION }}"
           PYTHON_MAX_NAME: "python${{ env.PY_MAX_VERSION }}"
 
+      - name: Upload build artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{runner.os}}
+          path: |
+            ${{runner.workspace}}/nrn/build/CMakeCache.txt
+            ${{runner.workspace}}/nrn/build/build.ninja
+            ${{runner.workspace}}/nrn/build/cmake_install.cmake
+
+      - name: Save compiler cache
+        id: save-compiler-cache
+        if: always() && steps.restore-compiler-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          key: ${{ steps.restore-compiler-cache.outputs.cache-primary-key }}
+          path: ${{ github.workspace }}/ccache
+
       # This step will set up an SSH connection on tmate.io for live debugging.
       # To enable it, you have to:
       #   * add 'live-debug-coverage' to your PR title