merge main to iclabel branch

.circleci/config.yml

@@ -2,12 +2,12 @@ version: 2.1
 
 orbs:
   browser-tools: circleci/[email protected]
-  codecov: codecov/codecov@3.2.4
+  codecov: codecov/codecov@5.3.0
 
 # Aliases to reuse
 _imageconfig: &imageconfig
   docker:
-    - image: cimg/base:2022.10-22.04
+    - image: cimg/python:3.12.4
 
 # https://circleci.com/docs/workflows/#executing-workflows-for-a-git-tag
 
@@ -47,10 +47,11 @@ jobs:
       - run:
           name: Get Python running
           command: |
-            pip install --upgrade --progress-bar off pip setuptools
-            pip install --upgrade --progress-bar off "autoreject @ https://api.github.com/repos/autoreject/autoreject/zipball/master" "mne[hdf5] @ git+https://github.com/mne-tools/mne-python@main" "mne-bids[full] @ https://api.github.com/repos/mne-tools/mne-bids/zipball/main" numba
+            pip install --upgrade --progress-bar off pip
+            # TODO: Restore https://api.github.com/repos/mne-tools/mne-bids/zipball/main pending https://github.com/mne-tools/mne-bids/pull/1349/files#r1885104885
+            pip install --upgrade --progress-bar off "autoreject @ https://api.github.com/repos/autoreject/autoreject/zipball/master" "mne[hdf5] @ git+https://github.com/mne-tools/mne-python@main" "mne-bids[full] @ git+https://github.com/mne-tools/mne-bids@main" numba
             pip install -ve .[tests]
-            pip install PyQt6
+            pip install "PyQt6!=6.6.1" "PyQt6-Qt6!=6.6.1,!=6.6.2,!=6.6.3,!=6.7.0"
       - run:
           name: Check Qt
           command: |
@@ -67,7 +68,7 @@ jobs:
             - project
             - mne_data
             - minimal_cmds
-            - python_env
+            - .pyenv
 
   cache_ds000117:
     <<: *imageconfig
@@ -76,15 +77,15 @@ jobs:
           at: ~/
       - restore_cache:
           keys:
-            - data-cache-ds000117-2
+            - data-cache-ds000117-3
       - bash_env
       - run:
           name: Get ds000117
           command: |
             $DOWNLOAD_DATA ds000117
       - codecov/upload
       - save_cache:
-          key: data-cache-ds000117-2
+          key: data-cache-ds000117-3
           paths:
             - ~/mne_data/ds000117
 
@@ -151,15 +152,15 @@ jobs:
           at: ~/
       - restore_cache:
           keys:
-            - data-cache-ds000246-2
+            - data-cache-ds000246-3
       - bash_env
       - run:
           name: Get ds000246
           command: |
             $DOWNLOAD_DATA ds000246
       - codecov/upload
       - save_cache:
-          key: data-cache-ds000246-2
+          key: data-cache-ds000246-3
           paths:
             - ~/mne_data/ds000246
 
@@ -170,15 +171,15 @@ jobs:
           at: ~/
       - restore_cache:
           keys:
-            - data-cache-ds000247-2
+            - data-cache-ds000247-3
       - bash_env
       - run:
           name: Get ds000247
           command: |
             $DOWNLOAD_DATA ds000247
       - codecov/upload
       - save_cache:
-          key: data-cache-ds000247-2
+          key: data-cache-ds000247-3
           paths:
             - ~/mne_data/ds000247
 
@@ -286,7 +287,6 @@ jobs:
           keys:
             - data-cache-eeg_matchingpennies-1
       - bash_env
-      - gitconfig  # email address is needed for datalad
       - run:
           name: Get eeg_matchingpennies
           command: |
@@ -297,6 +297,44 @@ jobs:
           paths:
             - ~/mne_data/eeg_matchingpennies
 
+  cache_MNE-funloc-data:
+    <<: *imageconfig
+    steps:
+      - attach_workspace:
+          at: ~/
+      - restore_cache:
+          keys:
+            - data-cache-MNE-funloc-data-5
+      - bash_env
+      - run:
+          name: Get MNE-funloc-data
+          command: |
+            $DOWNLOAD_DATA MNE-funloc-data
+      - codecov/upload
+      - save_cache:
+          key: data-cache-MNE-funloc-data-5
+          paths:
+            - ~/mne_data/MNE-funloc-data
+
+  cache_MNE-phantom-KIT-data:
+    <<: *imageconfig
+    steps:
+      - attach_workspace:
+          at: ~/
+      - restore_cache:
+          keys:
+            - data-cache-MNE-phantom-KIT-data-1
+      - bash_env
+      - run:
+          name: Get MNE-phantom-KIT-data
+          command: |
+            $DOWNLOAD_DATA MNE-phantom-KIT-data
+      - codecov/upload
+      - save_cache:
+          key: data-cache-MNE-phantom-KIT-data-1
+          paths:
+            - ~/mne_data/MNE-phantom-KIT-data
+
   cache_ERP_CORE:
     <<: *imageconfig
     steps:
@@ -324,7 +362,7 @@ jobs:
       - bash_env
       - restore_cache:
           keys:
-            - data-cache-ds000117-2
+            - data-cache-ds000117-3
       - run:
           name: test ds000117
           command: $RUN_TESTS ds000117
@@ -429,7 +467,7 @@ jobs:
       - bash_env
       - restore_cache:
           keys:
-            - data-cache-ds000246-2
+            - data-cache-ds000246-3
       - run:
           name: test ds000246
           no_output_timeout: 15m
@@ -457,7 +495,7 @@ jobs:
       - bash_env
       - restore_cache:
           keys:
-            - data-cache-ds000247-2
+            - data-cache-ds000247-3
       - run:
           name: test ds000247
           command: $RUN_TESTS ds000247
@@ -741,6 +779,7 @@ jobs:
 
   test_eeg_matchingpennies:
     <<: *imageconfig
+    resource_class: large  # memory for zapline
     steps:
       - attach_workspace:
           at: ~/
@@ -765,6 +804,59 @@ jobs:
           paths:
             - mne_data/derivatives/mne-bids-pipeline/eeg_matchingpennies/*/*/*.html
 
+  test_MNE-funloc-data:
+    <<: *imageconfig
+    resource_class: large
+    steps:
+      - attach_workspace:
+          at: ~/
+      - bash_env
+      - restore_cache:
+          keys:
+            - data-cache-MNE-funloc-data-5
+      - run:
+          name: test MNE-funloc-data
+          command: $RUN_TESTS MNE-funloc-data
+      - codecov/upload
+      - store_test_results:
+          path: ./test-results
+      - store_artifacts:
+          path: ./test-results
+          destination: test-results
+      - store_artifacts:
+          path: /home/circleci/reports/MNE-funloc-data
+          destination: reports/MNE-funloc-data
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - mne_data/derivatives/mne-bids-pipeline/MNE-funloc-data/*/*/*.html
+
+  test_MNE-phantom-KIT-data:
+    <<: *imageconfig
+    steps:
+      - attach_workspace:
+          at: ~/
+      - bash_env
+      - restore_cache:
+          keys:
+            - data-cache-MNE-phantom-KIT-data-1
+      - run:
+          name: test MNE-phantom-KIT-data
+          command: $RUN_TESTS MNE-phantom-KIT-data
+      - codecov/upload
+      - store_test_results:
+          path: ./test-results
+      - store_artifacts:
+          path: ./test-results
+          destination: test-results
+      - store_artifacts:
+          path: /home/circleci/reports/MNE-phantom-KIT-data
+          destination: reports/MNE-phantom-KIT-data
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - mne_data/derivatives/mne-bids-pipeline/MNE-phantom-KIT-data/*/*/*.html
+
   test_ERP_CORE_N400:
     <<: *imageconfig
     resource_class: large
@@ -986,6 +1078,10 @@ jobs:
       - attach_workspace:
           at: ~/
       - bash_env
+      - run:
+          name: Install dependencies
+          command: |
+            pip install -ve .[docs]
       - run:
           name: Build documentation
           command: |
@@ -1014,6 +1110,10 @@ jobs:
           at: ~/
       - bash_env
       - gitconfig
+      - run:
+          name: Install dependencies
+          command: |
+            pip install -ve .[docs]
       - run:
           # This is a bit computationally inefficient, but it should be much
           # faster to "cp" directly on the machine rather than persist
@@ -1191,6 +1291,24 @@ workflows:
             - cache_eeg_matchingpennies
           <<: *filter_tags
 
+      - cache_MNE-funloc-data:
+          requires:
+            - setup_env
+          <<: *filter_tags
+      - test_MNE-funloc-data:
+          requires:
+            - cache_MNE-funloc-data
+          <<: *filter_tags
+
+      - cache_MNE-phantom-KIT-data:
+          requires:
+            - setup_env
+          <<: *filter_tags
+      - test_MNE-phantom-KIT-data:
+          requires:
+            - cache_MNE-phantom-KIT-data
+          <<: *filter_tags
+
       - cache_ERP_CORE:
           requires:
             - setup_env
@@ -1242,6 +1360,8 @@ workflows:
             - test_ds003392
             - test_ds004229
             - test_eeg_matchingpennies
+            - test_MNE-funloc-data
+            - test_MNE-phantom-KIT-data
             - test_ERP_CORE_N400
             - test_ERP_CORE_ERN
             - test_ERP_CORE_LRP

.circleci/remove_examples.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -eo pipefail
+
+VER=$1
+if [ -z "$VER" ]; then
+  echo "Usage: $0 <version>"
+  exit 1
+fi
+ROOT="$PWD/$VER/examples/"
+if [ ! -d ${ROOT} ]; then
+  echo "Version directory does not exist or appears incorrect:"
+  echo
+  echo "$ROOT"
+  echo
+  echo "Are you on the gh-pages branch and is the ds000117 directory present?"
+  exit 1
+fi
+if [ ! -d ${ROOT}ds000117 ]; then
+  echo "Directory does not exist:"
+  echo
+  echo "${ROOT}ds000117"
+  echo
+  echo "Assuming already pruned and exiting."
+  exit 0
+fi
+echo "Pruning examples in ${ROOT} ..."
+
+find $ROOT -type d -name "*" | tail -n +2 | xargs rm -Rf
+find $ROOT -name "*.html" -exec sed -i /^\<h2\ id=\"generated-output\"\>Generated/,/^\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ $/{d} {} \;
+find $ROOT -name "*.html" -exec sed -i '/^  <a href="#generated-output"/,/^  <\/a>$/{d}' {} \;
+
+echo "Done"

.circleci/run_dataset_and_copy_files.sh

@@ -27,25 +27,32 @@ else
 fi
 
 SECONDS=0
+EMPH="\e[35m\e[1m"  # bold magenta foreground
+RESET="\e[0m"
 pytest mne_bids_pipeline --junit-xml=test-results/junit-results.xml -k ${DS_RUN}
-echo "Runtime: ${SECONDS} seconds"
+# Add emphasis and echo
+echo -e "${EMPH}Clean test runtime: ${SECONDS} seconds${RESET}"
+echo
 
 # rerun test (check caching)!
 SECONDS=0
+RERUN_LIMIT=60
 if [[ "$RERUN_TEST" == "false" ]]; then
-  echo "Skipping rerun test"
+  echo -e "${EMPH}Skipping cache rerun test${RESET}"
   RUN_TIME=0
 else
   pytest mne_bids_pipeline --cov-append -k $DS_RUN
   RUN_TIME=$SECONDS
-  echo "Runtime: ${RUN_TIME} seconds (should be < 20)"
+  echo -e "${EMPH}Cached test runtime: ${RUN_TIME} seconds (should be <= $RERUN_LIMIT)${RESET}"
 fi
-test $RUN_TIME -lt 20
+test $RUN_TIME -le $RERUN_LIMIT
 
 if [[ "$COPY_FILES" == "false" ]]; then
-  echo "Not copying files"
+  echo -e "${EMPH}Not copying files${RESET}"
   exit 0
 fi
+echo
+echo -e "${EMPH}Copying files${RESET}"
 mkdir -p ~/reports/${DS}
 # these should always exist
 cp -av ~/mne_data/derivatives/mne-bids-pipeline/${DS}/*/**/*.html ~/reports/${DS}/