From 40fae75558891ab5ff2b3e58d6b652ffe2dfd8c4 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 23 Sep 2025 14:16:40 -0400 Subject: [PATCH 01/86] Edit gitignore --- .gitignore | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 4a82b8f4..dcdcbd5d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,14 +6,12 @@ __pycache__/ *_env.txt *.tif *.zip -.vscode/settings.json +.vscode .mypy_cache -Deploy/dist -Deploy/build -Deploy/suite2p +.venv-*/ + +Deploy/ caiman_data/ -CaImAn/caiman_correlation_pnr_functions/CaImAn-b5932abe85c478fae4004e8df14535f3f5f7274a.zip -CaImAn/caiman_correlation_pnr_functions/pre_processing_comment.py -CaImAn/caiman_correlation_pnr_functions/summary_images_comment.py +CaImAn/caiman_correlation_pnr_functions/ From 7b25e940d512ca5dc8e55cdcdcdb18b3e50502f1 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 23 Sep 2025 14:17:25 -0400 Subject: [PATCH 02/86] Add workflow for deeplabcut --- .github/workflows/release-dlc.yml | 124 ++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 .github/workflows/release-dlc.yml diff --git a/.github/workflows/release-dlc.yml b/.github/workflows/release-dlc.yml new file mode 100644 index 00000000..ab5d2146 --- /dev/null +++ b/.github/workflows/release-dlc.yml @@ -0,0 +1,124 @@ +name: Release DeepLabCut + +on: + push: + tags: ['deeplabcut-v*'] + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: dlc-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: [self-hosted, Windows, X64, GPU] + timeout-minutes: 180 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Extract version from tag (cmd) + shell: cmd + run: | + set VERSION=%GITHUB_REF_NAME:deeplabcut-v=% + echo VERSION=%VERSION%>> %GITHUB_ENV% + + - name: Create venv + shell: cmd + run: | + C:\Python310\python.exe -V + C:\Python310\python.exe -m venv venv + + - name: Install deps + shell: cmd + run: | + venv\Scripts\python -m pip install --upgrade pip + venv\Scripts\pip install h5py pyinstaller deeplabcut==3.0.0rc8 + venv\Scripts\pip install --upgrade --index-url https://download.pytorch.org/whl/cu126 torch + venv\Scripts\python -c "import torch;print('Torch:',torch.__version__);print('CUDA build:',getattr(torch.version,'cuda',None));print('CUDA available:',torch.cuda.is_available())" + + - name: Clean old dist/build + shell: cmd + working-directory: Deploy + run: | + if exist dist rmdir /s /q dist + if exist build rmdir /s /q build + + - name: Build deeplabcut + shell: cmd + working-directory: Deploy + run: | + ..\venv\Scripts\pyinstaller --clean --noconfirm pack_deeplabcut_run.spec + echo === DIST TREE === + dir /s /b dist + + - name: Zip onedir output with build-info + shell: cmd + working-directory: Deploy + env: + GIT_SHA: ${{ github.sha }} + run: | + rem Locate the onedir app folder under dist + set "APPDIR=" + for /d %%D in (dist\*) do ( set "APPDIR=%%D" & goto :found ) + :found + if not defined APPDIR ( echo ERROR: no onedir folder found in Deploy\dist & exit /b 1 ) + + rem Write build-info into the app folder (included in the zip) + echo Version: %VERSION%> "%APPDIR%\build-info.txt" + echo Commit: %GIT_SHA%>> "%APPDIR%\build-info.txt" + echo Date: %DATE% %TIME%>> "%APPDIR%\build-info.txt" + + set "ZIPNAME=deeplabcut-%VERSION%.zip" + + rem One-line Python (no backslashes/newlines) + ..\venv\Scripts\python -c "import os,zipfile; a=r'%APPDIR%'; zf=r'%ZIPNAME%'; assert os.path.isdir(a), 'no APPDIR: '+a; z=zipfile.ZipFile(zf,'w',zipfile.ZIP_DEFLATED); [z.write(os.path.join(r,f), os.path.relpath(os.path.join(r,f),a)) for r,_,fs in os.walk(a) for f in fs]; z.close()" + echo Created zip: %ZIPNAME% + + - name: Show zip contents + shell: cmd + working-directory: Deploy + run: | + set "ZN=deeplabcut-%VERSION%.zip" + if not exist "%ZN%" ( echo ERROR: zip not found & exit /b 1 ) + ..\venv\Scripts\python -c "import os,zipfile; zn=r'%ZN%'; z=zipfile.ZipFile(zn,'r'); n=z.namelist(); print('Entries in',zn,':',len(n)); [print(' ',x) for x in n[:50]]; z.close()" + + + - name: SHA256 of release zip + shell: cmd + working-directory: Deploy + run: | + setlocal EnableDelayedExpansion + set ZIP=deeplabcut-%VERSION%.zip + for /f "usebackq skip=1 tokens=1" %%H in (`certutil -hashfile "%ZIP%" SHA256 ^| findstr /v /i "hash certutil"`) do ( + set HASH=%%H + goto :got + ) + :got + echo !HASH!> deeplabcut-%VERSION%.zip.sha256 + type deeplabcut-%VERSION%.zip.sha256 + + - name: Create GitHub Release (attach zip + checksum) + uses: softprops/action-gh-release@v2 + with: + name: deeplabcut ${{ env.VERSION }} + body: | + Automated deeplabcut release for tag ${{ github.ref_name }}. + - Version: ${{ env.VERSION }} + - Commit: ${{ github.sha }} + files: | + Deploy/deeplabcut-${{ env.VERSION }}.zip + Deploy/deeplabcut-${{ env.VERSION }}.zip.sha256 + + - name: Upload dist artifact (only on failure) + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: deeplabcut-dist-${{ env.VERSION }} + path: Deploy/dist/** + retention-days: 3 + compression-level: 1 From a2b34c638274f350e1ea9edfe3d94a121af98646 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 23 Sep 2025 14:53:32 -0400 Subject: [PATCH 03/86] Add yml draft for caiman releases --- .github/workflows/release-cmn.yml | 123 ++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 .github/workflows/release-cmn.yml diff --git a/.github/workflows/release-cmn.yml b/.github/workflows/release-cmn.yml new file mode 100644 index 00000000..a7dd52b5 --- /dev/null +++ b/.github/workflows/release-cmn.yml @@ -0,0 +1,123 @@ +name: Release CaImAn + +on: + push: + tags: ['caiman-v*'] + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: dlc-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: [self-hosted, Windows, X64, GPU] + timeout-minutes: 180 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Extract version from tag (cmd) + shell: cmd + run: | + set VERSION=%GITHUB_REF_NAME:caiman-v=% + echo VERSION=%VERSION%>> %GITHUB_ENV% + + - name: Create venv + shell: cmd + run: | + C:\Python310\python.exe -V + C:\Python310\python.exe -m venv venv + + - name: Install deps + shell: cmd + run: | + venv\Scripts\python -m pip install --upgrade pip + venv\Scripts\pip install pyinstaller caiman==1.11.4 + venv\Scripts\python -c "import caiman; caimanmanager.install()" + + - name: Clean old dist/build + shell: cmd + working-directory: Deploy + run: | + if exist dist rmdir /s /q dist + if exist build rmdir /s /q build + + - name: Build caiman + shell: cmd + working-directory: Deploy + run: | + ..\venv\Scripts\pyinstaller --clean --noconfirm pack_caiman_run.spec + echo === DIST TREE === + dir /s /b dist + + - name: Zip onedir output with build-info + shell: cmd + working-directory: Deploy + env: + GIT_SHA: ${{ github.sha }} + run: | + rem Locate the onedir app folder under dist + set "APPDIR=" + for /d %%D in (dist\*) do ( set "APPDIR=%%D" & goto :found ) + :found + if not defined APPDIR ( echo ERROR: no onedir folder found in Deploy\dist & exit /b 1 ) + + rem Write build-info into the app folder (included in the zip) + echo Version: %VERSION%> "%APPDIR%\build-info.txt" + echo Commit: %GIT_SHA%>> "%APPDIR%\build-info.txt" + echo Date: %DATE% %TIME%>> "%APPDIR%\build-info.txt" + + set "ZIPNAME=caiman-%VERSION%.zip" + + rem One-line Python (no backslashes/newlines) + ..\venv\Scripts\python -c "import os,zipfile; a=r'%APPDIR%'; zf=r'%ZIPNAME%'; assert os.path.isdir(a), 'no APPDIR: '+a; z=zipfile.ZipFile(zf,'w',zipfile.ZIP_DEFLATED); [z.write(os.path.join(r,f), os.path.relpath(os.path.join(r,f),a)) for r,_,fs in os.walk(a) for f in fs]; z.close()" + echo Created zip: %ZIPNAME% + + - name: Show zip contents + shell: cmd + working-directory: Deploy + run: | + set "ZN=caiman-%VERSION%.zip" + if not exist "%ZN%" ( echo ERROR: zip not found & exit /b 1 ) + ..\venv\Scripts\python -c "import os,zipfile; zn=r'%ZN%'; z=zipfile.ZipFile(zn,'r'); n=z.namelist(); print('Entries in',zn,':',len(n)); [print(' ',x) for x in n[:50]]; z.close()" + + + - name: SHA256 of release zip + shell: cmd + working-directory: Deploy + run: | + setlocal EnableDelayedExpansion + set ZIP=caiman-%VERSION%.zip + for /f "usebackq skip=1 tokens=1" %%H in (`certutil -hashfile "%ZIP%" SHA256 ^| findstr /v /i "hash certutil"`) do ( + set HASH=%%H + goto :got + ) + :got + echo !HASH!> caiman-%VERSION%.zip.sha256 + type caiman-%VERSION%.zip.sha256 + + - name: Create GitHub Release (attach zip + checksum) + uses: softprops/action-gh-release@v2 + with: + name: caiman ${{ env.VERSION }} + body: | + Automated caiman release for tag ${{ github.ref_name }}. + - Version: ${{ env.VERSION }} + - Commit: ${{ github.sha }} + files: | + Deploy/caiman-${{ env.VERSION }}.zip + Deploy/caiman-${{ env.VERSION }}.zip.sha256 + + - name: Upload dist artifact (only on failure) + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: caiman-dist-${{ env.VERSION }} + path: Deploy/dist/** + retention-days: 3 + compression-level: 1 From 0cea2ba0284753aa0ee600bbbff2b7ddf94f9dc3 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 23 Sep 2025 15:05:22 -0400 Subject: [PATCH 04/86] Edit caiman yml --- .github/workflows/release-cmn.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-cmn.yml b/.github/workflows/release-cmn.yml index a7dd52b5..833b73d9 100644 --- a/.github/workflows/release-cmn.yml +++ b/.github/workflows/release-cmn.yml @@ -9,7 +9,7 @@ permissions: contents: write concurrency: - group: dlc-${{ github.ref }} + group: cmn-${{ github.ref }} cancel-in-progress: true jobs: @@ -21,7 +21,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Extract version from tag (cmd) + - name: Extract version from tag shell: cmd run: | set VERSION=%GITHUB_REF_NAME:caiman-v=% @@ -33,12 +33,17 @@ jobs: C:\Python310\python.exe -V C:\Python310\python.exe -m venv venv - - name: Install deps + - name: Install Caiman deps (pip, no conda) shell: cmd run: | - venv\Scripts\python -m pip install --upgrade pip - venv\Scripts\pip install pyinstaller caiman==1.11.4 - venv\Scripts\python -c "import caiman; caimanmanager.install()" + venv\Scripts\python -m pip install --upgrade pip wheel setuptools + venv\Scripts\pip install "numpy<2" + venv\Scripts\pip install caiman + venv\Scripts\pip install opencv-python-headless + venv\Scripts\pip install pyinstaller + venv\Scripts\caimanmanager install + + venv\Scripts\python -c "import sys; import caiman, numpy as np; print('Python:', sys.version); print('Caiman:', getattr(caiman,'__version__','?')); print('NumPy:', np.__version__)" - name: Clean old dist/build shell: cmd @@ -86,13 +91,13 @@ jobs: if not exist "%ZN%" ( echo ERROR: zip not found & exit /b 1 ) ..\venv\Scripts\python -c "import os,zipfile; zn=r'%ZN%'; z=zipfile.ZipFile(zn,'r'); n=z.namelist(); print('Entries in',zn,':',len(n)); [print(' ',x) for x in n[:50]]; z.close()" - - name: SHA256 of release zip shell: cmd working-directory: Deploy run: | setlocal EnableDelayedExpansion set ZIP=caiman-%VERSION%.zip + if not exist "%ZIP%" ( echo ERROR: zip not found & exit /b 1 ) for /f "usebackq skip=1 tokens=1" %%H in (`certutil -hashfile "%ZIP%" SHA256 ^| findstr /v /i "hash certutil"`) do ( set HASH=%%H goto :got From 015a7093a8e7d73d893ebd0e48136e8931917896 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 23 Sep 2025 15:26:43 -0400 Subject: [PATCH 05/86] Try to use miniconda to install caiman --- .github/workflows/release-cmn.yml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release-cmn.yml b/.github/workflows/release-cmn.yml index 833b73d9..537be7f8 100644 --- a/.github/workflows/release-cmn.yml +++ b/.github/workflows/release-cmn.yml @@ -27,23 +27,26 @@ jobs: set VERSION=%GITHUB_REF_NAME:caiman-v=% echo VERSION=%VERSION%>> %GITHUB_ENV% - - name: Create venv + - name: Install Miniconda shell: cmd run: | - C:\Python310\python.exe -V - C:\Python310\python.exe -m venv venv + if not exist C:\Miniconda3 ( + curl -L -o miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe + start /wait "" miniconda.exe /InstallationType=AllUsers /AddToPath=1 /S /D=C:\Miniconda3 + ) + C:\Miniconda3\Scripts\conda.exe --version + + - name: Create conda env for caiman + shell: cmd + run: | + C:\Miniconda3\Scripts\conda.exe env remove -n caiman_release -y || echo "no old env" + C:\Miniconda3\Scripts\conda.exe create -n caiman_release python=3.10 -y - - name: Install Caiman deps (pip, no conda) + - name: Install caiman deps with conda-forge shell: cmd run: | - venv\Scripts\python -m pip install --upgrade pip wheel setuptools - venv\Scripts\pip install "numpy<2" - venv\Scripts\pip install caiman - venv\Scripts\pip install opencv-python-headless - venv\Scripts\pip install pyinstaller - venv\Scripts\caimanmanager install - - venv\Scripts\python -c "import sys; import caiman, numpy as np; print('Python:', sys.version); print('Caiman:', getattr(caiman,'__version__','?')); print('NumPy:', np.__version__)" + C:\Miniconda3\Scripts\conda.exe install -n caiman_release -c conda-forge caiman pyinstaller -y + C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import caiman; print('Caiman version:', getattr(caiman,'__version__','?'))" - name: Clean old dist/build shell: cmd From e9efcc35d0b93d79a5016903e23521d5fa9fd329 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 23 Sep 2025 15:38:51 -0400 Subject: [PATCH 06/86] Remove miniconda installation -- It was installed on the service --- .github/workflows/release-cmn.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/release-cmn.yml b/.github/workflows/release-cmn.yml index 537be7f8..8ecfdb0b 100644 --- a/.github/workflows/release-cmn.yml +++ b/.github/workflows/release-cmn.yml @@ -27,13 +27,9 @@ jobs: set VERSION=%GITHUB_REF_NAME:caiman-v=% echo VERSION=%VERSION%>> %GITHUB_ENV% - - name: Install Miniconda + - name: Verify Miniconda shell: cmd run: | - if not exist C:\Miniconda3 ( - curl -L -o miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe - start /wait "" miniconda.exe /InstallationType=AllUsers /AddToPath=1 /S /D=C:\Miniconda3 - ) C:\Miniconda3\Scripts\conda.exe --version - name: Create conda env for caiman From c00514ceb732f478cf2709fb4ae5c8a622dce0d2 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 23 Sep 2025 16:08:48 -0400 Subject: [PATCH 07/86] Update caiman installation --- .github/workflows/release-cmn.yml | 33 +++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release-cmn.yml b/.github/workflows/release-cmn.yml index 8ecfdb0b..e88232d4 100644 --- a/.github/workflows/release-cmn.yml +++ b/.github/workflows/release-cmn.yml @@ -29,8 +29,7 @@ jobs: - name: Verify Miniconda shell: cmd - run: | - C:\Miniconda3\Scripts\conda.exe --version + run: C:\Miniconda3\Scripts\conda.exe --version - name: Create conda env for caiman shell: cmd @@ -41,23 +40,37 @@ jobs: - name: Install caiman deps with conda-forge shell: cmd run: | - C:\Miniconda3\Scripts\conda.exe install -n caiman_release -c conda-forge caiman pyinstaller -y + C:\Miniconda3\Scripts\conda.exe install -n caiman_release -c conda-forge pyinstaller caiman==1.11.4 -y C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import caiman; print('Caiman version:', getattr(caiman,'__version__','?'))" - - name: Clean old dist/build + - name: Run caimanmanager install + shell: cmd + run: | + C:\Miniconda3\Scripts\conda.exe run -n caiman_release caimanmanager install + C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import os, pathlib, sys; home=str(pathlib.Path.home()); print('Caiman manager installed under:', home)" + + - name: Preflight paths (cmd) + shell: cmd + run: | + cd + dir + dir Deploy + dir Deploy\pack_caiman_run.spec + + - name: Clean old dist/build (cmd) shell: cmd working-directory: Deploy run: | - if exist dist rmdir /s /q dist - if exist build rmdir /s /q build + if exist dist rmdir /s /q dist + if exist build rmdir /s /q build - - name: Build caiman + - name: Build caiman exe (cmd) shell: cmd working-directory: Deploy run: | - ..\venv\Scripts\pyinstaller --clean --noconfirm pack_caiman_run.spec - echo === DIST TREE === - dir /s /b dist + C:\Miniconda3\Scripts\conda.exe run -n caiman_release pyinstaller --clean --noconfirm pack_caiman_run.spec + echo === DIST TREE === + dir /s /b dist - name: Zip onedir output with build-info shell: cmd From 957de359a79bfbed54f3b95305f520003eb691f8 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 23 Sep 2025 16:38:42 -0400 Subject: [PATCH 08/86] Change hard-coded path to the caiman_data --- Deploy/pack_caiman_run.spec | 48 ++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/Deploy/pack_caiman_run.spec b/Deploy/pack_caiman_run.spec index 052e05a6..dcae9b01 100644 --- a/Deploy/pack_caiman_run.spec +++ b/Deploy/pack_caiman_run.spec @@ -1,12 +1,12 @@ # -*- mode: python ; coding: utf-8 -*- -from PyInstaller.utils.hooks import collect_all -from PyInstaller.utils.hooks import copy_metadata -from PyInstaller.utils.hooks import collect_dynamic_libs +import os +from PyInstaller.utils.hooks import collect_all, copy_metadata +from PyInstaller.building.datastruct import Tree -# -# for maim CaimAn python script -# +_specdir = os.path.abspath(os.path.dirname(SPEC)) +distpath = os.path.join(_specdir, "dist") +workpath = os.path.join(_specdir, "build") block_cipher = None @@ -15,6 +15,37 @@ binaries = [] hiddenimports = [] excludes = [] +# Where caiman is installed: +cmn = importlib.import_module("caiman") +CAIMAN_PKG = Path(cmn.__file__).parent + +# Likely places for models/data created by 'caimanmanager install' +HOME = Path.home() +CANDIDATE_DIRS = [ + CAIMAN_PKG / "caiman_data" / "model", + CAIMAN_PKG / "data" / "model", + HOME / ".caimanmanager" / "caiman_data" / "model", + HOME / ".caimanmanager" / "data" / "model", + HOME / ".caimanmanager" / "model", +] + +def first_existing_dir(paths): + for p in paths: + if p.is_dir(): + return p + return None + +MODELS_DIR = first_existing_dir(CANDIDATE_DIRS) + +# Build datas list safely (don’t hard fail if missing) +datas = [] +if MODELS_DIR: + # Include the whole 'model' dir under 'caiman_data/model' in the bundle + datas += Tree(str(MODELS_DIR), prefix=os.path.join("caiman_data", "model")).toc + print(f"[spec] Including Caiman models from: {MODELS_DIR}") +else: + print("[spec] WARNING: could not find Caiman 'model' directory; skipping bundle of models") + tmp_ret = collect_all('caiman') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] @@ -34,9 +65,10 @@ datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('scipy') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] -datas += [( '../CaImAn/caiman_data/model', 'caiman_data/model')] +# datas += [( '../CaImAn/caiman_data/model', 'caiman_data/model')] -excludes = ["PyQt5", "Markdown", "jupyter", "panel", "matplotlib", "bokeh", "IPython", "ipyparallel", "ipywidgets", "tensorflow", "pyqtgraph"] +excludes = ["PyQt5", "Markdown", "jupyter", "panel", "matplotlib", "bokeh", "IPython", "ipyparallel", "ipywidgets", "tensorflow", "pyqtgraph", + "torch", "torchvision", "scipy._lib.array_api_compat.torch"] a_caimAn = Analysis( ['../CaImAn/caiman_run.py'], From 25ed141abc7d92897ab1d60c9ecd08cec93b8358 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 23 Sep 2025 16:48:27 -0400 Subject: [PATCH 09/86] Add missing imports in caiman spec --- Deploy/pack_caiman_run.spec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Deploy/pack_caiman_run.spec b/Deploy/pack_caiman_run.spec index dcae9b01..c61bc884 100644 --- a/Deploy/pack_caiman_run.spec +++ b/Deploy/pack_caiman_run.spec @@ -1,6 +1,7 @@ # -*- mode: python ; coding: utf-8 -*- -import os +import os, importlib +from pathlib import Path from PyInstaller.utils.hooks import collect_all, copy_metadata from PyInstaller.building.datastruct import Tree From 4b7c4e84431257f9120a83ef6df35a780bf1231c Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 23 Sep 2025 17:18:07 -0400 Subject: [PATCH 10/86] Add caiman_data to the environment --- .github/workflows/release-cmn.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-cmn.yml b/.github/workflows/release-cmn.yml index e88232d4..9ad00b75 100644 --- a/.github/workflows/release-cmn.yml +++ b/.github/workflows/release-cmn.yml @@ -43,11 +43,11 @@ jobs: C:\Miniconda3\Scripts\conda.exe install -n caiman_release -c conda-forge pyinstaller caiman==1.11.4 -y C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import caiman; print('Caiman version:', getattr(caiman,'__version__','?'))" - - name: Run caimanmanager install + - name: Run caimanmanager inplace install shell: cmd run: | - C:\Miniconda3\Scripts\conda.exe run -n caiman_release caimanmanager install - C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import os, pathlib, sys; home=str(pathlib.Path.home()); print('Caiman manager installed under:', home)" + C:\Miniconda3\Scripts\conda.exe run -n caiman_release caimanmanager install --inplace --force + C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import caiman, pathlib; print('Caiman package dir:', pathlib.Path(caiman.__file__).parent)" - name: Preflight paths (cmd) shell: cmd From 88f919f5fa10e4eea3ccde93430862e508517de4 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 09:49:19 -0400 Subject: [PATCH 11/86] Edit spec files: add missing imports, specify build and dist paths --- Deploy/pack_caiman_run.spec | 39 +++++++-------------------------- Deploy/pack_deeplabcut_run.spec | 19 ++++++++++------ Deploy/pack_minian_run.spec | 20 +++++++++-------- Deploy/pack_suite2p_run.spec | 18 ++++++++------- 4 files changed, 41 insertions(+), 55 deletions(-) diff --git a/Deploy/pack_caiman_run.spec b/Deploy/pack_caiman_run.spec index c61bc884..9f3ea93b 100644 --- a/Deploy/pack_caiman_run.spec +++ b/Deploy/pack_caiman_run.spec @@ -2,6 +2,8 @@ import os, importlib from pathlib import Path + +from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT from PyInstaller.utils.hooks import collect_all, copy_metadata from PyInstaller.building.datastruct import Tree @@ -9,43 +11,18 @@ _specdir = os.path.abspath(os.path.dirname(SPEC)) distpath = os.path.join(_specdir, "dist") workpath = os.path.join(_specdir, "build") -block_cipher = None +BLOCK_CIPHER = None datas = [] binaries = [] hiddenimports = [] excludes = [] -# Where caiman is installed: cmn = importlib.import_module("caiman") CAIMAN_PKG = Path(cmn.__file__).parent - -# Likely places for models/data created by 'caimanmanager install' -HOME = Path.home() -CANDIDATE_DIRS = [ - CAIMAN_PKG / "caiman_data" / "model", - CAIMAN_PKG / "data" / "model", - HOME / ".caimanmanager" / "caiman_data" / "model", - HOME / ".caimanmanager" / "data" / "model", - HOME / ".caimanmanager" / "model", -] - -def first_existing_dir(paths): - for p in paths: - if p.is_dir(): - return p - return None - -MODELS_DIR = first_existing_dir(CANDIDATE_DIRS) - -# Build datas list safely (don’t hard fail if missing) -datas = [] -if MODELS_DIR: - # Include the whole 'model' dir under 'caiman_data/model' in the bundle - datas += Tree(str(MODELS_DIR), prefix=os.path.join("caiman_data", "model")).toc - print(f"[spec] Including Caiman models from: {MODELS_DIR}") -else: - print("[spec] WARNING: could not find Caiman 'model' directory; skipping bundle of models") +INPLACE_DATA = CAIMAN_PKG / "caiman_data" +datas += Tree(str(INPLACE_DATA), prefix="caiman_data").toc +datas += Tree(str(INPLACE_DATA), prefix=os.path.join("caiman_data", "model")).toc tmp_ret = collect_all('caiman') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] @@ -83,11 +60,11 @@ a_caimAn = Analysis( excludes=excludes, win_no_prefer_redirects=False, win_private_assemblies=False, - cipher=block_cipher, + cipher=BLOCK_CIPHER, noarchive=False, ) -pyz_caimAn = PYZ(a_caimAn.pure, a_caimAn.zipped_data, cipher=block_cipher) +pyz_caimAn = PYZ(a_caimAn.pure, a_caimAn.zipped_data, cipher=BLOCK_CIPHER) exe_caimAn = EXE( pyz_caimAn, diff --git a/Deploy/pack_deeplabcut_run.spec b/Deploy/pack_deeplabcut_run.spec index 2f4f10c5..b8d6616d 100644 --- a/Deploy/pack_deeplabcut_run.spec +++ b/Deploy/pack_deeplabcut_run.spec @@ -1,18 +1,23 @@ # -*- mode: python ; coding: utf-8 -*- -from PyInstaller.utils.hooks import collect_all -from PyInstaller.utils.hooks import copy_metadata -from PyInstaller.utils.hooks import collect_dynamic_libs +import os +from pathlib import Path +from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT +from PyInstaller.utils.hooks import collect_all, copy_metadata -block_cipher = None +_specdir = os.path.abspath(os.path.dirname(SPEC)) +distpath = os.path.join(_specdir, "dist") +workpath = os.path.join(_specdir, "build") + +BLOCK_CIPHER = None datas = [] binaries = [] hiddenimports = [] excludes = [] -# datas += copy_metadata('deeplabcut', recursive=True) +datas += copy_metadata('deeplabcut', recursive=True) tmp_ret = collect_all('deeplabcut') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] @@ -32,7 +37,7 @@ a_deeplabcut = Analysis( excludes=excludes, win_no_prefer_redirects=False, win_private_assemblies=False, - cipher=block_cipher, + cipher=BLOCK_CIPHER, noarchive=False, ) @@ -58,7 +63,7 @@ exclude_binaries = [ a_deeplabcut.binaries= TOC([x for x in a_deeplabcut.binaries if not any(exclude in x[0] for exclude in exclude_binaries)]) -pyz_deeplabcut = PYZ(a_deeplabcut.pure, a_deeplabcut.zipped_data, cipher=block_cipher) +pyz_deeplabcut = PYZ(a_deeplabcut.pure, a_deeplabcut.zipped_data, cipher=BLOCK_CIPHER) exe_deeplabcut = EXE( pyz_deeplabcut, diff --git a/Deploy/pack_minian_run.spec b/Deploy/pack_minian_run.spec index 498f915c..87b00080 100644 --- a/Deploy/pack_minian_run.spec +++ b/Deploy/pack_minian_run.spec @@ -1,14 +1,16 @@ # -*- mode: python ; coding: utf-8 -*- -from PyInstaller.utils.hooks import collect_all -from PyInstaller.utils.hooks import copy_metadata -from PyInstaller.utils.hooks import collect_dynamic_libs +import os +from pathlib import Path -# -# for main MiniAn python script -# +from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT +from PyInstaller.utils.hooks import collect_all, collect_dynamic_libs -block_cipher = None +_specdir = os.path.abspath(os.path.dirname(SPEC)) +distpath = os.path.join(_specdir, "dist") +workpath = os.path.join(_specdir, "build") + +BLOCK_CIPHER = None datas = [] binaries = [] @@ -40,10 +42,10 @@ a_minian = Analysis( excludes=excludes, win_no_prefer_redirects=False, win_private_assemblies=False, - cipher=block_cipher, + cipher=BLOCK_CIPHER, noarchive=False, ) -pyz_minian = PYZ(a_minian.pure, a_minian.zipped_data, cipher=block_cipher) +pyz_minian = PYZ(a_minian.pure, a_minian.zipped_data, cipher=BLOCK_CIPHER) exe_minian = EXE( pyz_minian, diff --git a/Deploy/pack_suite2p_run.spec b/Deploy/pack_suite2p_run.spec index 441643b3..5e24734f 100644 --- a/Deploy/pack_suite2p_run.spec +++ b/Deploy/pack_suite2p_run.spec @@ -1,14 +1,16 @@ # -*- mode: python ; coding: utf-8 -*- +import os +from pathlib import Path + +from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT from PyInstaller.utils.hooks import collect_all -from PyInstaller.utils.hooks import copy_metadata -from PyInstaller.utils.hooks import collect_dynamic_libs -# -# for main MiniAn python script -# +_specdir = os.path.abspath(os.path.dirname(SPEC)) +distpath = os.path.join(_specdir, "dist") +workpath = os.path.join(_specdir, "build") -block_cipher = None +BLOCK_CIPHER = None datas = [] binaries = [] @@ -38,10 +40,10 @@ a_suite2p = Analysis( excludes=excludes, win_no_prefer_redirects=False, win_private_assemblies=False, - cipher=block_cipher, + cipher=BLOCK_CIPHER, noarchive=False, ) -pyz_suite2p = PYZ(a_suite2p.pure, a_suite2p.zipped_data, cipher=block_cipher) +pyz_suite2p = PYZ(a_suite2p.pure, a_suite2p.zipped_data, cipher=BLOCK_CIPHER) exe_suite2p = EXE( pyz_suite2p, From 316793bec5556c47e60a106feb34d37390f3f414 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 10:17:51 -0400 Subject: [PATCH 12/86] Draft release yml for MiniAn --- .github/workflows/release-mnn.yml | 134 ++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 .github/workflows/release-mnn.yml diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml new file mode 100644 index 00000000..dc4c4a9e --- /dev/null +++ b/.github/workflows/release-mnn.yml @@ -0,0 +1,134 @@ +name: Release MiniAn + +on: + push: + tags: ['minian-v*'] + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: mnn-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: [self-hosted, Windows, X64, GPU] + timeout-minutes: 180 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Extract version from tag + shell: cmd + run: | + set VERSION=%GITHUB_REF_NAME:minian-v=% + echo VERSION=%VERSION%>> %GITHUB_ENV% + + - name: Verify Miniconda + shell: cmd + run: C:\Miniconda3\Scripts\conda.exe --version + + - name: Create conda env for minian + shell: cmd + run: | + C:\Miniconda3\Scripts\conda.exe env remove -n minian_release -y || echo "no old env" + C:\Miniconda3\Scripts\conda.exe create -n minian_release python=3.10 -y + + - name: Install minian deps with conda-forge + shell: cmd + run: | + C:\Miniconda3\Scripts\conda.exe install -n minian_release -c bioconda -c conda-forge python=3.8 minian=1.2.1 h5py pyinstaller pefile=2023.2.7 -y + C:\Miniconda3\Scripts\conda.exe run -n minian_release python -c "import minian; print('MiniAn version:', getattr(minian,'__version__','?'))" + + - name: Preflight paths + shell: cmd + run: | + cd + dir + dir Deploy + dir Deploy\pack_minian_run.spec + + - name: Clean old dist/build + shell: cmd + working-directory: Deploy + run: | + if exist dist rmdir /s /q dist + if exist build rmdir /s /q build + + - name: Build minian exe + shell: cmd + working-directory: Deploy + run: | + C:\Miniconda3\Scripts\conda.exe run -n minian_release pyinstaller --clean --noconfirm pack_minian_run.spec + echo === DIST TREE === + dir /s /b dist + + - name: Zip onedir output with build-info + shell: cmd + working-directory: Deploy + env: + GIT_SHA: ${{ github.sha }} + run: | + rem Locate the onedir app folder under dist + set "APPDIR=" + for /d %%D in (dist\*) do ( set "APPDIR=%%D" & goto :found ) + :found + if not defined APPDIR ( echo ERROR: no onedir folder found in Deploy\dist & exit /b 1 ) + + rem Write build-info into the app folder (included in the zip) + echo Version: %VERSION%> "%APPDIR%\build-info.txt" + echo Commit: %GIT_SHA%>> "%APPDIR%\build-info.txt" + echo Date: %DATE% %TIME%>> "%APPDIR%\build-info.txt" + + set "ZIPNAME=minian-%VERSION%.zip" + + rem One-line Python (no backslashes/newlines) + ..\venv\Scripts\python -c "import os,zipfile; a=r'%APPDIR%'; zf=r'%ZIPNAME%'; assert os.path.isdir(a), 'no APPDIR: '+a; z=zipfile.ZipFile(zf,'w',zipfile.ZIP_DEFLATED); [z.write(os.path.join(r,f), os.path.relpath(os.path.join(r,f),a)) for r,_,fs in os.walk(a) for f in fs]; z.close()" + echo Created zip: %ZIPNAME% + + - name: Show zip contents + shell: cmd + working-directory: Deploy + run: | + set "ZN=minian-%VERSION%.zip" + if not exist "%ZN%" ( echo ERROR: zip not found & exit /b 1 ) + ..\venv\Scripts\python -c "import os,zipfile; zn=r'%ZN%'; z=zipfile.ZipFile(zn,'r'); n=z.namelist(); print('Entries in',zn,':',len(n)); [print(' ',x) for x in n[:50]]; z.close()" + + - name: SHA256 of release zip + shell: cmd + working-directory: Deploy + run: | + setlocal EnableDelayedExpansion + set ZIP=minian-%VERSION%.zip + if not exist "%ZIP%" ( echo ERROR: zip not found & exit /b 1 ) + for /f "usebackq skip=1 tokens=1" %%H in (`certutil -hashfile "%ZIP%" SHA256 ^| findstr /v /i "hash certutil"`) do ( + set HASH=%%H + goto :got + ) + :got + echo !HASH!> minian-%VERSION%.zip.sha256 + type minian-%VERSION%.zip.sha256 + + - name: Create GitHub Release (attach zip + checksum) + uses: softprops/action-gh-release@v2 + with: + name: minian ${{ env.VERSION }} + body: | + Automated minian release for tag ${{ github.ref_name }}. + - Version: ${{ env.VERSION }} + - Commit: ${{ github.sha }} + files: | + Deploy/minian-${{ env.VERSION }}.zip + Deploy/minian-${{ env.VERSION }}.zip.sha256 + + - name: Upload dist artifact (only on failure) + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: minian-dist-${{ env.VERSION }} + path: Deploy/dist/** + retention-days: 3 + compression-level: 1 From 8ba3a226dba91b8ac507a0a3deeec692f6a9c5de Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 10:31:18 -0400 Subject: [PATCH 13/86] Add draft of suite2p release yml --- .github/workflows/release-s2p.yml | 124 ++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 .github/workflows/release-s2p.yml diff --git a/.github/workflows/release-s2p.yml b/.github/workflows/release-s2p.yml new file mode 100644 index 00000000..e97ac829 --- /dev/null +++ b/.github/workflows/release-s2p.yml @@ -0,0 +1,124 @@ +name: Release Suite2p + +on: + push: + tags: ['suite2p-v*'] + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: s2p-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: [self-hosted, Windows, X64, GPU] + timeout-minutes: 180 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Extract version from tag + shell: cmd + run: | + set VERSION=%GITHUB_REF_NAME:suite2p-v=% + echo VERSION=%VERSION%>> %GITHUB_ENV% + + - name: Create venv + shell: cmd + run: | + C:\Python310\python.exe -V + C:\Python310\python.exe -m venv venv + + - name: Install deps + shell: cmd + run: | + venv\Scripts\python -m pip install --upgrade pip + venv\Scripts\pip install h5py pyinstaller suite2p + venv\Scripts\pip install --upgrade --index-url https://download.pytorch.org/whl/cu126 torch + venv\Scripts\python -c "import suite2p; print('Suite2p version:', getattr(suite2p,'__version__','?'))" + + - name: Clean old dist/build + shell: cmd + working-directory: Deploy + run: | + if exist dist rmdir /s /q dist + if exist build rmdir /s /q build + + - name: Build suite2p + shell: cmd + working-directory: Deploy + run: | + ..\venv\Scripts\pyinstaller --clean --noconfirm pack_suite2p_run.spec + echo === DIST TREE === + dir /s /b dist + + - name: Zip onedir output with build-info + shell: cmd + working-directory: Deploy + env: + GIT_SHA: ${{ github.sha }} + run: | + rem Locate the onedir app folder under dist + set "APPDIR=" + for /d %%D in (dist\*) do ( set "APPDIR=%%D" & goto :found ) + :found + if not defined APPDIR ( echo ERROR: no onedir folder found in Deploy\dist & exit /b 1 ) + + rem Write build-info into the app folder (included in the zip) + echo Version: %VERSION%> "%APPDIR%\build-info.txt" + echo Commit: %GIT_SHA%>> "%APPDIR%\build-info.txt" + echo Date: %DATE% %TIME%>> "%APPDIR%\build-info.txt" + + set "ZIPNAME=suite2p-%VERSION%.zip" + + rem One-line Python (no backslashes/newlines) + ..\venv\Scripts\python -c "import os,zipfile; a=r'%APPDIR%'; zf=r'%ZIPNAME%'; assert os.path.isdir(a), 'no APPDIR: '+a; z=zipfile.ZipFile(zf,'w',zipfile.ZIP_DEFLATED); [z.write(os.path.join(r,f), os.path.relpath(os.path.join(r,f),a)) for r,_,fs in os.walk(a) for f in fs]; z.close()" + echo Created zip: %ZIPNAME% + + - name: Show zip contents + shell: cmd + working-directory: Deploy + run: | + set "ZN=suite2p-%VERSION%.zip" + if not exist "%ZN%" ( echo ERROR: zip not found & exit /b 1 ) + ..\venv\Scripts\python -c "import os,zipfile; zn=r'%ZN%'; z=zipfile.ZipFile(zn,'r'); n=z.namelist(); print('Entries in',zn,':',len(n)); [print(' ',x) for x in n[:50]]; z.close()" + + - name: SHA256 of release zip + shell: cmd + working-directory: Deploy + run: | + setlocal EnableDelayedExpansion + set ZIP=suite2p-%VERSION%.zip + if not exist "%ZIP%" ( echo ERROR: zip not found & exit /b 1 ) + for /f "usebackq skip=1 tokens=1" %%H in (`certutil -hashfile "%ZIP%" SHA256 ^| findstr /v /i "hash certutil"`) do ( + set HASH=%%H + goto :got + ) + :got + echo !HASH!> suite2p-%VERSION%.zip.sha256 + type suite2p-%VERSION%.zip.sha256 + + - name: Create GitHub Release (attach zip + checksum) + uses: softprops/action-gh-release@v2 + with: + name: suite2p ${{ env.VERSION }} + body: | + Automated suite2p release for tag ${{ github.ref_name }}. + - Version: ${{ env.VERSION }} + - Commit: ${{ github.sha }} + files: | + Deploy/suite2p-${{ env.VERSION }}.zip + Deploy/suite2p-${{ env.VERSION }}.zip.sha256 + + - name: Upload dist artifact (only on failure) + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: suite2p-dist-${{ env.VERSION }} + path: Deploy/dist/** + retention-days: 3 + compression-level: 1 From 71896d7c8b7382de24fd6cb2b21c47524fcb5f9f Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 10:35:55 -0400 Subject: [PATCH 14/86] Remove torch installation from suite2p release --- .github/workflows/release-s2p.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release-s2p.yml b/.github/workflows/release-s2p.yml index e97ac829..011bbae1 100644 --- a/.github/workflows/release-s2p.yml +++ b/.github/workflows/release-s2p.yml @@ -38,7 +38,6 @@ jobs: run: | venv\Scripts\python -m pip install --upgrade pip venv\Scripts\pip install h5py pyinstaller suite2p - venv\Scripts\pip install --upgrade --index-url https://download.pytorch.org/whl/cu126 torch venv\Scripts\python -c "import suite2p; print('Suite2p version:', getattr(suite2p,'__version__','?'))" - name: Clean old dist/build From d6d3d0d1727511def8bd41179fc245c421b5e228 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 11:05:32 -0400 Subject: [PATCH 15/86] Edit minian release: try to use venv --- .github/workflows/release-mnn.yml | 39 +++++++++++-------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml index dc4c4a9e..f7e0df54 100644 --- a/.github/workflows/release-mnn.yml +++ b/.github/workflows/release-mnn.yml @@ -27,44 +27,33 @@ jobs: set VERSION=%GITHUB_REF_NAME:minian-v=% echo VERSION=%VERSION%>> %GITHUB_ENV% - - name: Verify Miniconda - shell: cmd - run: C:\Miniconda3\Scripts\conda.exe --version - - - name: Create conda env for minian - shell: cmd - run: | - C:\Miniconda3\Scripts\conda.exe env remove -n minian_release -y || echo "no old env" - C:\Miniconda3\Scripts\conda.exe create -n minian_release python=3.10 -y - - - name: Install minian deps with conda-forge + - name: Create venv shell: cmd run: | - C:\Miniconda3\Scripts\conda.exe install -n minian_release -c bioconda -c conda-forge python=3.8 minian=1.2.1 h5py pyinstaller pefile=2023.2.7 -y - C:\Miniconda3\Scripts\conda.exe run -n minian_release python -c "import minian; print('MiniAn version:', getattr(minian,'__version__','?'))" + C:\Python310\python.exe -V + C:\Python310\python.exe -m venv venv - - name: Preflight paths + - name: Install deps shell: cmd run: | - cd - dir - dir Deploy - dir Deploy\pack_minian_run.spec + venv\Scripts\python -m pip install --upgrade pip + venv\Scripts\pip install h5py pyinstaller minian + venv\Scripts\python -c "import minian; print('MiniAn version:', getattr(minian,'__version__','?'))" - - name: Clean old dist/build + - name: Clean old dist/build shell: cmd working-directory: Deploy run: | - if exist dist rmdir /s /q dist - if exist build rmdir /s /q build + if exist dist rmdir /s /q dist + if exist build rmdir /s /q build - - name: Build minian exe + - name: Build minian shell: cmd working-directory: Deploy run: | - C:\Miniconda3\Scripts\conda.exe run -n minian_release pyinstaller --clean --noconfirm pack_minian_run.spec - echo === DIST TREE === - dir /s /b dist + ..\venv\Scripts\pyinstaller --clean --noconfirm pack_minian_run.spec + echo === DIST TREE === + dir /s /b dist - name: Zip onedir output with build-info shell: cmd From 8792f41fce1786bb5514991b0fee4e7286308630 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 11:15:29 -0400 Subject: [PATCH 16/86] Revert to use conda for minian installation --- .github/workflows/release-mnn.yml | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml index f7e0df54..8883eea9 100644 --- a/.github/workflows/release-mnn.yml +++ b/.github/workflows/release-mnn.yml @@ -27,33 +27,36 @@ jobs: set VERSION=%GITHUB_REF_NAME:minian-v=% echo VERSION=%VERSION%>> %GITHUB_ENV% - - name: Create venv + - name: Verify Miniconda + shell: cmd + run: C:\Miniconda3\Scripts\conda.exe --version + + - name: Create conda env for minian shell: cmd run: | - C:\Python310\python.exe -V - C:\Python310\python.exe -m venv venv + C:\Miniconda3\Scripts\conda.exe env remove -n minian_release -y || echo "no old env" + C:\Miniconda3\Scripts\conda.exe create -n minian_release python=3.10 -y - - name: Install deps + - name: Install minian deps with conda-forge shell: cmd run: | - venv\Scripts\python -m pip install --upgrade pip - venv\Scripts\pip install h5py pyinstaller minian - venv\Scripts\python -c "import minian; print('MiniAn version:', getattr(minian,'__version__','?'))" + C:\Miniconda3\Scripts\conda.exe install -n minian_release -c conda-forge h5py pyinstaller minian -y + C:\Miniconda3\Scripts\conda.exe run -n minian_release python -c "import minian; print('MiniAn version:', getattr(minian,'__version__','?'))" - - name: Clean old dist/build + - name: Clean old dist/build shell: cmd working-directory: Deploy run: | - if exist dist rmdir /s /q dist - if exist build rmdir /s /q build + if exist dist rmdir /s /q dist + if exist build rmdir /s /q build - - name: Build minian + - name: Build minian exe shell: cmd working-directory: Deploy run: | - ..\venv\Scripts\pyinstaller --clean --noconfirm pack_minian_run.spec - echo === DIST TREE === - dir /s /b dist + C:\Miniconda3\Scripts\conda.exe run -n minian_release pyinstaller --clean --noconfirm pack_minian_run.spec + echo === DIST TREE === + dir /s /b dist - name: Zip onedir output with build-info shell: cmd From fd3f9315c7137258e265f7b580b406e565767d55 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 11:40:39 -0400 Subject: [PATCH 17/86] Set python version to 3.9 for minian release --- .github/workflows/release-mnn.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml index 8883eea9..36a0b4c7 100644 --- a/.github/workflows/release-mnn.yml +++ b/.github/workflows/release-mnn.yml @@ -35,12 +35,12 @@ jobs: shell: cmd run: | C:\Miniconda3\Scripts\conda.exe env remove -n minian_release -y || echo "no old env" - C:\Miniconda3\Scripts\conda.exe create -n minian_release python=3.10 -y + C:\Miniconda3\Scripts\conda.exe create -n minian_release python=3.9 -y - name: Install minian deps with conda-forge shell: cmd run: | - C:\Miniconda3\Scripts\conda.exe install -n minian_release -c conda-forge h5py pyinstaller minian -y + C:\Miniconda3\Scripts\conda.exe install -n minian_release -c conda-forge h5py pyinstaller minian -y C:\Miniconda3\Scripts\conda.exe run -n minian_release python -c "import minian; print('MiniAn version:', getattr(minian,'__version__','?'))" - name: Clean old dist/build From 4ceec5cfb2f8e72700415d98ea028b1b38d83eda Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 11:51:59 -0400 Subject: [PATCH 18/86] Change python to 3.8 for minian installation --- .github/workflows/release-mnn.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml index 36a0b4c7..d4822491 100644 --- a/.github/workflows/release-mnn.yml +++ b/.github/workflows/release-mnn.yml @@ -35,7 +35,7 @@ jobs: shell: cmd run: | C:\Miniconda3\Scripts\conda.exe env remove -n minian_release -y || echo "no old env" - C:\Miniconda3\Scripts\conda.exe create -n minian_release python=3.9 -y + C:\Miniconda3\Scripts\conda.exe create -n minian_release python=3.8 -y - name: Install minian deps with conda-forge shell: cmd From f2cfc60eabcf5bd90ea842b9ac81c8d7069aeadf Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 12:23:17 -0400 Subject: [PATCH 19/86] Edit caiman release file --- .github/workflows/release-cmn.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-cmn.yml b/.github/workflows/release-cmn.yml index 9ad00b75..b2a25766 100644 --- a/.github/workflows/release-cmn.yml +++ b/.github/workflows/release-cmn.yml @@ -40,7 +40,7 @@ jobs: - name: Install caiman deps with conda-forge shell: cmd run: | - C:\Miniconda3\Scripts\conda.exe install -n caiman_release -c conda-forge pyinstaller caiman==1.11.4 -y + C:\Miniconda3\Scripts\conda.exe install -n caiman_release -c conda-forge pyinstaller caiman -y C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import caiman; print('Caiman version:', getattr(caiman,'__version__','?'))" - name: Run caimanmanager inplace install @@ -49,7 +49,7 @@ jobs: C:\Miniconda3\Scripts\conda.exe run -n caiman_release caimanmanager install --inplace --force C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import caiman, pathlib; print('Caiman package dir:', pathlib.Path(caiman.__file__).parent)" - - name: Preflight paths (cmd) + - name: Preflight paths shell: cmd run: | cd @@ -57,14 +57,14 @@ jobs: dir Deploy dir Deploy\pack_caiman_run.spec - - name: Clean old dist/build (cmd) + - name: Clean old dist/build shell: cmd working-directory: Deploy run: | if exist dist rmdir /s /q dist if exist build rmdir /s /q build - - name: Build caiman exe (cmd) + - name: Build caiman exe shell: cmd working-directory: Deploy run: | From b8916e001eb698871fb1edf6eb8d168c70f78a9e Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 13:00:44 -0400 Subject: [PATCH 20/86] Fix caiman data dir --- Deploy/pack_caiman_run.spec | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Deploy/pack_caiman_run.spec b/Deploy/pack_caiman_run.spec index 9f3ea93b..f99a8d4d 100644 --- a/Deploy/pack_caiman_run.spec +++ b/Deploy/pack_caiman_run.spec @@ -18,11 +18,8 @@ binaries = [] hiddenimports = [] excludes = [] -cmn = importlib.import_module("caiman") -CAIMAN_PKG = Path(cmn.__file__).parent -INPLACE_DATA = CAIMAN_PKG / "caiman_data" -datas += Tree(str(INPLACE_DATA), prefix="caiman_data").toc -datas += Tree(str(INPLACE_DATA), prefix=os.path.join("caiman_data", "model")).toc +CAIMAN_DATA_DIR = os.environ.get("CAIMAN_DATA_DIR") +datas += Tree(CAIMAN_DATA_DIR, prefix="caiman_data").toc tmp_ret = collect_all('caiman') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] From b9dde9a177715ebf9df0c047c81ad6e5ef082bd5 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 14:10:41 -0400 Subject: [PATCH 21/86] Change place for caiman_data --- .github/workflows/release-cmn.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release-cmn.yml b/.github/workflows/release-cmn.yml index b2a25766..179d4c22 100644 --- a/.github/workflows/release-cmn.yml +++ b/.github/workflows/release-cmn.yml @@ -43,19 +43,21 @@ jobs: C:\Miniconda3\Scripts\conda.exe install -n caiman_release -c conda-forge pyinstaller caiman -y C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import caiman; print('Caiman version:', getattr(caiman,'__version__','?'))" - - name: Run caimanmanager inplace install + - name: Install Caiman data into workspace (no inplace) shell: cmd + working-directory: Deploy run: | - C:\Miniconda3\Scripts\conda.exe run -n caiman_release caimanmanager install --inplace --force - C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import caiman, pathlib; print('Caiman package dir:', pathlib.Path(caiman.__file__).parent)" + set "CAIMAN_DATA_DIR=%CD%\caiman_data" + if exist "%CAIMAN_DATA_DIR%" rmdir /s /q "%CAIMAN_DATA_DIR%" + mkdir "%CAIMAN_DATA_DIR%" + C:\Miniconda3\Scripts\conda.exe run -n caiman_release caimanmanager install --dir "%CAIMAN_DATA_DIR%" --force + echo CAIMAN_DATA_DIR=%CAIMAN_DATA_DIR%>> %GITHUB_ENV% - - name: Preflight paths + - name: Show installed Caiman data shell: cmd + working-directory: Deploy run: | - cd - dir - dir Deploy - dir Deploy\pack_caiman_run.spec + dir /s /b caiman_data | find /c /v "" - name: Clean old dist/build shell: cmd From bd2d9739a63b94a5e975f93002ff79b509b3b429 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 14:50:33 -0400 Subject: [PATCH 22/86] Resolve caiman_data location --- .github/workflows/release-cmn.yml | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-cmn.yml b/.github/workflows/release-cmn.yml index 179d4c22..44d10403 100644 --- a/.github/workflows/release-cmn.yml +++ b/.github/workflows/release-cmn.yml @@ -43,15 +43,23 @@ jobs: C:\Miniconda3\Scripts\conda.exe install -n caiman_release -c conda-forge pyinstaller caiman -y C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import caiman; print('Caiman version:', getattr(caiman,'__version__','?'))" - - name: Install Caiman data into workspace (no inplace) + - name: Run caimanmanager install (user home, force) shell: cmd - working-directory: Deploy run: | - set "CAIMAN_DATA_DIR=%CD%\caiman_data" - if exist "%CAIMAN_DATA_DIR%" rmdir /s /q "%CAIMAN_DATA_DIR%" - mkdir "%CAIMAN_DATA_DIR%" - C:\Miniconda3\Scripts\conda.exe run -n caiman_release caimanmanager install --dir "%CAIMAN_DATA_DIR%" --force - echo CAIMAN_DATA_DIR=%CAIMAN_DATA_DIR%>> %GITHUB_ENV% + rem Resolve the service account’s home (NetworkService on your runner) + for /f "delims=" %%H in ('C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import pathlib; print(pathlib.Path.home())"') do set "CM_HOME=%%H" + echo Home resolved to: %CM_HOME% + + rem Remove any previous installs so we get a clean copy + if exist "%CM_HOME%\caiman_data" rmdir /s /q "%CM_HOME%\caiman_data" + if exist "%CM_HOME%\.caimanmanager\caiman_data" rmdir /s /q "%CM_HOME%\.caimanmanager\caiman_data" + + rem Install into user home (default), overwrite if needed + C:\Miniconda3\Scripts\conda.exe run -n caiman_release caimanmanager install --force + + rem Show what was installed + if exist "%CM_HOME%\caiman_data" (dir /s /b "%CM_HOME%\caiman_data" | find /c /v "" ) + if exist "%CM_HOME%\.caimanmanager\caiman_data" (dir /s /b "%CM_HOME%\.caimanmanager\caiman_data" | find /c /v "" ) - name: Show installed Caiman data shell: cmd From 590e39cdd3aee839718335bafbf7580bf1f8b814 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 15:46:00 -0400 Subject: [PATCH 23/86] Try to resolve caiman_data location again --- .github/workflows/release-cmn.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-cmn.yml b/.github/workflows/release-cmn.yml index b2a25766..07b9239c 100644 --- a/.github/workflows/release-cmn.yml +++ b/.github/workflows/release-cmn.yml @@ -43,12 +43,19 @@ jobs: C:\Miniconda3\Scripts\conda.exe install -n caiman_release -c conda-forge pyinstaller caiman -y C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import caiman; print('Caiman version:', getattr(caiman,'__version__','?'))" - - name: Run caimanmanager inplace install + - name: caimanmanager install (detect path and export) shell: cmd run: | - C:\Miniconda3\Scripts\conda.exe run -n caiman_release caimanmanager install --inplace --force - C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import caiman, pathlib; print('Caiman package dir:', pathlib.Path(caiman.__file__).parent)" - + C:\Miniconda3\Scripts\conda.exe run -n caiman_release caimanmanager install --force + for /f "delims=" %%H in ('C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import pathlib; print(pathlib.Path.home())"') do set "CM_HOME=%%H" + if exist "%CM_HOME%\caiman_data" ( + echo CAIMAN_DATA_DIR=%CM_HOME%\caiman_data>> %GITHUB_ENV% + ) else if exist "%CM_HOME%\.caimanmanager\caiman_data" ( + echo CAIMAN_DATA_DIR=%CM_HOME%\.caimanmanager\caiman_data>> %GITHUB_ENV% + ) else ( + echo WARNING: caiman_data not found after install + ) + - name: Preflight paths shell: cmd run: | From f624697fc0ccea2e4e90b666c4f71b19745d8be8 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 16:15:37 -0400 Subject: [PATCH 24/86] Change caiman data location in spec file --- Deploy/pack_caiman_run.spec | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Deploy/pack_caiman_run.spec b/Deploy/pack_caiman_run.spec index f99a8d4d..77ca73ab 100644 --- a/Deploy/pack_caiman_run.spec +++ b/Deploy/pack_caiman_run.spec @@ -1,7 +1,6 @@ # -*- mode: python ; coding: utf-8 -*- -import os, importlib -from pathlib import Path +import os from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT from PyInstaller.utils.hooks import collect_all, copy_metadata @@ -19,7 +18,11 @@ hiddenimports = [] excludes = [] CAIMAN_DATA_DIR = os.environ.get("CAIMAN_DATA_DIR") -datas += Tree(CAIMAN_DATA_DIR, prefix="caiman_data").toc +if CAIMAN_DATA_DIR and os.path.isdir(CAIMAN_DATA_DIR): + datas.append(Tree(CAIMAN_DATA_DIR, prefix="caiman_data")) +else: + print("[spec] WARNING: CAIMAN_DATA_DIR not set or not a directory; models/configs not bundled.") + tmp_ret = collect_all('caiman') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] From f2cfc49e8e6102ea37267779ce554590a9fede4e Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 16:49:09 -0400 Subject: [PATCH 25/86] Add caiman_data to collect --- Deploy/pack_caiman_run.spec | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Deploy/pack_caiman_run.spec b/Deploy/pack_caiman_run.spec index 77ca73ab..6c4817a2 100644 --- a/Deploy/pack_caiman_run.spec +++ b/Deploy/pack_caiman_run.spec @@ -17,13 +17,6 @@ binaries = [] hiddenimports = [] excludes = [] -CAIMAN_DATA_DIR = os.environ.get("CAIMAN_DATA_DIR") -if CAIMAN_DATA_DIR and os.path.isdir(CAIMAN_DATA_DIR): - datas.append(Tree(CAIMAN_DATA_DIR, prefix="caiman_data")) -else: - print("[spec] WARNING: CAIMAN_DATA_DIR not set or not a directory; models/configs not bundled.") - - tmp_ret = collect_all('caiman') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] @@ -85,11 +78,19 @@ exe_caimAn = EXE( entitlements_file=None, ) +extra_nodes = [] +if CAIMAN_DATA_DIR and os.path.isdir(CAIMAN_DATA_DIR): + extra_nodes.append(Tree(CAIMAN_DATA_DIR, prefix="caiman_data")) +else: + print("[spec] WARNING: CAIMAN_DATA_DIR not set or not a directory; models/configs not bundled.") + + coll = COLLECT( exe_caimAn, a_caimAn.binaries, a_caimAn.zipfiles, a_caimAn.datas, + *extra_nodes, strip=False, upx=True, upx_exclude=[], From 36506b4bb3b31aeac65868d516376c01015f0fab Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 24 Sep 2025 17:05:47 -0400 Subject: [PATCH 26/86] Update location of the zip file for minian --- .github/workflows/release-mnn.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml index d4822491..89451b7f 100644 --- a/.github/workflows/release-mnn.yml +++ b/.github/workflows/release-mnn.yml @@ -77,8 +77,8 @@ jobs: set "ZIPNAME=minian-%VERSION%.zip" - rem One-line Python (no backslashes/newlines) - ..\venv\Scripts\python -c "import os,zipfile; a=r'%APPDIR%'; zf=r'%ZIPNAME%'; assert os.path.isdir(a), 'no APPDIR: '+a; z=zipfile.ZipFile(zf,'w',zipfile.ZIP_DEFLATED); [z.write(os.path.join(r,f), os.path.relpath(os.path.join(r,f),a)) for r,_,fs in os.walk(a) for f in fs]; z.close()" + rem Use conda env's python (single-line, no backslashes) + C:\Miniconda3\Scripts\conda.exe run -n minian_release python -c "import os,zipfile; a=r'%APPDIR%'; zf=r'%ZIPNAME%'; assert os.path.isdir(a), 'no APPDIR: '+a; z=zipfile.ZipFile(zf,'w',zipfile.ZIP_DEFLATED); [z.write(os.path.join(r,f), os.path.relpath(os.path.join(r,f),a)) for r,_,fs in os.walk(a) for f in fs]; z.close()" echo Created zip: %ZIPNAME% - name: Show zip contents @@ -87,7 +87,7 @@ jobs: run: | set "ZN=minian-%VERSION%.zip" if not exist "%ZN%" ( echo ERROR: zip not found & exit /b 1 ) - ..\venv\Scripts\python -c "import os,zipfile; zn=r'%ZN%'; z=zipfile.ZipFile(zn,'r'); n=z.namelist(); print('Entries in',zn,':',len(n)); [print(' ',x) for x in n[:50]]; z.close()" + C:\Miniconda3\Scripts\conda.exe run -n minian_release python -c "import os,zipfile; zn=r'%ZN%'; z=zipfile.ZipFile(zn,'r'); n=z.namelist(); print('Entries in',zn,':',len(n)); [print(' ',x) for x in n[:50]]; z.close()" - name: SHA256 of release zip shell: cmd From e0925824f3cb7331b5c7468bbcf9adad1e33ca0d Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 25 Sep 2025 09:24:03 -0400 Subject: [PATCH 27/86] Fix caiman zip in the workflow --- .github/workflows/release-cmn.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-cmn.yml b/.github/workflows/release-cmn.yml index 07b9239c..2c8a2019 100644 --- a/.github/workflows/release-cmn.yml +++ b/.github/workflows/release-cmn.yml @@ -98,8 +98,8 @@ jobs: set "ZIPNAME=caiman-%VERSION%.zip" - rem One-line Python (no backslashes/newlines) - ..\venv\Scripts\python -c "import os,zipfile; a=r'%APPDIR%'; zf=r'%ZIPNAME%'; assert os.path.isdir(a), 'no APPDIR: '+a; z=zipfile.ZipFile(zf,'w',zipfile.ZIP_DEFLATED); [z.write(os.path.join(r,f), os.path.relpath(os.path.join(r,f),a)) for r,_,fs in os.walk(a) for f in fs]; z.close()" + rem Use conda env's python (single-line, no backslashes) + C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import os,zipfile; a=r'%APPDIR%'; zf=r'%ZIPNAME%'; assert os.path.isdir(a), 'no APPDIR: '+a; z=zipfile.ZipFile(zf,'w',zipfile.ZIP_DEFLATED); [z.write(os.path.join(r,f), os.path.relpath(os.path.join(r,f),a)) for r,_,fs in os.walk(a) for f in fs]; z.close()" echo Created zip: %ZIPNAME% - name: Show zip contents @@ -108,7 +108,7 @@ jobs: run: | set "ZN=caiman-%VERSION%.zip" if not exist "%ZN%" ( echo ERROR: zip not found & exit /b 1 ) - ..\venv\Scripts\python -c "import os,zipfile; zn=r'%ZN%'; z=zipfile.ZipFile(zn,'r'); n=z.namelist(); print('Entries in',zn,':',len(n)); [print(' ',x) for x in n[:50]]; z.close()" + C:\Miniconda3\Scripts\conda.exe run -n caiman_release python -c "import os,zipfile; zn=r'%ZN%'; z=zipfile.ZipFile(zn,'r'); n=z.namelist(); print('Entries in',zn,':',len(n)); [print(' ',x) for x in n[:50]]; z.close()" - name: SHA256 of release zip shell: cmd From 6e6f86a4a6eb324e77624da449303c6b4581588f Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 25 Sep 2025 09:46:07 -0400 Subject: [PATCH 28/86] Define missing caiman data dir in spec file --- Deploy/pack_caiman_run.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Deploy/pack_caiman_run.spec b/Deploy/pack_caiman_run.spec index 6c4817a2..7e110cab 100644 --- a/Deploy/pack_caiman_run.spec +++ b/Deploy/pack_caiman_run.spec @@ -10,6 +10,8 @@ _specdir = os.path.abspath(os.path.dirname(SPEC)) distpath = os.path.join(_specdir, "dist") workpath = os.path.join(_specdir, "build") +CAIMAN_DATA_DIR = os.environ.get("CAIMAN_DATA_DIR") + BLOCK_CIPHER = None datas = [] From f9b7504bef8e245c2a2e03bd881d65480efac6f9 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 2 Oct 2025 11:07:39 -0400 Subject: [PATCH 29/86] Add constraints for deeplabcut installation. Fix git ignore --- .github/workflows/release-dlc.yml | 4 ++-- .gitignore | 3 ++- Deploy/constraints/deeplabcut-pins.txt | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 Deploy/constraints/deeplabcut-pins.txt diff --git a/.github/workflows/release-dlc.yml b/.github/workflows/release-dlc.yml index ab5d2146..fd7f46ed 100644 --- a/.github/workflows/release-dlc.yml +++ b/.github/workflows/release-dlc.yml @@ -37,8 +37,8 @@ jobs: shell: cmd run: | venv\Scripts\python -m pip install --upgrade pip - venv\Scripts\pip install h5py pyinstaller deeplabcut==3.0.0rc8 - venv\Scripts\pip install --upgrade --index-url https://download.pytorch.org/whl/cu126 torch + venv\Scripts\pip install --constraint Deploy\constraints\deeplabcut-pins.txt h5py pyinstaller deeplabcut==3.0.0rc8 + venv\Scripts\pip install --constraint Deploy\constraints\deeplabcut-pins.txt --upgrade --index-url https://download.pytorch.org/whl/cu126 torch torchvision venv\Scripts\python -c "import torch;print('Torch:',torch.__version__);print('CUDA build:',getattr(torch.version,'cuda',None));print('CUDA available:',torch.cuda.is_available())" - name: Clean old dist/build diff --git a/.gitignore b/.gitignore index dcdcbd5d..47463b4f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,8 @@ __pycache__/ .venv-*/ -Deploy/ +Deploy/build/ +Deploy/dist/ caiman_data/ CaImAn/caiman_correlation_pnr_functions/ diff --git a/Deploy/constraints/deeplabcut-pins.txt b/Deploy/constraints/deeplabcut-pins.txt new file mode 100644 index 00000000..8ac51b36 --- /dev/null +++ b/Deploy/constraints/deeplabcut-pins.txt @@ -0,0 +1,3 @@ +llvmlite==0.44.0 +numba==0.61.2 +numpy==1.26.4 From 9c92e1f1c1e49397546846e1360013b227145f35 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 7 Oct 2025 16:22:59 -0400 Subject: [PATCH 30/86] Fix error llvmpy for suite2p --- .github/workflows/release-dlc.yml | 4 ++-- .github/workflows/release-s2p.yml | 2 +- Deploy/constraints/{deeplabcut-pins.txt => pins.txt} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename Deploy/constraints/{deeplabcut-pins.txt => pins.txt} (100%) diff --git a/.github/workflows/release-dlc.yml b/.github/workflows/release-dlc.yml index fd7f46ed..3ca39694 100644 --- a/.github/workflows/release-dlc.yml +++ b/.github/workflows/release-dlc.yml @@ -37,8 +37,8 @@ jobs: shell: cmd run: | venv\Scripts\python -m pip install --upgrade pip - venv\Scripts\pip install --constraint Deploy\constraints\deeplabcut-pins.txt h5py pyinstaller deeplabcut==3.0.0rc8 - venv\Scripts\pip install --constraint Deploy\constraints\deeplabcut-pins.txt --upgrade --index-url https://download.pytorch.org/whl/cu126 torch torchvision + venv\Scripts\pip install --constraint Deploy\constraints\pins.txt h5py pyinstaller deeplabcut==3.0.0rc8 + venv\Scripts\pip install --constraint Deploy\constraints\pins.txt --upgrade --index-url https://download.pytorch.org/whl/cu126 torch torchvision venv\Scripts\python -c "import torch;print('Torch:',torch.__version__);print('CUDA build:',getattr(torch.version,'cuda',None));print('CUDA available:',torch.cuda.is_available())" - name: Clean old dist/build diff --git a/.github/workflows/release-s2p.yml b/.github/workflows/release-s2p.yml index 011bbae1..bb72440f 100644 --- a/.github/workflows/release-s2p.yml +++ b/.github/workflows/release-s2p.yml @@ -37,7 +37,7 @@ jobs: shell: cmd run: | venv\Scripts\python -m pip install --upgrade pip - venv\Scripts\pip install h5py pyinstaller suite2p + venv\Scripts\pip install --constraint Deploy\constraints\pins.txt h5py pyinstaller suite2p venv\Scripts\python -c "import suite2p; print('Suite2p version:', getattr(suite2p,'__version__','?'))" - name: Clean old dist/build diff --git a/Deploy/constraints/deeplabcut-pins.txt b/Deploy/constraints/pins.txt similarity index 100% rename from Deploy/constraints/deeplabcut-pins.txt rename to Deploy/constraints/pins.txt From 341c3604d446cf8fdfff2452c1e2859da3924f89 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 7 Oct 2025 16:50:43 -0400 Subject: [PATCH 31/86] Add collect all for h5py in minian --- Deploy/pack_minian_run.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Deploy/pack_minian_run.spec b/Deploy/pack_minian_run.spec index 87b00080..8cdb7716 100644 --- a/Deploy/pack_minian_run.spec +++ b/Deploy/pack_minian_run.spec @@ -26,6 +26,9 @@ datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('skimage') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('h5py') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] + binaries += collect_dynamic_libs('llvmlite',destdir='.\\Library\\bin') excludes = ["IPython", "PyQt5", "Markdown", "jupyter", "panel", "matplotlib", "notebook", "bokeh"] @@ -75,4 +78,4 @@ coll = COLLECT( upx=True, upx_exclude=[], name='minian', -) \ No newline at end of file +) From 42840e13d0cfa084a7b95335b6f87c7e6cb75f81 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 16 Oct 2025 12:28:16 -0400 Subject: [PATCH 32/86] Make sure to collect all dlls for h5py library in minian release --- .gitignore | 2 +- Deploy/pack_minian_run.spec | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 47463b4f..e8a57ffe 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ __pycache__/ .vscode .mypy_cache -.venv-*/ +.venv*/ Deploy/build/ Deploy/dist/ diff --git a/Deploy/pack_minian_run.spec b/Deploy/pack_minian_run.spec index 8cdb7716..fd5a5fe7 100644 --- a/Deploy/pack_minian_run.spec +++ b/Deploy/pack_minian_run.spec @@ -1,6 +1,7 @@ # -*- mode: python ; coding: utf-8 -*- import os +import sys from pathlib import Path from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT @@ -29,6 +30,26 @@ datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('h5py') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +binaries += collect_dynamic_libs('h5py', destdir='h5py') +_conda_prefix = Path(os.environ.get("CONDA_PREFIX", sys.prefix)).resolve() +_conda_bin_dir = _conda_prefix / "Library" / "bin" +if _conda_bin_dir.is_dir(): + _dll_patterns = ["hdf5*.dll", "szip.dll", "zlib.dll", "libaec*.dll"] + _existing = {Path(src).resolve(): dest for src, dest in binaries} + _added = 0 + for _pattern in _dll_patterns: + for _dll in _conda_bin_dir.glob(_pattern): + _dll = _dll.resolve() + if _dll in _existing: + continue + binaries.append((str(_dll), os.path.join('h5py', _dll.name))) + _existing[_dll] = os.path.join('h5py', _dll.name) + _added += 1 + if _added == 0: + print(f"[spec] WARNING: No extra HDF5 DLLs matched in {_conda_bin_dir}") +else: + print(f"[spec] WARNING: Could not locate conda Library/bin under {_conda_prefix}") + binaries += collect_dynamic_libs('llvmlite',destdir='.\\Library\\bin') excludes = ["IPython", "PyQt5", "Markdown", "jupyter", "panel", "matplotlib", "notebook", "bokeh"] From 3d99e026bdd420a7aefdac444c45fe43f3e93374 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 16 Oct 2025 14:15:24 -0400 Subject: [PATCH 33/86] Print dlls that were taken when building minian --- Deploy/pack_minian_run.spec | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Deploy/pack_minian_run.spec b/Deploy/pack_minian_run.spec index fd5a5fe7..7bdcf8b4 100644 --- a/Deploy/pack_minian_run.spec +++ b/Deploy/pack_minian_run.spec @@ -33,7 +33,9 @@ datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] binaries += collect_dynamic_libs('h5py', destdir='h5py') _conda_prefix = Path(os.environ.get("CONDA_PREFIX", sys.prefix)).resolve() _conda_bin_dir = _conda_prefix / "Library" / "bin" +print(f"[spec] HDF5 lookup using CONDA_PREFIX={_conda_prefix}") if _conda_bin_dir.is_dir(): + print(f"[spec] Scanning {_conda_bin_dir} for HDF5 runtime DLLs") _dll_patterns = ["hdf5*.dll", "szip.dll", "zlib.dll", "libaec*.dll"] _existing = {Path(src).resolve(): dest for src, dest in binaries} _added = 0 @@ -45,6 +47,7 @@ if _conda_bin_dir.is_dir(): binaries.append((str(_dll), os.path.join('h5py', _dll.name))) _existing[_dll] = os.path.join('h5py', _dll.name) _added += 1 + print(f"[spec] + bundled {_dll.name}") if _added == 0: print(f"[spec] WARNING: No extra HDF5 DLLs matched in {_conda_bin_dir}") else: From 03ad35edd4a17e95f4a9aca2a20ddb6a904e5583 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 16 Oct 2025 15:38:12 -0400 Subject: [PATCH 34/86] Include all dlls for h5py in minian release --- Deploy/pack_minian_run.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Deploy/pack_minian_run.spec b/Deploy/pack_minian_run.spec index 7bdcf8b4..3d9d0648 100644 --- a/Deploy/pack_minian_run.spec +++ b/Deploy/pack_minian_run.spec @@ -35,8 +35,8 @@ _conda_prefix = Path(os.environ.get("CONDA_PREFIX", sys.prefix)).resolve() _conda_bin_dir = _conda_prefix / "Library" / "bin" print(f"[spec] HDF5 lookup using CONDA_PREFIX={_conda_prefix}") if _conda_bin_dir.is_dir(): - print(f"[spec] Scanning {_conda_bin_dir} for HDF5 runtime DLLs") - _dll_patterns = ["hdf5*.dll", "szip.dll", "zlib.dll", "libaec*.dll"] + print(f"[spec] Scanning {_conda_bin_dir} for runtime DLLs") + _dll_patterns = ["*.dll"] _existing = {Path(src).resolve(): dest for src, dest in binaries} _added = 0 for _pattern in _dll_patterns: From 853b4b248f0aac7c1a42a932adaaeaed1a880e5a Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 16 Oct 2025 17:06:33 -0400 Subject: [PATCH 35/86] Collect all data from pyarrow --- Deploy/pack_deeplabcut_run.spec | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Deploy/pack_deeplabcut_run.spec b/Deploy/pack_deeplabcut_run.spec index b8d6616d..239bcf62 100644 --- a/Deploy/pack_deeplabcut_run.spec +++ b/Deploy/pack_deeplabcut_run.spec @@ -1,10 +1,8 @@ # -*- mode: python ; coding: utf-8 -*- import os -from pathlib import Path - from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT -from PyInstaller.utils.hooks import collect_all, copy_metadata +from PyInstaller.utils.hooks import collect_all _specdir = os.path.abspath(os.path.dirname(SPEC)) distpath = os.path.join(_specdir, "dist") @@ -17,13 +15,17 @@ binaries = [] hiddenimports = [] excludes = [] -datas += copy_metadata('deeplabcut', recursive=True) tmp_ret = collect_all('deeplabcut') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] -excludes = ["tensorflow","PySide6", "PyQT6", "PyQT5", "IPython", "Markdown", "jupyter", "napari", - "napari_deeplabcut", "napari_console", "npe2", "napari_plugin_engine", "napari_svg", "matplotlib" - ] +tmp_ret = collect_all('pyarrow') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +hiddenimports += ['pyarrow._generated_version'] + +excludes = [ + "tensorflow","PySide6", "PyQT6", "PyQT5", "IPython", "Markdown", "jupyter", "napari", + "napari_deeplabcut", "napari_console", "npe2", "napari_plugin_engine", "napari_svg", "matplotlib" + ] a_deeplabcut = Analysis( ['../DeepLabCut/deeplabcut_run.py'], From ecc67c993e7f905b27b8d2e310f57bf79dfa87da Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 21 Oct 2025 14:08:54 -0400 Subject: [PATCH 36/86] Collect data from skipped libraries --- Deploy/pack_deeplabcut_run.spec | 41 +++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/Deploy/pack_deeplabcut_run.spec b/Deploy/pack_deeplabcut_run.spec index 239bcf62..b6ab0601 100644 --- a/Deploy/pack_deeplabcut_run.spec +++ b/Deploy/pack_deeplabcut_run.spec @@ -1,7 +1,8 @@ # -*- mode: python ; coding: utf-8 -*- import os -from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT +import importlib.util +from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT, TOC from PyInstaller.utils.hooks import collect_all _specdir = os.path.abspath(os.path.dirname(SPEC)) @@ -10,21 +11,37 @@ workpath = os.path.join(_specdir, "build") BLOCK_CIPHER = None +def _safe_collect_all(package_name): + """ + Collect binaries/datas for optional dependencies without failing the build + when a dependency is absent from the environment. + """ + if importlib.util.find_spec(package_name) is None: + return [], [], [] + return collect_all(package_name) + +packages = [ + 'deeplabcut', + 'charset_normalizer', + 'dateutil', + 'numpy', + 'pandas', + 'safetensors', + 'scipy', + 'shapely', + 'tables', + 'tkinter', +] + datas = [] binaries = [] hiddenimports = [] -excludes = [] - -tmp_ret = collect_all('deeplabcut') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - -tmp_ret = collect_all('pyarrow') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] -hiddenimports += ['pyarrow._generated_version'] +for package in packages: + tmp_ret = _safe_collect_all(package) + datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] excludes = [ - "tensorflow","PySide6", "PyQT6", "PyQT5", "IPython", "Markdown", "jupyter", "napari", - "napari_deeplabcut", "napari_console", "npe2", "napari_plugin_engine", "napari_svg", "matplotlib" + "matplotlib" ] a_deeplabcut = Analysis( @@ -61,7 +78,7 @@ exclude_binaries = [ 'cudnn_heuristic64_9.dll', 'cudnn_ops64_9.dll', 'cudnn64_9.dll' - ] +] a_deeplabcut.binaries= TOC([x for x in a_deeplabcut.binaries if not any(exclude in x[0] for exclude in exclude_binaries)]) From d89eb2c78d00810321d8aa75726c3668b5bd73b7 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 22 Oct 2025 14:02:36 -0400 Subject: [PATCH 37/86] Add torch, torchvision and pyarrow --- Deploy/pack_deeplabcut_run.spec | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Deploy/pack_deeplabcut_run.spec b/Deploy/pack_deeplabcut_run.spec index b6ab0601..214a159f 100644 --- a/Deploy/pack_deeplabcut_run.spec +++ b/Deploy/pack_deeplabcut_run.spec @@ -20,26 +20,36 @@ def _safe_collect_all(package_name): return [], [], [] return collect_all(package_name) -packages = [ - 'deeplabcut', +required_packages = ['deeplabcut'] +optional_packages = [ 'charset_normalizer', 'dateutil', 'numpy', 'pandas', + 'pyarrow' 'safetensors', 'scipy', 'shapely', 'tables', 'tkinter', + 'torch', + 'torchvision', ] -datas = [] -binaries = [] -hiddenimports = [] -for package in packages: +datas = [] +binaries = [] +hiddenimports = [] + +for package in required_packages: + tmp_ret = collect_all(package) + datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] + +for package in optional_packages: tmp_ret = _safe_collect_all(package) datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +hiddenimports += ['pyarrow._generated_version'] + excludes = [ "matplotlib" ] From ee3ad6686fc031c5616cc83a9902e2d838fec993 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 22 Oct 2025 16:07:49 -0400 Subject: [PATCH 38/86] Remove unnecessary import in suite2p pack --- Deploy/pack_suite2p_run.spec | 2 -- 1 file changed, 2 deletions(-) diff --git a/Deploy/pack_suite2p_run.spec b/Deploy/pack_suite2p_run.spec index 5e24734f..7c8dc157 100644 --- a/Deploy/pack_suite2p_run.spec +++ b/Deploy/pack_suite2p_run.spec @@ -1,8 +1,6 @@ # -*- mode: python ; coding: utf-8 -*- import os -from pathlib import Path - from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT from PyInstaller.utils.hooks import collect_all From b19f1a306692072e1db335e6d3449e4625cf735a Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 22 Oct 2025 16:08:13 -0400 Subject: [PATCH 39/86] Add hook for torch --- Deploy/constraints/torch_openmp_env.py | 7 +++++++ Deploy/pack_deeplabcut_run.spec | 13 ++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 Deploy/constraints/torch_openmp_env.py diff --git a/Deploy/constraints/torch_openmp_env.py b/Deploy/constraints/torch_openmp_env.py new file mode 100644 index 00000000..91868e95 --- /dev/null +++ b/Deploy/constraints/torch_openmp_env.py @@ -0,0 +1,7 @@ +"""Runtime hook to prevent Intel OpenMP initialization conflicts in frozen builds.""" + +import os + +# Allow duplicate Intel OpenMP runtimes that ship with torch/numpy/scipy +# to coexist inside the bundled application. +os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE") diff --git a/Deploy/pack_deeplabcut_run.spec b/Deploy/pack_deeplabcut_run.spec index 214a159f..3cf4ac2b 100644 --- a/Deploy/pack_deeplabcut_run.spec +++ b/Deploy/pack_deeplabcut_run.spec @@ -26,14 +26,12 @@ optional_packages = [ 'dateutil', 'numpy', 'pandas', - 'pyarrow' 'safetensors', 'scipy', 'shapely', 'tables', 'tkinter', - 'torch', - 'torchvision', + 'pyarrow', ] datas = [] @@ -48,7 +46,8 @@ for package in optional_packages: tmp_ret = _safe_collect_all(package) datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] -hiddenimports += ['pyarrow._generated_version'] +if importlib.util.find_spec('pyarrow') is not None: + hiddenimports += ['pyarrow._generated_version'] excludes = [ "matplotlib" @@ -62,7 +61,7 @@ a_deeplabcut = Analysis( hiddenimports=hiddenimports, hookspath=[], hooksconfig={}, - runtime_hooks=[], + runtime_hooks=['constraints/torch_openmp_env.py'], excludes=excludes, win_no_prefer_redirects=False, win_private_assemblies=False, @@ -102,7 +101,7 @@ exe_deeplabcut = EXE( debug=False, bootloader_ignore_signals=False, strip=False, - upx=True, + upx=False, upx_exclude=[], runtime_tmpdir=None, console=True, @@ -119,7 +118,7 @@ coll = COLLECT( a_deeplabcut.zipfiles, a_deeplabcut.datas, strip=False, - upx=True, + upx=False, upx_exclude=[], name='deeplabcut', ) From b3f09106d367a7b9b6e642c7e68eb165b572d871 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 22 Oct 2025 16:08:49 -0400 Subject: [PATCH 40/86] Add early fail if torch is installed incorrectly --- .github/workflows/release-dlc.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release-dlc.yml b/.github/workflows/release-dlc.yml index 3ca39694..17e1d6d5 100644 --- a/.github/workflows/release-dlc.yml +++ b/.github/workflows/release-dlc.yml @@ -56,6 +56,11 @@ jobs: echo === DIST TREE === dir /s /b dist + - name: Smoke test torch/deeplabcut import + shell: cmd + run: | + venv\Scripts\python -c "import torch; import deeplabcut; print('torch', torch.__version__)" + - name: Zip onedir output with build-info shell: cmd working-directory: Deploy From 1cabbe00df4dc6027980b2dffc31f33c41fa455f Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 22 Oct 2025 16:41:56 -0400 Subject: [PATCH 41/86] Use 2.8 torch version --- .github/workflows/release-dlc.yml | 2 +- Deploy/pack_deeplabcut_run.spec | 20 ++++++++------------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release-dlc.yml b/.github/workflows/release-dlc.yml index 17e1d6d5..186eb66e 100644 --- a/.github/workflows/release-dlc.yml +++ b/.github/workflows/release-dlc.yml @@ -38,7 +38,7 @@ jobs: run: | venv\Scripts\python -m pip install --upgrade pip venv\Scripts\pip install --constraint Deploy\constraints\pins.txt h5py pyinstaller deeplabcut==3.0.0rc8 - venv\Scripts\pip install --constraint Deploy\constraints\pins.txt --upgrade --index-url https://download.pytorch.org/whl/cu126 torch torchvision + venv\Scripts\pip install --constraint Deploy\constraints\pins.txt --upgrade --index-url https://download.pytorch.org/whl/cu126 torch==2.8.0+cu126 torchvision==0.23.0+cu126 venv\Scripts\python -c "import torch;print('Torch:',torch.__version__);print('CUDA build:',getattr(torch.version,'cuda',None));print('CUDA available:',torch.cuda.is_available())" - name: Clean old dist/build diff --git a/Deploy/pack_deeplabcut_run.spec b/Deploy/pack_deeplabcut_run.spec index 3cf4ac2b..f126c64a 100644 --- a/Deploy/pack_deeplabcut_run.spec +++ b/Deploy/pack_deeplabcut_run.spec @@ -11,16 +11,11 @@ workpath = os.path.join(_specdir, "build") BLOCK_CIPHER = None -def _safe_collect_all(package_name): - """ - Collect binaries/datas for optional dependencies without failing the build - when a dependency is absent from the environment. - """ - if importlib.util.find_spec(package_name) is None: - return [], [], [] - return collect_all(package_name) - -required_packages = ['deeplabcut'] +required_packages = [ + 'deeplabcut', + 'torch', + 'torchvision' +] optional_packages = [ 'charset_normalizer', 'dateutil', @@ -43,8 +38,9 @@ for package in required_packages: datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] for package in optional_packages: - tmp_ret = _safe_collect_all(package) - datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] + if importlib.util.find_spec(package) is not None: + tmp_ret = collect_all(package) + datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] if importlib.util.find_spec('pyarrow') is not None: hiddenimports += ['pyarrow._generated_version'] From 8a660f936ec51c3660c8101c05d6f2d108b5e781 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 22 Oct 2025 16:56:37 -0400 Subject: [PATCH 42/86] Rearrange package collection for suite2p --- Deploy/pack_suite2p_run.spec | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/Deploy/pack_suite2p_run.spec b/Deploy/pack_suite2p_run.spec index 7c8dc157..8b7f8fea 100644 --- a/Deploy/pack_suite2p_run.spec +++ b/Deploy/pack_suite2p_run.spec @@ -10,21 +10,25 @@ workpath = os.path.join(_specdir, "build") BLOCK_CIPHER = None +packages = [ + 'suite2p', + 'ScanImageTiffReader', +] + +excludes = [ + "IPython", + "PyQt6", + "PyQt5", + "Markdown", + "jupyter" +] + datas = [] binaries = [] hiddenimports = [] -excludes = [] - -# datas += copy_metadata('suite2p', recursive=True) -tmp_ret = collect_all('suite2p') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - -tmp_ret = collect_all('ScanImageTiffReader') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - -# binaries += collect_dynamic_libs('ScanImageTiffReaderAPI', destdir='.\\Library\\bin') - -excludes = ["IPython", "PyQt6", "PyQt5", "Markdown", "jupyter"] +for package in packages: + tmp_ret = collect_all(package) + datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] a_suite2p = Analysis( ['../Suite2p/suite2p_run.py'], From e66091d1b0bf0c673899fb38640fd7d2a58a9e8e Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 22 Oct 2025 16:59:39 -0400 Subject: [PATCH 43/86] Edit package collection in minian --- Deploy/pack_minian_run.spec | 38 ++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/Deploy/pack_minian_run.spec b/Deploy/pack_minian_run.spec index 3d9d0648..073e7893 100644 --- a/Deploy/pack_minian_run.spec +++ b/Deploy/pack_minian_run.spec @@ -13,22 +13,30 @@ workpath = os.path.join(_specdir, "build") BLOCK_CIPHER = None +packages = [ + 'minian', + 'distributed', + 'skimage', + 'h5py' +] + +excludes = [ + "IPython", + "PyQt5", + "Markdown", + "jupyter", + "panel", + "matplotlib", + "notebook", + "bokeh" +] + datas = [] binaries = [] hiddenimports = [] -excludes = [] - -tmp_ret = collect_all('minian') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - -tmp_ret = collect_all('distributed') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - -tmp_ret = collect_all('skimage') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - -tmp_ret = collect_all('h5py') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +for package in packages: + tmp_ret = collect_all(package) + datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] binaries += collect_dynamic_libs('h5py', destdir='h5py') _conda_prefix = Path(os.environ.get("CONDA_PREFIX", sys.prefix)).resolve() @@ -53,10 +61,6 @@ if _conda_bin_dir.is_dir(): else: print(f"[spec] WARNING: Could not locate conda Library/bin under {_conda_prefix}") -binaries += collect_dynamic_libs('llvmlite',destdir='.\\Library\\bin') - -excludes = ["IPython", "PyQt5", "Markdown", "jupyter", "panel", "matplotlib", "notebook", "bokeh"] - a_minian = Analysis( ['../MiniAn/minian_run.py'], pathex=['../'], From a9d9423c8c419c97902325755998a9aa744922e3 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 22 Oct 2025 17:03:17 -0400 Subject: [PATCH 44/86] Edit package collection for caiman --- Deploy/pack_caiman_run.spec | 55 +++++++++++++++++++------------------ Deploy/pack_minian_run.spec | 2 ++ 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/Deploy/pack_caiman_run.spec b/Deploy/pack_caiman_run.spec index 7e110cab..62ddc2de 100644 --- a/Deploy/pack_caiman_run.spec +++ b/Deploy/pack_caiman_run.spec @@ -14,34 +14,38 @@ CAIMAN_DATA_DIR = os.environ.get("CAIMAN_DATA_DIR") BLOCK_CIPHER = None +packages = [ + 'caiman', + 'hdmf', + 'pynwb', + 'param', + 'skimage', + 'scipy' +] + +excludes = [ + "PyQt5", + "Markdown", + "jupyter", + "panel", + "matplotlib", + "bokeh", + "IPython", + "ipyparallel", + "ipywidgets", + "tensorflow", + "pyqtgraph", + "torch", + "torchvision", + "scipy._lib.array_api_compat.torch" +] + datas = [] binaries = [] hiddenimports = [] -excludes = [] - -tmp_ret = collect_all('caiman') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - -tmp_ret = collect_all('hdmf') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - -tmp_ret = collect_all('pynwb') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - -datas += copy_metadata('param', recursive=True) -tmp_ret = collect_all('param') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - -tmp_ret = collect_all('skimage') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - -tmp_ret = collect_all('scipy') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - -# datas += [( '../CaImAn/caiman_data/model', 'caiman_data/model')] - -excludes = ["PyQt5", "Markdown", "jupyter", "panel", "matplotlib", "bokeh", "IPython", "ipyparallel", "ipywidgets", "tensorflow", "pyqtgraph", - "torch", "torchvision", "scipy._lib.array_api_compat.torch"] +for package in packages: + tmp_ret = collect_all(package) + datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] a_caimAn = Analysis( ['../CaImAn/caiman_run.py'], @@ -86,7 +90,6 @@ if CAIMAN_DATA_DIR and os.path.isdir(CAIMAN_DATA_DIR): else: print("[spec] WARNING: CAIMAN_DATA_DIR not set or not a directory; models/configs not bundled.") - coll = COLLECT( exe_caimAn, a_caimAn.binaries, diff --git a/Deploy/pack_minian_run.spec b/Deploy/pack_minian_run.spec index 073e7893..040c1010 100644 --- a/Deploy/pack_minian_run.spec +++ b/Deploy/pack_minian_run.spec @@ -38,6 +38,8 @@ for package in packages: tmp_ret = collect_all(package) datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +binaries += collect_dynamic_libs('llvmlite',destdir='.\\Library\\bin') + binaries += collect_dynamic_libs('h5py', destdir='h5py') _conda_prefix = Path(os.environ.get("CONDA_PREFIX", sys.prefix)).resolve() _conda_bin_dir = _conda_prefix / "Library" / "bin" From 2382a9f2b8ed81b78666b777083e424800dc1fa7 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 22 Oct 2025 17:15:44 -0400 Subject: [PATCH 45/86] Minor edits in spec files --- Deploy/pack_caiman_run.spec | 8 ++++++-- Deploy/pack_deeplabcut_run.spec | 12 +++++++++--- Deploy/pack_minian_run.spec | 7 ++++++- Deploy/pack_suite2p_run.spec | 8 ++++++-- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/Deploy/pack_caiman_run.spec b/Deploy/pack_caiman_run.spec index 62ddc2de..8f82920f 100644 --- a/Deploy/pack_caiman_run.spec +++ b/Deploy/pack_caiman_run.spec @@ -3,8 +3,8 @@ import os from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT -from PyInstaller.utils.hooks import collect_all, copy_metadata from PyInstaller.building.datastruct import Tree +from PyInstaller.utils.hooks import collect_all _specdir = os.path.abspath(os.path.dirname(SPEC)) distpath = os.path.join(_specdir, "dist") @@ -63,7 +63,11 @@ a_caimAn = Analysis( noarchive=False, ) -pyz_caimAn = PYZ(a_caimAn.pure, a_caimAn.zipped_data, cipher=BLOCK_CIPHER) +pyz_caimAn = PYZ( + a_caimAn.pure, + a_caimAn.zipped_data, + cipher=BLOCK_CIPHER +) exe_caimAn = EXE( pyz_caimAn, diff --git a/Deploy/pack_deeplabcut_run.spec b/Deploy/pack_deeplabcut_run.spec index f126c64a..94310c06 100644 --- a/Deploy/pack_deeplabcut_run.spec +++ b/Deploy/pack_deeplabcut_run.spec @@ -85,9 +85,15 @@ exclude_binaries = [ 'cudnn64_9.dll' ] -a_deeplabcut.binaries= TOC([x for x in a_deeplabcut.binaries if not any(exclude in x[0] for exclude in exclude_binaries)]) - -pyz_deeplabcut = PYZ(a_deeplabcut.pure, a_deeplabcut.zipped_data, cipher=BLOCK_CIPHER) +a_deeplabcut.binaries= TOC([ + x for x in a_deeplabcut.binaries if not any(exclude in x[0] for exclude in exclude_binaries) +]) + +pyz_deeplabcut = PYZ( + a_deeplabcut.pure, + a_deeplabcut.zipped_data, + cipher=BLOCK_CIPHER +) exe_deeplabcut = EXE( pyz_deeplabcut, diff --git a/Deploy/pack_minian_run.spec b/Deploy/pack_minian_run.spec index 040c1010..030f4a09 100644 --- a/Deploy/pack_minian_run.spec +++ b/Deploy/pack_minian_run.spec @@ -78,7 +78,12 @@ a_minian = Analysis( cipher=BLOCK_CIPHER, noarchive=False, ) -pyz_minian = PYZ(a_minian.pure, a_minian.zipped_data, cipher=BLOCK_CIPHER) + +pyz_minian = PYZ( + a_minian.pure, + a_minian.zipped_data, + cipher=BLOCK_CIPHER +) exe_minian = EXE( pyz_minian, diff --git a/Deploy/pack_suite2p_run.spec b/Deploy/pack_suite2p_run.spec index 8b7f8fea..6e13c5db 100644 --- a/Deploy/pack_suite2p_run.spec +++ b/Deploy/pack_suite2p_run.spec @@ -45,7 +45,11 @@ a_suite2p = Analysis( cipher=BLOCK_CIPHER, noarchive=False, ) -pyz_suite2p = PYZ(a_suite2p.pure, a_suite2p.zipped_data, cipher=BLOCK_CIPHER) +pyz_suite2p = PYZ( + a_suite2p.pure, + a_suite2p.zipped_data, + cipher=BLOCK_CIPHER +) exe_suite2p = EXE( pyz_suite2p, @@ -75,4 +79,4 @@ coll = COLLECT( upx=True, upx_exclude=[], name='suite2p', -) \ No newline at end of file +) From 426f362eabaea959ad5da5f5f950d79064d257bd Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 22 Oct 2025 17:20:17 -0400 Subject: [PATCH 46/86] Remove heavy libraries from the hook that are already collected by the pyinstaller --- Deploy/pack_deeplabcut_run.spec | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Deploy/pack_deeplabcut_run.spec b/Deploy/pack_deeplabcut_run.spec index 94310c06..d67ad5ad 100644 --- a/Deploy/pack_deeplabcut_run.spec +++ b/Deploy/pack_deeplabcut_run.spec @@ -12,17 +12,12 @@ workpath = os.path.join(_specdir, "build") BLOCK_CIPHER = None required_packages = [ - 'deeplabcut', - 'torch', - 'torchvision' + 'deeplabcut' ] optional_packages = [ 'charset_normalizer', 'dateutil', - 'numpy', - 'pandas', 'safetensors', - 'scipy', 'shapely', 'tables', 'tkinter', From 70ac7155938072eb1687600b51e1917e3bc8b352 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 22 Oct 2025 17:27:23 -0400 Subject: [PATCH 47/86] Space edits in spec files --- Deploy/pack_caiman_run.spec | 4 ++-- Deploy/pack_deeplabcut_run.spec | 6 +----- Deploy/pack_suite2p_run.spec | 4 ++-- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Deploy/pack_caiman_run.spec b/Deploy/pack_caiman_run.spec index 8f82920f..0083e417 100644 --- a/Deploy/pack_caiman_run.spec +++ b/Deploy/pack_caiman_run.spec @@ -64,8 +64,8 @@ a_caimAn = Analysis( ) pyz_caimAn = PYZ( - a_caimAn.pure, - a_caimAn.zipped_data, + a_caimAn.pure, + a_caimAn.zipped_data, cipher=BLOCK_CIPHER ) diff --git a/Deploy/pack_deeplabcut_run.spec b/Deploy/pack_deeplabcut_run.spec index d67ad5ad..2e1e7b2c 100644 --- a/Deploy/pack_deeplabcut_run.spec +++ b/Deploy/pack_deeplabcut_run.spec @@ -27,7 +27,6 @@ optional_packages = [ datas = [] binaries = [] hiddenimports = [] - for package in required_packages: tmp_ret = collect_all(package) datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] @@ -42,7 +41,7 @@ if importlib.util.find_spec('pyarrow') is not None: excludes = [ "matplotlib" - ] +] a_deeplabcut = Analysis( ['../DeepLabCut/deeplabcut_run.py'], @@ -119,6 +118,3 @@ coll = COLLECT( upx_exclude=[], name='deeplabcut', ) - - - diff --git a/Deploy/pack_suite2p_run.spec b/Deploy/pack_suite2p_run.spec index 6e13c5db..a1069466 100644 --- a/Deploy/pack_suite2p_run.spec +++ b/Deploy/pack_suite2p_run.spec @@ -46,8 +46,8 @@ a_suite2p = Analysis( noarchive=False, ) pyz_suite2p = PYZ( - a_suite2p.pure, - a_suite2p.zipped_data, + a_suite2p.pure, + a_suite2p.zipped_data, cipher=BLOCK_CIPHER ) From 8fe14561303d87a5463cfd850d637774b33804be Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 22 Oct 2025 17:29:29 -0400 Subject: [PATCH 48/86] Rename spec files --- .github/workflows/release-cmn.yml | 2 +- .github/workflows/release-dlc.yml | 2 +- .github/workflows/release-mnn.yml | 2 +- .github/workflows/release-s2p.yml | 2 +- Deploy/{pack_caiman_run.spec => pack_caiman.spec} | 0 Deploy/{pack_deeplabcut_run.spec => pack_deeplabcut.spec} | 0 Deploy/{pack_minian_run.spec => pack_minian.spec} | 0 Deploy/{pack_suite2p_run.spec => pack_suite2p.spec} | 0 8 files changed, 4 insertions(+), 4 deletions(-) rename Deploy/{pack_caiman_run.spec => pack_caiman.spec} (100%) rename Deploy/{pack_deeplabcut_run.spec => pack_deeplabcut.spec} (100%) rename Deploy/{pack_minian_run.spec => pack_minian.spec} (100%) rename Deploy/{pack_suite2p_run.spec => pack_suite2p.spec} (100%) diff --git a/.github/workflows/release-cmn.yml b/.github/workflows/release-cmn.yml index 2c8a2019..bfeeba2f 100644 --- a/.github/workflows/release-cmn.yml +++ b/.github/workflows/release-cmn.yml @@ -62,7 +62,7 @@ jobs: cd dir dir Deploy - dir Deploy\pack_caiman_run.spec + dir Deploy\pack_caiman.spec - name: Clean old dist/build shell: cmd diff --git a/.github/workflows/release-dlc.yml b/.github/workflows/release-dlc.yml index 186eb66e..c83862c8 100644 --- a/.github/workflows/release-dlc.yml +++ b/.github/workflows/release-dlc.yml @@ -52,7 +52,7 @@ jobs: shell: cmd working-directory: Deploy run: | - ..\venv\Scripts\pyinstaller --clean --noconfirm pack_deeplabcut_run.spec + ..\venv\Scripts\pyinstaller --clean --noconfirm pack_deeplabcut.spec echo === DIST TREE === dir /s /b dist diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml index 89451b7f..15c27940 100644 --- a/.github/workflows/release-mnn.yml +++ b/.github/workflows/release-mnn.yml @@ -54,7 +54,7 @@ jobs: shell: cmd working-directory: Deploy run: | - C:\Miniconda3\Scripts\conda.exe run -n minian_release pyinstaller --clean --noconfirm pack_minian_run.spec + C:\Miniconda3\Scripts\conda.exe run -n minian_release pyinstaller --clean --noconfirm pack_minian.spec echo === DIST TREE === dir /s /b dist diff --git a/.github/workflows/release-s2p.yml b/.github/workflows/release-s2p.yml index bb72440f..50c29391 100644 --- a/.github/workflows/release-s2p.yml +++ b/.github/workflows/release-s2p.yml @@ -51,7 +51,7 @@ jobs: shell: cmd working-directory: Deploy run: | - ..\venv\Scripts\pyinstaller --clean --noconfirm pack_suite2p_run.spec + ..\venv\Scripts\pyinstaller --clean --noconfirm pack_suite2p.spec echo === DIST TREE === dir /s /b dist diff --git a/Deploy/pack_caiman_run.spec b/Deploy/pack_caiman.spec similarity index 100% rename from Deploy/pack_caiman_run.spec rename to Deploy/pack_caiman.spec diff --git a/Deploy/pack_deeplabcut_run.spec b/Deploy/pack_deeplabcut.spec similarity index 100% rename from Deploy/pack_deeplabcut_run.spec rename to Deploy/pack_deeplabcut.spec diff --git a/Deploy/pack_minian_run.spec b/Deploy/pack_minian.spec similarity index 100% rename from Deploy/pack_minian_run.spec rename to Deploy/pack_minian.spec diff --git a/Deploy/pack_suite2p_run.spec b/Deploy/pack_suite2p.spec similarity index 100% rename from Deploy/pack_suite2p_run.spec rename to Deploy/pack_suite2p.spec From b0d5319b91655c8c9b496fed405379414b0828b9 Mon Sep 17 00:00:00 2001 From: maximeB Date: Thu, 23 Oct 2025 09:50:24 -0400 Subject: [PATCH 49/86] add sss for test --- CaImAn/caiman_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CaImAn/caiman_main.py b/CaImAn/caiman_main.py index 48bc7498..1ee25006 100644 --- a/CaImAn/caiman_main.py +++ b/CaImAn/caiman_main.py @@ -15,7 +15,7 @@ import caiman_parameters as cm_params from caiman.base.rois import register_multisession -# Import for CaimAn lib +# Import for CaimAn libsss import caiman as cm # Import for PyInstaller From d350fc7d2830eb98a588e230c591692528ba12fd Mon Sep 17 00:00:00 2001 From: maximeB Date: Thu, 23 Oct 2025 09:52:14 -0400 Subject: [PATCH 50/86] remove test commit --- CaImAn/caiman_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CaImAn/caiman_main.py b/CaImAn/caiman_main.py index 1ee25006..48bc7498 100644 --- a/CaImAn/caiman_main.py +++ b/CaImAn/caiman_main.py @@ -15,7 +15,7 @@ import caiman_parameters as cm_params from caiman.base.rois import register_multisession -# Import for CaimAn libsss +# Import for CaimAn lib import caiman as cm # Import for PyInstaller From a846de216ea83f56c9196644c5e4a15d84c55005 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 23 Oct 2025 16:53:49 -0400 Subject: [PATCH 51/86] Remove extra hooks for h5py for minian --- Deploy/pack_minian.spec | 46 +++++++++-------------------------------- 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/Deploy/pack_minian.spec b/Deploy/pack_minian.spec index 030f4a09..79175426 100644 --- a/Deploy/pack_minian.spec +++ b/Deploy/pack_minian.spec @@ -1,8 +1,6 @@ # -*- mode: python ; coding: utf-8 -*- import os -import sys -from pathlib import Path from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT from PyInstaller.utils.hooks import collect_all, collect_dynamic_libs @@ -17,18 +15,17 @@ packages = [ 'minian', 'distributed', 'skimage', - 'h5py' ] excludes = [ - "IPython", - "PyQt5", + "bokeh", + "IPython", + "jupyter", "Markdown", - "jupyter", - "panel", - "matplotlib", - "notebook", - "bokeh" + "matplotlib", + "notebook", + "panel", + "PyQt5", ] datas = [] @@ -38,30 +35,7 @@ for package in packages: tmp_ret = collect_all(package) datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] -binaries += collect_dynamic_libs('llvmlite',destdir='.\\Library\\bin') - -binaries += collect_dynamic_libs('h5py', destdir='h5py') -_conda_prefix = Path(os.environ.get("CONDA_PREFIX", sys.prefix)).resolve() -_conda_bin_dir = _conda_prefix / "Library" / "bin" -print(f"[spec] HDF5 lookup using CONDA_PREFIX={_conda_prefix}") -if _conda_bin_dir.is_dir(): - print(f"[spec] Scanning {_conda_bin_dir} for runtime DLLs") - _dll_patterns = ["*.dll"] - _existing = {Path(src).resolve(): dest for src, dest in binaries} - _added = 0 - for _pattern in _dll_patterns: - for _dll in _conda_bin_dir.glob(_pattern): - _dll = _dll.resolve() - if _dll in _existing: - continue - binaries.append((str(_dll), os.path.join('h5py', _dll.name))) - _existing[_dll] = os.path.join('h5py', _dll.name) - _added += 1 - print(f"[spec] + bundled {_dll.name}") - if _added == 0: - print(f"[spec] WARNING: No extra HDF5 DLLs matched in {_conda_bin_dir}") -else: - print(f"[spec] WARNING: Could not locate conda Library/bin under {_conda_prefix}") +binaries += collect_dynamic_libs('llvmlite', destdir='.\\Library\\bin') a_minian = Analysis( ['../MiniAn/minian_run.py'], @@ -80,8 +54,8 @@ a_minian = Analysis( ) pyz_minian = PYZ( - a_minian.pure, - a_minian.zipped_data, + a_minian.pure, + a_minian.zipped_data, cipher=BLOCK_CIPHER ) From 95ef4d9240a08e82b2f27ccd86c47a8d4394a7eb Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 28 Oct 2025 12:13:29 -0400 Subject: [PATCH 52/86] Try to collect more binaries for minian --- Deploy/pack_minian.spec | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Deploy/pack_minian.spec b/Deploy/pack_minian.spec index 79175426..34eb1fdf 100644 --- a/Deploy/pack_minian.spec +++ b/Deploy/pack_minian.spec @@ -15,6 +15,7 @@ packages = [ 'minian', 'distributed', 'skimage', + 'scipy', ] excludes = [ @@ -35,7 +36,17 @@ for package in packages: tmp_ret = collect_all(package) datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] -binaries += collect_dynamic_libs('llvmlite', destdir='.\\Library\\bin') +for dyn_pkg, dest in [ + ('llvmlite', os.path.join('Library', 'bin')), + ('scipy', os.path.join('scipy', '.libs')), + ('h5py', '.'), + ('SimpleITK', '.'), + ('cv2', '.'), +]: + try: + binaries += collect_dynamic_libs(dyn_pkg, destdir=dest) + except ImportError: + pass a_minian = Analysis( ['../MiniAn/minian_run.py'], From ee0dc4bd5757ca2fd81befe6c5bb3412b709d71b Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 28 Oct 2025 14:21:20 -0400 Subject: [PATCH 53/86] Capture the environment during github release for minian --- .github/workflows/release-mnn.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml index 15c27940..3fbe969b 100644 --- a/.github/workflows/release-mnn.yml +++ b/.github/workflows/release-mnn.yml @@ -43,6 +43,18 @@ jobs: C:\Miniconda3\Scripts\conda.exe install -n minian_release -c conda-forge h5py pyinstaller minian -y C:\Miniconda3\Scripts\conda.exe run -n minian_release python -c "import minian; print('MiniAn version:', getattr(minian,'__version__','?'))" + - name: Snapshot conda packages + shell: cmd + run: | + C:\Miniconda3\Scripts\conda.exe list --explicit -n minian_release > minian-conda-explicit.txt + type minian-conda-explicit.txt + + - name: Upload conda manifest + uses: actions/upload-artifact@v4 + with: + name: minian-conda-explicit-${{ env.VERSION }} + path: minian-conda-explicit.txt + - name: Clean old dist/build shell: cmd working-directory: Deploy From d0dce714cbfcdfba4d4d249f053ade091afa19f3 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 28 Oct 2025 16:04:06 -0400 Subject: [PATCH 54/86] Try to reproduce the same environment as local for minian release --- .github/workflows/release-mnn.yml | 10 +++++++--- environment-minian.txt | Bin 0 -> 47558 bytes 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 environment-minian.txt diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml index 3fbe969b..216de2dd 100644 --- a/.github/workflows/release-mnn.yml +++ b/.github/workflows/release-mnn.yml @@ -35,12 +35,16 @@ jobs: shell: cmd run: | C:\Miniconda3\Scripts\conda.exe env remove -n minian_release -y || echo "no old env" - C:\Miniconda3\Scripts\conda.exe create -n minian_release python=3.8 -y + set LOCK_FILE=%GITHUB_WORKSPACE%\environment-minian.txt + if not exist "%LOCK_FILE%" ( + echo ERROR: environment-minian.txt not found at %LOCK_FILE% + exit /b 1 + ) + C:\Miniconda3\Scripts\conda.exe create -n minian_release --file "%LOCK_FILE%" -y - - name: Install minian deps with conda-forge + - name: Verify minian import shell: cmd run: | - C:\Miniconda3\Scripts\conda.exe install -n minian_release -c conda-forge h5py pyinstaller minian -y C:\Miniconda3\Scripts\conda.exe run -n minian_release python -c "import minian; print('MiniAn version:', getattr(minian,'__version__','?'))" - name: Snapshot conda packages diff --git a/environment-minian.txt b/environment-minian.txt new file mode 100644 index 0000000000000000000000000000000000000000..1b193bdf4733a827705ca557a9bbbe29555dd538 GIT binary patch literal 47558 zcmd6wNlzrp6@_zc$^YS5_Lgqu;pyqglI4XZFTC)=J5*E^4c$zSf@b*fw$B%bOJrmf zgI<%7DO6At1)RLYxkKED{NI27-2Bx1yE$o2n}wbo>+4xF*R%6x(md$vQTpyyKbhrX@*7p;A$9pHug+9;q`MZAqTJN~f@ALhQUgvjD_54fobMwdM4-NnEFM3SV z`)2y>?%!K$*7W{K`WrtrzdZfrU-dqIk}>|Gzu!gv>+}D((v^VlSpVl*f9q%c{#`<{ z*6cRh&+ltr%S?a&NY~AK+xRw{o#s=sr^kBwy}vepZ~oK#tNCa1kLGWB{5!pRqW^Z& zTLi7<1|Tu*U-R1 zKiL#AqukK5Pr|#V-`m!2>`8A2>5322PcjCy(I;lE1{+w@13jz>$-Mb0)Zb+lYr>6G zK!2Xzd8ucc!oDTUpr1$=o4SU%aIZH;f1_8g^`4{VQ9rAH1z55rxDaMr7@FBiwgFC% z@9AT%k6F5waUcipZL+7iFmerOV@>vo$6E61!n;4(E4Y=`Z<9q~Jy`j^@UD_K+UV2H z+dT4F*1HgAEM$+C>xCqXP0S_ROW`=tbNq8dUbrrM-bgau5XQc(xx{&u{B4#nS>GFC zhEECii5O!~-)#!pk)(R4XZ^~sGi+|~n+;A`KuRq8Le9yZSsXh$?8X_S28yZ%sNsm+p!GpPsdd zoeziTOxAbwWP51xL{>*;MJDn^Kf?maJJ z-VlmIp*16)O0q-~Ud0Y=i93@0dh%d$5!i*uxToibdMDSDZKYN$?^@_QVK9S}O`WH! z7~WE+!nLLh1N#WfXwUZGd}B?re2?{hoYq6bu*`|{IgO2B>txua)X4F^OI|QF zQUjm0Ih9ZFIFig@y1sd?#5I>LzlbL;gbS8{iJh%%ON;0c=A8)_sI$I4o_|YR$CAj6 zBAroz3>;MYTs*UunnmCEN$4|U#u0^x>27bCNtvcq`_tR5p zi^>P-gsrV7X|0Pn$WDjwgRj!vTo$*MG>4SB_K&XX+1c0TK5+(d)1JB7E9;`7t41%! z)^C3tUU$weCc*|X_<)BQmgwq!z_Bc@Hj5eAkz{r$*5ErV=v*IEh&fIj==rk6;UD!Z z&RFTDz#Hz#g*2_{ef{KEKi^AiHiVZN<6OQ$7l5h)R?8U(y-DK3L^$8aY#)=y85h!t zdf)W}TAc~cp1gQNSdUUHsIOJq9i8pasQY8oKja`=LJ998&Ds`sro4^*g`*Um5i=b< z(_MWW$$t+9!geV;qw{a6QNcQbcCppXjcjdnLT9qKTS+f7iMw)SAT!AkU!WJ}Ea~mK zI_v(&YlJ}gTjD)xQb&y)=`+}ws%s`4^BNV<5OTZ|vLor4HIs>S?fRe2M9w&8Lg(D% zeXR2Fa(sNw<#LuD)obY|;LiWQ?|j%=ev~a0JMj zbzmbwR?c4NOw6TUs#@X+b=$gLtLZIafApNF!koY4+IH9uN}yXvdS_2oi2OEEy}O*B z)@AFmuZ(_OnX_hLIp%t@8=sMIJx*N64<=$7R)g%zmZI&Dj>^6)Z%bc~pCa??*ghsl zT{NpBu`?R_KnHqkr_7;tS7jl7CYyIJ$}+<yAYUe{-;_N$H>G!1 zJ9EB&=8ui$Px4Q=glIvg?3syf%pkL`%f!YA8hR8)MW>y06S7kBJ$_Ob)q`y8Jh2<> zg;b1eSFf`UPmh>&J>QfGu{L8CqyH2YE!8X@dMpo!E@%GFbu#sKFN$7IWSgKIPZz37$Fd@T?wO@xt((=9A*!_8X#wANJ;?Iln{jnruAXJ;n7b|6 z5_6gJVU3ox6=H5(zKMRuB1Qkm>%b6M3RxZb6R~87*k**epF!3+t}e!XioKDq(-DnW zLYHbwa@&<=U7gj;*rSWV`mr@jE#b_VjyG$xaOGTjMXGh9@XOVARJrD!_s&t|nDgV& z?q1znCSx!4;T7&N@yVk!#@frdY9G&&At`59%;Qd_V{$&CaqW!HHWyg}aU&{=A^Ulc zs{L&ZZnUKl_0NJZ6trG-{ZcQ z#@TffbCvh<;9Gr%gpeZBMyuplSJ%rZXNtqjzZ)rN9r3#Q24`JQcNWvlT$uUwBmwz+ zU0$TO!rU~w&e+%DHG-%$STo&|uBkWR#`+Q)Co9JMzB61T7+ot*bep-4#4pbN_*$E} z9r1{9cz`1@xXKB_O<5lCcw1kIF0eK4&$)Y@evxo zxbXIU{gj-GouK`*O&22wuM;*_MZM-hWny&JDzzu`leN(?aWj?ln724fIX01uNShJT zFZTpLQ|d!7O}iUyxy2s8oO6*IB2o7Cq4O=F-bw68wcPh4XQ^|f$*u!;Nxd_c+K(xX zSdk{zA?nx)Ch~6b30A+Tp@}qgeW~0N9z2A-!kMgW2G(KyDRy%(Cq{LEx{hq29 zhLp_$5i7^Q0<2Up1IkLar)AsPAHu`B7E2X-E5Bg}1vr^Bw;Zv>m$lz_PSnLomlKV% zipsli?k0nc= zz2X%h`%TR4j|C|?9Jvym(o69W-;vYV$Hya4CLi!@x@mL=Sb=7h<5sWOR>`1pbRRZi zf)-1?O=$BR$(Z;E|D_j+1;9z6b!J9$jj=kH@7tS7W@3-Yx%BM4YV;mMj&RkPayQm$ z_QeFOl~c{RcR&Ql?DO%xgOZz$CBa2P>CQAYB~g+}ydBNC8fL=isO|}A-6#Yr(SN3< z%$fYlxkg=dCz7A5qq6&Fwj4fZJ4rl=x3I?KbFO?Q1#%iw1Ui+4F_9a2&{#Cgv_bCn z9E!!^AM>Qw(vf6Y4R^U5@2}_AUxm(?(L2*mbTnY7Swio489flMXullY`@~(c<~Jea z6`yv+v95>i<`9!8Q}Me`VM6ott?9QO2oF;z?2jI%;s#1je)J=?Pg5+#jFmnC&P1R=pHMr$D8M= z6VvX-MP;#k(i*!E>S9}@`aE)^b}Y>n%06d)V!g8u8!56ow#I(b(uc@Fh>+Ckc#-w} zmKXJ@?Jhn};RG4g(1R?KRToyD@prTvthBlk>Jw?noUv0apW1<45|cp41POcE_vJaP z^$y|Rk-JZ1qdqqlFA9YA@Xdsh{cB9G8D*a`EVT_n&Uj>c$fwZ~gIxQf?-J_}{JxCW zb%yOR_QxX2IvK|*Jb3de_Fg5gnz|(y^C`$}_J(ojT1Jc+8fEVFGgVLgj#xv-(I zGuq^qtXk45W)_0H*gaW$SU$7V{8`(2%NRtKnJvT7F-J(}8=hd?lC&{2e1-EEW$QjmpjKCOaB^Mv?*O7BgUuVt2GK_4) zY}BqOYWqu5UKdYD!J)%hXqSl|in9=y zmI!V)4|V5JkZ+&X`8vL-c;34aVx0v%!AM5TnbdG5^Q|@Jl3qg08XGe+Aaoz0#MzZ> z-+sT0Gw*1R7!-CnObl8^$oXEsB^iRyl5AJy?7(6Lh}yjV8GYAc&YAefh$2F^xyN$c zlpQjWd#q}h6SeAU+Ve*Aw#vPBR6*@XwAy>fJp1m(m2A)4`n_bze|BdmGx@Icy`B8K zUN5_g?n3x{USIS#952UF8AXqX2|n{Q+9R?@g^m{ygR)dtMfch|_Y4BM_x^J8X2hcU zU87R#%s6|Uz$<}mA0Hl@dv9cMR6*vUw|j1NH5}Z-o?kqBSP7Pq#xqd#Sm|lS-4WVx zhJIVS>yO9RmLDA~R(qIz^J=MA(p-~|gMFNxxsITAycbfcAMZu-oI7~)-WAv5-6`U} zBV_5_cVbE`VkWfVKd|JGtI%D9Ocf$YIGuhB(@>EE+YVl`0IsCZjC~bSq7bvVUh^ya zF?fGF@Ay6ZNpIrE?{RaN>PDYsg2nl+9T69EK2-A#H(hDA=OZqX4Rn1f&-jeVUa`=_ zWT)7`;Zyb*Cs#WHgvGL7ms32bh?okYgImvHx%UHg<@h4I?@Dp?gly;@W@(x91aHJa zM5U`|*7Ve4UWs0nS=Q4mt5grtE-{^m{r1#wJ#4T!d#}AqlGC2xOsX`bmt$n~3|QkR zciPmAk>tN1rq>o3(pxn|J!!W0Ex{_R3cR#$X*u4&Mtx^y$GQ z!*cmDYVIIL7l8=GU0QrkU3v&aqF7B|E%ACR|5doCeauR}cjxQV;x)06@p5lpY}M5x zaV1!3S$4dquAhIEesdoSbZ9qv{>upa&M?ZDSWbwuiXY60?+_saX|Bv_uJe?fS)z@3 z_T?2{WJvBA(S2FU=%G(LEv0Oh!U~8`&Wl+I0G}sXJ)g;Gvz$lC2_2s0tdBA##@Pqh zK64a_VGBN6j+1d~-uQ@yc^gsdJ; zWzQ8?;`|<~eZ7O+x7C(X_H_PTVuWvLbqMy1I&-KgO`?9Bcj$_3DE*c`&Rzv}2wbYG z$zAEn49@Jy$|)zRbLVDyF7NATe@x#Ed~`Luvu~NzhwHbwawMmt9~5;4dw}@fJdWMP zYamM42aysPF=Sfl<$Ql8%#hdp|?0{f`)nlBh(o;D*5E6W>AK5_1) zJ0I+FZ(|!agKV$!uE{dHYf`X~Ga@+Cxq5BHWu#=n^@}+2BB9Cs*P~?Tji z`pk#OWR@%Ex(r-a{M}s!cqjQ`)Fi9X_AUL=W?buY67p-+{-)&bYQQ@ihpP6$NDb<< z#*k&?TG5vvdnJoymXiJTHISB3ICcs%Cv+z+aZNCmDYbU>awwnjofTGBFU$5>x1uK> zB>p;Mpj!0~j+2D?Tgqe2uEBZ148vw+rH3)sM70R|=vJAB9rd_MM-Sh{ne+6?&gyh5 zO02#=gLiHc4T&BwC?_y(f=ier4n17H{iVb4aHt|D%7tVeV+{50JW?=s9o0%TRj{p zDlyF%H8=hf92EUjb|i3T5B5+E!SB>GBCSj&&DfOl;( zqvI$FBM)YEtPe1a_If!?L(Y?o{@Vt4{zT}9il+W9s8(Ds15gp*A`JPYC5jt@z=^x zJt6E47Bxwphv1YDy6X6R{ZnC#lGxNfL(M&`Hp*eS)2trnu1wXjc5ldevO9wKmFIrz z`q1jtTfCZntMiX`HWw^9m$rALb9NrqVXHE$V-h@0XP|o$r>c=<1d9%x(K$;(mcj5{ zjh_fB=k0JhnWJX?EXgQgmdv1_C{LBP-n*wZ_gyiUStq>_RscM=VP?-6k`WFe&(bnI zMZg(Qq4gn83+YPmL0&_a#JYIaazvewAzr;Ft2|DuL_ScLmB=*iq-W2(u)0fJN8{~^ zEx4%6=y+G(XiJT=cJ`YpBJVj+T}qQM}BInHS#IV_jO0D!p$x zmexsRyno6;a|8-MFVS+>O75b>`7mThHF&esx_OB6Jl(GkoZ(BWHC|Vu&Wyt%?eBqa zEZ(!cs^BwCxDNvv2w5kOEj@=9R?D?sW=!T$?&m|U?#aH6v99htG3N7J$%wE|F2jBx9HLZIbyNGYk`4=A|5= z@O|VP5o+j}ILG=}^#U!c9jw)137mqRv0B2ugYmw8zL%mYI_TT$OPc^Qaf6MZ6z85GCo-mpOw4mqmu~7kcs*UxzU5eYy{DLc9$aMjpbQ4B zlPPs|iwwx&&m}qJmrv*?=45oMa}un!t`zAaQzv!L0kyu{@?0H}>Z0Wy#@G6oJlC8) zH7ImG28@qS7`?LLUDBB^Yxn0$pD8=?*yr^VSC%~xx;6OsmVBN&uMELM&OXVIeJ2Lu zJCTVR)VVU86Y+Cd^EF~;>HTkKOLelNE~D6N9lnGh%1t=A6Cn)d8bGCS$68${Z-Kw?s^A&$(L5Lv#*Hn{4+#9Amn9+8E{; zIiZ35(Xiy9yoImryTW)+OL)*S17WkZtM!sEqmN9LITp?={e2b7!%@R?dlx5r{#XU3 zW~Ud%sy5zcKZ8?;TaD>rb>D{71E$u{z4N(TZ47U%U3sE)>dU-ya@d{oOeeChj!I~* z9z8XirHHOG%I-bZcu(D|xa=z$CEW8eGkMy&Pi`vt-uo|3l^MsWP{=T{;>bOy$mxBM JcfOQy{vV6pM}7bR literal 0 HcmV?d00001 From 7b8ff4ac1467118763ef04db69cd741df13fd346 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 28 Oct 2025 16:25:10 -0400 Subject: [PATCH 55/86] Try to use evironment yml from minian github --- .github/workflows/release-mnn.yml | 8 ++--- Deploy/constraints/environment-minian.yml | 44 +++++++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 Deploy/constraints/environment-minian.yml diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml index 216de2dd..700b8a82 100644 --- a/.github/workflows/release-mnn.yml +++ b/.github/workflows/release-mnn.yml @@ -35,12 +35,12 @@ jobs: shell: cmd run: | C:\Miniconda3\Scripts\conda.exe env remove -n minian_release -y || echo "no old env" - set LOCK_FILE=%GITHUB_WORKSPACE%\environment-minian.txt - if not exist "%LOCK_FILE%" ( - echo ERROR: environment-minian.txt not found at %LOCK_FILE% + set "ENV_FILE=%GITHUB_WORKSPACE%\Deploy\constraints\environment-minian.yml" + if not exist "%ENV_FILE%" ( + echo ERROR: environment file not found at %ENV_FILE% exit /b 1 ) - C:\Miniconda3\Scripts\conda.exe create -n minian_release --file "%LOCK_FILE%" -y + C:\Miniconda3\Scripts\conda.exe env create -n minian_release -f "%ENV_FILE%" --yes - name: Verify minian import shell: cmd diff --git a/Deploy/constraints/environment-minian.yml b/Deploy/constraints/environment-minian.yml new file mode 100644 index 00000000..65e1d44d --- /dev/null +++ b/Deploy/constraints/environment-minian.yml @@ -0,0 +1,44 @@ +name: minian +channels: + - bioconda + - conda-forge + - defaults +dependencies: + - bokeh=1.4.0 + - cvxpy>=1.1.11 + - dask=2021.2.0 + - datashader=0.12.1 + - distributed=2021.2.0 + - ecos>=2.0.7 + - ffmpeg + - ffmpeg-python>=0.2.0 + - fftw + - holoviews=1.12.7 + - jupyter + - matplotlib-base=3.2 + - natsort + - netcdf4 + - networkx=2.4 + - numba=0.52.0 + - numpy=1.20.2 + - opencv=4.2.0 + - pandas=1.2.3 + - panel=0.8.0 + - param=1.9 + - pyfftw=0.12.0 + - python=3.8 + - scikit-image=0.18.1 + - scikit-learn=0.22.1 + - scipy>=1.4.1 + - scs + - simpleitk=2.0.2 + - sk-video + - statsmodels>=0.11.1 + - tifffile + - xarray=0.16.2 + - zarr + - sparse=0.11.2 + - pymetis=2020.1 + - rechunker=0.3.3 + - medpy=0.4.0 + - jinja2=2.11.3 From ab3d5c52ea17a490bddebdac4ce2a16cca1e1cd7 Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 28 Oct 2025 16:35:24 -0400 Subject: [PATCH 56/86] Add missing minian installation --- .github/workflows/release-mnn.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml index 700b8a82..46e48a26 100644 --- a/.github/workflows/release-mnn.yml +++ b/.github/workflows/release-mnn.yml @@ -42,9 +42,10 @@ jobs: ) C:\Miniconda3\Scripts\conda.exe env create -n minian_release -f "%ENV_FILE%" --yes - - name: Verify minian import + - name: Install minian deps with conda-forge shell: cmd run: | + C:\Miniconda3\Scripts\conda.exe install -n minian_release -c conda-forge h5py pyinstaller minian -y C:\Miniconda3\Scripts\conda.exe run -n minian_release python -c "import minian; print('MiniAn version:', getattr(minian,'__version__','?'))" - name: Snapshot conda packages From 1a00dcff4eaa404cfb6b8dc4aa1ea947e960e68b Mon Sep 17 00:00:00 2001 From: katemartian Date: Tue, 28 Oct 2025 17:21:15 -0400 Subject: [PATCH 57/86] Collect all hdf5 dlls for minian release --- Deploy/pack_minian.spec | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Deploy/pack_minian.spec b/Deploy/pack_minian.spec index 34eb1fdf..55485931 100644 --- a/Deploy/pack_minian.spec +++ b/Deploy/pack_minian.spec @@ -1,6 +1,7 @@ # -*- mode: python ; coding: utf-8 -*- import os +from pathlib import Path from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT from PyInstaller.utils.hooks import collect_all, collect_dynamic_libs @@ -48,6 +49,23 @@ for dyn_pkg, dest in [ except ImportError: pass +conda_prefix = os.environ.get('MINIAN_CONDA_PREFIX') or os.environ.get('CONDA_PREFIX') +if conda_prefix: + library_bin = Path(conda_prefix) / 'Library' / 'bin' + if library_bin.is_dir(): + dll_patterns = [ + 'hdf*.dll', + 'hdf5*.dll', + ] + seen = set() + for pattern in dll_patterns: + for dll in library_bin.glob(pattern): + rel_path = os.path.join('Library', 'bin', dll.name) + if rel_path in seen: + continue + binaries.append((rel_path, str(dll), 'BINARY')) + seen.add(rel_path) + a_minian = Analysis( ['../MiniAn/minian_run.py'], pathex=['../'], From 67513de06ae4e2084328e30fc8b993d047251ea3 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 29 Oct 2025 15:20:10 -0400 Subject: [PATCH 58/86] Install SimpleITK with pip to avoid errors --- Deploy/constraints/environment-minian.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Deploy/constraints/environment-minian.yml b/Deploy/constraints/environment-minian.yml index 65e1d44d..ede9ea5f 100644 --- a/Deploy/constraints/environment-minian.yml +++ b/Deploy/constraints/environment-minian.yml @@ -31,7 +31,6 @@ dependencies: - scikit-learn=0.22.1 - scipy>=1.4.1 - scs - - simpleitk=2.0.2 - sk-video - statsmodels>=0.11.1 - tifffile @@ -41,4 +40,8 @@ dependencies: - pymetis=2020.1 - rechunker=0.3.3 - medpy=0.4.0 + - pip - jinja2=2.11.3 + - pip: + # Install SimpleITK via pip to avoid clobber conflicts between libitk and hdf5 on Windows builds + - SimpleITK==2.0.2 From 61870d1bdbfffa6a537423c4b2da6c00b8390385 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 29 Oct 2025 15:22:12 -0400 Subject: [PATCH 59/86] Update conda tooling and clean cache --- .github/workflows/release-mnn.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml index 46e48a26..872eba26 100644 --- a/.github/workflows/release-mnn.yml +++ b/.github/workflows/release-mnn.yml @@ -31,6 +31,13 @@ jobs: shell: cmd run: C:\Miniconda3\Scripts\conda.exe --version + - name: Update conda tooling and clean cache + shell: cmd + run: | + C:\Miniconda3\Scripts\conda.exe update -n base -c defaults conda conda-package-handling -y + C:\Miniconda3\Scripts\conda.exe config --set conda_pkg_format 1 + C:\Miniconda3\Scripts\conda.exe clean --packages --tarballs -y + - name: Create conda env for minian shell: cmd run: | @@ -40,11 +47,14 @@ jobs: echo ERROR: environment file not found at %ENV_FILE% exit /b 1 ) + set "CONDA_PKGS_DIRS=%GITHUB_WORKSPACE%\conda_pkgs" + if not exist "%CONDA_PKGS_DIRS%" mkdir "%CONDA_PKGS_DIRS%" C:\Miniconda3\Scripts\conda.exe env create -n minian_release -f "%ENV_FILE%" --yes - name: Install minian deps with conda-forge shell: cmd run: | + set "CONDA_PKGS_DIRS=%GITHUB_WORKSPACE%\conda_pkgs" C:\Miniconda3\Scripts\conda.exe install -n minian_release -c conda-forge h5py pyinstaller minian -y C:\Miniconda3\Scripts\conda.exe run -n minian_release python -c "import minian; print('MiniAn version:', getattr(minian,'__version__','?'))" From 3dc09a9bcc3c2ff812fca01019f16e2aecd6ba55 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 29 Oct 2025 15:36:17 -0400 Subject: [PATCH 60/86] Pin versions of minian, h5py and pyinstaller --- .github/workflows/release-mnn.yml | 3 +-- Deploy/constraints/environment-minian.yml | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml index 872eba26..855bd0b1 100644 --- a/.github/workflows/release-mnn.yml +++ b/.github/workflows/release-mnn.yml @@ -51,11 +51,10 @@ jobs: if not exist "%CONDA_PKGS_DIRS%" mkdir "%CONDA_PKGS_DIRS%" C:\Miniconda3\Scripts\conda.exe env create -n minian_release -f "%ENV_FILE%" --yes - - name: Install minian deps with conda-forge + - name: Verify minian deps shell: cmd run: | set "CONDA_PKGS_DIRS=%GITHUB_WORKSPACE%\conda_pkgs" - C:\Miniconda3\Scripts\conda.exe install -n minian_release -c conda-forge h5py pyinstaller minian -y C:\Miniconda3\Scripts\conda.exe run -n minian_release python -c "import minian; print('MiniAn version:', getattr(minian,'__version__','?'))" - name: Snapshot conda packages diff --git a/Deploy/constraints/environment-minian.yml b/Deploy/constraints/environment-minian.yml index ede9ea5f..2d9b77a6 100644 --- a/Deploy/constraints/environment-minian.yml +++ b/Deploy/constraints/environment-minian.yml @@ -40,6 +40,9 @@ dependencies: - pymetis=2020.1 - rechunker=0.3.3 - medpy=0.4.0 + - h5py=3.1.0 + - minian=1.2.1 + - pyinstaller=4.5.1 - pip - jinja2=2.11.3 - pip: From 9d6b8124ccc40584960ce1d629127d85570b44e0 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 29 Oct 2025 17:05:43 -0400 Subject: [PATCH 61/86] Place minian, h5py, and pyinstaller as the last libraries to install --- Deploy/constraints/environment-minian.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Deploy/constraints/environment-minian.yml b/Deploy/constraints/environment-minian.yml index 2d9b77a6..8e75a648 100644 --- a/Deploy/constraints/environment-minian.yml +++ b/Deploy/constraints/environment-minian.yml @@ -40,11 +40,10 @@ dependencies: - pymetis=2020.1 - rechunker=0.3.3 - medpy=0.4.0 - - h5py=3.1.0 - - minian=1.2.1 - - pyinstaller=4.5.1 - pip - jinja2=2.11.3 - pip: - # Install SimpleITK via pip to avoid clobber conflicts between libitk and hdf5 on Windows builds - SimpleITK==2.0.2 + - minian=1.2.1 + - h5py=3.1.0 + - pyinstaller=4.5.1 From 32f7185c7f81b2468ce58815dbcda0f56ae76230 Mon Sep 17 00:00:00 2001 From: katemartian Date: Wed, 29 Oct 2025 17:06:19 -0400 Subject: [PATCH 62/86] Fix parameters that are passed to pyinstaller Analysis in minian --- Deploy/pack_minian.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Deploy/pack_minian.spec b/Deploy/pack_minian.spec index 55485931..8a731057 100644 --- a/Deploy/pack_minian.spec +++ b/Deploy/pack_minian.spec @@ -63,7 +63,7 @@ if conda_prefix: rel_path = os.path.join('Library', 'bin', dll.name) if rel_path in seen: continue - binaries.append((rel_path, str(dll), 'BINARY')) + binaries.append((str(dll), rel_path)) seen.add(rel_path) a_minian = Analysis( From 32d41bc5b7cff1cb7a45534f9bdca4d2bf9422a0 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 30 Oct 2025 09:49:53 -0400 Subject: [PATCH 63/86] Hide binary collection that wasn't here before. Change pyinstaller version to the one in local environment --- Deploy/pack_minian.spec | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/Deploy/pack_minian.spec b/Deploy/pack_minian.spec index 8a731057..55b66fb0 100644 --- a/Deploy/pack_minian.spec +++ b/Deploy/pack_minian.spec @@ -40,31 +40,31 @@ for package in packages: for dyn_pkg, dest in [ ('llvmlite', os.path.join('Library', 'bin')), ('scipy', os.path.join('scipy', '.libs')), - ('h5py', '.'), - ('SimpleITK', '.'), - ('cv2', '.'), + # ('h5py', '.'), + # ('SimpleITK', '.'), + # ('cv2', '.'), ]: try: binaries += collect_dynamic_libs(dyn_pkg, destdir=dest) except ImportError: pass -conda_prefix = os.environ.get('MINIAN_CONDA_PREFIX') or os.environ.get('CONDA_PREFIX') -if conda_prefix: - library_bin = Path(conda_prefix) / 'Library' / 'bin' - if library_bin.is_dir(): - dll_patterns = [ - 'hdf*.dll', - 'hdf5*.dll', - ] - seen = set() - for pattern in dll_patterns: - for dll in library_bin.glob(pattern): - rel_path = os.path.join('Library', 'bin', dll.name) - if rel_path in seen: - continue - binaries.append((str(dll), rel_path)) - seen.add(rel_path) +# conda_prefix = os.environ.get('MINIAN_CONDA_PREFIX') or os.environ.get('CONDA_PREFIX') +# if conda_prefix: +# library_bin = Path(conda_prefix) / 'Library' / 'bin' +# if library_bin.is_dir(): +# dll_patterns = [ +# 'hdf*.dll', +# 'hdf5*.dll', +# ] +# seen = set() +# for pattern in dll_patterns: +# for dll in library_bin.glob(pattern): +# rel_path = os.path.join('Library', 'bin', dll.name) +# if rel_path in seen: +# continue +# binaries.append((str(dll), rel_path)) +# seen.add(rel_path) a_minian = Analysis( ['../MiniAn/minian_run.py'], From ed5ebdd9bc3226c2d663afe5d7fd3583227a5746 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 30 Oct 2025 09:50:03 -0400 Subject: [PATCH 64/86] Missed file --- Deploy/constraints/environment-minian.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Deploy/constraints/environment-minian.yml b/Deploy/constraints/environment-minian.yml index 8e75a648..e4b7868b 100644 --- a/Deploy/constraints/environment-minian.yml +++ b/Deploy/constraints/environment-minian.yml @@ -46,4 +46,4 @@ dependencies: - SimpleITK==2.0.2 - minian=1.2.1 - h5py=3.1.0 - - pyinstaller=4.5.1 + - pyinstaller=6.7.0 From caed8586ab459793cce5b056d0833827eca25abb Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 30 Oct 2025 10:39:42 -0400 Subject: [PATCH 65/86] Return back the collection of the hdf dlls --- Deploy/pack_minian.spec | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Deploy/pack_minian.spec b/Deploy/pack_minian.spec index 55b66fb0..26118dc2 100644 --- a/Deploy/pack_minian.spec +++ b/Deploy/pack_minian.spec @@ -49,22 +49,22 @@ for dyn_pkg, dest in [ except ImportError: pass -# conda_prefix = os.environ.get('MINIAN_CONDA_PREFIX') or os.environ.get('CONDA_PREFIX') -# if conda_prefix: -# library_bin = Path(conda_prefix) / 'Library' / 'bin' -# if library_bin.is_dir(): -# dll_patterns = [ -# 'hdf*.dll', -# 'hdf5*.dll', -# ] -# seen = set() -# for pattern in dll_patterns: -# for dll in library_bin.glob(pattern): -# rel_path = os.path.join('Library', 'bin', dll.name) -# if rel_path in seen: -# continue -# binaries.append((str(dll), rel_path)) -# seen.add(rel_path) +conda_prefix = os.environ.get('MINIAN_CONDA_PREFIX') or os.environ.get('CONDA_PREFIX') +if conda_prefix: + library_bin = Path(conda_prefix) / 'Library' / 'bin' + if library_bin.is_dir(): + dll_patterns = [ + 'hdf*.dll', + 'hdf5*.dll', + ] + seen = set() + for pattern in dll_patterns: + for dll in library_bin.glob(pattern): + rel_path = os.path.join('Library', 'bin', dll.name) + if rel_path in seen: + continue + binaries.append((str(dll), rel_path)) + seen.add(rel_path) a_minian = Analysis( ['../MiniAn/minian_run.py'], From a341a40d0c06b783df385724ea51d9c4caad6dd4 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 30 Oct 2025 11:28:24 -0400 Subject: [PATCH 66/86] Freeze release of h5py and pyinstaller to the ones before last minian release --- Deploy/constraints/environment-minian.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Deploy/constraints/environment-minian.yml b/Deploy/constraints/environment-minian.yml index e4b7868b..56ea9708 100644 --- a/Deploy/constraints/environment-minian.yml +++ b/Deploy/constraints/environment-minian.yml @@ -1,4 +1,4 @@ -name: minian +name: minian # MiniAn last release was v1.2.1 on Feb 10, 2022, they stopped updating conda package channels: - bioconda - conda-forge @@ -44,6 +44,6 @@ dependencies: - jinja2=2.11.3 - pip: - SimpleITK==2.0.2 - - minian=1.2.1 - - h5py=3.1.0 - - pyinstaller=6.7.0 + - minian=1.2.1 # Feb 10, 2022 release + - h5py=3.6.0 # Nov 16, 2021 release + - pyinstaller=4.9.0 # Feb 3, 2022 release From deefa870582669b4ed2224017f4629ab42bf3dbd Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 30 Oct 2025 12:22:59 -0400 Subject: [PATCH 67/86] Uncomment dynamic lib collection --- Deploy/pack_minian.spec | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Deploy/pack_minian.spec b/Deploy/pack_minian.spec index 26118dc2..7a3ea9c4 100644 --- a/Deploy/pack_minian.spec +++ b/Deploy/pack_minian.spec @@ -2,7 +2,6 @@ import os from pathlib import Path - from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT from PyInstaller.utils.hooks import collect_all, collect_dynamic_libs @@ -40,9 +39,9 @@ for package in packages: for dyn_pkg, dest in [ ('llvmlite', os.path.join('Library', 'bin')), ('scipy', os.path.join('scipy', '.libs')), - # ('h5py', '.'), - # ('SimpleITK', '.'), - # ('cv2', '.'), + ('h5py', '.'), + ('SimpleITK', '.'), + ('cv2', '.'), ]: try: binaries += collect_dynamic_libs(dyn_pkg, destdir=dest) From dec0d4267f4c9226fb9532d0f55bc87d1b705907 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 30 Oct 2025 15:48:33 -0400 Subject: [PATCH 68/86] Set upx to false in minian --- Deploy/pack_minian.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Deploy/pack_minian.spec b/Deploy/pack_minian.spec index 7a3ea9c4..80fee294 100644 --- a/Deploy/pack_minian.spec +++ b/Deploy/pack_minian.spec @@ -95,7 +95,7 @@ exe_minian = EXE( debug=False, bootloader_ignore_signals=False, strip=False, - upx=True, + upx=False, upx_exclude=[], runtime_tmpdir=None, console=True, @@ -112,7 +112,7 @@ coll = COLLECT( a_minian.zipfiles, a_minian.datas, strip=False, - upx=True, + upx=False, upx_exclude=[], name='minian', ) From ba90745dfc322a40426782be145c4cef2c91b00e Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 30 Oct 2025 16:22:36 -0400 Subject: [PATCH 69/86] Set release of h5py and pyinstaller to the ones used locally for minian --- Deploy/constraints/environment-minian.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Deploy/constraints/environment-minian.yml b/Deploy/constraints/environment-minian.yml index 56ea9708..e40e493b 100644 --- a/Deploy/constraints/environment-minian.yml +++ b/Deploy/constraints/environment-minian.yml @@ -45,5 +45,5 @@ dependencies: - pip: - SimpleITK==2.0.2 - minian=1.2.1 # Feb 10, 2022 release - - h5py=3.6.0 # Nov 16, 2021 release - - pyinstaller=4.9.0 # Feb 3, 2022 release + - h5py=3.7.0 # May 24, 2022 # 3.6.0 -- Nov 16, 2021 release + - pyinstaller=6.7.0 # May 21, 2024 # 4.9.0 -- Feb 3, 2022 release From ea1d7eaf33e61759ae9cedf79c13615092ca2681 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 30 Oct 2025 16:58:03 -0400 Subject: [PATCH 70/86] Pin versions of h5py and pyinstaller suggested by Codex for minian release --- Deploy/constraints/environment-minian.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Deploy/constraints/environment-minian.yml b/Deploy/constraints/environment-minian.yml index e40e493b..ff1a3b8a 100644 --- a/Deploy/constraints/environment-minian.yml +++ b/Deploy/constraints/environment-minian.yml @@ -45,5 +45,12 @@ dependencies: - pip: - SimpleITK==2.0.2 - minian=1.2.1 # Feb 10, 2022 release - - h5py=3.7.0 # May 24, 2022 # 3.6.0 -- Nov 16, 2021 release - - pyinstaller=6.7.0 # May 21, 2024 # 4.9.0 -- Feb 3, 2022 release +# Releases suggested by Codex + - h5py=3.1.0 # Nov 6, 2020 release + - pyinstaller=4.5.1 # Aug 6, 2021 release +# Local versions +# - h5py=3.7.0 # May 24, 2022 release +# - pyinstaller=6.7.0 # May 21, 2024 release +# Releases just before the last update of minian conda package +# - h5py=3.6.0 # Nov 16, 2021 release +# - pyinstaller=4.9.0 # Feb 3, 2022 release From 23ee317f875b7c3ae064baca80c9d5cab0a446ef Mon Sep 17 00:00:00 2001 From: katemartian Date: Fri, 31 Oct 2025 10:47:15 -0400 Subject: [PATCH 71/86] Set all versions to the local ones for minian release --- Deploy/constraints/environment-minian.yml | 91 ++++++++++++----------- 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/Deploy/constraints/environment-minian.yml b/Deploy/constraints/environment-minian.yml index ff1a3b8a..c7f98af5 100644 --- a/Deploy/constraints/environment-minian.yml +++ b/Deploy/constraints/environment-minian.yml @@ -4,53 +4,54 @@ channels: - conda-forge - defaults dependencies: - - bokeh=1.4.0 - - cvxpy>=1.1.11 - - dask=2021.2.0 - - datashader=0.12.1 - - distributed=2021.2.0 - - ecos>=2.0.7 - - ffmpeg - - ffmpeg-python>=0.2.0 - - fftw - - holoviews=1.12.7 - - jupyter - - matplotlib-base=3.2 - - natsort - - netcdf4 - - networkx=2.4 - - numba=0.52.0 - - numpy=1.20.2 - - opencv=4.2.0 - - pandas=1.2.3 - - panel=0.8.0 - - param=1.9 - - pyfftw=0.12.0 - - python=3.8 - - scikit-image=0.18.1 - - scikit-learn=0.22.1 - - scipy>=1.4.1 - - scs - - sk-video - - statsmodels>=0.11.1 - - tifffile - - xarray=0.16.2 - - zarr - - sparse=0.11.2 - - pymetis=2020.1 - - rechunker=0.3.3 - - medpy=0.4.0 - - pip - - jinja2=2.11.3 + - bokeh=1.4.0 # =1.4.0 + - cvxpy=1.2.1 # >=1.1.11 + - dask=2021.2.0 # =2021.2.0 + - datashader=0.12.1 # =0.12.1 + - distributed=2021.2.0 # =2021.2.0 + - ecos=2.0.10 # >=2.0.7 + - ffmpeg=6.1.0 # + - ffmpeg-python=0.2.0 # >=0.2.0 + - fftw=3.3.10 # + - holoviews=1.12.7 # =1.12.7 + - jupyter # + - matplotlib-base=3.2.2 # =3.2 + - natsort=8.4.0 # + - netcdf4=1.6.0 # + - networkx=2.4 # =2.4 + - numba=0.52.0 # =0.52.0 + - numpy=1.20.2 # =1.20.2 + - opencv=4.2.0 # =4.2.0 + - pandas=1.2.3 # =1.2.3 + - panel=0.8.0 # =0.8.0 + - param=1.9.3 # =1.9 + - pyfftw=0.12.0 # =0.12.0 + - python=3.8 # =3.8 + - scikit-image=0.18.1 # =0.18.1 + - scikit-learn=0.22.1 # =0.22.1 + - scipy=1.9.1 # >=1.4.1 + - scs=3.2.0 # + - sk-video=1.1.10 # + - statsmodels=0.13.2 # >=0.11.1 + - tifffile=2021.11.2 # + - xarray=0.16.2 # =0.16.2 + - zarr=2.17.1 # + - sparse=0.11.2 # =0.11.2 + - pymetis=2020.1 # =2020.1 + - rechunker=0.3.3 # =0.3.3 + - medpy=0.4.0 # =0.4.0 + - pip=24.2 # + - jinja2=2.11.3 # =2.11.3 + - pefile-2024.8.26 - pip: - - SimpleITK==2.0.2 + - SimpleITK==2.0.2 # ==2.0.2 - minian=1.2.1 # Feb 10, 2022 release + # Local versions + - h5py=3.7.0 # May 24, 2022 release + - pyinstaller=6.7.0 # May 21, 2024 release # Releases suggested by Codex - - h5py=3.1.0 # Nov 6, 2020 release - - pyinstaller=4.5.1 # Aug 6, 2021 release -# Local versions -# - h5py=3.7.0 # May 24, 2022 release -# - pyinstaller=6.7.0 # May 21, 2024 release + # - h5py=3.1.0 # Nov 6, 2020 release + # - pyinstaller=4.5.1 # Aug 6, 2021 release # Releases just before the last update of minian conda package # - h5py=3.6.0 # Nov 16, 2021 release -# - pyinstaller=4.9.0 # Feb 3, 2022 release +# - pyinstaller=4.9.0 # Feb 3, 2022 release \ No newline at end of file From dbce7231811b468b72904e020ceeff400512e328 Mon Sep 17 00:00:00 2001 From: katemartian Date: Fri, 31 Oct 2025 10:58:20 -0400 Subject: [PATCH 72/86] Fix typo in the yml file --- Deploy/constraints/environment-minian.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Deploy/constraints/environment-minian.yml b/Deploy/constraints/environment-minian.yml index c7f98af5..3b0ab862 100644 --- a/Deploy/constraints/environment-minian.yml +++ b/Deploy/constraints/environment-minian.yml @@ -42,7 +42,7 @@ dependencies: - medpy=0.4.0 # =0.4.0 - pip=24.2 # - jinja2=2.11.3 # =2.11.3 - - pefile-2024.8.26 + - pefile=2024.8.26 - pip: - SimpleITK==2.0.2 # ==2.0.2 - minian=1.2.1 # Feb 10, 2022 release From ea251d6f8eeac7f3cc25f69077d5882818d8e521 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 6 Nov 2025 12:18:03 -0500 Subject: [PATCH 73/86] Add missing dlls for minian release --- Deploy/pack_minian.spec | 80 ++++++++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/Deploy/pack_minian.spec b/Deploy/pack_minian.spec index 80fee294..aef20b79 100644 --- a/Deploy/pack_minian.spec +++ b/Deploy/pack_minian.spec @@ -11,14 +11,15 @@ workpath = os.path.join(_specdir, "build") BLOCK_CIPHER = None -packages = [ +PACKAGES = [ 'minian', 'distributed', 'skimage', 'scipy', + 'pyviz_comms', ] -excludes = [ +EXCLUDES = [ "bokeh", "IPython", "jupyter", @@ -32,39 +33,54 @@ excludes = [ datas = [] binaries = [] hiddenimports = [] -for package in packages: +for package in PACKAGES: tmp_ret = collect_all(package) - datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] + datas += tmp_ret[0] + binaries += tmp_ret[1] + hiddenimports += tmp_ret[2] -for dyn_pkg, dest in [ - ('llvmlite', os.path.join('Library', 'bin')), - ('scipy', os.path.join('scipy', '.libs')), - ('h5py', '.'), - ('SimpleITK', '.'), - ('cv2', '.'), -]: - try: - binaries += collect_dynamic_libs(dyn_pkg, destdir=dest) - except ImportError: - pass +binaries += collect_dynamic_libs('llvmlite', destdir=os.path.join('Library', 'bin')) -conda_prefix = os.environ.get('MINIAN_CONDA_PREFIX') or os.environ.get('CONDA_PREFIX') -if conda_prefix: - library_bin = Path(conda_prefix) / 'Library' / 'bin' - if library_bin.is_dir(): - dll_patterns = [ - 'hdf*.dll', - 'hdf5*.dll', - ] - seen = set() - for pattern in dll_patterns: - for dll in library_bin.glob(pattern): - rel_path = os.path.join('Library', 'bin', dll.name) - if rel_path in seen: - continue - binaries.append((str(dll), rel_path)) - seen.add(rel_path) +DLL_PREFIXES = { + "ff", + "hdf", + "icu", + "itk", + "lib", + "open", + "qt5", +} + +DLL_EXACT = { + "freetype.dll", + "lerc.dll", + "mfhdf.dll", + "mkl_pgi_thread.2.dll", + "netcdf.dll", + "pthreadvse2.dll", + "szip.dll", + "tiff.dll", + "xdr.dll", + "yaml.dll", + "zip.dll", + "zstd.dll", +} +conda_prefix = os.environ.get('MINIAN_CONDA_PREFIX') or os.environ.get('CONDA_PREFIX') +library_bin = Path(conda_prefix) / 'Library' / 'bin' +if library_bin.is_dir(): + existing = {os.path.basename(src).lower() for src, _ in binaries} + selected = [] + for dll in sorted(library_bin.glob('*.dll'), key=lambda p: p.name.lower()): + name = dll.name.lower() + if name not in DLL_EXACT and not any(name.startswith(prefix) for prefix in DLL_PREFIXES): + continue + if name in existing: + continue + selected.append((str(dll), '.')) + existing.add(name) + binaries.extend(selected) + a_minian = Analysis( ['../MiniAn/minian_run.py'], pathex=['../'], @@ -74,7 +90,7 @@ a_minian = Analysis( hookspath=[], hooksconfig={}, runtime_hooks=[], - excludes=excludes, + excludes=EXCLUDES, win_no_prefer_redirects=False, win_private_assemblies=False, cipher=BLOCK_CIPHER, From 20c20d5e956ff7f8c900a2161cfc9c9ded130cf5 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 6 Nov 2025 12:33:28 -0500 Subject: [PATCH 74/86] Fix version of torch for cuda 12.8 in deeplabcut release --- .github/workflows/release-dlc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-dlc.yml b/.github/workflows/release-dlc.yml index c83862c8..9bb05ad1 100644 --- a/.github/workflows/release-dlc.yml +++ b/.github/workflows/release-dlc.yml @@ -38,7 +38,7 @@ jobs: run: | venv\Scripts\python -m pip install --upgrade pip venv\Scripts\pip install --constraint Deploy\constraints\pins.txt h5py pyinstaller deeplabcut==3.0.0rc8 - venv\Scripts\pip install --constraint Deploy\constraints\pins.txt --upgrade --index-url https://download.pytorch.org/whl/cu126 torch==2.8.0+cu126 torchvision==0.23.0+cu126 + venv\Scripts\pip install --constraint Deploy\constraints\pins.txt --upgrade --index-url https://download.pytorch.org/whl/cu128 torch==2.8.0+cu128 torchvision==0.23.0+cu128 venv\Scripts\python -c "import torch;print('Torch:',torch.__version__);print('CUDA build:',getattr(torch.version,'cuda',None));print('CUDA available:',torch.cuda.is_available())" - name: Clean old dist/build From 073ce45891481ec2f28b462a1e48a398bbbd77ed Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 6 Nov 2025 12:33:48 -0500 Subject: [PATCH 75/86] Freeze version of suite2p --- .github/workflows/release-s2p.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-s2p.yml b/.github/workflows/release-s2p.yml index 50c29391..3f06b0dd 100644 --- a/.github/workflows/release-s2p.yml +++ b/.github/workflows/release-s2p.yml @@ -37,7 +37,7 @@ jobs: shell: cmd run: | venv\Scripts\python -m pip install --upgrade pip - venv\Scripts\pip install --constraint Deploy\constraints\pins.txt h5py pyinstaller suite2p + venv\Scripts\pip install --constraint Deploy\constraints\pins.txt h5py pyinstaller suite2p==0.14.4 venv\Scripts\python -c "import suite2p; print('Suite2p version:', getattr(suite2p,'__version__','?'))" - name: Clean old dist/build From 01f457ea44b59cb1048efa34ae3f0f6d96b93b74 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 6 Nov 2025 12:36:01 -0500 Subject: [PATCH 76/86] Minor edits in the caiman and suite2p spec files --- Deploy/pack_caiman.spec | 12 +++++++----- Deploy/pack_suite2p.spec | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Deploy/pack_caiman.spec b/Deploy/pack_caiman.spec index 0083e417..586fd1f6 100644 --- a/Deploy/pack_caiman.spec +++ b/Deploy/pack_caiman.spec @@ -14,7 +14,7 @@ CAIMAN_DATA_DIR = os.environ.get("CAIMAN_DATA_DIR") BLOCK_CIPHER = None -packages = [ +PACKAGES = [ 'caiman', 'hdmf', 'pynwb', @@ -23,7 +23,7 @@ packages = [ 'scipy' ] -excludes = [ +EXCLUDES = [ "PyQt5", "Markdown", "jupyter", @@ -43,9 +43,11 @@ excludes = [ datas = [] binaries = [] hiddenimports = [] -for package in packages: +for package in PACKAGES: tmp_ret = collect_all(package) - datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] + datas += tmp_ret[0] + binaries += tmp_ret[1] + hiddenimports += tmp_ret[2] a_caimAn = Analysis( ['../CaImAn/caiman_run.py'], @@ -56,7 +58,7 @@ a_caimAn = Analysis( hookspath=[], hooksconfig={}, runtime_hooks=[], - excludes=excludes, + excludes=EXCLUDES, win_no_prefer_redirects=False, win_private_assemblies=False, cipher=BLOCK_CIPHER, diff --git a/Deploy/pack_suite2p.spec b/Deploy/pack_suite2p.spec index a1069466..32b34596 100644 --- a/Deploy/pack_suite2p.spec +++ b/Deploy/pack_suite2p.spec @@ -10,12 +10,12 @@ workpath = os.path.join(_specdir, "build") BLOCK_CIPHER = None -packages = [ +PACKAGES = [ 'suite2p', 'ScanImageTiffReader', ] -excludes = [ +EXCLUDES = [ "IPython", "PyQt6", "PyQt5", @@ -26,9 +26,11 @@ excludes = [ datas = [] binaries = [] hiddenimports = [] -for package in packages: +for package in PACKAGES: tmp_ret = collect_all(package) - datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] + datas += tmp_ret[0] + binaries += tmp_ret[1] + hiddenimports += tmp_ret[2] a_suite2p = Analysis( ['../Suite2p/suite2p_run.py'], @@ -39,7 +41,7 @@ a_suite2p = Analysis( hookspath=[], hooksconfig={}, runtime_hooks=[], - excludes=excludes, + excludes=EXCLUDES, win_no_prefer_redirects=False, win_private_assemblies=False, cipher=BLOCK_CIPHER, From ad6f002e92a8071678bf862161a4414c88ac3df2 Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 6 Nov 2025 12:36:35 -0500 Subject: [PATCH 77/86] Test deeplabcut release including all the optional packages --- Deploy/pack_deeplabcut.spec | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/Deploy/pack_deeplabcut.spec b/Deploy/pack_deeplabcut.spec index 2e1e7b2c..58a259c8 100644 --- a/Deploy/pack_deeplabcut.spec +++ b/Deploy/pack_deeplabcut.spec @@ -1,7 +1,7 @@ # -*- mode: python ; coding: utf-8 -*- import os -import importlib.util +# import importlib.util from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT, TOC from PyInstaller.utils.hooks import collect_all @@ -11,10 +11,11 @@ workpath = os.path.join(_specdir, "build") BLOCK_CIPHER = None -required_packages = [ +PACKAGES = [ 'deeplabcut' -] -optional_packages = [ +# ] + +# optional_packages = [ 'charset_normalizer', 'dateutil', 'safetensors', @@ -27,19 +28,21 @@ optional_packages = [ datas = [] binaries = [] hiddenimports = [] -for package in required_packages: +for package in PACKAGES: tmp_ret = collect_all(package) - datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] + datas += tmp_ret[0] + binaries += tmp_ret[1] + hiddenimports += tmp_ret[2] -for package in optional_packages: - if importlib.util.find_spec(package) is not None: - tmp_ret = collect_all(package) - datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +# for package in optional_packages: +# if importlib.util.find_spec(package) is not None: +# tmp_ret = collect_all(package) +# datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] -if importlib.util.find_spec('pyarrow') is not None: - hiddenimports += ['pyarrow._generated_version'] +# if importlib.util.find_spec('pyarrow') is not None: +hiddenimports += ['pyarrow._generated_version'] -excludes = [ +EXCLUDE_PACKAGES = [ "matplotlib" ] @@ -52,14 +55,14 @@ a_deeplabcut = Analysis( hookspath=[], hooksconfig={}, runtime_hooks=['constraints/torch_openmp_env.py'], - excludes=excludes, + excludes=EXCLUDE_PACKAGES, win_no_prefer_redirects=False, win_private_assemblies=False, cipher=BLOCK_CIPHER, noarchive=False, ) -exclude_binaries = [ +EXCLUDE_BINARIES = [ 'cublasLt64_12.dll', 'cublas64_12.dll', 'cudart64_12.dll', @@ -80,7 +83,7 @@ exclude_binaries = [ ] a_deeplabcut.binaries= TOC([ - x for x in a_deeplabcut.binaries if not any(exclude in x[0] for exclude in exclude_binaries) + x for x in a_deeplabcut.binaries if not any(exclude in x[0] for exclude in EXCLUDE_BINARIES) ]) pyz_deeplabcut = PYZ( From d614de82d6e7a999f65391e9f84d7fa8e2f826ef Mon Sep 17 00:00:00 2001 From: katemartian Date: Thu, 6 Nov 2025 13:42:50 -0500 Subject: [PATCH 78/86] Mock matplotlib.patches.Ellipse --- DeepLabCut/deeplabcut_run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/DeepLabCut/deeplabcut_run.py b/DeepLabCut/deeplabcut_run.py index b5517e51..938b434a 100644 --- a/DeepLabCut/deeplabcut_run.py +++ b/DeepLabCut/deeplabcut_run.py @@ -15,6 +15,7 @@ 'matplotlib.image', 'matplotlib.lines', 'matplotlib.patches', + 'matplotlib.patches.Ellipse', 'matplotlib.container', 'matplotlib.transforms', 'matplotlib.tri._triangulation', From 4b2f406bd8f364aab721d6e7b6eb5cdf9cae48ba Mon Sep 17 00:00:00 2001 From: maximeB Date: Fri, 7 Nov 2025 12:52:43 -0500 Subject: [PATCH 79/86] missing coma --- Deploy/pack_deeplabcut.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Deploy/pack_deeplabcut.spec b/Deploy/pack_deeplabcut.spec index 58a259c8..aaffb2d7 100644 --- a/Deploy/pack_deeplabcut.spec +++ b/Deploy/pack_deeplabcut.spec @@ -12,7 +12,7 @@ workpath = os.path.join(_specdir, "build") BLOCK_CIPHER = None PACKAGES = [ - 'deeplabcut' + 'deeplabcut', # ] # optional_packages = [ From 5f69baf4a2d8939b840ce113cbef95a5c8290583 Mon Sep 17 00:00:00 2001 From: maximeB Date: Fri, 7 Nov 2025 13:34:31 -0500 Subject: [PATCH 80/86] remove commented code in spec deeplabcut --- Deploy/pack_deeplabcut.spec | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Deploy/pack_deeplabcut.spec b/Deploy/pack_deeplabcut.spec index aaffb2d7..51488bd1 100644 --- a/Deploy/pack_deeplabcut.spec +++ b/Deploy/pack_deeplabcut.spec @@ -13,9 +13,6 @@ BLOCK_CIPHER = None PACKAGES = [ 'deeplabcut', -# ] - -# optional_packages = [ 'charset_normalizer', 'dateutil', 'safetensors', @@ -34,12 +31,6 @@ for package in PACKAGES: binaries += tmp_ret[1] hiddenimports += tmp_ret[2] -# for package in optional_packages: -# if importlib.util.find_spec(package) is not None: -# tmp_ret = collect_all(package) -# datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - -# if importlib.util.find_spec('pyarrow') is not None: hiddenimports += ['pyarrow._generated_version'] EXCLUDE_PACKAGES = [ From 60b1edc650faa6de9ec5f685690de48177db62d0 Mon Sep 17 00:00:00 2001 From: maximeB Date: Fri, 7 Nov 2025 13:38:16 -0500 Subject: [PATCH 81/86] remove commented import --- Deploy/pack_deeplabcut.spec | 1 - 1 file changed, 1 deletion(-) diff --git a/Deploy/pack_deeplabcut.spec b/Deploy/pack_deeplabcut.spec index 51488bd1..09652b57 100644 --- a/Deploy/pack_deeplabcut.spec +++ b/Deploy/pack_deeplabcut.spec @@ -1,7 +1,6 @@ # -*- mode: python ; coding: utf-8 -*- import os -# import importlib.util from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT, TOC from PyInstaller.utils.hooks import collect_all From 146ff88c23f90a40e6be590c474bee6e5b39248f Mon Sep 17 00:00:00 2001 From: katemartian Date: Fri, 14 Nov 2025 12:33:25 -0500 Subject: [PATCH 82/86] Remove unnecessary comment --- .github/workflows/release-s2p.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release-s2p.yml b/.github/workflows/release-s2p.yml index 3f06b0dd..54b73a78 100644 --- a/.github/workflows/release-s2p.yml +++ b/.github/workflows/release-s2p.yml @@ -74,7 +74,6 @@ jobs: set "ZIPNAME=suite2p-%VERSION%.zip" - rem One-line Python (no backslashes/newlines) ..\venv\Scripts\python -c "import os,zipfile; a=r'%APPDIR%'; zf=r'%ZIPNAME%'; assert os.path.isdir(a), 'no APPDIR: '+a; z=zipfile.ZipFile(zf,'w',zipfile.ZIP_DEFLATED); [z.write(os.path.join(r,f), os.path.relpath(os.path.join(r,f),a)) for r,_,fs in os.walk(a) for f in fs]; z.close()" echo Created zip: %ZIPNAME% From eb3a362e6332665d060733c33b6a5e6387e64b82 Mon Sep 17 00:00:00 2001 From: katemartian Date: Fri, 14 Nov 2025 14:37:24 -0500 Subject: [PATCH 83/86] Update pins for suite2p --- .github/workflows/release-dlc.yml | 4 ++-- .github/workflows/release-s2p.yml | 2 +- Deploy/constraints/{pins.txt => pins-dlc.txt} | 0 Deploy/constraints/pins-s2p.txt | 4 ++++ 4 files changed, 7 insertions(+), 3 deletions(-) rename Deploy/constraints/{pins.txt => pins-dlc.txt} (100%) create mode 100644 Deploy/constraints/pins-s2p.txt diff --git a/.github/workflows/release-dlc.yml b/.github/workflows/release-dlc.yml index 9bb05ad1..6a04b221 100644 --- a/.github/workflows/release-dlc.yml +++ b/.github/workflows/release-dlc.yml @@ -37,8 +37,8 @@ jobs: shell: cmd run: | venv\Scripts\python -m pip install --upgrade pip - venv\Scripts\pip install --constraint Deploy\constraints\pins.txt h5py pyinstaller deeplabcut==3.0.0rc8 - venv\Scripts\pip install --constraint Deploy\constraints\pins.txt --upgrade --index-url https://download.pytorch.org/whl/cu128 torch==2.8.0+cu128 torchvision==0.23.0+cu128 + venv\Scripts\pip install --constraint Deploy\constraints\pins-dlc.txt h5py pyinstaller deeplabcut==3.0.0rc8 + venv\Scripts\pip install --constraint Deploy\constraints\pins-dlc.txt --upgrade --index-url https://download.pytorch.org/whl/cu128 torch==2.8.0+cu128 torchvision==0.23.0+cu128 venv\Scripts\python -c "import torch;print('Torch:',torch.__version__);print('CUDA build:',getattr(torch.version,'cuda',None));print('CUDA available:',torch.cuda.is_available())" - name: Clean old dist/build diff --git a/.github/workflows/release-s2p.yml b/.github/workflows/release-s2p.yml index 54b73a78..a20deea7 100644 --- a/.github/workflows/release-s2p.yml +++ b/.github/workflows/release-s2p.yml @@ -37,7 +37,7 @@ jobs: shell: cmd run: | venv\Scripts\python -m pip install --upgrade pip - venv\Scripts\pip install --constraint Deploy\constraints\pins.txt h5py pyinstaller suite2p==0.14.4 + venv\Scripts\pip install --constraint Deploy\constraints\pins-s2p.txt h5py pyinstaller suite2p==0.14.4 tbb venv\Scripts\python -c "import suite2p; print('Suite2p version:', getattr(suite2p,'__version__','?'))" - name: Clean old dist/build diff --git a/Deploy/constraints/pins.txt b/Deploy/constraints/pins-dlc.txt similarity index 100% rename from Deploy/constraints/pins.txt rename to Deploy/constraints/pins-dlc.txt diff --git a/Deploy/constraints/pins-s2p.txt b/Deploy/constraints/pins-s2p.txt new file mode 100644 index 00000000..d4467595 --- /dev/null +++ b/Deploy/constraints/pins-s2p.txt @@ -0,0 +1,4 @@ +llvmlite==0.44.0 +numba==0.61.2 +numpy==1.26.4 +tbb==2022.3.0 From 02e9d07f40e099cc9fd480a91329a1a1f30e2e3a Mon Sep 17 00:00:00 2001 From: katemartian Date: Fri, 14 Nov 2025 14:37:51 -0500 Subject: [PATCH 84/86] Remove redundnt spaces --- Deploy/pack_suite2p.spec | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Deploy/pack_suite2p.spec b/Deploy/pack_suite2p.spec index 32b34596..db8fdf72 100644 --- a/Deploy/pack_suite2p.spec +++ b/Deploy/pack_suite2p.spec @@ -16,10 +16,10 @@ PACKAGES = [ ] EXCLUDES = [ - "IPython", - "PyQt6", - "PyQt5", - "Markdown", + "IPython", + "PyQt6", + "PyQt5", + "Markdown", "jupyter" ] From f5bb2a5a97f00f376b7bbdfb9723759e62fb101a Mon Sep 17 00:00:00 2001 From: katemartian Date: Fri, 14 Nov 2025 15:02:49 -0500 Subject: [PATCH 85/86] Copy release zips to the shared directory on the workstation --- .github/workflows/release-cmn.yml | 19 +++++++++++++++++++ .github/workflows/release-dlc.yml | 19 +++++++++++++++++++ .github/workflows/release-mnn.yml | 19 +++++++++++++++++++ .github/workflows/release-s2p.yml | 19 +++++++++++++++++++ 4 files changed, 76 insertions(+) diff --git a/.github/workflows/release-cmn.yml b/.github/workflows/release-cmn.yml index bfeeba2f..7d037421 100644 --- a/.github/workflows/release-cmn.yml +++ b/.github/workflows/release-cmn.yml @@ -125,6 +125,25 @@ jobs: echo !HASH!> caiman-%VERSION%.zip.sha256 type caiman-%VERSION%.zip.sha256 + - name: Copy release zip + shell: cmd + working-directory: Deploy + env: + SHARE_DIR: "C:\\DoricSoftware\\danse\\Libraries\\CaImAn" + run: | + set "ZIP=caiman-%VERSION%.zip" + if not exist "%ZIP%" ( + echo ERROR: zip not found + exit /b 1 + ) + set "DEST=%SHARE_DIR%" + if not exist "%DEST%" ( + echo Creating %DEST% + mkdir "%DEST%" + ) + copy /Y "%ZIP%" "%DEST%\" + echo Copied %ZIP% to %DEST% + - name: Create GitHub Release (attach zip + checksum) uses: softprops/action-gh-release@v2 with: diff --git a/.github/workflows/release-dlc.yml b/.github/workflows/release-dlc.yml index 6a04b221..ebc78e76 100644 --- a/.github/workflows/release-dlc.yml +++ b/.github/workflows/release-dlc.yml @@ -107,6 +107,25 @@ jobs: echo !HASH!> deeplabcut-%VERSION%.zip.sha256 type deeplabcut-%VERSION%.zip.sha256 + - name: Copy release zip to network share + shell: cmd + working-directory: Deploy + env: + SHARE_DIR: "C:\\DoricSoftware\\danse\\Libraries\\DeepLabCut" + run: | + set "ZIP=deeplabcut-%VERSION%.zip" + if not exist "%ZIP%" ( + echo ERROR: zip not found + exit /b 1 + ) + set "DEST=%SHARE_DIR%" + if not exist "%DEST%" ( + echo Creating %DEST% + mkdir "%DEST%" + ) + copy /Y "%ZIP%" "%DEST%\" + echo Copied %ZIP% to %DEST% + - name: Create GitHub Release (attach zip + checksum) uses: softprops/action-gh-release@v2 with: diff --git a/.github/workflows/release-mnn.yml b/.github/workflows/release-mnn.yml index 855bd0b1..e9f10634 100644 --- a/.github/workflows/release-mnn.yml +++ b/.github/workflows/release-mnn.yml @@ -130,6 +130,25 @@ jobs: echo !HASH!> minian-%VERSION%.zip.sha256 type minian-%VERSION%.zip.sha256 + - name: Copy release zip to network share + shell: cmd + working-directory: Deploy + env: + SHARE_DIR: "C:\\DoricSoftware\\danse\\Libraries\\MiniAn" + run: | + set "ZIP=minian-%VERSION%.zip" + if not exist "%ZIP%" ( + echo ERROR: zip not found + exit /b 1 + ) + set "DEST=%SHARE_DIR%" + if not exist "%DEST%" ( + echo Creating %DEST% + mkdir "%DEST%" + ) + copy /Y "%ZIP%" "%DEST%\" + echo Copied %ZIP% to %DEST% + - name: Create GitHub Release (attach zip + checksum) uses: softprops/action-gh-release@v2 with: diff --git a/.github/workflows/release-s2p.yml b/.github/workflows/release-s2p.yml index a20deea7..171c4031 100644 --- a/.github/workflows/release-s2p.yml +++ b/.github/workflows/release-s2p.yml @@ -100,6 +100,25 @@ jobs: echo !HASH!> suite2p-%VERSION%.zip.sha256 type suite2p-%VERSION%.zip.sha256 + - name: Copy release zip to network share + shell: cmd + working-directory: Deploy + env: + SHARE_DIR: "C:\\DoricSoftware\\danse\\Libraries\\Suite2p" + run: | + set "ZIP=suite2p-%VERSION%.zip" + if not exist "%ZIP%" ( + echo ERROR: zip not found + exit /b 1 + ) + set "DEST=%SHARE_DIR%" + if not exist "%DEST%" ( + echo Creating %DEST% + mkdir "%DEST%" + ) + copy /Y "%ZIP%" "%DEST%\" + echo Copied %ZIP% to %DEST% + - name: Create GitHub Release (attach zip + checksum) uses: softprops/action-gh-release@v2 with: From 593f590a9c5ebfcbe8ff2439c05fd15b4cd2b0b9 Mon Sep 17 00:00:00 2001 From: katemartian Date: Fri, 14 Nov 2025 15:37:32 -0500 Subject: [PATCH 86/86] Collect all the tbb dlls for suite2p release --- Deploy/pack_suite2p.spec | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Deploy/pack_suite2p.spec b/Deploy/pack_suite2p.spec index db8fdf72..88929f50 100644 --- a/Deploy/pack_suite2p.spec +++ b/Deploy/pack_suite2p.spec @@ -1,6 +1,8 @@ # -*- mode: python ; coding: utf-8 -*- import os +import sys +from pathlib import Path from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT from PyInstaller.utils.hooks import collect_all @@ -32,6 +34,22 @@ for package in PACKAGES: binaries += tmp_ret[1] hiddenimports += tmp_ret[2] +# Ship Intel TBB runtime that PyTorch depends on. +TBB_DLLS = [ + "tbb12.dll", + "tbbmalloc.dll", + "tbbmalloc_proxy.dll", + "tbbbind.dll", + "tbbbind_2_0.dll", + "tbbbind_2_5.dll", +] +library_bin = Path(sys.prefix) / "Library" / "bin" +if library_bin.is_dir(): + for dll_name in TBB_DLLS: + dll_path = library_bin / dll_name + if dll_path.is_file(): + binaries.append((str(dll_path), ".")) + a_suite2p = Analysis( ['../Suite2p/suite2p_run.py'], pathex=['../'],