merge: sync with upstream main and resolve experiment.py conflict

Author: binary69

.github/workflows/ruff.yml

@@ -0,0 +1,29 @@
+name: CI - Ruff Linting
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  ruff:
+    name: Ruff Linting
+    runs-on: ubuntu-latest
+    if: |
+      github.event_name == 'push' ||
+      (
+        github.event_name == 'pull_request' &&
+        github.repository != github.event.pull_request.head.repo.full_name
+      )
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Ruff
+        uses: astral-sh/ruff-action@v3
+        with:
+          version: "0.15.6"
+          args: "check"

.gitignore

@@ -129,6 +129,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+*env/
 
 # Spyder project settings
 .spyderproject
@@ -164,4 +165,4 @@ cython_debug/
 *.sif
 *.bak
 
-docs/source/generated/
+docs/source/generated/

docs/source/conf.py

@@ -78,7 +78,6 @@
     "cv2",
     "pandas",
     "hydra",
-    "omegaconf",
     "jaxtyping",
     "plotly",
     "optree",

examples/dandi_experanto_example.ipynb

@@ -22,7 +22,6 @@
     "\n",
     "# Standard imports\n",
     "import sys\n",
-    "import os\n",
     "from pathlib import Path\n",
     "\n",
     "# Add project root to path if needed\n",
@@ -45,13 +44,9 @@
    "outputs": [],
    "source": [
     "import matplotlib.pyplot as plt\n",
-    "from os import path\n",
     "\n",
-    "from tqdm import tqdm\n",
-    "import torch\n",
-    "from omegaconf import OmegaConf, open_dict\n",
+    "from omegaconf import OmegaConf\n",
     "\n",
-    "from experanto.datasets import ChunkDataset\n",
     "from experanto.dataloaders import get_multisession_dataloader"
    ]
   },
@@ -198,7 +193,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "3eb3e584-077f-4124-abb2-4fb8d43070c0",
    "metadata": {},
    "outputs": [
@@ -211,7 +206,7 @@
     }
    ],
    "source": [
-    "ls /data/test_upsampling_without_hamming_30.0Hz/dynamic29515-10-12-Video-021a75e56847d574b9acbcc06c675055_30hz"
+    "!ls /data/test_upsampling_without_hamming_30.0Hz/dynamic29515-10-12-Video-021a75e56847d574b9acbcc06c675055_30hz"
    ]
   },
   {
@@ -238,7 +233,6 @@
     }
    ],
    "source": [
-    "from experanto.dataloaders import get_multisession_dataloader\n",
     "\n",
     "paths = [\"/data/test_upsampling_without_hamming_30.0Hz/dynamic29515-10-12-Video-021a75e56847d574b9acbcc06c675055_30hz\"]\n",
     "train_dl = get_multisession_dataloader(paths, cfg)"
@@ -321,7 +315,7 @@
     "fig, axs = plt.subplots(1, 4, figsize=(10, 2))\n",
     "for i, ax in enumerate(axs.ravel()):\n",
     "    ax.imshow(batch[\"screen\"][i, 0, 0])  # video frames that the mouse sees\n",
-    "    ax.axis(\"off\");"
+    "    ax.axis(\"off\")"
    ]
   },
   {

examples/demo.ipynb

@@ -14,7 +14,6 @@
     "import numpy as np\n",
     "import matplotlib.pyplot as plt\n",
     "import matplotlib.animation as animation\n",
-    "from pathlib import Path\n",
     "from IPython.display import HTML"
    ]
   },
@@ -33,9 +32,7 @@
     "if p not in sys.path:\n",
     "    sys.path.append(p)\n",
     "\n",
-    "from experanto.experiment import Experiment\n",
-    "from experanto.interpolators import Interpolator\n",
-    "from experanto.interpolators import ScreenInterpolator"
+    "from experanto.experiment import Experiment # noqa: E402"
    ]
   },
   {
@@ -93,7 +90,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "plt.plot(data[\"running_speed\"], \"o-k\")"
+    "plt.plot(data[\"running_speed\"], \"o-k\") # noqa: F821 - `data` is defined in prior cell"
    ]
   },
   {
@@ -103,8 +100,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "vid1 = e[b : b + time_steps][\"screen\"]\n",
-    "vid2 = e[b : b + time_steps, \"screen\"]\n",
+    "vid1 = e[b : b + time_steps][\"screen\"] # noqa: F821 - `b` is defined in prior cell\n",
+    "vid2 = e[b : b + time_steps, \"screen\"] # noqa: F821\n",
     "np.sum(vid1 - vid2)"
    ]
   },

examples/sensorium/data.ipynb

@@ -30,7 +30,7 @@
     "    sys.path.append(p)\n",
     "\n",
     "# from experanto.experiment import ImageInterpolator\n",
-    "from experanto.interpolators import ImageInterpolator"
+    "from experanto.interpolators import ImageInterpolator # noqa: E402"
    ]
   },
   {

examples/sensorium/experiment.ipynb

@@ -48,13 +48,13 @@
    "source": [
     "import yaml\n",
     "\n",
-    "meta = dict(\n",
-    "    modality=\"time_series\",\n",
-    "    start_time=float(si.timestamps[0]),\n",
-    "    end_time=float(si.timestamps[-1]),\n",
-    "    time_delta=float((si.timestamps[-1] - si.timestamps[0]) / (len(si.timestamps) - 1)),\n",
-    "    phase_shift_per_signal=False,\n",
-    ")\n",
+    "meta = {\n",
+    "    \"modality\": \"time_series\",\n",
+    "    \"start_time\": float(si.timestamps[0]),\n",
+    "    \"end_time\": float(si.timestamps[-1]),\n",
+    "    \"time_delta\": float((si.timestamps[-1] - si.timestamps[0]) / (len(si.timestamps) - 1)),\n",
+    "    \"phase_shift_per_signal\": False,\n",
+    "}\n",
     "\n",
     "with open(\"/Users/fabee/Data/sinzlab-data/dataset0/eye_tracker/meta.yml\", \"w\") as f:\n",
     "    yaml.safe_dump(meta, f)"

examples/sensorium/images.ipynb

@@ -32,7 +32,6 @@
     "import numpy as np\n",
     "from tqdm import tqdm\n",
     "import matplotlib.pyplot as plt\n",
-    "from torch.utils.data import DataLoader\n",
     "from collections import OrderedDict"
    ]
   },
@@ -429,80 +428,78 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": null,
    "id": "70c2c2fa",
    "metadata": {},
    "outputs": [],
    "source": [
     "seed = 42\n",
-    "import sys\n",
+    "import sys # noqa: E402\n",
     "\n",
     "sys.path.append(\"/srv/user/turishcheva/sensorium_replicate/sensorium_2023/\")\n",
     "sys.path.append(\"/srv/user/turishcheva/sensorium_replicate/neuralpredictors/\")\n",
-    "import torch\n",
-    "from nnfabrik.utility.nn_helpers import set_random_seed\n",
+    "import torch # noqa: E402\n",
+    "from nnfabrik.utility.nn_helpers import set_random_seed # noqa: E402\n",
     "\n",
     "set_random_seed(seed)\n",
     "\n",
-    "from sensorium.datasets.mouse_video_loaders import mouse_video_loader\n",
-    "from sensorium.utility.scores import get_correlations\n",
-    "from nnfabrik.builder import get_trainer\n",
-    "from sensorium.models.make_model import make_video_model"
+    "from nnfabrik.builder import get_trainer # noqa: E402\n",
+    "from sensorium.models.make_model import make_video_model # noqa: E402"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": null,
    "id": "78705901",
    "metadata": {},
    "outputs": [],
    "source": [
-    "factorised_3D_core_dict = dict(\n",
-    "    input_channels=1,  # increase if behaviour is used\n",
-    "    hidden_channels=[32, 64, 128],\n",
-    "    spatial_input_kernel=(11, 11),\n",
-    "    temporal_input_kernel=11,\n",
-    "    spatial_hidden_kernel=(5, 5),\n",
-    "    temporal_hidden_kernel=5,\n",
-    "    stride=1,\n",
-    "    layers=3,\n",
-    "    gamma_input_spatial=10,\n",
-    "    gamma_input_temporal=0.01,\n",
-    "    bias=True,\n",
-    "    hidden_nonlinearities=\"elu\",\n",
-    "    x_shift=0,\n",
-    "    y_shift=0,\n",
-    "    batch_norm=True,\n",
-    "    laplace_padding=None,\n",
-    "    input_regularizer=\"LaplaceL2norm\",\n",
-    "    padding=False,\n",
-    "    final_nonlin=True,\n",
-    "    momentum=0.7,\n",
-    ")\n",
+    "factorised_3D_core_dict = {\n",
+    "    \"input_channels\": 1,  # increase if behaviour is used\n",
+    "    \"hidden_channels\": [32, 64, 128],\n",
+    "    \"spatial_input_kernel\": (11, 11),\n",
+    "    \"temporal_input_kernel\": 11,\n",
+    "    \"spatial_hidden_kernel\": (5, 5),\n",
+    "    \"temporal_hidden_kernel\": 5,\n",
+    "    \"stride\": 1,\n",
+    "    \"layers\": 3,\n",
+    "    \"gamma_input_spatial\": 10,\n",
+    "    \"gamma_input_temporal\": 0.01,\n",
+    "    \"bias\": True,\n",
+    "    \"hidden_nonlinearities\": \"elu\",\n",
+    "    \"x_shift\": 0,\n",
+    "    \"y_shift\": 0,\n",
+    "    \"batch_norm\": True,\n",
+    "    \"laplace_padding\": None,\n",
+    "    \"input_regularizer\": \"LaplaceL2norm\",\n",
+    "    \"padding\": False,\n",
+    "    \"final_nonlin\": True,\n",
+    "    \"momentum\": 0.7,\n",
+    "}\n",
     "\n",
     "\n",
     "shifter_dict = None\n",
     "\n",
     "\n",
-    "readout_dict = dict(\n",
-    "    bias=True,\n",
-    "    init_mu_range=0.2,\n",
-    "    init_sigma=1.0,\n",
-    "    gamma_readout=0.0,\n",
-    "    gauss_type=\"full\",\n",
-    "    #     grid_mean_predictor=None,\n",
-    "    grid_mean_predictor={\n",
+    "readout_dict = {\n",
+    "    \"bias\": True,\n",
+    "    \"init_mu_range\": 0.2,\n",
+    "    \"init_sigma\": 1.0,\n",
+    "    \"gamma_readout\": 0.0,\n",
+    "    \"gauss_type\": \"full\",\n",
+    "    # grid_mean_predictor=None,\n",
+    "    \"grid_mean_predictor\": {\n",
     "        \"type\": \"cortex\",\n",
     "        \"input_dimensions\": 2,\n",
     "        \"hidden_layers\": 1,\n",
     "        \"hidden_features\": 30,\n",
     "        \"final_tanh\": True,\n",
     "    },\n",
-    "    share_features=False,\n",
-    "    share_grid=False,\n",
-    "    shared_match_ids=None,\n",
-    "    gamma_grid_dispersion=0.0,\n",
-    ")"
+    "    \"share_features\": False,\n",
+    "    \"share_grid\": False,\n",
+    "    \"shared_match_ids\": None,\n",
+    "    \"gamma_grid_dispersion\": 0.0,\n",
+    "}"
    ]
   },
   {
@@ -744,7 +741,6 @@
    "source": [
     "import yaml\n",
     "import os\n",
-    "from tqdm import tqdm\n",
     "from collections import Counter"
    ]
   },
@@ -763,6 +759,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import pickle\n",
+    "\n",
     "path_to_old = ...\n",
     "path_to_new = ...\n",
     "path_to_save_matching = ...\n",
@@ -782,7 +780,7 @@
     "    )\n",
     "\n",
     "    for file in tqdm(os.listdir(yaml_pre_path)):\n",
-    "        with open(f\"{yaml_pre_path}{file}\", \"r\") as f:\n",
+    "        with open(f\"{yaml_pre_path}{file}\") as f:\n",
     "            data = yaml.safe_load(f)\n",
     "        if data[\"modality\"] != \"blank\":\n",
     "            if len(np.where(trial_idx_prev == data[\"trial_idx\"])[0]) == 0:\n",
@@ -817,6 +815,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import pickle\n",
+    "\n",
     "for m in [\n",
     "    \"dynamic29623-4-9-Video-full\",\n",
     "    \"dynamic29647-19-8-Video-full\",\n",
@@ -845,7 +845,7 @@
     "    trial_idx_prev = np.asarray(trial_idx_prev)\n",
     "\n",
     "    for file in tqdm(os.listdir(yaml_pre_path)):\n",
-    "        with open(f\"{yaml_pre_path}{file}\", \"r\") as f:\n",
+    "        with open(f\"{yaml_pre_path}{file}\") as f:\n",
     "            data = yaml.safe_load(f)\n",
     "        if data[\"modality\"] != \"blank\":\n",
     "            if len(np.where(trial_idx_prev == data[\"trial_idx\"])[0]) == 0:\n",