diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index f31fb09ce..99e5adab4 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -51,7 +51,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ["3.10", "3.13"] # Oldest and newest supported versions + python-version: ["3.11", "3.13"] # Oldest and newest supported versions steps: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 @@ -129,14 +129,14 @@ jobs: # # 8 checks: # for each machine type (ubuntu, macos, windows): - # 3.10 / mne-stable / full / validator-stable + # 3.11 / mne-stable / full / validator-stable # 3.13 / mne-stable / full / validator-stable # # 4 additional checks with alternative MNE-Python and BIDS validator versions: # ubuntu / 3.13 / mne-main / full / validator-main --> to test cutting edge of everything - # ubuntu / 3.10 / mne-prev / full / validator-stable --> to test last supported of everything + # ubuntu / 3.11 / mne-prev / full / validator-stable --> to test last supported of everything # ubuntu / 3.13 / mne-stable / minimal / validator-stable --> to test a minimal installation - # ubuntu / 3.12 / mne-stable / full / validator-main-schema --> to test next-gen BIDS validator + # ubuntu / 3.13 / mne-stable / full / validator-main-schema --> to test next-gen BIDS validator # macos-15-intel / 3.13 / mne-stable / full / validator-stable --> to test macos x86_64 timeout-minutes: 60 runs-on: ${{ matrix.os }} @@ -144,7 +144,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.10", "3.13"] # Oldest and newest supported versions + python-version: ["3.11", "3.13"] # Oldest and newest supported versions mne-version: [mne-stable] mne-bids-install: [full] bids-validator-version: [validator-stable] @@ -159,7 +159,7 @@ jobs: bids-validator-version: validator-main # Test previous MNE stable version - os: ubuntu-latest - python-version: "3.10" + python-version: "3.11" mne-version: mne-prev-stable mne-bids-install: full bids-validator-version: validator-stable @@ -171,7 +171,7 @@ jobs: bids-validator-version: validator-stable # Test next gen BIDS schema validator - os: ubuntu-latest - python-version: "3.12" + python-version: "3.13" mne-version: mne-stable mne-bids-install: full bids-validator-version: validator-main-schema diff --git a/doc/install.rst b/doc/install.rst index f79f7af10..ccd25479b 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -32,7 +32,7 @@ Optional: We recommend installing ``mne-bids`` into an isolated Python environment, for example created via ``conda`` (may be obtained through `miniconda `_). -We require that you **use Python 3.10 or higher**. +We require that you **use Python 3.11 or higher**. You may choose to install ``mne-bids`` into your isolated Python environment `via pip <#installation-via-pip>`_ or `via conda <#installation-via-conda>`_. diff --git a/doc/whats_new.rst b/doc/whats_new.rst index ae32abe1b..cbfb6e433 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -51,6 +51,7 @@ Detailed list of changes 🛠 Requirements ^^^^^^^^^^^^^^^ +- MNE-BIDS now requires Python 3.11 or higher. - Including ``filelock`` as a dependency to handle atomic file writing and parallel processing support by `Bruno Aristimunha`_ (:gh:`1451`) - None yet @@ -63,6 +64,7 @@ Detailed list of changes - Made the lock helpers skip reference counting when the optional ``filelock`` dependency is missing, preventing spurious ``AttributeError`` crashes during reads, by `Bruno Aristimunha`_ (:gh:`1469`) - Fixed a bug in :func:`mne_bids.read_raw_bids` that caused it to fail when reading BIDS datasets where the acquisition time was specified in local time rather than UTC only in Windows, by `Bruno Aristimunha`_ (:gh:`1452`) - Fixed bug in :func:`~mne_bids.write_raw_bids` where incorrect unit was sometimes written into ``channels.tsv`` file when converting data to BrainVision, EDF, or EEGLAB formats, by `Daniel McCloy`_ (:gh:`1475`) +- Converting data to EDF format while also anonymizing it will now yield valid EDF files (with ``startdate`` set to 1985-01-01 00:00:00); the BIDS-compliant anonymized date will still be stored in ``scans.tsv`` file and restored to the in-memory Raw object when read using :func:`~mne_bids.read_raw_bids`, by `Daniel McCloy`_ (:gh:`1479`) ⚕️ Code health ^^^^^^^^^^^^^^ diff --git a/examples/convert_empty_room.py b/examples/convert_empty_room.py index 7d522a015..ffa2b0f44 100644 --- a/examples/convert_empty_room.py +++ b/examples/convert_empty_room.py @@ -29,7 +29,7 @@ # Let us first import mne_bids. import shutil -from datetime import datetime, timezone +from datetime import UTC, datetime import mne from mne.datasets import sample @@ -98,7 +98,7 @@ for date in dates: er_bids_path.update(session=date) er_meas_date = datetime.strptime(date, "%Y%m%d") - er_raw.set_meas_date(er_meas_date.replace(tzinfo=timezone.utc)) + er_raw.set_meas_date(er_meas_date.replace(tzinfo=UTC)) write_raw_bids(er_raw, er_bids_path, overwrite=True) # %% diff --git a/mne_bids/read.py b/mne_bids/read.py index d2d4d736c..979db4e5d 100644 --- a/mne_bids/read.py +++ b/mne_bids/read.py @@ -6,7 +6,7 @@ import json import os import re -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from difflib import get_close_matches from pathlib import Path @@ -417,14 +417,14 @@ def _handle_scans_reading(scans_fname, raw, bids_path): if acq_time_is_utc: # Enforce setting timezone to UTC without additonal conversion - acq_time = acq_time.replace(tzinfo=timezone.utc) + acq_time = acq_time.replace(tzinfo=UTC) else: # Convert time offset to UTC if acq_time.tzinfo is None: # Windows needs an explicit local tz for naive, pre-epoch times. - local_tz = datetime.now().astimezone().tzinfo or timezone.utc + local_tz = datetime.now().astimezone().tzinfo or UTC acq_time = acq_time.replace(tzinfo=local_tz) - acq_time = acq_time.astimezone(timezone.utc) + acq_time = acq_time.astimezone(UTC) logger.debug(f"Loaded {scans_fname} scans file to set acq_time as {acq_time}.") # First set measurement date to None and then call call anonymize() to diff --git a/mne_bids/tests/test_path.py b/mne_bids/tests/test_path.py index b320eb61b..bb8505e8f 100644 --- a/mne_bids/tests/test_path.py +++ b/mne_bids/tests/test_path.py @@ -8,7 +8,7 @@ import shutil import shutil as sh import timeit -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path import mne @@ -1435,7 +1435,7 @@ def test_find_empty_room(return_bids_test_dir, tmp_path): for date in dates: er_bids_path.update(session=date) er_meas_date = datetime.strptime(date, "%Y%m%d") - er_meas_date = er_meas_date.replace(tzinfo=timezone.utc) + er_meas_date = er_meas_date.replace(tzinfo=UTC) er_raw.set_meas_date(er_meas_date) write_raw_bids(er_raw, er_bids_path, verbose=False) @@ -1460,8 +1460,8 @@ def test_find_empty_room(return_bids_test_dir, tmp_path): bids_root = tmp_path / "associated-empty-room" bids_root.mkdir() raw = _read_raw_fif(raw_fname) - meas_date = datetime(year=2020, month=1, day=10, tzinfo=timezone.utc) - er_date = datetime(year=2010, month=1, day=1, tzinfo=timezone.utc) + meas_date = datetime(year=2020, month=1, day=10, tzinfo=UTC) + er_date = datetime(year=2010, month=1, day=1, tzinfo=UTC) raw.set_meas_date(meas_date) er_raw_matching_date = er_raw.copy().set_meas_date(meas_date) @@ -1543,7 +1543,7 @@ def test_find_emptyroom_ties(tmp_path): ) er_dir = er_dir_path.mkdir().directory - meas_date = datetime.strptime(session, "%Y%m%d").replace(tzinfo=timezone.utc) + meas_date = datetime.strptime(session, "%Y%m%d").replace(tzinfo=UTC) raw = _read_raw_fif(raw_fname) diff --git a/mne_bids/tests/test_read.py b/mne_bids/tests/test_read.py index 7d273e34a..a19685714 100644 --- a/mne_bids/tests/test_read.py +++ b/mne_bids/tests/test_read.py @@ -13,7 +13,7 @@ import shutil as sh from collections import OrderedDict from contextlib import nullcontext -from datetime import date, datetime, timezone +from datetime import UTC, date, datetime from pathlib import Path import mne @@ -108,7 +108,7 @@ def _make_parallel_raw(subject, *, seed=None): info = mne.create_info(["MEG0113"], 100, ch_types="mag") data = rng.standard_normal((1, 100)) * 1e-12 raw = mne.io.RawArray(data, info) - raw.set_meas_date(datetime(2020, 1, 1, tzinfo=timezone.utc)) + raw.set_meas_date(datetime(2020, 1, 1, tzinfo=UTC)) raw.info["line_freq"] = 60 raw.info["subject_info"] = { "his_id": subject, @@ -802,7 +802,7 @@ def test_handle_scans_reading(tmp_path): scans_tsv = _from_tsv(scans_path) acq_time_str = scans_tsv["acq_time"][0] acq_time = datetime.strptime(acq_time_str, "%Y-%m-%dT%H:%M:%S.%fZ") - acq_time = acq_time.replace(tzinfo=timezone.utc) + acq_time = acq_time.replace(tzinfo=UTC) new_acq_time = acq_time_str.split(".")[0] + "Z" assert acq_time == raw_01.info["meas_date"] scans_tsv["acq_time"][0] = new_acq_time @@ -813,7 +813,7 @@ def test_handle_scans_reading(tmp_path): raw_02 = read_raw_bids(bids_path) new_acq_time = new_acq_time.replace("Z", ".0Z") new_acq_time = datetime.strptime(new_acq_time, "%Y-%m-%dT%H:%M:%S.%fZ") - new_acq_time = new_acq_time.replace(tzinfo=timezone.utc) + new_acq_time = new_acq_time.replace(tzinfo=UTC) assert raw_02.info["meas_date"] == new_acq_time assert new_acq_time != raw_01.info["meas_date"] @@ -833,7 +833,7 @@ def test_handle_scans_reading(tmp_path): # from the original date and the same as the newly altered date raw_03 = read_raw_bids(bids_path) new_acq_time = datetime.strptime(new_acq_time_str, date_format) - assert raw_03.info["meas_date"] == new_acq_time.astimezone(timezone.utc) + assert raw_03.info["meas_date"] == new_acq_time.astimezone(UTC) # Regression for naive, pre-epoch acquisition times (Windows bug GH-1399) pre_epoch_str = "1950-06-15T13:45:30" @@ -842,10 +842,8 @@ def test_handle_scans_reading(tmp_path): raw_pre_epoch = read_raw_bids(bids_path) pre_epoch_naive = datetime.strptime(pre_epoch_str, "%Y-%m-%dT%H:%M:%S") - local_tz = datetime.now().astimezone().tzinfo or timezone.utc - expected_pre_epoch = pre_epoch_naive.replace(tzinfo=local_tz).astimezone( - timezone.utc - ) + local_tz = datetime.now().astimezone().tzinfo or UTC + expected_pre_epoch = pre_epoch_naive.replace(tzinfo=local_tz).astimezone(UTC) assert raw_pre_epoch.info["meas_date"] == expected_pre_epoch if raw_pre_epoch.annotations.orig_time is not None: assert raw_pre_epoch.annotations.orig_time == expected_pre_epoch diff --git a/mne_bids/tests/test_write.py b/mne_bids/tests/test_write.py index d214af262..82fef4626 100644 --- a/mne_bids/tests/test_write.py +++ b/mne_bids/tests/test_write.py @@ -15,7 +15,7 @@ import time import warnings from concurrent.futures import ProcessPoolExecutor -from datetime import date, datetime, timedelta, timezone +from datetime import UTC, date, datetime, timedelta from glob import glob from pathlib import Path @@ -24,7 +24,7 @@ import pandas as pd import pytest from mne.datasets import testing -from mne.io import anonymize_info +from mne.io import anonymize_info, read_raw_edf from mne.io.constants import FIFF from mne.io.kit.kit import get_kit_info from mne.utils import check_version @@ -115,7 +115,7 @@ def _make_parallel_raw(subject, *, seed=None): info = mne.create_info(["MEG0113"], 100, ch_types="mag") data = rng.standard_normal((1, 100)) * 1e-12 raw = mne.io.RawArray(data, info, verbose=False) - raw.set_meas_date(datetime(2020, 1, 1, tzinfo=timezone.utc)) + raw.set_meas_date(datetime(2020, 1, 1, tzinfo=UTC)) raw.info["line_freq"] = 60 raw.info["subject_info"] = { "his_id": subject, @@ -267,7 +267,7 @@ def test_write_participants(_bids_validate, tmp_path): raw = _read_raw_fif(raw_fname) # add fake participants data - raw.set_meas_date(datetime(year=1994, month=1, day=26, tzinfo=timezone.utc)) + raw.set_meas_date(datetime(year=1994, month=1, day=26, tzinfo=UTC)) birthday = (1993, 1, 26) birthday = date(*birthday) raw.info["subject_info"] = { @@ -478,10 +478,10 @@ def test_stamp_to_dt(): """Test conversions of meas_date to datetime objects.""" meas_date = (1346981585, 835782) meas_datetime = _stamp_to_dt(meas_date) - assert meas_datetime == datetime(2012, 9, 7, 1, 33, 5, 835782, tzinfo=timezone.utc) + assert meas_datetime == datetime(2012, 9, 7, 1, 33, 5, 835782, tzinfo=UTC) meas_date = (1346981585,) meas_datetime = _stamp_to_dt(meas_date) - assert meas_datetime == datetime(2012, 9, 7, 1, 33, 5, 0, tzinfo=timezone.utc) + assert meas_datetime == datetime(2012, 9, 7, 1, 33, 5, 0, tzinfo=UTC) @testing.requires_testing_data @@ -688,7 +688,7 @@ def test_fif(_bids_validate, tmp_path): raw = _read_raw_fif(raw_fname) meas_date = raw.info["meas_date"] if not isinstance(meas_date, datetime): - meas_date = datetime.fromtimestamp(meas_date[0], tz=timezone.utc) + meas_date = datetime.fromtimestamp(meas_date[0], tz=UTC) er_date = meas_date.strftime("%Y%m%d") er_bids_path = BIDSPath( subject="emptyroom", session=er_date, task="noise", root=bids_root @@ -3457,6 +3457,43 @@ def test_convert_eeg_formats(dir_name, fmt, fname, reader, tmp_path): assert_array_almost_equal(raw.get_data(), raw2.get_data()[:, :orig_len], decimal=6) +@testing.requires_testing_data +def test_anonymize_and_convert_to_edf(tmp_path): + """Test anonymization if converting to EDF (different codepath than EDF → EDF).""" + bids_root = tmp_path / "EDF" + raw_fname = data_path / "NihonKohden" / "MB0400FU.EEG" + bids_path = _bids_path.copy().update(root=bids_root, datatype="eeg") + raw = _read_raw_nihon(raw_fname) + + with ( + pytest.warns(RuntimeWarning, match="limits `startdate` to dates after 1985"), + pytest.warns(RuntimeWarning, match="Converting data files to EDF format"), + ): + bids_output_path = write_raw_bids( + raw=raw, + format="EDF", + bids_path=bids_path, + overwrite=True, + verbose=False, + anonymize=dict(daysback=41234), + ) + # make sure the written EDF file is valid + direct_read = read_raw_edf(bids_output_path.fpath) + assert direct_read.info["meas_date"] == datetime(1985, 1, 1, tzinfo=UTC) + # make sure MNE-BIDS replaced the 1985-1-1 date with the date from scans.tsv + bids_read = read_raw_bids(bids_output_path) + bids_output_path.update( + suffix="scans", + extension=".tsv", + task=None, + acquisition=None, + run=None, + datatype=None, + ) + scans = _from_tsv(bids_output_path.fpath) + assert bids_read.info["meas_date"] == datetime.fromisoformat(scans["acq_time"][0]) + + @pytest.mark.parametrize("dir_name, fmt, fname, reader", test_converteeg_data) @pytest.mark.filterwarnings( warning_str["channel_unit_changed"], @@ -3699,7 +3736,7 @@ def test_write_associated_emptyroom(_bids_validate, tmp_path, empty_room_dtype): bids_root = tmp_path / "bids1" raw_fname = data_path / "MEG" / "sample" / "sample_audvis_trunc_raw.fif" raw = _read_raw_fif(raw_fname) - meas_date = datetime(year=2020, month=1, day=10, tzinfo=timezone.utc) + meas_date = datetime(year=2020, month=1, day=10, tzinfo=UTC) if empty_room_dtype == "BIDSPath": # First write "empty-room" data diff --git a/mne_bids/utils.py b/mne_bids/utils.py index 0a6434464..73d505cbc 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -6,7 +6,7 @@ import json import os import re -from datetime import date, datetime, timedelta, timezone +from datetime import UTC, date, datetime, timedelta from pathlib import Path import numpy as np @@ -469,7 +469,7 @@ def _stamp_to_dt(utc_stamp): stamp = [int(s) for s in utc_stamp] if len(stamp) == 1: # In case there is no microseconds information stamp.append(0) - return datetime.fromtimestamp(0, tz=timezone.utc) + timedelta( + return datetime.fromtimestamp(0, tz=UTC) + timedelta( 0, stamp[0], stamp[1] ) # day, sec, μs diff --git a/mne_bids/write.py b/mne_bids/write.py index 835a42b92..418aed1a1 100644 --- a/mne_bids/write.py +++ b/mne_bids/write.py @@ -12,7 +12,7 @@ import sys import warnings from collections import OrderedDict, defaultdict -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from pathlib import Path import mne @@ -551,11 +551,11 @@ def _participants_tsv(raw, subject_id, fname, overwrite=False): meas_date = meas_date[0] if meas_date is not None and age is not None: - bday = datetime(age.year, age.month, age.day, tzinfo=timezone.utc) + bday = datetime(age.year, age.month, age.day, tzinfo=UTC) if isinstance(meas_date, datetime): meas_datetime = meas_date else: - meas_datetime = datetime.fromtimestamp(meas_date, tz=timezone.utc) + meas_datetime = datetime.fromtimestamp(meas_date, tz=UTC) subject_age = _age_on_date(bday, meas_datetime) else: subject_age = "n/a" @@ -1338,7 +1338,7 @@ def _write_raw_brainvision(raw, bids_fname, events, overwrite): ) -def _write_raw_edf(raw, bids_fname, overwrite): +def _write_raw_edf_bdf(raw, bids_fname, overwrite): """Store data as EDF. Parameters @@ -1350,7 +1350,23 @@ def _write_raw_edf(raw, bids_fname, overwrite): overwrite : bool Whether to overwrite an existing file or not. """ - assert str(bids_fname).endswith(".edf") + ext = bids_fname.suffix[1:].upper() + assert ext in ("EDF", "BDF") + if raw.info["meas_date"] is not None and raw.info["meas_date"].year < 1985: + warn( + f"Attempting to write a {ext} file with a meas_date of " + f"{raw.info['meas_date']}. This is not supported; {ext} limits `startdate` " + "to dates after 1985-01-01. Setting `startdate` to 1985-01-01 00:00:00 in " + f"the {ext} file; the original anonymized date will be written to scans.tsv" + ) + # make a copy, so that anonymized meas_date is unchanged in orig raw, + # and thus scans.tsv ends up with the properly anonymized meas_date + raw = raw.copy() + raw.set_meas_date( + raw.info["meas_date"].replace( + year=1985, month=1, day=1, hour=0, minute=0, second=0, microsecond=0 + ) + ) raw.export(bids_fname, overwrite=overwrite) @@ -2013,7 +2029,7 @@ def write_raw_bids( meas_date = raw.info.get("meas_date", None) if meas_date is not None: if not isinstance(meas_date, datetime): - meas_date = datetime.fromtimestamp(meas_date[0], tz=timezone.utc) + meas_date = datetime.fromtimestamp(meas_date[0], tz=UTC) if anonymize is not None and "daysback" in anonymize: meas_date = meas_date - timedelta(anonymize["daysback"]) @@ -2361,7 +2377,7 @@ def write_raw_bids( ) elif bids_path.datatype in ["eeg", "ieeg"] and format == "EDF": warn("Converting data files to EDF format") - _write_raw_edf(raw, bids_path.fpath, overwrite=overwrite) + _write_raw_edf_bdf(raw, bids_path.fpath, overwrite=overwrite) elif bids_path.datatype in ["eeg", "ieeg"] and format == "EEGLAB": warn("Converting data files to EEGLAB format") _write_raw_eeglab(raw, bids_path.fpath, overwrite=overwrite) diff --git a/pyproject.toml b/pyproject.toml index 147bd170d..754caf85a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,13 +11,11 @@ classifiers = [ "Operating System :: MacOS", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python", "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering", "Topic :: Software Development", ] dependencies = ["mne>=1.8", "numpy>=1.23", "scipy>=1.9"] @@ -38,7 +36,7 @@ maintainers = [ ] name = "mne-bids" readme = {content-type = "text/markdown", file = "README.md"} -requires-python = ">=3.10" +requires-python = ">=3.11" scripts = {mne_bids = "mne_bids.commands.run:main"} [project.optional-dependencies]