From fe2d2b351b91ed81d6357752420cde6d5b0bf606 Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Mon, 15 Dec 2025 09:36:51 +0100 Subject: [PATCH 01/47] APIS-1444 ASR processor module (no auto-calib) --- src/explorepy/explore.py | 11 +++++++++++ src/explorepy/packet.py | 15 +++++++++++++++ src/explorepy/stream_processor.py | 14 ++++++++++++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/explorepy/explore.py b/src/explorepy/explore.py index 6ae74f53..1671383c 100644 --- a/src/explorepy/explore.py +++ b/src/explorepy/explore.py @@ -734,3 +734,14 @@ def is_bt_link_unstable(self): def get_channel_mask(self): return SettingsManager(self.device_name).get_adc_mask() + + def calibrate_asr(self, length=-1.0): + if self.stream_processor.asr_processor is not None and self.stream_processor.asr_processor.is_initialized: + self.stream_processor.asr_processor.start_calibration(length) + + def start_asr(self, window=None): + if self.stream_processor.asr_processor is not None and self.stream_processor.asr_processor.is_initialized: + if self.stream_processor.asr_processor.calibration_data_available: + self.stream_processor.asr_processor.start_cleaning(window) + + diff --git a/src/explorepy/packet.py b/src/explorepy/packet.py index 1314ba5f..2a19158e 100644 --- a/src/explorepy/packet.py +++ b/src/explorepy/packet.py @@ -175,6 +175,21 @@ def __str__(self): return f"{binascii.hexlify(bytearray(self.bin_data))}" +class CleanEEG(Packet): + def __init__(self, timestamp, payload): + self.timestamps = timestamp + self.data = payload + + def _convert(self, bin_data): + self.data = bin_data + + def get_data(self): + return self.timestamps, self.data + + def __str__(self): + return f"Timestamps: {self.timestamps}, samples: {self.data}" + + class EEG(Packet): """EEG packet class""" diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index 318618b1..c96dda05 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -14,7 +14,6 @@ ) import numpy as np - from explorepy.command import ( DeviceConfiguration, ZMeasurementDisable, @@ -25,6 +24,7 @@ EEG, CalibrationInfo, CalibrationInfo_USBC, + CleanEEG, CommandRCV, CommandStatus, DeviceInfo, @@ -44,9 +44,10 @@ is_usb_mode ) +from explorepy.asr_processor import AsrProcessor TOPICS =\ - Enum('Topics', 'raw_ExG filtered_ExG device_info marker raw_orn cmd_ack env cmd_status imp packet_bin') + Enum('Topics', 'raw_ExG filtered_ExG asr_ExG device_info marker raw_orn cmd_ack env cmd_status imp packet_bin') logger = logging.getLogger(__name__) lock = Lock() @@ -56,6 +57,7 @@ class StreamProcessor: def __init__(self, debug=False): self.parser = None + self.asr_processor = None self.filters = [] self.device_info = {} self.old_device_info = {} @@ -315,6 +317,7 @@ def process(self, packet): self._update_last_time_point(packet, received_time) self.dispatch(topic=TOPICS.raw_ExG, packet=packet) self.packet_count += 1 + if self._is_imp_mode and self.imp_calculator: packet_imp = self.imp_calculator.measure_imp( packet=copy.deepcopy(packet)) @@ -331,6 +334,12 @@ def process(self, packet): self.dispatch(topic=TOPICS.filtered_ExG, packet=packet) self.dispatch(topic=TOPICS.filtered_ExG, packet=packet) + if not self._is_imp_mode and self.imp_calculator is None: + if self.asr_processor.cleaned_data_available: + clean_packet = CleanEEG(timestamp=self.asr_processor.cleaned_data['timestamps'], + payload=self.asr_processor.cleaned_data['data']) + self.dispatch(topic=TOPICS.asr_ExG, packet=clean_packet) + self.asr_processor.clear_cleaned_data() elif isinstance(packet, DeviceInfo): self.old_device_info = self.device_info.copy() print(self.old_device_info) @@ -340,6 +349,7 @@ def process(self, packet): self.device_info["device_name"]) settings_manager.update_device_settings(packet.get_info()) self.dispatch(topic=TOPICS.device_info, packet=packet) + self.asr_processor = AsrProcessor(self, TOPICS.filtered_ExG) elif isinstance(packet, CommandRCV): self.dispatch(topic=TOPICS.cmd_ack, packet=packet) elif isinstance(packet, CommandStatus): From e7ddab193b6faaf8dfded1e4f773a598880c7f8f Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Tue, 16 Dec 2025 16:54:45 +0100 Subject: [PATCH 02/47] APIS-1443 Split calib step from process step, introduced larger buffer for ASR cleaning --- src/explorepy/explore.py | 47 +++++++++++++++++++++++++++++-- src/explorepy/stream_processor.py | 4 +-- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/explorepy/explore.py b/src/explorepy/explore.py index 1671383c..17d307e3 100644 --- a/src/explorepy/explore.py +++ b/src/explorepy/explore.py @@ -735,13 +735,56 @@ def is_bt_link_unstable(self): def get_channel_mask(self): return SettingsManager(self.device_name).get_adc_mask() + def is_asr_processor_available(self): + return self.stream_processor.asr_processor is not None and self.stream_processor.asr_processor.is_initialized + def calibrate_asr(self, length=-1.0): - if self.stream_processor.asr_processor is not None and self.stream_processor.asr_processor.is_initialized: + if self.is_asr_processor_available(): self.stream_processor.asr_processor.start_calibration(length) + def is_asr_calibration_data_available(self): + if self.is_asr_processor_available(): + return self.stream_processor.asr_processor.calibration_data_available + else: + return False + + def is_asr_calibrating(self): + if self.is_asr_processor_available(): + return self.stream_processor.asr_processor.is_calibrating + else: + return False + + def is_asr_running(self): + if self.is_asr_processor_available(): + return self.stream_processor.asr_processor.is_cleaning + else: + return False + def start_asr(self, window=None): - if self.stream_processor.asr_processor is not None and self.stream_processor.asr_processor.is_initialized: + print("Starting asr") + if self.is_asr_processor_available(): + print("ASR processor is available") if self.stream_processor.asr_processor.calibration_data_available: + print("calibration data is available") self.stream_processor.asr_processor.start_cleaning(window) + def stop_asr(self): + if self.is_asr_processor_available(): + if self.stream_processor.asr_processor.calibration_data_available: + self.stream_processor.asr_processor.stop_cleaning() + + def set_asr_cutoff(self, new_cutoff: float): + if self.is_asr_processor_available(): + self.stream_processor.asr_processor.cutoff = new_cutoff + + def get_asr_cutoff(self): + if self.is_asr_processor_available(): + return self.stream_processor.asr_processor.cutoff + + def set_asr_refresh_window(self, new_window): + if self.is_asr_processor_available(): + self.stream_processor.asr_processor.refresh_window = new_window + def get_asr_refresh_window(self): + if self.is_asr_processor_available(): + return self.stream_processor.asr_processor.refresh_window diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index c96dda05..1fa2ee2c 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -336,8 +336,8 @@ def process(self, packet): self.dispatch(topic=TOPICS.filtered_ExG, packet=packet) if not self._is_imp_mode and self.imp_calculator is None: if self.asr_processor.cleaned_data_available: - clean_packet = CleanEEG(timestamp=self.asr_processor.cleaned_data['timestamps'], - payload=self.asr_processor.cleaned_data['data']) + clean_packet = CleanEEG(timestamp=self.asr_processor.cleaned_data_ts, + payload=self.asr_processor.cleaned_data) self.dispatch(topic=TOPICS.asr_ExG, packet=clean_packet) self.asr_processor.clear_cleaned_data() elif isinstance(packet, DeviceInfo): From 3c6664eb9d8ff03605061489b146333776797b4f Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Thu, 15 Jan 2026 17:12:57 +0100 Subject: [PATCH 03/47] Add CSV client support for data streaming Introduced CsvClient and CsvServer classes to enable reading data from CSV sources. Updated parser to support 'csv' as a new bt_interface option and route streaming logic accordingly. This allows the system to simulate device data input from CSV for testing or offline analysis. --- src/explorepy/__init__.py | 2 +- src/explorepy/csv_client.py | 111 ++++++++++++++++++++++++++++++++++++ src/explorepy/parser.py | 15 ++++- 3 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 src/explorepy/csv_client.py diff --git a/src/explorepy/__init__.py b/src/explorepy/__init__.py index e3fc1238..bce1b38f 100644 --- a/src/explorepy/__init__.py +++ b/src/explorepy/__init__.py @@ -24,7 +24,7 @@ this = sys.modules[__name__] # TODO appropriate library -bt_interface_list = ['sdk', 'ble', 'mock', 'pyserial', 'usb'] +bt_interface_list = ['sdk', 'ble', 'mock', 'pyserial', 'usb', 'csv'] this._bt_interface = 'ble' if not sys.version_info >= (3, 6): diff --git a/src/explorepy/csv_client.py b/src/explorepy/csv_client.py new file mode 100644 index 00000000..3c972337 --- /dev/null +++ b/src/explorepy/csv_client.py @@ -0,0 +1,111 @@ +import time +from enum import Enum, auto + +import numpy as np +from pyarrow import timestamp + +from explorepy.packet import BleImpedancePacket, DeviceInfoBLE + +class ClientState(Enum): + DISCONNECTED = auto() + CONNECTED = auto() + STREAMING = auto() + STOPPED = auto() + + +class CsvClient: + def __init__(self): + self.server = CsvServer(8) + self._state = ClientState.DISCONNECTED + + # -------- state management -------- + def set_state(self, state: ClientState): + if not isinstance(state, ClientState): + raise ValueError("Invalid client state") + + self._state = state + + def get_state(self) -> ClientState: + return self._state + + # -------- device-like API -------- + def connect(self): + if self._state != ClientState.DISCONNECTED: + return False + self.set_state(ClientState.CONNECTED) + return True + + def disconnect(self): + self.set_state(ClientState.DISCONNECTED) + return True + + def start_streaming(self): + if self._state != ClientState.CONNECTED: + raise RuntimeError( + f"Cannot start streaming from state {self._state}" + ) + self.set_state(ClientState.STREAMING) + + def stop_streaming(self): + if self._state == ClientState.STREAMING: + self.set_state(ClientState.STOPPED) + + def read(self): + if self._state != ClientState.STREAMING: + if self._state == ClientState.CONNECTED: + self.set_state(ClientState.STREAMING) + device_info = self.server.read_device_info() + device_info_packet = DeviceInfoMock(timestamp=self.server.ts, payload=None) + device_info_packet.set_info(self.server.device_info_ble_32ch) + return device_info_packet + else: + self.server.ts += self.server.time_period + data = self.server.read_sample() + eeg_packet = BleImpedancePacket(timestamp=self.server.ts, payload=None) + eeg_packet.data = self.server.eeg_32 + return eeg_packet + + def write(self, bytes): + # TODO parse the bytes and hadndle the state + pass + + +class CsvServer: + def __init__(self, channel_count): + self.device_info_ble_8ch = {} + self.eeg_32 = [ + -17.83, -400000.05, -400000.05, -400000.05, + -400000.05, -400000.05, -400000.05, -400000.05, + -400000.05, -400000.05, -400000.05, -400000.05, + -400000.05, -400000.05, -400000.05, -400000.05, + -400000.05, -400000.05, -400000.05, -400000.05, + -400000.05, -400000.05, -400000.05, -400000.05, + -400000.05, -400000.05, -400000.05, -400000.05, + -400000.05, -400000.05, -400000.05, -400000.05 + ] + self.eeg_32 = np.array(self.eeg_32).reshape(32, 1) + self.device_info_ble_32ch = {'device_name': 'Explore_DABD', 'firmware_version': '9.1.0', + 'adc_mask': [1, 1, 1, 1, 1, 1, 1, 1], 'sampling_rate': 250, 'is_imp_mode': False, + 'board_id': 'PCB_304_801p2_X', 'memory_info': 1, 'max_online_sps': 250, + 'max_offline_sps': 2000} + self.time_period = 0.004 + self.ts = 193.5597 # as dummy + + def read_sample(self): + return self.ts, self.eeg_32 + + def read_device_info(self): + return self.device_info_ble_8ch + +class DeviceInfoMock(DeviceInfoBLE): + + def __init__(self, timestamp, payload, time_offset=0): + self.timestamp = timestamp + + def _convert(self, bin_data): + pass + def set_info(self, info): + self.info = info + + def get_info(self): + return self.info diff --git a/src/explorepy/parser.py b/src/explorepy/parser.py index a79c7311..f8ba72ee 100644 --- a/src/explorepy/parser.py +++ b/src/explorepy/parser.py @@ -67,18 +67,23 @@ def __init__(self, callback, progress_callback=None, mode='device', debug=True): self.total_packet_size_read = 0 self.progress = 0 self.progress_callback = progress_callback + self._generate_packet_impl = None self.header_len = 0 self.data_len = 0 def start_streaming(self, device_name, mac_address): """Start streaming data from Explore device""" self.device_name = device_name + #explorepy.set_bt_interface('csv') if is_ble_mode(): from explorepy.BLEClient import BLEClient self.stream_interface = BLEClient(device_name=device_name, mac_address=mac_address) elif explorepy.get_bt_interface() == 'mock': from explorepy.bt_mock_client import MockBtClient self.stream_interface = MockBtClient(device_name=device_name, mac_address=mac_address) + elif explorepy.get_bt_interface() == 'csv': + from explorepy.csv_client import CsvClient + self.stream_interface = CsvClient() elif is_usb_mode(): from explorepy.serial_client import SerialStream self.stream_interface = SerialStream(device_name=device_name) @@ -87,6 +92,10 @@ def start_streaming(self, device_name, mac_address): "Please use the following command to use ExplorePy with a legacy device\n" "pip install explorepy==3.2.1\n" "https://explorepy.readthedocs.io/en/latest/explore_legacy_devices\n") + if explorepy.get_bt_interface() == 'csv': + self._generate_packet_impl = self._generate_packet_from_csv + else: + self._generate_packet_impl = self._generate_packet self.stream_interface.connect() self._stream() @@ -159,7 +168,7 @@ def _stream_loop(self): asyncio.set_event_loop(asyncio.new_event_loop()) while self._do_streaming: try: - packet, packet_size = self._generate_packet() + packet, packet_size = self._generate_packet_impl() self.total_packet_size_read += packet_size self.callback(packet=packet) except ReconnectionFlowError: @@ -365,6 +374,10 @@ def get_header_bytes(self): else: return self.stream_interface.read(self.header_len) + def _generate_packet_from_csv(self): + packet = self.stream_interface.read() + return packet, 30 + class FileHandler: """Binary file handler with conditional memory mapping for improved performance""" From 99922a125bb7ad6e5e88c97b0ef12caeafd4d310 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Fri, 16 Jan 2026 15:51:04 +0100 Subject: [PATCH 04/47] Refactor CsvClient for channel count and packet size CsvClient and CsvServer now support configurable channel counts (8 or 32), with packet size tracking via a new PacketSize enum. Parser updated to instantiate CsvClient with channel count and to use packet_size for CSV packet length. Orientation packet handling and device info logic improved for mock streaming. --- src/explorepy/csv_client.py | 132 ++++++++++++++++++++++++------------ src/explorepy/parser.py | 6 +- 2 files changed, 92 insertions(+), 46 deletions(-) diff --git a/src/explorepy/csv_client.py b/src/explorepy/csv_client.py index 3c972337..7cacd2a1 100644 --- a/src/explorepy/csv_client.py +++ b/src/explorepy/csv_client.py @@ -1,10 +1,10 @@ import time from enum import Enum, auto - import numpy as np -from pyarrow import timestamp +from explorepy.packet import BleImpedancePacket, DeviceInfoBLE, OrientationV1, OrientationV2 +from pylsl import local_clock +from typing_extensions import override -from explorepy.packet import BleImpedancePacket, DeviceInfoBLE class ClientState(Enum): DISCONNECTED = auto() @@ -12,23 +12,25 @@ class ClientState(Enum): STREAMING = auto() STOPPED = auto() +class PacketSize(Enum): + EEG_8 = 40 + EEG_32 = 112 + ORN = 50 + DEVICE_INFO = 38 class CsvClient: - def __init__(self): - self.server = CsvServer(8) + def __init__(self, channel_count): + self.server = CsvServer(channel_count) self._state = ClientState.DISCONNECTED - # -------- state management -------- def set_state(self, state: ClientState): if not isinstance(state, ClientState): raise ValueError("Invalid client state") - self._state = state def get_state(self) -> ClientState: return self._state - # -------- device-like API -------- def connect(self): if self._state != ClientState.DISCONNECTED: return False @@ -41,9 +43,7 @@ def disconnect(self): def start_streaming(self): if self._state != ClientState.CONNECTED: - raise RuntimeError( - f"Cannot start streaming from state {self._state}" - ) + raise RuntimeError(f"Cannot start streaming from state {self._state}") self.set_state(ClientState.STREAMING) def stop_streaming(self): @@ -54,58 +54,104 @@ def read(self): if self._state != ClientState.STREAMING: if self._state == ClientState.CONNECTED: self.set_state(ClientState.STREAMING) - device_info = self.server.read_device_info() device_info_packet = DeviceInfoMock(timestamp=self.server.ts, payload=None) - device_info_packet.set_info(self.server.device_info_ble_32ch) + device_info_packet.packet_size = PacketSize.DEVICE_INFO + device_info_packet.set_info(self.server.device_info) return device_info_packet - else: - self.server.ts += self.server.time_period - data = self.server.read_sample() - eeg_packet = BleImpedancePacket(timestamp=self.server.ts, payload=None) - eeg_packet.data = self.server.eeg_32 - return eeg_packet + + + self.server.tick += 1 + if self.server.tick % 5 == 0: + orn_packet = OrientationMock(timestamp=self.server.ts, payload=None) + orn_packet.set_data(self.server.orn_value) + orn_packet.packet_size = PacketSize.ORN + return orn_packet + + self.server.ts += self.server.time_period + while local_clock() < self.server.ts: + continue + eeg_packet = BleImpedancePacket(timestamp=self.server.ts, payload=None) + eeg_packet.data = self.server.eeg_data + eeg_packet.packet_size = self.server.packet_size + return eeg_packet def write(self, bytes): - # TODO parse the bytes and hadndle the state pass - class CsvServer: - def __init__(self, channel_count): - self.device_info_ble_8ch = {} - self.eeg_32 = [ - -17.83, -400000.05, -400000.05, -400000.05, - -400000.05, -400000.05, -400000.05, -400000.05, - -400000.05, -400000.05, -400000.05, -400000.05, - -400000.05, -400000.05, -400000.05, -400000.05, - -400000.05, -400000.05, -400000.05, -400000.05, - -400000.05, -400000.05, -400000.05, -400000.05, - -400000.05, -400000.05, -400000.05, -400000.05, - -400000.05, -400000.05, -400000.05, -400000.05 - ] - self.eeg_32 = np.array(self.eeg_32).reshape(32, 1) - self.device_info_ble_32ch = {'device_name': 'Explore_DABD', 'firmware_version': '9.1.0', - 'adc_mask': [1, 1, 1, 1, 1, 1, 1, 1], 'sampling_rate': 250, 'is_imp_mode': False, - 'board_id': 'PCB_304_801p2_X', 'memory_info': 1, 'max_online_sps': 250, - 'max_offline_sps': 2000} - self.time_period = 0.004 - self.ts = 193.5597 # as dummy + def __init__(self, channel_count: int): + if channel_count not in (8, 32): + raise ValueError("Only 8 or 32 channels supported") + self.channel_count = channel_count + self.device_info_ble_32ch = { + 'device_name': 'Explore_DABD', + 'firmware_version': '9.1.0', + 'adc_mask': [1] * 8, + 'sampling_rate': 250, + 'is_imp_mode': False, + 'board_id': 'PCB_304_801p2_X', + 'memory_info': 1, + 'max_online_sps': 250, + 'max_offline_sps': 2000 + } + self.device_info_ble_8ch = { + 'device_name': 'Explore_AAAQ', + 'firmware_version': '7.1.0', + 'adc_mask': [1] * 8, + 'sampling_rate': 250, + 'is_imp_mode': False, + 'board_id': 'PCB_303_801E_XX', + 'memory_info': 1, + 'max_online_sps': 1000, + 'max_offline_sps': 8000 + } + self.eeg_32 = np.array([-17.83] + [-400000.05] * 31).reshape(32, 1) + self.eeg_8 = np.array([-17.83] + [-400000.05] * 7).reshape(8, 1) + if self.channel_count == 8: + self.device_info = self.device_info_ble_8ch + self.eeg_data = self.eeg_8 + self.packet_size = PacketSize.EEG_8 + else: + self.device_info = self.device_info_ble_32ch + self.eeg_data = self.eeg_32 + self.packet_size = PacketSize.EEG_32 + self.fs = 250 + self.time_period = np.round(1 / self.fs, 3) + self.ts = local_clock() + self.tick = 0 + self.orn_value = [5.002,-3.904,1001.01,420.0,-70.0,210.0,103.36,804.08,-532.0,-0.0023,-0.0028,0.0371,0.9993] def read_sample(self): - return self.ts, self.eeg_32 + return self.eeg_data def read_device_info(self): - return self.device_info_ble_8ch + return self.device_info class DeviceInfoMock(DeviceInfoBLE): - def __init__(self, timestamp, payload, time_offset=0): self.timestamp = timestamp def _convert(self, bin_data): pass + def set_info(self, info): self.info = info def get_info(self): return self.info + + +class OrientationMock(OrientationV2): + """Orientation data packet""" + + def __init__(self, timestamp, payload, time_offset=0): + self.timestamp = timestamp + + def _convert(self, bin_data): + pass + + def get_data(self, srate=None): + return [self.timestamp], self.data + + def set_data(self, data): + self.data = data diff --git a/src/explorepy/parser.py b/src/explorepy/parser.py index f8ba72ee..4c354ece 100644 --- a/src/explorepy/parser.py +++ b/src/explorepy/parser.py @@ -74,7 +74,7 @@ def __init__(self, callback, progress_callback=None, mode='device', debug=True): def start_streaming(self, device_name, mac_address): """Start streaming data from Explore device""" self.device_name = device_name - #explorepy.set_bt_interface('csv') + explorepy.set_bt_interface('csv') if is_ble_mode(): from explorepy.BLEClient import BLEClient self.stream_interface = BLEClient(device_name=device_name, mac_address=mac_address) @@ -83,7 +83,7 @@ def start_streaming(self, device_name, mac_address): self.stream_interface = MockBtClient(device_name=device_name, mac_address=mac_address) elif explorepy.get_bt_interface() == 'csv': from explorepy.csv_client import CsvClient - self.stream_interface = CsvClient() + self.stream_interface = CsvClient(8) elif is_usb_mode(): from explorepy.serial_client import SerialStream self.stream_interface = SerialStream(device_name=device_name) @@ -376,7 +376,7 @@ def get_header_bytes(self): def _generate_packet_from_csv(self): packet = self.stream_interface.read() - return packet, 30 + return packet, packet.packet_size.value class FileHandler: From 08265325018214315ef5ef38be4414d33ed6c3a7 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Fri, 23 Jan 2026 11:06:14 +0100 Subject: [PATCH 05/47] read samples from csv file in data server --- src/explorepy/csv_client.py | 80 ++++++++++++++++++++++++++++--------- src/explorepy/parser.py | 5 ++- 2 files changed, 65 insertions(+), 20 deletions(-) diff --git a/src/explorepy/csv_client.py b/src/explorepy/csv_client.py index 7cacd2a1..b06da7ed 100644 --- a/src/explorepy/csv_client.py +++ b/src/explorepy/csv_client.py @@ -1,3 +1,4 @@ +import os import time from enum import Enum, auto import numpy as np @@ -20,7 +21,12 @@ class PacketSize(Enum): class CsvClient: def __init__(self, channel_count): - self.server = CsvServer(channel_count) + print(os.getcwd()) + self.server = server = CsvServer( + channel_count=channel_count, + csv_path="../../../explore-desktop/exploredesktop/tests_ExG.csv", + loop=True +) self._state = ClientState.DISCONNECTED def set_state(self, state: ClientState): @@ -68,21 +74,44 @@ def read(self): return orn_packet self.server.ts += self.server.time_period - while local_clock() < self.server.ts: - continue + sleep_time = self.server.ts - local_clock() + if sleep_time > 0: + time.sleep(sleep_time) eeg_packet = BleImpedancePacket(timestamp=self.server.ts, payload=None) - eeg_packet.data = self.server.eeg_data + try: + eeg_packet.data = self.server.read_sample() + except StopIteration: + self.set_state(ClientState.STOPPED) + return None eeg_packet.packet_size = self.server.packet_size return eeg_packet def write(self, bytes): pass +import numpy as np +from pylsl import local_clock + + class CsvServer: - def __init__(self, channel_count: int): + def __init__(self, channel_count: int, csv_path: str, loop: bool = True): if channel_count not in (8, 32): raise ValueError("Only 8 or 32 channels supported") + self.channel_count = channel_count + self.loop = loop + self.csv_data = np.loadtxt(csv_path, delimiter=',', skiprows=1) # skip row 0 + + self.csv_data = self.csv_data[:, 1:] + if self.csv_data.shape[1] != channel_count: + print('######################', self.csv_data.shape) + raise ValueError( + f"CSV has {self.csv_data.shape[1]} columns, " + f"expected {channel_count}" + ) + + self.row_idx = 0 + self.num_rows = self.csv_data.shape[0] self.device_info_ble_32ch = { 'device_name': 'Explore_DABD', 'firmware_version': '9.1.0', @@ -94,6 +123,7 @@ def __init__(self, channel_count: int): 'max_online_sps': 250, 'max_offline_sps': 2000 } + self.device_info_ble_8ch = { 'device_name': 'Explore_AAAQ', 'firmware_version': '7.1.0', @@ -105,24 +135,38 @@ def __init__(self, channel_count: int): 'max_online_sps': 1000, 'max_offline_sps': 8000 } - self.eeg_32 = np.array([-17.83] + [-400000.05] * 31).reshape(32, 1) - self.eeg_8 = np.array([-17.83] + [-400000.05] * 7).reshape(8, 1) - if self.channel_count == 8: - self.device_info = self.device_info_ble_8ch - self.eeg_data = self.eeg_8 - self.packet_size = PacketSize.EEG_8 - else: - self.device_info = self.device_info_ble_32ch - self.eeg_data = self.eeg_32 - self.packet_size = PacketSize.EEG_32 - self.fs = 250 + + self.device_info = ( + self.device_info_ble_8ch + if channel_count == 8 + else self.device_info_ble_32ch + ) + + self.fs = self.device_info['sampling_rate'] self.time_period = np.round(1 / self.fs, 3) self.ts = local_clock() self.tick = 0 - self.orn_value = [5.002,-3.904,1001.01,420.0,-70.0,210.0,103.36,804.08,-532.0,-0.0023,-0.0028,0.0371,0.9993] + + self.packet_size = ( + PacketSize.EEG_8 if channel_count == 8 else PacketSize.EEG_32 + ) + + self.orn_value = [ + 5.002, -3.904, 1001.01, 420.0, -70.0, 210.0, + 103.36, 804.08, -532.0, -0.0023, + -0.0028, 0.0371, 0.9993 + ] def read_sample(self): - return self.eeg_data + if self.row_idx >= self.num_rows: + if not self.loop: + raise StopIteration("End of CSV reached") + self.row_idx = 0 + + sample = self.csv_data[self.row_idx] + self.row_idx += 1 + + return sample.reshape(self.channel_count, 1) def read_device_info(self): return self.device_info diff --git a/src/explorepy/parser.py b/src/explorepy/parser.py index 4c354ece..c09b8025 100644 --- a/src/explorepy/parser.py +++ b/src/explorepy/parser.py @@ -74,7 +74,6 @@ def __init__(self, callback, progress_callback=None, mode='device', debug=True): def start_streaming(self, device_name, mac_address): """Start streaming data from Explore device""" self.device_name = device_name - explorepy.set_bt_interface('csv') if is_ble_mode(): from explorepy.BLEClient import BLEClient self.stream_interface = BLEClient(device_name=device_name, mac_address=mac_address) @@ -83,7 +82,9 @@ def start_streaming(self, device_name, mac_address): self.stream_interface = MockBtClient(device_name=device_name, mac_address=mac_address) elif explorepy.get_bt_interface() == 'csv': from explorepy.csv_client import CsvClient - self.stream_interface = CsvClient(8) + device_id = str.split(device_name, '_')[1] # split by underscore and take second part + ch_count = 32 if device_id[0] == 'D' else 8 #dynamic channel selection based on first character + self.stream_interface = CsvClient(ch_count) elif is_usb_mode(): from explorepy.serial_client import SerialStream self.stream_interface = SerialStream(device_name=device_name) From 69975cb51cd65d48b4b8db7e5756f4881be36092 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Fri, 23 Jan 2026 13:31:20 +0100 Subject: [PATCH 06/47] Update CSV file paths and firmware versions Changed CSV file path construction in CsvClient to use a test-specific file based on channel count. Updated firmware versions in device info dictionaries. Added setting of BT interface to 'csv' and a debug print of current working directory in Parser. --- src/explorepy/csv_client.py | 10 +++++----- src/explorepy/parser.py | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/explorepy/csv_client.py b/src/explorepy/csv_client.py index b06da7ed..760ab873 100644 --- a/src/explorepy/csv_client.py +++ b/src/explorepy/csv_client.py @@ -1,4 +1,3 @@ -import os import time from enum import Enum, auto import numpy as np @@ -21,10 +20,11 @@ class PacketSize(Enum): class CsvClient: def __init__(self, channel_count): - print(os.getcwd()) + file_path = "../../explorepy/tests/sample_data/" + file_name = "test_" + str(channel_count) + ".csv" self.server = server = CsvServer( channel_count=channel_count, - csv_path="../../../explore-desktop/exploredesktop/tests_ExG.csv", + csv_path=file_path + file_name, loop=True ) self._state = ClientState.DISCONNECTED @@ -114,7 +114,7 @@ def __init__(self, channel_count: int, csv_path: str, loop: bool = True): self.num_rows = self.csv_data.shape[0] self.device_info_ble_32ch = { 'device_name': 'Explore_DABD', - 'firmware_version': '9.1.0', + 'firmware_version': '9.6.9', 'adc_mask': [1] * 8, 'sampling_rate': 250, 'is_imp_mode': False, @@ -126,7 +126,7 @@ def __init__(self, channel_count: int, csv_path: str, loop: bool = True): self.device_info_ble_8ch = { 'device_name': 'Explore_AAAQ', - 'firmware_version': '7.1.0', + 'firmware_version': '7.6.9', 'adc_mask': [1] * 8, 'sampling_rate': 250, 'is_imp_mode': False, diff --git a/src/explorepy/parser.py b/src/explorepy/parser.py index c09b8025..88d6a305 100644 --- a/src/explorepy/parser.py +++ b/src/explorepy/parser.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import os """Parser module""" import asyncio import binascii @@ -74,6 +75,8 @@ def __init__(self, callback, progress_callback=None, mode='device', debug=True): def start_streaming(self, device_name, mac_address): """Start streaming data from Explore device""" self.device_name = device_name + explorepy.set_bt_interface('csv') + print(os.getcwd()) if is_ble_mode(): from explorepy.BLEClient import BLEClient self.stream_interface = BLEClient(device_name=device_name, mac_address=mac_address) From 52e08efcb0c0ff5bdac0034b953dcef6bf2c7997 Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Wed, 11 Feb 2026 11:25:04 +0100 Subject: [PATCH 07/47] Changed csv_client path to artifact file, added calib setting to ASR processor --- src/explorepy/csv_client.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/explorepy/csv_client.py b/src/explorepy/csv_client.py index 760ab873..9fc4963c 100644 --- a/src/explorepy/csv_client.py +++ b/src/explorepy/csv_client.py @@ -20,8 +20,9 @@ class PacketSize(Enum): class CsvClient: def __init__(self, channel_count): - file_path = "../../explorepy/tests/sample_data/" - file_name = "test_" + str(channel_count) + ".csv" + file_path = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/" + file_name = "32channel_semidry_artefacts_ExG.csv" + #file_name = "Explore_DABH_Artefacts_60sCalibration_ExG.csv" self.server = server = CsvServer( channel_count=channel_count, csv_path=file_path + file_name, @@ -79,7 +80,7 @@ def read(self): time.sleep(sleep_time) eeg_packet = BleImpedancePacket(timestamp=self.server.ts, payload=None) try: - eeg_packet.data = self.server.read_sample() + eeg_packet.data, eeg_packet.timestamp = self.server.read_sample() except StopIteration: self.set_state(ClientState.STOPPED) return None @@ -102,7 +103,9 @@ def __init__(self, channel_count: int, csv_path: str, loop: bool = True): self.loop = loop self.csv_data = np.loadtxt(csv_path, delimiter=',', skiprows=1) # skip row 0 + self.csv_ts = self.csv_data[:, 0] self.csv_data = self.csv_data[:, 1:] + if self.csv_data.shape[1] != channel_count: print('######################', self.csv_data.shape) raise ValueError( @@ -163,10 +166,11 @@ def read_sample(self): raise StopIteration("End of CSV reached") self.row_idx = 0 + ts = self.csv_ts[self.row_idx] sample = self.csv_data[self.row_idx] self.row_idx += 1 - return sample.reshape(self.channel_count, 1) + return sample.reshape(self.channel_count, 1), ts def read_device_info(self): return self.device_info From 0ed62a14ade6131d38824333308cfec367d31572 Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Wed, 11 Feb 2026 11:33:47 +0100 Subject: [PATCH 08/47] Added WIP file to clean from file and compare cleaned, matched by timestamp data --- examples/clean_data_from_exg.py | 162 ++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 examples/clean_data_from_exg.py diff --git a/examples/clean_data_from_exg.py b/examples/clean_data_from_exg.py new file mode 100644 index 00000000..6db96f8e --- /dev/null +++ b/examples/clean_data_from_exg.py @@ -0,0 +1,162 @@ +# Note: This is a WIP + +import explorepy +from explorepy.stream_processor import TOPICS +from eegprep import clean_asr +from eegprep.utils.asr import asr_calibrate +import time +import numpy as np +import polars as pl + +import matplotlib.pyplot as plt + +n_ch = 32 + +assumed_sr = 250 + +calib_time = 30. +asr_window = 0.05 +t = 180. + +ts_buffer = [] +data_buffer = [[] for _ in range(n_ch)] +ts_buffer_no_asr = [] +data_buffer_no_asr = [[] for _ in range(n_ch)] + +now = time.time() + +def compare_cleaned_with_uncleaned(): + file_path_raw = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/32channel_semidry_artefacts_ExG.csv" + raw = pl.read_csv(file_path_raw) + + file_path_cleaned = f"./asr_cleaned-data_calib-{calib_time}_window-{asr_window}_t-{t}-extra.csv" + comp_cleaned_df = pl.read_csv(file_path_cleaned) + + calib_file = f"asr_calibration-data_calib-{calib_time}.csv" + calib_file_df = pl.read_csv(calib_file) + + as_np = calib_file_df.to_numpy() + as_np = as_np.swapaxes(1, 0) + + cleaned_df = call_clean_from_eegprep(raw, comp_cleaned_df, as_np, sr=assumed_sr, cutoff=asr_window, n_chan=32) + + dataframes = [(comp_cleaned_df, "b"), (cleaned_df, "s")] + + plot_comp_from_dataframes(dataframes) + + +def plot_comp_from_dataframes(dataframes: list[tuple[pl.DataFrame, str]]): + """Plots comparison of any dataframes' first channel from a list of tuples + + Args: + dataframes: list of tuples with the first element in the tuple being a polars DataFrame and the second element + in the tuple being a colour string to use for plotting with matplotlib, i.e. "b" for blue etc. + """ + fig, ax = plt.subplots(1, 1) + for tup in dataframes: + assert type(tup[0]) is pl.DataFrame + assert type(tup[1]) is str + ax.plot(tup[0]["TimeStamp"][:], tup[0][tup[0].columns[1]][:], tup[1]) + plt.show() + + +def plot_comp(): + no_asr = pl.read_csv(f"filtered_exg_calib-30.0_window-0.05_t-120.0.csv") + no_asr = no_asr.to_numpy().swapaxes(1, 0) + asr_one = pl.read_csv(f"asr_cleaned-data_calib-30.0_window-0.05_t-120.0.csv") + asr_one = asr_one.to_numpy().swapaxes(1, 0) + asr_two = pl.read_csv(f"asr_cleaned-data_calib-30.0_window-1.0_t-120.0.csv") + asr_two = asr_two.to_numpy().swapaxes(1, 0) + fig, ax = plt.subplots(1, 1) + #ax.plot(asr_one[0][:1000], asr_one[1][:1000], 'r') + #ax.plot(asr_two[0][:1000], asr_two[1][:1000], 'b') + ax.plot(no_asr[0][:], no_asr[1][:], 'g') + plt.show() + + +def on_asr_received(packet): + data = packet.get_data() + ts = data[0] + ts_buffer.extend(ts) + for i in range(n_ch): + data_buffer[i].extend(data[1][i, :]) + + +def on_filtered_received(packet): + data_filtered = packet.get_data() + ts_filtered = data_filtered[0] + ts_buffer_no_asr.extend(ts_filtered) + for i in range(n_ch): + data_buffer_no_asr[i].extend(data_filtered[1][i, :]) + + +def call_clean_from_eegprep(raw_data, comp_cleaned, calib_data, sr, cutoff, n_chan): + first_ts_cleaned = comp_cleaned["TimeStamp"][0] + last_ts_cleaned = comp_cleaned["TimeStamp"][-1] + + matched_raw = raw_data.remove(pl.col("TimeStamp") < first_ts_cleaned) + matched_raw = matched_raw.remove(pl.col("TimeStamp") > last_ts_cleaned) + as_np_raw = matched_raw.to_numpy() + as_np_raw = as_np_raw.swapaxes(1, 0) + as_np_raw_ts = as_np_raw[0, :] + as_np_raw = as_np_raw[1:, :] + as_dict = {"data": as_np_raw, "srate": sr, "nbchan": n_chan} + cleaned_dict = clean_asr(as_dict, cutoff=cutoff, ref_maxbadchannels=calib_data) + cleaned_df = cleaned_dict["data"] + cleaned_df = cleaned_df.swapaxes(1, 0) + cleaned_df = pl.DataFrame(cleaned_df) + s = pl.Series("TimeStamp", as_np_raw_ts) + cleaned_df.insert_column(0, s) + + return cleaned_df + + +def clean_data_from_file(): + dev = explorepy.Explore() + dev.connect("Explore_DABC") + dev.stream_processor.add_filter(cutoff_freq=50., filter_type="notch") + dev.stream_processor.add_filter(cutoff_freq=(1., 30.), filter_type="bandpass") + + time.sleep(5.0) + + dev.calibrate_asr(calib_time) + dev.stream_processor.subscribe(on_filtered_received, topic=TOPICS.filtered_ExG) + dev.stream_processor.subscribe(on_asr_received, topic=TOPICS.asr_ExG) + + time.sleep(calib_time + 5.) + + dev.stream_processor.asr_processor.calibration_data_input = as_np + dev.stream_processor.asr_processor.set_state_from_calibration_data(calib_data=as_np) + + time.sleep(1.) + + # Uncomment to write calibration data to file after it has been "recorded" + + # calib_data = dev.stream_processor.asr_processor.calibration_data_input + # calib_df = pl.DataFrame(calib_data.swapaxes(1, 0)) + # calib_df.write_csv(calib_file) + + dev.start_asr(window=asr_window) + + time.sleep(t) + + ts_buffer_np = np.array(ts_buffer) + data_buffer_np = np.array(data_buffer) + + ts_buffer_no_asr_np = np.array(ts_buffer_no_asr) + data_buffer_no_asr_np = np.array(data_buffer_no_asr) + + n = ["TimeStamp"] + n.extend([f"ch{i + 1}" for i in range(n_ch)]) + ret = np.vstack((ts_buffer_np, data_buffer_np)) + df = pl.DataFrame(ret.swapaxes(1, 0), schema=n) + df.write_csv(f"asr_cleaned-data_calib-{calib_time}_window-{asr_window}_t-{t}.csv") # cleaned from filtered + + ret_two = np.vstack((ts_buffer_no_asr_np, data_buffer_no_asr_np)) + df_no_asr = pl.DataFrame(ret_two.swapaxes(1, 0), schema=n) + df_no_asr.write_csv(f"filtered_exg_calib-{calib_time}_window-{asr_window}_t-{t}.csv") # uncleaned but filtered + + +if __name__ == '__main__': + compare_cleaned_with_uncleaned() + # clean_data_from_file() From b6b35847fc574070bc77c402f71c88f9bc22d73f Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Thu, 12 Feb 2026 16:26:27 +0100 Subject: [PATCH 09/47] Rewrote code for matrix testing --- examples/clean_data_from_exg.py | 178 ++++++++++++++++++++++++-------- 1 file changed, 134 insertions(+), 44 deletions(-) diff --git a/examples/clean_data_from_exg.py b/examples/clean_data_from_exg.py index 6db96f8e..78a9389b 100644 --- a/examples/clean_data_from_exg.py +++ b/examples/clean_data_from_exg.py @@ -1,9 +1,9 @@ # Note: This is a WIP +import random import explorepy from explorepy.stream_processor import TOPICS from eegprep import clean_asr -from eegprep.utils.asr import asr_calibrate import time import numpy as np import polars as pl @@ -18,6 +18,10 @@ asr_window = 0.05 t = 180. +max_plot_rows = 4 +max_plot_cols = 2 +max_plot_indices = 1000 + ts_buffer = [] data_buffer = [[] for _ in range(n_ch)] ts_buffer_no_asr = [] @@ -25,13 +29,22 @@ now = time.time() + def compare_cleaned_with_uncleaned(): - file_path_raw = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/32channel_semidry_artefacts_ExG.csv" + file_path_raw = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/32channel_semidry_artefacts_ExG_1ch_corrupted.csv" raw = pl.read_csv(file_path_raw) - file_path_cleaned = f"./asr_cleaned-data_calib-{calib_time}_window-{asr_window}_t-{t}-extra.csv" + file_path_cleaned = f"./corrupted_asr_cleaned-data_calib-{calib_time}_window-{asr_window}_t-{t}.csv" comp_cleaned_df = pl.read_csv(file_path_cleaned) + first_ts_cleaned = comp_cleaned_df["TimeStamp"][0] + last_ts_cleaned = comp_cleaned_df["TimeStamp"][-1] + + file_path_filtered = f"corrupted_filtered_exg_calib-{calib_time}_window-{asr_window}_t-{t}.csv" + filtered = pl.read_csv(file_path_filtered) + matched_filtered = filtered.remove(pl.col("TimeStamp") < first_ts_cleaned) + matched_filtered = matched_filtered.remove(pl.col("TimeStamp") > last_ts_cleaned) + calib_file = f"asr_calibration-data_calib-{calib_time}.csv" calib_file_df = pl.read_csv(calib_file) @@ -40,37 +53,69 @@ def compare_cleaned_with_uncleaned(): cleaned_df = call_clean_from_eegprep(raw, comp_cleaned_df, as_np, sr=assumed_sr, cutoff=asr_window, n_chan=32) - dataframes = [(comp_cleaned_df, "b"), (cleaned_df, "s")] - - plot_comp_from_dataframes(dataframes) + dataframes = [(matched_filtered, "r", "Uncleaned (filtered)"), + (comp_cleaned_df, "b", f"Cleaned with window = {asr_window}s"), + (cleaned_df, "g", "Cleaned with clean_asr")] + + plot_comp_from_dataframes(dataframes, [1, 2, 3, 4, 5, 6, 7, 8]) + + +def add_dead_channels_to_dataframe(dataframe, count): + n_channels = len(dataframe.columns) - 1 + drop_indices = random.sample(range(1, n_channels), count) + + for idx in drop_indices: + column_name = dataframe.columns[idx] + dataframe = dataframe.replace_column(idx, pl.Series(name=column_name, + values=np.full(dataframe.shape[0], -400000.05))) + return dataframe -def plot_comp_from_dataframes(dataframes: list[tuple[pl.DataFrame, str]]): + +def plot_comp_from_dataframes(dataframes: list[tuple[pl.DataFrame, str, str]], channels: list[int]=None): """Plots comparison of any dataframes' first channel from a list of tuples Args: dataframes: list of tuples with the first element in the tuple being a polars DataFrame and the second element in the tuple being a colour string to use for plotting with matplotlib, i.e. "b" for blue etc. + channels: list of ints that defines which channels to plot """ - fig, ax = plt.subplots(1, 1) - for tup in dataframes: - assert type(tup[0]) is pl.DataFrame - assert type(tup[1]) is str - ax.plot(tup[0]["TimeStamp"][:], tup[0][tup[0].columns[1]][:], tup[1]) - plt.show() - - -def plot_comp(): - no_asr = pl.read_csv(f"filtered_exg_calib-30.0_window-0.05_t-120.0.csv") - no_asr = no_asr.to_numpy().swapaxes(1, 0) - asr_one = pl.read_csv(f"asr_cleaned-data_calib-30.0_window-0.05_t-120.0.csv") - asr_one = asr_one.to_numpy().swapaxes(1, 0) - asr_two = pl.read_csv(f"asr_cleaned-data_calib-30.0_window-1.0_t-120.0.csv") - asr_two = asr_two.to_numpy().swapaxes(1, 0) - fig, ax = plt.subplots(1, 1) - #ax.plot(asr_one[0][:1000], asr_one[1][:1000], 'r') - #ax.plot(asr_two[0][:1000], asr_two[1][:1000], 'b') - ax.plot(no_asr[0][:], no_asr[1][:], 'g') + if channels is None: + print("No channels supplied for plotting, exiting...") + return + n_channels = len(channels) + + if n_channels > max_plot_cols * max_plot_rows: + print(f"Too many channels to reasonably plot (got: {n_channels}, " + f"max: {max_plot_cols * max_plot_rows}), exiting...") + + plot_cols = 1 + for i in range(max_plot_cols): + if n_channels > (i+1) * max_plot_rows: + plot_cols += 1 + plot_rows = min(n_channels, max_plot_rows) + print(f"n_channels: {n_channels}, plot_cols: {plot_cols}, plot_rows: {plot_rows}") + fig, axs = plt.subplots(plot_rows, plot_cols, layout="constrained") + print(len(axs)) + print(axs) + it = 0 + ax = None + for idx_to_plot in channels: + r = it%max_plot_rows + c = it//max_plot_rows + print(f"r: {r}, c: {c}, idx_to_plot: {idx_to_plot}") + ax = axs[r, c] + for tup in dataframes: + assert type(tup[0]) is pl.DataFrame + assert type(tup[1]) is str + assert type(tup[2]) is str + ax.plot(tup[0]["TimeStamp"][:max_plot_indices], tup[0][tup[0].columns[idx_to_plot+1]][:max_plot_indices], + tup[1], label=tup[2]) + ax.title.set_text(f"Channel {idx_to_plot}") + it += 1 + if ax is not None: + legend_handles, legend_labels = ax.get_legend_handles_labels() + fig.legend(legend_handles, legend_labels, loc='upper center') plt.show() @@ -111,34 +156,57 @@ def call_clean_from_eegprep(raw_data, comp_cleaned, calib_data, sr, cutoff, n_ch return cleaned_df -def clean_data_from_file(): +def set_up_explore_device(t_calib=30., dev_name="Explore_DABC", notch=50., bp=(1., 30.)): + """Connects to a device, sets up filters and performs ASR calibration""" dev = explorepy.Explore() - dev.connect("Explore_DABC") - dev.stream_processor.add_filter(cutoff_freq=50., filter_type="notch") - dev.stream_processor.add_filter(cutoff_freq=(1., 30.), filter_type="bandpass") + dev.connect(dev_name) + + dev.stream_processor.add_filter(cutoff_freq=notch, filter_type="notch") + dev.stream_processor.add_filter(cutoff_freq=bp, filter_type="bandpass") time.sleep(5.0) - dev.calibrate_asr(calib_time) + dev.calibrate_asr(t_calib) dev.stream_processor.subscribe(on_filtered_received, topic=TOPICS.filtered_ExG) dev.stream_processor.subscribe(on_asr_received, topic=TOPICS.asr_ExG) - time.sleep(calib_time + 5.) + time.sleep(t_calib + 5.) + + return dev + - dev.stream_processor.asr_processor.calibration_data_input = as_np - dev.stream_processor.asr_processor.set_state_from_calibration_data(calib_data=as_np) +def write_calibration_data(t_calib=30., calib_file=None): + dev = set_up_explore_device(t_calib=t_calib) + if calib_file is None: + calib_file = f"asr_calibration-data_calib-{t_calib}.csv" time.sleep(1.) - # Uncomment to write calibration data to file after it has been "recorded" + calib_data = dev.stream_processor.asr_processor.calibration_data_input + calib_df = pl.DataFrame(calib_data.swapaxes(1, 0)) + calib_df.write_csv(calib_file) + dev.disconnect() + time.sleep(1.) + + return calib_file + + +def clean_data_from_file(t_calib=30., t_window=0.05, rec_length=180., calib_file=None): + dev = set_up_explore_device(t_calib=t_calib) - # calib_data = dev.stream_processor.asr_processor.calibration_data_input - # calib_df = pl.DataFrame(calib_data.swapaxes(1, 0)) - # calib_df.write_csv(calib_file) + if calib_file is not None: + calib_file_df = pl.read_csv(calib_file) + as_np = calib_file_df.to_numpy() + as_np = as_np.swapaxes(1, 0) + + dev.stream_processor.asr_processor.calibration_data_input = as_np + dev.stream_processor.asr_processor.set_state_from_calibration_data(calib_data=as_np) + + time.sleep(1.) - dev.start_asr(window=asr_window) + dev.start_asr(window=t_window) - time.sleep(t) + time.sleep(rec_length) ts_buffer_np = np.array(ts_buffer) data_buffer_np = np.array(data_buffer) @@ -150,13 +218,35 @@ def clean_data_from_file(): n.extend([f"ch{i + 1}" for i in range(n_ch)]) ret = np.vstack((ts_buffer_np, data_buffer_np)) df = pl.DataFrame(ret.swapaxes(1, 0), schema=n) - df.write_csv(f"asr_cleaned-data_calib-{calib_time}_window-{asr_window}_t-{t}.csv") # cleaned from filtered + f_name_cleaned = f"asr_cleaned-data_calib-{t_calib}_window-{t_window}_t-{rec_length}.csv" + df.write_csv(f_name_cleaned) # cleaned from filtered ret_two = np.vstack((ts_buffer_no_asr_np, data_buffer_no_asr_np)) df_no_asr = pl.DataFrame(ret_two.swapaxes(1, 0), schema=n) - df_no_asr.write_csv(f"filtered_exg_calib-{calib_time}_window-{asr_window}_t-{t}.csv") # uncleaned but filtered + f_name_uncleaned = f"filtered_exg_calib-{t_calib}_window-{t_window}_t-{rec_length}.csv" + df_no_asr.write_csv(f_name_uncleaned) # uncleaned but filtered + + dev.disconnect() + time.sleep(1.) + + return f_name_cleaned, f_name_uncleaned if __name__ == '__main__': - compare_cleaned_with_uncleaned() - # clean_data_from_file() + t_windows_to_test = [0.01, 0.05] + t_calib_to_test = [30.] + rec_length_to_test = [20.] + + for t_calib in t_calib_to_test: + calib_file_path = write_calibration_data(t_calib=t_calib) + + for t_window in t_windows_to_test: + for rec_length in rec_length_to_test: + cleaned_path, uncleaned_path = clean_data_from_file(t_calib=t_calib, t_window=t_window, + rec_length=rec_length, calib_file=calib_file_path) + + # TODO gather files + metadata for comparison plots + # TODO clear buffers between runs + # TODO add channel dropping to matrix test + + # compare_cleaned_with_uncleaned() From f57beb821cb5199cf1e470691dded90d97e050f2 Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Fri, 13 Feb 2026 12:37:11 +0100 Subject: [PATCH 10/47] Cleaned up plotting / matrix test for ASR --- examples/clean_data_from_exg.py | 440 +++++++++++++++++++------------- 1 file changed, 260 insertions(+), 180 deletions(-) diff --git a/examples/clean_data_from_exg.py b/examples/clean_data_from_exg.py index 78a9389b..babc7522 100644 --- a/examples/clean_data_from_exg.py +++ b/examples/clean_data_from_exg.py @@ -10,54 +10,205 @@ import matplotlib.pyplot as plt -n_ch = 32 - -assumed_sr = 250 - calib_time = 30. asr_window = 0.05 t = 180. -max_plot_rows = 4 -max_plot_cols = 2 -max_plot_indices = 1000 - -ts_buffer = [] -data_buffer = [[] for _ in range(n_ch)] -ts_buffer_no_asr = [] -data_buffer_no_asr = [[] for _ in range(n_ch)] - now = time.time() +class DataRecorder: + def __init__(self, n_channels): + self.dev = None + self.n_channels = n_channels + self.ts_buffer = [] + self.data_buffer = [[] for _ in range(self.n_channels)] + self.ts_buffer_no_asr = [] + self.data_buffer_no_asr = [[] for _ in range(self.n_channels)] -def compare_cleaned_with_uncleaned(): - file_path_raw = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/32channel_semidry_artefacts_ExG_1ch_corrupted.csv" - raw = pl.read_csv(file_path_raw) + def clear_buffers(self): + self.ts_buffer = [] + self.data_buffer = [[] for _ in range(self.n_channels)] + self.ts_buffer_no_asr = [] + self.data_buffer_no_asr = [[] for _ in range(self.n_channels)] - file_path_cleaned = f"./corrupted_asr_cleaned-data_calib-{calib_time}_window-{asr_window}_t-{t}.csv" - comp_cleaned_df = pl.read_csv(file_path_cleaned) + def on_asr_received(self, packet): + data = packet.get_data() + ts = data[0] + self.ts_buffer.extend(ts) + for i in range(self.n_channels): + self.data_buffer[i].extend(data[1][i, :]) - first_ts_cleaned = comp_cleaned_df["TimeStamp"][0] - last_ts_cleaned = comp_cleaned_df["TimeStamp"][-1] + def on_filtered_received(self, packet): + data_filtered = packet.get_data() + ts_filtered = data_filtered[0] + self.ts_buffer_no_asr.extend(ts_filtered) + for i in range(self.n_channels): + self.data_buffer_no_asr[i].extend(data_filtered[1][i, :]) - file_path_filtered = f"corrupted_filtered_exg_calib-{calib_time}_window-{asr_window}_t-{t}.csv" - filtered = pl.read_csv(file_path_filtered) - matched_filtered = filtered.remove(pl.col("TimeStamp") < first_ts_cleaned) - matched_filtered = matched_filtered.remove(pl.col("TimeStamp") > last_ts_cleaned) + def set_up_explore_device(self, t_calib=30., dev_name="Explore_DABC", notch=50., bp=(1., 30.), record_raw_data=False): + """Connects to a device, sets up filters and performs ASR calibration""" + self.dev = explorepy.Explore() + self.dev.connect(dev_name) - calib_file = f"asr_calibration-data_calib-{calib_time}.csv" - calib_file_df = pl.read_csv(calib_file) + self.dev.stream_processor.add_filter(cutoff_freq=notch, filter_type="notch") + self.dev.stream_processor.add_filter(cutoff_freq=bp, filter_type="bandpass") - as_np = calib_file_df.to_numpy() - as_np = as_np.swapaxes(1, 0) + time.sleep(5.0) + + self.dev.calibrate_asr(t_calib) + if record_raw_data: + self.dev.stream_processor.subscribe(self.on_filtered_received, topic=TOPICS.filtered_ExG) + self.dev.stream_processor.subscribe(self.on_asr_received, topic=TOPICS.asr_ExG) + + time.sleep(t_calib + 5.) - cleaned_df = call_clean_from_eegprep(raw, comp_cleaned_df, as_np, sr=assumed_sr, cutoff=asr_window, n_chan=32) + return self.dev - dataframes = [(matched_filtered, "r", "Uncleaned (filtered)"), - (comp_cleaned_df, "b", f"Cleaned with window = {asr_window}s"), - (cleaned_df, "g", "Cleaned with clean_asr")] + def write_calibration_data(self, t_calib=30., calib_file=None): + self.dev = self.set_up_explore_device(t_calib=t_calib) + if calib_file is None: + calib_file = f"asr_calibration-data_calib-{t_calib}.csv" + + time.sleep(1.) - plot_comp_from_dataframes(dataframes, [1, 2, 3, 4, 5, 6, 7, 8]) + calib_data = self.dev.stream_processor.asr_processor.calibration_data_input + calib_df = pl.DataFrame(calib_data.swapaxes(1, 0)) + calib_df.write_csv(calib_file) + self.dev.disconnect() + time.sleep(1.) + + self.clear_buffers() + + return calib_file + + def clean_data_from_file(self, cutoff=5.0, t_calib=30., t_window=0.05, rec_length=180., calib_file=None, record_raw_data=False): + self.dev = self.set_up_explore_device(t_calib=t_calib, record_raw_data=record_raw_data) + + if calib_file is not None: + calib_file_df = pl.read_csv(calib_file) + as_np = calib_file_df.to_numpy() + as_np = as_np.swapaxes(1, 0) + + self.dev.stream_processor.asr_processor.calibration_data_input = as_np + self.dev.stream_processor.asr_processor.set_state_from_calibration_data(calib_data=as_np) + + time.sleep(1.) + + self.dev.stream_processor.asr_processor.cutoff = cutoff + + self.dev.start_asr(window=t_window) + + time.sleep(rec_length) + + ts_buffer_np = np.array(self.ts_buffer) + data_buffer_np = np.array(self.data_buffer) + + n = ["TimeStamp"] + n.extend([f"ch{i + 1}" for i in range(self.n_channels)]) + ret = np.vstack((ts_buffer_np, data_buffer_np)) + df = pl.DataFrame(ret.swapaxes(1, 0), schema=n) + f_name_cleaned = f"asr_cleaned-data_calib-{t_calib}_window-{t_window}_t-{rec_length}.csv" + df.write_csv(f_name_cleaned) # cleaned from filtered + + f_name_uncleaned = None + + if record_raw_data: + ts_buffer_no_asr_np = np.array(self.ts_buffer_no_asr) + data_buffer_no_asr_np = np.array(self.data_buffer_no_asr) + + ret_two = np.vstack((ts_buffer_no_asr_np, data_buffer_no_asr_np)) + df_no_asr = pl.DataFrame(ret_two.swapaxes(1, 0), schema=n) + f_name_uncleaned = f"filtered_exg_calib-{t_calib}_window-{t_window}_t-{rec_length}.csv" + df_no_asr.write_csv(f_name_uncleaned) # uncleaned but filtered + + self.dev.disconnect() + time.sleep(1.) + self.dev = None + + self.clear_buffers() + + return f_name_cleaned, f_name_uncleaned + + +class DataComparator: + def __init__(self): + self.dataframes = [] + self.max_plot_rows = 4 + self.max_plot_cols = 2 + self.max_plot_indices = 1000 + + def get_first_and_last_ts(self): + all_first_ts = [] + all_last_ts = [] + for tup in self.dataframes: + all_first_ts.append(tup[0]["TimeStamp"][0]) + all_last_ts.append(tup[0]["TimeStamp"][-1]) + first_ts = max(all_first_ts) + last_ts = min(all_last_ts) + return first_ts, last_ts + + def cull_dataframes(self): + culled_dataframes = [] + first_ts, last_ts = self.get_first_and_last_ts() + for tup in self.dataframes: + df = tup[0] + df = df.remove(pl.col("TimeStamp") < first_ts) + df = df.remove(pl.col("TimeStamp") > last_ts) + culled_dataframes.append((df, tup[1], tup[2])) + self.dataframes = culled_dataframes + + def add_dataframe_from_file(self, file_path, colour, label): + dataframe = pl.read_csv(file_path) + self.add_dataframe(dataframe, colour, label) + + def add_dataframe(self, dataframe, colour, label): + self.dataframes.append((dataframe, colour, label)) + + def clear_dataframes(self): + self.dataframes = [] + + def plot_comp_from_dataframes(self, channels: list[int] = None): + """Plots comparison of any dataframes' first channel from a list of tuples + + Args: + dataframes: list of tuples with the first element in the tuple being a polars DataFrame and the second element + in the tuple being a colour string to use for plotting with matplotlib, i.e. "b" for blue etc. + channels: list of ints that defines which channels to plot + """ + if channels is None: + print("No channels supplied for plotting, exiting...") + return + n_channels = len(channels) + + if n_channels > self.max_plot_cols * self.max_plot_rows: + print(f"Too many channels to reasonably plot (got: {n_channels}, " + f"max: {self.max_plot_cols * self.max_plot_rows}), exiting...") + + plot_cols = 1 + for i in range(self.max_plot_cols): + if n_channels > (i + 1) * self.max_plot_rows: + plot_cols += 1 + plot_rows = min(n_channels, self.max_plot_rows) + fig, axs = plt.subplots(plot_rows, plot_cols, layout="constrained") + it = 0 + ax = None + for idx_to_plot in channels: + r = it % self.max_plot_rows + c = it // self.max_plot_rows + ax = axs[r, c] + for tup in self.dataframes: + assert type(tup[0]) is pl.DataFrame + assert type(tup[1]) is str + assert type(tup[2]) is str + ax.plot(tup[0]["TimeStamp"][:self.max_plot_indices], + tup[0][tup[0].columns[idx_to_plot + 1]][:self.max_plot_indices], + tup[1], label=tup[2]) + ax.title.set_text(f"Channel {idx_to_plot}") + it += 1 + if ax is not None: + legend_handles, legend_labels = ax.get_legend_handles_labels() + fig.legend(legend_handles, legend_labels, loc='upper center') + plt.show() def add_dead_channels_to_dataframe(dataframe, count): @@ -72,178 +223,107 @@ def add_dead_channels_to_dataframe(dataframe, count): return dataframe -def plot_comp_from_dataframes(dataframes: list[tuple[pl.DataFrame, str, str]], channels: list[int]=None): - """Plots comparison of any dataframes' first channel from a list of tuples - - Args: - dataframes: list of tuples with the first element in the tuple being a polars DataFrame and the second element - in the tuple being a colour string to use for plotting with matplotlib, i.e. "b" for blue etc. - channels: list of ints that defines which channels to plot - """ - if channels is None: - print("No channels supplied for plotting, exiting...") - return - n_channels = len(channels) - - if n_channels > max_plot_cols * max_plot_rows: - print(f"Too many channels to reasonably plot (got: {n_channels}, " - f"max: {max_plot_cols * max_plot_rows}), exiting...") - - plot_cols = 1 - for i in range(max_plot_cols): - if n_channels > (i+1) * max_plot_rows: - plot_cols += 1 - plot_rows = min(n_channels, max_plot_rows) - print(f"n_channels: {n_channels}, plot_cols: {plot_cols}, plot_rows: {plot_rows}") - fig, axs = plt.subplots(plot_rows, plot_cols, layout="constrained") - print(len(axs)) - print(axs) - it = 0 - ax = None - for idx_to_plot in channels: - r = it%max_plot_rows - c = it//max_plot_rows - print(f"r: {r}, c: {c}, idx_to_plot: {idx_to_plot}") - ax = axs[r, c] - for tup in dataframes: - assert type(tup[0]) is pl.DataFrame - assert type(tup[1]) is str - assert type(tup[2]) is str - ax.plot(tup[0]["TimeStamp"][:max_plot_indices], tup[0][tup[0].columns[idx_to_plot+1]][:max_plot_indices], - tup[1], label=tup[2]) - ax.title.set_text(f"Channel {idx_to_plot}") - it += 1 - if ax is not None: - legend_handles, legend_labels = ax.get_legend_handles_labels() - fig.legend(legend_handles, legend_labels, loc='upper center') - plt.show() - - -def on_asr_received(packet): - data = packet.get_data() - ts = data[0] - ts_buffer.extend(ts) - for i in range(n_ch): - data_buffer[i].extend(data[1][i, :]) - - -def on_filtered_received(packet): - data_filtered = packet.get_data() - ts_filtered = data_filtered[0] - ts_buffer_no_asr.extend(ts_filtered) - for i in range(n_ch): - data_buffer_no_asr[i].extend(data_filtered[1][i, :]) - - -def call_clean_from_eegprep(raw_data, comp_cleaned, calib_data, sr, cutoff, n_chan): - first_ts_cleaned = comp_cleaned["TimeStamp"][0] - last_ts_cleaned = comp_cleaned["TimeStamp"][-1] - - matched_raw = raw_data.remove(pl.col("TimeStamp") < first_ts_cleaned) - matched_raw = matched_raw.remove(pl.col("TimeStamp") > last_ts_cleaned) - as_np_raw = matched_raw.to_numpy() - as_np_raw = as_np_raw.swapaxes(1, 0) - as_np_raw_ts = as_np_raw[0, :] - as_np_raw = as_np_raw[1:, :] - as_dict = {"data": as_np_raw, "srate": sr, "nbchan": n_chan} - cleaned_dict = clean_asr(as_dict, cutoff=cutoff, ref_maxbadchannels=calib_data) +def call_clean_from_eegprep(raw_data_file, calib_file, sr, cutoff, n_chan, first_ts, last_ts): + calib_file_df = pl.read_csv(calib_file) + calib_as_np = calib_file_df.to_numpy() + calib_as_np = calib_as_np.swapaxes(1, 0) + + raw_data = pl.read_csv(raw_data_file) + culled_raw_data = raw_data.remove(pl.col("TimeStamp") < first_ts) + culled_raw_data = culled_raw_data.remove(pl.col("TimeStamp") > last_ts) + culled_raw_data = culled_raw_data.to_numpy() + culled_raw_data = culled_raw_data.swapaxes(1, 0) + + culled_raw_ts = culled_raw_data[0, :] + culled_raw_data = culled_raw_data[1:, :] + + as_dict = {"data": culled_raw_data, "srate": sr, "nbchan": n_chan} + + cleaned_dict = clean_asr(as_dict, cutoff=cutoff, ref_maxbadchannels=calib_as_np) cleaned_df = cleaned_dict["data"] cleaned_df = cleaned_df.swapaxes(1, 0) cleaned_df = pl.DataFrame(cleaned_df) - s = pl.Series("TimeStamp", as_np_raw_ts) + s = pl.Series("TimeStamp", culled_raw_ts) cleaned_df.insert_column(0, s) return cleaned_df -def set_up_explore_device(t_calib=30., dev_name="Explore_DABC", notch=50., bp=(1., 30.)): - """Connects to a device, sets up filters and performs ASR calibration""" - dev = explorepy.Explore() - dev.connect(dev_name) - - dev.stream_processor.add_filter(cutoff_freq=notch, filter_type="notch") - dev.stream_processor.add_filter(cutoff_freq=bp, filter_type="bandpass") - - time.sleep(5.0) - - dev.calibrate_asr(t_calib) - dev.stream_processor.subscribe(on_filtered_received, topic=TOPICS.filtered_ExG) - dev.stream_processor.subscribe(on_asr_received, topic=TOPICS.asr_ExG) - - time.sleep(t_calib + 5.) - - return dev - - -def write_calibration_data(t_calib=30., calib_file=None): - dev = set_up_explore_device(t_calib=t_calib) - if calib_file is None: - calib_file = f"asr_calibration-data_calib-{t_calib}.csv" - - time.sleep(1.) - - calib_data = dev.stream_processor.asr_processor.calibration_data_input - calib_df = pl.DataFrame(calib_data.swapaxes(1, 0)) - calib_df.write_csv(calib_file) - dev.disconnect() - time.sleep(1.) - - return calib_file - - -def clean_data_from_file(t_calib=30., t_window=0.05, rec_length=180., calib_file=None): - dev = set_up_explore_device(t_calib=t_calib) +def compare_cleaned_with_uncleaned(self): + file_path_raw = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/32channel_semidry_artefacts_ExG_1ch_corrupted.csv" + raw = pl.read_csv(file_path_raw) - if calib_file is not None: - calib_file_df = pl.read_csv(calib_file) - as_np = calib_file_df.to_numpy() - as_np = as_np.swapaxes(1, 0) + file_path_cleaned = f"./corrupted_asr_cleaned-data_calib-{calib_time}_window-{asr_window}_t-{t}.csv" + comp_cleaned_df = pl.read_csv(file_path_cleaned) - dev.stream_processor.asr_processor.calibration_data_input = as_np - dev.stream_processor.asr_processor.set_state_from_calibration_data(calib_data=as_np) + first_ts_cleaned = comp_cleaned_df["TimeStamp"][0] + last_ts_cleaned = comp_cleaned_df["TimeStamp"][-1] - time.sleep(1.) + file_path_filtered = f"corrupted_filtered_exg_calib-{calib_time}_window-{asr_window}_t-{t}.csv" + filtered = pl.read_csv(file_path_filtered) + matched_filtered = filtered.remove(pl.col("TimeStamp") < first_ts_cleaned) + matched_filtered = matched_filtered.remove(pl.col("TimeStamp") > last_ts_cleaned) - dev.start_asr(window=t_window) + calib_file = f"asr_calibration-data_calib-{calib_time}.csv" - time.sleep(rec_length) + cleaned_df = call_clean_from_eegprep(raw, calib_file=calib_file, sr=250, cutoff=asr_window, n_chan=32, first_ts=None, last_ts=None) - ts_buffer_np = np.array(ts_buffer) - data_buffer_np = np.array(data_buffer) + dataframes = [(matched_filtered, "r", "Uncleaned (filtered)"), + (comp_cleaned_df, "b", f"Cleaned with window = {asr_window}s"), + (cleaned_df, "g", "Cleaned with clean_asr")] - ts_buffer_no_asr_np = np.array(ts_buffer_no_asr) - data_buffer_no_asr_np = np.array(data_buffer_no_asr) + self.plot_comp_from_dataframes([1, 2, 3, 4, 5, 6, 7, 8]) - n = ["TimeStamp"] - n.extend([f"ch{i + 1}" for i in range(n_ch)]) - ret = np.vstack((ts_buffer_np, data_buffer_np)) - df = pl.DataFrame(ret.swapaxes(1, 0), schema=n) - f_name_cleaned = f"asr_cleaned-data_calib-{t_calib}_window-{t_window}_t-{rec_length}.csv" - df.write_csv(f_name_cleaned) # cleaned from filtered - ret_two = np.vstack((ts_buffer_no_asr_np, data_buffer_no_asr_np)) - df_no_asr = pl.DataFrame(ret_two.swapaxes(1, 0), schema=n) - f_name_uncleaned = f"filtered_exg_calib-{t_calib}_window-{t_window}_t-{rec_length}.csv" - df_no_asr.write_csv(f_name_uncleaned) # uncleaned but filtered +def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window_list: list, rec_length_list: list): + data_writer = DataRecorder(number_channels) + data_comparator = DataComparator() - dev.disconnect() - time.sleep(1.) + uncleaned_file = None + cleaned_files = [] + calib_file_path = data_writer.write_calibration_data(t_calib=t_calib_length) - return f_name_cleaned, f_name_uncleaned + uncleaned_plot_colour = "gray" + eegprep_plot_colour = "red" + asr_processor_plot_colours = ["green", "skyblue", "violet", "gold", "aquamarine", "mediumpurple"] + it = 0 + for t_window in t_window_list: + for rec_length in rec_length_list: + cleaned_path, uncleaned_path = data_writer.clean_data_from_file(cutoff=cutoff, + t_calib=t_calib_length, + t_window=t_window, + rec_length=rec_length, + calib_file=calib_file_path, + record_raw_data=uncleaned_file is None) + if uncleaned_path is not None and uncleaned_file is None: + uncleaned_file = (uncleaned_path, uncleaned_plot_colour, "Uncleaned (filtered)") + cleaned_files.append((cleaned_path, + asr_processor_plot_colours[it%len(asr_processor_plot_colours)], + f"Cleaned, cutoff={cutoff}, t_window={t_window}, calib={t_calib_length}, rec_length={rec_length}")) + it += 1 + + for cleaned_file in cleaned_files: + data_comparator.add_dataframe_from_file(*cleaned_file) + data_comparator.add_dataframe_from_file(*uncleaned_file) + data_comparator.cull_dataframes() + first_ts, last_ts = data_comparator.get_first_and_last_ts() + eegprep_cleaned = call_clean_from_eegprep(raw_data_file=uncleaned_file[0], calib_file=calib_file_path, sr=250, + cutoff=5.0, n_chan=32, first_ts=first_ts, last_ts=last_ts) + eeg_prep_tup = (eegprep_cleaned, eegprep_plot_colour, f"Cleaned (eegprep), cutoff={cutoff}") + cleaned_files.append(eeg_prep_tup) + data_comparator.add_dataframe(*eeg_prep_tup) + data_comparator.plot_comp_from_dataframes(channels=[1,2,3,4,5,6,7,8,]) if __name__ == '__main__': + n_ch = 32 + cutoff = 5.0 + t_windows_to_test = [0.01, 0.05] - t_calib_to_test = [30.] + t_calib_to_test = 10. rec_length_to_test = [20.] - for t_calib in t_calib_to_test: - calib_file_path = write_calibration_data(t_calib=t_calib) - - for t_window in t_windows_to_test: - for rec_length in rec_length_to_test: - cleaned_path, uncleaned_path = clean_data_from_file(t_calib=t_calib, t_window=t_window, - rec_length=rec_length, calib_file=calib_file_path) + perform_matrix_test(n_ch, cutoff, t_calib_to_test, t_windows_to_test, rec_length_to_test) # TODO gather files + metadata for comparison plots # TODO clear buffers between runs From c13338fee5471a6121589842672a9c64dbe54b55 Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Mon, 16 Feb 2026 17:03:25 +0100 Subject: [PATCH 11/47] Added setting file to csv_client, extended example script with corrupting channel --- examples/clean_data_from_exg.py | 65 +++++++++++++++++++++---------- src/explorepy/csv_client.py | 19 ++++++--- src/explorepy/explore.py | 4 +- src/explorepy/parser.py | 4 +- src/explorepy/stream_processor.py | 4 +- 5 files changed, 64 insertions(+), 32 deletions(-) diff --git a/examples/clean_data_from_exg.py b/examples/clean_data_from_exg.py index babc7522..991a5c03 100644 --- a/examples/clean_data_from_exg.py +++ b/examples/clean_data_from_exg.py @@ -1,4 +1,5 @@ # Note: This is a WIP +import os import random import explorepy @@ -45,13 +46,15 @@ def on_filtered_received(self, packet): for i in range(self.n_channels): self.data_buffer_no_asr[i].extend(data_filtered[1][i, :]) - def set_up_explore_device(self, t_calib=30., dev_name="Explore_DABC", notch=50., bp=(1., 30.), record_raw_data=False): + def set_up_explore_device(self, t_calib=30., dev_name="Explore_DABC", notch=50., bp=(1., 30.), record_raw_data=False, file=None): """Connects to a device, sets up filters and performs ASR calibration""" self.dev = explorepy.Explore() - self.dev.connect(dev_name) + self.dev.connect(dev_name, file_path=file) self.dev.stream_processor.add_filter(cutoff_freq=notch, filter_type="notch") self.dev.stream_processor.add_filter(cutoff_freq=bp, filter_type="bandpass") + f_name = self.dev.stream_processor.parser.stream_interface.file_name + print(f"Working on {f_name}") time.sleep(5.0) @@ -64,16 +67,20 @@ def set_up_explore_device(self, t_calib=30., dev_name="Explore_DABC", notch=50., return self.dev - def write_calibration_data(self, t_calib=30., calib_file=None): - self.dev = self.set_up_explore_device(t_calib=t_calib) + def write_calibration_data(self, t_calib=30., calib_file=None, file=None, root_folder=None, overwrite=False): + self.dev = self.set_up_explore_device(t_calib=t_calib, file=file) if calib_file is None: calib_file = f"asr_calibration-data_calib-{t_calib}.csv" + if root_folder is not None: + calib_file = os.path.join(root_folder, calib_file) + time.sleep(1.) calib_data = self.dev.stream_processor.asr_processor.calibration_data_input calib_df = pl.DataFrame(calib_data.swapaxes(1, 0)) - calib_df.write_csv(calib_file) + if not os.path.exists(calib_file) or overwrite: + calib_df.write_csv(calib_file) self.dev.disconnect() time.sleep(1.) @@ -81,8 +88,8 @@ def write_calibration_data(self, t_calib=30., calib_file=None): return calib_file - def clean_data_from_file(self, cutoff=5.0, t_calib=30., t_window=0.05, rec_length=180., calib_file=None, record_raw_data=False): - self.dev = self.set_up_explore_device(t_calib=t_calib, record_raw_data=record_raw_data) + def clean_data_from_file(self, cutoff=5.0, t_calib=30., t_window=0.05, rec_length=180., calib_file=None, record_raw_data=False, file=None, root_folder=None, overwrite=False): + self.dev = self.set_up_explore_device(t_calib=t_calib, record_raw_data=record_raw_data, file=file) if calib_file is not None: calib_file_df = pl.read_csv(calib_file) @@ -108,7 +115,10 @@ def clean_data_from_file(self, cutoff=5.0, t_calib=30., t_window=0.05, rec_lengt ret = np.vstack((ts_buffer_np, data_buffer_np)) df = pl.DataFrame(ret.swapaxes(1, 0), schema=n) f_name_cleaned = f"asr_cleaned-data_calib-{t_calib}_window-{t_window}_t-{rec_length}.csv" - df.write_csv(f_name_cleaned) # cleaned from filtered + if root_folder is not None: + f_name_cleaned = os.path.join(root_folder, f_name_cleaned) + if not os.path.exists(f_name_cleaned) or overwrite: + df.write_csv(f_name_cleaned) # cleaned from filtered f_name_uncleaned = None @@ -119,7 +129,10 @@ def clean_data_from_file(self, cutoff=5.0, t_calib=30., t_window=0.05, rec_lengt ret_two = np.vstack((ts_buffer_no_asr_np, data_buffer_no_asr_np)) df_no_asr = pl.DataFrame(ret_two.swapaxes(1, 0), schema=n) f_name_uncleaned = f"filtered_exg_calib-{t_calib}_window-{t_window}_t-{rec_length}.csv" - df_no_asr.write_csv(f_name_uncleaned) # uncleaned but filtered + if root_folder is not None: + f_name_uncleaned = os.path.join(root_folder, f_name_uncleaned) + if not os.path.exists(f_name_uncleaned) or overwrite: + df_no_asr.write_csv(f_name_uncleaned) # uncleaned but filtered self.dev.disconnect() time.sleep(1.) @@ -203,7 +216,7 @@ def plot_comp_from_dataframes(self, channels: list[int] = None): ax.plot(tup[0]["TimeStamp"][:self.max_plot_indices], tup[0][tup[0].columns[idx_to_plot + 1]][:self.max_plot_indices], tup[1], label=tup[2]) - ax.title.set_text(f"Channel {idx_to_plot}") + ax.title.set_text(f"Channel {idx_to_plot + 1}") # start label for channels at 1 it += 1 if ax is not None: legend_handles, legend_labels = ax.get_legend_handles_labels() @@ -275,13 +288,19 @@ def compare_cleaned_with_uncleaned(self): self.plot_comp_from_dataframes([1, 2, 3, 4, 5, 6, 7, 8]) -def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window_list: list, rec_length_list: list): +def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window_list: list, rec_length_list: list, input_file=None): data_writer = DataRecorder(number_channels) data_comparator = DataComparator() + root_folder = None + if input_file is not None: + root_folder = os.path.splitext(os.path.split(input_file)[1])[0] + if not os.path.exists(root_folder): + os.mkdir(root_folder) + uncleaned_file = None cleaned_files = [] - calib_file_path = data_writer.write_calibration_data(t_calib=t_calib_length) + calib_file_path = data_writer.write_calibration_data(t_calib=t_calib_length, file=input_file, root_folder=root_folder) uncleaned_plot_colour = "gray" eegprep_plot_colour = "red" @@ -295,7 +314,9 @@ def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window t_window=t_window, rec_length=rec_length, calib_file=calib_file_path, - record_raw_data=uncleaned_file is None) + record_raw_data=uncleaned_file is None, + file=input_file, + root_folder=root_folder) if uncleaned_path is not None and uncleaned_file is None: uncleaned_file = (uncleaned_path, uncleaned_plot_colour, "Uncleaned (filtered)") cleaned_files.append((cleaned_path, @@ -313,20 +334,24 @@ def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window eeg_prep_tup = (eegprep_cleaned, eegprep_plot_colour, f"Cleaned (eegprep), cutoff={cutoff}") cleaned_files.append(eeg_prep_tup) data_comparator.add_dataframe(*eeg_prep_tup) - data_comparator.plot_comp_from_dataframes(channels=[1,2,3,4,5,6,7,8,]) + data_comparator.plot_comp_from_dataframes(channels=[1,2,3,4,5,6,7,8,]) # Note that this starts from the second channel! if __name__ == '__main__': n_ch = 32 cutoff = 5.0 - t_windows_to_test = [0.01, 0.05] + t_windows_to_test = [0.01, 0.05, 0.5, 1.0, 5.0] t_calib_to_test = 10. rec_length_to_test = [20.] - perform_matrix_test(n_ch, cutoff, t_calib_to_test, t_windows_to_test, rec_length_to_test) + input_file = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/32channel_semidry_artefacts_ExG.csv" + + perform_matrix_test(n_ch, cutoff, t_calib_to_test, t_windows_to_test, rec_length_to_test, input_file=input_file) + + as_pl_df = pl.read_csv(input_file) + as_pl_df_with_dead_channel = add_dead_channels_to_dataframe(as_pl_df, 1) + input_file_corrupted = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/32channel_semidry_artefacts_ExG_1ch_corrupted.csv" - # TODO gather files + metadata for comparison plots - # TODO clear buffers between runs - # TODO add channel dropping to matrix test + as_pl_df_with_dead_channel.write_csv(input_file_corrupted) - # compare_cleaned_with_uncleaned() + perform_matrix_test(n_ch, cutoff, t_calib_to_test, t_windows_to_test, rec_length_to_test, input_file=input_file_corrupted) diff --git a/src/explorepy/csv_client.py b/src/explorepy/csv_client.py index 9fc4963c..d8278021 100644 --- a/src/explorepy/csv_client.py +++ b/src/explorepy/csv_client.py @@ -1,4 +1,5 @@ import time +import os from enum import Enum, auto import numpy as np from explorepy.packet import BleImpedancePacket, DeviceInfoBLE, OrientationV1, OrientationV2 @@ -19,13 +20,19 @@ class PacketSize(Enum): DEVICE_INFO = 38 class CsvClient: - def __init__(self, channel_count): - file_path = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/" - file_name = "32channel_semidry_artefacts_ExG.csv" - #file_name = "Explore_DABH_Artefacts_60sCalibration_ExG.csv" - self.server = server = CsvServer( + def __init__(self, channel_count, file_path: str=None): + if file_path is None: + self.file_name = "32channel_semidry_artefacts_ExG.csv" + file_path = os.path.join("/Users/sonjastefani/Documents/dev/explore-desktop/test-data/", self.file_name) + else: + self.file_name = os.path.split(file_path)[-1] + + if not os.path.isfile(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + + self.server = CsvServer( channel_count=channel_count, - csv_path=file_path + file_name, + csv_path=file_path, loop=True ) self._state = ClientState.DISCONNECTED diff --git a/src/explorepy/explore.py b/src/explorepy/explore.py index 17d307e3..27911e0f 100644 --- a/src/explorepy/explore.py +++ b/src/explorepy/explore.py @@ -83,7 +83,7 @@ def is_measuring_imp(self): imp_mode = self.stream_processor._is_imp_mode return imp_mode - def connect(self, device_name=None, mac_address=None): + def connect(self, device_name=None, mac_address=None, file_path=None): r""" Connects to the nearby device. If there are more than one device, the user is asked to choose one of them. @@ -100,7 +100,7 @@ def connect(self, device_name=None, mac_address=None): self.stream_processor = StreamProcessor( debug=True if self.debug else False) self.stream_processor.start( - device_name=device_name, mac_address=mac_address) + device_name=device_name, mac_address=mac_address, file_path=file_path) cnt = 0 cnt_limit = 20 if self.debug else 15 while "adc_mask" not in self.stream_processor.device_info: diff --git a/src/explorepy/parser.py b/src/explorepy/parser.py index 88d6a305..525cedac 100644 --- a/src/explorepy/parser.py +++ b/src/explorepy/parser.py @@ -72,7 +72,7 @@ def __init__(self, callback, progress_callback=None, mode='device', debug=True): self.header_len = 0 self.data_len = 0 - def start_streaming(self, device_name, mac_address): + def start_streaming(self, device_name, mac_address, file_path=None): """Start streaming data from Explore device""" self.device_name = device_name explorepy.set_bt_interface('csv') @@ -87,7 +87,7 @@ def start_streaming(self, device_name, mac_address): from explorepy.csv_client import CsvClient device_id = str.split(device_name, '_')[1] # split by underscore and take second part ch_count = 32 if device_id[0] == 'D' else 8 #dynamic channel selection based on first character - self.stream_interface = CsvClient(ch_count) + self.stream_interface = CsvClient(ch_count, file_path=file_path) elif is_usb_mode(): from explorepy.serial_client import SerialStream self.stream_interface = SerialStream(device_name=device_name) diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index 590ed860..ce61003c 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -103,7 +103,7 @@ def unsubscribe(self, callback, topic): logger.debug(f"Unsubscribe {callback} from {topic}") self.subscribers[topic].discard(callback) - def start(self, device_name=None, mac_address=None): + def start(self, device_name=None, mac_address=None, file_path=None): """Start streaming from Explore device Args: @@ -116,7 +116,7 @@ def start(self, device_name=None, mac_address=None): self.device_info["device_name"] = device_name self.parser = Parser(callback=self.process, mode='device', debug=self.debug) - self.parser.start_streaming(device_name, mac_address) + self.parser.start_streaming(device_name, mac_address, file_path=file_path) self.is_connected = True self._device_configurator = DeviceConfiguration( bt_interface=self.parser.stream_interface) From ca82aa72c574c40318c403e7258fc967f73c1d59 Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Tue, 17 Feb 2026 11:18:54 +0100 Subject: [PATCH 12/47] Updated asr example, update asr processor - ASR example now saves channels that are corrupted so they (and the channels around them) are passed to plotting - ASR processor updated with bigger time window range (0.01 to 30.0 seconds) --- examples/clean_data_from_exg.py | 57 ++++++++++++--------------------- 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/examples/clean_data_from_exg.py b/examples/clean_data_from_exg.py index 991a5c03..bb304344 100644 --- a/examples/clean_data_from_exg.py +++ b/examples/clean_data_from_exg.py @@ -233,7 +233,7 @@ def add_dead_channels_to_dataframe(dataframe, count): dataframe = dataframe.replace_column(idx, pl.Series(name=column_name, values=np.full(dataframe.shape[0], -400000.05))) - return dataframe + return dataframe, drop_indices def call_clean_from_eegprep(raw_data_file, calib_file, sr, cutoff, n_chan, first_ts, last_ts): @@ -262,33 +262,7 @@ def call_clean_from_eegprep(raw_data_file, calib_file, sr, cutoff, n_chan, first return cleaned_df -def compare_cleaned_with_uncleaned(self): - file_path_raw = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/32channel_semidry_artefacts_ExG_1ch_corrupted.csv" - raw = pl.read_csv(file_path_raw) - - file_path_cleaned = f"./corrupted_asr_cleaned-data_calib-{calib_time}_window-{asr_window}_t-{t}.csv" - comp_cleaned_df = pl.read_csv(file_path_cleaned) - - first_ts_cleaned = comp_cleaned_df["TimeStamp"][0] - last_ts_cleaned = comp_cleaned_df["TimeStamp"][-1] - - file_path_filtered = f"corrupted_filtered_exg_calib-{calib_time}_window-{asr_window}_t-{t}.csv" - filtered = pl.read_csv(file_path_filtered) - matched_filtered = filtered.remove(pl.col("TimeStamp") < first_ts_cleaned) - matched_filtered = matched_filtered.remove(pl.col("TimeStamp") > last_ts_cleaned) - - calib_file = f"asr_calibration-data_calib-{calib_time}.csv" - - cleaned_df = call_clean_from_eegprep(raw, calib_file=calib_file, sr=250, cutoff=asr_window, n_chan=32, first_ts=None, last_ts=None) - - dataframes = [(matched_filtered, "r", "Uncleaned (filtered)"), - (comp_cleaned_df, "b", f"Cleaned with window = {asr_window}s"), - (cleaned_df, "g", "Cleaned with clean_asr")] - - self.plot_comp_from_dataframes([1, 2, 3, 4, 5, 6, 7, 8]) - - -def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window_list: list, rec_length_list: list, input_file=None): +def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window_list: list, rec_length_list: list, input_file=None, plot_idx=None): data_writer = DataRecorder(number_channels) data_comparator = DataComparator() @@ -300,7 +274,7 @@ def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window uncleaned_file = None cleaned_files = [] - calib_file_path = data_writer.write_calibration_data(t_calib=t_calib_length, file=input_file, root_folder=root_folder) + calib_file_path = data_writer.write_calibration_data(t_calib=t_calib_length, file=input_file, root_folder=root_folder, overwrite=True) uncleaned_plot_colour = "gray" eegprep_plot_colour = "red" @@ -316,7 +290,8 @@ def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window calib_file=calib_file_path, record_raw_data=uncleaned_file is None, file=input_file, - root_folder=root_folder) + root_folder=root_folder, + overwrite=True) if uncleaned_path is not None and uncleaned_file is None: uncleaned_file = (uncleaned_path, uncleaned_plot_colour, "Uncleaned (filtered)") cleaned_files.append((cleaned_path, @@ -334,24 +309,34 @@ def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window eeg_prep_tup = (eegprep_cleaned, eegprep_plot_colour, f"Cleaned (eegprep), cutoff={cutoff}") cleaned_files.append(eeg_prep_tup) data_comparator.add_dataframe(*eeg_prep_tup) - data_comparator.plot_comp_from_dataframes(channels=[1,2,3,4,5,6,7,8,]) # Note that this starts from the second channel! + if plot_idx is None or type(plot_idx) is not list: + print("plot_idx is not give or the wrong time, using fall back index list...") + plot_idx = [0, 1, 2, 3, 4, 5, 6, 7,] + data_comparator.plot_comp_from_dataframes(channels=plot_idx) if __name__ == '__main__': n_ch = 32 cutoff = 5.0 - t_windows_to_test = [0.01, 0.05, 0.5, 1.0, 5.0] - t_calib_to_test = 10. - rec_length_to_test = [20.] + t_windows_to_test = [0.01, 0.05, 1.0, 5.0, 15., 30.] + t_calib_to_test = 30. + rec_length_to_test = [180.] input_file = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/32channel_semidry_artefacts_ExG.csv" perform_matrix_test(n_ch, cutoff, t_calib_to_test, t_windows_to_test, rec_length_to_test, input_file=input_file) as_pl_df = pl.read_csv(input_file) - as_pl_df_with_dead_channel = add_dead_channels_to_dataframe(as_pl_df, 1) + as_pl_df_with_dead_channel, dropped_channels = add_dead_channels_to_dataframe(as_pl_df, 2) + print(f"Dropped channels: {dropped_channels}") + to_plot = np.array([dropped_channels[0]-1, dropped_channels[0], dropped_channels[0]+1, + dropped_channels[1]-1, dropped_channels[1], dropped_channels[1]+1]) + to_plot = to_plot[to_plot >= 0] + to_plot = to_plot[to_plot < n_ch] + to_plot = np.unique(to_plot) + print(f"Indices to plot: {to_plot}") input_file_corrupted = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/32channel_semidry_artefacts_ExG_1ch_corrupted.csv" as_pl_df_with_dead_channel.write_csv(input_file_corrupted) - perform_matrix_test(n_ch, cutoff, t_calib_to_test, t_windows_to_test, rec_length_to_test, input_file=input_file_corrupted) + perform_matrix_test(n_ch, cutoff, t_calib_to_test, t_windows_to_test, rec_length_to_test, input_file=input_file_corrupted, plot_idx=list(to_plot)) From 613d3bd82e7f95d124c3356875c08f0051104026 Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Wed, 18 Feb 2026 16:33:24 +0100 Subject: [PATCH 13/47] Added skipping already recorded files in clean example --- examples/clean_data_from_exg.py | 101 +++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 21 deletions(-) diff --git a/examples/clean_data_from_exg.py b/examples/clean_data_from_exg.py index bb304344..5a0f4102 100644 --- a/examples/clean_data_from_exg.py +++ b/examples/clean_data_from_exg.py @@ -67,10 +67,26 @@ def set_up_explore_device(self, t_calib=30., dev_name="Explore_DABC", notch=50., return self.dev + @staticmethod + def get_calibration_filename(t_calib): + return f"asr_calibration-data_calib-{t_calib}.csv" + + @staticmethod + def get_clean_filename(t_calib, t_window, rec_length): + return f"asr_cleaned-data_calib-{t_calib}_window-{t_window}_t-{rec_length}.csv" + + @staticmethod + def get_filtered_filename(rec_length): + return f"filtered_exg_calib-{rec_length}.csv" + + @staticmethod + def get_eegprep_filename(t_calib): + return f"eegprep_cleaned_exg_calib-{t_calib}.csv" + def write_calibration_data(self, t_calib=30., calib_file=None, file=None, root_folder=None, overwrite=False): self.dev = self.set_up_explore_device(t_calib=t_calib, file=file) if calib_file is None: - calib_file = f"asr_calibration-data_calib-{t_calib}.csv" + calib_file = self.get_calibration_filename(t_calib) if root_folder is not None: calib_file = os.path.join(root_folder, calib_file) @@ -80,6 +96,7 @@ def write_calibration_data(self, t_calib=30., calib_file=None, file=None, root_f calib_data = self.dev.stream_processor.asr_processor.calibration_data_input calib_df = pl.DataFrame(calib_data.swapaxes(1, 0)) if not os.path.exists(calib_file) or overwrite: + print(f"Writing calibration data to {calib_file}") calib_df.write_csv(calib_file) self.dev.disconnect() time.sleep(1.) @@ -114,10 +131,11 @@ def clean_data_from_file(self, cutoff=5.0, t_calib=30., t_window=0.05, rec_lengt n.extend([f"ch{i + 1}" for i in range(self.n_channels)]) ret = np.vstack((ts_buffer_np, data_buffer_np)) df = pl.DataFrame(ret.swapaxes(1, 0), schema=n) - f_name_cleaned = f"asr_cleaned-data_calib-{t_calib}_window-{t_window}_t-{rec_length}.csv" + f_name_cleaned = self.get_clean_filename(t_calib, t_window, rec_length) if root_folder is not None: f_name_cleaned = os.path.join(root_folder, f_name_cleaned) if not os.path.exists(f_name_cleaned) or overwrite: + print(f"Writing cleaned data to {f_name_cleaned}") df.write_csv(f_name_cleaned) # cleaned from filtered f_name_uncleaned = None @@ -128,10 +146,11 @@ def clean_data_from_file(self, cutoff=5.0, t_calib=30., t_window=0.05, rec_lengt ret_two = np.vstack((ts_buffer_no_asr_np, data_buffer_no_asr_np)) df_no_asr = pl.DataFrame(ret_two.swapaxes(1, 0), schema=n) - f_name_uncleaned = f"filtered_exg_calib-{t_calib}_window-{t_window}_t-{rec_length}.csv" + f_name_uncleaned = self.get_filtered_filename(rec_length) if root_folder is not None: f_name_uncleaned = os.path.join(root_folder, f_name_uncleaned) if not os.path.exists(f_name_uncleaned) or overwrite: + print(f"Writing uncleaned data to {f_name_uncleaned}") df_no_asr.write_csv(f_name_uncleaned) # uncleaned but filtered self.dev.disconnect() @@ -226,7 +245,7 @@ def plot_comp_from_dataframes(self, channels: list[int] = None): def add_dead_channels_to_dataframe(dataframe, count): n_channels = len(dataframe.columns) - 1 - drop_indices = random.sample(range(1, n_channels), count) + drop_indices = random.sample(range(1, n_channels), count) # from 1 as col[0] is "TimeStamp" for idx in drop_indices: column_name = dataframe.columns[idx] @@ -262,7 +281,9 @@ def call_clean_from_eegprep(raw_data_file, calib_file, sr, cutoff, n_chan, first return cleaned_df -def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window_list: list, rec_length_list: list, input_file=None, plot_idx=None): +def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window_list: list, rec_length_list: list, + input_file=None, plot_idx=None, overwrite_calib=False, overwrite_processor_cleaned=False, + overwrite_eegprep_cleaned=False): data_writer = DataRecorder(number_channels) data_comparator = DataComparator() @@ -274,7 +295,19 @@ def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window uncleaned_file = None cleaned_files = [] - calib_file_path = data_writer.write_calibration_data(t_calib=t_calib_length, file=input_file, root_folder=root_folder, overwrite=True) + expected_calib_path = os.path.join(root_folder if root_folder is not None else ".", + data_writer.get_calibration_filename(t_calib_length)) + if overwrite_calib or not os.path.exists(expected_calib_path): + calib_file_path = data_writer.write_calibration_data(t_calib=t_calib_length, + file=input_file, + root_folder=root_folder, + overwrite=True) + else: + print("\n--- Skipping write_calibration_data ---\n") + calib_file_path = expected_calib_path + print(f"Calibration file path: {calib_file_path}") + + print(f"Running cleaning with calibration file: {calib_file_path}") uncleaned_plot_colour = "gray" eegprep_plot_colour = "red" @@ -283,15 +316,28 @@ def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window it = 0 for t_window in t_window_list: for rec_length in rec_length_list: - cleaned_path, uncleaned_path = data_writer.clean_data_from_file(cutoff=cutoff, - t_calib=t_calib_length, - t_window=t_window, - rec_length=rec_length, - calib_file=calib_file_path, - record_raw_data=uncleaned_file is None, - file=input_file, - root_folder=root_folder, - overwrite=True) + expected_clean_path = os.path.join(root_folder if root_folder is not None else ".", + data_writer.get_clean_filename(t_calib_length, t_window, rec_length)) + expected_unclean_path = os.path.join(root_folder if root_folder is not None else ".", + data_writer.get_filtered_filename(rec_length)) + if (overwrite_processor_cleaned + or not os.path.exists(expected_clean_path) + or not os.path.exists(expected_unclean_path)): + print(f"Uncleaned_file is None? {uncleaned_file is None}") + cleaned_path, uncleaned_path = data_writer.clean_data_from_file(cutoff=cutoff, + t_calib=t_calib_length, + t_window=t_window, + rec_length=rec_length, + calib_file=calib_file_path, + record_raw_data=uncleaned_file is None, + file=input_file, + root_folder=root_folder, + overwrite=True) + else: + print(f"\n--- Skipping clean_data_from_file for t_window {t_window} and rec_length {rec_length} ---\n") + cleaned_path = expected_clean_path if os.path.exists(expected_clean_path) else None + uncleaned_path = expected_unclean_path if os.path.exists(expected_unclean_path) else None + print(f"Cleaned path: {cleaned_path}\nUncleaned path: {uncleaned_path}") if uncleaned_path is not None and uncleaned_file is None: uncleaned_file = (uncleaned_path, uncleaned_plot_colour, "Uncleaned (filtered)") cleaned_files.append((cleaned_path, @@ -304,8 +350,15 @@ def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window data_comparator.add_dataframe_from_file(*uncleaned_file) data_comparator.cull_dataframes() first_ts, last_ts = data_comparator.get_first_and_last_ts() - eegprep_cleaned = call_clean_from_eegprep(raw_data_file=uncleaned_file[0], calib_file=calib_file_path, sr=250, - cutoff=5.0, n_chan=32, first_ts=first_ts, last_ts=last_ts) + eegprep_cleaned_path = os.path.join(root_folder if root_folder is not None else ".", + data_writer.get_eegprep_filename(t_calib_length)) + if overwrite_eegprep_cleaned or not os.path.exists(eegprep_cleaned_path): + eegprep_cleaned = call_clean_from_eegprep(raw_data_file=uncleaned_file[0], calib_file=calib_file_path, sr=250, + cutoff=5.0, n_chan=32, first_ts=first_ts, last_ts=last_ts) + eegprep_cleaned.write_csv(eegprep_cleaned_path) + else: + print(f"\n--- Skipping call_clean_from_eegprep, loading instead from {eegprep_cleaned_path} ---\n") + eegprep_cleaned = pl.read_csv(eegprep_cleaned_path) eeg_prep_tup = (eegprep_cleaned, eegprep_plot_colour, f"Cleaned (eegprep), cutoff={cutoff}") cleaned_files.append(eeg_prep_tup) data_comparator.add_dataframe(*eeg_prep_tup) @@ -318,24 +371,30 @@ def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window n_ch = 32 cutoff = 5.0 - t_windows_to_test = [0.01, 0.05, 1.0, 5.0, 15., 30.] + t_windows_to_test = [0.05, 1.0] t_calib_to_test = 30. - rec_length_to_test = [180.] + rec_length_to_test = [10.] + + # replace with valid path to test recording (csv only) + test_file_name = "32channel_semidry_artefacts_ExG" + test_file_root = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data" - input_file = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/32channel_semidry_artefacts_ExG.csv" + input_file = os.path.join(test_file_root, f"{test_file_name}.csv") perform_matrix_test(n_ch, cutoff, t_calib_to_test, t_windows_to_test, rec_length_to_test, input_file=input_file) as_pl_df = pl.read_csv(input_file) as_pl_df_with_dead_channel, dropped_channels = add_dead_channels_to_dataframe(as_pl_df, 2) print(f"Dropped channels: {dropped_channels}") + dropped_channels = [e-1 for e in dropped_channels] to_plot = np.array([dropped_channels[0]-1, dropped_channels[0], dropped_channels[0]+1, dropped_channels[1]-1, dropped_channels[1], dropped_channels[1]+1]) to_plot = to_plot[to_plot >= 0] to_plot = to_plot[to_plot < n_ch] to_plot = np.unique(to_plot) print(f"Indices to plot: {to_plot}") - input_file_corrupted = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data/32channel_semidry_artefacts_ExG_1ch_corrupted.csv" + + input_file_corrupted = os.path.join(test_file_root, f"{test_file_name}_ch-{"-".join(map(str, dropped_channels))}_corrupted.csv") as_pl_df_with_dead_channel.write_csv(input_file_corrupted) From 2d746d58fc5670ae6383f47f333b909f4e2559fd Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Tue, 24 Feb 2026 10:53:49 +0100 Subject: [PATCH 14/47] Added plotting from folder to ASR example, made filenames less hard-coded --- examples/clean_data_from_exg.py | 95 ++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 12 deletions(-) diff --git a/examples/clean_data_from_exg.py b/examples/clean_data_from_exg.py index 5a0f4102..94d2ec99 100644 --- a/examples/clean_data_from_exg.py +++ b/examples/clean_data_from_exg.py @@ -1,6 +1,8 @@ # Note: This is a WIP +import glob import os import random +from typing import Union import explorepy from explorepy.stream_processor import TOPICS @@ -11,6 +13,18 @@ import matplotlib.pyplot as plt +CALIB_LENGTH_STR = "calib" +WINDOW_STR = "window" +REC_TIME_STR = "t" + +EEGPREP_STR = "eegprep" +CALIBRATION_DATA_STR = "calibdata" +FILTERED_STR = "filtered" +ASR_CLEANED_STR = "asrcleaned" + +BLOCK_SEPARATOR = "_" +VALUE_SEPARATOR = "-" + calib_time = 30. asr_window = 0.05 t = 180. @@ -51,6 +65,8 @@ def set_up_explore_device(self, t_calib=30., dev_name="Explore_DABC", notch=50., self.dev = explorepy.Explore() self.dev.connect(dev_name, file_path=file) + time.sleep(4.0) + self.dev.stream_processor.add_filter(cutoff_freq=notch, filter_type="notch") self.dev.stream_processor.add_filter(cutoff_freq=bp, filter_type="bandpass") f_name = self.dev.stream_processor.parser.stream_interface.file_name @@ -69,28 +85,41 @@ def set_up_explore_device(self, t_calib=30., dev_name="Explore_DABC", notch=50., @staticmethod def get_calibration_filename(t_calib): - return f"asr_calibration-data_calib-{t_calib}.csv" + return (f"{CALIBRATION_DATA_STR}" + f"{BLOCK_SEPARATOR}" + f"{CALIB_LENGTH_STR}{VALUE_SEPARATOR}{t_calib}.csv") @staticmethod def get_clean_filename(t_calib, t_window, rec_length): - return f"asr_cleaned-data_calib-{t_calib}_window-{t_window}_t-{rec_length}.csv" + return (f"{ASR_CLEANED_STR}" + f"{BLOCK_SEPARATOR}" + f"{CALIB_LENGTH_STR}{VALUE_SEPARATOR}{t_calib}" + f"{BLOCK_SEPARATOR}" + f"{WINDOW_STR}{VALUE_SEPARATOR}{t_window}" + f"{BLOCK_SEPARATOR}" + f"{REC_TIME_STR}{VALUE_SEPARATOR}{rec_length}.csv") @staticmethod def get_filtered_filename(rec_length): - return f"filtered_exg_calib-{rec_length}.csv" + return (f"{FILTERED_STR}" + f"{BLOCK_SEPARATOR}" + f"{CALIB_LENGTH_STR}{VALUE_SEPARATOR}{rec_length}.csv") @staticmethod def get_eegprep_filename(t_calib): - return f"eegprep_cleaned_exg_calib-{t_calib}.csv" + return (f"{EEGPREP_STR}" + f"{BLOCK_SEPARATOR}" + f"{CALIB_LENGTH_STR}{VALUE_SEPARATOR}{t_calib}.csv") def write_calibration_data(self, t_calib=30., calib_file=None, file=None, root_folder=None, overwrite=False): - self.dev = self.set_up_explore_device(t_calib=t_calib, file=file) if calib_file is None: calib_file = self.get_calibration_filename(t_calib) if root_folder is not None: calib_file = os.path.join(root_folder, calib_file) + self.dev = self.set_up_explore_device(t_calib=t_calib, file=file) + time.sleep(1.) calib_data = self.dev.stream_processor.asr_processor.calibration_data_input @@ -113,6 +142,8 @@ def clean_data_from_file(self, cutoff=5.0, t_calib=30., t_window=0.05, rec_lengt as_np = calib_file_df.to_numpy() as_np = as_np.swapaxes(1, 0) + print(f"Setting calib data from {calib_file}") + self.dev.stream_processor.asr_processor.calibration_data_input = as_np self.dev.stream_processor.asr_processor.set_state_from_calibration_data(calib_data=as_np) @@ -199,7 +230,7 @@ def add_dataframe(self, dataframe, colour, label): def clear_dataframes(self): self.dataframes = [] - def plot_comp_from_dataframes(self, channels: list[int] = None): + def plot_comp_from_dataframes(self, channels: list[int] = None, max_samples : Union[str, int]="default"): """Plots comparison of any dataframes' first channel from a list of tuples Args: @@ -207,6 +238,13 @@ def plot_comp_from_dataframes(self, channels: list[int] = None): in the tuple being a colour string to use for plotting with matplotlib, i.e. "b" for blue etc. channels: list of ints that defines which channels to plot """ + n_samples_to_plot = self.max_plot_indices + if max_samples: + if type(max_samples) == int: + n_samples_to_plot = max(max_samples, 1) + elif type(max_samples) == str: + if max_samples in ["all", "full"]: + n_samples_to_plot = None if channels is None: print("No channels supplied for plotting, exiting...") return @@ -232,8 +270,8 @@ def plot_comp_from_dataframes(self, channels: list[int] = None): assert type(tup[0]) is pl.DataFrame assert type(tup[1]) is str assert type(tup[2]) is str - ax.plot(tup[0]["TimeStamp"][:self.max_plot_indices], - tup[0][tup[0].columns[idx_to_plot + 1]][:self.max_plot_indices], + ax.plot(tup[0]["TimeStamp"][:n_samples_to_plot], + tup[0][tup[0].columns[idx_to_plot + 1]][:n_samples_to_plot], tup[1], label=tup[2]) ax.title.set_text(f"Channel {idx_to_plot + 1}") # start label for channels at 1 it += 1 @@ -323,7 +361,7 @@ def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window if (overwrite_processor_cleaned or not os.path.exists(expected_clean_path) or not os.path.exists(expected_unclean_path)): - print(f"Uncleaned_file is None? {uncleaned_file is None}") + print(f"\n--- Cleaning data with t_window={t_window}, rec_length: {rec_length}, cutoff: {cutoff}---\n") cleaned_path, uncleaned_path = data_writer.clean_data_from_file(cutoff=cutoff, t_calib=t_calib_length, t_window=t_window, @@ -367,16 +405,49 @@ def perform_matrix_test(number_channels, cutoff, t_calib_length: float, t_window plot_idx = [0, 1, 2, 3, 4, 5, 6, 7,] data_comparator.plot_comp_from_dataframes(channels=plot_idx) +def plot_recordings_from_folder(folder, idx=(0, 1, 2, 3, 4, 5, 6, 7)): + data_comparator = DataComparator() + files = glob.glob(os.path.join(folder, "*.csv")) + files_for_comparator = [] + + asr_processor_plot_colours = ["green", "skyblue", "violet", "gold", "aquamarine", "mediumpurple"] + + it = 0 + for f_name in files: + if CALIBRATION_DATA_STR in f_name: + pass + else: + if EEGPREP_STR in f_name: + colour = "red" + desc = "Eegprep cleaned" + elif FILTERED_STR in f_name: + colour = "gray" + desc = "Filtered, uncleaned" + else: + blocks = os.path.splitext(os.path.basename(f_name))[0].split("_") + desc = ", ".join(blocks[1:]) + desc = f"ASR cleaned - {desc}" + colour = asr_processor_plot_colours[it % len(asr_processor_plot_colours)] + files_for_comparator.append((f_name, colour, desc)) + it += 1 + + for cleaned_file in files_for_comparator: + data_comparator.add_dataframe_from_file(*cleaned_file) + data_comparator.cull_dataframes() + data_comparator.plot_comp_from_dataframes(channels=idx, max_samples="all") + + if __name__ == '__main__': n_ch = 32 cutoff = 5.0 - t_windows_to_test = [0.05, 1.0] + t_windows_to_test = [0.05, 0.1, 1.0, 5.0, 10.0] t_calib_to_test = 30. - rec_length_to_test = [10.] + rec_length_to_test = [180.] # calib_t + rec_t should be lower than length of test file + # replace with valid path to test recording (csv only) - test_file_name = "32channel_semidry_artefacts_ExG" + test_file_name = "32channel_semidry_artefacts_ExG" # t = 243.996 test_file_root = "/Users/sonjastefani/Documents/dev/explore-desktop/test-data" input_file = os.path.join(test_file_root, f"{test_file_name}.csv") From dcb7bbb66447575d21c48219643cfbfe8863f443 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Tue, 24 Feb 2026 15:31:54 +0100 Subject: [PATCH 15/47] lower order for 1000 Hz filter --- src/explorepy/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/explorepy/filters.py b/src/explorepy/filters.py index f6c309e5..8b605bc1 100644 --- a/src/explorepy/filters.py +++ b/src/explorepy/filters.py @@ -31,7 +31,7 @@ def __init__(self, cutoff_freq, filter_type, s_rate, n_chan, order=5): self.s_rate = float(s_rate) self.filter_type = filter_type self.filter_param = None - order = 2 if self.s_rate > 1000 else order + order = 2 if self.s_rate >= 1000 else order a, b, zi = self.get_filter_coeffs(cutoff_freq, filter_type, s_rate, n_chan, order) self.filter_param = {'a': a, 'b': b, 'zi': zi} From 3cd48e3cf6ea756ba8a43bbaa7e5ba60518717c0 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Tue, 24 Feb 2026 16:34:47 +0100 Subject: [PATCH 16/47] add bumpversion configs to toml --- .bumpversion.cfg | 26 -------------------------- pyproject.toml | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 26 deletions(-) delete mode 100644 .bumpversion.cfg diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 9d795327..00000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,26 +0,0 @@ -[bumpversion] -current_version = 4.3.1 -commit = False -tag = False - -[bumpversion:file:pyproject.toml] - -[bumpversion:file:README.rst] -search = v{current_version}. -replace = v{new_version}. - -[bumpversion:file:docs/conf.py] -search = version = release = '{current_version}' -replace = version = release = '{new_version}' - -[bumpversion:file:src/explorepy/__init__.py] -search = __version__ = '{current_version}' -replace = __version__ = '{new_version}' - -[bumpversion:file:installer/windows/installer.cfg] -search = explorepy=={current_version} -replace = explorepy=={new_version} - -[bumpversion:file (second):installer/windows/installer.cfg] -search = version={current_version} -replace = version={new_version} diff --git a/pyproject.toml b/pyproject.toml index ecec706d..82be6745 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,3 +60,39 @@ test = [ [project.scripts] explorepy = "explorepy.cli:cli" + +# bumpoversion config +[tool.bumpversion] +current_version = "4.4.0" +commit = false +tag = false + +[[tool.bumpversion.files]] +filename = "pyproject.toml" +search = 'version = "{current_version}"' +replace = 'version = "{new_version}"' + +[[tool.bumpversion.files]] +filename = "README.rst" +search = "v{current_version}." +replace = "v{new_version}." + +[[tool.bumpversion.files]] +filename = "docs/conf.py" +search = "version = release = '{current_version}'" +replace = "version = release = '{new_version}'" + +[[tool.bumpversion.files]] +filename = "src/explorepy/__init__.py" +search = "__version__ = '{current_version}'" +replace = "__version__ = '{new_version}'" + +[[tool.bumpversion.files]] +filename = "installer/windows/installer.cfg" +search = "explorepy=={current_version}" +replace = "explorepy=={new_version}" + +[[tool.bumpversion.files]] +filename = "installer/windows/installer.cfg" +search = "version={current_version}" +replace = "version={new_version}" From ed296c4978b3e2a0bad7a23fd5412374407010bd Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Mon, 2 Mar 2026 15:56:16 +0100 Subject: [PATCH 17/47] separate notch filter from general workflow --- src/explorepy/stream_processor.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index ef647004..cbbc9884 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -78,6 +78,7 @@ def __init__(self, debug=False): self.reset_timer() self.packet_count = 0 self.progress = 0 + self._notch_filter = None def subscribe(self, callback, topic): """Subscribe a function to a topic @@ -312,13 +313,17 @@ def process(self, packet): self.last_exg_packet_timestamp = get_local_time() missing_timestamps = self.fill_missing_packet(packet) self._update_last_time_point(packet, received_time) - self.dispatch(topic=TOPICS.raw_ExG, packet=packet) + self.packet_count += 1 if self._is_imp_mode and self.imp_calculator: packet_imp = self.imp_calculator.measure_imp( packet=copy.deepcopy(packet)) if packet_imp is not None: self.dispatch(topic=TOPICS.imp, packet=packet_imp) + + self.dispatch(topic=TOPICS.raw_ExG, packet=self._notch_filter.apply(packet=packet, in_place=False)) + else: + self.dispatch(topic=TOPICS.raw_ExG, packet=packet) try: self.apply_filters(packet=packet) except ValueError as error: @@ -583,3 +588,11 @@ def fill_missing_packet(self, packet): timestamps = np.linspace(self._last_packet_timestamp + sps, packet.timestamp, num=missing_samples, endpoint=True) return timestamps[:-1] + + def _add_notch_filter(self): + self._notch_filter = ExGFilter( + cutoff_freq=(61, 64), + filter_type='notch_imp', + s_rate=250, + n_chan=SettingsManager(self.device_info['device_name']).get_channel_count() + ) From c30e74a21b2d83e9c4ae934f8b164f910644e7f6 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Mon, 2 Mar 2026 18:07:17 +0100 Subject: [PATCH 18/47] check if notch filter exists --- src/explorepy/stream_processor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index cbbc9884..e432f487 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -313,15 +313,15 @@ def process(self, packet): self.last_exg_packet_timestamp = get_local_time() missing_timestamps = self.fill_missing_packet(packet) self._update_last_time_point(packet, received_time) - self.packet_count += 1 if self._is_imp_mode and self.imp_calculator: packet_imp = self.imp_calculator.measure_imp( packet=copy.deepcopy(packet)) if packet_imp is not None: self.dispatch(topic=TOPICS.imp, packet=packet_imp) - - self.dispatch(topic=TOPICS.raw_ExG, packet=self._notch_filter.apply(packet=packet, in_place=False)) + if self._notch_filter: + self._notch_filter.apply(packet) + self.dispatch(topic=TOPICS.raw_ExG, packet=packet) else: self.dispatch(topic=TOPICS.raw_ExG, packet=packet) try: @@ -590,7 +590,7 @@ def fill_missing_packet(self, packet): return timestamps[:-1] def _add_notch_filter(self): - self._notch_filter = ExGFilter( + self._notch_filter = ExGFilter( cutoff_freq=(61, 64), filter_type='notch_imp', s_rate=250, From c0d1f1ae9af72caa077c1be2c9d83bc2aa830fff Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Tue, 3 Mar 2026 16:05:40 +0100 Subject: [PATCH 19/47] get power line frequency value --- src/explorepy/explore.py | 8 +++++--- src/explorepy/filters.py | 4 ++++ src/explorepy/stream_processor.py | 17 +++++++++++++++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/explorepy/explore.py b/src/explorepy/explore.py index 6ae74f53..1e30e4a3 100644 --- a/src/explorepy/explore.py +++ b/src/explorepy/explore.py @@ -182,13 +182,15 @@ def record_data( if file_type not in ['edf', 'csv']: raise ValueError( '{} is not a supported file extension!'.format(file_type)) + if imp_mode: + if not self.stream_processor.is_imp_running(): + raise ValueError('Impedance measurement not running!') if file_type == 'edf': raise ValueError( '{} is not a supported file extension for recording impedance!'.format(file_type)) - if notch_freq is None: - raise ValueError( - 'Missing notch frequency argument, please provide the notch frequency to get live impedance values') + + notch_freq = self.stream_processor.get_power_line_freq() or 50 duration = self._check_duration(duration) diff --git a/src/explorepy/filters.py b/src/explorepy/filters.py index f6c309e5..08bc18e3 100644 --- a/src/explorepy/filters.py +++ b/src/explorepy/filters.py @@ -31,6 +31,7 @@ def __init__(self, cutoff_freq, filter_type, s_rate, n_chan, order=5): self.s_rate = float(s_rate) self.filter_type = filter_type self.filter_param = None + self.cutoff_freq = cutoff_freq order = 2 if self.s_rate > 1000 else order a, b, zi = self.get_filter_coeffs(cutoff_freq, filter_type, s_rate, n_chan, order) self.filter_param = {'a': a, 'b': b, 'zi': zi} @@ -110,6 +111,9 @@ def get_notch_coeffs(cutoff, nyquist, n_channels, order, quality=30, imp_mode=Fa zi = np.zeros(shape=(n_channels, 2)) return a, b, zi + def get_cutoff_freq(self): + return self.cutoff_freq + def apply(self, input_data, in_place=True): """Apply filter diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index e432f487..0770a73b 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -314,7 +314,7 @@ def process(self, packet): missing_timestamps = self.fill_missing_packet(packet) self._update_last_time_point(packet, received_time) self.packet_count += 1 - if self._is_imp_mode and self.imp_calculator: + if self.is_imp_running(): packet_imp = self.imp_calculator.measure_imp( packet=copy.deepcopy(packet)) if packet_imp is not None: @@ -459,9 +459,10 @@ def imp_initialize(self, notch_freq, calibration=False): cmd = ZMeasurementEnable() if self.configure_device(cmd): self.imp_calib_info['calibration'] = calibration + self._add_notch_filter() self.imp_calculator = ImpedanceMeasurement(device_info=self.device_info, calib_param=self.imp_calib_info, - notch_freq=notch_freq) + notch_freq=self.get_power_line_freq()) self._is_imp_mode = True else: raise ConnectionError('Device configuration process failed!') @@ -475,6 +476,7 @@ def disable_imp(self): return True print("WARNING: Couldn't disable impedance measurement mode. " "Please restart your device manually.") + self._notch_filter = None return False def set_marker(self, marker_string, time_lsl=None, name='mkr', soft_marker=True): @@ -596,3 +598,14 @@ def _add_notch_filter(self): s_rate=250, n_chan=SettingsManager(self.device_info['device_name']).get_channel_count() ) + + def is_imp_running(self): + return self._is_imp_mode and self.imp_calculator + + def get_power_line_freq(self): + match = next( + (item for item in self.filters if item.filter_type == 'notch'), + None + ) + + return match.cutoff_freq if match else None From 09f9730f52d7df86540bb67796cbb2910c4eab56 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Tue, 3 Mar 2026 18:53:42 +0100 Subject: [PATCH 20/47] disable impedance state change from recorder, remove debug print --- src/explorepy/explore.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/explorepy/explore.py b/src/explorepy/explore.py index 1e30e4a3..d2709eb4 100644 --- a/src/explorepy/explore.py +++ b/src/explorepy/explore.py @@ -262,7 +262,6 @@ def handle_impedance_packet(packet): impedance_values = packet.get_impedances() real_values = np.array(impedance_values) / 2 - print("Impedance:", real_values.tolist()) if file_type == 'csv': row_data = [float(packet.timestamp), *real_values] @@ -273,7 +272,6 @@ def handle_impedance_packet(packet): self.stream_processor.subscribe(callback=handle_exg_impedance_packet, topic=TOPICS.raw_ExG) self.stream_processor.subscribe(callback=handle_impedance_packet, topic=TOPICS.imp) - self.stream_processor.imp_initialize(notch_freq=notch_freq) logger.info("Recording with impedance mode...") else: self.stream_processor.subscribe(callback=self.recorders['exg'].write_data, topic=TOPICS.raw_ExG) @@ -303,8 +301,6 @@ def stop_recording(self): if is_impedance_mode: try: - self.stream_processor.disable_imp() - if 'handle_exg_impedance_callback' in self.recorders: self.stream_processor.unsubscribe( callback=self.recorders['handle_exg_impedance_callback'], topic=TOPICS.raw_ExG) From 60bab29b0a4449e03eb2730b8b9bfaba11f13100 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Tue, 3 Mar 2026 18:54:25 +0100 Subject: [PATCH 21/47] fallback to default notch on failure, set cutoff freq for imp --- src/explorepy/stream_processor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index 0770a73b..5f8a7f5d 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -462,7 +462,7 @@ def imp_initialize(self, notch_freq, calibration=False): self._add_notch_filter() self.imp_calculator = ImpedanceMeasurement(device_info=self.device_info, calib_param=self.imp_calib_info, - notch_freq=self.get_power_line_freq()) + notch_freq=self.get_power_line_freq() or notch_freq) self._is_imp_mode = True else: raise ConnectionError('Device configuration process failed!') @@ -593,7 +593,7 @@ def fill_missing_packet(self, packet): def _add_notch_filter(self): self._notch_filter = ExGFilter( - cutoff_freq=(61, 64), + cutoff_freq=62.5, filter_type='notch_imp', s_rate=250, n_chan=SettingsManager(self.device_info['device_name']).get_channel_count() From 7e1ca8f82e315118a8a15c33f3f9bce07f20b88e Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Fri, 27 Mar 2026 07:10:47 +0100 Subject: [PATCH 22/47] raw exg topic for asr subscriber --- src/explorepy/stream_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index ce61003c..b0b9068d 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -350,7 +350,7 @@ def process(self, packet): self.device_info["device_name"]) settings_manager.update_device_settings(packet.get_info()) self.dispatch(topic=TOPICS.device_info, packet=packet) - self.asr_processor = AsrProcessor(self, TOPICS.filtered_ExG) + self.asr_processor = AsrProcessor(self, TOPICS.raw_ExG) elif isinstance(packet, CommandRCV): self.dispatch(topic=TOPICS.cmd_ack, packet=packet) elif isinstance(packet, CommandStatus): From 32675f88591d8c5ab2a004ff418c475903fb08c2 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Wed, 15 Apr 2026 13:44:47 +0200 Subject: [PATCH 23/47] remove setting bt interface to csv, remove debug print --- src/explorepy/parser.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/explorepy/parser.py b/src/explorepy/parser.py index 88d6a305..af59f96b 100644 --- a/src/explorepy/parser.py +++ b/src/explorepy/parser.py @@ -75,8 +75,6 @@ def __init__(self, callback, progress_callback=None, mode='device', debug=True): def start_streaming(self, device_name, mac_address): """Start streaming data from Explore device""" self.device_name = device_name - explorepy.set_bt_interface('csv') - print(os.getcwd()) if is_ble_mode(): from explorepy.BLEClient import BLEClient self.stream_interface = BLEClient(device_name=device_name, mac_address=mac_address) From efc052b018eb2582f361441f4d22d8afcf5426f8 Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Wed, 15 Apr 2026 15:18:19 +0200 Subject: [PATCH 24/47] Linted code, sorted imports --- src/explorepy/csv_client.py | 35 ++++++++++++++++++------------- src/explorepy/parser.py | 5 ++--- src/explorepy/stream_processor.py | 7 +++---- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/explorepy/csv_client.py b/src/explorepy/csv_client.py index d8278021..a4ff2fe9 100644 --- a/src/explorepy/csv_client.py +++ b/src/explorepy/csv_client.py @@ -1,10 +1,18 @@ -import time import os -from enum import Enum, auto +import time +from enum import ( + Enum, + auto +) + import numpy as np -from explorepy.packet import BleImpedancePacket, DeviceInfoBLE, OrientationV1, OrientationV2 from pylsl import local_clock -from typing_extensions import override + +from explorepy.packet import ( + BleImpedancePacket, + DeviceInfoBLE, + OrientationV2 +) class ClientState(Enum): @@ -13,14 +21,16 @@ class ClientState(Enum): STREAMING = auto() STOPPED = auto() + class PacketSize(Enum): EEG_8 = 40 EEG_32 = 112 ORN = 50 DEVICE_INFO = 38 + class CsvClient: - def __init__(self, channel_count, file_path: str=None): + def __init__(self, channel_count, file_path: str = None): if file_path is None: self.file_name = "32channel_semidry_artefacts_ExG.csv" file_path = os.path.join("/Users/sonjastefani/Documents/dev/explore-desktop/test-data/", self.file_name) @@ -31,10 +41,10 @@ def __init__(self, channel_count, file_path: str=None): raise FileNotFoundError(f"File not found: {file_path}") self.server = CsvServer( - channel_count=channel_count, - csv_path=file_path, - loop=True -) + channel_count=channel_count, + csv_path=file_path, + loop=True + ) self._state = ClientState.DISCONNECTED def set_state(self, state: ClientState): @@ -73,7 +83,6 @@ def read(self): device_info_packet.set_info(self.server.device_info) return device_info_packet - self.server.tick += 1 if self.server.tick % 5 == 0: orn_packet = OrientationMock(timestamp=self.server.ts, payload=None) @@ -97,9 +106,6 @@ def read(self): def write(self, bytes): pass -import numpy as np -from pylsl import local_clock - class CsvServer: def __init__(self, channel_count: int, csv_path: str, loop: bool = True): @@ -108,7 +114,7 @@ def __init__(self, channel_count: int, csv_path: str, loop: bool = True): self.channel_count = channel_count self.loop = loop - self.csv_data = np.loadtxt(csv_path, delimiter=',', skiprows=1) # skip row 0 + self.csv_data = np.loadtxt(csv_path, delimiter=',', skiprows=1) # skip row 0 self.csv_ts = self.csv_data[:, 0] self.csv_data = self.csv_data[:, 1:] @@ -182,6 +188,7 @@ def read_sample(self): def read_device_info(self): return self.device_info + class DeviceInfoMock(DeviceInfoBLE): def __init__(self, timestamp, payload, time_offset=0): self.timestamp = timestamp diff --git a/src/explorepy/parser.py b/src/explorepy/parser.py index 4c484a5d..e1183d36 100644 --- a/src/explorepy/parser.py +++ b/src/explorepy/parser.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import os """Parser module""" import asyncio import binascii @@ -83,8 +82,8 @@ def start_streaming(self, device_name, mac_address, file_path=None): self.stream_interface = MockBtClient(device_name=device_name, mac_address=mac_address) elif explorepy.get_bt_interface() == 'csv': from explorepy.csv_client import CsvClient - device_id = str.split(device_name, '_')[1] # split by underscore and take second part - ch_count = 32 if device_id[0] == 'D' else 8 #dynamic channel selection based on first character + device_id = str.split(device_name, '_')[1] # split by underscore and take second part + ch_count = 32 if device_id[0] == 'D' else 8 # dynamic channel selection based on first character self.stream_interface = CsvClient(ch_count, file_path=file_path) elif is_usb_mode(): from explorepy.serial_client import SerialStream diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index f03f8e47..ea660c8c 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -14,6 +14,8 @@ ) import numpy as np + +from explorepy.asr_processor import AsrProcessor from explorepy.command import ( DeviceConfiguration, ZMeasurementDisable, @@ -22,10 +24,8 @@ from explorepy.filters import ExGFilter from explorepy.packet import ( EEG, - CalibrationInfo, - CalibrationInfo_USBC, - CleanEEG, CalibrationInfoBase, + CleanEEG, CommandRCV, CommandStatus, DeviceInfo, @@ -45,7 +45,6 @@ is_usb_mode ) -from explorepy.asr_processor import AsrProcessor TOPICS =\ Enum('Topics', 'raw_ExG filtered_ExG asr_ExG device_info marker raw_orn cmd_ack env cmd_status imp packet_bin') From ada7ac3add14ad25fd43e5616e3398640a1a000a Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Tue, 21 Apr 2026 11:31:00 +0200 Subject: [PATCH 25/47] Use getter for cutoff frequency --- src/explorepy/stream_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index ea660c8c..1fd8ce05 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -618,4 +618,4 @@ def get_power_line_freq(self): None ) - return match.cutoff_freq if match else None + return match.get_cutoff_freq() if match else None From 4b74a26a43e95b69be78121536d4d18b47f92fba Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Tue, 21 Apr 2026 12:27:04 +0200 Subject: [PATCH 26/47] Prevent ASR while impedance mode is active Add ImpedanceModeActiveError and guard ASR availability when impedance is running. StreamProcessor gains is_asr_processor_available() which raises ImpedanceModeActiveError if impedance mode is active and otherwise checks the ASR processor initialization. Explore.is_asr_processor_available() now delegates to the stream processor. Also add the new exception import. --- src/explorepy/_exceptions.py | 10 ++++++++++ src/explorepy/explore.py | 2 +- src/explorepy/stream_processor.py | 6 ++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/explorepy/_exceptions.py b/src/explorepy/_exceptions.py index 616133d7..d15009bf 100644 --- a/src/explorepy/_exceptions.py +++ b/src/explorepy/_exceptions.py @@ -62,6 +62,16 @@ class BleDisconnectionFailedError(Exception): pass +class ImpedanceModeActiveError(Exception): + """ + Exception for ASR, raised when impedance is running + """ + + def __init__(self, message="ASR can not run because impedance mode is active.\n" + "Please disable impedance and try again\n"): + super().__init__(message) + + class ExplorePyDeprecationError(Exception): def __init__(self, message="Explorepy support for legacy devices is deprecated.\n" "Please install explorepy 3.2.1 from Github or use the following command from Anaconda " diff --git a/src/explorepy/explore.py b/src/explorepy/explore.py index 0575282d..87f0978e 100644 --- a/src/explorepy/explore.py +++ b/src/explorepy/explore.py @@ -734,7 +734,7 @@ def get_channel_mask(self): return SettingsManager(self.device_name).get_adc_mask() def is_asr_processor_available(self): - return self.stream_processor.asr_processor is not None and self.stream_processor.asr_processor.is_initialized + return self.stream_processor.is_asr_processor_available() def calibrate_asr(self, length=-1.0): if self.is_asr_processor_available(): diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index ea660c8c..43d09622 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -15,6 +15,7 @@ import numpy as np +from explorepy._exceptions import ImpedanceModeActiveError from explorepy.asr_processor import AsrProcessor from explorepy.command import ( DeviceConfiguration, @@ -619,3 +620,8 @@ def get_power_line_freq(self): ) return match.cutoff_freq if match else None + + def is_asr_processor_available(self): + if self.is_imp_running(): + raise ImpedanceModeActiveError() + return self.asr_processor is not None and self.asr_processor.is_initialized From 2cd40957481853f2c6591f132ae02bbddbd5bffe Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Wed, 6 May 2026 16:27:04 +0200 Subject: [PATCH 27/47] use non-causal filter --- src/explorepy/filters.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/explorepy/filters.py b/src/explorepy/filters.py index 41f1ef66..e6faf08e 100644 --- a/src/explorepy/filters.py +++ b/src/explorepy/filters.py @@ -4,6 +4,7 @@ import logging import numpy as np +from scipy import signal from scipy.signal import ( butter, iirfilter, @@ -17,6 +18,13 @@ logger = logging.getLogger(__name__) +@staticmethod +def bp_filter(exg, lf, hf, fs, btype='bandpass'): + N = 5 + b, a = signal.butter(N, [lf, hf], btype=btype, fs=fs) + return signal.filtfilt(b, a, exg, axis=1) + + class ExGFilter: """Filter class for ExG signals""" def __init__(self, cutoff_freq, filter_type, s_rate, n_chan, order=5): From 7dad21ea57467785f603e380ad8ff2e3a09df203 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Wed, 6 May 2026 16:27:45 +0200 Subject: [PATCH 28/47] add eegprep to toml --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 82be6745..c746cfc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,8 @@ dependencies = [ 'pyyaml', 'bleak==0.22.3', 'pylsl', - 'numba'] + 'numba', + 'eegprep'] [tool.setuptools] package-dir = {"" = "src"} From 410413088756e948bcf7eebf70601169912bdec4 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Mon, 11 May 2026 13:10:43 +0200 Subject: [PATCH 29/47] rename is_asr_processor_available method --- src/explorepy/explore.py | 2 +- src/explorepy/stream_processor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/explorepy/explore.py b/src/explorepy/explore.py index 87f0978e..b3287749 100644 --- a/src/explorepy/explore.py +++ b/src/explorepy/explore.py @@ -734,7 +734,7 @@ def get_channel_mask(self): return SettingsManager(self.device_name).get_adc_mask() def is_asr_processor_available(self): - return self.stream_processor.is_asr_processor_available() + return self.stream_processor.ensure_asr_processor_available() def calibrate_asr(self, length=-1.0): if self.is_asr_processor_available(): diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index 43d09622..3c01a7fd 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -621,7 +621,7 @@ def get_power_line_freq(self): return match.cutoff_freq if match else None - def is_asr_processor_available(self): + def ensure_asr_processor_available(self): if self.is_imp_running(): raise ImpedanceModeActiveError() return self.asr_processor is not None and self.asr_processor.is_initialized From 3480832e0c2cf8b1964d0081b2208d211c41ba62 Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Tue, 12 May 2026 16:46:07 +0200 Subject: [PATCH 30/47] Added exg_fs argument to get_data for CleanEEG --- src/explorepy/packet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/explorepy/packet.py b/src/explorepy/packet.py index ec739162..4194362d 100644 --- a/src/explorepy/packet.py +++ b/src/explorepy/packet.py @@ -184,7 +184,7 @@ def __init__(self, timestamp, payload): def _convert(self, bin_data): self.data = bin_data - def get_data(self): + def get_data(self, exg_fs=None): return self.timestamps, self.data def __str__(self): From 5d89ea2d51345cc1bb6b50f1ed69844efbbff565 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Fri, 15 May 2026 10:15:18 +0200 Subject: [PATCH 31/47] hotfix for missing cutoff-freq variable in filt class --- src/explorepy/filters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/explorepy/filters.py b/src/explorepy/filters.py index e6faf08e..cc5b9b46 100644 --- a/src/explorepy/filters.py +++ b/src/explorepy/filters.py @@ -39,6 +39,7 @@ def __init__(self, cutoff_freq, filter_type, s_rate, n_chan, order=5): self.s_rate = float(s_rate) self.filter_type = filter_type self.filter_param = None + self.cutoff_freq = cutoff_freq order = 2 if self.s_rate >= 1000 else order a, b, zi = self.get_filter_coeffs(cutoff_freq, filter_type, s_rate, n_chan, order) self.filter_param = {'a': a, 'b': b, 'zi': zi} From a4931f3f4f436d0cc8effcf70dbda2788dc845a7 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Fri, 15 May 2026 13:52:48 +0200 Subject: [PATCH 32/47] Set binary time (#432) * set time as bin file name * Shorten device-info wait and set binary time Reduce the polling sleep interval from 1s to 0.5s while waiting for the device info packet to speed up startup. After receiving device info, call self.set_binary_time() to initialize device time immediately. Also remove the redundant self._check_connection() call from set_binary_time to avoid an unnecessary connection verification. * Revert "Shorten device-info wait and set binary time" This reverts commit 7fb6792f38eefd02d35f0fbdf83d1a65421b0c75. * Reapply "Shorten device-info wait and set binary time" This reverts commit d0de9e71fe75e12ea87904649be248e19805f7c0. * set time on connect with fw version check * add new dev info and adjust time cmd * fix flake8 errors * add packet id to header length check * refactor class name to 64b command, fix docs --- src/explorepy/command.py | 44 +++++++++++++++++++++++++++++++++++++++- src/explorepy/explore.py | 16 ++++++++++++++- src/explorepy/packet.py | 14 ++++++++++++- src/explorepy/parser.py | 7 ++++++- 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/src/explorepy/command.py b/src/explorepy/command.py index 72ccda61..9a4f69c1 100644 --- a/src/explorepy/command.py +++ b/src/explorepy/command.py @@ -16,6 +16,7 @@ class CommandID(Enum): """Command ID enum class""" API2BCMD = b'\xA0' API4BCMD = b'\xB0' + API64BCMD = b'\xC0' class OpcodeID(Enum): @@ -27,6 +28,7 @@ class OpcodeID(Enum): CMD_ZM_ENABLE = b'\xA7' CMD_SOFT_RESET = b'\xA8' CMD_TEST_SIG = b'\xAA' + CMD_SET_EMMC_TIME = b'\xAB' class DeviceConfiguration: @@ -182,6 +184,19 @@ def __str__(self): """prints the appropriate info about the command. """ +class Command64B(Command): + """An abstract base class for Explore 40 Byte command data length packets""" + + def __init__(self): + super().__init__() + self.pid = CommandID.API64BCMD + self.payload_length = int2bytearray(40, 2) + + @abc.abstractmethod + def __str__(self): + """prints the appropriate info about the command. """ + + class SetSPS(Command2B): """Set the sampling rate of ExG device""" @@ -213,6 +228,32 @@ def __str__(self): return "Set sampling rate command" +class SetBinaryTime(Command64B): + def __init__(self): + super().__init__() + self.opcode = OpcodeID.CMD_SET_EMMC_TIME + from datetime import datetime + now = datetime.now() + date_time = bytes( + [ + now.year - 2000, + now.month, + now.day, + now.weekday(), + now.hour, + now.minute, + now.second, + 1, + 2, + 3, + ] + ) + self.param = date_time + bytes(51) + + def __str__(self): + return "set time cmd" + + class MemoryFormat(Command2B): """Format device memory""" @@ -282,7 +323,8 @@ def __str__(self): COMMAND_CLASS_DICT = { CommandID.API2BCMD: Command2B, - CommandID.API4BCMD: Command4B + CommandID.API4BCMD: Command4B, + CommandID.API64BCMD: Command64B } diff --git a/src/explorepy/explore.py b/src/explorepy/explore.py index b3287749..ee0024f2 100644 --- a/src/explorepy/explore.py +++ b/src/explorepy/explore.py @@ -26,6 +26,7 @@ import explorepy from explorepy.command import ( MemoryFormat, + SetBinaryTime, SetChTest, SetSPS, SoftReset @@ -105,11 +106,13 @@ def connect(self, device_name=None, mac_address=None, file_path=None): cnt_limit = 20 if self.debug else 15 while "adc_mask" not in self.stream_processor.device_info: logger.info("Waiting for device info packet...") - time.sleep(1) + time.sleep(.5) if cnt >= cnt_limit: raise ConnectionAbortedError( "Could not get info packet from the device") cnt += 1 + if 'time_cmd' in self.stream_processor.device_info: + self.set_binary_time() if self.stream_processor.device_info['is_imp_mode'] is True: self.stream_processor.disable_imp() logger.info( @@ -652,6 +655,17 @@ def set_sampling_rate(self, sampling_rate): SettingsManager(self.device_name).set_sampling_rate(sampling_rate) return True + def set_binary_time(self): + """Set binary file to current time + + Returns: + bool: True for success, False otherwise + """ + cmd = SetBinaryTime() + if self.stream_processor.configure_device(cmd): + logger.info('Device set to current time.') + return True + def reset_soft(self): """Reset the device to the default settings diff --git a/src/explorepy/packet.py b/src/explorepy/packet.py index ec739162..c3501ceb 100644 --- a/src/explorepy/packet.py +++ b/src/explorepy/packet.py @@ -25,6 +25,7 @@ class PACKET_ID(IntEnum): # Info packet from BLE devices, applies to Explore Pro INFO_BLE = 98 INFO_HYP = 99 + INFO_TIME_CMD = 100 # New info packet containing memory and board ID: this applies to all Explore+ systems INFO_V2 = 97 INFO = 96 @@ -696,7 +697,17 @@ def get_info(self): class DeviceInfoHyp(DeviceInfoBLE): - pass + def get_info(self): + as_dict = super().get_info() + as_dict['is_hypersync'] = True + return as_dict + + +class DeviceInfoBinTimeCmd(DeviceInfoHyp): + def get_info(self): + as_dict = super().get_info() + as_dict['time_cmd'] = True + return as_dict class CommandRCV(Packet): @@ -837,6 +848,7 @@ def __str__(self): PACKET_ID.INFO_V2: DeviceInfoV2, PACKET_ID.INFO_BLE: DeviceInfoBLE, PACKET_ID.INFO_HYP: DeviceInfoHyp, + PACKET_ID.INFO_TIME_CMD: DeviceInfoBinTimeCmd, PACKET_ID.EEG94: EEG94, PACKET_ID.EEG98: EEG98, PACKET_ID.EEG99: EEG99, diff --git a/src/explorepy/parser.py b/src/explorepy/parser.py index e1183d36..e47f0659 100644 --- a/src/explorepy/parser.py +++ b/src/explorepy/parser.py @@ -26,6 +26,7 @@ ) from explorepy.packet import ( PACKET_CLASS_DICT, + PACKET_ID, DeviceInfo, Packet, PacketBIN @@ -373,7 +374,11 @@ def get_header_bytes(self): break if pid_bin is None: raise ValueError - self.header_len = 12 if pid_bin[0] == 99 else 8 + self.header_len = ( + 12 + if pid_bin[0] in (PACKET_ID.INFO_HYP, PACKET_ID.INFO_TIME_CMD) + else 8 + ) self.data_len = self.header_len - 4 return pid_bin + self.stream_interface.read(self.header_len - 1) else: From cecc24040995490db595c1d98935e8cce65900df Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Tue, 19 May 2026 11:26:10 +0200 Subject: [PATCH 33/47] return bool in is_asr_processor_available --- src/explorepy/explore.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/explorepy/explore.py b/src/explorepy/explore.py index ee0024f2..01426483 100644 --- a/src/explorepy/explore.py +++ b/src/explorepy/explore.py @@ -49,6 +49,7 @@ local_clock, setup_usb_marker_port ) +from explorepy._exceptions import ImpedanceModeActiveError logger = logging.getLogger(__name__) @@ -748,7 +749,10 @@ def get_channel_mask(self): return SettingsManager(self.device_name).get_adc_mask() def is_asr_processor_available(self): - return self.stream_processor.ensure_asr_processor_available() + try: + return self.stream_processor.ensure_asr_processor_available() + except ImpedanceModeActiveError: + return False def calibrate_asr(self, length=-1.0): if self.is_asr_processor_available(): @@ -773,11 +777,8 @@ def is_asr_running(self): return False def start_asr(self, window=None): - print("Starting asr") if self.is_asr_processor_available(): - print("ASR processor is available") if self.stream_processor.asr_processor.calibration_data_available: - print("calibration data is available") self.stream_processor.asr_processor.start_cleaning(window) def stop_asr(self): From 41ffbf7d67c82d1c35db93537ec5413eadf12300 Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Tue, 19 May 2026 11:50:19 +0200 Subject: [PATCH 34/47] Sorted imports --- src/explorepy/explore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/explorepy/explore.py b/src/explorepy/explore.py index 01426483..a5697249 100644 --- a/src/explorepy/explore.py +++ b/src/explorepy/explore.py @@ -24,6 +24,7 @@ from scipy import signal as scipy_signal import explorepy +from explorepy._exceptions import ImpedanceModeActiveError from explorepy.command import ( MemoryFormat, SetBinaryTime, @@ -49,7 +50,6 @@ local_clock, setup_usb_marker_port ) -from explorepy._exceptions import ImpedanceModeActiveError logger = logging.getLogger(__name__) From 72eee663d96909b166d0aaf6c489cd8c8ee1cce6 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Tue, 19 May 2026 14:25:00 +0200 Subject: [PATCH 35/47] add new binary time set packet ID in binary converter (#437) --- src/explorepy/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/explorepy/tools.py b/src/explorepy/tools.py index 0b120961..a687acd4 100644 --- a/src/explorepy/tools.py +++ b/src/explorepy/tools.py @@ -730,7 +730,7 @@ def setup_usb_marker_port(): def check_bin_compatibility(file_name): with open(file_name, "rb") as f: b = f.read(1).hex() - if b != "62" and b != "63": + if b not in {"62", "63", "64"}: raise ExplorePyDeprecationError() From fc15632f0234b0a9cac9ba497bc5806ae1802f66 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Tue, 19 May 2026 16:31:22 +0200 Subject: [PATCH 36/47] Rc 4.5.0 (#435) * bumpversion minor * pip deps on toml * remove sps from args in imp recorder (#436) --- README.rst | 4 ++-- docs/conf.py | 2 +- installer/windows/installer.cfg | 4 ++-- pyproject.toml | 14 +++++++------- src/explorepy/__init__.py | 2 +- src/explorepy/explore.py | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 1389bd39..e08584be 100644 --- a/README.rst +++ b/README.rst @@ -17,9 +17,9 @@ :target: https://pypi.org/project/explorepy -.. |commits-since| image:: https://img.shields.io/github/commits-since/Mentalab-hub/explorepy/v4.4.0.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/Mentalab-hub/explorepy/v4.5.0.svg :alt: Commits since latest release - :target: https://github.com/Mentalab-hub/explorepy/compare/v4.4.0...master + :target: https://github.com/Mentalab-hub/explorepy/compare/v4.5.0...master .. |wheel| image:: https://img.shields.io/pypi/wheel/explorepy.svg diff --git a/docs/conf.py b/docs/conf.py index f763e66f..2a8ed6c0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,7 @@ year = '2018-2025' author = 'Mentalab GmbH.' copyright = '{0}, {1}'.format(year, author) -version = release = '4.4.0' +version = release = '4.5.0' pygments_style = 'trac' templates_path = ['.'] extlinks = { diff --git a/installer/windows/installer.cfg b/installer/windows/installer.cfg index 49abaf2e..5a4ea517 100644 --- a/installer/windows/installer.cfg +++ b/installer/windows/installer.cfg @@ -1,6 +1,6 @@ [Application] name=MentaLab ExplorePy -version=4.4.0 +version=4.5.0 entry_point=explorepy.cli:cli console=true icon=mentalab.ico @@ -26,7 +26,7 @@ pypi_wheels = decorator==5.1.1 distlib==0.3.7 eeglabio==0.0.2.post4 - explorepy==4.4.0 + explorepy==4.5.0 fonttools==4.42.1 idna==3.4 importlib-resources==6.0.1 diff --git a/pyproject.toml b/pyproject.toml index c746cfc0..bdadc8be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'explorepy' -version = "4.4.0" +version = "4.5.0" license = { text = "MIT" } readme = { file = "README.rst", content-type = "text/markdown" } authors = [ @@ -29,11 +29,11 @@ classifiers = [ ] dependencies = [ - 'numpy', - 'scipy', + 'numpy==2.1.3', + 'scipy==1.17.1', 'pyEDFlib==0.1.38', 'click==7.1.2', - 'appdirs==1.4.3', + 'appdirs==1.4.4', 'sentry_sdk==2.8.0', 'mne', 'eeglabio', @@ -41,9 +41,9 @@ dependencies = [ 'pyserial', 'pyyaml', 'bleak==0.22.3', - 'pylsl', + 'pylsl==1.18.2', 'numba', - 'eegprep'] + 'eegprep==0.2.23'] [tool.setuptools] package-dir = {"" = "src"} @@ -64,7 +64,7 @@ explorepy = "explorepy.cli:cli" # bumpoversion config [tool.bumpversion] -current_version = "4.4.0" +current_version = "4.5.0" commit = false tag = false diff --git a/src/explorepy/__init__.py b/src/explorepy/__init__.py index 6b022a80..cfa4ad24 100644 --- a/src/explorepy/__init__.py +++ b/src/explorepy/__init__.py @@ -20,7 +20,7 @@ __all__ = ["Explore", "command", "tools", "log_config"] -__version__ = '4.4.0' +__version__ = '4.5.0' this = sys.modules[__name__] # TODO appropriate library diff --git a/src/explorepy/explore.py b/src/explorepy/explore.py index a5697249..18ec91a2 100644 --- a/src/explorepy/explore.py +++ b/src/explorepy/explore.py @@ -353,7 +353,7 @@ def __init__(self, timestamps, signals): self.timestamps = timestamps self.signals = signals - def get_data(self, fs): + def get_data(self): return self.timestamps, self.signals timestamps = exg_array[:, 0] From aa728fd16cee5c570f9bac001f506b2327b00136 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Tue, 19 May 2026 17:32:07 +0200 Subject: [PATCH 37/47] Handle invalid time value set (#439) * bumpversion minor * pip deps on toml * remove sps from args in imp recorder * handle invalid binary time value set only sends the time command when the time is gregorian calendar, else raise an exception --- src/explorepy/command.py | 2 ++ src/explorepy/explore.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/explorepy/command.py b/src/explorepy/command.py index 9a4f69c1..b248b0c1 100644 --- a/src/explorepy/command.py +++ b/src/explorepy/command.py @@ -234,6 +234,8 @@ def __init__(self): self.opcode = OpcodeID.CMD_SET_EMMC_TIME from datetime import datetime now = datetime.now() + if not (2020 < now.year < 2099): + raise ValueError(f"Unsupported year: {now.year}") date_time = bytes( [ now.year - 2000, diff --git a/src/explorepy/explore.py b/src/explorepy/explore.py index 18ec91a2..81c04247 100644 --- a/src/explorepy/explore.py +++ b/src/explorepy/explore.py @@ -662,10 +662,14 @@ def set_binary_time(self): Returns: bool: True for success, False otherwise """ - cmd = SetBinaryTime() - if self.stream_processor.configure_device(cmd): - logger.info('Device set to current time.') - return True + try: + cmd = SetBinaryTime() + if self.stream_processor.configure_device(cmd): + logger.info('Device set to current time.') + return True + except ValueError as e: + logger.error('Error on setting the binary file time: {}'.format(e)) + return False def reset_soft(self): """Reset the device to the default settings From 9961b4f80a0224f501461efefa6e1aba321ee294 Mon Sep 17 00:00:00 2001 From: Sonja Stefani Date: Wed, 20 May 2026 18:05:43 +0200 Subject: [PATCH 38/47] Set imp_calculator to None on disable_imp --- src/explorepy/stream_processor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index 7394e04e..608e5d7e 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -483,6 +483,7 @@ def disable_imp(self): cmd = ZMeasurementDisable() if self.configure_device(cmd): self._is_imp_mode = False + self.imp_calculator = None print("Impedance measurement mode has been disabled.") return True print("WARNING: Couldn't disable impedance measurement mode. " From 90980fb039f17bd94f4e7b065c393ce7d2b1fcfd Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Wed, 20 May 2026 18:19:32 +0200 Subject: [PATCH 39/47] Create LICENSE_eegprep.txt (#441) --- licenses/LICENSE_eegprep.txt | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 licenses/LICENSE_eegprep.txt diff --git a/licenses/LICENSE_eegprep.txt b/licenses/LICENSE_eegprep.txt new file mode 100644 index 00000000..2fba5ba8 --- /dev/null +++ b/licenses/LICENSE_eegprep.txt @@ -0,0 +1,25 @@ +BSD 2-Clause License + +Copyright (c) 2025, Swartz Center for Computational Neuroscience + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. From 220500604b50e33562e306b8ca1092b138646d26 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Fri, 22 May 2026 10:29:36 +0200 Subject: [PATCH 40/47] change padding bytes length (#440) --- src/explorepy/command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/explorepy/command.py b/src/explorepy/command.py index b248b0c1..1c333379 100644 --- a/src/explorepy/command.py +++ b/src/explorepy/command.py @@ -250,7 +250,7 @@ def __init__(self): 3, ] ) - self.param = date_time + bytes(51) + self.param = date_time + bytes(41) def __str__(self): return "set time cmd" From 8fd1bae24797d0b6b7d710973dd102e8afefb3b1 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Tue, 26 May 2026 12:25:25 +0200 Subject: [PATCH 41/47] change depricated function (#443) --- src/explorepy/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/explorepy/tools.py b/src/explorepy/tools.py index a687acd4..d656293a 100644 --- a/src/explorepy/tools.py +++ b/src/explorepy/tools.py @@ -272,7 +272,7 @@ def _init_edf_channels(self): ch_info_list.append({ 'label': ch[0], 'dimension': ch[1], - 'sample_rate': self._fs, + 'sample_frequency': self._fs, 'physical_max': ch[2], 'physical_min': ch[3], 'digital_max': 8388607, From 2e086cbef1420fa2b4bfe74d572948d6336edc00 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Wed, 27 May 2026 16:53:35 +0200 Subject: [PATCH 42/47] Hotfix bdf recording (#444) * change depricated function * Flush EDF buffer at quarter-second intervals Reduce the threshold for writing EDF samples from _fs to _fs // 4 so buffered data is flushed more frequently. Previously the FileRecorder only wrote when the buffer exceeded one second of samples; this change triggers writes at roughly quarter-second increments to lower memory usage and latency while retaining the existing behavior of writing full _fs-sized chunks and updating annotations/timestamps under the same lock. --- src/explorepy/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/explorepy/tools.py b/src/explorepy/tools.py index d656293a..b6953ec8 100644 --- a/src/explorepy/tools.py +++ b/src/explorepy/tools.py @@ -362,7 +362,7 @@ def _write_edf(self, packet): self._data = np.concatenate((self._data, data), axis=1) self._timestamps += list(data[0, :]) with lock: - if self._data.shape[1] > self._fs: + if self._data.shape[1] > self._fs // 4: self._file_obj.writeSamples(list(self._data[:, :self._fs])) self._write_edf_anno() self._data = self._data[:, self._fs:] From d1e84e1f9d9bdc20315ebb3e85848869c987cd6a Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Thu, 28 May 2026 15:00:01 +0200 Subject: [PATCH 43/47] Hotfix bdf recording (#445) * change depricated function * Flush EDF buffer at quarter-second intervals Reduce the threshold for writing EDF samples from _fs to _fs // 4 so buffered data is flushed more frequently. Previously the FileRecorder only wrote when the buffer exceeded one second of samples; this change triggers writes at roughly quarter-second increments to lower memory usage and latency while retaining the existing behavior of writing full _fs-sized chunks and updating annotations/timestamps under the same lock. * Write full-second sample blocks to file Replace the previous arbitrary threshold write logic with computation of whole-second sample blocks: compute num_samples_to_write as (n_samples // fs) * fs and only write when > 0. This ensures written chunks are multiples of the sampling frequency, avoids partial-frame writes and potential timestamp/data misalignment, and then drops the written samples from the buffer. --- src/explorepy/tools.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/explorepy/tools.py b/src/explorepy/tools.py index b6953ec8..21dc70cd 100644 --- a/src/explorepy/tools.py +++ b/src/explorepy/tools.py @@ -362,10 +362,11 @@ def _write_edf(self, packet): self._data = np.concatenate((self._data, data), axis=1) self._timestamps += list(data[0, :]) with lock: - if self._data.shape[1] > self._fs // 4: - self._file_obj.writeSamples(list(self._data[:, :self._fs])) + num_samples_to_write = (self._data.shape[1] // self._fs) * self._fs + if num_samples_to_write > 0: + self._file_obj.writeSamples(list(self._data[:, :num_samples_to_write])) self._write_edf_anno() - self._data = self._data[:, self._fs:] + self._data = self._data[:, num_samples_to_write:] def _process_packet_data(self, packet): """Helper function to extract and format data from a packet.""" From 6bcfd488b03f90b01ba8277074817d723bb66048 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Mon, 1 Jun 2026 17:44:40 +0200 Subject: [PATCH 44/47] remove unused functions --- src/explorepy/explore.py | 55 ------------------------------- src/explorepy/stream_processor.py | 16 --------- 2 files changed, 71 deletions(-) diff --git a/src/explorepy/explore.py b/src/explorepy/explore.py index 81c04247..337b9ea7 100644 --- a/src/explorepy/explore.py +++ b/src/explorepy/explore.py @@ -24,7 +24,6 @@ from scipy import signal as scipy_signal import explorepy -from explorepy._exceptions import ImpedanceModeActiveError from explorepy.command import ( MemoryFormat, SetBinaryTime, @@ -751,57 +750,3 @@ def is_bt_link_unstable(self): def get_channel_mask(self): return SettingsManager(self.device_name).get_adc_mask() - - def is_asr_processor_available(self): - try: - return self.stream_processor.ensure_asr_processor_available() - except ImpedanceModeActiveError: - return False - - def calibrate_asr(self, length=-1.0): - if self.is_asr_processor_available(): - self.stream_processor.asr_processor.start_calibration(length) - - def is_asr_calibration_data_available(self): - if self.is_asr_processor_available(): - return self.stream_processor.asr_processor.calibration_data_available - else: - return False - - def is_asr_calibrating(self): - if self.is_asr_processor_available(): - return self.stream_processor.asr_processor.is_calibrating - else: - return False - - def is_asr_running(self): - if self.is_asr_processor_available(): - return self.stream_processor.asr_processor.is_cleaning - else: - return False - - def start_asr(self, window=None): - if self.is_asr_processor_available(): - if self.stream_processor.asr_processor.calibration_data_available: - self.stream_processor.asr_processor.start_cleaning(window) - - def stop_asr(self): - if self.is_asr_processor_available(): - if self.stream_processor.asr_processor.calibration_data_available: - self.stream_processor.asr_processor.stop_cleaning() - - def set_asr_cutoff(self, new_cutoff: float): - if self.is_asr_processor_available(): - self.stream_processor.asr_processor.cutoff = new_cutoff - - def get_asr_cutoff(self): - if self.is_asr_processor_available(): - return self.stream_processor.asr_processor.cutoff - - def set_asr_refresh_window(self, new_window): - if self.is_asr_processor_available(): - self.stream_processor.asr_processor.refresh_window = new_window - - def get_asr_refresh_window(self): - if self.is_asr_processor_available(): - return self.stream_processor.asr_processor.refresh_window diff --git a/src/explorepy/stream_processor.py b/src/explorepy/stream_processor.py index 608e5d7e..64b55280 100644 --- a/src/explorepy/stream_processor.py +++ b/src/explorepy/stream_processor.py @@ -15,8 +15,6 @@ import numpy as np -from explorepy._exceptions import ImpedanceModeActiveError -from explorepy.asr_processor import AsrProcessor from explorepy.command import ( DeviceConfiguration, ZMeasurementDisable, @@ -26,7 +24,6 @@ from explorepy.packet import ( EEG, CalibrationInfoBase, - CleanEEG, CommandRCV, CommandStatus, DeviceInfo, @@ -58,7 +55,6 @@ class StreamProcessor: def __init__(self, debug=False): self.parser = None - self.asr_processor = None self.filters = [] self.device_info = {} self.old_device_info = {} @@ -339,12 +335,6 @@ def process(self, packet): self.dispatch(topic=TOPICS.filtered_ExG, packet=packet) self.dispatch(topic=TOPICS.filtered_ExG, packet=packet) - if not self._is_imp_mode and self.imp_calculator is None: - if self.asr_processor.cleaned_data_available: - clean_packet = CleanEEG(timestamp=self.asr_processor.cleaned_data_ts, - payload=self.asr_processor.cleaned_data) - self.dispatch(topic=TOPICS.asr_ExG, packet=clean_packet) - self.asr_processor.clear_cleaned_data() elif isinstance(packet, DeviceInfo): self.old_device_info = self.device_info.copy() print(self.old_device_info) @@ -354,7 +344,6 @@ def process(self, packet): self.device_info["device_name"]) settings_manager.update_device_settings(packet.get_info()) self.dispatch(topic=TOPICS.device_info, packet=packet) - self.asr_processor = AsrProcessor(self, TOPICS.raw_ExG) elif isinstance(packet, CommandRCV): self.dispatch(topic=TOPICS.cmd_ack, packet=packet) elif isinstance(packet, CommandStatus): @@ -620,8 +609,3 @@ def get_power_line_freq(self): None ) return match.get_cutoff_freq() if match else None - - def ensure_asr_processor_available(self): - if self.is_imp_running(): - raise ImpedanceModeActiveError() - return self.asr_processor is not None and self.asr_processor.is_initialized From f6202a1e24cc98b13e8a7eb0de5a191c8497f5c9 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Fri, 5 Jun 2026 19:03:00 +0200 Subject: [PATCH 45/47] Rc 4.5.0 (#446) * Bump pyEDFlib and update changelog Add 4.5.0 changelog entry (Live impedance, LSL push for preprocessed data, support for new FW version). Bump pyEDFlib from 0.1.38 to 0.1.42 and remove the eegprep==0.2.23 dependency from pyproject.toml * fix pypi rendering --- CHANGELOG.rst | 5 +++++ README.rst | 15 ++++++++------- pyproject.toml | 7 +++---- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 00526788..4af26a76 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,11 @@ Changelog ========= +4.5.0 (9.6.2025) +------------------ +* Live impedance +* Allow pushing preprocessed data to lab streaming layer +* Support new FW version 4.3.1 (17.9.2025) ------------------ diff --git a/README.rst b/README.rst index e08584be..63f94ae5 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,6 @@ .. image:: https://raw.githubusercontent.com/Mentalab-hub/explorepy/master/docs/logo.jpg - :scale: 100 % - :align: left - + :alt: Explorepy + :target: https://github.com/Mentalab-hub/explorepy .. start-badges @@ -39,10 +38,10 @@ ========================= -``explorepy`` overview +ExplorePy overview ========================= -``explorepy`` is an open-source Python API designed to collect and process ExG data using Mentalab's Explore device. Amongst other things, ``explorepy`` provides the following features: +ExplorePy is an open-source Python API designed to collect and process ExG data using Mentalab's Explore device. Amongst other things, ExplorePy provides the following features: * Real-time streaming of ExG, orientation and environmental data. * Data recording in CSV and BDF+ formats. @@ -61,7 +60,7 @@ Requirements Detailed installation instructions can be found on the `installation page `_. -To install ``explorepy`` from PyPI run: +To install ExplorePy from PyPI run: :: pip install explorepy @@ -78,12 +77,14 @@ Get started CLI command ----------- -To check ``explorepy`` is running use: +To check ExplorePy is running use: :: + explorepy acquire -n Explore_XXXX For help, use: :: + explorepy -h diff --git a/pyproject.toml b/pyproject.toml index bdadc8be..817ab259 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = 'setuptools.build_meta' name = 'explorepy' version = "4.5.0" license = { text = "MIT" } -readme = { file = "README.rst", content-type = "text/markdown" } +readme = { file = "README.rst", content-type = "text/x-rst"} authors = [ { name = "MentaLab Hub", email = "support@mentab.org" }, ] @@ -31,7 +31,7 @@ classifiers = [ dependencies = [ 'numpy==2.1.3', 'scipy==1.17.1', - 'pyEDFlib==0.1.38', + 'pyEDFlib==0.1.42', 'click==7.1.2', 'appdirs==1.4.4', 'sentry_sdk==2.8.0', @@ -42,8 +42,7 @@ dependencies = [ 'pyyaml', 'bleak==0.22.3', 'pylsl==1.18.2', - 'numba', - 'eegprep==0.2.23'] + 'numba'] [tool.setuptools] package-dir = {"" = "src"} From 7cd3f0f53c5c2a64117e6185dd2bfbf520cf0d07 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Fri, 5 Jun 2026 19:08:01 +0200 Subject: [PATCH 46/47] remove unused file (#447) --- licenses/LICENSE_eegprep.txt | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 licenses/LICENSE_eegprep.txt diff --git a/licenses/LICENSE_eegprep.txt b/licenses/LICENSE_eegprep.txt deleted file mode 100644 index 2fba5ba8..00000000 --- a/licenses/LICENSE_eegprep.txt +++ /dev/null @@ -1,25 +0,0 @@ -BSD 2-Clause License - -Copyright (c) 2025, Swartz Center for Computational Neuroscience - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, -this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF -THE POSSIBILITY OF SUCH DAMAGE. From 46530c0794bda6fbdb911d61e8b4bd69d5479ab1 Mon Sep 17 00:00:00 2001 From: Salman Rahman Date: Fri, 5 Jun 2026 19:11:53 +0200 Subject: [PATCH 47/47] Fix release date for version 4.5.0 Updated the release date for version 4.5.0 in the changelog. --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4af26a76..8ed0ee15 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ Changelog ========= -4.5.0 (9.6.2025) +4.5.0 (5.6.2025) ------------------ * Live impedance * Allow pushing preprocessed data to lab streaming layer