Add add_started parsing key, i.e. use absolute timestamps for columns of interest. Add docs + minor fixes here and there

Author: marcelzwiers

.github/workflows/tests.yaml renamed to .github/workflows/tests.yml

@@ -36,7 +36,7 @@ jobs:
         if: runner.os != 'Windows'
         run: |
           if [ "$RUNNER_OS" == "Linux" ]; then
-            sudo apt update && sudo apt install qt6-base-dev
+            apt update && apt install qt6-base-dev
             mkdir dcm2niix_install/ && cd dcm2niix_install/
             curl -fLO https://github.com/rordenlab/dcm2niix/releases/latest/download/dcm2niix_lnx.zip
             unzip dcm2niix*.zip

bidscoin/bcoin.py

@@ -481,7 +481,7 @@ def test_bidscoin(bidsmapfile, options: dict=None, testplugins: bool=True, testg
             root = tk.Tk()
             root.withdraw()                         # Don't show the window
         except tk.TclError as display_error:
-            LOGGER.error(f"Cannot open a grahical display on your system:\n{display_error}")
+            LOGGER.error(f"Cannot open a graphical display on your system:\n{display_error}")
             success = False
         try:
             from PyQt6.QtWidgets import QApplication, QPushButton

bidscoin/heuristics/bidsmap_bids2bids.yaml

@@ -20,7 +20,7 @@ Options:
 # General BIDScoin and plugin options
 # --------------------------------------------------------------------------------
   bidscoin:
-    version: 4.5.1.dev                  # BIDScoin version (should correspond with the version in pyproject.toml)
+    version: 4.5.2.dev                  # BIDScoin version (should correspond with the version in pyproject.toml)
     subprefix: sub-                 # The subject prefix of the source data
     sesprefix: ses-                 # The session prefix of the source data
     bidsignore: [extra_data/, sub-*_ct.*]    # List of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv]

bidscoin/heuristics/bidsmap_dccn.yaml

@@ -21,7 +21,7 @@ Options:
 # General BIDScoin and plugin options
 # -----------------------------------
   bidscoin:
-    version: 4.5.1.dev                  # BIDScoin version (should correspond with the version in pyproject.toml)
+    version: 4.5.2.dev                  # BIDScoin version (should correspond with the version in pyproject.toml)
     subprefix: sub-                 # The subject prefix of the source data
     sesprefix: ses-                 # The session prefix of the source data
     bidsignore: [extra_data/, sub-*_ct.*]   # List of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv]
@@ -1161,7 +1161,8 @@ Psychopy:
     events: &psychopy_events
       parsing:                          # The settings to parse the source table from the log file
         table: [long-wide, pivot, 1]    # The raw source table or a pivoted 'onset', 'duration', 'event_type' version
-        expand: scannerPulse.rt         # Expands lists into columns for each array item
+        add_started:                    # Columns for which the .started time is added, i.e. that are converted to absolute timestamps (e.g. the .rt columns)
+        expand:                         # Expands lists into columns for each array item
       columns:                          # Columns that are included in the output table, i.e. {output column: input column}
       - onset: onset                    # The mapping for the first required column 'onset'
       - duration: duration              # The mapping for the second required column 'duration'

bidscoin/heuristics/bidsmap_sst.yaml

@@ -20,7 +20,7 @@ Options:
 # General BIDScoin and plugin options
 # -----------------------------------
   bidscoin:
-    version: 4.5.1.dev                  # BIDScoin version (should correspond with the version in pyproject.toml)
+    version: 4.5.2.dev                  # BIDScoin version (should correspond with the version in pyproject.toml)
     subprefix: sub-                 # The subject prefix of the source data
     sesprefix: ses-                 # The session prefix of the source data
     bidsignore: [extra_data/, sub-*_ct.*]   # List of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv]
@@ -1163,7 +1163,8 @@ Psychopy:
     events: &psychopy_events
       parsing:                          # The settings to parse the source table from the log file
         table: [long-wide, pivot, 1]    # The raw source table or a pivoted 'onset', 'duration', 'event_type' version
-        expand: scannerPulse.rt         # Expands lists into columns for each array item
+        add_started:                    # Columns for which the .started time is added, i.e. that are converted to absolute timestamps (e.g. the .rt columns)
+        expand:                         # Expands lists into columns for each array item
       columns:                          # Columns that are included in the output table, i.e. {output column: input column}
       - onset: onset                    # The mapping for the first required column 'onset'
       - duration: duration              # The mapping for the second required column 'duration'
@@ -1173,6 +1174,8 @@ Psychopy:
           event_type: '.*'
       time:
         cols: ['(?i).*time.*', '(?i).*duration.*', '(?i).*onset.*', '(?i).*start.*', '(?i).*stop.*', '.*\.rt']
+        start:
+          event_type:                   # E.g. scannerPulse.rt_0
 
   eeg:        # ----------------------- All EEG run-items ---------------------------
   - attributes: *psychopy_attr

bidscoin/plugins/__init__.py

@@ -285,7 +285,7 @@ def eventstable(self) -> pd.DataFrame:
             # Add the matching rows to the grand rows group
             rows |= rowgroup
 
-        return df.loc[rows].sort_values(by='onset')
+        return df.loc[rows].sort_values(by='onset', key=lambda x: pd.to_numeric(x, errors='coerce'))
 
     @property
     def parsing(self) -> dict:

bidscoin/plugins/events2bids.py

@@ -316,21 +316,35 @@ def logtable(self) -> pd.DataFrame:
         """Returns the Psychopy log-table"""
 
         # Start with a fresh data frame
-        df = self._sourcetable.copy()
+        df: pd.DataFrame = self._sourcetable.copy()
         if not len(df):
             return df
 
         # Get the table name
         table = self.parsing.get('table', ['long-wide', 'pivot', 1])
         table = table[table[-1]]
 
+        # Add .started, i.e. use absolute timestamps
+        def add_started(time, started: float):
+            try:
+                if isinstance(time, str) and time.startswith('[') and time.endswith(']'):
+                    return [float(val) + started for val in ast.literal_eval(time)]
+                return float(time) + started
+            except (ValueError, TypeError, SyntaxError) as error:
+                return time
+
+        # Identify columns that match the regex and have a corresponding `.started` column, and apply the add_started transformations
+        for add2col, started in {col: f"{col.rsplit('.', 1)[0]}.started" for col in df
+                                 if re.fullmatch(self.parsing.get('add_started') or '', col) and f"{col.rsplit('.', 1)[0]}.started" in df}.items():
+            df[add2col] = df.apply(lambda row: add_started(row[add2col], row[started]), axis=1)
+
         # Expand the array items in the source data, e.g. scannerPulse.rt = ["[493.15245039993897, 494.6524632999208, 496.15445999987423, 497.65245070005767, 499.1524501000531]"]
         try:
             for expand in set(col for col in df if re.fullmatch(self.parsing.get('expand') or '', col)):
-                ds  = df[expand].apply(lambda x: ast.literal_eval(x) if isinstance(x,str) and x.startswith('[') else [])    # Convert string representation of lists into actual Python lists
-                df_ = ds.apply(pd.Series).add_prefix(f"{expand}{'.started' if '.' in expand and table=='pivot' else ''}_")  # Append `.started` to pivot the data into the onset column
+                ds  = df[expand].apply(lambda x: ast.literal_eval(x) if isinstance(x,str) and x.startswith('[') else x)     # Convert string representation of lists into actual Python lists
+                df_ = ds.apply(pd.Series).add_prefix(f"{expand}{'.started' if '.' in expand and table=='pivot' else ''}_")  # Expand each item into its own column and append `.started` to pivot the data into the onset column
                 if '.' in expand:                                                                                           # Time columns should have a `.` in their name
-                    df_ = df_.rename(columns=lambda col: re.sub(r'(.*)\.(\w+)_(\d+)', r'\1_\3.\2', col))        # Put e.g. `.rt` or `.started` back at the end
+                    df_ = df_.rename(columns=lambda col: re.sub(r'(.*)\.(\w+)_(\d+)', r'\1_\3.\2', col))        # Put e.g. `.rt` or `.started` back at the end of the column name
                 if not df_.empty:
                     df = pd.concat([df.drop(columns=[expand]), df_], axis=1)
         except (re.error, TypeError) as pattern_error:

docs/CHANGELOG.md

@@ -2,7 +2,7 @@
 
 *All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)*
 
-## [4.5.1.dev]
+## [4.5.2.dev]
 
 ## [4.5.0] - 2025-02-05
 

docs/plugins.rst

@@ -24,10 +24,10 @@ Nibabel2bids: a generic plugin for imaging data
 
 The nibabel2bids plugin wraps around the versatile `nibabel <https://nipy.org/nibabel>`__ tool to convert a wide variety of data formats into NIfTI-files. Currently, the default template bidsmap is tailored to NIfTI source data only (but this can readily be extended), and BIDS sidecar files are not automatically produced by nibabel (but see the note further below). Please cite: `DOI: 10.5281/zenodo.591597 <https://doi.org/10.5281/zenodo.591597>`__
 
-Events2bids: a plugin for NeuroBS Presentation log data
--------------------------------------------------------
+Events2bids: a plugin for stimulus presentation log data
+--------------------------------------------------------
 
-The events2bids plugin parses `NeuroBS <https://www.neurobs.com/>`__ stimulus Presentation log files to BIDS task events files. See the `workflow page <./workflow.html#stimulus-events>`__ for usage.
+The events2bids plugin parses `NeuroBS <https://www.neurobs.com/>`__ Presentation, `PsychoPy <https://psychopy.org/>`__, as well as generic behavioural log files to BIDS task events files. See the `workflow page <./workflow.html#stimulus-events>`__ for usage.
 
 .. note::
    Out of the box, BIDScoin plugins typically produce sidecar files that contain metadata from the source headers. However, when such metadata is missing (e.g. as for nibabel2bids), or when it needs to be appended or overruled, then users can add sidecar files to the source data (as explained `here <./bidsmap_indepth.html#run-items>`__) or add that metadata using the bidseditor (the latter takes precedence).

docs/workflow.rst

@@ -176,24 +176,43 @@ Finally, if all BIDS output names in the main window look correct, click the [Sa
 
 Stimulus events
 ```````````````
-If your dataset contain (stimulus) events logfiles and you are using e.g. the `events2bids <./plugins.html#events2bids-a-plugin-for-neurobs-presentation-log-data>`__ plugin to convert them to `BIDS events <https://bids-specification.readthedocs.io/en/stable/modality-specific-files/task-events.html>`__, you will get a ``Presentation`` tab in the main window. The edit window for the run-items in there will include an additional ``BIDS output data`` table, providing a preview of the output data. ``Edit`` the output data if needed.
+If your dataset contain behavioural/stimulus presentation logfiles and you are using e.g. the `events2bids <./plugins.html#events2bids-a-plugin-for-stimulus-presentation-log-data>`__ plugin to convert them to `BIDS events <https://bids-specification.readthedocs.io/en/stable/modality-specific-files/task-events.html>`__, you will get a ``Presentation``, ``PsychoPy`` and/or a generic ``Logdata`` tab in the main window. If you open a run-items in there, you will get the same edit window as explained before, except that an extra ``BIDS output data`` table is appended on the right side. This table provides a preview of your log data after conversion to BIDS. Click on the ``Edit`` button next to the output table to adjust its content to your needs.
 
 .. dropdown:: More details...
 
-   The edit window will then show the input data (left column), the mapping tables to convert the input data (middle column), and the preview of the converted output data. Tweak the mapping tables until the conversion is correct, and click on ``Done``. The mapping tables include tables for selecting the ``Columns`` and ``Rows``, along with a ``Timing`` table for calibration of the clock:
+   The edit window displays the parsed input data on the left, BIDS conversion settings in the center, and a preview of the BIDS output on the right.
 
-   * The **'Columns'** table specifies which input column names are included (left) and how they should be named in the output table (right). You can add, edit and remove column names as needed
-   * The **'Rows'** table specifies which input rows are included in the output table. A ``condition`` (left) is a dictionary with columns names as keys and regular expression patterns as values. Rows are included if the pattern matches with the column value, e.g. when an experimental condition is met. Note that the patterns within a condition act as AND operators, i.e. the more patterns you add, the less rows your include in the output table. The ``output column`` is optional and can be useful, e.g. to create a new output column or (contrast) regressor for your design matrix (see the screenshot below). Each row in the rows table defines a condition. Conditions acts as OR operators, i.e. the more conditions you add to the table, the more rows are included in the output table.
-   * The **'Timing'** table contains settings for converting input time values to BIDS compliant output values:
+   - **Left panel**: Configure how your raw log files are transformed into a tabular format, which is then processed using the settings from the center panel.
 
-     * **columns** -- A list of input column names that hold time values.
-     * **units/sec** -- The number of source data time units per second (e.g., 10000 for clock times with a precision of 0.1 milliseconds).
-     * **start** -- A dictionary with column names and event-codes that define the start of the run (time zero), e.g. the column name and event-code that log the scanner pulses.
+     - For **Presentation log files**, you can select from the drop-down menu which table within your log file to use.
+     - For **PsychoPy log files**, you can select from the drop-down menu whether to use the raw long-wide data, or to use a pivoted version in which all ``.started`` and ``.stopped`` timestamps are stored in the ``onset``, ``duration``, and ``event_type`` columns.
+     - Additionally, for **PsychoPy**, you can specify values for ``add_started`` and ``expand`` as regex patterns to select relevant input columns.
+
+       - **add_started** columns: The corresponding ``.started`` times will be added to these columns. This is useful for converting relative times (e.g., ``.rt`` reaction times) into absolute timestamps.
+       - **expand** columns: If a column contains list data (e.g., ``"['0', '2.1', '4.2']"``), it will be split into multiple new columns. These expanded columns are assumed to store time onsets and will be included when pivoting the data.
+
+   - **Center panel**: Define the mapping parameters that transform the input table (left panel) into the output table (right panel).
+     The mapping parameters are organized into three tables:
+
+     1. **'Columns' table** – Specifies which input column names are included (left) and how they should be named in the output table (right). You can add, edit, and remove column names as needed.
+     2. **'Rows' table** – Specifies which input rows are included in the output table. A **condition** (left) is a dictionary where column names serve as keys and values as regular expression patterns. Rows are included if the pattern matches a column value, i.e. when a specific experimental condition is met.
+
+       - Within a condition, patterns act as **AND** operators, meaning that the more patterns you add to the dictionary, the fewer rows are included in the output.
+       - Between conditions, they act as **OR** operators, meaning that adding more conditions increases the number of included rows.
+       - The **'output column'** is optional and can be used, for example, to create new output columns or contrast regressors for a design matrix (see the screenshot below).
+
+     3. **'Timing' table** – Contains settings for converting input time values to BIDS-compliant output values:
+
+       - **columns** – A list of input column names that contain time values.
+       - **units/sec** – The number of time units per second in the source data (e.g., 10000 for clock times with a precision of 0.1 milliseconds).
+       - **start** – A dictionary specifying column names and event codes that define the start of the run (time zero), such as the column name and event code that log scanner pulses.
 
    .. figure:: ./_static/bidseditor_edit_events.png
       :width: 100%
 
-   *Edit window for conversion of Presentation log data to BIDS output data. Note that, since the first row condition has a non-selective matching pattern* ``.*``, *all input rows are included. Also note that, for selected rows, each of the two subsequent conditions add data ("go" and "stop") to the new* ``task`` *output column.*
+   *Edit window for converting Presentation log data to BIDS output. Note that, since the first row condition uses the non-selective matching pattern* ``.*``, *all input rows are included. Additionally, for selected rows, each of the two subsequent conditions adds data ("go" and "stop") to the new* ``task`` *output column.*
+
+   Adjust the mapping tables until the transformation is correct, then click on **"Done"**.
 
 .. tip::
    The BIDScoin GUI provides several tools to help you set the correct values: