Merge remote-tracking branch 'upstream/main'

Author: Thinh Nguyen

.github/workflows/u24_element_release_call.yaml

@@ -17,7 +17,6 @@ jobs:
     secrets:
       TWINE_USERNAME: ${{secrets.TWINE_TEST_USERNAME}}
       TWINE_PASSWORD: ${{secrets.TWINE_TEST_PASSWORD}}
-      GOOGLE_ANALYTICS_KEY: ${{secrets.GOOGLE_ANALYTICS_KEY}}
   call_u24_elements_release_alpine:
     if: >-
       github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'datajoint' && !contains(github.event.workflow_run.head_branch, 'test')
@@ -27,4 +26,3 @@ jobs:
     secrets:
       TWINE_USERNAME: ${{secrets.TWINE_USERNAME}}
       TWINE_PASSWORD: ${{secrets.TWINE_PASSWORD}}
-      GOOGLE_ANALYTICS_KEY: ${{secrets.GOOGLE_ANALYTICS_KEY}}

CHANGELOG.md

@@ -3,6 +3,26 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
+## [0.6.0] - 2023-07-26
+
++ Update - `prairieviewreader.py` -> `prairie_view_loader.py`
++ Update - `get_pv_metadata()` -> `get_prairieview_metadata()`
++ Update - Internal variable names within `prairie_view_loader.py`
+
+## [0.5.4] - 2023-05-25
+
++ Fix - DANDI URL for uploads where `staging=False`.
++ Fix - Docs to remove Google Analytics Key and add Markdown extensions
+
+## [0.5.3] - 2023-05-11
+
++ Fix - `.ipynb` dark mode output for all notebooks.
++ Fix - Remove `GOOGLE_ANALYTICS_KEY` from `u24_element_release_call.yml`.
+
+## [0.5.2] - 2023-04-28
+
++ Fix - `.ipynb` output in tutorials is not visible in dark mode.
+
 ## [0.5.1] - 2023-03-15
 
 + Fix - ingestion routine for multiple Z devices in `prairieviewreader.py`.
@@ -57,6 +77,10 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 
 + Add - Readers for: `ScanImage`, `Suite2p`, `CaImAn`.
 
+[0.6.0]: https://github.com/datajoint/element-interface/releases/tag/0.6.0
+[0.5.4]: https://github.com/datajoint/element-interface/releases/tag/0.5.4
+[0.5.3]: https://github.com/datajoint/element-interface/releases/tag/0.5.3
+[0.5.2]: https://github.com/datajoint/element-interface/releases/tag/0.5.2
 [0.5.1]: https://github.com/datajoint/element-interface/releases/tag/0.5.1
 [0.5.0]: https://github.com/datajoint/element-interface/releases/tag/0.5.0
 [0.4.2]: https://github.com/datajoint/element-interface/releases/tag/0.4.2

docs/.docker/pip_requirements.txt

@@ -7,4 +7,6 @@ mdx-truly-sane-lists
 mkdocs-gen-files
 mkdocs-literate-nav
 mkdocs-exclude-search
-mkdocs-markdownextradata-plugin
+mkdocs-markdownextradata-plugin
+mkdocs-jupyter
+mkdocs-section-index

docs/mkdocs.yaml

@@ -42,17 +42,14 @@ nav:
 #     UPSTREAM_REPO=https://github.com/datajoint/element-{ELEMENT}.git \
 #     HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build
 #     ```
-# 02. Site analytics depend on a local environment variable GOOGLE_ANALYTICS_KEY
-#     You can find this in LastPass or declare with any string to suppress errors
-# 03. The API section will pull docstrings.
+# 02. The API section will pull docstrings.
 #     A. Follow google styleguide e.g., 
 #        https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html
 #        With typing suggestions: https://docs.python.org/3/library/typing.html
 #     B. To pull a specific workflow fork, change ./docs/src/api/make_pages.py#L19
-# 04. To see your fork of the workflow-{element} in this render, change the
+# 03. To see your fork of the workflow-{element} in this render, change the
 #     URL in ./docs/src/api/make_pages.py#L19 to your fork.
-# 05. For redirecting options For redirect options, see 'redirects' below.
-# 06. To deploy this site on your fork, 
+# 04. To deploy this site on your fork, 
 #     A. declare a branch called gh-pages
 #     B. go to the your fork > settings > pages 
 #     C. direct pages to render from the gh-pages branch at root
@@ -87,9 +84,6 @@ theme:
 plugins:
   - markdownextradata: {}
   - search
-  # - redirects: # OPTIONAL REDIRECTS
-  #     redirect_maps:
-  #       "index.md": "getting_started.md"
   - mkdocstrings:
       default_handler: python
   - gen-files:
@@ -120,13 +114,12 @@ markdown_extensions:
       linenums: true
   - pymdownx.inlinehilite
   - pymdownx.snippets
-  
+  - pymdownx.magiclink # Displays bare URLs as links
+  - pymdownx.tasklist: # Renders check boxes in tasks lists
+      custom_checkbox: true
 extra:
   PATCH_VERSION: !ENV PATCH_VERSION
   generator: false # Disable watermark
-  analytics:
-    provider: google
-    property: !ENV GOOGLE_ANALYTICS_KEY
   version:
     provider: mike
   social:

docs/src/.overrides/assets/stylesheets/extra.css

@@ -91,3 +91,8 @@ html a[title="YouTube"].md-social__link svg {
     /* previous/next text */
     /* --md-footer-fg-color: var(--dj-white); */
 }
+
+[data-md-color-scheme="slate"] td,
+th {
+    color: var(--dj-black)
+}

element_interface/prairieviewreader.py renamed to element_interface/prairie_view_loader.py

@@ -5,107 +5,110 @@
 import numpy as np
 
 
-def get_pv_metadata(pvtiffile: str) -> dict:
-    """Extract metadata for scans generated by PrairieView acquisition software.
+def get_prairieview_metadata(ome_tif_filepath: str) -> dict:
+    """Extract metadata for scans generated by Prairie View acquisition software.
 
-    The PrairieView software generates one .ome.tif imaging file per frame acquired. The
-    metadata for all frames is contained one .xml file. This function locates the .xml
-    file and generates a dictionary necessary to populate the DataJoint ScanInfo and
-    Field tables. PrairieView works with resonance scanners with a single field.
-    PrairieView does not support bidirectional x and y scanning. ROI information is not
-    contained in the .xml file. All images generated using PrairieView have square
-    dimensions(e.g. 512x512).
+    The Prairie View software generates one `.ome.tif` imaging file per frame
+    acquired. The metadata for all frames is contained in one .xml file. This
+    function locates the .xml file and generates a dictionary necessary to
+    populate the DataJoint `ScanInfo` and `Field` tables. Prairie View works
+    with resonance scanners with a single field. Prairie View does not support
+    bidirectional x and y scanning. ROI information is not contained in the
+    `.xml` file. All images generated using Prairie View have square dimensions(e.g. 512x512).
 
     Args:
-        pvtiffile: An absolute path to the .ome.tif image file.
+        ome_tif_filepath: An absolute path to the .ome.tif image file.
 
     Raises:
         FileNotFoundError: No .xml file containing information about the acquired scan
-            was found at path in parent directory at `pvtiffile`.
+            was found at path in parent directory at `ome_tif_filepath`.
 
     Returns:
         metainfo: A dict mapping keys to corresponding metadata values fetched from the
             .xml file.
     """
 
     # May return multiple xml files. Only need one that contains scan metadata.
-    xml_files = pathlib.Path(pvtiffile).parent.glob("*.xml")
+    xml_files_list = pathlib.Path(ome_tif_filepath).parent.glob("*.xml")
 
-    for xml_file in xml_files:
-        tree = ET.parse(xml_file)
-        root = tree.getroot()
-        if root.find(".//Sequence"):
+    for file in xml_files_list:
+        xml_tree = ET.parse(file)
+        xml_file = xml_tree.getroot()
+        if xml_file.find(".//Sequence"):
             break
     else:
         raise FileNotFoundError(
-            f"No PrarieView metadata XML file found at {pvtiffile.parent}"
+            f"No PrarieView metadata .xml file found at {pathlib.Path(ome_tif_filepath).parent}"
         )
 
     bidirectional_scan = False  # Does not support bidirectional
     roi = 0
     n_fields = 1  # Always contains 1 field
-    record_start_time = root.find(".//Sequence/[@cycle='1']").attrib.get("time")
+    recording_start_time = xml_file.find(".//Sequence/[@cycle='1']").attrib.get("time")
 
     # Get all channels and find unique values
     channel_list = [
         int(channel.attrib.get("channel"))
-        for channel in root.iterfind(".//Sequence/Frame/File/[@channel]")
+        for channel in xml_file.iterfind(".//Sequence/Frame/File/[@channel]")
     ]
     n_channels = len(set(channel_list))
-    n_frames = len(root.findall(".//Sequence/Frame"))
+    n_frames = len(xml_file.findall(".//Sequence/Frame"))
     framerate = 1 / float(
-        root.findall('.//PVStateValue/[@key="framePeriod"]')[0].attrib.get("value")
+        xml_file.findall('.//PVStateValue/[@key="framePeriod"]')[0].attrib.get("value")
     )  # rate = 1/framePeriod
 
     usec_per_line = (
         float(
-            root.findall(".//PVStateValue/[@key='scanLinePeriod']")[0].attrib.get(
+            xml_file.findall(".//PVStateValue/[@key='scanLinePeriod']")[0].attrib.get(
                 "value"
             )
         )
         * 1e6
     )  # Convert from seconds to microseconds
 
-    scan_datetime = datetime.strptime(root.attrib.get("date"), "%m/%d/%Y %I:%M:%S %p")
+    scan_datetime = datetime.strptime(
+        xml_file.attrib.get("date"), "%m/%d/%Y %I:%M:%S %p"
+    )
 
-    total_duration = float(
-        root.findall(".//Sequence/Frame")[-1].attrib.get("relativeTime")
+    total_scan_duration = float(
+        xml_file.findall(".//Sequence/Frame")[-1].attrib.get("relativeTime")
     )
 
-    px_height = int(
-        root.findall(".//PVStateValue/[@key='pixelsPerLine']")[0].attrib.get("value")
+    pixel_height = int(
+        xml_file.findall(".//PVStateValue/[@key='pixelsPerLine']")[0].attrib.get(
+            "value"
+        )
     )
     # All PrairieView-acquired images have square dimensions (512 x 512; 1024 x 1024)
-    px_width = px_height
+    pixel_width = pixel_height
 
     um_per_pixel = float(
-        root.find(
+        xml_file.find(
             ".//PVStateValue/[@key='micronsPerPixel']/IndexedValue/[@index='XAxis']"
         ).attrib.get("value")
     )
 
-    um_height = um_width = float(px_height) * um_per_pixel
+    um_height = um_width = float(pixel_height) * um_per_pixel
 
     # x and y coordinate values for the center of the field
     x_field = float(
-        root.find(
+        xml_file.find(
             ".//PVStateValue/[@key='currentScanCenter']/IndexedValue/[@index='XAxis']"
         ).attrib.get("value")
     )
     y_field = float(
-        root.find(
+        xml_file.find(
             ".//PVStateValue/[@key='currentScanCenter']/IndexedValue/[@index='YAxis']"
         ).attrib.get("value")
     )
     if (
-        root.find(
+        xml_file.find(
             ".//Sequence/[@cycle='1']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']"
         )
         is None
     ):
-
         z_fields = np.float64(
-            root.find(
+            xml_file.find(
                 ".//PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue"
             ).attrib.get("value")
         )
@@ -114,46 +117,53 @@ def get_pv_metadata(pvtiffile: str) -> dict:
         bidirection_z = False
 
     else:
+        bidirection_z = (
+            xml_file.find(".//Sequence").attrib.get("bidirectionalZ") == "True"
+        )
 
-        bidirection_z = root.find(".//Sequence").attrib.get("bidirectionalZ") == "True"
-
-        # One "Frame" per depth. Gets number of frames in first sequence
+        # One "Frame" per depth in the .xml file. Gets number of frames in first sequence
         planes = [
             int(plane.attrib.get("index"))
-            for plane in root.findall(".//Sequence/[@cycle='1']/Frame")
+            for plane in xml_file.findall(".//Sequence/[@cycle='1']/Frame")
         ]
         n_depths = len(set(planes))
 
-        z_controllers = root.findall(
+        z_controllers = xml_file.findall(
             ".//Sequence/[@cycle='1']/Frame/[@index='1']/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue"
         )
-        if len(z_controllers) > 1:
 
+        # If more than one Z-axis controllers are found, 
+        # check which controller is changing z_field depth. Only 1 controller
+        # must change depths.
+        if len(z_controllers) > 1:
             z_repeats = []
-            for controller in root.findall(
-                ".//Sequence/[@cycle='1']/Frame/[@index='1']/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/"):
+            for controller in xml_file.findall(
+                ".//Sequence/[@cycle='1']/Frame/[@index='1']/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/"
+            ):
                 z_repeats.append(
                     [
                         float(z.attrib.get("value"))
-                        for z in root.findall(
+                        for z in xml_file.findall(
                             ".//Sequence/[@cycle='1']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue/[@subindex='{0}']".format(
                                 controller.attrib.get("subindex")
                             )
                         )
                     ]
                 )
-    
-
-            controller_assert = [not all(z == z_controller[0] for z in z_controller) for z_controller in z_repeats]
-
-            assert sum(controller_assert)==1, "Multiple controllers changing z depth is not supported"
+            controller_assert = [
+                not all(z == z_controller[0] for z in z_controller)
+                for z_controller in z_repeats
+            ]
+            assert (
+                sum(controller_assert) == 1
+            ), "Multiple controllers changing z depth is not supported"
 
             z_fields = z_repeats[controller_assert.index(True)]
-            
+
         else:
             z_fields = [
                 z.attrib.get("value")
-                for z in root.findall(
+                for z in xml_file.findall(
                     ".//Sequence/[@cycle='1']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue/[@subindex='0']"
                 )
             ]
@@ -176,15 +186,15 @@ def get_pv_metadata(pvtiffile: str) -> dict:
         bidirectional_z=bidirection_z,
         scan_datetime=scan_datetime,
         usecs_per_line=usec_per_line,
-        scan_duration=total_duration,
-        height_in_pixels=px_height,
-        width_in_pixels=px_width,
+        scan_duration=total_scan_duration,
+        height_in_pixels=pixel_height,
+        width_in_pixels=pixel_width,
         height_in_um=um_height,
         width_in_um=um_width,
         fieldX=x_field,
         fieldY=y_field,
         fieldZ=z_fields,
-        recording_time=record_start_time,
+        recording_time=recording_start_time,
     )
 
     return metainfo

element_interface/version.py

@@ -1,3 +1,3 @@
 """Package metadata"""
 
-__version__ = "0.5.1"
+__version__ = "0.6.0"