Merge branch 'master' into girder-5

Author: manthey

CHANGELOG.md

@@ -4,11 +4,17 @@
 
 ### Improvements
 
-- Speed up annotation centroid queries ([#1981](../../pull/1981))
+- Speed up annotation centroid queries ([#1981](../../pull/1981), [#1990](../../pull/1990))
 - Improve large annotation load and display speed ([#1982](../../pull/1982))
 - Denormalize some values in the annotation collection to support improving display speed ([#1984](../../pull/1984))
 - Improve indices for annotationelement queries ([#1985](../../pull/1985))
 - Use some raw bson handling to speed up annotationelement serialization ([#1986](../../pull/1986))
+- Improve indices used in large image associated files in girder ([#1988](../../pull/1988))
+
+### Changes
+
+- Skip adjacent DICOM file search during Girder import ([#1987](../../pull/1987))
+- Update to work with the most recent wsidicom ([#1989](../../pull/1989))
 
 ### Bug Fixes
 

girder/girder_large_image/models/image_item.py

@@ -52,14 +52,16 @@ def initialize(self):
         self.ensureIndices(['largeImage.fileId'])
         File().ensureIndices([
             ([
-                ('isLargeImageThumbnail', pymongo.ASCENDING),
-                ('attachedToType', pymongo.ASCENDING),
                 ('attachedToId', pymongo.ASCENDING),
+                ('attachedToType', pymongo.ASCENDING),
+                ('isLargeImageThumbnail', pymongo.ASCENDING),
+                ('thumbnailKey', pymongo.ASCENDING),
             ], {}),
             ([
-                ('isLargeImageData', pymongo.ASCENDING),
-                ('attachedToType', pymongo.ASCENDING),
                 ('attachedToId', pymongo.ASCENDING),
+                ('attachedToType', pymongo.ASCENDING),
+                ('isLargeImageData', pymongo.ASCENDING),
+                ('thumbnailKey', pymongo.ASCENDING),
             ], {}),
         ])
 

girder_annotation/girder_large_image_annotation/models/annotationelement.py

@@ -352,7 +352,7 @@ def __getitem__(self, key):
                     props[prop] = len(props)
                     proplist.append([element.get(key, defaultProps.get(key)) for key in propskeys])
                 yield [
-                    str(element.get('id') or entry['_id']),
+                    element.get('id', entry['_id']),
                     (bbox['lowx'] + bbox['highx']) / 2,
                     (bbox['lowy'] + bbox['highy']) / 2,
                     bbox['size'] if entry.get('type') != 'point' else 0,

girder_annotation/girder_large_image_annotation/rest/annotation.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 ##############################################################################
 
+import binascii
 import json
 import logging
 import struct
@@ -219,6 +220,14 @@ def _getAnnotation(self, annotation, params):
         :param params: paging and region parameters for the annotation.
         :returns: a function that will return a generator.
         """
+        import gc
+
+        # We increase the threshold2 value for python's gc, because we may
+        # return a massive number of annotations and the garbage collection
+        # full scan is _slow_ but probably not useful.
+        threshold = gc.get_threshold()
+        if threshold[2] < 100:
+            gc.set_threshold(threshold[0], threshold[1], 100)
         # Ensure that we have read access to the parent item.  We could fail
         # faster when there are permissions issues if we didn't load the
         # annotation elements before checking the item access permissions.
@@ -245,6 +254,7 @@ def generateResult():
             if centroids:
                 # Add a null byte to indicate the start of the binary data
                 yield b'\x00'
+                packing = struct.Struct('<fffl')
             for element in Annotationelement().yieldElements(annotation, params, info):
                 # The json conversion is fastest if we use defaults as much as
                 # possible.  The only value in an annotation element that needs
@@ -261,9 +271,9 @@ def generateResult():
                             [int(x) if isinstance(x, float) and x.is_integer() else x for x in sub]
                             for sub in hole] for hole in element['holes']]
                 else:
-                    element = struct.pack(
-                        '>QL', int(element[0][:16], 16), int(element[0][16:24], 16),
-                    ) + struct.pack('<fffl', *element[1:])
+                    element = (
+                        binascii.unhexlify(element[0]) if isinstance(element[0], str) else
+                        element[0].binary) + packing.pack(*element[1:])
                 # Use orjson; it is much faster.  The standard json library
                 # could be used in its most default mode instead like so:
                 #   result = json.dumps(element, separators=(',', ':'))

sources/dicom/large_image_source_dicom/dicomweb_utils.py

@@ -26,7 +26,10 @@ def get_dicomweb_metadata(client, study_uid, series_uid):
 
 def get_first_wsi_volume_metadata(client, study_uid, series_uid):
     # Find the first WSI Volume and return the DICOMweb metadata
-    from wsidicom.uid import WSI_SOP_CLASS_UID
+    try:
+        from pydicom.uid import VLWholeSlideMicroscopyImageStorage
+    except ImportError:
+        from wsidicom.uid import WSI_SOP_CLASS_UID as VLWholeSlideMicroscopyImageStorage
 
     image_type_tag = dicom_key_to_tag('ImageType')
     instance_uid_tag = dicom_key_to_tag('SOPInstanceUID')
@@ -46,7 +49,7 @@ def get_first_wsi_volume_metadata(client, study_uid, series_uid):
     volume_instance = None
     for instance in wsi_instances:
         class_type = instance.get(class_uid_tag, {}).get('Value')
-        if not class_type or class_type[0] != WSI_SOP_CLASS_UID:
+        if not class_type or class_type[0] != VLWholeSlideMicroscopyImageStorage:
             # Only look at WSI classes
             continue
 

sources/dicom/large_image_source_dicom/girder_source.py

@@ -21,6 +21,11 @@ class DICOMGirderTileSource(DICOMFileTileSource, GirderTileSource):
 
     _mayHaveAdjacentFiles = True
 
+    def __init__(self, item, *args, **kwargs):
+        # This must be set before super() because parent's __init__ calls _getLargeImagePath()
+        self._noCache = kwargs.get('noCache', False)
+        super().__init__(item, *args, **kwargs)
+
     def _getAssetstore(self):
         files = Item().childFiles(self.item, limit=1)
         if not files:
@@ -45,6 +50,13 @@ def _getFilesystemLargeImagePath(self):
         filelist = [
             File().getLocalFilePath(file) for file in Item().childFiles(self.item)
             if self._pathMightBeDicom(file['name'])]
+
+        if self._noCache:
+            # Skip expensive folder scan during import checks.
+            # During import, just return files from current item.
+            # Adjacent files will be found later when actually viewing.
+            return filelist if filelist else None
+
         if len(filelist) != 1:
             return filelist
         try:

tox.ini

@@ -33,7 +33,7 @@ deps =
   pytest-cov
   pytest-asyncio
   pytest-custom-exit-code
-  pytest-girder>=5.0.0a10
+  pytest-girder>=5.0.0a13
   pytest-rerunfailures
   pytest-xdist
 allowlist_externals =
@@ -75,7 +75,7 @@ deps =
   pytest-cov
   pytest-asyncio
   pytest-custom-exit-code
-  pytest-girder==5.0.0a10
+  pytest-girder>=5.0.0a13
   pytest-rerunfailures
   pytest-xdist
 allowlist_externals = {[testenv:test]allowlist_externals}
@@ -95,7 +95,7 @@ deps =
   pytest-cov
   pytest-asyncio
   pytest-custom-exit-code
-  pytest-girder==5.0.0a10
+  pytest-girder>=5.0.0a13
   pytest-monkeytype
   pytest-rerunfailures
   pytest-xdist