Remove panimg from main environment (#4541)

This PR removes panimg as a direct dependency and puts everything into
the virtual environment. As vips is no longer in the main environment
this should solve the issue with pycurl installation on all platforms.
Tests are removed that were only testing panimg functionality, I checked
that they are present in the panimg repo - these were left over from the
extraction of panimg from this codebase a few years ago.

If this all works then we should create another package for the shared
models between panimg and grand challenge, for now they are copied in.

See DIAGNijmegen/rse-cloud-infrastructure#212

Author: jmsmkn

app/grandchallenge/cases/models.py

@@ -27,12 +27,16 @@
 from django_deprecate_fields import deprecate_field
 from grand_challenge_dicom_de_identifier.deidentifier import DicomDeidentifier
 from guardian.shortcuts import assign_perm, get_groups_with_perms, remove_perm
-from panimg.models import MAXIMUM_SEGMENTS_LENGTH, ColorSpace, ImageType
 from pydantic import ConfigDict, Field, field_validator
 from pydantic.alias_generators import to_camel
 from pydantic.dataclasses import dataclass
 from storages.utils import clean_name
 
+from grandchallenge.cases.panimg_models import (
+    MAXIMUM_SEGMENTS_LENGTH,
+    ColorSpace,
+    ImageType,
+)
 from grandchallenge.core.error_handlers import (
     DICOMImageSetUploadErrorHandler,
     RawImageUploadSessionErrorHandler,

app/grandchallenge/cases/panimg.py

@@ -3,9 +3,51 @@
 from pathlib import Path
 
 from django.utils._os import safe_join
-from panimg.models import PanImgFile, PostProcessorResult
 from pydantic import TypeAdapter
 
+from grandchallenge.cases.panimg_models import (
+    PanImgFile,
+    PanImgResult,
+    PostProcessorOptions,
+    PostProcessorResult,
+)
+
+
+def convert(*, input_directory, output_directory, builders):
+    builders_command = []
+    for builder in builders:
+        builders_command.extend(["--image-builder", builder])
+
+    panimg_command = shlex.join(
+        [
+            "panimg",
+            "convert",
+            "--input-dir",
+            str(input_directory.resolve()),
+            "--output-dir",
+            str(output_directory.resolve()),
+            *builders_command,
+            "--no-post-processing",
+        ]
+    )
+
+    cli_result = subprocess.run(
+        [
+            "bash",
+            "-c",
+            f"source /opt/virtualenvs/panimg/bin/activate && {panimg_command}",
+        ],
+        text=True,
+        check=True,
+        capture_output=True,
+    )
+
+    panimg_result: PanImgResult = TypeAdapter(PanImgResult).validate_json(
+        cli_result.stdout.splitlines()[-1]
+    )
+
+    return panimg_result
+
 
 def post_process(*, image_file, output_directory):
     panimg_file = _download_image_file(
@@ -23,7 +65,7 @@ def post_process(*, image_file, output_directory):
             "--input-file",
             str(panimg_file.file),
             "--post-processor",
-            "DZI",
+            PostProcessorOptions.DZI,
         ]
     )
 

app/grandchallenge/cases/panimg_models.py

@@ -0,0 +1,95 @@
+# These models are extracted from panimg itself and need to be used
+# in both places. It will be best to create a new python package
+# for this so that it can be used by both grand challenge and panimg
+
+from enum import Enum
+from pathlib import Path
+from uuid import UUID
+
+from pydantic.dataclasses import dataclass
+
+
+class ImageBuilderOptions(str, Enum):
+    MHD = "MHD"
+    NIFTI = "NIFTI"
+    NRRD = "NRRD"
+    DICOM = "DICOM"
+    TIFF = "TIFF"
+    OCT = "OCT"
+    FALLBACK = "FALLBACK"
+
+
+class PostProcessorOptions(str, Enum):
+    DZI = "DZI"
+
+
+# NOTE: Only int8 or uint8 data types are checked for segments
+# so the true maximum is 256
+MAXIMUM_SEGMENTS_LENGTH = 64
+
+
+class ColorSpace(str, Enum):
+    GRAY = "GRAY"
+    RGB = "RGB"
+    RGBA = "RGBA"
+    YCBCR = "YCBCR"
+
+
+ITK_COLOR_SPACE_MAP = {
+    1: ColorSpace.GRAY,
+    3: ColorSpace.RGB,
+    4: ColorSpace.RGBA,
+}
+
+
+class ImageType(str, Enum):
+    MHD = "MHD"
+    TIFF = "TIFF"
+    DZI = "DZI"
+
+
+class EyeChoice(str, Enum):
+    OCULUS_DEXTER = "OD"
+    OCULUS_SINISTER = "OS"
+    UNKNOWN = "U"
+    NOT_APPLICABLE = "NA"
+
+
+@dataclass(frozen=True)
+class PanImg:
+    pk: UUID
+    name: str
+    width: int
+    height: int
+    depth: int | None
+    voxel_width_mm: float | None
+    voxel_height_mm: float | None
+    voxel_depth_mm: float | None
+    timepoints: int | None
+    resolution_levels: int | None
+    window_center: float | None
+    window_width: float | None
+    color_space: ColorSpace
+    eye_choice: EyeChoice
+    segments: frozenset[int] | None = None
+
+
+@dataclass(frozen=True)
+class PanImgFile:
+    image_id: UUID
+    image_type: ImageType
+    file: Path
+    directory: Path | None = None
+
+
+@dataclass
+class PanImgResult:
+    new_images: set[PanImg]
+    new_image_files: set[PanImgFile]
+    consumed_files: set[Path]
+    file_errors: dict[Path, list[str]]
+
+
+@dataclass
+class PostProcessorResult:
+    new_image_files: set[PanImgFile]

app/grandchallenge/cases/tasks.py

@@ -1,9 +1,10 @@
 import re
 import zipfile
-from collections.abc import Callable, Sequence
+from collections.abc import Sequence
 from dataclasses import asdict, dataclass
 from pathlib import Path
 from shutil import rmtree
+from subprocess import CalledProcessError
 from tempfile import TemporaryDirectory
 
 import boto3
@@ -22,8 +23,6 @@
 from grand_challenge_dicom_de_identifier.exceptions import (
     RejectedDICOMFileError,
 )
-from panimg import convert
-from panimg.models import PanImgResult
 
 from grandchallenge.cases.models import (
     DICOMImageSetUpload,
@@ -34,7 +33,11 @@
     PostProcessImageTaskStatusChoices,
     RawImageUploadSession,
 )
-from grandchallenge.cases.panimg import post_process
+from grandchallenge.cases.panimg import convert, post_process
+from grandchallenge.cases.panimg_models import (
+    ImageBuilderOptions,
+    PanImgResult,
+)
 from grandchallenge.components.backends.exceptions import RetryStep
 from grandchallenge.components.backends.utils import UUID4_REGEX, safe_extract
 from grandchallenge.components.models import ComponentInterface
@@ -189,8 +192,8 @@ def _handle_error(*, error_message):
                 base_directory=tmp_dir,
                 upload_session=upload_session,
             )
-    except RuntimeError as error:
-        if "std::bad_alloc" in str(error):
+    except CalledProcessError as error:
+        if error.returncode == 137:
             _handle_error(
                 error_message=(
                     "The uploaded images were too large to process, "
@@ -297,15 +300,14 @@ def import_images(
     *,
     input_directory: Path,
     origin: RawImageUploadSession | None = None,
-    builders: Sequence[Callable] | None = None,
-    recurse_subdirectories: bool = True,
+    builders: Sequence[str] | None = None,
 ) -> ImporterResult:
     """
     Creates Image objects from a set of files.
 
     Parameters
     ----------
-    files
+    input_directory
         A Set of files that can form one or many Images
     origin
         The RawImageUploadSession (if any) that was the source of these files
@@ -318,13 +320,24 @@ def import_images(
         any file errors
 
     """
+    if builders is None:
+        builders = [
+            ImageBuilderOptions.MHD,
+            ImageBuilderOptions.NIFTI,
+            ImageBuilderOptions.NRRD,
+            ImageBuilderOptions.DICOM,
+            ImageBuilderOptions.TIFF,
+            ImageBuilderOptions.OCT,
+            ImageBuilderOptions.FALLBACK,
+        ]
+
     with TemporaryDirectory() as output_directory:
+        panimg_output_dir = Path(output_directory) / "output"
+
         panimg_result = convert(
             input_directory=input_directory,
-            output_directory=output_directory,
+            output_directory=panimg_output_dir,
             builders=builders,
-            post_processors=[],  # Do the post-processing later
-            recurse_subdirectories=recurse_subdirectories,
         )
 
         _check_all_ids(panimg_result=panimg_result)

app/grandchallenge/components/backends/base.py

@@ -13,6 +13,7 @@
 from json import JSONDecodeError
 from math import ceil
 from pathlib import Path
+from subprocess import CalledProcessError
 from tempfile import SpooledTemporaryFile, TemporaryDirectory
 from typing import NamedTuple
 from uuid import UUID
@@ -30,10 +31,10 @@
 from django.db import transaction
 from django.utils._os import safe_join
 from django.utils.functional import cached_property
-from panimg.image_builders import image_builder_mhd, image_builder_tiff
 from pydantic import BaseModel, ConfigDict
 from pydantic_core import to_json
 
+from grandchallenge.cases.panimg_models import ImageBuilderOptions
 from grandchallenge.cases.tasks import import_images
 from grandchallenge.components.backends.exceptions import (
     ComponentException,
@@ -892,17 +893,24 @@ def _create_images_result(self, *, interface):
             )
 
         with TemporaryDirectory() as tmpdir:
+            input_directory = Path(tmpdir)
+
             self._download_output_files(
-                output_files=output_files, tmpdir=tmpdir, prefix=prefix
+                output_files=output_files,
+                target_directory=input_directory,
+                prefix=prefix,
             )
 
             try:
                 importer_result = import_images(
-                    input_directory=tmpdir,
-                    builders=[image_builder_mhd, image_builder_tiff],
+                    input_directory=input_directory,
+                    builders=[
+                        ImageBuilderOptions.MHD,
+                        ImageBuilderOptions.TIFF,
+                    ],
                 )
-            except RuntimeError as error:
-                if "std::bad_alloc" in str(error):
+            except CalledProcessError as error:
+                if error.returncode == 137:
                     raise ComponentException(
                         "The output image was too large to process, "
                         "please try again with smaller images"
@@ -932,11 +940,15 @@ def _create_images_result(self, *, interface):
 
         return civ
 
-    def _download_output_files(self, *, output_files, tmpdir, prefix):
+    def _download_output_files(
+        self, *, output_files, target_directory, prefix
+    ):
         for file in output_files:
             try:
                 root_key = safe_join("/", file["Key"])
-                dest = safe_join(tmpdir, Path(root_key).relative_to(prefix))
+                dest = safe_join(
+                    target_directory, Path(root_key).relative_to(prefix)
+                )
             except (SuspiciousFileOperation, ValueError):
                 logger.warning(f"Skipping {file=}")
                 continue

app/grandchallenge/components/models.py

@@ -34,7 +34,6 @@
 from django.utils.translation import gettext_lazy as _
 from django_deprecate_fields import deprecate_field
 from django_extensions.db.fields import AutoSlugField
-from panimg.models import MAXIMUM_SEGMENTS_LENGTH
 from pydantic_core import MISSING
 
 from grandchallenge.cases.models import (
@@ -43,6 +42,7 @@
     ImageFile,
     RawImageUploadSession,
 )
+from grandchallenge.cases.panimg_models import MAXIMUM_SEGMENTS_LENGTH
 from grandchallenge.charts.specs import components_line
 from grandchallenge.components.backends.exceptions import (
     CIVNotEditableException,

app/grandchallenge/workstation_configs/models.py

@@ -9,8 +9,8 @@
 from django.db.models import PositiveSmallIntegerField
 from django_extensions.db.models import TitleSlugDescriptionModel
 from guardian.shortcuts import assign_perm
-from panimg.models import MAXIMUM_SEGMENTS_LENGTH
 
+from grandchallenge.cases.panimg_models import MAXIMUM_SEGMENTS_LENGTH
 from grandchallenge.core.fields import HexColorField
 from grandchallenge.core.guardian import (
     GroupObjectPermissionBase,