Feature/optional pin white and black (#118)

davisadam10 · web-flow · commit fb2608db4bb2 · 2025-10-19T08:33:42.000+13:00
* Introduce an optional white and black measure to help getting more accurate results at the extremes
Black - with the lens cap on
White - with the lens off

These are optional samples which can be provided or not

* In our log camera generator we insert the black as the first reference and sample
In our log camera generator if we have a white sample we use this as our max

* Move the verify into the overall validate settings so non matter if we are coming from a zip file or a directory all validations happen during the process function

* Start to cleanup the code now we do the same verification on all pathways

* Ensure we have file types before we do an iter

* Ensure we validate the project_settings before we try and process anything
diff --git a/aces/idt/application.py b/aces/idt/application.py
@@ -241,6 +241,34 @@ def _verify_directory(self, root_directory: Path | str) -> None:
         else:
             self.project_settings.data[DirectoryStructure.GREY_CARD] = []
 
+        if self.project_settings.data.get(DirectoryStructure.BLACK, []):
+            images = [
+                Path(root_directory) / image
+                for image in self.project_settings.data.get(
+                    DirectoryStructure.BLACK, []
+                )
+            ]
+            for image in images:
+                attest(image.exists())
+
+            self.project_settings.data[DirectoryStructure.BLACK] = images
+        else:
+            self.project_settings.data[DirectoryStructure.BLACK] = []
+
+        if self.project_settings.data.get(DirectoryStructure.WHITE, []):
+            images = [
+                Path(root_directory) / image
+                for image in self.project_settings.data.get(
+                    DirectoryStructure.WHITE, []
+                )
+            ]
+            for image in images:
+                attest(image.exists())
+
+            self.project_settings.data[DirectoryStructure.WHITE] = images
+        else:
+            self.project_settings.data[DirectoryStructure.WHITE] = []
+
     def _verify_file_type(self) -> None:
         """
         Verify that the *IDT* archive contains a unique file type and set the
diff --git a/aces/idt/core/constants.py b/aces/idt/core/constants.py
@@ -51,6 +51,8 @@ class DirectoryStructure:
     COLOUR_CHECKER: ClassVar[str] = "colour_checker"
     GREY_CARD: ClassVar[str] = "grey_card"
     FLATFIELD: ClassVar[str] = "flatfield"
+    WHITE: ClassVar[str] = "white"
+    BLACK: ClassVar[str] = "black"
 
 
 class UITypes:
diff --git a/aces/idt/framework/project_settings.py b/aces/idt/framework/project_settings.py
@@ -697,6 +697,8 @@ def from_directory(cls, directory: str) -> IDTProjectSettings:
         data = {
             DirectoryStructure.COLOUR_CHECKER: {},
             DirectoryStructure.GREY_CARD: [],
+            DirectoryStructure.WHITE: [],
+            DirectoryStructure.BLACK: [],
         }
 
         # Validate folder paths for colour_checker and grey_card exist
@@ -707,6 +709,15 @@ def from_directory(cls, directory: str) -> IDTProjectSettings:
             directory, DirectoryStructure.DATA, DirectoryStructure.GREY_CARD
         )
 
+        white_path = os.path.join(
+            directory, DirectoryStructure.DATA, DirectoryStructure.WHITE
+        )
+
+        black_path = os.path.join(
+            directory, DirectoryStructure.DATA, DirectoryStructure.BLACK
+        )
+
+        # Check for non optional paths
         if not os.path.exists(colour_checker_path) or not os.path.exists(
             grey_card_path
         ):
@@ -740,20 +751,46 @@ def from_directory(cls, directory: str) -> IDTProjectSettings:
 
         data[DirectoryStructure.COLOUR_CHECKER] = sorted_colour_checker
 
-        # Populate grey_card data
-        for root, _, files in os.walk(grey_card_path):
-            files.sort()
-            for file in files:
-                if os.path.basename(file).startswith("."):
-                    continue
-                absolute_file_path = os.path.join(root, file)
-                data[DirectoryStructure.GREY_CARD].append(
-                    os.path.relpath(absolute_file_path, start=directory)
-                )
+        # Populate grey_card data along with white and black data if we have them
+        cls.populate_data_from_path(
+            grey_card_path, data, directory, DirectoryStructure.GREY_CARD
+        )
+        cls.populate_data_from_path(
+            white_path, data, directory, DirectoryStructure.WHITE
+        )
+        cls.populate_data_from_path(
+            black_path, data, directory, DirectoryStructure.BLACK
+        )
 
         instance.data = data
         return instance
 
+    @classmethod
+    def populate_data_from_path(
+        cls, folder_path: str, data: dict, directory: str, data_key: DirectoryStructure
+    ) -> None:
+        """
+
+        Parameters
+        ----------
+        folder_path: The folder path to the sub folder for the given sequence
+        data: The data dict that holds the paths to the image sequences
+        directory
+            The directory to the project root containing the image sequence
+            directories.
+        data_key: the key we want to store the data under in the data dict
+        """
+        if os.path.exists(folder_path):
+            for root, _, files in os.walk(folder_path):
+                files.sort()
+                for file in files:
+                    if os.path.basename(file).startswith("."):
+                        continue
+                    absolute_file_path = os.path.join(root, file)
+                    data[data_key].append(
+                        os.path.relpath(absolute_file_path, start=directory)
+                    )
+
     def __str__(self) -> str:
         """
         Return a formatted string representation of the project settings.
diff --git a/aces/idt/generators/base_generator.py b/aces/idt/generators/base_generator.py
@@ -86,6 +86,8 @@ class IDTBaseGenerator(ABC):
     -   :attr:`~aces.idt.IDTBaseGenerator.image_grey_card_sampling`
     -   :attr:`~aces.idt.IDTBaseGenerator.samples_camera`
     -   :attr:`~aces.idt.IDTBaseGenerator.samples_reference`
+    -   :attr:`~aces.idt.IDTBaseGenerator.samples_white`
+    -   :attr:`~aces.idt.IDTBaseGenerator.samples_black`
     -   :attr:`~aces.idt.IDTBaseGenerator.LUT_unfiltered`
     -   :attr:`~aces.idt.IDTBaseGenerator.LUT_filtered`
     -   :attr:`~aces.idt.IDTBaseGenerator.LUT_decoding`
@@ -117,6 +119,10 @@ def __init__(self, project_settings: IDTProjectSettings) -> None:
 
         self._samples_camera = None
         self._samples_reference = None
+
+        self._samples_black = None
+        self._samples_white = None
+
         self._baseline_exposure = 0
 
         self._LUT_unfiltered = None
@@ -201,6 +207,32 @@ def samples_camera(self) -> NDArrayFloat | None:
 
         return self._samples_camera
 
+    @property
+    def samples_black(self) -> NDArrayFloat | None:
+        """
+        Getter property for the samples of the camera for the measured black which
+        is optional
+
+        Returns
+        -------
+        :class:`NDArray` or :py:data:`None`
+            Samples of the camera for the measured black level.
+        """
+        return self._samples_black
+
+    @property
+    def samples_white(self) -> NDArrayFloat | None:
+        """
+        Getter property for the samples of the camera for the measured white which
+        is optional
+
+        Returns
+        -------
+        :class:`NDArray` or :py:data:`None`
+            Samples of the camera for the measured white level.
+        """
+        return self._samples_white
+
     @property
     def samples_reference(self) -> NDArrayFloat | None:
         """
@@ -599,6 +631,62 @@ def _reformat_image(image: ArrayLike) -> NDArrayInt | NDArrayFloat:
                 0,
             ).tolist()
 
+        if self.project_settings.data.get(DirectoryStructure.BLACK, []):
+            self._samples_analysis[DirectoryStructure.BLACK] = {}
+            self._samples_analysis[DirectoryStructure.BLACK]["samples_median"] = []
+            for path in self.project_settings.data[DirectoryStructure.BLACK]:
+                with working_directory(self.project_settings.working_directory):
+                    LOGGER.info(
+                        'Reading "Black" from "%s"...',
+                        path,
+                    )
+
+                    image = _reformat_image(read_image(path))
+                    self._samples_analysis[DirectoryStructure.BLACK][
+                        "samples_median"
+                    ].append(
+                        np.median(
+                            as_float_array(image),
+                            (0, 1),
+                        ).tolist()
+                    )
+                    self._samples_black = np.median(
+                        as_float_array(
+                            self._samples_analysis[DirectoryStructure.BLACK][
+                                "samples_median"
+                            ]
+                        ),
+                        axis=0,
+                    ).tolist()
+
+        if self.project_settings.data.get(DirectoryStructure.WHITE, []):
+            self._samples_analysis[DirectoryStructure.WHITE] = {}
+            self._samples_analysis[DirectoryStructure.WHITE]["samples_median"] = []
+            for path in self.project_settings.data[DirectoryStructure.WHITE]:
+                with working_directory(self.project_settings.working_directory):
+                    LOGGER.info(
+                        'Reading "WHITE" from "%s"...',
+                        path,
+                    )
+
+                    image = _reformat_image(read_image(path))
+                    self._samples_analysis[DirectoryStructure.WHITE][
+                        "samples_median"
+                    ].append(
+                        np.median(
+                            as_float_array(image),
+                            (0, 1),
+                        ).tolist()
+                    )
+                    self._samples_white = np.median(
+                        as_float_array(
+                            self._samples_analysis[DirectoryStructure.WHITE][
+                                "samples_median"
+                            ]
+                        ),
+                        axis=0,
+                    ).tolist()
+
         if self.project_settings.cleanup:
             shutil.rmtree(self.project_settings.working_directory)
 
diff --git a/aces/idt/generators/log_camera.py b/aces/idt/generators/log_camera.py
@@ -206,7 +206,13 @@ def generate_LUT(self) -> LUT3x1D:
         LOGGER.info('Generating unfiltered "LUT3x1D" with "%s" size...', size)
 
         self._LUT_unfiltered = LUT3x1D(size=size, name="LUT - Unfiltered")
-
+        if self.samples_black:
+            self._samples_camera = np.insert(
+                self._samples_camera, 0, self.samples_black, axis=0
+            )
+            self._samples_reference = np.insert(
+                self._samples_reference, 0, np.zeros(3), axis=0
+            )
         for i in range(3):
             x = self._samples_camera[..., i] * (size - 1)
             y = self._samples_reference[..., i]
@@ -218,12 +224,14 @@ def generate_LUT(self) -> LUT3x1D:
                 LinearInterpolator(x, y), method="Constant"
             )(samples)
 
+            max_value = np.max(self._samples_camera)
+            if self.samples_white:
+                max_value = self.samples_white[i]
+
             # Searching for the index of ~middle camera code value * 125%
             # We are trying to find the logarithmic slope of the camera middle
             # range.
-            index_middle = np.searchsorted(
-                samples / size, np.max(self._samples_camera) / 2 * 1.25
-            )
+            index_middle = np.searchsorted(samples / size, max_value / 2 * 1.25)
             padding = index_middle // 2
             samples_middle = np.log(np.copy(samples_linear))
             samples_middle[: index_middle - padding] = samples_middle[
@@ -242,7 +250,7 @@ def generate_LUT(self) -> LUT3x1D:
             # Preparing the mask to blend the logarithmic slope with the
             # extrapolated data.
             edge_left = index_middle - padding
-            edge_right = np.searchsorted(samples / size, np.max(self._samples_camera))
+            edge_right = np.searchsorted(samples / size, max_value)
             mask_samples = smoothstep_function(
                 samples, edge_left, edge_right, clip=True
             )
diff --git a/tests/resources/download_manifest.json b/tests/resources/download_manifest.json
@@ -3,8 +3,16 @@
     "URL": "https://www.dropbox.com/scl/fi/srna8mw98wxn47np0adoj/FULL_STOPS_EXR.zip?rlkey=e2n0t59g0olu40gxk31mm01sh&dl=0",
     "SHA256": "7f4bb72394e502815e85ef4a45c4a1c027d111e6a54b058b9d4d75e5b76e1127"
   },
+  "FULL_STOPS.zip": {
+    "URL": "",
+    "SHA256": "9a8f53d0ee191b040e1270d5e170bab6ad0b6002ee5d42fc7d71b97e81ff5710"
+  },
   "PTZ_160.zip": {
     "URL": "https://www.dropbox.com/scl/fi/9ni9vcpwahrx8s3l1zqkf/PTZ_160.zip?rlkey=m3ddjysrhmkbfxx7ktgpmgfft&dl=0",
     "SHA256": "4c3c85a293166ef4896713fc99f28870b90821c1f1d0118c6ceae3b40520d4c2"
+  },
+  "DPX_10_log.zip": {
+    "URL": "",
+    "SHA256": "7eef2787b5e4571810b508bc14d5673637e0507f638df8242d49830714dab9fa"
   }
 }
diff --git a/tests/resources/example_from_folder.json b/tests/resources/example_from_folder.json
@@ -75,6 +75,8 @@
       "data/grey_card/grey_card_0001.tif",
       "data/grey_card/grey_card_0002.tif",
       "data/grey_card/grey_card_0003.tif"
-    ]
+    ],
+    "white": [],
+    "black": []
   }
 }
diff --git a/tests/resources/synthetic_001/test_project.json b/tests/resources/synthetic_001/test_project.json
@@ -75,6 +75,8 @@
       "data/grey_card/grey_card_0001.tif",
       "data/grey_card/grey_card_0002.tif",
       "data/grey_card/grey_card_0003.tif"
-    ]
+    ],
+    "white": [],
+    "black": []
   }
 }
diff --git a/tests/test_application.py b/tests/test_application.py

Original file line number	Diff line number	Diff line change
`@@ -75,6 +75,8 @@`
`75`	`75`	`"data/grey_card/grey_card_0001.tif",`
`76`	`76`	`"data/grey_card/grey_card_0002.tif",`
`77`	`77`	`"data/grey_card/grey_card_0003.tif"`
`78`		`- ]`
	`78`	`+ ],`
	`79`	`+ "white": [],`
	`80`	`+ "black": []`
`79`	`81`	`}`
`80`	`82`	`}`