WIP: PERF: OOC-optimized algorithm variants for 30+ filters

CMakeLists.txt

@@ -53,6 +53,36 @@ option(SIMPLNX_DOWNLOAD_TEST_FILES "Download the test files" ON)
 # ------------------------------------------------------------------------------
 option(SIMPLNX_WRITE_TEST_OUTPUT "Write unit test output files" OFF)
 
+# ------------------------------------------------------------------------------
+# Controls which algorithm paths are exercised by dual-dispatch unit tests.
+#   0 (Both)       - tests run with forceOoc=false AND forceOoc=true (default)
+#   1 (OocOnly)    - tests run with forceOoc=true only (use for OOC builds)
+#   2 (InCoreOnly) - tests run with forceOoc=false only (quick validation)
+# ------------------------------------------------------------------------------
+set(SIMPLNX_TEST_ALGORITHM_PATH "0" CACHE STRING "Algorithm paths to test: 0=Both, 1=OocOnly, 2=InCoreOnly")
+
+# ------------------------------------------------------------------------------
+# Out-of-core support compile-time switch
+# ------------------------------------------------------------------------------
+option(SIMPLNX_USE_OOC "Compile out-of-core support into simplnx (requires SIMPLNX_OOC_SOURCE_DIR)" OFF)
+set(SIMPLNX_OOC_SOURCE_DIR "" CACHE PATH "Path to the private SimplnxOoc source directory (required when SIMPLNX_USE_OOC=ON)")
+
+if(SIMPLNX_USE_OOC)
+  if(NOT SIMPLNX_OOC_SOURCE_DIR OR NOT EXISTS "${SIMPLNX_OOC_SOURCE_DIR}")
+    message(FATAL_ERROR
+      "SIMPLNX_USE_OOC=ON requires SIMPLNX_OOC_SOURCE_DIR to point at the private SimplnxOoc repo. "
+      "Set -DSIMPLNX_OOC_SOURCE_DIR=/path/to/SimplnxOoc/SimplnxOoc")
+  endif()
+else()
+  # With OOC compiled out there are no out-of-core algorithm paths to exercise,
+  # so pin the test path to InCoreOnly. Forcing it avoids a stale cached 0/1
+  # (Both/OocOnly) producing tests that can never run.
+  if(NOT SIMPLNX_TEST_ALGORITHM_PATH STREQUAL "2")
+    message(STATUS "SIMPLNX_USE_OOC=OFF: forcing SIMPLNX_TEST_ALGORITHM_PATH=2 (InCoreOnly).")
+    set(SIMPLNX_TEST_ALGORITHM_PATH "2" CACHE STRING "Algorithm paths to test: 0=Both, 1=OocOnly, 2=InCoreOnly" FORCE)
+  endif()
+endif()
+
 # ------------------------------------------------------------------------------
 # Is the SimplnxCore Plugin enabled [DEFAULT=ON]
 # ------------------------------------------------------------------------------
@@ -261,6 +291,7 @@ if(SIMPLNX_ENABLE_MULTICORE)
   target_link_libraries(simplnx PUBLIC TBB::tbb)
 endif()
 
+
 target_link_libraries(simplnx
   PUBLIC
     fmt::fmt
@@ -293,6 +324,11 @@ if(SIMPLNX_ENABLE_LINK_FILESYSTEM)
 endif()
 
 set(SIMPLNX_GENERATED_DIR ${simplnx_BINARY_DIR}/generated)
+
+configure_file(
+  ${simplnx_SOURCE_DIR}/cmake/SimplnxConfig.hpp.in
+  ${SIMPLNX_GENERATED_DIR}/simplnx/Common/SimplnxConfig.hpp
+  @ONLY)
 set(SIMPLNX_GENERATED_HEADER_DIR ${simplnx_BINARY_DIR}/generated/simplnx)
 set(SIMPLNX_EXPORT_HEADER ${SIMPLNX_GENERATED_HEADER_DIR}/simplnx_export.hpp)
 
@@ -351,6 +387,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Common/Constants.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/DataTypeUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/DataVector.hpp
+  ${SIMPLNX_SOURCE_DIR}/Common/Extent.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/EulerAngle.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/Filesystem.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/IteratorUtility.hpp
@@ -465,6 +502,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/DataStructure/DynamicListArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyDataStore.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyListStore.hpp
+  ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyStringStore.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IDataArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IDataStore.hpp
@@ -546,6 +584,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataGroupUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataObjectUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataStoreUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/AlgorithmDispatch.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/FilePathGenerator.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/ColorTableUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/FileUtilities.hpp
@@ -556,6 +595,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Utilities/HistogramUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MaskCompareUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryBudgetManager.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MessageHelper.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/StringUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/StringInterpretationUtilities.hpp
@@ -568,6 +608,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Utilities/SamplingUtils.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/SegmentFeatures.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/TimeUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/UnionFind.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/TooltipGenerator.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/TooltipRowItem.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/OStreamUtilities.hpp
@@ -770,6 +811,7 @@ set(SIMPLNX_SRCS
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataStoreUtilities.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MaskCompareUtilities.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.cpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryBudgetManager.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MessageHelper.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/IParallelAlgorithm.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/ParallelDataAlgorithm.cpp
@@ -893,6 +935,49 @@ target_include_directories(simplnx
     $<INSTALL_INTERFACE:include>
 )
 
+# ------------------------------------------------------------------------------
+# Out-of-core sources (SIMPLNX_USE_OOC=ON)
+#
+# Copy the private SimplnxOoc sources into simplnx's build tree and compile them
+# directly into libsimplnx. This keeps the public simplnx source repo free of
+# OOC code while letting simplnx core call SimplnxOoc:: functions directly with
+# no separate library and no simplnx->SimplnxOoc->simplnx link cycle. The copy
+# target lives under SIMPLNX_GENERATED_DIR, which is already on simplnx's PUBLIC
+# include path, so `#include "SimplnxOoc/X.hpp"` resolves for every consumer.
+# ------------------------------------------------------------------------------
+if(SIMPLNX_USE_OOC)
+  file(GLOB SIMPLNX_OOC_HDRS CONFIGURE_DEPENDS "${SIMPLNX_OOC_SOURCE_DIR}/*.hpp")
+  file(GLOB SIMPLNX_OOC_SRCS CONFIGURE_DEPENDS "${SIMPLNX_OOC_SOURCE_DIR}/*.cpp")
+
+  # The partitioned store (unstructured/poly OOC) is deferred — exclude it from
+  # the compiled set so the deferred, #if 0'd code never reaches the compiler.
+  list(FILTER SIMPLNX_OOC_HDRS EXCLUDE REGEX "HDF5PartitionedStore")
+  list(FILTER SIMPLNX_OOC_SRCS EXCLUDE REGEX "HDF5PartitionedStore")
+
+  set(SIMPLNX_OOC_COPY_DIR "${SIMPLNX_GENERATED_DIR}/SimplnxOoc")
+  set(SIMPLNX_OOC_COPIED_SRCS "")
+  foreach(_oocSrc ${SIMPLNX_OOC_HDRS} ${SIMPLNX_OOC_SRCS})
+    get_filename_component(_oocName "${_oocSrc}" NAME)
+    configure_file("${_oocSrc}" "${SIMPLNX_OOC_COPY_DIR}/${_oocName}" COPYONLY)
+    if(_oocSrc MATCHES "\\.cpp$")
+      list(APPEND SIMPLNX_OOC_COPIED_SRCS "${SIMPLNX_OOC_COPY_DIR}/${_oocName}")
+    endif()
+  endforeach()
+
+  # Generated shim: the OOC code compiles into libsimplnx, so its export macro
+  # maps onto simplnx's own export macro.
+  file(WRITE "${SIMPLNX_OOC_COPY_DIR}/SimplnxOoc_export.hpp"
+    "#pragma once\n#include \"simplnx/simplnx_export.hpp\"\n#define SIMPLNXOOC_EXPORT SIMPLNX_EXPORT\n")
+
+  target_sources(simplnx PRIVATE ${SIMPLNX_OOC_COPIED_SRCS})
+
+  # The chunked-store .cpp files explicitly instantiate many template classes,
+  # exceeding the default COFF section limit on MSVC.
+  if(MSVC)
+    set_source_files_properties(${SIMPLNX_OOC_COPIED_SRCS} PROPERTIES COMPILE_OPTIONS "/bigobj")
+  endif()
+endif()
+
 cmake_dependent_option(SIMPLNX_DOWNLOAD_TEST_FILES_FIRST "Forces test files to download before simplnx builds" OFF "SIMPLNX_DOWNLOAD_TEST_FILES" OFF)
 if(SIMPLNX_DOWNLOAD_TEST_FILES_FIRST)
   add_dependencies(simplnx Fetch_Remote_Data_Files)
@@ -1063,6 +1148,13 @@ if(SIMPLNX_BUILD_TESTS)
   find_package(ZLIB REQUIRED)
   include(CTest)
   add_subdirectory(test)
+
+  # Build the private SimplnxOoc test suite against libsimplnx (which carries
+  # the OOC symbols when SIMPLNX_USE_OOC=ON). This single rule serves both ON
+  # contexts: standalone simplnx and the simplnx subdirectory inside DREAM3D-NX.
+  if(SIMPLNX_USE_OOC)
+    add_subdirectory("${SIMPLNX_OOC_SOURCE_DIR}/test" ${simplnx_BINARY_DIR}/SimplnxOoc_test)
+  endif()
 endif()
 
 if(SIMPLNX_BUILD_PYTHON)

cmake/Plugin.cmake

@@ -383,6 +383,7 @@ function(create_simplnx_plugin_unit_test)
   target_compile_definitions(${UNIT_TEST_TARGET}
     PRIVATE
       SIMPLNX_BUILD_DIR="$<TARGET_FILE_DIR:simplnx_test>"
+      SIMPLNX_TEST_ALGORITHM_PATH=${SIMPLNX_TEST_ALGORITHM_PATH}
   )
 
   target_compile_options(${UNIT_TEST_TARGET}

cmake/SimplnxConfig.hpp.in

@@ -0,0 +1,11 @@
+#pragma once
+
+// This header is generated from cmake/SimplnxConfig.hpp.in by configure_file().
+// It carries compile-time configuration into every translation unit that links
+// simplnx, including MOC, via simplnx's PUBLIC generated include directory.
+// Delivering the macro through a header (not per-target compile definitions)
+// keeps the value identical across all consumers, which is required for ODR
+// safety when the macro gates inline code in public headers.
+
+// Defined (to 1) when out-of-core support is compiled into libsimplnx.
+#cmakedefine SIMPLNX_USE_OOC

src/Plugins/ITKImageProcessing/src/ITKImageProcessing/Common/ITKArrayHelper.hpp

@@ -855,7 +855,7 @@ Result<OutputActions> DataCheck(const DataStructure& dataStructure, const DataPa
   const auto& inputArray = dataStructure.getDataRefAs<IDataArray>(inputArrayPath);
   const auto& inputDataStore = inputArray.getIDataStoreRef();
 
-  if(!inputArray.getDataFormat().empty())
+  if(inputArray.getStoreType() == IDataStore::StoreType::OutOfCore)
   {
     return MakeErrorResult<OutputActions>(Constants::k_OutOfCoreDataNotSupported,
                                           fmt::format("Input Array '{}' utilizes out-of-core data. This is not supported within ITK filters.", inputArrayPath.toString()));
@@ -877,7 +877,7 @@ Result<detail::ITKFilterFunctorResult_t<FilterCreationFunctorT>> Execute(DataStr
 
   using ResultT = detail::ITKFilterFunctorResult_t<FilterCreationFunctorT>;
 
-  if(!inputArray.getDataFormat().empty())
+  if(inputArray.getStoreType() == IDataStore::StoreType::OutOfCore)
   {
     return MakeErrorResult(Constants::k_OutOfCoreDataNotSupported, fmt::format("Input Array '{}' utilizes out-of-core data. This is not supported within ITK filters.", inputArrayPath.toString()));
   }

src/Plugins/ITKImageProcessing/test/ITKTestBase.cpp

@@ -30,7 +30,7 @@ std::string ComputeMD5HashTyped(const IDataArray& outputDataArray)
   usize arraySize = dataArray.getSize();
 
   MD5 md5;
-  if(outputDataArray.getDataFormat().empty())
+  if(outputDataArray.getIDataStoreRef().getStoreType() != IDataStore::StoreType::OutOfCore)
   {
     const T* dataPtr = dataArray.template getIDataStoreRefAs<DataStore<T>>().data();
     md5.update(reinterpret_cast<const uint8*>(dataPtr), arraySize * sizeof(T));
@@ -135,47 +135,7 @@ namespace ITKTestBase
 bool IsArrayInMemory(DataStructure& dataStructure, const DataPath& outputDataPath)
 {
   const auto& outputDataArray = dataStructure.getDataRefAs<IDataArray>(outputDataPath);
-  DataType outputDataType = outputDataArray.getDataType();
-
-  switch(outputDataType)
-  {
-  case DataType::float32: {
-    return dynamic_cast<const DataArray<float32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::float64: {
-    return dynamic_cast<const DataArray<float64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int8: {
-    return dynamic_cast<const DataArray<int8>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint8: {
-    return dynamic_cast<const DataArray<uint8>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int16: {
-    return dynamic_cast<const DataArray<int16>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint16: {
-    return dynamic_cast<const DataArray<uint16>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int32: {
-    return dynamic_cast<const DataArray<int32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint32: {
-    return dynamic_cast<const DataArray<uint32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int64: {
-    return dynamic_cast<const DataArray<int64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint64: {
-    return dynamic_cast<const DataArray<uint64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::boolean: {
-    [[fallthrough]];
-  }
-  default: {
-    return {};
-  }
-  }
+  return outputDataArray.getIDataStoreRef().getStoreType() != IDataStore::StoreType::OutOfCore;
 }
 //------------------------------------------------------------------------------
 std::string ComputeMd5Hash(DataStructure& dataStructure, const DataPath& outputDataPath)

src/Plugins/OrientationAnalysis/CMakeLists.txt

@@ -172,6 +172,8 @@ set(filter_algorithms
   AlignSectionsMisorientation
   AlignSectionsMutualInformation
   BadDataNeighborOrientationCheck
+  BadDataNeighborOrientationCheckScanline
+  BadDataNeighborOrientationCheckWorklist
   CAxisSegmentFeatures
   ComputeAvgCAxes
   ComputeAvgOrientations
@@ -186,9 +188,12 @@ set(filter_algorithms
   ComputeFZQuaternions
   ComputeGBCD
   ComputeGBCDMetricBased
-  ComputeGBCDPoleFigure
+  ComputeGBCDPoleFigureDirect
+  ComputeGBCDPoleFigureScanline
   ComputeGBPDMetricBased
   ComputeIPFColors
+  ComputeIPFColorsDirect
+  ComputeIPFColorsScanline
   ComputeKernelAvgMisorientations
   ComputeMisorientations
   ComputeQuaternionConjugate

src/Plugins/OrientationAnalysis/docs/AlignSectionsMisorientationFilter.md

@@ -45,6 +45,20 @@ In this new structure, what follows is what the created structures represent:
 In previous versions a file would have been produced instead. If you wish to recreate this, you can write the Attribute Matrix as a CSV/Text file.
 
 
+## Algorithm
+
+### In-Core Path
+
+For each pair of adjacent Z-sections, the algorithm computes the misorientation between voxels across the section boundary. It tests candidate X-Y shifts to find the shift that minimizes the total misorientation between the two sections. All voxel comparisons use direct array indexing with `operator[]`.
+
+### Out-of-Core Path
+
+Reads pairs of adjacent Z-slices into local memory buffers using `copyIntoBuffer()`. All misorientation comparisons for a given slice pair operate entirely on the in-memory buffers. This converts what would otherwise be random cross-slice element access into sequential bulk reads, avoiding chunk thrashing when data is stored on disk in compressed chunks.
+
+### Performance
+
+The OOC optimization matters most for large datasets that exceed available RAM. By reading entire Z-slices in bulk rather than accessing individual voxels across slice boundaries, the algorithm avoids repeatedly decompressing the same disk chunks. For in-memory datasets, the two paths produce identical results with negligible overhead difference.
+
 % Auto generated parameter table will be inserted here
 
 ## Example Pipelines

src/Plugins/OrientationAnalysis/docs/AlignSectionsMutualInformationFilter.md

@@ -49,6 +49,20 @@ In this new structure, what follows is what the created structures represent:
 
 In previous versions a file would have been produced instead. If you wish to recreate this, you can write the Attribute Matrix as a CSV/Text file.
 
+## Algorithm
+
+### In-Core Path
+
+Aligns Z-sections by maximizing the mutual information of orientation data between adjacent slices. The algorithm segments each slice into temporary features, bins the orientations, and computes joint and marginal histograms to evaluate mutual information at each candidate shift position. All orientation and feature ID data is accessed through direct array indexing with `operator[]`.
+
+### Out-of-Core Path
+
+Reads the orientation and phase data for each pair of adjacent Z-slices into local memory buffers using `copyIntoBuffer()` before computing histograms. This eliminates per-voxel out-of-core reads during the histogram binning and mutual information calculation, replacing them with two sequential bulk reads per slice pair.
+
+### Performance
+
+The OOC optimization matters most for large datasets that exceed available RAM. Histogram computation requires visiting every voxel in both slices multiple times (once per candidate shift), so eliminating per-element OOC access prevents repeated decompression of the same disk chunks. For in-memory datasets, the two paths produce identical results with negligible overhead difference.
+
 % Auto generated parameter table will be inserted here
 
 ## References

src/Plugins/OrientationAnalysis/docs/BadDataNeighborOrientationCheckFilter.md

@@ -41,6 +41,32 @@ since there are no neighbors is the +-Z directions.
 
 Only the *Mask* value defining the cell as *good* or *bad* is changed. No other cell level array is modified.
 
+## Algorithm
+
+The algorithm operates in a multi-level iterative scheme, starting at level 6 (all 6 face-neighbors must agree) and decrementing to the user-specified *Required Number of Neighbors*. At each level, bad voxels are flipped to good if they have at least that many good face-neighbors with matching crystallographic orientation (misorientation below the tolerance). Starting strict and relaxing ensures high-confidence flips happen first, which can cascade to enable additional flips.
+
+### In-Core Path (BadDataNeighborOrientationCheckWorklist)
+
+When all arrays reside in contiguous in-memory storage, the algorithm uses a two-phase worklist approach:
+
+1. **Phase 1 (Initial count)**: A single linear scan counts matching good face-neighbors for every bad voxel, storing the count in a per-voxel array.
+2. **Phase 2 (Worklist propagation)**: For each level, a deque is seeded with all bad voxels meeting the threshold. As each voxel is flipped, its still-bad neighbors' counts are incremented. If a neighbor's count now meets the threshold, it is enqueued. This breadth-first flood-fill processes each voxel at most once per level, achieving O(flipped) amortized cost.
+
+### Out-of-Core Path (BadDataNeighborOrientationCheckScanline)
+
+When any of the quaternion, mask, or phase arrays are backed by chunked (OOC) disk storage, the algorithm uses a 3-slice rolling window over the Z axis:
+
+1. Three Z-slices of quaternions, phases, and mask data are maintained in memory (previous, current, next).
+2. For each bad voxel in the current slice, the count of matching good face-neighbors is recomputed on-the-fly using the rolling window buffers.
+3. If a voxel is flipped, the mask change is written back to the OOC store per-slice via `copyFromBuffer()`.
+4. The window shifts forward one Z-slice at a time, with only one new slice loaded per step.
+
+This approach trades recomputation (no persistent neighbor-count array) for strictly sequential I/O that avoids the random-access chunk thrashing that would occur with the worklist variant's BFS pattern.
+
+### Performance
+
+The in-core worklist variant is significantly faster for datasets that fit in RAM because each voxel is processed at most once per level (O(flipped) cost vs. O(N * passes) for the scanline variant). The OOC scanline variant is slower in absolute terms but avoids catastrophic performance degradation on disk-backed datasets where the worklist's random access pattern would trigger continuous chunk load/evict cycles. Memory usage is O(N) for the worklist variant vs. O(3 * sliceSize) for the scanline variant.
+
 ## Example Data
 
 | Example Input Image                                                       | Example Output Image                                                                                                                          |