Vulkan common filter (RGBA to YCbCr) update and adding drm format modifier and import/export support

.gitignore

@@ -33,8 +33,15 @@
 *.out
 .cproject
 .project
-build
-build32
+build*/
+_bin*/
+lib/
+misc/
+bin/
+testingAq/
 __pycache__
 vk_video_decoder/bin/libs/ffmpeg/
 .ccls-cache/
+.claude/
+CLAUDE.md
+vk_video_encoder/test/gop_structure_test

CMakeLists.txt

@@ -531,4 +531,15 @@ if(BUILD_ENCODER)
         add_subdirectory(vk_video_encoder/demos)
     endif()
 endif()
-############ VULKAN_VIDEO_ENCODER_LIB ######################################
+############ VULKAN_VIDEO_ENCODER_LIB ######################################
+
+############ VK_FILTER_TEST - Filter Unit Tests ######################################
+if(BUILD_TESTS AND NOT DEFINED DEQP_TARGET)
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/common/libs/tests")
+        message(STATUS "Building VulkanFilterYuvCompute test application")
+        add_subdirectory(common/libs/tests)
+    else()
+        message(STATUS "Filter tests directory not found, skipping...")
+    endif()
+endif()
+############ VK_FILTER_TEST ######################################

common/include/nvidia_utils/vulkan/ycbcr_utils.h

@@ -214,6 +214,36 @@ static inline YcbcrPrimariesConstants GetYcbcrPrimariesConstants(YcbcrBtStandard
     return YcbcrBtStandardBtUnsupported;
 }
 
+#ifdef VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709
+/**
+ * @brief Convert VkSamplerYcbcrModelConversion to YcbcrBtStandard
+ *
+ * This function provides a centralized mapping from Vulkan YCbCr model conversion
+ * enum values to the internal YcbcrBtStandard enum. Use this function instead of
+ * implementing local switch statements to ensure consistency across the codebase.
+ *
+ * @param modelConversion Vulkan YCbCr model conversion value
+ * @return Corresponding YcbcrBtStandard value, or YcbcrBtStandardUnknown if not supported
+ */
+static inline YcbcrBtStandard VkYcbcrModelToYcbcrBtStandard(VkSamplerYcbcrModelConversion modelConversion)
+{
+    switch (modelConversion) {
+    case VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709:
+        return YcbcrBtStandardBt709;
+    case VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_601:
+        return YcbcrBtStandardBt601Ebu;
+    case VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_2020:
+        return YcbcrBtStandardBt2020;
+    case VK_SAMPLER_YCBCR_MODEL_CONVERSION_RGB_IDENTITY:
+    case VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_IDENTITY:
+    default:
+        break;
+    }
+
+    return YcbcrBtStandardUnknown;
+}
+#endif // VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709
+
 //                                                  alpha         beta        gamma     kCoef    reGamma
 #define ITU_BT_GAMMA_COEFFICIENTS()              { 1.0993f,      0.0181f,     0.45f,     4.5f,    true  }
 #define SMPTE170M_GAMMA_COEFFICIENTS()           { 1.0993f,      0.0181f,      2.2f,     4.5f,    false }

common/libs/VkCodecUtils/Helpers.cpp

@@ -16,6 +16,10 @@
 
 #include "VkCodecUtils/Helpers.h"
 
+#if defined(VK_USE_PLATFORM_ANDROID_KHR) || defined(VK_PLATFORM_IS_UNIX)
+#include <unistd.h>  // for close()
+#endif
+
 using namespace vk;
 
 NativeHandle::NativeHandle (void) :

common/libs/VkCodecUtils/VkImageResource.h

@@ -29,6 +29,28 @@ class VkImageResource : public VkVideoRefCountBase
                            VkMemoryPropertyFlags memoryPropertyFlags,
                            VkSharedBaseObj<VkImageResource>& imageResource);
 
+    /**
+     * @brief Create image resource with external memory export support
+     *
+     * This overload creates an image that can be exported for cross-process sharing
+     * via DMA-BUF (Linux) or NT handles (Windows). If a DRM format modifier is specified,
+     * the image is created with VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT.
+     *
+     * @param vkDevCtx              Vulkan device context
+     * @param pImageCreateInfo      Image creation parameters (tiling will be overridden if drmFormatModifier != 0)
+     * @param memoryPropertyFlags   Required memory property flags
+     * @param exportHandleTypes     External memory handle types (e.g., VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT)
+     * @param drmFormatModifier     DRM format modifier (0 for no modifier, non-zero creates with DRM_FORMAT_MODIFIER tiling)
+     * @param imageResource         Output image resource
+     * @return VK_SUCCESS on success
+     */
+    static VkResult CreateExportable(const VulkanDeviceContext* vkDevCtx,
+                                     const VkImageCreateInfo* pImageCreateInfo,
+                                     VkMemoryPropertyFlags memoryPropertyFlags,
+                                     VkExternalMemoryHandleTypeFlags exportHandleTypes,
+                                     uint64_t drmFormatModifier,
+                                     VkSharedBaseObj<VkImageResource>& imageResource);
+
     bool IsCompatible ( VkDevice,
                         const VkImageCreateInfo* pImageCreateInfo)
     {
@@ -80,6 +102,7 @@ class VkImageResource : public VkVideoRefCountBase
     VkImage GetImage() const { return m_image; }
     VkDevice GetDevice() const { return *m_vkDevCtx; }
     VkDeviceMemory GetDeviceMemory() const { return *m_vulkanDeviceMemory; }
+    VkDeviceMemory GetImageDeviceMemory() const { return *m_vulkanDeviceMemory; }
 
     VkSharedBaseObj<VulkanDeviceMemoryImpl>& GetMemory() { return m_vulkanDeviceMemory; }
 
@@ -88,9 +111,101 @@ class VkImageResource : public VkVideoRefCountBase
 
     const VkImageCreateInfo& GetImageCreateInfo() const { return m_imageCreateInfo; }
 
+    /**
+     * @brief Get the image tiling mode
+     */
+    VkImageTiling GetImageTiling() const { return m_imageCreateInfo.tiling; }
+
+    /**
+     * @brief Check if this is a linear tiled image
+     */
+    bool IsLinearImage() const { return m_isLinearImage; }
+
     const VkSubresourceLayout* GetSubresourceLayout() const {
         return m_isLinearImage ? m_layouts : nullptr;
     }
+
+    /**
+     * @brief Get plane layout for LINEAR images
+     * 
+     * For LINEAR images, returns the layout queried via vkGetImageSubresourceLayout.
+     * For DRM modifier images, use GetMemoryPlaneLayout() instead.
+     * 
+     * @param planeIndex The format plane index (0-2)
+     * @param layout Output layout structure
+     * @return true if layout is available
+     */
+    bool GetPlaneLayout(uint32_t planeIndex, VkSubresourceLayout& layout) const {
+        if (!m_isLinearImage || planeIndex > 2) return false;
+        layout = m_layouts[planeIndex];
+        return (layout.size > 0 || layout.rowPitch > 0);
+    }
+
+    //-------------------------------------------------------------------------
+    // External Memory / DRM Format Modifier Support
+    //-------------------------------------------------------------------------
+
+    /**
+     * @brief Check if this image was created with external memory export support
+     */
+    bool IsExportable() const;
+
+    /**
+     * @brief Get the DRM format modifier for this image (0 if not using DRM modifier tiling)
+     *
+     * Returns the actual DRM modifier used by the driver, which may differ from what was
+     * requested if the driver chose a different compatible modifier from the list.
+     */
+    uint64_t GetDrmFormatModifier() const { return m_drmFormatModifier; }
+
+    /**
+     * @brief Check if this image uses DRM format modifier tiling
+     */
+    bool UsesDrmFormatModifier() const { return m_usesDrmFormatModifier; }
+
+    /**
+     * @brief Get the number of memory planes for this image
+     *
+     * For images with DRM format modifiers, this is the number of memory planes
+     * reported by the driver, which may differ from the number of color planes.
+     */
+    uint32_t GetMemoryPlaneCount() const { return m_memoryPlaneCount; }
+
+    /**
+     * @brief Get plane layout information using MEMORY_PLANE aspect bits
+     *
+     * For DRM format modifier images, use MEMORY_PLANE aspects (not PLANE aspects).
+     * All planes are typically packed in a single memory allocation.
+     *
+     * @param planeIndex Memory plane index (0-based)
+     * @param layout     Output subresource layout
+     * @return true if layout was retrieved successfully
+     */
+    bool GetMemoryPlaneLayout(uint32_t planeIndex, VkSubresourceLayout& layout) const;
+
+    /**
+     * @brief Export native handle (DMA-BUF FD on Linux, NT handle on Windows)
+     *
+     * The image must have been created with CreateExportable() and appropriate handle types.
+     * The caller is responsible for closing the returned handle when done.
+     *
+     * @param handleType The handle type to export (e.g., VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT)
+     * @param outHandle  Output native handle
+     * @return VK_SUCCESS on success, VK_ERROR_FEATURE_NOT_PRESENT if not exportable
+     */
+#ifdef _WIN32
+    VkResult ExportNativeHandle(VkExternalMemoryHandleTypeFlagBits handleType, void** outHandle) const;
+#else
+    VkResult ExportNativeHandle(VkExternalMemoryHandleTypeFlagBits handleType, int* outHandle) const;
+#endif
+
+    /**
+     * @brief Get the memory type index used for this image's allocation
+     *
+     * This is needed for cross-process sharing with opaque FD fallback,
+     * where the importer must allocate memory with the same parameters.
+     */
+    uint32_t GetMemoryTypeIndex() const;
 
 private:
     std::atomic<int32_t>    m_refCount;
@@ -101,15 +216,21 @@ class VkImageResource : public VkVideoRefCountBase
     VkDeviceSize            m_imageSize;
     VkSharedBaseObj<VulkanDeviceMemoryImpl> m_vulkanDeviceMemory;
     VkSubresourceLayout     m_layouts[3]; // per plane layout for linear images
+    VkSubresourceLayout     m_memoryPlaneLayouts[4]; // per memory plane layout for DRM modifier images
+    uint64_t                m_drmFormatModifier;     // DRM format modifier (0 if not used)
+    uint32_t                m_memoryPlaneCount;      // Number of memory planes
     uint32_t                m_isLinearImage : 1;
     uint32_t                m_is16Bit : 1;
     uint32_t                m_isSubsampledX : 1;
     uint32_t                m_isSubsampledY : 1;
+    uint32_t                m_usesDrmFormatModifier : 1;
 
     VkImageResource(const VulkanDeviceContext* vkDevCtx,
                     const VkImageCreateInfo* pImageCreateInfo,
                     VkImage image, VkDeviceSize imageOffset, VkDeviceSize imageSize,
-                    VkSharedBaseObj<VulkanDeviceMemoryImpl>& vulkanDeviceMemory);
+                    VkSharedBaseObj<VulkanDeviceMemoryImpl>& vulkanDeviceMemory,
+                    uint64_t drmFormatModifier = 0,
+                    uint32_t memoryPlaneCount = 0);
 
     void Destroy();
 
@@ -124,6 +245,44 @@ class VkImageResourceView : public VkVideoRefCountBase
                            VkImageSubresourceRange &imageSubresourceRange,
                            VkSharedBaseObj<VkImageResourceView>& imageResourceView);
 
+    // Overload with optional plane usage override for storage-compatible plane views
+    // When planeUsageOverride is non-zero, plane views will be created with VkImageViewUsageCreateInfo
+    // specifying that usage. This is needed when the base format doesn't support storage but
+    // individual planes (R8, RG8) do via VK_IMAGE_CREATE_EXTENDED_USAGE_BIT.
+    static VkResult Create(const VulkanDeviceContext* vkDevCtx,
+                           VkSharedBaseObj<VkImageResource>& imageResource,
+                           VkImageSubresourceRange &imageSubresourceRange,
+                           VkImageUsageFlags planeUsageOverride,
+                           VkSharedBaseObj<VkImageResourceView>& imageResourceView);
+
+    /**
+     * @brief Create image view with YCbCr sampler conversion support
+     *
+     * Creates both:
+     * 1. A combined YCbCr view with VkSamplerYcbcrConversionInfo attached (for display sampling)
+     * 2. Per-plane views with optional usage override (for compute storage)
+     *
+     * This is needed when an image needs to be both:
+     * - Written by a compute shader (storage views per-plane)
+     * - Sampled with YCbCr conversion for display (combined sampled view)
+     *
+     * @param vkDevCtx              Vulkan device context
+     * @param imageResource         Image resource to create view for
+     * @param imageSubresourceRange Subresource range for the view
+     * @param planeUsageOverride    Usage flags for per-plane views (e.g., VK_IMAGE_USAGE_STORAGE_BIT)
+     * @param ycbcrConversion       YCbCr sampler conversion object (or VK_NULL_HANDLE if none)
+     * @param combinedViewUsage     Usage flags for combined view (e.g., VK_IMAGE_USAGE_SAMPLED_BIT)
+     * @param imageResourceView     Output image resource view
+     * @return VK_SUCCESS on success
+     */
+    static VkResult Create(const VulkanDeviceContext* vkDevCtx,
+                           VkSharedBaseObj<VkImageResource>& imageResource,
+                           VkImageSubresourceRange &imageSubresourceRange,
+                           VkImageUsageFlags planeUsageOverride,
+                           VkSamplerYcbcrConversion ycbcrConversion,
+                           VkImageUsageFlags combinedViewUsage,
+                           VkSharedBaseObj<VkImageResourceView>& imageResourceView);
+
 
     virtual int32_t AddRef()
     {
@@ -140,8 +299,14 @@ class VkImageResourceView : public VkVideoRefCountBase
         return ret;
     }
 
-    operator VkImageView() const { return m_imageViews[0]; }
-    VkImageView GetImageView() const { return m_imageViews[0]; }
+    operator VkImageView() const {
+        // Fall back to first plane view if combined view is null (storage-only case)
+        return m_imageViews[0] ? m_imageViews[0] : (m_numPlanes > 0 ? m_imageViews[1] : VK_NULL_HANDLE);
+    }
+    VkImageView GetImageView() const {
+        // Fall back to first plane view if combined view is null (storage-only case)
+        return m_imageViews[0] ? m_imageViews[0] : (m_numPlanes > 0 ? m_imageViews[1] : VK_NULL_HANDLE);
+    }
     uint32_t GetNumberOfPlanes() const { return m_numPlanes; }
     VkImageView GetPlaneImageView(uint32_t planeIndex = 0) const {
         if (m_numPlanes == 1) {

common/libs/VkCodecUtils/VulkanComputePipeline.h

@@ -55,23 +55,23 @@ class VulkanComputePipeline
     // Destroy Graphics Pipeline
     void DestroyPipeline()
     {
-        if (m_pipeline) {
+        if (m_pipeline && m_vkDevCtx) {
             m_vkDevCtx->DestroyPipeline(*m_vkDevCtx, m_pipeline, nullptr);
             m_pipeline = VkPipeline(0);
         }
     }
 
     void DestroyPipelineCache()
     {
-        if (m_pipelineCache) {
+        if (m_pipelineCache && m_vkDevCtx) {
             m_vkDevCtx->DestroyPipelineCache(*m_vkDevCtx, m_pipelineCache, nullptr);
             m_pipelineCache = VkPipelineCache(0);
         }
     }
 
     void DestroyShaderModule()
     {
-        if (m_shaderModule) {
+        if (m_shaderModule && m_vkDevCtx) {
             m_vkDevCtx->DestroyShaderModule(*m_vkDevCtx, m_shaderModule, nullptr);
             m_shaderModule = VkShaderModule();
         }