Vulkan validation layer and decoder errors and assert fixes

common/libs/VkCodecUtils/VkImageResource.cpp

@@ -566,25 +566,51 @@ VkResult VkImageResourceView::Create(const VulkanDeviceContext* vkDevCtx,
     viewInfo.flags = 0;
 
     const VkMpFormatInfo* mpInfo = YcbcrVkFormatInfo(viewInfo.format);
+    const VkImageCreateInfo& imageCreateInfo = imageResource->GetImageCreateInfo();
 
     // For multi-planar formats with planeUsageOverride, skip the combined view (index 0)
     // as the combined format may not support the requested usage (e.g., STORAGE)
     bool skipCombinedView = (mpInfo != nullptr) && (planeUsageOverride != 0);
 
+    // Setup VkImageViewUsageCreateInfo - required when using EXTENDED_USAGE_BIT per Khronos issue #4624:
+    // Combined views must restrict usage to what the multi-planar format supports (exclude STORAGE)
+    // Per-plane views must restrict usage to what the plane format supports (exclude VIDEO_ENCODE_SRC)
+    VkImageViewUsageCreateInfo usageCreateInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO};
+    usageCreateInfo.pNext = nullptr;
+    usageCreateInfo.usage = planeUsageOverride;
+
     if (!skipCombinedView) {
+        // For multi-planar formats, combined views without YCbCr conversion cannot have:
+        // - STORAGE_BIT: multi-planar formats don't support storage (only per-plane views do)
+        // - SAMPLED_BIT: multi-planar formats require YCbCr conversion for sampling
+        // Must use VkImageViewUsageCreateInfo to restrict usage.
+        // Reference: https://gitlab.khronos.org/vulkan/vulkan/-/issues/4624
+        if (mpInfo) {
+            VkImageUsageFlags combinedUsage = imageCreateInfo.usage;
+            // Remove usages not supported by multi-planar base format without YCbCr conversion
+            VkImageUsageFlags unsupportedUsages = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
+            if (combinedUsage & unsupportedUsages) {
+                combinedUsage &= ~unsupportedUsages;
+                usageCreateInfo.usage = combinedUsage;
+                viewInfo.pNext = &usageCreateInfo;
+            }
+        }
+
         VkResult result = vkDevCtx->CreateImageView(device, &viewInfo, nullptr, &imageViews[numViews]);
         if (result != VK_SUCCESS) {
             return result;
         }
         numViews++;
+
+        // Reset pNext for subsequent views
+        viewInfo.pNext = nullptr;
     } else {
         // Set placeholder for combined view
         imageViews[numViews] = VK_NULL_HANDLE;
         numViews++;
     }
 
-    // Setup VkImageViewUsageCreateInfo for plane views when planeUsageOverride is set
-    VkImageViewUsageCreateInfo usageCreateInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO};
+    // Reset usage for plane views
     usageCreateInfo.usage = planeUsageOverride;
 
     if (mpInfo) { // Is this a YCbCr format

common/libs/VkCodecUtils/VulkanDeviceContext.cpp

@@ -795,7 +795,13 @@ VkResult VulkanDeviceContext::CreateVulkanDevice(int32_t numDecodeQueues,
                                                                                   VK_FALSE
                                                                                 };
 
-        VkPhysicalDeviceFeatures2 deviceFeatures { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &intraRefreshFeatures};
+        // Required for creating YCbCr samplers used with multi-planar video formats
+        VkPhysicalDeviceSamplerYcbcrConversionFeatures samplerYcbcrFeatures { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLER_YCBCR_CONVERSION_FEATURES,
+                                                                               &intraRefreshFeatures,
+                                                                               VK_FALSE
+                                                                             };
+
+        VkPhysicalDeviceFeatures2 deviceFeatures { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &samplerYcbcrFeatures};
         GetPhysicalDeviceFeatures2(m_physDevice, &deviceFeatures);
 
         assert(timelineSemaphoreFeatures.timelineSemaphore);

vk_video_decoder/libs/NvVideoParser/src/VulkanAV1Decoder.cpp

@@ -2291,6 +2291,12 @@ bool VulkanAV1Decoder::ParseObuTileGroup(const AV1ObuHeader& hdr)
             skip_bits((uint32_t)(tileSize * 8));
         }
 
+        // Bounds check to prevent buffer overflow into pStdSps and other fields
+        if (m_PicData.khr_info.tileCount >= 64) {
+            nvParserErrorLog("s", "\nError: Tile count exceeds maximum of 64 tiles\n");
+            return false;
+        }
+
         m_PicData.tileSizes[m_PicData.khr_info.tileCount] = (uint32_t)tileSize;
         m_PicData.khr_info.tileCount++;
     }

vk_video_decoder/libs/VkVideoParser/VulkanVideoParser.cpp

@@ -1274,6 +1274,15 @@ int32_t VulkanVideoParser::BeginSequence(const VkParserSequenceInfo* pnvsi)
         assert(!"m_pDecoderHandler is NULL");
     }
 
+    // When starting a new sequence (not reconfigure), reset the picture-to-DPB slot mapping
+    // to avoid freeing slots from the old, larger DPB after it has been deinitialized.
+    if (!sequenceUpdate) {
+        m_dpbSlotsMask = 0;
+        for (uint32_t picId = 0; picId < MAX_FRM_CNT; picId++) {
+            m_pictureToDpbSlotMap[picId] = -1;
+        }
+    }
+
     m_maxNumDpbSlots = m_dpb.Init(configDpbSlots, sequenceUpdate /* reconfigure the DPB size if true */);
 
     return m_maxNumDecodeSurfaces;

vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp

@@ -596,5 +596,12 @@ bool EncoderConfigH264::GetRateControlParameters(VkVideoEncodeRateControlInfoKHR
     pRateControlInfoH264->gopFrameCount = (gopStructure.GetGopFrameCount() > 0) ? gopStructure.GetGopFrameCount() : (uint32_t)GOP_LENGTH_DEFAULT;
     pRateControlInfoH264->idrPeriod = (gopStructure.GetIdrPeriod() > 0) ? gopStructure.GetIdrPeriod() : (uint32_t)IDR_PERIOD_DEFAULT;
 
+    // Validation: If idrPeriod is not 0, it must be >= gopFrameCount
+    // If gopFrameCount is very large (e.g., UINT32_MAX for infinite GOP), set idrPeriod to 0
+    if (pRateControlInfoH264->idrPeriod != 0 &&
+        pRateControlInfoH264->idrPeriod < pRateControlInfoH264->gopFrameCount) {
+        pRateControlInfoH264->idrPeriod = 0;  // Disable IDR period when GOP is larger
+    }
+
     return true;
 }

vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp

@@ -2088,8 +2088,16 @@ bool VkVideoEncoder::HandleCtrlCmd(VkSharedBaseObj<VkVideoEncodeFrameInfo>& enco
             encodeFrameInfo->rateControlLayersInfo[layerIndx].sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_RATE_CONTROL_LAYER_INFO_KHR;
         }
 
-        encodeFrameInfo->rateControlInfo.pLayers = encodeFrameInfo->rateControlLayersInfo;
-        encodeFrameInfo->rateControlInfo.layerCount = 1;
+        // layerCount must be 0 when rateControlMode is DEFAULT or DISABLED
+        VkVideoEncodeRateControlModeFlagBitsKHR rcMode = encodeFrameInfo->rateControlInfo.rateControlMode;
+        if (rcMode == VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DEFAULT_KHR ||
+            rcMode == VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR) {
+            encodeFrameInfo->rateControlInfo.pLayers = nullptr;
+            encodeFrameInfo->rateControlInfo.layerCount = 0;
+        } else {
+            encodeFrameInfo->rateControlInfo.pLayers = encodeFrameInfo->rateControlLayersInfo;
+            encodeFrameInfo->rateControlInfo.layerCount = 1;
+        }
         m_beginRateControlInfo = encodeFrameInfo->rateControlInfo;
 
         if (pNext != nullptr) {

vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.cpp

@@ -227,11 +227,12 @@ VkResult VkVideoEncoderAV1::ProcessDpb(VkSharedBaseObj<VkVideoEncodeFrameInfo>&
     assert(dpbIndx >= 0);
 
     m_dpbAV1->ConfigureRefBufUpdate(pFrameInfo->bShownKeyFrameOrSwitch, pFrameInfo->bShowExistingFrame, frameUpdateType);
-    assert(pFrameInfo->frameEncodeEncodeOrderNum <= std::numeric_limits<uint32_t>::max());
-    m_dpbAV1->InvalidateStaleReferenceFrames(static_cast<uint32_t>(pFrameInfo->frameEncodeEncodeOrderNum), pFrameInfo->picOrderCntVal, &m_stateAV1.m_sequenceHeader);
-    pFrameInfo->stdPictureInfo.refresh_frame_flags = (uint8_t)m_dpbAV1->GetRefreshFrameFlags(pFrameInfo->bShownKeyFrameOrSwitch, pFrameInfo->bShowExistingFrame);
 
+    // For show existing frame, skip stale reference invalidation and return early.
+    // Show existing frames don't have a valid frameEncodeEncodeOrderNum since they
+    // don't represent a new encoded frame - they just display a previously encoded one.
     if (pFrameInfo->bShowExistingFrame) {
+        pFrameInfo->stdPictureInfo.refresh_frame_flags = (uint8_t)m_dpbAV1->GetRefreshFrameFlags(pFrameInfo->bShownKeyFrameOrSwitch, pFrameInfo->bShowExistingFrame);
         m_dpbAV1->DpbPictureEnd(dpbIndx, encodeFrameInfo->setupImageResource/*nullptr*/, &m_stateAV1.m_sequenceHeader,
                                 pFrameInfo->bShowExistingFrame, pFrameInfo->bShownKeyFrameOrSwitch,
                                 pFrameInfo->stdPictureInfo.flags.error_resilient_mode,
@@ -240,6 +241,10 @@ VkResult VkVideoEncoderAV1::ProcessDpb(VkSharedBaseObj<VkVideoEncodeFrameInfo>&
         return VK_SUCCESS;
     }
 
+    assert(pFrameInfo->frameEncodeEncodeOrderNum <= std::numeric_limits<uint32_t>::max());
+    m_dpbAV1->InvalidateStaleReferenceFrames(static_cast<uint32_t>(pFrameInfo->frameEncodeEncodeOrderNum), pFrameInfo->picOrderCntVal, &m_stateAV1.m_sequenceHeader);
+    pFrameInfo->stdPictureInfo.refresh_frame_flags = (uint8_t)m_dpbAV1->GetRefreshFrameFlags(pFrameInfo->bShownKeyFrameOrSwitch, pFrameInfo->bShowExistingFrame);
+
     // setup recon picture (pSetupReferenceSlot)
     bool success = m_dpbImagePool->GetAvailableImage(encodeFrameInfo->setupImageResource,
                                                      VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR);