decoder: Fix AV1 encoder crash with B-frames (show existing frame handling)

When encoding AV1 with B-frames (consecutiveBFrameCount > 0), the
encoder creates "show existing frame" P-frames that display previously
encoded frames. These frames have bShowExistingFrame=true.

The issue was in ProcessDpb():
- frameEncodeEncodeOrderNum is only assigned in StartOfVideoCodingEncodeOrder
  when bShowExistingFrame=false
- ProcessDpb called InvalidateStaleReferenceFrames() with the uninitialized
  frameEncodeEncodeOrderNum (UINT64_MAX) before checking bShowExistingFrame
- This triggered the assertion: frameEncodeEncodeOrderNum <= UINT32_MAX

Fix: Move the bShowExistingFrame early return check before the assertion
and InvalidateStaleReferenceFrames() call. Show existing frames don't need
stale reference invalidation since they don't encode a new frame.

Command line that triggered this crash:
  vk-video-enc-test -i input.yuv -c av1 --inputWidth 352 --inputHeight 288 \
    --inputChromaSubsampling 420 --numFrames 30 -o output.ivf \
    --gopFrameCount 16 --consecutiveBFrameCount 3

This fixes the AV1_gop_16_b3 encoder test.

Signed-off-by: Tony Zlatinski <tzlatinski@nvidia.com>

Author: zlatinski

vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.cpp

@@ -227,11 +227,12 @@ VkResult VkVideoEncoderAV1::ProcessDpb(VkSharedBaseObj<VkVideoEncodeFrameInfo>&
     assert(dpbIndx >= 0);
 
     m_dpbAV1->ConfigureRefBufUpdate(pFrameInfo->bShownKeyFrameOrSwitch, pFrameInfo->bShowExistingFrame, frameUpdateType);
-    assert(pFrameInfo->frameEncodeEncodeOrderNum <= std::numeric_limits<uint32_t>::max());
-    m_dpbAV1->InvalidateStaleReferenceFrames(static_cast<uint32_t>(pFrameInfo->frameEncodeEncodeOrderNum), pFrameInfo->picOrderCntVal, &m_stateAV1.m_sequenceHeader);
-    pFrameInfo->stdPictureInfo.refresh_frame_flags = (uint8_t)m_dpbAV1->GetRefreshFrameFlags(pFrameInfo->bShownKeyFrameOrSwitch, pFrameInfo->bShowExistingFrame);
 
+    // For show existing frame, skip stale reference invalidation and return early.
+    // Show existing frames don't have a valid frameEncodeEncodeOrderNum since they
+    // don't represent a new encoded frame - they just display a previously encoded one.
     if (pFrameInfo->bShowExistingFrame) {
+        pFrameInfo->stdPictureInfo.refresh_frame_flags = (uint8_t)m_dpbAV1->GetRefreshFrameFlags(pFrameInfo->bShownKeyFrameOrSwitch, pFrameInfo->bShowExistingFrame);
         m_dpbAV1->DpbPictureEnd(dpbIndx, encodeFrameInfo->setupImageResource/*nullptr*/, &m_stateAV1.m_sequenceHeader,
                                 pFrameInfo->bShowExistingFrame, pFrameInfo->bShownKeyFrameOrSwitch,
                                 pFrameInfo->stdPictureInfo.flags.error_resilient_mode,
@@ -240,6 +241,10 @@ VkResult VkVideoEncoderAV1::ProcessDpb(VkSharedBaseObj<VkVideoEncodeFrameInfo>&
         return VK_SUCCESS;
     }
 
+    assert(pFrameInfo->frameEncodeEncodeOrderNum <= std::numeric_limits<uint32_t>::max());
+    m_dpbAV1->InvalidateStaleReferenceFrames(static_cast<uint32_t>(pFrameInfo->frameEncodeEncodeOrderNum), pFrameInfo->picOrderCntVal, &m_stateAV1.m_sequenceHeader);
+    pFrameInfo->stdPictureInfo.refresh_frame_flags = (uint8_t)m_dpbAV1->GetRefreshFrameFlags(pFrameInfo->bShownKeyFrameOrSwitch, pFrameInfo->bShowExistingFrame);
+
     // setup recon picture (pSetupReferenceSlot)
     bool success = m_dpbImagePool->GetAvailableImage(encodeFrameInfo->setupImageResource,
                                                      VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR);