decoder: Default to valid profiles when demuxer reports unknown profile

When FFmpeg's demuxer reports an invalid or unknown profile (e.g.,
profile=0 for raw .265/.264 files without container metadata, or
mis-tagged Baseline for interlaced H.264), default to a safe profile:

  H.264: Default to HIGH (100) -- superset of Baseline/Main, handles
    interlaced, CABAC, B-slices, weighted prediction. Matches NVCUVID.
  H.265: Default to MAIN (1) -- covers most 8-bit 4:2:0 content.
  AV1: Default to MAIN (0).

The fix is in FFmpegDemuxer::GetProfileIdc() so it covers both
VulkanVideoProcessor::Initialize() and VkVideoDecoder::StartVideoSequence()
code paths. A warning is printed when a default is used.

Additionally, VkVideoDecoder::StartVideoSequence() retries with
upgraded profiles (Baseline→Main→High for H.264, Main→Main10 for
H.265) if the initial capabilities query fails, as a second line of
defense when the parser-reported profile differs from the demuxer.

Fixes: Assert on 1080i-25-H264.mkv (interlaced Baseline)
Fixes: Assert on 2024-05-03_14-55-55_1080p_p1_vbv2_5Mbps.265 (raw H.265)
Signed-off-by: Tony Zlatinski <tzlatinski@nvidia.com>

Author: zlatinski

vk_video_decoder/libs/VkDecoderUtils/FFmpegDemuxer.cpp

@@ -404,7 +404,12 @@ class FFmpegDemuxer : public VideoStreamDemuxer {
                     case STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE:
                         break;
                     default:
-                        std::cerr << "\nInvalid h.264 profile: " << profile << std::endl;
+                        // Unknown/invalid H.264 profile from FFmpeg (e.g., profile=0 for
+                        // some streams). Default to High which is a superset of all lower
+                        // profiles and handles Baseline/Main/interlaced content correctly.
+                        std::cerr << "WARNING: Unknown H.264 profile_idc=" << profile
+                                  << " from demuxer, defaulting to HIGH (100)" << std::endl;
+                        return STD_VIDEO_H264_PROFILE_IDC_HIGH;
                 }
             }
             break;
@@ -418,7 +423,11 @@ class FFmpegDemuxer : public VideoStreamDemuxer {
                     case STD_VIDEO_H265_PROFILE_IDC_SCC_EXTENSIONS:
                         break;
                     default:
-                        std::cerr << "\nInvalid h.265 profile: " << profile << std::endl;
+                        // Unknown/invalid H.265 profile from FFmpeg (e.g., profile=0 for
+                        // raw .265 files without container metadata). Default to Main.
+                        std::cerr << "WARNING: Unknown H.265 profile_idc=" << profile
+                                  << " from demuxer, defaulting to MAIN (1)" << std::endl;
+                        return STD_VIDEO_H265_PROFILE_IDC_MAIN;
                 }
             }
             break;
@@ -430,7 +439,9 @@ class FFmpegDemuxer : public VideoStreamDemuxer {
                     case STD_VIDEO_AV1_PROFILE_PROFESSIONAL:
                         break;
                     default:
-                        std::cerr << "\nInvalid AV1 profile: " << profile << std::endl;
+                        std::cerr << "WARNING: Unknown AV1 profile=" << profile
+                                  << " from demuxer, defaulting to MAIN (0)" << std::endl;
+                        return STD_VIDEO_AV1_PROFILE_MAIN;
                 }
             }
             break;

vk_video_decoder/libs/VkVideoDecoder/VkVideoDecoder.cpp

@@ -210,6 +210,35 @@ int32_t VkVideoDecoder::StartVideoSequence(VkParserDetectedVideoFormat* pVideoFo
             }
         }
     }
+    if (result != VK_SUCCESS && videoCodec == VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR) {
+        // The reported H.265 profile may be invalid (e.g., FFmpeg reports profile=0
+        // for some streams, which maps to an invalid StdVideoH265ProfileIdc).
+        // Try Main and Main10 which cover the vast majority of H.265 content.
+        static constexpr uint32_t h265ProfileUpgradePath[] = {
+            STD_VIDEO_H265_PROFILE_IDC_MAIN,
+            STD_VIDEO_H265_PROFILE_IDC_MAIN_10,
+        };
+        for (uint32_t upgradedProfile : h265ProfileUpgradePath) {
+            std::cerr << "WARNING: H.265 profile_idc=" << pVideoFormat->codecProfile
+                      << " capabilities query failed (result=" << result
+                      << "). Retrying with profile_idc=" << upgradedProfile << std::endl;
+
+            videoProfile = VkVideoCoreProfile(videoCodec,
+                                              pVideoFormat->chromaSubsampling,
+                                              pVideoFormat->lumaBitDepth,
+                                              pVideoFormat->chromaBitDepth,
+                                              upgradedProfile);
+
+            result = VulkanVideoCapabilities::GetVideoDecodeCapabilities(m_vkDevCtx, videoProfile,
+                                                                         videoCapabilities,
+                                                                         videoDecodeCapabilities);
+            if (result == VK_SUCCESS) {
+                std::cerr << "WARNING: H.265 profile upgraded to profile_idc="
+                          << upgradedProfile << " successfully." << std::endl;
+                break;
+            }
+        }
+    }
     if (result != VK_SUCCESS) {
         std::cout << "*** Could not get Video Capabilities :" << result << " ***" << std::endl;
         assert(!"Could not get Video Capabilities!");