Skip to content

encode: codec profile fixes #203

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
srinathkr-nv wants to merge 6 commits into
base: main
Choose a base branch
from
Open

encode: codec profile fixes #203

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
fe11559
EncoderConfigH264: fix InitProfileLevel to depend only on user config
srinathkr-nv Feb 5, 2026
6b305a7
EncoderConfigH264: Call InitProfileLevel in InitializeParameters
srinathkr-nv Feb 5, 2026
a84c0da
EncoderConfigH265: Add InitProfileLevel for common profile, level, ti…
srinathkr-nv Feb 6, 2026
31b7a63
EncoderConfigH265: rename GetLevelTier to DetermineLevelTier
srinathkr-nv Feb 6, 2026
9dbc7a3
EncoderConfigAV1: Add InitProfileLevel for common profile, level, tie…
srinathkr-nv Feb 6, 2026
1080d06
encode: remove videoProfileIdc and GetDefaultVideoProfileIdc()
srinathkr-nv Feb 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 4 additions & 7 deletions vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -994,11 +994,8 @@ void EncoderConfig::InitVideoProfile()
encodeBitDepthChroma = encodeBitDepthLuma;
}

// If videoProfileIdc is not explicitly set (default -1), select appropriate profile
// based on bit depth and chroma format to ensure correct profile for 10-bit/12-bit content
if (videoProfileIdc == (uint32_t)-1) {
videoProfileIdc = GetDefaultVideoProfileIdc();
}
// Get the codec-specific profile (already set by InitProfileLevel)
uint32_t codecProfile = GetCodecProfile();

VkVideoEncodeUsageInfoKHR encodeUsageInfo = {};
encodeUsageInfo.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_USAGE_INFO_KHR;
Expand All @@ -1007,11 +1004,11 @@ void EncoderConfig::InitVideoProfile()
encodeUsageInfo.videoContentHints = encodeContentHints;
encodeUsageInfo.tuningMode = tuningMode;

// update the video profile
// Create video profile with the codec-specific profile
videoCoreProfile = VkVideoCoreProfile(codec, encodeChromaSubsampling,
GetComponentBitDepthFlagBits(encodeBitDepthLuma),
GetComponentBitDepthFlagBits(encodeBitDepthChroma),
videoProfileIdc,
codecProfile,
VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_PROGRESSIVE_KHR, // interlaced video is not supported with encode
encodeUsageInfo);
}
Expand Down
5 changes: 2 additions & 3 deletions vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -757,7 +757,6 @@ struct EncoderConfig : public VkVideoRefCountBase {
int32_t queueId;
VkVideoCodecOperationFlagBitsKHR codec;
bool useDpbArray;
uint32_t videoProfileIdc;
uint32_t numInputImages;
EncoderInputImageParameters input;
uint8_t encodeBitDepthLuma;
Expand Down Expand Up @@ -880,7 +879,6 @@ struct EncoderConfig : public VkVideoRefCountBase {
, queueId(0)
, codec(VK_VIDEO_CODEC_OPERATION_NONE_KHR)
, useDpbArray(false)
, videoProfileIdc((uint32_t)-1)
, numInputImages(DEFAULT_NUM_INPUT_IMAGES)
, input()
, encodeBitDepthLuma(0)
Expand Down Expand Up @@ -1070,7 +1068,8 @@ struct EncoderConfig : public VkVideoRefCountBase {
// These functions should be overwritten from the codec-specific classes
virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) { return VK_ERROR_INITIALIZATION_FAILED; };

virtual uint32_t GetDefaultVideoProfileIdc() { return 0; };
// Returns the codec-specific profile identifier (must be set by InitProfileLevel first)
virtual uint32_t GetCodecProfile() = 0;

virtual int8_t InitDpbCount() { return 16; };

Expand Down
26 changes: 24 additions & 2 deletions vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,29 @@ VkResult EncoderConfigAV1::InitDeviceCapabilities(const VulkanDeviceContext* vkD
return VK_SUCCESS;
}

void EncoderConfigAV1::InitProfileLevel()
{
// If profile hasn't been specified, determine it based on bit depth and chroma
if (profile == STD_VIDEO_AV1_PROFILE_INVALID) {
// PROFESSIONAL is required for 12-bit or 422
if ((input.bpp > 10) ||
(encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_422_BIT_KHR)) {
profile = STD_VIDEO_AV1_PROFILE_PROFESSIONAL;
}
// HIGH is required for 444 chroma
else if (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_444_BIT_KHR) {
profile = STD_VIDEO_AV1_PROFILE_HIGH;
}
// MAIN supports 8-bit and 10-bit with 420
else {
profile = STD_VIDEO_AV1_PROFILE_MAIN;
}
}

// Determine level and tier based on encoder configuration
DetermineLevelTier();
}

int8_t EncoderConfigAV1::InitDpbCount()
{
dpbCount = STD_VIDEO_AV1_NUM_REF_FRAMES + 1; // BUFFER_POOL_MAX_SIZE = Number of frames in buffer pool = 10
Expand Down Expand Up @@ -351,8 +374,7 @@ bool EncoderConfigAV1::DetermineLevelTier()

bool EncoderConfigAV1::InitRateControl()
{
DetermineLevelTier();

// Level and tier are already initialized by InitProfileLevel()
// use level max values for now. Limit it to 120Mbits/sec
uint32_t levelBitrate = std::min(GetLevelBitrate(level, tier), 120000000u);

Expand Down
26 changes: 8 additions & 18 deletions vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ struct EncoderConfigAV1 : public EncoderConfig {
picHeightInSbs = DivUp<uint32_t>(encodeHeight, SUPERBLOCK_SIZE);

if ((picWidthInSbs > 0) && (picHeightInSbs > 0)) {
// Initialize profile, level, and tier based on encoder configuration
InitProfileLevel();
return VK_SUCCESS;
}

Expand All @@ -113,23 +115,11 @@ struct EncoderConfigAV1 : public EncoderConfig {

virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override;

virtual uint32_t GetDefaultVideoProfileIdc() override {
// AV1 profile selection based on bit depth and chroma format
// AV1 Main Profile: 8-bit and 10-bit 4:2:0
// AV1 High Profile: 8-bit and 10-bit 4:2:0 and 4:4:4
// AV1 Professional: 8-bit, 10-bit, 12-bit for all chroma formats including 4:2:2
void InitProfileLevel();

// PROFESSIONAL is required for 12-bit or 422
if (encodeBitDepthLuma > 10 || encodeBitDepthChroma > 10 ||
encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_422_BIT_KHR) {
return STD_VIDEO_AV1_PROFILE_PROFESSIONAL;
}
// HIGH is required for 444 chroma
if (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_444_BIT_KHR) {
return STD_VIDEO_AV1_PROFILE_HIGH;
}
// MAIN supports 8-bit and 10-bit with 420
return STD_VIDEO_AV1_PROFILE_MAIN;
virtual uint32_t GetCodecProfile() override {
assert(profile != STD_VIDEO_AV1_PROFILE_INVALID);
return static_cast<uint32_t>(profile);
}

virtual int8_t InitDpbCount() override;
Expand Down Expand Up @@ -204,8 +194,8 @@ struct EncoderConfigAV1 : public EncoderConfig {
return ((encodeWidth * encodeHeight * picSizeProfileFactor) >> 3);
}

StdVideoAV1Profile profile{ STD_VIDEO_AV1_PROFILE_MAIN };
StdVideoAV1Level level{ STD_VIDEO_AV1_LEVEL_5_0 };
StdVideoAV1Profile profile{ STD_VIDEO_AV1_PROFILE_INVALID };
StdVideoAV1Level level{ STD_VIDEO_AV1_LEVEL_INVALID };
uint8_t tier{};
VkVideoEncodeAV1CapabilitiesKHR av1EncodeCapabilities{ VK_STRUCTURE_TYPE_VIDEO_ENCODE_AV1_CAPABILITIES_KHR };
VkVideoEncodeAV1QualityLevelPropertiesKHR av1QualityLevelProperties{ VK_STRUCTURE_TYPE_VIDEO_ENCODE_AV1_QUALITY_LEVEL_PROPERTIES_KHR };
Expand Down
7 changes: 4 additions & 3 deletions vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -464,15 +464,16 @@ void EncoderConfigH264::InitProfileLevel()
if (profileIdc == STD_VIDEO_H264_PROFILE_IDC_INVALID) {
profileIdc = STD_VIDEO_H264_PROFILE_IDC_BASELINE;

if ((GetMaxBFrameCount() > 0) || (entropyCodingMode == ENTROPY_CODING_MODE_CABAC))
// Upgrade to MAIN profile if using B-frames or CABAC entropy coding
if ((gopStructure.GetConsecutiveBFrameCount() > 0) || (entropyCodingMode == ENTROPY_CODING_MODE_CABAC))
profileIdc = STD_VIDEO_H264_PROFILE_IDC_MAIN;

if (use8x8Transform) {
profileIdc = STD_VIDEO_H264_PROFILE_IDC_HIGH;
}

if (((tuningMode == VK_VIDEO_ENCODE_TUNING_MODE_LOSSLESS_KHR) &&
(rateControlMode == VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR)) ||
// Upgrade to HIGH_444_PREDICTIVE for lossless encoding or 4:4:4 chroma
if ((tuningMode == VK_VIDEO_ENCODE_TUNING_MODE_LOSSLESS_KHR) ||
(input.chromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_444_BIT_KHR)) {
profileIdc = STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE;
}
Expand Down
15 changes: 6 additions & 9 deletions vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,8 @@ struct EncoderConfigH264 : public EncoderConfig {
pic_height_in_map_units = DivUp<uint32_t>(encodeHeight, 16);

if ((pic_width_in_mbs > 0) && (pic_height_in_map_units > 0)) {
// Initialize codec profile and level based on encoder configuration
InitProfileLevel();
return VK_SUCCESS;
}

Expand All @@ -179,15 +181,10 @@ struct EncoderConfigH264 : public EncoderConfig {

virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override;

virtual uint32_t GetDefaultVideoProfileIdc() override {
// H.264 profile selection based on bit depth and chroma format
// HIGH_444_PREDICTIVE is required for 10-bit or 444 chroma
if (encodeBitDepthLuma > 8 || encodeBitDepthChroma > 8 ||
encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_444_BIT_KHR) {
return STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE;
}
return STD_VIDEO_H264_PROFILE_IDC_HIGH;
};
virtual uint32_t GetCodecProfile() override {
assert(profileIdc != STD_VIDEO_H264_PROFILE_IDC_INVALID);
return static_cast<uint32_t>(profileIdc);
}

// 1. First h.264 determine the number of the Dpb buffers required
virtual int8_t InitDpbCount() override;
Expand Down
69 changes: 35 additions & 34 deletions vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,53 +460,67 @@ void EncoderConfigH265::InitializeSpsRefPicSet(SpsH265 *pSps)
memset(&pSps->longTermRefPicsSps.lt_ref_pic_poc_lsb_sps, 0, sizeof(pSps->longTermRefPicsSps.lt_ref_pic_poc_lsb_sps));
}

StdVideoH265ProfileTierLevel EncoderConfigH265::GetLevelTier()
void EncoderConfigH265::DetermineLevelTier()
{
StdVideoH265ProfileTierLevel profileTierLevel{};
profileTierLevel.general_profile_idc = STD_VIDEO_H265_PROFILE_IDC_INVALID;
profileTierLevel.general_level_idc = STD_VIDEO_H265_LEVEL_IDC_INVALID;
levelIdc = STD_VIDEO_H265_LEVEL_IDC_INVALID;
general_tier_flag = 0;
uint32_t levelIdx = 0;
for (; levelIdx < levelLimitsTblSize; levelIdx++) {

if (IsSuitableLevel(levelIdx, 0)) {
profileTierLevel.general_level_idc = levelLimits[levelIdx].stdLevel;
profileTierLevel.flags.general_tier_flag = 0; // Main tier
levelIdc = levelLimits[levelIdx].stdLevel;
general_tier_flag = 0; // Main tier
break;
}

if ((levelLimits[levelIdx].levelIdc >= 120) && // level 4.0 and above
IsSuitableLevel(levelIdx, 1)) {

profileTierLevel.general_level_idc = levelLimits[levelIdx].stdLevel;
profileTierLevel.flags.general_tier_flag = 1; // Main tier
levelIdc = levelLimits[levelIdx].stdLevel;
general_tier_flag = 1; // High tier
break;
}
}

if (levelIdx >= levelLimitsTblSize) {
assert(!"No suitable level selected");
}
}

void EncoderConfigH265::InitProfileLevel()
{
// If profile hasn't been specified, determine it based on bit depth and chroma
if (profile == STD_VIDEO_H265_PROFILE_IDC_INVALID) {
if (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_420_BIT_KHR) {
if (input.bpp == 8) {
profile = STD_VIDEO_H265_PROFILE_IDC_MAIN;
} else if (input.bpp <= 10) {
profile = STD_VIDEO_H265_PROFILE_IDC_MAIN_10;
} else {
profile = STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS;
}
} else {
// 4:2:2 or 4:4:4 requires Range Extensions profile
profile = STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS;
}
}

return profileTierLevel;
// Determine level and tier based on resolution, bitrate, etc.
DetermineLevelTier();
}

bool EncoderConfigH265::InitRateControl()
{
StdVideoH265ProfileTierLevel profileTierLevel = GetLevelTier();
// Assigning default main profile if invalid.
if (profileTierLevel.general_profile_idc == STD_VIDEO_H265_PROFILE_IDC_INVALID) {
profileTierLevel.general_profile_idc = STD_VIDEO_H265_PROFILE_IDC_MAIN;
}
uint32_t level = profileTierLevel.general_level_idc;
if (level >= levelLimitsTblSize) {
// Level is already initialized by InitProfileLevel()
if (levelIdc >= levelLimitsTblSize) {
assert(!"The h.265 level index is invalid");
return false;
}
uint32_t cpbVclFactor = GetCpbVclFactor();

// Safe default maximum bitrate
uint32_t levelBitRate = std::max(averageBitrate, hrdBitrate);
levelBitRate = std::max(levelBitRate, std::min(levelLimits[level].maxBitRateMainTier * 800u, 120000000u));
levelBitRate = std::max(levelBitRate, std::min(levelLimits[levelIdc].maxBitRateMainTier * 800u, 120000000u));

// If no bitrate is specified, use the level limit
if (averageBitrate == 0) {
Expand Down Expand Up @@ -540,7 +554,7 @@ bool EncoderConfigH265::InitRateControl()

// Use the main tier level limit for the max VBV buffer size, and no more than 8 seconds at peak rate
if (vbvBufferSize == 0) {
vbvBufferSize = std::min(levelLimits[level].maxCPBSizeMainTier * cpbVclFactor, 100000000u);
vbvBufferSize = std::min(levelLimits[levelIdc].maxCPBSizeMainTier * cpbVclFactor, 100000000u);
if (rateControlMode != VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR) {
if ((vbvBufferSize >> 3) > hrdBitrate) {
vbvBufferSize = hrdBitrate << 3;
Expand Down Expand Up @@ -607,7 +621,9 @@ bool EncoderConfigH265::InitParamameters(VpsH265 *vpsInfo, SpsH265 *spsInfo,
spsInfo->decPicBufMgr.max_num_reorder_pics[i] = gopStructure.GetConsecutiveBFrameCount() ? 1 : 0;
}

spsInfo->profileTierLevel = GetLevelTier();
// Use pre-initialized profile, level, and tier from InitProfileLevel()
spsInfo->profileTierLevel.general_profile_idc = profile;
spsInfo->profileTierLevel.general_level_idc = levelIdc;
spsInfo->profileTierLevel.flags.general_tier_flag = general_tier_flag;

// Always insert profile tier flags assuming frame mode
Expand All @@ -617,21 +633,6 @@ bool EncoderConfigH265::InitParamameters(VpsH265 *vpsInfo, SpsH265 *spsInfo,
spsInfo->profileTierLevel.flags.general_non_packed_constraint_flag = 0;
spsInfo->profileTierLevel.flags.general_frame_only_constraint_flag = 1;

// Assigning default main profile if invalid.
if (spsInfo->profileTierLevel.general_profile_idc == STD_VIDEO_H265_PROFILE_IDC_INVALID) {
if (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_420_BIT_KHR) {
if (input.bpp == 8) {
spsInfo->profileTierLevel.general_profile_idc = STD_VIDEO_H265_PROFILE_IDC_MAIN;
} else if (input.bpp == 10) {
spsInfo->profileTierLevel.general_profile_idc = STD_VIDEO_H265_PROFILE_IDC_MAIN_10;
} else {
spsInfo->profileTierLevel.general_profile_idc = STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS;
}
} else {
spsInfo->profileTierLevel.general_profile_idc = STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS;
}
}

const uint32_t ctbLog2SizeY = cuSize + 3;
const uint32_t minCbLog2SizeY = cuMinSize + 3;
const uint32_t log2MinTransformBlockSize = std::min(minCbLog2SizeY, minTransformUnitSize + 2U);
Expand Down
29 changes: 12 additions & 17 deletions vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ struct EncoderConfigH265 : public EncoderConfig {
size_t levelLimitsTblSize;

EncoderConfigH265()
: profile(STD_VIDEO_H265_PROFILE_IDC_MAIN)
, levelIdc(STD_VIDEO_H265_LEVEL_IDC_5_2)
: profile(STD_VIDEO_H265_PROFILE_IDC_INVALID)
, levelIdc(STD_VIDEO_H265_LEVEL_IDC_INVALID)
, h265EncodeCapabilities()
, general_tier_flag(false)
, numRefL0(1)
Expand Down Expand Up @@ -146,26 +146,21 @@ struct EncoderConfigH265 : public EncoderConfig {

hrdBitrate = maxBitrate;

// Initialize profile, level, and tier based on encoder configuration
InitProfileLevel();

// TODO: more h.265 parameters init ...
return VK_SUCCESS;
}

void InitProfileLevel();

virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override;

virtual uint32_t GetDefaultVideoProfileIdc() override {
// Select profile based on bit depth and chroma format
// 12-bit or 444 chroma requires Range Extensions profile
if (encodeBitDepthLuma > 10 || encodeBitDepthChroma > 10 ||
encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_444_BIT_KHR) {
return STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS;
}
// 10-bit with 420/422 uses Main10 profile
if (encodeBitDepthLuma > 8 || encodeBitDepthChroma > 8) {
return STD_VIDEO_H265_PROFILE_IDC_MAIN_10;
}
// 8-bit with 420/422 uses Main profile
return STD_VIDEO_H265_PROFILE_IDC_MAIN;
};
virtual uint32_t GetCodecProfile() override {
assert(profile != STD_VIDEO_H265_PROFILE_IDC_INVALID);
return static_cast<uint32_t>(profile);
}

// 1. First h.265 determine the number of the Dpb buffers required
virtual int8_t InitDpbCount() override;
Expand Down Expand Up @@ -203,7 +198,7 @@ struct EncoderConfigH265 : public EncoderConfig {
StdVideoH265SequenceParameterSetVui* vui = nullptr);

bool IsSuitableLevel(uint32_t levelIdx, bool highTier);
StdVideoH265ProfileTierLevel GetLevelTier();
void DetermineLevelTier();
void InitializeSpsRefPicSet(SpsH265 *pSps);

};
Expand Down
3 changes: 0 additions & 3 deletions vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,6 @@ VkResult VkVideoEncoderH264::InitEncoderCodec(VkSharedBaseObj<EncoderConfig>& en
return VK_ERROR_VIDEO_PROFILE_CODEC_NOT_SUPPORTED_KHR;
}

// Initialize the codec profile and level before other encoder configuration
m_encoderConfig->InitProfileLevel();

VkResult result = InitEncoder(encoderConfig);
if (result != VK_SUCCESS) {
fprintf(stderr, "\nERROR: InitEncoder() failed with ret(%d)\n", result);
Expand Down