Skip to content

Commit a4f96f6

Browse files
authored
Merge pull request #17 from openapv/hotfix_enc_medium
[Bug Fix] My cause overflow on QP index
2 parents 9680d7c + 510fd91 commit a4f96f6

File tree

2 files changed

+649
-641
lines changed

2 files changed

+649
-641
lines changed

src/oapv.c

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -500,7 +500,15 @@ static double enc_block_rdo_medium(oapve_ctx_t* ctx, oapve_core_t* core, int log
500500
continue;
501501
}
502502
}
503-
int q_setp = (core->q_mat_dec[c][scanp[j]] + (1 << (core->dq_shift[c] - 1))) >> core->dq_shift[c];
503+
int q_step = 0;
504+
if (core->dq_shift[c] > 0)
505+
{
506+
q_step = (core->q_mat_dec[c][scanp[j]] + (1 << (core->dq_shift[c] - 1))) >> core->dq_shift[c];
507+
}
508+
else
509+
{
510+
q_step = (core->q_mat_dec[c][scanp[j]]) << (-core->dq_shift[c]);
511+
}
504512

505513
for (int i = 1; i < adj_rng && !zero_dist; i++) {
506514
if (i > 2) {
@@ -517,8 +525,8 @@ static double enc_block_rdo_medium(oapve_ctx_t* ctx, oapve_core_t* core, int log
517525

518526
s16 test_coef = org_coef + map_idx_diff[i];
519527
coeff[scanp[j]] = test_coef;
520-
int step_diff = q_setp * map_idx_diff[i];
521-
ctx->fn_itx_adj[0](rec_ups, rec_tmp, j, step_diff, 16);
528+
int step_diff = q_step * map_idx_diff[i];
529+
ctx->fn_itx_adj[0](rec_ups, rec_tmp, j, step_diff, 9);
522530
for (int k = 0; k < 64; k++)
523531
{
524532
recon[k] = (rec_tmp[k] + 512) >> 10;

0 commit comments

Comments
 (0)