Skip to content

Support changing frame size during encoding #1069

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 16 commits into
base: main
Choose a base branch
from
Draft

Support changing frame size during encoding #1069

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
864c34e
Support changing frame size during encoding
tongyuantongyu Aug 26, 2022
3f8f3b3
clang-format
tongyuantongyu Aug 30, 2022
b8356ea
test on ssim
tongyuantongyu Aug 31, 2022
a677a39
review changes
tongyuantongyu Sep 2, 2022
d0d7290
Merge remote-tracking branch 'upstream/main' into var-frame-size
tongyuantongyu Jan 15, 2023
d7fe311
Add tests for more encoding cases
tongyuantongyu Jan 15, 2023
5e208db
AOM version check for encoding
tongyuantongyu Jan 15, 2023
35af2e6
skip test if libyuv is not available
tongyuantongyu Jan 15, 2023
ea312fa
Merge branch 'main' into var-frame-size
tongyuantongyu Mar 3, 2023
86382ae
Two (failing) progressive encoding tests
tongyuantongyu Feb 8, 2023
956a3fd
Add test
tongyuantongyu Mar 7, 2023
b69e549
Update branch
tongyuantongyu Sep 13, 2023
382a42a
Update test (still has crash from AOM)
tongyuantongyu Sep 13, 2023
f19fa0e
Fix gainmap, adjust member location
tongyuantongyu Sep 13, 2023
e7d768f
Fix gainmap test
tongyuantongyu Sep 13, 2023
0086558
Fix test compilation
tongyuantongyu Sep 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions include/avif/avif.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1046,6 +1046,9 @@ struct avifCodecSpecificOptions;
// image in less bytes. AVIF_SPEED_DEFAULT means "Leave the AV1 codec to its default speed settings"./
// If avifEncoder uses rav1e, the speed value is directly passed through (0-10). If libaom is used,
// a combination of settings are tweaked to simulate this speed range.
// * Width and height: width and height of encoded image. Default value 0 means infer from first frame.
// For grid image, this is the size of one cell. Value must not be smaller than the largest frame
// to be encoded.
Copy link
Collaborator

@wantehchang wantehchang Sep 1, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I need to think about what this value means for grid images.

Copy link
Contributor Author

@tongyuantongyu tongyuantongyu Sep 2, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the easiest choice: encoder->width and encoder-height overrides image->width and image->height, if present.

Alternatively this can be defined as size of the whole grid, and we verify and compute size of each cell internally.

// * Some encoder settings can be changed after encoding starts. Changes will take effect in the next
// call to avifEncoderAddImage().
typedef struct avifEncoder
Expand All @@ -1058,6 +1061,8 @@ typedef struct avifEncoder
int speed;
int keyframeInterval; // How many frames between automatic forced keyframes; 0 to disable (default).
uint64_t timescale; // timescale of the media (Hz)
uint32_t width;
uint32_t height;
// changeable encoder settings
int minQuantizer;
int maxQuantizer;
Expand Down
7 changes: 5 additions & 2 deletions src/codec_aom.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -666,6 +666,8 @@ static avifResult aomCodecEncodeImage(avifCodec * codec,
cfg->g_input_bit_depth = image->depth;
cfg->g_w = image->width;
cfg->g_h = image->height;
cfg->g_forced_max_frame_width = encoder->width;
cfg->g_forced_max_frame_height = encoder->height;
if (addImageFlags & AVIF_ADD_IMAGE_FLAG_SINGLE) {
// Set the maximum number of frames to encode to 1. This instructs
// libaom to set still_picture and reduced_still_picture_header to
Expand Down Expand Up @@ -761,8 +763,9 @@ static avifResult aomCodecEncodeImage(avifCodec * codec,
} else {
avifBool dimensionsChanged = AVIF_FALSE;
if ((cfg->g_w != image->width) || (cfg->g_h != image->height)) {
// We are not ready for dimension change for now.
return AVIF_RESULT_NOT_IMPLEMENTED;
cfg->g_w = image->width;
cfg->g_h = image->height;
dimensionsChanged = AVIF_TRUE;
}
if (alpha) {
if (encoderChanges & (AVIF_ENCODER_CHANGE_MIN_QUANTIZER_ALPHA | AVIF_ENCODER_CHANGE_MAX_QUANTIZER_ALPHA)) {
Expand Down
5 changes: 5 additions & 0 deletions src/codec_rav1e.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,11 @@ static avifResult rav1eCodecEncodeImage(avifCodec * codec,
return AVIF_RESULT_NOT_IMPLEMENTED;
}

// rav1e does not support overriding maximum frame width/height in sequence header
if (encoder->width || encoder->height) {
return AVIF_RESULT_NOT_IMPLEMENTED;
}

avifResult result = AVIF_RESULT_UNKNOWN_ERROR;

RaConfig * rav1eConfig = NULL;
Expand Down
2 changes: 2 additions & 0 deletions src/codec_svt.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,8 @@ static avifResult svtCodecEncodeImage(avifCodec * codec,

svt_config->source_width = image->width;
svt_config->source_height = image->height;
svt_config->forced_max_frame_width = encoder->width;
svt_config->forced_max_frame_height = encoder->height;
svt_config->logical_processors = encoder->maxThreads;
svt_config->enable_adaptive_quantization = AVIF_FALSE;
// disable 2-pass
Expand Down
57 changes: 35 additions & 22 deletions src/write.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,8 @@ avifEncoder * avifEncoderCreate(void)
encoder->speed = AVIF_SPEED_DEFAULT;
encoder->keyframeInterval = 0;
encoder->timescale = 1;
encoder->width = 0;
encoder->height = 0;
encoder->minQuantizer = AVIF_QUANTIZER_LOSSLESS;
encoder->maxQuantizer = AVIF_QUANTIZER_LOSSLESS;
encoder->minQuantizerAlpha = AVIF_QUANTIZER_LOSSLESS;
Expand Down Expand Up @@ -332,12 +334,15 @@ static void avifEncoderBackupSettings(avifEncoder * encoder)
lastEncoder->speed = encoder->speed;
lastEncoder->keyframeInterval = encoder->keyframeInterval;
lastEncoder->timescale = encoder->timescale;
lastEncoder->width = encoder->width;
lastEncoder->height = encoder->height;
lastEncoder->minQuantizer = encoder->minQuantizer;
lastEncoder->maxQuantizer = encoder->maxQuantizer;
lastEncoder->minQuantizerAlpha = encoder->minQuantizerAlpha;
lastEncoder->maxQuantizerAlpha = encoder->maxQuantizerAlpha;
lastEncoder->tileRowsLog2 = encoder->tileRowsLog2;
lastEncoder->tileColsLog2 = encoder->tileColsLog2;
lastEncoder->speed = encoder->speed;
Copy link
Collaborator

@wantehchang wantehchang Aug 31, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Delete this. This line has been moved to line 334.

Copy link
Contributor Author

@tongyuantongyu tongyuantongyu Sep 2, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

}

// This function detects changes made on avifEncoder. It returns true on success (i.e., if every
Expand All @@ -355,7 +360,8 @@ static avifBool avifEncoderDetectChanges(const avifEncoder * encoder, avifEncode

if ((lastEncoder->codecChoice != encoder->codecChoice) || (lastEncoder->maxThreads != encoder->maxThreads) ||
(lastEncoder->speed != encoder->speed) || (lastEncoder->keyframeInterval != encoder->keyframeInterval) ||
(lastEncoder->timescale != encoder->timescale)) {
(lastEncoder->timescale != encoder->timescale) || (lastEncoder->width != encoder->width) ||
(lastEncoder->height != encoder->height)) {
return AVIF_FALSE;
}

Expand Down Expand Up @@ -694,6 +700,10 @@ static avifResult avifEncoderAddImageInternal(avifEncoder * encoder,
return AVIF_RESULT_NO_CONTENT;
}

if ((encoder->width && (encoder->width < firstCell->width)) || (encoder->height && (encoder->height < firstCell->height))) {
return AVIF_RESULT_INCOMPATIBLE_IMAGE;
}

if ((cellCount > 1) && !avifAreGridDimensionsValid(firstCell->yuvFormat,
gridCols * firstCell->width,
gridRows * firstCell->height,
Expand Down Expand Up @@ -997,6 +1007,9 @@ avifResult avifEncoderFinish(avifEncoder * encoder, avifRWData * output)
// Begin write stream

const avifImage * imageMetadata = encoder->data->imageMetadata;
const uint32_t cellWidth = encoder->width ? encoder->width : imageMetadata->width;
const uint32_t cellHeight = encoder->height ? encoder->height : imageMetadata->height;

// The epoch for creation_time and modification_time is midnight, Jan. 1,
// 1904, in UTC time. Add the number of seconds between that epoch and the
// Unix epoch.
Expand Down Expand Up @@ -1200,11 +1213,11 @@ avifResult avifEncoderFinish(avifEncoder * encoder, avifRWData * output)
}
}

uint32_t imageWidth = imageMetadata->width;
uint32_t imageHeight = imageMetadata->height;
uint32_t imageWidth = cellWidth;
uint32_t imageHeight = cellHeight;
if (isGrid) {
imageWidth = imageMetadata->width * item->gridCols;
imageHeight = imageMetadata->height * item->gridRows;
imageWidth = imageWidth * item->gridCols;
imageHeight = imageHeight * item->gridRows;
Copy link
Collaborator

@wantehchang wantehchang Sep 1, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Optional: Should we use the *= notation?

            imageWidth *=  item->gridCols;
            imageHeight *= item->gridRows;

Copy link
Contributor Author

@tongyuantongyu tongyuantongyu Sep 2, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks better. Done.

}

// Properties all av01 items need
Expand Down Expand Up @@ -1361,8 +1374,8 @@ avifResult avifEncoderFinish(avifEncoder * encoder, avifRWData * output)
avifRWStreamWriteU16(&s, 0); // template int(16) volume = {if track_is_audio 0x0100 else 0};
avifRWStreamWriteU16(&s, 0); // const unsigned int(16) reserved = 0;
avifRWStreamWrite(&s, unityMatrix, sizeof(unityMatrix)); // template int(32)[9] matrix= // { 0x00010000,0,0,0,0x00010000,0,0,0,0x40000000 };
avifRWStreamWriteU32(&s, imageMetadata->width << 16); // unsigned int(32) width;
avifRWStreamWriteU32(&s, imageMetadata->height << 16); // unsigned int(32) height;
avifRWStreamWriteU32(&s, cellWidth << 16); // unsigned int(32) width;
avifRWStreamWriteU32(&s, cellHeight << 16); // unsigned int(32) height;
avifRWStreamFinishBox(&s, tkhd);

if (item->irefToID != 0) {
Expand Down Expand Up @@ -1470,21 +1483,21 @@ avifResult avifEncoderFinish(avifEncoder * encoder, avifRWData * output)
avifBoxMarker stsd = avifRWStreamWriteFullBox(&s, "stsd", AVIF_BOX_SIZE_TBD, 0, 0);
avifRWStreamWriteU32(&s, 1); // unsigned int(32) entry_count;
avifBoxMarker av01 = avifRWStreamWriteBox(&s, "av01", AVIF_BOX_SIZE_TBD);
avifRWStreamWriteZeros(&s, 6); // const unsigned int(8)[6] reserved = 0;
avifRWStreamWriteU16(&s, 1); // unsigned int(16) data_reference_index;
avifRWStreamWriteU16(&s, 0); // unsigned int(16) pre_defined = 0;
avifRWStreamWriteU16(&s, 0); // const unsigned int(16) reserved = 0;
avifRWStreamWriteZeros(&s, sizeof(uint32_t) * 3); // unsigned int(32)[3] pre_defined = 0;
avifRWStreamWriteU16(&s, (uint16_t)imageMetadata->width); // unsigned int(16) width;
avifRWStreamWriteU16(&s, (uint16_t)imageMetadata->height); // unsigned int(16) height;
avifRWStreamWriteU32(&s, 0x00480000); // template unsigned int(32) horizresolution
avifRWStreamWriteU32(&s, 0x00480000); // template unsigned int(32) vertresolution
avifRWStreamWriteU32(&s, 0); // const unsigned int(32) reserved = 0;
avifRWStreamWriteU16(&s, 1); // template unsigned int(16) frame_count = 1;
avifRWStreamWriteChars(&s, "\012AOM Coding", 11); // string[32] compressorname;
avifRWStreamWriteZeros(&s, 32 - 11); //
avifRWStreamWriteU16(&s, 0x0018); // template unsigned int(16) depth = 0x0018;
avifRWStreamWriteU16(&s, (uint16_t)0xffff); // int(16) pre_defined = -1;
avifRWStreamWriteZeros(&s, 6); // const unsigned int(8)[6] reserved = 0;
avifRWStreamWriteU16(&s, 1); // unsigned int(16) data_reference_index;
avifRWStreamWriteU16(&s, 0); // unsigned int(16) pre_defined = 0;
avifRWStreamWriteU16(&s, 0); // const unsigned int(16) reserved = 0;
avifRWStreamWriteZeros(&s, sizeof(uint32_t) * 3); // unsigned int(32)[3] pre_defined = 0;
avifRWStreamWriteU16(&s, (uint16_t)cellWidth); // unsigned int(16) width;
avifRWStreamWriteU16(&s, (uint16_t)cellHeight); // unsigned int(16) height;
avifRWStreamWriteU32(&s, 0x00480000); // template unsigned int(32) horizresolution
avifRWStreamWriteU32(&s, 0x00480000); // template unsigned int(32) vertresolution
avifRWStreamWriteU32(&s, 0); // const unsigned int(32) reserved = 0;
avifRWStreamWriteU16(&s, 1); // template unsigned int(16) frame_count = 1;
avifRWStreamWriteChars(&s, "\012AOM Coding", 11); // string[32] compressorname;
avifRWStreamWriteZeros(&s, 32 - 11); //
avifRWStreamWriteU16(&s, 0x0018); // template unsigned int(16) depth = 0x0018;
avifRWStreamWriteU16(&s, (uint16_t)0xffff); // int(16) pre_defined = -1;
writeConfigBox(&s, &item->av1C);
if (!item->alpha) {
avifEncoderWriteColorProperties(&s, imageMetadata, NULL, NULL);
Expand Down
63 changes: 63 additions & 0 deletions tests/gtest/avifchangesettingtest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,5 +143,68 @@ TEST(ChangeSettingTest, UnchangeableSetting) {
AVIF_RESULT_CANNOT_CHANGE_SETTING);
}

TEST(ChangeSettingTest, DISABLED_ChangeDimension) {
if (avifCodecName(AVIF_CODEC_CHOICE_AOM, AVIF_CODEC_FLAG_CAN_ENCODE) ==
nullptr) {
GTEST_SKIP() << "Codec unavailable, skip test.";
}

const uint32_t size_small = 256;
const uint32_t size_display = 512;

testutil::AvifImagePtr first =
testutil::CreateImage(size_small, size_small, 8, AVIF_PIXEL_FORMAT_YUV420,
AVIF_PLANES_YUV, AVIF_RANGE_FULL);
ASSERT_NE(first, nullptr);
testutil::FillImageGradient(first.get());

testutil::AvifImagePtr second = testutil::CreateImage(
size_display, size_display, 8, AVIF_PIXEL_FORMAT_YUV420, AVIF_PLANES_YUV,
AVIF_RANGE_FULL);
ASSERT_NE(second, nullptr);
testutil::FillImageGradient(second.get());

testutil::AvifRwData encodedAvif;

// Encode
{
testutil::AvifEncoderPtr encoder(avifEncoderCreate(), avifEncoderDestroy);
ASSERT_NE(encoder, nullptr);
encoder->codecChoice = AVIF_CODEC_CHOICE_AOM;
encoder->speed = AVIF_SPEED_FASTEST;
encoder->timescale = 1;
encoder->minQuantizer = 63;
encoder->maxQuantizer = 63;
encoder->width = size_display;
encoder->height = size_display;

// avifEncoderSetCodecSpecificOption(encoder.get(), "tune", "psnr");

ASSERT_EQ(avifEncoderAddImage(encoder.get(), first.get(), 1, 0),
AVIF_RESULT_OK);

ASSERT_EQ(avifEncoderAddImage(encoder.get(), second.get(), 1, 0),
AVIF_RESULT_OK);

ASSERT_EQ(avifEncoderFinish(encoder.get(), &encodedAvif), AVIF_RESULT_OK);
}

// Decode
{
testutil::AvifDecoderPtr decoder(avifDecoderCreate(), avifDecoderDestroy);
ASSERT_NE(decoder, nullptr);

avifDecoderSetIOMemory(decoder.get(), encodedAvif.data, encodedAvif.size);
ASSERT_EQ(avifDecoderParse(decoder.get()), AVIF_RESULT_OK);
ASSERT_EQ(avifDecoderNextImage(decoder.get()), AVIF_RESULT_OK);
// libavif scales frame automatically.
ASSERT_EQ(decoder->image->width, size_display);
ASSERT_EQ(decoder->image->height, size_display);
ASSERT_EQ(avifDecoderNextImage(decoder.get()), AVIF_RESULT_OK);
ASSERT_EQ(decoder->image->width, size_display);
ASSERT_EQ(decoder->image->height, size_display);
}
}

} // namespace
} // namespace libavif