libvmaf/motion_v2: add motion_five_frame_window

Author: kylophone

libvmaf/src/feature/feature_extractor.h

@@ -98,7 +98,8 @@ typedef struct VmafFeatureExtractor {
     #endif
 
     VmafFrameSyncContext *framesync;
-    VmafPicture prev_ref; ///< Previous reference picture, set by framework.
+    VmafPicture prev_ref; ///< Previous reference picture (n-1), set by framework.
+    VmafPicture prev_prev_ref; ///< Reference picture from two frames ago (n-2), set by framework.
 
 } VmafFeatureExtractor;
 

libvmaf/src/feature/integer_motion_v2.c

@@ -62,6 +62,7 @@ typedef struct MotionV2State {
     unsigned w, h, bpc;
     motion_pipeline_fn pipeline;
     double motion_max_val;
+    bool motion_five_frame_window;
     VmafDictionary *feature_name_dict;
 } MotionV2State;
 
@@ -77,6 +78,15 @@ static const VmafOption options[] = {
         .flags = VMAF_OPT_FLAG_FEATURE_PARAM,
         .alias = "mmxv",
     },
+    {
+        .name = "motion_five_frame_window",
+        .alias = "mffw",
+        .help = "use five-frame temporal window",
+        .offset = offsetof(MotionV2State, motion_five_frame_window),
+        .type = VMAF_OPT_TYPE_BOOL,
+        .default_val.b = false,
+        .flags = VMAF_OPT_FLAG_FEATURE_PARAM,
+    },
     { 0 }
 };
 
@@ -239,21 +249,25 @@ static int extract(VmafFeatureExtractor *fex,
     (void) ref_pic_90;
     (void) dist_pic_90;
 
-    if (index == 0) {
+    const unsigned min_idx = s->motion_five_frame_window ? 2 : 1;
+    if (index < min_idx) {
         return vmaf_feature_collector_append_with_dict(feature_collector,
                 s->feature_name_dict,
                 "VMAF_integer_feature_motion_v2_sad_score", 0., index);
     }
 
-    if (!fex->prev_ref.ref)
+    const VmafPicture *prev = s->motion_five_frame_window
+            ? &fex->prev_prev_ref
+            : &fex->prev_ref;
+    if (!prev->ref)
         return -EINVAL;
 
     const unsigned w = s->w;
     const unsigned h = s->h;
-    const uint8_t *prev_data = (const uint8_t *)fex->prev_ref.data[0];
+    const uint8_t *prev_data = (const uint8_t *)prev->data[0];
     const uint8_t *cur_data = (const uint8_t *)ref_pic->data[0];
 
-    uint64_t sad = s->pipeline(prev_data, fex->prev_ref.stride[0],
+    uint64_t sad = s->pipeline(prev_data, prev->stride[0],
                                cur_data, ref_pic->stride[0],
                                s->y_row, w, h, s->bpc);
 
@@ -276,6 +290,12 @@ static int flush(VmafFeatureExtractor *fex,
 {
     MotionV2State *s = fex->priv;
 
+    if (!s->feature_name_dict) {
+        s->feature_name_dict = vmaf_feature_name_dict_from_provided_features(
+                fex->provided_features, fex->options, s);
+        if (!s->feature_name_dict) return -ENOMEM;
+    }
+
     VmafDictionaryEntry *e_sad = vmaf_dictionary_get(&s->feature_name_dict,
             "VMAF_integer_feature_motion_v2_sad_score", 0);
     const char *sad_name =
@@ -287,20 +307,29 @@ static int flush(VmafFeatureExtractor *fex,
             sad_name, &dummy, n_frames))
         n_frames++;
 
-    if (n_frames < 2) return 1;
+    const unsigned stride = s->motion_five_frame_window ? 2 : 1;
+    const unsigned min_idx = s->motion_five_frame_window ? 2 : 1;
+    if (n_frames == 0) return 1;
 
     for (unsigned i = 0; i < n_frames; i++) {
-        double score_cur, score_next;
-        vmaf_feature_collector_get_score(feature_collector,
-            sad_name, &score_cur, i);
-
         double motion2;
-        if (i + 1 < n_frames) {
-            vmaf_feature_collector_get_score(feature_collector,
-                sad_name, &score_next, i + 1);
-            motion2 = score_cur < score_next ? score_cur : score_next;
+
+        if (i < min_idx) {
+            motion2 = 0.;
+        } else if (i == n_frames - 1) {
+            vmaf_feature_collector_get_score(feature_collector, sad_name, &motion2, i);
         } else {
-            motion2 = score_cur;
+            const int lo_idx = (int)i - (int)(stride - 1);
+            const int hi_idx = (int)i + 1;
+            double hi;
+            vmaf_feature_collector_get_score(feature_collector, sad_name, &hi, hi_idx);
+            if (lo_idx >= (int)min_idx) {
+                double lo;
+                vmaf_feature_collector_get_score(feature_collector, sad_name, &lo, lo_idx);
+                motion2 = lo < hi ? lo : hi;
+            } else {
+                motion2 = hi;
+            }
         }
 
         vmaf_feature_collector_append_with_dict(feature_collector,

libvmaf/src/libvmaf.c

@@ -90,7 +90,8 @@ typedef struct VmafContext {
     } pic_params;
     unsigned pic_cnt;
     bool flushed;
-    VmafPicture prev_ref; // previous ref pic for PREV_REF extractors (in-order only)
+    VmafPicture prev_ref;      // n-1 ref pic for PREV_REF extractors (in-order only)
+    VmafPicture prev_prev_ref; // n-2 ref pic for PREV_REF extractors (in-order only)
 } VmafContext;
 
 #ifdef VMAF_BATCH_THREADING
@@ -299,8 +300,7 @@ static int check_picture_pool(VmafContext *vmaf)
     if (!vmaf->thread_pool) return 0;
     if (vmaf->picture_pool) return 0;
 
-    // Default to 2x thread count if not explicitly preallocated
-    const unsigned pic_cnt = vmaf->cfg.n_threads * 2;
+    const unsigned pic_cnt = vmaf->cfg.n_threads * 2 + 2;
 
     int err = prepare_picture_pool(vmaf, pic_cnt,
                                    vmaf->pic_params.w,
@@ -351,6 +351,8 @@ int vmaf_close(VmafContext *vmaf)
     vmaf_thread_pool_wait(vmaf->thread_pool);
     if (vmaf->prev_ref.ref)
         vmaf_picture_unref(&vmaf->prev_ref);
+    if (vmaf->prev_prev_ref.ref)
+        vmaf_picture_unref(&vmaf->prev_prev_ref);
     vmaf_framesync_destroy(vmaf->framesync);
     feature_extractor_vector_destroy(&(vmaf->registered_feature_extractors));
     vmaf_feature_collector_destroy(vmaf->feature_collector);
@@ -486,7 +488,7 @@ int vmaf_use_features_from_model_collection(VmafContext *vmaf,
 
 struct ThreadData {
     VmafFeatureExtractorContext *fex_ctx;
-    VmafPicture ref, dist, prev_ref;
+    VmafPicture ref, dist, prev_ref, prev_prev_ref;
     unsigned index;
     VmafFeatureCollector *feature_collector;
     VmafFeatureExtractorContextPool *fex_ctx_pool;
@@ -500,15 +502,21 @@ static void threaded_extract_func(void *e, void **thread_data)
 
     if (f->prev_ref.ref)
         f->fex_ctx->fex->prev_ref = f->prev_ref;
+    if (f->prev_prev_ref.ref)
+        f->fex_ctx->fex->prev_prev_ref = f->prev_prev_ref;
 
     f->err = vmaf_feature_extractor_context_extract(f->fex_ctx, &f->ref, NULL,
                                                     &f->dist, NULL, f->index,
                                                     f->feature_collector);
 
     if (f->prev_ref.ref) {
-        memset(&f->fex_ctx->fex->prev_ref, 0, sizeof(f->fex_ctx->fex->prev_ref));
+        f->fex_ctx->fex->prev_ref = (VmafPicture){0};
         vmaf_picture_unref(&f->prev_ref);
     }
+    if (f->prev_prev_ref.ref) {
+        f->fex_ctx->fex->prev_prev_ref = (VmafPicture){0};
+        vmaf_picture_unref(&f->prev_prev_ref);
+    }
 
     f->err = vmaf_fex_ctx_pool_release(f->fex_ctx_pool, f->fex_ctx);
     vmaf_picture_unref(&f->ref);
@@ -517,7 +525,7 @@ static void threaded_extract_func(void *e, void **thread_data)
 
 #ifdef VMAF_BATCH_THREADING
 struct ThreadDataBatch {
-    VmafPicture ref, dist, prev_ref;
+    VmafPicture ref, dist, prev_ref, prev_prev_ref;
     unsigned index;
     VmafFeatureCollector *feature_collector;
     RegisteredFeatureExtractors *registered_fex;
@@ -567,6 +575,8 @@ static void threaded_extract_batch_func(void *e, void **thread_data)
         if (fex->flags & VMAF_FEATURE_EXTRACTOR_PREV_REF) {
             if (f->prev_ref.ref)
                 td->fex_ctx[i]->fex->prev_ref = f->prev_ref;
+            if (f->prev_prev_ref.ref)
+                td->fex_ctx[i]->fex->prev_prev_ref = f->prev_prev_ref;
         }
 
         int err = vmaf_feature_extractor_context_extract(td->fex_ctx[i],
@@ -575,9 +585,10 @@ static void threaded_extract_batch_func(void *e, void **thread_data)
                                                          f->index,
                                                          f->feature_collector);
 
-        if (fex->flags & VMAF_FEATURE_EXTRACTOR_PREV_REF)
-            memset(&td->fex_ctx[i]->fex->prev_ref, 0,
-                   sizeof(td->fex_ctx[i]->fex->prev_ref));
+        if (fex->flags & VMAF_FEATURE_EXTRACTOR_PREV_REF) {
+            td->fex_ctx[i]->fex->prev_ref = (VmafPicture){0};
+            td->fex_ctx[i]->fex->prev_prev_ref = (VmafPicture){0};
+        }
 
         if (err) {
             f->err = err;
@@ -588,6 +599,8 @@ static void threaded_extract_batch_func(void *e, void **thread_data)
 unref:
     if (f->prev_ref.ref)
         vmaf_picture_unref(&f->prev_ref);
+    if (f->prev_prev_ref.ref)
+        vmaf_picture_unref(&f->prev_prev_ref);
     vmaf_picture_unref(&f->ref);
     vmaf_picture_unref(&f->dist);
 }
@@ -622,21 +635,23 @@ static int threaded_read_pictures(VmafContext *vmaf, VmafPicture *ref,
                                        &fex_ctx);
         if (err) return err;
 
-        VmafPicture pic_a, pic_b, prev_ref = { 0 };
+        VmafPicture pic_a, pic_b, prev_ref = { 0 }, prev_prev_ref = { 0 };
         vmaf_picture_ref(&pic_a, ref);
         vmaf_picture_ref(&pic_b, dist);
 
-        if ((fex->flags & VMAF_FEATURE_EXTRACTOR_PREV_REF) &&
-            vmaf->prev_ref.ref)
-        {
-            vmaf_picture_ref(&prev_ref, &vmaf->prev_ref);
+        if (fex->flags & VMAF_FEATURE_EXTRACTOR_PREV_REF) {
+            if (vmaf->prev_ref.ref)
+                vmaf_picture_ref(&prev_ref, &vmaf->prev_ref);
+            if (vmaf->prev_prev_ref.ref)
+                vmaf_picture_ref(&prev_prev_ref, &vmaf->prev_prev_ref);
         }
 
         struct ThreadData data = {
             .fex_ctx = fex_ctx,
             .ref = pic_a,
             .dist = pic_b,
             .prev_ref = prev_ref,
+            .prev_prev_ref = prev_prev_ref,
             .index = index,
             .feature_collector = vmaf->feature_collector,
             .fex_ctx_pool = vmaf->fex_ctx_pool,
@@ -649,12 +664,17 @@ static int threaded_read_pictures(VmafContext *vmaf, VmafPicture *ref,
             vmaf_picture_unref(&pic_a);
             vmaf_picture_unref(&pic_b);
             if (prev_ref.ref) vmaf_picture_unref(&prev_ref);
+            if (prev_prev_ref.ref) vmaf_picture_unref(&prev_prev_ref);
             return err;
         }
     }
 
-    if (vmaf->prev_ref.ref)
-        vmaf_picture_unref(&vmaf->prev_ref);
+    if (vmaf->prev_prev_ref.ref)
+        vmaf_picture_unref(&vmaf->prev_prev_ref);
+    if (vmaf->prev_ref.ref) {
+        vmaf->prev_prev_ref = vmaf->prev_ref;
+        vmaf->prev_ref = (VmafPicture){0};
+    }
     vmaf_picture_ref(&vmaf->prev_ref, ref);
 
     return vmaf_picture_unref(ref) | vmaf_picture_unref(dist);
@@ -670,17 +690,20 @@ static int threaded_read_pictures_batch(VmafContext *vmaf, VmafPicture *ref,
 
     int err = 0;
 
-    VmafPicture pic_a, pic_b, prev_ref = { 0 };
+    VmafPicture pic_a, pic_b, prev_ref = { 0 }, prev_prev_ref = { 0 };
     vmaf_picture_ref(&pic_a, ref);
     vmaf_picture_ref(&pic_b, dist);
 
     if (vmaf->prev_ref.ref)
         vmaf_picture_ref(&prev_ref, &vmaf->prev_ref);
+    if (vmaf->prev_prev_ref.ref)
+        vmaf_picture_ref(&prev_prev_ref, &vmaf->prev_prev_ref);
 
     struct ThreadDataBatch data = {
         .ref = pic_a,
         .dist = pic_b,
         .prev_ref = prev_ref,
+        .prev_prev_ref = prev_prev_ref,
         .index = index,
         .feature_collector = vmaf->feature_collector,
         .registered_fex = &vmaf->registered_feature_extractors,
@@ -694,11 +717,16 @@ static int threaded_read_pictures_batch(VmafContext *vmaf, VmafPicture *ref,
         vmaf_picture_unref(&pic_a);
         vmaf_picture_unref(&pic_b);
         if (prev_ref.ref) vmaf_picture_unref(&prev_ref);
+        if (prev_prev_ref.ref) vmaf_picture_unref(&prev_prev_ref);
         return err;
     }
 
-    if (vmaf->prev_ref.ref)
-        vmaf_picture_unref(&vmaf->prev_ref);
+    if (vmaf->prev_prev_ref.ref)
+        vmaf_picture_unref(&vmaf->prev_prev_ref);
+    if (vmaf->prev_ref.ref) {
+        vmaf->prev_prev_ref = vmaf->prev_ref;
+        vmaf->prev_ref = (VmafPicture){0};
+    }
     vmaf_picture_ref(&vmaf->prev_ref, ref);
 
     return vmaf_picture_unref(ref) | vmaf_picture_unref(dist);
@@ -980,18 +1008,21 @@ int vmaf_read_pictures(VmafContext *vmaf, VmafPicture *ref, VmafPicture *dist,
             &dist_device : &dist_host;
 #endif
 
-        if ((fex_ctx->fex->flags & VMAF_FEATURE_EXTRACTOR_PREV_REF) &&
-            vmaf->prev_ref.ref)
-        {
-            fex_ctx->fex->prev_ref = vmaf->prev_ref;
+        if (fex_ctx->fex->flags & VMAF_FEATURE_EXTRACTOR_PREV_REF) {
+            if (vmaf->prev_ref.ref)
+                fex_ctx->fex->prev_ref = vmaf->prev_ref;
+            if (vmaf->prev_prev_ref.ref)
+                fex_ctx->fex->prev_prev_ref = vmaf->prev_prev_ref;
         }
 
         err = vmaf_feature_extractor_context_extract(fex_ctx, ref, NULL, dist,
                                                      NULL, index,
                                                      vmaf->feature_collector);
 
-        if (fex_ctx->fex->flags & VMAF_FEATURE_EXTRACTOR_PREV_REF)
-            memset(&fex_ctx->fex->prev_ref, 0, sizeof(fex_ctx->fex->prev_ref));
+        if (fex_ctx->fex->flags & VMAF_FEATURE_EXTRACTOR_PREV_REF) {
+            fex_ctx->fex->prev_ref = (VmafPicture){0};
+            fex_ctx->fex->prev_prev_ref = (VmafPicture){0};
+        }
 
         if (err) return err;
     }
@@ -1011,9 +1042,12 @@ int vmaf_read_pictures(VmafContext *vmaf, VmafPicture *ref, VmafPicture *dist,
 #endif
     }
 
-    if (vmaf->prev_ref.ref)
-        vmaf_picture_unref(&vmaf->prev_ref);
-    
+    if (vmaf->prev_prev_ref.ref)
+        vmaf_picture_unref(&vmaf->prev_prev_ref);
+    if (vmaf->prev_ref.ref) {
+        vmaf->prev_prev_ref = vmaf->prev_ref;
+        vmaf->prev_ref = (VmafPicture){0};
+    }
     if (ref && ref->ref)
         vmaf_picture_ref(&vmaf->prev_ref, ref);
 #ifdef HAVE_CUDA

libvmaf/tools/vmaf.c

@@ -291,7 +291,7 @@ int main(int argc, char *argv[])
             .bpc = common_bitdepth,
             .pix_fmt = pix_fmt_map(info.pixel_fmt),
         },
-        .pic_cnt = c.thread_cnt > 0 ? (c.thread_cnt + 1) * 2 : 3,
+        .pic_cnt = c.thread_cnt > 0 ? (c.thread_cnt + 1) * 2 + 1 : 4,
     };
 
     err = vmaf_preallocate_pictures(vmaf, pic_cfg);