cefsrc: Add is-live=false - produces frames OnPaint

Defaults to is-live=true (previous functionality)

Author: aiden-jeffrey

gstcefsrc.cc

@@ -2,6 +2,7 @@
 #include "gst/gstelement.h"
 #include "gst/gstinfo.h"
 #include "gst/gstobject.h"
+#include "gst/gstpad.h"
 #include <cstdio>
 #include <glib-object.h>
 #include <glib.h>
@@ -44,6 +45,7 @@ GST_DEBUG_CATEGORY_STATIC (cef_src_debug);
 
 GST_DEBUG_CATEGORY_STATIC (cef_console_debug);
 
+#define DEFAULT_IS_LIVE TRUE
 #define DEFAULT_WIDTH 1920
 #define DEFAULT_HEIGHT 1080
 #define DEFAULT_FPS_N 30
@@ -126,6 +128,7 @@ enum
 {
   PROP_0,
   PROP_URL,
+  PROP_IS_LIVE,
   PROP_GPU,
   PROP_CHROMIUM_DEBUG_PORT,
   PROP_CHROME_EXTRA_FLAGS,
@@ -267,6 +270,14 @@ class RenderHandler : public CefRenderHandler
       gst_buffer_unref (new_buffer);
       GST_OBJECT_UNLOCK (src);
 
+      if (!src->is_live) {
+        g_mutex_lock(&src->on_paint_lock);
+        src->painted = TRUE;
+
+        g_cond_signal(&src->on_paint_cond);
+        g_mutex_unlock(&src->on_paint_lock);
+      }
+
       GST_LOG_OBJECT (src, "done painting");
     }
 
@@ -345,7 +356,7 @@ class AudioHandler : public CefAudioHandler
       // NB: audio latency is required because of the way the audio data is muxed into the video
       //     buffers
       // TODO: is there a better way to determine the required audio latency here?
-      GstClockTimeDiff audio_latency = (dt * 4);
+      GstClockTimeDiff audio_latency = (dt * 10);
 
       GstClockTime vpts = gst_util_uint64_scale (src->n_frames + 1, src->vinfo.fps_d * GST_SECOND, src->vinfo.fps_n);
       cefGstOffset = GST_CLOCK_DIFF (cef_pts, vpts) + audio_latency;
@@ -720,6 +731,25 @@ static GstFlowReturn gst_cef_src_create(GstPushSrc *push_src, GstBuffer **buf)
   GstCefSrc *src = GST_CEF_SRC (push_src);
   GList *tmp;
 
+  if (!src->is_live) {
+    g_mutex_lock(&src->on_paint_lock);
+    while (!src->painted && !src->flushing) {
+      g_cond_wait(&src->on_paint_cond, &src->on_paint_lock);
+    }
+
+    if (src->flushing) {
+      g_mutex_unlock(&src->on_paint_lock);
+      GST_DEBUG_OBJECT(src, "Flushing");
+      return GST_FLOW_FLUSHING;
+    }
+
+    if (src->painted) {
+      src->painted = FALSE;
+    }
+
+    g_mutex_unlock(&src->on_paint_lock);
+  }
+
   GST_OBJECT_LOCK (src);
 
   if (src->audio_events) {
@@ -753,6 +783,27 @@ static GstFlowReturn gst_cef_src_create(GstPushSrc *push_src, GstBuffer **buf)
   return GST_FLOW_OK;
 }
 
+static gboolean gst_cef_src_unlock(GstBaseSrc *bsrc) {
+  GstCefSrc *src = GST_CEF_SRC (bsrc);
+
+  g_mutex_lock(&src->on_paint_lock);
+  src->flushing = TRUE;
+  g_cond_signal(&src->on_paint_cond);
+  g_mutex_unlock(&src->on_paint_lock);
+
+  return TRUE;
+}
+
+static gboolean gst_cef_src_unlock_stop(GstBaseSrc *bsrc) {
+  GstCefSrc *src = GST_CEF_SRC (bsrc);
+
+  g_mutex_lock(&src->on_paint_lock);
+  src->flushing = FALSE;
+  g_mutex_unlock(&src->on_paint_lock);
+
+  return TRUE;
+}
+
 /* Once we have started a first cefsrc for this process, we start
  * a UI thread and never shut it down. We could probably refine this
  * to stop and restart the thread as needed, but this updated approach
@@ -978,6 +1029,13 @@ gst_cef_src_start(GstBaseSrc *base_src)
 
   GST_ELEMENT_PROGRESS(src, START, "open", ("Creating CEF browser client"));
 
+  if (!src->is_live) {
+    g_mutex_lock(&src->on_paint_lock);
+    src->flushing = FALSE;
+    g_cond_signal(&src->on_paint_cond);
+    g_mutex_unlock(&src->on_paint_lock);
+  }
+
   CefRefPtr<BrowserClient> browserClient = new BrowserClient(src);
 
   /* Make sure CEF is initialized before posting a task */
@@ -1055,6 +1113,13 @@ gst_cef_src_stop (GstBaseSrc *base_src)
 
   GST_INFO_OBJECT (src, "Stopping");
 
+  if (!src->is_live) {
+    g_mutex_lock(&src->on_paint_lock);
+    src->flushing = TRUE;
+    g_cond_signal(&src->on_paint_cond);
+    g_mutex_unlock(&src->on_paint_lock);
+  }
+
   if (src->browser) {
     gst_cef_src_close_browser(src);
 #ifdef __APPLE__
@@ -1193,6 +1258,11 @@ gst_cef_src_set_property (GObject * object, guint prop_id, const GValue * value,
       src->chrome_extra_flags = g_value_dup_string (value);
       break;
     }
+    case PROP_IS_LIVE:
+    {
+      src->is_live = g_value_get_boolean (value);
+      break;
+    }
     case PROP_GPU:
     {
       GST_WARNING_OBJECT(
@@ -1279,6 +1349,9 @@ gst_cef_src_get_property (GObject * object, guint prop_id, GValue * value,
     case PROP_CHROME_EXTRA_FLAGS:
       g_value_set_string (value, src->chrome_extra_flags);
       break;
+    case PROP_IS_LIVE:
+      g_value_set_boolean (value, src->is_live);
+      break;
     case PROP_GPU:
       g_value_set_boolean (value, src->gpu);
       break;
@@ -1324,13 +1397,22 @@ gst_cef_src_finalize (GObject *object)
 
   g_cond_clear(&src->state_cond);
   g_mutex_clear(&src->state_lock);
+
+  g_cond_clear (&src->on_paint_cond);
+  g_mutex_clear (&src->on_paint_lock);
 }
 
 static void
 gst_cef_src_init (GstCefSrc * src)
 {
   GstBaseSrc *base_src = GST_BASE_SRC (src);
 
+  src->is_live = DEFAULT_IS_LIVE;
+  g_cond_init (&src->on_paint_cond);
+  g_mutex_init (&src->on_paint_lock);
+  src->flushing = FALSE;
+  src->painted = FALSE;
+
   src->n_frames = 0;
   src->current_buffer = NULL;
   src->audio_buffers = NULL;
@@ -1367,6 +1449,11 @@ gst_cef_src_class_init (GstCefSrcClass * klass)
           "The URL to display",
           DEFAULT_URL, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT)));
 
+  g_object_class_install_property (gobject_class, PROP_IS_LIVE,
+    g_param_spec_boolean ("is-live", "is-live",
+          "Operate in live mode (if true, we only sample the cef video buffer, if false we produce one frame per OnPaint)",
+          DEFAULT_IS_LIVE, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
   g_object_class_install_property (gobject_class, PROP_GPU,
     g_param_spec_boolean ("gpu", "gpu",
           "Enable GPU usage in chromium (Improves performance if you have GPU) - "
@@ -1435,6 +1522,9 @@ gst_cef_src_class_init (GstCefSrcClass * klass)
   base_src_class->get_times = GST_DEBUG_FUNCPTR(gst_cef_src_get_times);
   base_src_class->query = GST_DEBUG_FUNCPTR(gst_cef_src_query);
 
+  base_src_class->unlock = GST_DEBUG_FUNCPTR(gst_cef_src_unlock);
+  base_src_class->unlock_stop = GST_DEBUG_FUNCPTR(gst_cef_src_unlock_stop);
+
   gstelement_class->change_state = GST_DEBUG_FUNCPTR(gst_cef_src_change_state);
 
   push_src_class->create = GST_DEBUG_FUNCPTR(gst_cef_src_create);

gstcefsrc.h

@@ -64,6 +64,12 @@ struct _GstCefSrc {
   CefRefPtr<CefBrowser> browser;
   CefRefPtr<CefApp> app;
 
+  gboolean is_live;
+  GCond on_paint_cond;
+  GMutex on_paint_lock;
+  gboolean painted;
+  gboolean flushing;
+
   GCond state_cond;
   GMutex state_lock;
   CefSrcState state;

html/raf_test.html

@@ -32,6 +32,8 @@
       }
     </style>
     <script language="JavaScript">
+      const params = new URLSearchParams(window.location.search);
+
       let frameNum = 0; // s
       let fps = 0;
       let t = undefined;
@@ -72,21 +74,25 @@
         requestAnimationFrame(animate);
       }
 
-      const audioDelay = 1000;
-
-      document.addEventListener("DOMContentLoaded", () => {
-        setTimeout(() => {
-          const audio = new Audio("sounds/test.mp3");
-          // audio.loop = true;
-          audio.play();
-          audio.onended = () => {
-            setTimeout(() => {
-              audio.currentTime = 0;
-              audio.play();
-            }, audioDelay);
-          }
-        }, audioDelay);
-      });
+      const audioDelayStr = params.get("audioDelay") ?? "1000";
+      const audioDelay = parseInt(audioDelayStr);
+      console.log("audioDelay:", audioDelay);
+      if (audioDelay >= 0) {
+
+        document.addEventListener("DOMContentLoaded", () => {
+          setTimeout(() => {
+            const audio = new Audio("sounds/test.mp3");
+            // audio.loop = true;
+            audio.play();
+            audio.onended = () => {
+              setTimeout(() => {
+                audio.currentTime = 0;
+                audio.play();
+              }, audioDelay);
+            }
+          }, audioDelay);
+        });
+      }
 
       requestAnimationFrame(animate);
     </script>