WebRTC on IGPU

.cargo/config.toml

@@ -14,4 +14,7 @@ rustflags = ["-C", "link-args=-ObjC"]
 rustflags = ["-C", "link-args=-ObjC"]
 
 [target.aarch64-apple-ios-sim]
-rustflags = ["-C", "link-args=-ObjC"]
+rustflags = ["-C", "link-args=-ObjC"]
+
+[env]
+LK_CUSTOM_WEBRTC = { value = "webrtc-sys/libwebrtc/linux-x64-release", relative = true }

Cargo.toml

@@ -9,3 +9,9 @@ members = [
     "webrtc-sys",
     "webrtc-sys/build",
 ]
+
+[profile.release]
+opt-level = 3        # Maximum optimization (0-3, or "s" for size, "z" for even smaller size)
+lto = "fat"          # Link Time Optimization (LTO): "thin" (faster) or "fat" (better optimization)
+codegen-units = 1    # Improves performance by reducing parallel compilation
+debug = false        # Remove debug symbols for smaller binaries. Set this to true for profiling with debug symbols

README.md

@@ -1,12 +1,80 @@
-<!--BEGIN_BANNER_IMAGE-->
+# ANYbotics forked version of LiveKit Rust SDK
+The Rust SDK is using the Google WebRTC repository. The Rust SDK repository has been modified for patching WebRTC. The patches add support for accelerating the encoding process on Intel GPUs, by utilizing the libvpl, vaapi and one of the two Intel GPU runtimes (MediaSDK or Intel VPL GPU RT).
+During initialization of WebRTC encoders it is checked whether HW acceleration is possible and if HW acceleration was initialized successfuly. If so, the HW accelerated encoder will take place automatically. Otherwise, the software implementation is used.
+**Note**: The design that includes the accelerated encoder implementation is not ideal, however the goal was to modify the Livekit stack as less as possible.
+
+To make use of the accelerated version of WebRTC, we need to build Livekit from source. To achieve this we need to execute the `/webrtc-sys/libwebrtc/build_linux.sh`. This script checks if the WebRTC repository has already been cloned locally. If not, it fetches it, along with all each submodules and applies some Livekit patches on it. Livekit uses a certain WebRTC version, and not the latest one.
+To fetch and build WebRTC:
+```
+./build_linux.sh --arch x64 --profile release
+```
+In case WebRTC is already present locally, after executing the above script, some warning will be shown regarding the failure of Livekit patches. That's because the patches have already been applied.
+Once WebRTC is present locally, we can apply our patches. Navigate to the root of the repository and execute:
+
+```
+apply_vpl_patches.sh
+```
+
+The implementation of the accelerated encoder is part of the Rust SDK repo, which we have forked, and it is under `libwebrtc-hw`. Ideally the implementation would be part of the WebRTC repository, but it is more complicated. WebRTC repository is huge and includes a lot of sub-modules, which we would have to fork. For now, Rust SDK repository has all the required changes to use Livekit with accelerated encoding.
+
+Once we have WebRTC patched and built, a static library is generated. The next step is to build the Rust part and link it against the generated static library.
+Under the root folder execute:
+```
+cargo clean
+cargo build --release
+```
+Whenever we build again WebRTC, we need to do a cargo clean and rebuild.
+At this point a `liblivekit_ffi.so` has been generated, and our application needs to make use of it, to have our accelerated version. To achieve this, we need to expose its path to an environment variable. Along with this environment variable, we need to expose a couple of other ones as well and the application should start in a context that can parse them:
+```
+export LIVEKIT_URL=wss://192.168.0.6/webrtc/
+export LIVEKIT_API_KEY=ads-admin
+export LIVEKIT_API_SECRET=livekit-server-secret-for-ads-server
+export LIVEKIT_LIB_PATH="/home/integration/gpu_acceleration_ws/anybotics-python-sdks/livekit-rtc/rust-sdks/target/release/liblivekit_ffi.so"
+export LD_LIBRARY_PATH=/home/integration/libvpl/_build:$LD_LIBRARY_PATH
+export LIBVA_DRIVER_NAME=iHD
+export LIBVA_DRIVERS_PATH=/home/integration/media-driver-workspace/build_media/media_driver
+```
+The above values are indicative.
+**LIVEKIT_URL** should include the IP of the desired pc.
+**LIVEKIT_API_KEY** and **LIVEKIT_API_SECRET** are the ones that we use to generate a token.
+**LIVEKIT_LIB_PATH** should be set accordingly, depending on where we have install the `liblivekit_ffi.so`.
+**LD_LIBRARY_PATH** exposes the Intel libvpl and we need to set the path to the installed location.
+**LIBVA_DRIVER_NAME** indicates the Intel driver. iHD is the appropriate one for our HW on anymal D.
+**LIBVA_DRIVERS_PATH** exposes the path in which we have installed the Intel runtimes (MediaSDK or Intel® VPL)
+
+
+## Updating patches
+To update the patches, navigate to `webrtc-sys/libwebrtc/src` and execute
+```
+git diff original_commit new_commit --src-prefix=org/webrtc-sys/libwebrtc/src/ --dst-prefix=update/webrtc-sys/libwebrtc/src/ ./path/file > ./../../../libwebrtc-patches/file.patch
+```
+
+## Developing and testing
+If the development takes place on a PC, we need to start a local server with
+```
+livekit-server --dev
+```
+Use the Livekit web client to receive the stream
+```
+https://meet.livekit.io/?tab=custom
+```
+Add the required URL of the server.
+In case we test on a PC, use the loopback IP
+```
+ws://localhost:7880
+```
+In case we test on an anymal use the corresponding IP. The example below is from dobby:
+```
+wss://192.168.0.6/webrtc/
+```
+
+For a token we need to use the Livekit CLI:
+```
+lk token create --api-key ads-admin --api-secret livekit-server-secret-for-ads-server --join --room dobby --identity test_user --valid-for 24h
+```
+adapt the arguments accordingly.
 
-<picture>
-  <source media="(prefers-color-scheme: dark)" srcset="/.github/banner_dark.png">
-  <source media="(prefers-color-scheme: light)" srcset="/.github/banner_light.png">
-  <img style="width:100%;" alt="The LiveKit icon, the name of the repository and some sample code in the background." src="https://raw.githubusercontent.com/livekit/rust-sdks/main/.github/banner_light.png">
-</picture>
 
-<!--END_BANNER_IMAGE-->
 
 # 📹🎙️🦀 Rust Client SDK for LiveKit
 

apply_vpl_patches.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+git apply -p1 libwebrtc-patches/h264_encoder_impl_cc.patch
+if [ $? -eq 0 ]; then
+  echo "h264_encoder_impl_cc.patch applied successfully."
+else
+  echo "Failed to apply h264_encoder_impl_cc.patch."
+  exit 1
+fi
+
+git apply -p1 libwebrtc-patches/h264_encoder_impl_h.patch
+if [ $? -eq 0 ]; then
+  echo "h264_encoder_impl_h.patch applied successfully."
+else
+  echo "Failed to apply h264_encoder_impl_h.patch."
+  exit 1
+fi
+
+git apply -p1 libwebrtc-patches/build_gn.patch
+if [ $? -eq 0 ]; then
+  echo "build_gn.patch applied successfully."
+else
+  echo "Failed to apply build_gn.patch."
+  exit 1
+fi

examples/basic_room/src/main.rs

@@ -1,9 +1,30 @@
 use livekit::prelude::*;
 use livekit_api::access_token;
 use std::env;
+use image::GenericImageView;
+use std::sync::Arc;
+use tokio::time::{sleep, Duration};
+use livekit::options::TrackPublishOptions;
+// use livekit_ffi::livekit::proto::track::VideoCodec;
+use livekit::options::VideoCodec;
+use tokio::signal;
+use livekit::webrtc::video_source::RtcVideoSource;
+use livekit::webrtc::video_source::VideoResolution;
+use livekit::webrtc::{
+    native::yuv_helper,
+    video_frame::{I420Buffer, VideoFrame, VideoRotation},
+    video_source::native::NativeVideoSource,
+};
+use tokio::sync::Notify;
+use std::time::{Instant};
 
 // Connect to a room using the specified env variables
 // and print all incoming events
+// const WIDTH: usize = 1280;
+// const HEIGHT: usize = 720;
+
+const WIDTH: usize = 1440;
+const HEIGHT: usize = 1080;
 
 #[tokio::main]
 async fn main() {
@@ -18,7 +39,7 @@ async fn main() {
         .with_name("Rust Bot")
         .with_grants(access_token::VideoGrants {
             room_join: true,
-            room: "my-room".to_string(),
+            room: "dobby".to_string(),
             ..Default::default()
         })
         .to_jwt()
@@ -27,16 +48,123 @@ async fn main() {
     let (room, mut rx) = Room::connect(&url, &token, RoomOptions::default()).await.unwrap();
     log::info!("Connected to room: {} - {}", room.name(), String::from(room.sid().await));
 
-    room.local_participant()
-        .publish_data(DataPacket {
-            payload: "Hello world".to_owned().into_bytes(),
-            reliable: true,
-            ..Default::default()
-        })
-        .await
-        .unwrap();
+    // Create a video source and track
+    let source = NativeVideoSource::new(VideoResolution {
+        width: WIDTH as u32,
+        height: HEIGHT as u32,
+    });
+    let track = LocalVideoTrack::create_video_track(
+        "image",
+        RtcVideoSource::Native(source.clone()),
+    );
 
-    while let Some(msg) = rx.recv().await {
-        log::info!("Event: {:?}", msg);
-    }
+    // Publish the track
+    let options = TrackPublishOptions {
+        source: TrackSource::Camera,
+        video_codec: VideoCodec::H264,
+        ..Default::default()
+    };
+    let publication = room.local_participant().publish_track(LocalTrack::Video(track.clone()), options).await.unwrap();
+    println!("Published track with SID: {}", publication.sid());
+
+    // Start displaying the image
+    // tokio::spawn(display_image(source));
+    // Create a Notify object to signal termination
+    let notify = Arc::new(Notify::new());
+    let notify_clone = notify.clone();
+    tokio::spawn(async move {
+        display_image(source, notify_clone).await;
+    });
+
+    // Wait for termination signals
+    signal::ctrl_c().await.unwrap();
+    // room.disconnect().await?;
+    // Ok(())
 }
+
+async fn display_image(video_source: NativeVideoSource, notify: Arc<Notify>) {
+    let frame_duration = Duration::from_millis(33); // Approx. 30 FPS
+
+    // Load the image
+    let img1 = image::open("/home/integration/test_image1.png").expect("Failed to open image");
+    let img1 = img1.resize_exact(WIDTH as u32, HEIGHT as u32, image::imageops::FilterType::Nearest);
+    let img2 = image::open("/home/integration/test_image2.png").expect("Failed to open image");
+    let img2 = img2.resize_exact(WIDTH as u32, HEIGHT as u32, image::imageops::FilterType::Nearest);
+
+    let mut argb_frame = vec![0u8; WIDTH * HEIGHT * 4];
+    let mut y_plane = vec![0u8; WIDTH * HEIGHT];
+    let mut u_plane = vec![0u8; WIDTH * HEIGHT / 4];
+    let mut v_plane = vec![0u8; WIDTH * HEIGHT / 4];
+
+    let mut last_switch = Instant::now();
+
+    let mut current_img = &img1;
+
+    loop {
+        let start_time = tokio::time::Instant::now();
+
+        tokio::select! {
+            _ = notify.notified() => {
+                log::info!("Shutting down display_image loop");
+                break;
+            }
+            _ = async {
+
+        // Check if 5 seconds have passed
+        if last_switch.elapsed() >= Duration::from_secs(5) {
+            // Switch the image
+            if current_img == &img1 {
+                current_img = &img2;
+            } else {
+                current_img = &img1;
+            }
+            // Reset the timer
+            last_switch = Instant::now();
+        }
+
+        // Fill the frame buffer with the image data
+        for (x, y, pixel) in current_img.pixels() {
+            let i = (y as usize * WIDTH + x as usize) * 4;
+            argb_frame[i] = pixel[2]; // B
+            argb_frame[i + 1] = pixel[1]; // G
+            argb_frame[i + 2] = pixel[0]; // R
+            argb_frame[i + 3] = 255; // A
+        }
+
+        let mut video_frame = VideoFrame {
+            rotation: VideoRotation::VideoRotation0,
+            buffer: I420Buffer::new(WIDTH as u32, HEIGHT as u32),
+            timestamp_us: 0,
+        };
+
+        let i420_buffer = &mut video_frame.buffer;
+
+        let (stride_y, stride_u, stride_v) = i420_buffer.strides();
+        let (data_y, data_u, data_v) = i420_buffer.data_mut();
+
+        // Convert ARGB to I420 using abgr_to_i420
+        yuv_helper::abgr_to_i420(
+            &argb_frame,
+            (WIDTH * 4) as u32,
+            data_y,
+            stride_y,
+            data_u,
+            stride_u,
+            data_v,
+            stride_v,
+            WIDTH as i32,
+            HEIGHT as i32,
+        );
+
+        video_source.capture_frame(&video_frame);
+    } => {},
+}
+
+
+        // Sleep to maintain the frame rate
+        let elapsed = start_time.elapsed();
+        if frame_duration > elapsed {
+            sleep(frame_duration - elapsed).await;
+        }
+    }
+}

libwebrtc-hw/.clang-format

@@ -0,0 +1,15 @@
+# The ANYbotics style guide is based on google: https://anybotics.github.io/styleguide/cppguide.html
+BasedOnStyle : Google
+# Maximum line with is 140 characters (default: 80)
+ColumnLimit : 140
+# Indent of two spaces, no tabs.
+IndentWidth: 2
+# Always attach braces to surrounding context.
+BreakBeforeBraces: Attach
+# Force pointer alignement with the type (left).
+DerivePointerAlignment: false
+PointerAlignment: Left
+# Only merge functions defined inside a class.
+AllowShortFunctionsOnASingleLine: Inline
+# Sort each include block separately.
+IncludeBlocks: Preserve