diff --git a/.cargo/config.toml b/.cargo/config.toml
index 2dc4a5047..f7de21c89 100644
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@@ -14,4 +14,7 @@ rustflags = ["-C", "link-args=-ObjC"]
rustflags = ["-C", "link-args=-ObjC"]
[target.aarch64-apple-ios-sim]
-rustflags = ["-C", "link-args=-ObjC"]
\ No newline at end of file
+rustflags = ["-C", "link-args=-ObjC"]
+
+[env]
+LK_CUSTOM_WEBRTC = { value = "webrtc-sys/libwebrtc/linux-x64-release", relative = true }
diff --git a/Cargo.toml b/Cargo.toml
index 47c7ce8f7..ba4986681 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,3 +9,9 @@ members = [
"webrtc-sys",
"webrtc-sys/build",
]
+
+[profile.release]
+opt-level = 3 # Maximum optimization (0-3, or "s" for size, "z" for even smaller size)
+lto = "fat" # Link Time Optimization (LTO): "thin" (faster) or "fat" (better optimization)
+codegen-units = 1 # Improves performance by reducing parallel compilation
+debug = false # Remove debug symbols for smaller binaries. Set this to true for profiling with debug symbols
diff --git a/README.md b/README.md
index e305bd727..32cf36553 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,80 @@
-
+# ANYbotics forked version of LiveKit Rust SDK
+The Rust SDK is using the Google WebRTC repository. The Rust SDK repository has been modified for patching WebRTC. The patches add support for accelerating the encoding process on Intel GPUs, by utilizing the libvpl, vaapi and one of the two Intel GPU runtimes (MediaSDK or Intel VPL GPU RT).
+During initialization of WebRTC encoders it is checked whether HW acceleration is possible and if HW acceleration was initialized successfuly. If so, the HW accelerated encoder will take place automatically. Otherwise, the software implementation is used.
+**Note**: The design that includes the accelerated encoder implementation is not ideal, however the goal was to modify the Livekit stack as less as possible.
+
+To make use of the accelerated version of WebRTC, we need to build Livekit from source. To achieve this we need to execute the `/webrtc-sys/libwebrtc/build_linux.sh`. This script checks if the WebRTC repository has already been cloned locally. If not, it fetches it, along with all each submodules and applies some Livekit patches on it. Livekit uses a certain WebRTC version, and not the latest one.
+To fetch and build WebRTC:
+```
+./build_linux.sh --arch x64 --profile release
+```
+In case WebRTC is already present locally, after executing the above script, some warning will be shown regarding the failure of Livekit patches. That's because the patches have already been applied.
+Once WebRTC is present locally, we can apply our patches. Navigate to the root of the repository and execute:
+
+```
+apply_vpl_patches.sh
+```
+
+The implementation of the accelerated encoder is part of the Rust SDK repo, which we have forked, and it is under `libwebrtc-hw`. Ideally the implementation would be part of the WebRTC repository, but it is more complicated. WebRTC repository is huge and includes a lot of sub-modules, which we would have to fork. For now, Rust SDK repository has all the required changes to use Livekit with accelerated encoding.
+
+Once we have WebRTC patched and built, a static library is generated. The next step is to build the Rust part and link it against the generated static library.
+Under the root folder execute:
+```
+cargo clean
+cargo build --release
+```
+Whenever we build again WebRTC, we need to do a cargo clean and rebuild.
+At this point a `liblivekit_ffi.so` has been generated, and our application needs to make use of it, to have our accelerated version. To achieve this, we need to expose its path to an environment variable. Along with this environment variable, we need to expose a couple of other ones as well and the application should start in a context that can parse them:
+```
+export LIVEKIT_URL=wss://192.168.0.6/webrtc/
+export LIVEKIT_API_KEY=ads-admin
+export LIVEKIT_API_SECRET=livekit-server-secret-for-ads-server
+export LIVEKIT_LIB_PATH="/home/integration/gpu_acceleration_ws/anybotics-python-sdks/livekit-rtc/rust-sdks/target/release/liblivekit_ffi.so"
+export LD_LIBRARY_PATH=/home/integration/libvpl/_build:$LD_LIBRARY_PATH
+export LIBVA_DRIVER_NAME=iHD
+export LIBVA_DRIVERS_PATH=/home/integration/media-driver-workspace/build_media/media_driver
+```
+The above values are indicative.
+**LIVEKIT_URL** should include the IP of the desired pc.
+**LIVEKIT_API_KEY** and **LIVEKIT_API_SECRET** are the ones that we use to generate a token.
+**LIVEKIT_LIB_PATH** should be set accordingly, depending on where we have install the `liblivekit_ffi.so`.
+**LD_LIBRARY_PATH** exposes the Intel libvpl and we need to set the path to the installed location.
+**LIBVA_DRIVER_NAME** indicates the Intel driver. iHD is the appropriate one for our HW on anymal D.
+**LIBVA_DRIVERS_PATH** exposes the path in which we have installed the Intel runtimes (MediaSDK or Intel® VPL)
+
+
+## Updating patches
+To update the patches, navigate to `webrtc-sys/libwebrtc/src` and execute
+```
+git diff original_commit new_commit --src-prefix=org/webrtc-sys/libwebrtc/src/ --dst-prefix=update/webrtc-sys/libwebrtc/src/ ./path/file > ./../../../libwebrtc-patches/file.patch
+```
+
+## Developing and testing
+If the development takes place on a PC, we need to start a local server with
+```
+livekit-server --dev
+```
+Use the Livekit web client to receive the stream
+```
+https://meet.livekit.io/?tab=custom
+```
+Add the required URL of the server.
+In case we test on a PC, use the loopback IP
+```
+ws://localhost:7880
+```
+In case we test on an anymal use the corresponding IP. The example below is from dobby:
+```
+wss://192.168.0.6/webrtc/
+```
+
+For a token we need to use the Livekit CLI:
+```
+lk token create --api-key ads-admin --api-secret livekit-server-secret-for-ads-server --join --room dobby --identity test_user --valid-for 24h
+```
+adapt the arguments accordingly.
-
-
-
-
-
-
# 📹🎙️🦀 Rust Client SDK for LiveKit
diff --git a/apply_vpl_patches.sh b/apply_vpl_patches.sh
new file mode 100755
index 000000000..ce86a21e8
--- /dev/null
+++ b/apply_vpl_patches.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+git apply -p1 libwebrtc-patches/h264_encoder_impl_cc.patch
+if [ $? -eq 0 ]; then
+ echo "h264_encoder_impl_cc.patch applied successfully."
+else
+ echo "Failed to apply h264_encoder_impl_cc.patch."
+ exit 1
+fi
+
+git apply -p1 libwebrtc-patches/h264_encoder_impl_h.patch
+if [ $? -eq 0 ]; then
+ echo "h264_encoder_impl_h.patch applied successfully."
+else
+ echo "Failed to apply h264_encoder_impl_h.patch."
+ exit 1
+fi
+
+git apply -p1 libwebrtc-patches/build_gn.patch
+if [ $? -eq 0 ]; then
+ echo "build_gn.patch applied successfully."
+else
+ echo "Failed to apply build_gn.patch."
+ exit 1
+fi
diff --git a/examples/basic_room/src/main.rs b/examples/basic_room/src/main.rs
index e82b784c2..401815025 100644
--- a/examples/basic_room/src/main.rs
+++ b/examples/basic_room/src/main.rs
@@ -1,9 +1,30 @@
use livekit::prelude::*;
use livekit_api::access_token;
use std::env;
+use image::GenericImageView;
+use std::sync::Arc;
+use tokio::time::{sleep, Duration};
+use livekit::options::TrackPublishOptions;
+// use livekit_ffi::livekit::proto::track::VideoCodec;
+use livekit::options::VideoCodec;
+use tokio::signal;
+use livekit::webrtc::video_source::RtcVideoSource;
+use livekit::webrtc::video_source::VideoResolution;
+use livekit::webrtc::{
+ native::yuv_helper,
+ video_frame::{I420Buffer, VideoFrame, VideoRotation},
+ video_source::native::NativeVideoSource,
+};
+use tokio::sync::Notify;
+use std::time::{Instant};
// Connect to a room using the specified env variables
// and print all incoming events
+// const WIDTH: usize = 1280;
+// const HEIGHT: usize = 720;
+
+const WIDTH: usize = 1440;
+const HEIGHT: usize = 1080;
#[tokio::main]
async fn main() {
@@ -18,7 +39,7 @@ async fn main() {
.with_name("Rust Bot")
.with_grants(access_token::VideoGrants {
room_join: true,
- room: "my-room".to_string(),
+ room: "dobby".to_string(),
..Default::default()
})
.to_jwt()
@@ -27,16 +48,123 @@ async fn main() {
let (room, mut rx) = Room::connect(&url, &token, RoomOptions::default()).await.unwrap();
log::info!("Connected to room: {} - {}", room.name(), String::from(room.sid().await));
- room.local_participant()
- .publish_data(DataPacket {
- payload: "Hello world".to_owned().into_bytes(),
- reliable: true,
- ..Default::default()
- })
- .await
- .unwrap();
+ // Create a video source and track
+ let source = NativeVideoSource::new(VideoResolution {
+ width: WIDTH as u32,
+ height: HEIGHT as u32,
+ });
+ let track = LocalVideoTrack::create_video_track(
+ "image",
+ RtcVideoSource::Native(source.clone()),
+ );
- while let Some(msg) = rx.recv().await {
- log::info!("Event: {:?}", msg);
- }
+ // Publish the track
+ let options = TrackPublishOptions {
+ source: TrackSource::Camera,
+ video_codec: VideoCodec::H264,
+ ..Default::default()
+ };
+ let publication = room.local_participant().publish_track(LocalTrack::Video(track.clone()), options).await.unwrap();
+ println!("Published track with SID: {}", publication.sid());
+
+ // Start displaying the image
+ // tokio::spawn(display_image(source));
+ // Create a Notify object to signal termination
+ let notify = Arc::new(Notify::new());
+ let notify_clone = notify.clone();
+ tokio::spawn(async move {
+ display_image(source, notify_clone).await;
+ });
+
+ // Wait for termination signals
+ signal::ctrl_c().await.unwrap();
+ // room.disconnect().await?;
+ // Ok(())
}
+
+async fn display_image(video_source: NativeVideoSource, notify: Arc) {
+ let frame_duration = Duration::from_millis(33); // Approx. 30 FPS
+
+ // Load the image
+ let img1 = image::open("/home/integration/test_image1.png").expect("Failed to open image");
+ let img1 = img1.resize_exact(WIDTH as u32, HEIGHT as u32, image::imageops::FilterType::Nearest);
+ let img2 = image::open("/home/integration/test_image2.png").expect("Failed to open image");
+ let img2 = img2.resize_exact(WIDTH as u32, HEIGHT as u32, image::imageops::FilterType::Nearest);
+
+ let mut argb_frame = vec![0u8; WIDTH * HEIGHT * 4];
+ let mut y_plane = vec![0u8; WIDTH * HEIGHT];
+ let mut u_plane = vec![0u8; WIDTH * HEIGHT / 4];
+ let mut v_plane = vec![0u8; WIDTH * HEIGHT / 4];
+
+ let mut last_switch = Instant::now();
+
+ let mut current_img = &img1;
+
+ loop {
+ let start_time = tokio::time::Instant::now();
+
+ tokio::select! {
+ _ = notify.notified() => {
+ log::info!("Shutting down display_image loop");
+ break;
+ }
+ _ = async {
+
+ // Check if 5 seconds have passed
+ if last_switch.elapsed() >= Duration::from_secs(5) {
+ // Switch the image
+ if current_img == &img1 {
+ current_img = &img2;
+ } else {
+ current_img = &img1;
+ }
+ // Reset the timer
+ last_switch = Instant::now();
+ }
+
+ // Fill the frame buffer with the image data
+ for (x, y, pixel) in current_img.pixels() {
+ let i = (y as usize * WIDTH + x as usize) * 4;
+ argb_frame[i] = pixel[2]; // B
+ argb_frame[i + 1] = pixel[1]; // G
+ argb_frame[i + 2] = pixel[0]; // R
+ argb_frame[i + 3] = 255; // A
+ }
+
+ let mut video_frame = VideoFrame {
+ rotation: VideoRotation::VideoRotation0,
+ buffer: I420Buffer::new(WIDTH as u32, HEIGHT as u32),
+ timestamp_us: 0,
+ };
+
+ let i420_buffer = &mut video_frame.buffer;
+
+ let (stride_y, stride_u, stride_v) = i420_buffer.strides();
+ let (data_y, data_u, data_v) = i420_buffer.data_mut();
+
+ // Convert ARGB to I420 using abgr_to_i420
+ yuv_helper::abgr_to_i420(
+ &argb_frame,
+ (WIDTH * 4) as u32,
+ data_y,
+ stride_y,
+ data_u,
+ stride_u,
+ data_v,
+ stride_v,
+ WIDTH as i32,
+ HEIGHT as i32,
+ );
+
+ video_source.capture_frame(&video_frame);
+ } => {},
+}
+
+
+ // Sleep to maintain the frame rate
+ let elapsed = start_time.elapsed();
+ if frame_duration > elapsed {
+ sleep(frame_duration - elapsed).await;
+ }
+ }
+}
\ No newline at end of file
diff --git a/libwebrtc-hw/.clang-format b/libwebrtc-hw/.clang-format
new file mode 100644
index 000000000..1dbcc1a56
--- /dev/null
+++ b/libwebrtc-hw/.clang-format
@@ -0,0 +1,15 @@
+# The ANYbotics style guide is based on google: https://anybotics.github.io/styleguide/cppguide.html
+BasedOnStyle : Google
+# Maximum line with is 140 characters (default: 80)
+ColumnLimit : 140
+# Indent of two spaces, no tabs.
+IndentWidth: 2
+# Always attach braces to surrounding context.
+BreakBeforeBraces: Attach
+# Force pointer alignement with the type (left).
+DerivePointerAlignment: false
+PointerAlignment: Left
+# Only merge functions defined inside a class.
+AllowShortFunctionsOnASingleLine: Inline
+# Sort each include block separately.
+IncludeBlocks: Preserve
diff --git a/libwebrtc-hw/vpl_session.cpp b/libwebrtc-hw/vpl_session.cpp
new file mode 100644
index 000000000..4e54e1026
--- /dev/null
+++ b/libwebrtc-hw/vpl_session.cpp
@@ -0,0 +1,156 @@
+#include "vpl_session.h"
+
+#include
+
+#include
+#include
+#include "va/va.h"
+#include "va/va_drm.h"
+
+namespace {
+constexpr char* GPU_RENDER_NODE = "/dev/dri/renderD128";
+}
+
+namespace any_vpl {
+
+VplSession::~VplSession() {
+ MFXClose(session_);
+ MFXUnload(loader_);
+}
+
+mfxSession VplSession::GetSession() const {
+ return session_;
+}
+
+bool VplSession::Initialize() {
+ mfxStatus sts = MFX_ERR_NONE;
+
+ loader_ = MFXLoad();
+ if (loader_ == nullptr) {
+ RTC_LOG(LS_ERROR) << "MFXLoad failed";
+ return false;
+ }
+
+ constexpr char* implementationDescription = "mfxImplDescription.Impl";
+ MFX_ADD_PROPERTY_U32(loader_, implementationDescription, MFX_IMPL_TYPE_HARDWARE);
+
+ sts = MFXCreateSession(loader_, 0, &session_);
+ if (sts != MFX_ERR_NONE) {
+ RTC_LOG(LS_ERROR) << "MFXCreateSession failed: sts=" << sts;
+ return false;
+ }
+
+ // Query selected implementation
+ mfxIMPL implementation;
+ sts = MFXQueryIMPL(session_, &implementation);
+ if (sts != MFX_ERR_NONE) {
+ RTC_LOG(LS_ERROR) << "MFXQueryIMPL failed: sts=" << sts;
+ return false;
+ }
+
+ InitAcceleratorHandle(implementation);
+
+ mfxVersion version;
+ sts = MFXQueryVersion(session_, &version);
+ if (sts != MFX_ERR_NONE) {
+ RTC_LOG(LS_ERROR) << "MFXQueryVersion failed: sts=" << sts;
+ return false;
+ }
+
+ RTC_LOG(LS_INFO) << "Intel VPL Implementation: " << (implementation == MFX_IMPL_SOFTWARE ? "SOFTWARE" : "HARDWARE");
+ RTC_LOG(LS_INFO) << "Intel VPL Version: " << version.Major << "." << version.Minor;
+ ShowImplementationInfo(0);
+
+ return true;
+}
+
+void VplSession::ShowImplementationInfo(mfxU32 implnum) {
+ mfxImplDescription* idesc = nullptr;
+ mfxStatus sts;
+ // Loads info about implementation at specified list location
+ sts = MFXEnumImplementations(loader_, implnum, MFX_IMPLCAPS_IMPLDESCSTRUCTURE, (mfxHDL*)&idesc);
+ if (!idesc || (sts != MFX_ERR_NONE)) {
+ return;
+ }
+
+ RTC_LOG(LS_INFO) << "Implementation details:\n";
+ RTC_LOG(LS_INFO) << " ApiVersion: " << idesc->ApiVersion.Major << "." << idesc->ApiVersion.Minor;
+ RTC_LOG(LS_INFO) << " AccelerationMode via: ";
+ switch (idesc->AccelerationMode) {
+ case MFX_ACCEL_MODE_NA:
+ RTC_LOG(LS_INFO) << "NA";
+ break;
+ case MFX_ACCEL_MODE_VIA_D3D9:
+ RTC_LOG(LS_INFO) << "D3D9";
+ break;
+ case MFX_ACCEL_MODE_VIA_D3D11:
+ RTC_LOG(LS_INFO) << "D3D11";
+ break;
+ case MFX_ACCEL_MODE_VIA_VAAPI:
+ RTC_LOG(LS_INFO) << "VAAPI";
+ break;
+ case MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET:
+ RTC_LOG(LS_INFO) << "VAAPI_DRM_MODESET";
+ break;
+ case MFX_ACCEL_MODE_VIA_VAAPI_GLX:
+ RTC_LOG(LS_INFO) << "VAAPI_GLX";
+ break;
+ case MFX_ACCEL_MODE_VIA_VAAPI_X11:
+ RTC_LOG(LS_INFO) << "VAAPI_X11";
+ break;
+ case MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND:
+ RTC_LOG(LS_INFO) << "VAAPI_WAYLAND";
+ break;
+ case MFX_ACCEL_MODE_VIA_HDDLUNITE:
+ RTC_LOG(LS_INFO) << "HDDLUNITE";
+ break;
+ default:
+ RTC_LOG(LS_INFO) << "unknown";
+ break;
+ }
+ RTC_LOG(LS_INFO) << " DeviceID: " << idesc->Dev.DeviceID;
+ MFXDispReleaseImplDescription(loader_, idesc);
+
+#if (MFX_VERSION >= 2004)
+ // Show implementation path, added in 2.4 API
+ mfxHDL implPath = nullptr;
+ sts = MFXEnumImplementations(loader_, implnum, MFX_IMPLCAPS_IMPLPATH, &implPath);
+ if (!implPath || (sts != MFX_ERR_NONE)) {
+ return;
+ }
+
+ RTC_LOG(LS_INFO) << " Path: " << reinterpret_cast(implPath);
+ MFXDispReleaseImplDescription(loader_, implPath);
+#endif
+}
+
+void VplSession::InitAcceleratorHandle(mfxIMPL implementation) {
+ if ((implementation & MFX_IMPL_VIA_VAAPI) != MFX_IMPL_VIA_VAAPI) {
+ return;
+ }
+ // initialize VAAPI context and set session handle (req in Linux)
+ accelratorFD_ = open(GPU_RENDER_NODE, O_RDWR);
+ if (accelratorFD_ < 0) {
+ RTC_LOG(LS_ERROR) << "Failed to open GPU render node: " << GPU_RENDER_NODE;
+ return;
+ }
+
+ vaDisplay_ = vaGetDisplayDRM(accelratorFD_);
+ if (!vaDisplay_) {
+ RTC_LOG(LS_ERROR) << "Failed to get VA display from GPU render node: " << GPU_RENDER_NODE;
+ return;
+ }
+
+ int majorVersion = 0, minorVersion = 0;
+ if (VA_STATUS_SUCCESS != vaInitialize(vaDisplay_, &majorVersion, &minorVersion)) {
+ RTC_LOG(LS_ERROR) << "Failed to initialize VA library";
+ return;
+ }
+
+ RTC_LOG(LS_INFO) << "VAAPI initialized. Version: " << majorVersion << "." << minorVersion;
+ if (MFXVideoCORE_SetHandle(session_, static_cast(MFX_HANDLE_VA_DISPLAY), vaDisplay_) != MFX_ERR_NONE) {
+ RTC_LOG(LS_ERROR) << "Failed to set VA display handle for the VA library to use";
+ }
+}
+
+} // namespace any_vpl
diff --git a/libwebrtc-hw/vpl_session.h b/libwebrtc-hw/vpl_session.h
new file mode 100644
index 000000000..e9981f963
--- /dev/null
+++ b/libwebrtc-hw/vpl_session.h
@@ -0,0 +1,62 @@
+#ifndef ANY_VPL_SESSION_IMPL_H_
+#define ANY_VPL_SESSION_IMPL_H_
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace any_vpl {
+
+/**
+ * @brief Wraps an Intel® VPL session.
+ *
+ */
+class VplSession {
+ public:
+ VplSession() = default;
+ ~VplSession();
+
+ /**
+ * @brief Handles all the required initializations for the VPL session.
+ * MFXLoad, which enumerates and initializes all available runtimes
+ * MFXCreateSession, which creates a session for the selected runtime
+ * MFXQueryIMPL, returns the implementation type of a given session
+ *
+ */
+ bool Initialize();
+
+ /**
+ * @brief Get the Vpl Session
+ *
+ * @return mfxSession The Vpl Session
+ */
+ mfxSession GetSession() const;
+
+ private:
+ mfxLoader loader_{nullptr};
+ mfxSession session_{nullptr};
+ int accelratorFD_{0};
+ VADisplay vaDisplay_{nullptr};
+
+ /**
+ * @brief If the hardware acceleration goes through the Linux* VA-API infrastructure, this function initializes the VA-API context and
+ * sets the session handle.
+ *
+ * @param implementation The implementation type
+ */
+ void InitAcceleratorHandle(mfxIMPL implementation);
+
+ /**
+ * @brief Shows implementation info with Intel® VPL
+ *
+ * @param implnum
+ */
+ void ShowImplementationInfo(mfxU32 implnum);
+};
+
+} // namespace any_vpl
+
+#endif
diff --git a/libwebrtc-hw/vpl_utils.cpp b/libwebrtc-hw/vpl_utils.cpp
new file mode 100644
index 000000000..2dca7e31b
--- /dev/null
+++ b/libwebrtc-hw/vpl_utils.cpp
@@ -0,0 +1,54 @@
+#include "vpl_utils.h"
+
+#include
+#include
+#include
+#include
-