fix(📹): add e2e tests for Platform Buffers and fix iOS implementation (…

…#2351)

example/android/build.gradle

@@ -3,7 +3,7 @@
 buildscript {
     ext {
         buildToolsVersion = "33.0.0"
-        minSdkVersion = 21
+        minSdkVersion = 26
         compileSdkVersion = 33
         targetSdkVersion = 33
 

package/android/CMakeLists.txt

@@ -47,6 +47,7 @@ add_library(
         "${PROJECT_SOURCE_DIR}/cpp/rnskia-android/RNSkOpenGLCanvasProvider.cpp"
         "${PROJECT_SOURCE_DIR}/cpp/rnskia-android/SkiaOpenGLSurfaceFactory.cpp"
         "${PROJECT_SOURCE_DIR}/cpp/rnskia-android/GrAHardwareBufferUtils.cpp"
+        "${PROJECT_SOURCE_DIR}/cpp/rnskia-android/AHardwareBufferUtils.cpp"
 
         "${PROJECT_SOURCE_DIR}/../cpp/jsi/JsiHostObject.cpp"
         "${PROJECT_SOURCE_DIR}/../cpp/jsi/JsiValue.cpp"

package/android/cpp/rnskia-android/AHardwareBufferUtils.cpp

@@ -0,0 +1,31 @@
+#if __ANDROID_API__ >= 26
+
+#include "AHardwareBufferUtils.h"
+#include <android/hardware_buffer.h>
+
+namespace RNSkia {
+
+uint32_t GetBufferFormatFromSkColorType(SkColorType bufferFormat) {
+  switch (bufferFormat) {
+  case kRGBA_8888_SkColorType:
+    return AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
+  case kRGB_888x_SkColorType:
+    return AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM;
+  case kRGBA_F16_SkColorType:
+    return AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT;
+  case kRGB_565_SkColorType:
+    return AHARDWAREBUFFER_FORMAT_R5G6B5_UNORM;
+  case kRGBA_1010102_SkColorType:
+    return AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM;
+#if __ANDROID_API__ >= 33
+  case kAlpha_8_SkColorType:
+    return AHARDWAREBUFFER_FORMAT_R8_UNORM;
+#endif
+  default:
+    return AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
+  }
+}
+
+} // namespace RNSkia
+
+#endif

package/android/cpp/rnskia-android/AHardwareBufferUtils.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "include/core/SkColorType.h"
+
+#if __ANDROID_API__ >= 26
+
+namespace RNSkia {
+
+uint32_t GetBufferFormatFromSkColorType(SkColorType bufferFormat);
+
+} // namespace RNSkia
+
+#endif

package/android/cpp/rnskia-android/RNSkAndroidPlatformContext.h

@@ -1,10 +1,15 @@
 #pragma once
 
+// TODO: Add android flags
+#if __ANDROID_API__ >= 26
+#include <android/hardware_buffer.h>
+#endif
 #include <exception>
 #include <functional>
 #include <memory>
 #include <string>
 
+#include "AHardwareBufferUtils.h"
 #include "JniPlatformContext.h"
 #include "RNSkPlatformContext.h"
 #include "SkiaOpenGLSurfaceFactory.h"
@@ -52,6 +57,72 @@ class RNSkAndroidPlatformContext : public RNSkPlatformContext {
     return SkiaOpenGLSurfaceFactory::makeImageFromHardwareBuffer(buffer);
   }
 
+  void releasePlatformBuffer(uint64_t pointer) override {
+#if __ANDROID_API__ >= 26
+    AHardwareBuffer *buffer = reinterpret_cast<AHardwareBuffer *>(pointer);
+    AHardwareBuffer_release(buffer);
+#endif
+  }
+
+  uint64_t makePlatformBuffer(sk_sp<SkImage> image) override {
+#if __ANDROID_API__ >= 26
+    auto bytesPerPixel = image->imageInfo().bytesPerPixel();
+    int bytesPerRow = image->width() * bytesPerPixel;
+    auto buf = SkData::MakeUninitialized(image->width() * image->height() *
+                                         bytesPerPixel);
+    SkImageInfo info =
+        SkImageInfo::Make(image->width(), image->height(), image->colorType(),
+                          image->alphaType());
+    image->readPixels(nullptr, info, const_cast<void *>(buf->data()),
+                      bytesPerRow, 0, 0);
+    const void *pixelData = buf->data();
+
+    // Define the buffer description
+    AHardwareBuffer_Desc desc = {};
+    // TODO: use image info here
+    desc.width = image->width();
+    desc.height = image->height();
+    desc.layers = 1; // Single image layer
+    desc.format = GetBufferFormatFromSkColorType(
+        image->colorType()); // Assuming the image
+                             // is in this format
+    desc.usage = AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN |
+                 AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN |
+                 AHARDWAREBUFFER_USAGE_GPU_COLOR_OUTPUT |
+                 AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE;
+    desc.stride = bytesPerRow; // Stride in pixels, not in bytes
+
+    // Allocate the buffer
+    AHardwareBuffer *buffer = nullptr;
+    if (AHardwareBuffer_allocate(&desc, &buffer) != 0) {
+      // Handle allocation failure
+      return 0;
+    }
+
+    // Map the buffer to gain access to its memory
+    void *mappedBuffer = nullptr;
+    AHardwareBuffer_lock(buffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN, -1,
+                         nullptr, &mappedBuffer);
+    if (mappedBuffer == nullptr) {
+      // Handle mapping failure
+      AHardwareBuffer_release(buffer);
+      return 0;
+    }
+
+    // Copy the image data to the buffer
+    memcpy(mappedBuffer, pixelData, desc.height * bytesPerRow);
+
+    // Unmap the buffer
+    AHardwareBuffer_unlock(buffer, nullptr);
+
+    // Return the buffer pointer as a uint64_t. It's the caller's responsibility
+    // to manage this buffer.
+    return reinterpret_cast<uint64_t>(buffer);
+#else
+    return 0;
+#endif
+  }
+
   sk_sp<SkFontMgr> createFontMgr() override {
     return SkFontMgr_New_Android(nullptr);
   }

package/android/cpp/rnskia-android/SkiaOpenGLSurfaceFactory.cpp

@@ -18,6 +18,15 @@ thread_local SkiaOpenGLContext ThreadContextHolder::ThreadSkiaOpenGLContext;
 sk_sp<SkImage>
 SkiaOpenGLSurfaceFactory::makeImageFromHardwareBuffer(void *buffer) {
 #if __ANDROID_API__ >= 26
+  // Setup OpenGL and Skia:
+  if (!SkiaOpenGLHelper::createSkiaDirectContextIfNecessary(
+          &ThreadContextHolder::ThreadSkiaOpenGLContext)) {
+
+    RNSkLogger::logToConsole(
+        "Could not create Skia Surface from native window / surface. "
+        "Failed creating Skia Direct Context");
+    return nullptr;
+  }
   const AHardwareBuffer *hardwareBuffer =
       static_cast<AHardwareBuffer *>(buffer);
   DeleteImageProc deleteImageProc = nullptr;

package/cpp/api/JsiPlatformBuffer.h

@@ -0,0 +1,51 @@
+#pragma once
+
+#include <memory>
+#include <utility>
+
+#include <jsi/jsi.h>
+
+#include "JsiSkImage.h"
+
+namespace RNSkia {
+
+namespace jsi = facebook::jsi;
+
+/**
+ Implementation of the ParagraphBuilderFactory for making ParagraphBuilder JSI
+ object
+ */
+class JsiPlatformBufferFactory : public JsiSkHostObject {
+public:
+  JSI_HOST_FUNCTION(MakeFromImage) {
+    auto image = JsiSkImage::fromValue(runtime, arguments[0]);
+    image->makeNonTextureImage();
+
+    uint64_t pointer = getContext()->makePlatformBuffer(image);
+    jsi::HostFunctionType deleteFunc =
+        [=](jsi::Runtime &runtime, const jsi::Value &thisArg,
+            const jsi::Value *args, size_t count) -> jsi::Value {
+      getContext()->releasePlatformBuffer(pointer);
+      return jsi::Value::undefined();
+    };
+    return jsi::BigInt::fromUint64(runtime, pointer);
+  }
+
+  JSI_HOST_FUNCTION(Release) {
+
+    jsi::BigInt pointer = arguments[0].asBigInt(runtime);
+    const uintptr_t platformBufferPointer = pointer.asUint64(runtime);
+
+    getContext()->releasePlatformBuffer(platformBufferPointer);
+    return jsi::Value::undefined();
+  }
+
+  JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiPlatformBufferFactory, Release),
+                       JSI_EXPORT_FUNC(JsiPlatformBufferFactory, MakeFromImage))
+
+  explicit JsiPlatformBufferFactory(
+      std::shared_ptr<RNSkPlatformContext> context)
+      : JsiSkHostObject(std::move(context)) {}
+};
+
+} // namespace RNSkia

package/cpp/api/JsiSkApi.h

@@ -6,6 +6,7 @@
 
 #include "JsiSkHostObjects.h"
 
+#include "JsiPlatformBuffer.h"
 #include "JsiSkAnimatedImage.h"
 #include "JsiSkAnimatedImageFactory.h"
 #include "JsiSkColor.h"
@@ -122,6 +123,9 @@ class JsiSkApi : public JsiSkHostObject {
     installReadonlyProperty(
         "ParagraphBuilder",
         std::make_shared<JsiSkParagraphBuilderFactory>(context));
+
+    installReadonlyProperty(
+        "PlatformBuffer", std::make_shared<JsiPlatformBufferFactory>(context));
   }
 };
 } // namespace RNSkia

package/cpp/api/JsiSkImageFactory.h

@@ -82,7 +82,7 @@ class JsiSkImageFactory : public JsiSkHostObject {
                        JSI_EXPORT_FUNC(JsiSkImageFactory, MakeImageFromViewTag),
                        JSI_EXPORT_FUNC(JsiSkImageFactory,
                                        MakeImageFromPlatformBuffer),
-                       JSI_EXPORT_FUNC(JsiSkImageFactory, MakeImage), )
+                       JSI_EXPORT_FUNC(JsiSkImageFactory, MakeImage))
 
   explicit JsiSkImageFactory(std::shared_ptr<RNSkPlatformContext> context)
       : JsiSkHostObject(std::move(context)) {}

package/cpp/rnskia/RNSkPlatformContext.h

@@ -142,6 +142,10 @@ class RNSkPlatformContext {
    */
   virtual sk_sp<SkImage> makeImageFromPlatformBuffer(void *buffer) = 0;
 
+  virtual void releasePlatformBuffer(uint64_t pointer) = 0;
+
+  virtual uint64_t makePlatformBuffer(sk_sp<SkImage> image) = 0;
+
   /**
    * Return the Platform specific font manager
    */

package/ios/RNSkia-iOS/RNSkiOSPlatformContext.h

@@ -61,6 +61,10 @@ class RNSkiOSPlatformContext : public RNSkPlatformContext {
 
   sk_sp<SkImage> makeImageFromPlatformBuffer(void *buffer) override;
 
+  uint64_t makePlatformBuffer(sk_sp<SkImage> image) override;
+
+  void releasePlatformBuffer(uint64_t pointer) override;
+
   virtual void performStreamOperation(
       const std::string &sourceUri,
       const std::function<void(std::unique_ptr<SkStreamAsset>)> &op) override;

package/ios/RNSkia-iOS/RNSkiOSPlatformContext.mm

@@ -57,6 +57,89 @@
   std::thread(loader).detach();
 }
 
+void RNSkiOSPlatformContext::releasePlatformBuffer(uint64_t pointer) {
+  CMSampleBufferRef sampleBuffer = reinterpret_cast<CMSampleBufferRef>(pointer);
+  CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+  if (sampleBuffer) {
+    CFRelease(sampleBuffer);
+  }
+  if (pixelBuffer) {
+    CFRelease(pixelBuffer);
+  }
+}
+
+uint64_t RNSkiOSPlatformContext::makePlatformBuffer(sk_sp<SkImage> image) {
+  auto bytesPerPixel = image->imageInfo().bytesPerPixel();
+  int bytesPerRow = image->width() * bytesPerPixel;
+  auto buf = SkData::MakeUninitialized(image->width() * image->height() *
+                                       bytesPerPixel);
+  SkImageInfo info = SkImageInfo::Make(image->width(), image->height(),
+                                       image->colorType(), image->alphaType());
+  image->readPixels(nullptr, info, const_cast<void *>(buf->data()), bytesPerRow,
+                    0, 0);
+  auto pixelData = const_cast<void *>(buf->data());
+
+  // Create a CVPixelBuffer from the raw pixel data
+  CVPixelBufferRef pixelBuffer = nullptr;
+  // OSType pixelFormatType = MapSkColorTypeToOSType(image->colorType());
+
+  // You will need to fill in the details for creating the pixel buffer
+  // CVPixelBufferCreateWithBytes or CVPixelBufferCreateWithPlanarBytes
+  // Create the CVPixelBuffer with the image data
+  void *context = static_cast<void *>(
+      new sk_sp<SkData>(buf)); // Create a copy for the context
+  CVReturn r = CVPixelBufferCreateWithBytes(
+      nullptr, // allocator
+      image->width(), image->height(), kCVPixelFormatType_32BGRA,
+      pixelData,                                         // pixel data
+      bytesPerRow,                                       // bytes per row
+      [](void *releaseRefCon, const void *baseAddress) { // release callback
+        auto buf = static_cast<sk_sp<SkData> *>(releaseRefCon);
+        buf->reset(); // This effectively calls unref on the SkData object
+        delete buf;   // Cleanup the dynamically allocated context
+      },
+      context,     // release callback context
+      nullptr,     // pixel buffer attributes
+      &pixelBuffer // the newly created pixel buffer
+  );
+
+  if (r != kCVReturnSuccess) {
+    return 0; // or handle error appropriately
+  }
+
+  // Wrap the CVPixelBuffer in a CMSampleBuffer
+  CMSampleBufferRef sampleBuffer = nullptr;
+
+  CMFormatDescriptionRef formatDescription = nullptr;
+  CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer,
+                                               &formatDescription);
+
+  // Assuming no specific timing is required, we initialize the timing info to
+  // zero.
+  CMSampleTimingInfo timingInfo = {0};
+  timingInfo.duration = kCMTimeInvalid; // Indicate an unknown duration.
+  timingInfo.presentationTimeStamp = kCMTimeZero; // Start at time zero.
+  timingInfo.decodeTimeStamp = kCMTimeInvalid;    // No specific decode time.
+
+  // Create the sample buffer.
+  OSStatus status = CMSampleBufferCreateReadyWithImageBuffer(
+      kCFAllocatorDefault, pixelBuffer, formatDescription, &timingInfo,
+      &sampleBuffer);
+
+  if (status != noErr) {
+    if (formatDescription) {
+      CFRelease(formatDescription);
+    }
+    if (pixelBuffer) {
+      CFRelease(pixelBuffer);
+    }
+    return 0;
+  }
+
+  // Return sampleBuffer casted to uint64_t
+  return reinterpret_cast<uint64_t>(sampleBuffer);
+}
+
 void RNSkiOSPlatformContext::raiseError(const std::exception &err) {
   RCTFatal(RCTErrorWithMessage([NSString stringWithUTF8String:err.what()]));
 }
@@ -69,7 +152,32 @@
 sk_sp<SkImage>
 RNSkiOSPlatformContext::makeImageFromPlatformBuffer(void *buffer) {
   CMSampleBufferRef sampleBuffer = (CMSampleBufferRef)buffer;
-  return SkiaMetalSurfaceFactory::makeImageFromCMSampleBuffer(sampleBuffer);
+  // DO the CPU transfer (debugging only)
+  //  Step 1: Extract the CVPixelBufferRef from the CMSampleBufferRef
+  CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+
+  // Step 2: Lock the pixel buffer to access the raw pixel data
+  CVPixelBufferLockBaseAddress(pixelBuffer, 0);
+
+  // Step 3: Get information about the image
+  void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer);
+  size_t width = CVPixelBufferGetWidth(pixelBuffer);
+  size_t height = CVPixelBufferGetHeight(pixelBuffer);
+  size_t bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
+
+  // Assuming the pixel format is 32BGRA, which is common for iOS video frames.
+  // You might need to adjust this based on the actual pixel format.
+  SkImageInfo info = SkImageInfo::Make(width, height, kRGBA_8888_SkColorType,
+                                       kUnpremul_SkAlphaType);
+
+  // Step 4: Create an SkImage from the pixel buffer
+  sk_sp<SkData> data =
+      SkData::MakeWithoutCopy(baseAddress, height * bytesPerRow);
+  sk_sp<SkImage> image = SkImages::RasterFromData(info, data, bytesPerRow);
+  auto texture = SkiaMetalSurfaceFactory::makeTextureFromImage(image);
+  // Step 5: Unlock the pixel buffer
+  CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
+  return texture;
 }
 
 sk_sp<SkFontMgr> RNSkiOSPlatformContext::createFontMgr() {