Fixes

Author: aritchie

src/Shiny.Maui.Controls.Camera/CameraView.Properties.cs

@@ -60,6 +60,39 @@ public partial class CameraView
     public static readonly BindableProperty ShowDetectionOverlayProperty = BindableProperty.Create(
         nameof(ShowDetectionOverlay), typeof(bool), typeof(CameraView), true);
 
+    /// <summary>
+    /// Whether a frame analyzer is handed the frame with the effects on it. Default <c>false</c>.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// Off by default because an analyzer is usually a detector, and a detector wants the picture
+    /// the sensor produced: a barcode reader given a Noir grade, or a face detector given an
+    /// inverted frame, is being asked to work against the effect rather than with it.
+    /// </para>
+    /// <para>
+    /// On for the other kind of analyzer - the one that is a second pair of eyes rather than a
+    /// detector. A remote viewfinder taps the frame stream to show what the camera is seeing on
+    /// another screen, and it should show what the local preview shows and what the captured file
+    /// will contain, effects included. Without this it is the only surface in the app that quietly
+    /// disagrees with the other two.
+    /// </para>
+    /// <para>
+    /// It costs a render into a reusable buffer per delivered frame, and only while there is an
+    /// effect chain to apply - with no effects, the analyzer gets the buffer straight from the
+    /// camera either way.
+    /// </para>
+    /// </remarks>
+    public static readonly BindableProperty AnalyzerSeesEffectsProperty = BindableProperty.Create(
+        nameof(AnalyzerSeesEffects), typeof(bool), typeof(CameraView), false,
+        propertyChanged: (b, _, _) => ((CameraView)b).RebuildEffectChain());
+
+    /// <inheritdoc cref="AnalyzerSeesEffectsProperty"/>
+    public bool AnalyzerSeesEffects
+    {
+        get => (bool)this.GetValue(AnalyzerSeesEffectsProperty);
+        set => this.SetValue(AnalyzerSeesEffectsProperty, value);
+    }
+
     /// <summary>
     /// Live color filter applied to the preview. Default <see cref="CameraFilter.None"/>.
     /// </summary>

src/Shiny.Maui.Controls.Camera/CameraViewHandler.cs

@@ -63,6 +63,7 @@ private protected void TeardownPipeline()
             [nameof(CameraView.ScanWindow)] = MapOverlay,
             [nameof(CameraView.Analyzer)] = MapAnalyzer,
             [nameof(CameraView.Filter)] = MapEffects,
+            [nameof(CameraView.AnalyzerSeesEffects)] = MapEffects,
             [nameof(CameraView.Effects)] = MapEffects,
             [nameof(CameraView.PhotoQuality)] = MapPhotoQuality,
             [nameof(CameraView.VideoQuality)] = MapVideoQuality,

src/Shiny.Maui.Controls.Camera/Platforms/Apple/CameraViewHandler.Apple.cs

@@ -882,6 +882,7 @@ void ApplyEffects(CameraEffectChain chain)
 
         var filters = AppleCameraFilters.Create(chain);
         this.frameDelegate.Filters = filters;
+        this.frameDelegate.AnalyzerSeesEffects = this.MaybeVirtualView?.AnalyzerSeesEffects == true;
         this.UpdateFrameDelivery();
 
         // Show the filtered-frame overlay on top of the live preview while any effect is active. We must NOT

src/Shiny.Maui.Controls.Camera/Platforms/Apple/VideoFrameDelegate.cs

@@ -57,6 +57,21 @@ public CIFilter[] Filters
     /// <summary>Set by the handler so wrapped frames carry mirroring metadata.</summary>
     public volatile bool Mirrored;
 
+    /// <summary>Whether the analyzer is handed the filtered frame - see CameraView.AnalyzerSeesEffects.</summary>
+    public volatile bool AnalyzerSeesEffects;
+
+    /// <summary>Where a filtered frame is drawn before it is handed on. Created on first use.</summary>
+    readonly FilteredFrameBuffer filtered = new();
+
+    /// <summary>Releases the scratch buffer with the delegate that owns it.</summary>
+    protected override void Dispose(bool disposing)
+    {
+        if (disposing)
+            this.filtered.Dispose();
+
+        base.Dispose(disposing);
+    }
+
     /// <summary>When set, each frame is composited with the overlay and appended to the burn-in recording.</summary>
     public volatile AppleVideoOverlayRecorder? Recorder;
 
@@ -78,6 +93,19 @@ public override void DidOutputSampleBuffer(AVCaptureOutput captureOutput, CMSamp
                 var recorder = this.Recorder;
                 if (this.OnFrame != null && this.WantFrames?.Invoke() == true)
                 {
+                    // The frame as the preview draws it, for an analyzer that has asked to see what
+                    // the camera looks like rather than what the sensor produced. The buffer is this
+                    // delegate's own and is reused, which is safe because the pipeline runs one
+                    // analysis at a time - see FilteredFrameBuffer.
+                    if (this.AnalyzerSeesEffects
+                        && chain.Length > 0
+                        && this.RenderForAnalyzer(chain, pixelBuffer) is { } effected)
+                    {
+                        this.OnFrame(AppleCameraFrame.Wrap(effected, rotation: 0, mirrored: this.Mirrored));
+                        recorder?.AppendVideo(sampleBuffer);
+                        return;
+                    }
+
                     // ⚠️ Borrow only when nothing is going to write to this buffer. The recorder composites
                     // effects and the burn-in overlay back into it (AppleVideoOverlayRecorder.Composite), so
                     // with one attached a borrowed frame would be read by the analyzer on one thread while
@@ -122,6 +150,40 @@ public override void DidOutputSampleBuffer(AVCaptureOutput captureOutput, CMSamp
     // Reused across frames so a steady-state render allocates nothing here.
     readonly List<CIImage> produced = [];
 
+    /// <summary>
+    /// Runs the effect chain into the scratch buffer, for an analyzer rather than for the screen.
+    /// </summary>
+    /// <remarks>
+    /// Separate from <see cref="RenderFiltered"/> because the destinations have nothing in common:
+    /// that one produces an image for a view, this one fills a pixel buffer another consumer will
+    /// read. The recipe is identical, and both render at the source extent for the reason spelled
+    /// out there.
+    /// </remarks>
+    CVPixelBuffer? RenderForAnalyzer(CIFilter[] chain, CVPixelBuffer pixelBuffer)
+    {
+        using var input = new CIImage(pixelBuffer);
+
+        this.produced.Clear();
+        var output = AppleCameraFilters.Apply(input, chain, this.produced);
+
+        try
+        {
+            return output is null ? null : this.filtered.Render(this.context, output, input.Extent);
+        }
+        catch (Exception)
+        {
+            // One frame without the effect on it beats taking the capture pipeline down.
+            return null;
+        }
+        finally
+        {
+            foreach (var image in this.produced)
+                image.Dispose();
+
+            this.produced.Clear();
+        }
+    }
+
     void RenderFiltered(CIFilter[] chain, CVPixelBuffer pixelBuffer, UIImageView view)
     {
         using var input = new CIImage(pixelBuffer);

src/Shiny.Maui.Controls.Camera/Platforms/AppleShared/AppleCameraFrame.cs

@@ -119,6 +119,17 @@ public static AppleCameraFrame Borrow(CMSampleBuffer sampleBuffer, CVPixelBuffer
     /// the recorder writing into the original — and it replaces a full-frame CPU copy rather than adding to
     /// one.
     /// </remarks>
+    /// <summary>
+    /// Wraps a buffer this library owns and will reuse, without copying it.
+    /// </summary>
+    /// <remarks>
+    /// For the filtered frame handed to an analyzer - see <see cref="FilteredFrameBuffer"/>. There
+    /// is no CMSampleBuffer behind it and nothing to release: the buffer belongs to the delegate
+    /// that rendered it, and is not touched again until the analysis has returned.
+    /// </remarks>
+    internal static AppleCameraFrame Wrap(CVPixelBuffer pixelBuffer, int rotation, bool mirrored)
+        => new(owned: null, pixelBuffer, bgra: null, rotation, mirrored);
+
     public static AppleCameraFrame Copy(CVPixelBuffer pixelBuffer, int rotation, bool mirrored)
     {
         if (!IsBiplanarFormat(pixelBuffer.PixelFormatType))

src/Shiny.Maui.Controls.Camera/Platforms/AppleShared/FilteredFrameBuffer.cs

@@ -0,0 +1,64 @@
+using CoreGraphics;
+using CoreImage;
+using CoreVideo;
+
+namespace Shiny.Maui.Controls.Camera;
+
+/// <summary>
+/// A reusable surface for handing an analyzer the frame as the preview draws it.
+/// </summary>
+/// <remarks>
+/// <para>
+/// One buffer, reused for the life of the delegate and reallocated only when the frame size
+/// changes. That is safe because the pipeline analyses one frame at a time - <c>WantsFrame</c>
+/// refuses a frame while a pass is in flight - so the buffer handed out is never the one being
+/// rendered into.
+/// </para>
+/// <para>
+/// BGRA because that is what everything downstream of <see cref="AppleCameraFrame"/> expects, and
+/// IOSurface-backed so Core Image can render into it on the GPU rather than reading it back.
+/// </para>
+/// </remarks>
+sealed class FilteredFrameBuffer : IDisposable
+{
+    CVPixelBuffer? buffer;
+    nint width;
+    nint height;
+
+    /// <summary>Draws a filtered image into the scratch buffer, or null if it could not be made.</summary>
+    public CVPixelBuffer? Render(CIContext context, CIImage image, CGRect extent)
+    {
+        var w = (nint)extent.Width;
+        var h = (nint)extent.Height;
+
+        if (w <= 0 || h <= 0)
+            return null;
+
+        if (this.buffer is null || this.width != w || this.height != h)
+        {
+            this.buffer?.Dispose();
+            this.buffer = new CVPixelBuffer(
+                w,
+                h,
+                CVPixelFormatType.CV32BGRA,
+                new CVPixelBufferAttributes { PixelFormatType = CVPixelFormatType.CV32BGRA }
+            );
+
+            this.width = w;
+            this.height = h;
+        }
+
+        if (this.buffer is null)
+            return null;
+
+        // Rendered at the source extent for the same reason the preview is - see RenderFiltered.
+        context.Render(image, this.buffer, extent, null);
+        return this.buffer;
+    }
+
+    public void Dispose()
+    {
+        this.buffer?.Dispose();
+        this.buffer = null;
+    }
+}