Fix iOS native capture and release Flutter 0.8.17

docs/releases/compatibility.md

@@ -2,6 +2,7 @@
 
 | Adapter | Adapter version | Native runtime |
 | --- | --- | --- |
+| Flutter `tugboat` | 0.8.17 | Android `capture-runtime` `0.1.0`; Apple `TugboatCaptureRuntime` `0.1.1`; blank iOS engine captures retry with explicit hierarchy coverage. |
 | Apple `TugboatCaptureRuntime` | 0.1.1 | Rejects transparent and near-white captures and validates explicit view-hierarchy capture before encoding. |
 | Flutter `tugboat` | 0.8.16 | Same hosted runtimes as 0.8.15. iOS plugin looks up `registrar.viewController` at runtime so Flutter 3.35 hosts compile. |
 | Flutter `tugboat` | 0.8.15 | Android `com.gettugboat.sdk:capture-runtime:0.1.0` from Maven Central. Apple `TugboatCaptureRuntime` `0.1.0` from CocoaPods trunk. Plugin iOS floor 15. |

sdks/flutter/packages/tugboat/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 0.8.17
+
+### Fixed
+
+- Reject blank iOS engine-surface captures and retry with explicit hierarchy
+  capture before falling back to Flutter capture.
+- Resolve the live `FlutterView` without selecting an unrelated controller
+  view.
+
+### Changed
+
+- Raise the default degraded capture scale from `0.67` to `0.80` to retain
+  more screenshot detail when the capture budget is degraded.
+- Depend on CocoaPods `TugboatCaptureRuntime` `0.1.1`.
+
 ## 0.8.16
 
 ### Fixed

sdks/flutter/packages/tugboat/android/build.gradle

@@ -1,5 +1,5 @@
 group = "com.tugboat.flutter"
-version = "0.8.16"
+version = "0.8.17"
 
 buildscript {
     ext.kotlin_version = "2.2.20"

sdks/flutter/packages/tugboat/example/pubspec.yaml

@@ -32,7 +32,7 @@ resolution: workspace
 dependencies:
   flutter:
     sdk: flutter
-  tugboat: ^0.8.16
+  tugboat: ^0.8.17
 
   # The following adds the Cupertino Icons font to your application.
   # Use with the CupertinoIcons class for iOS style icons.

sdks/flutter/packages/tugboat/ios/Classes/TugboatPlugin.swift

@@ -3,65 +3,158 @@ import TugboatCaptureRuntime
 import UIKit
 
 public class TugboatPlugin: NSObject, FlutterPlugin, NativeCaptureHostApi {
-  private var runtime: CaptureRuntime?
+  private var engineRuntime: CaptureRuntime?
+  private var hierarchyRuntime: CaptureRuntime?
   private weak var registrar: FlutterPluginRegistrar?
   private let callbackQueue = DispatchQueue.main
+  private let stateLock = NSLock()
+  private var disposed = false
 
   public static func register(with registrar: FlutterPluginRegistrar) {
     let instance = TugboatPlugin()
     instance.registrar = registrar
-    instance.runtime = CaptureRuntime()
+    instance.engineRuntime = CaptureRuntime()
     NativeCaptureHostApiSetup.setUp(binaryMessenger: registrar.messenger(), api: instance)
   }
 
   func getCapabilities() throws -> NativeCaptureCapabilities {
-    NativeCaptureMapping.capabilities(requireRuntime().capabilities())
+    guard let runtime = engineCaptureRuntime() else {
+      return NativeCaptureCapabilities(
+        nativeCaptureSupported: false,
+        apiLevel: Int64(ProcessInfo.processInfo.operatingSystemVersion.majorVersion),
+        minNativeApi: Int64(CaptureRuntime.minNativeApi)
+      )
+    }
+    return NativeCaptureMapping.capabilities(runtime.capabilities())
   }
 
   func capture(
     request: NativeCaptureRequest,
     completion: @escaping (Result<NativeCaptureResult, Error>) -> Void
   ) {
+    guard let runtime = engineCaptureRuntime() else {
+      complete(NativeCaptureMapping.failed(request, .disposed), completion: completion)
+      return
+    }
     guard let view = flutterView() else {
-      completion(.success(NativeCaptureMapping.failed(request, .surfaceUnavailable)))
+      complete(
+        NativeCaptureMapping.failed(request, .surfaceUnavailable),
+        completion: completion
+      )
       return
     }
-    requireRuntime().capture(view: view, request: NativeCaptureMapping.request(request)) { result in
-      self.callbackQueue.async {
-        completion(.success(NativeCaptureMapping.result(result)))
+    let started = DispatchTime.now()
+    let nativeRequest = NativeCaptureMapping.request(request)
+    runtime.capture(view: view, request: nativeRequest) { result in
+      guard result.status == .pixelCopyFailed else {
+        self.complete(result, completion: completion)
+        return
+      }
+
+      let remainingMs = self.remainingTimeoutMs(since: started)
+      guard remainingMs > 0 else {
+        self.complete(NativeCaptureMapping.failed(request, .timeout), completion: completion)
+        return
+      }
+      guard let hierarchyRuntime = self.makeHierarchyRuntime(timeoutMs: remainingMs) else {
+        self.complete(NativeCaptureMapping.failed(request, .disposed), completion: completion)
+        return
+      }
+      hierarchyRuntime.capture(view: view, request: nativeRequest) { retry in
+        self.complete(retry, completion: completion)
       }
     }
   }
 
   func cancel(requestId: Int64) throws {
-    runtime?.cancel(requestId: requestId)
+    let runtimes = captureRuntimes()
+    runtimes.engine?.cancel(requestId: requestId)
+    runtimes.hierarchy?.cancel(requestId: requestId)
   }
 
   func dispose() throws {
-    runtime?.dispose()
-    runtime = nil
+    stateLock.lock()
+    disposed = true
+    let engine = engineRuntime
+    let hierarchy = hierarchyRuntime
+    engineRuntime = nil
+    hierarchyRuntime = nil
+    stateLock.unlock()
+    engine?.dispose()
+    hierarchy?.dispose()
+  }
+
+  private func complete(
+    _ result: CaptureResult,
+    completion: @escaping (Result<NativeCaptureResult, Error>) -> Void
+  ) {
+    complete(NativeCaptureMapping.result(result), completion: completion)
+  }
+
+  private func complete(
+    _ result: NativeCaptureResult,
+    completion: @escaping (Result<NativeCaptureResult, Error>) -> Void
+  ) {
+    callbackQueue.async {
+      completion(.success(result))
+    }
   }
 
-  private func requireRuntime() -> CaptureRuntime {
-    if let runtime {
-      return runtime
+  private func engineCaptureRuntime() -> CaptureRuntime? {
+    stateLock.lock()
+    defer { stateLock.unlock() }
+    guard !disposed else { return nil }
+    if let engineRuntime {
+      return engineRuntime
     }
     let created = CaptureRuntime()
-    runtime = created
+    engineRuntime = created
     return created
   }
 
+  /// The controller serializes capture requests. Replace the prior retry runtime
+  /// so this attempt uses only the time left in the original request budget.
+  private func makeHierarchyRuntime(timeoutMs: Int64) -> CaptureRuntime? {
+    stateLock.lock()
+    defer { stateLock.unlock() }
+    guard !disposed else { return nil }
+    let created = CaptureRuntime(timeoutMs: timeoutMs, coverage: .viewHierarchy)
+    hierarchyRuntime?.dispose()
+    hierarchyRuntime = created
+    return created
+  }
+
+  private func captureRuntimes() -> (engine: CaptureRuntime?, hierarchy: CaptureRuntime?) {
+    stateLock.lock()
+    defer { stateLock.unlock() }
+    return (engineRuntime, hierarchyRuntime)
+  }
+
+  private func remainingTimeoutMs(since started: DispatchTime) -> Int64 {
+    let elapsedMs = Int64(
+      (DispatchTime.now().uptimeNanoseconds - started.uptimeNanoseconds) / 1_000_000
+    )
+    return max(0, CaptureRuntime.defaultTimeoutMs - elapsedMs)
+  }
+
   private func flutterView() -> UIView? {
-    guard let controller = hostViewController() else {
-      return nil
+    if let controller = registrarViewController(),
+      let view = findFlutterView(controller.view)
+    {
+      return view
     }
-    return findFlutterView(controller.view) ?? controller.view
+
+    for window in foregroundWindows() {
+      if let view = findFlutterView(window) {
+        return view
+      }
+    }
+    return nil
   }
 
   /// Flutter 3.38+ exposes `registrar.viewController`. The package floor is
-  /// Flutter 3.35, which does not declare that property, so look it up at
-  /// runtime and fall back to the key window.
-  private func hostViewController() -> UIViewController? {
+  /// Flutter 3.35, which does not declare that property, so look it up at runtime.
+  private func registrarViewController() -> UIViewController? {
     if let registrar {
       let selector = NSSelectorFromString("viewController")
       if registrar.responds(to: selector),
@@ -71,15 +164,27 @@ public class TugboatPlugin: NSObject, FlutterPlugin, NativeCaptureHostApi {
         return controller
       }
     }
-    return keyWindowRootViewController()
+    return nil
   }
 
-  private func keyWindowRootViewController() -> UIViewController? {
-    let windows = UIApplication.shared.connectedScenes
+  /// Prefer the active key window. Do not use an arbitrary controller view when
+  /// FlutterView is absent because that can produce a valid but unrelated frame.
+  private func foregroundWindows() -> [UIWindow] {
+    let scenes = UIApplication.shared.connectedScenes
       .compactMap { $0 as? UIWindowScene }
-      .flatMap(\.windows)
-    let keyWindow = windows.first(where: \.isKeyWindow) ?? windows.first
-    return keyWindow?.rootViewController
+    let activeScenes = scenes.filter { $0.activationState == .foregroundActive }
+    let inactiveScenes = scenes.filter { $0.activationState == .foregroundInactive }
+
+    return (activeScenes + inactiveScenes).flatMap { scene in
+      scene.windows
+        .filter { !$0.isHidden && $0.alpha > 0 }
+        .sorted { left, right in
+          if left.isKeyWindow != right.isKeyWindow {
+            return left.isKeyWindow
+          }
+          return left.windowLevel.rawValue < right.windowLevel.rawValue
+        }
+    }
   }
 
   private func findFlutterView(_ view: UIView) -> UIView? {

sdks/flutter/packages/tugboat/ios/tugboat.podspec

@@ -4,7 +4,7 @@
 #
 Pod::Spec.new do |s|
   s.name             = 'tugboat'
-  s.version          = '0.8.16'
+  s.version          = '0.8.17'
   s.summary          = 'Screenshot-based session replay with compact interaction anchors for Tugboat.'
   s.description      = <<-DESC
     Flutter adapter for Tugboat session replay. Native CPU capture is experimental
@@ -17,7 +17,7 @@ depends on CocoaPods TugboatCaptureRuntime and requires iOS 15.
   s.author           = { 'Tugboat' => 'dev@bijatech.com' }
   s.source           = { :path => '.' }
   s.dependency 'Flutter'
-  s.dependency 'TugboatCaptureRuntime', '0.1.0'
+  s.dependency 'TugboatCaptureRuntime', '0.1.1'
   s.platform = :ios, '15.0'
   s.static_framework = true
   s.swift_version = '5.9'

sdks/flutter/packages/tugboat/lib/src/replay_config.dart

@@ -94,7 +94,7 @@ class TugboatReplayConfig {
     this.capturePixelRatio = 0.75,
     this.captureMaxWidth,
     this.captureMaxHeight,
-    this.degradedCaptureScale = 0.67,
+    this.degradedCaptureScale = 0.80,
     this.enableGlobalPointerCapture = true,
     this.explorationCollectorUrl,
     this.explorationRunId,

sdks/flutter/packages/tugboat/lib/src/sdk_version.dart

@@ -1,3 +1,3 @@
 // Keep this in sync with this package's pubspec.yaml. The SDK version test
 // reads pubspec.yaml directly so release bumps fail fast if this drifts.
-const tugboatSdkVersion = '0.8.16';
+const tugboatSdkVersion = '0.8.17';

sdks/flutter/packages/tugboat/pubspec.yaml

@@ -1,7 +1,7 @@
 name: tugboat
 description: >-
   Screenshot-based session replay with compact interaction anchors for Tugboat.
-version: 0.8.16
+version: 0.8.17
 repository: https://github.com/blendto/tugboat-flutter
 issue_tracker: https://github.com/blendto/tugboat-flutter/issues
 homepage: https://github.com/blendto/tugboat-flutter

sdks/flutter/packages/tugboat/test/replay/native_cpu_backend_test.dart

@@ -213,6 +213,34 @@ void main() {
     },
   );
 
+  test('native hierarchy retry reports its actual coverage', () async {
+    final api = _FakeHostApi(
+      captureHandler: (request) async => nativeCaptureResult(
+        requestId: request.requestId,
+        status: NativeCaptureStatus.ok,
+        jpeg: Uint8List.fromList(const [1, 2, 3, 4]),
+        width: 8,
+        height: 8,
+        dHash: '1' * 64,
+        contentHash: 'abc',
+        coverage: NativeCaptureCoverage.viewHierarchy,
+      ),
+    );
+    final fallback = _RecordingFallback();
+    final source = NativeCpuExperimentalPixelSource(
+      client: NativeCaptureClient(api: api),
+      fallback: fallback,
+    );
+
+    final result = await source.acquire(
+      _pixelRequest(boundary: RenderRepaintBoundary()),
+    );
+
+    expect(fallback.calls, 0);
+    expect(result.disposition, ScreenshotPixelDisposition.captured);
+    expect(result.trace.coverage, 'viewHierarchy');
+  });
+
   test('native fallback status uses Flutter path once', () async {
     final api = _FakeHostApi(
       captureHandler: (request) async => nativeCaptureResult(

sdks/flutter/packages/tugboat/test/replay/screenshot_fresh_paint_test.dart

@@ -428,6 +428,36 @@ void main() {
     expect(capture.result!.height, 160);
   });
 
+  testWidgets('default degraded scale retains more screenshot detail', (
+    tester,
+  ) async {
+    const config = TugboatReplayConfig();
+    final boundaryKey = GlobalKey();
+    final capturer = ScreenshotCapturer(
+      boundaryKey: boundaryKey,
+      maskLevel: TugboatScreenshotMaskLevel.explicitOnly,
+      anchorResolver: AnchorResolver(rootKey: boundaryKey),
+      pixelRatio: config.capturePixelRatio,
+      degradedScale: config.degradedCaptureScale,
+      frameWaiter: () => Future<void>.value(),
+      encoder: InlineScreenshotEncoder(),
+    );
+    addTearDown(capturer.dispose);
+    await tester.pumpWidget(_scene(boundaryKey, Colors.red));
+
+    final degraded = await tester.runAsync(
+      () => capturer.captureAttempt(force: true, degraded: true),
+    );
+
+    expect(degraded, isNotNull);
+    expect(degraded!.result, isNotNull);
+    final expectedRatio =
+        config.capturePixelRatio * config.degradedCaptureScale;
+    final expectedDimension = (80 * expectedRatio).ceil();
+    expect(degraded.result!.width, expectedDimension);
+    expect(degraded.result!.height, expectedDimension);
+  });
+
   test('copyWith can clear screenshot dimension bounds', () {
     const bounded = TugboatReplayConfig(
       captureMaxWidth: 540,

sdks/flutter/packages/tugboat_dio/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.8.17
+
+### Changed
+
+- Compatibility release for `tugboat` 0.8.17. The Dio adapter has no runtime
+  behavior change.
+
 ## 0.8.16
 
 ### Changed

sdks/flutter/packages/tugboat_dio/pubspec.yaml

@@ -2,7 +2,7 @@ name: tugboat_dio
 description: >-
   Dio interceptor that records safe, bounded network evidence into an active
   Tugboat capture session.
-version: 0.8.16
+version: 0.8.17
 repository: https://github.com/blendto/tugboat-flutter
 issue_tracker: https://github.com/blendto/tugboat-flutter/issues
 homepage: https://github.com/blendto/tugboat-flutter
@@ -18,7 +18,7 @@ dependencies:
   dio: ^5.4.0
   flutter:
     sdk: flutter
-  tugboat: ^0.8.16
+  tugboat: ^0.8.17
 
 dev_dependencies:
   flutter_lints: ^5.0.0