Skip to content

Commit c689845

Browse files
vaindvaind
authored
feat: ensure repaint on replay screenshot capture (#2527)
* feat: ensure repaint on replay screenshot capture * chore: update changelog * linter issue
1 parent 19a9adb commit c689845

File tree

4 files changed

+123
-112
lines changed

4 files changed

+123
-112
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
- Check `SentryTracer` type in TTFD tracker ([#2508](https://github.com/getsentry/sentry-dart/pull/2508))
1717
- Warning (in a debug build) if a potentially sensitive widget is not masked or unmasked explicitly ([#2375](https://github.com/getsentry/sentry-dart/pull/2375))
18+
- Replay: ensure visual update before capturing screenshots ([#2527](https://github.com/getsentry/sentry-dart/pull/2527))
1819

1920
### Dependencies
2021

‎flutter/lib/src/native/cocoa/sentry_native_cocoa.dart

+28-15
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import 'dart:async';
22
import 'dart:ffi';
3+
import 'dart:typed_data';
34
import 'dart:ui';
45

56
import 'package:meta/meta.dart';
@@ -45,22 +46,34 @@ class SentryNativeCocoa extends SentryNativeChannel {
4546
});
4647
}
4748

48-
return _replayRecorder?.capture((screenshot) async {
49-
final image = screenshot.image;
50-
final imageData =
51-
await image.toByteData(format: ImageByteFormat.png);
52-
if (imageData != null) {
53-
options.logger(
54-
SentryLevel.debug,
55-
'Replay: captured screenshot ('
56-
'${image.width}x${image.height} pixels, '
57-
'${imageData.lengthInBytes} bytes)');
58-
return imageData.buffer.asUint8List();
59-
} else {
60-
options.logger(SentryLevel.warning,
61-
'Replay: failed to convert screenshot to PNG');
62-
}
49+
final widgetsBinding = options.bindingUtils.instance;
50+
if (widgetsBinding == null) {
51+
options.logger(SentryLevel.warning,
52+
'Replay: failed to capture screenshot, WidgetsBinding.instance is null');
53+
return null;
54+
}
55+
56+
final completer = Completer<Uint8List?>();
57+
widgetsBinding.ensureVisualUpdate();
58+
widgetsBinding.addPostFrameCallback((_) {
59+
_replayRecorder?.capture((screenshot) async {
60+
final image = screenshot.image;
61+
final imageData =
62+
await image.toByteData(format: ImageByteFormat.png);
63+
if (imageData != null) {
64+
options.logger(
65+
SentryLevel.debug,
66+
'Replay: captured screenshot ('
67+
'${image.width}x${image.height} pixels, '
68+
'${imageData.lengthInBytes} bytes)');
69+
return imageData.buffer.asUint8List();
70+
} else {
71+
options.logger(SentryLevel.warning,
72+
'Replay: failed to convert screenshot to PNG');
73+
}
74+
}).then(completer.complete, onError: completer.completeError);
6375
});
76+
return completer.future;
6477
default:
6578
throw UnimplementedError('Method ${call.method} not implemented');
6679
}

‎flutter/lib/src/replay/scheduled_recorder.dart

+79-71
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ class ScheduledScreenshotRecorder extends ReplayScreenshotRecorder {
2020
late final ScheduledScreenshotRecorderCallback _callback;
2121
var _status = _Status.running;
2222
late final Duration _frameDuration;
23-
late final _idleFrameFiller = _IdleFrameFiller(_frameDuration, _onScreenshot);
23+
// late final _idleFrameFiller = _IdleFrameFiller(_frameDuration, _onScreenshot);
2424

2525
@override
2626
@protected
@@ -35,8 +35,7 @@ class ScheduledScreenshotRecorder extends ReplayScreenshotRecorder {
3535
_frameDuration = Duration(milliseconds: 1000 ~/ config.frameRate);
3636
assert(_frameDuration.inMicroseconds > 0);
3737

38-
_scheduler = Scheduler(_frameDuration, _capture,
39-
options.bindingUtils.instance!.addPostFrameCallback);
38+
_scheduler = Scheduler(_frameDuration, _capture, _addPostFrameCallback);
4039

4140
if (callback != null) {
4241
_callback = callback;
@@ -47,6 +46,12 @@ class ScheduledScreenshotRecorder extends ReplayScreenshotRecorder {
4746
_callback = callback;
4847
}
4948

49+
void _addPostFrameCallback(FrameCallback callback) {
50+
options.bindingUtils.instance!
51+
..ensureVisualUpdate()
52+
..addPostFrameCallback(callback);
53+
}
54+
5055
void start() {
5156
assert(() {
5257
// The following fails if callback hasn't been provided
@@ -74,14 +79,15 @@ class ScheduledScreenshotRecorder extends ReplayScreenshotRecorder {
7479
Future<void> stop() async {
7580
options.logger(SentryLevel.debug, "$logName: stopping capture.");
7681
_status = _Status.stopped;
77-
await Future.wait([_scheduler.stop(), _idleFrameFiller.stop()]);
82+
await _scheduler.stop();
83+
// await Future.wait([_scheduler.stop(), _idleFrameFiller.stop()]);
7884
options.logger(SentryLevel.debug, "$logName: capture stopped.");
7985
}
8086

8187
Future<void> pause() async {
8288
if (_status == _Status.running) {
8389
_status = _Status.paused;
84-
_idleFrameFiller.pause();
90+
// _idleFrameFiller.pause();
8591
await _scheduler.stop();
8692
}
8793
}
@@ -90,7 +96,7 @@ class ScheduledScreenshotRecorder extends ReplayScreenshotRecorder {
9096
if (_status == _Status.paused) {
9197
_status = _Status.running;
9298
_startScheduler();
93-
_idleFrameFiller.resume();
99+
// _idleFrameFiller.resume();
94100
}
95101
}
96102

@@ -104,7 +110,7 @@ class ScheduledScreenshotRecorder extends ReplayScreenshotRecorder {
104110
final screenshot = ScreenshotPng(
105111
image.width, image.height, imageData, capturedScreenshot.timestamp);
106112
await _onScreenshot(screenshot, true);
107-
_idleFrameFiller.actualFrameReceived(screenshot);
113+
// _idleFrameFiller.actualFrameReceived(screenshot);
108114
} else {
109115
options.logger(
110116
SentryLevel.debug,
@@ -140,69 +146,71 @@ class ScreenshotPng {
140146

141147
const ScreenshotPng(this.width, this.height, this.data, this.timestamp);
142148
}
143-
144-
// Workaround for https://github.com/getsentry/sentry-java/issues/3677
145-
// In short: when there are no postFrameCallbacks issued by Flutter (because
146-
// there are no animations or user interactions), the replay recorder will
147-
// need to get screenshots at a fixed frame rate. This class is responsible for
148-
// filling the gaps between actual frames with the most recent frame.
149-
class _IdleFrameFiller {
150-
final Duration _interval;
151-
final ScheduledScreenshotRecorderCallback _callback;
152-
var _status = _Status.running;
153-
Future<void>? _scheduled;
154-
ScreenshotPng? _mostRecent;
155-
156-
_IdleFrameFiller(this._interval, this._callback);
157-
158-
void actualFrameReceived(ScreenshotPng screenshot) {
159-
// We store the most recent frame but only repost it when the most recent
160-
// one is the same instance (unchanged).
161-
_mostRecent = screenshot;
162-
// Also, the initial reposted frame will be delayed to allow actual frames
163-
// to cancel the reposting.
164-
repostLater(_interval * 1.5, screenshot);
165-
}
166-
167-
Future<void> stop() async {
168-
_status = _Status.stopped;
169-
final scheduled = _scheduled;
170-
_scheduled = null;
171-
_mostRecent = null;
172-
await scheduled;
173-
}
174-
175-
void pause() {
176-
if (_status == _Status.running) {
177-
_status = _Status.paused;
178-
}
179-
}
180-
181-
void resume() {
182-
if (_status == _Status.paused) {
183-
_status = _Status.running;
184-
}
185-
}
186-
187-
void repostLater(Duration delay, ScreenshotPng screenshot) {
188-
_scheduled = Future.delayed(delay, () async {
189-
if (_status == _Status.stopped) {
190-
return;
191-
}
192-
193-
// Only repost if the screenshot haven't changed.
194-
if (screenshot == _mostRecent) {
195-
if (_status == _Status.running) {
196-
// We don't strictly need to await here but it helps to reduce load.
197-
// If the callback takes a long time, we still wait between calls,
198-
// based on the configured rate.
199-
await _callback(screenshot, false);
200-
}
201-
// On subsequent frames, we stick to the actual frame rate.
202-
repostLater(_interval, screenshot);
203-
}
204-
});
205-
}
206-
}
149+
// TODO this is currently unused because we've decided to capture on every
150+
// frame. Consider removing if we don't reverse the decision in the future.
151+
152+
// /// Workaround for https://github.com/getsentry/sentry-java/issues/3677
153+
// /// In short: when there are no postFrameCallbacks issued by Flutter (because
154+
// /// there are no animations or user interactions), the replay recorder will
155+
// /// need to get screenshots at a fixed frame rate. This class is responsible for
156+
// /// filling the gaps between actual frames with the most recent frame.
157+
// class _IdleFrameFiller {
158+
// final Duration _interval;
159+
// final ScheduledScreenshotRecorderCallback _callback;
160+
// var _status = _Status.running;
161+
// Future<void>? _scheduled;
162+
// ScreenshotPng? _mostRecent;
163+
164+
// _IdleFrameFiller(this._interval, this._callback);
165+
166+
// void actualFrameReceived(ScreenshotPng screenshot) {
167+
// // We store the most recent frame but only repost it when the most recent
168+
// // one is the same instance (unchanged).
169+
// _mostRecent = screenshot;
170+
// // Also, the initial reposted frame will be delayed to allow actual frames
171+
// // to cancel the reposting.
172+
// repostLater(_interval * 1.5, screenshot);
173+
// }
174+
175+
// Future<void> stop() async {
176+
// _status = _Status.stopped;
177+
// final scheduled = _scheduled;
178+
// _scheduled = null;
179+
// _mostRecent = null;
180+
// await scheduled;
181+
// }
182+
183+
// void pause() {
184+
// if (_status == _Status.running) {
185+
// _status = _Status.paused;
186+
// }
187+
// }
188+
189+
// void resume() {
190+
// if (_status == _Status.paused) {
191+
// _status = _Status.running;
192+
// }
193+
// }
194+
195+
// void repostLater(Duration delay, ScreenshotPng screenshot) {
196+
// _scheduled = Future.delayed(delay, () async {
197+
// if (_status == _Status.stopped) {
198+
// return;
199+
// }
200+
201+
// // Only repost if the screenshot haven't changed.
202+
// if (screenshot == _mostRecent) {
203+
// if (_status == _Status.running) {
204+
// // We don't strictly need to await here but it helps to reduce load.
205+
// // If the callback takes a long time, we still wait between calls,
206+
// // based on the configured rate.
207+
// await _callback(screenshot, false);
208+
// }
209+
// // On subsequent frames, we stick to the actual frame rate.
210+
// repostLater(_interval, screenshot);
211+
// }
212+
// });
213+
// }
214+
// }
207215

208216
enum _Status { stopped, running, paused }

‎flutter/test/replay/replay_native_test.dart

+15-26
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ void main() {
2424
MockPlatform.android(),
2525
MockPlatform.iOs(),
2626
]) {
27-
group('$SentryNativeBinding ($mockPlatform)', () {
27+
group('$SentryNativeBinding (${mockPlatform.operatingSystem})', () {
2828
late SentryNativeBinding sut;
2929
late NativeChannelFixture native;
3030
late SentryFlutterOptions options;
@@ -45,7 +45,7 @@ void main() {
4545
'directory': 'dir',
4646
'width': 800,
4747
'height': 600,
48-
'frameRate': 10,
48+
'frameRate': 1000,
4949
};
5050
}
5151

@@ -116,14 +116,15 @@ void main() {
116116
testWidgets('captures images', (tester) async {
117117
await tester.runAsync(() async {
118118
when(hub.configureScope(captureAny)).thenReturn(null);
119+
await pumpTestElement(tester);
120+
pumpAndSettle() => tester.pumpAndSettle(const Duration(seconds: 1));
119121

120122
if (mockPlatform.isAndroid) {
121123
var callbackFinished = Completer<void>();
122124

123125
nextFrame({bool wait = true}) async {
124126
final future = callbackFinished.future;
125-
tester.binding.scheduleFrame();
126-
await tester.pumpAndSettle(const Duration(seconds: 1));
127+
await pumpAndSettle();
127128
await future.timeout(Duration(milliseconds: wait ? 1000 : 100),
128129
onTimeout: () {
129130
if (wait) {
@@ -132,29 +133,24 @@ void main() {
132133
});
133134
}
134135

135-
imageInfo(File file) => file.readAsBytesSync().length;
136-
137-
fileToImageMap(Iterable<File> files) =>
138-
{for (var file in files) file.path: imageInfo(file)};
136+
imageSizeBytes(File file) => file.readAsBytesSync().length;
139137

140138
final capturedImages = <String, int>{};
141139
when(native.handler('addReplayScreenshot', any))
142-
.thenAnswer((invocation) async {
140+
.thenAnswer((invocation) {
143141
final path =
144142
invocation.positionalArguments[1]["path"] as String;
145-
capturedImages[path] = imageInfo(fs.file(path));
143+
capturedImages[path] = imageSizeBytes(fs.file(path));
146144
callbackFinished.complete();
147145
callbackFinished = Completer<void>();
148146
return null;
149147
});
150148

151149
fsImages() {
152150
final files = replayDir.listSync().map((f) => f as File);
153-
return fileToImageMap(files);
151+
return {for (var f in files) f.path: imageSizeBytes(f)};
154152
}
155153

156-
await pumpTestElement(tester);
157-
158154
await nextFrame(wait: false);
159155
expect(fsImages(), isEmpty);
160156
verifyNever(native.handler('addReplayScreenshot', any));
@@ -202,24 +198,17 @@ void main() {
202198
expect(capturedImages, equals(fsImages()));
203199
expect(capturedImages.length, count);
204200
} else if (mockPlatform.isIOS) {
205-
nextFrame() async {
206-
tester.binding.scheduleFrame();
207-
await Future<void>.delayed(const Duration(milliseconds: 100));
208-
await tester.pumpAndSettle(const Duration(seconds: 1));
209-
}
210-
211-
await pumpTestElement(tester);
212-
await nextFrame();
213-
214-
final imagaData = await native.invokeFromNative(
201+
var imagaData = native.invokeFromNative(
215202
'captureReplayScreenshot', replayConfig);
216-
expect(imagaData?.lengthInBytes, greaterThan(3000));
203+
await pumpAndSettle();
204+
expect((await imagaData)?.lengthInBytes, greaterThan(3000));
217205

218206
// Happens if the session-replay rate is 0.
219207
replayConfig['replayId'] = null;
220-
final imagaData2 = await native.invokeFromNative(
208+
imagaData = native.invokeFromNative(
221209
'captureReplayScreenshot', replayConfig);
222-
expect(imagaData2?.lengthInBytes, greaterThan(3000));
210+
await pumpAndSettle();
211+
expect((await imagaData)?.lengthInBytes, greaterThan(3000));
223212
} else {
224213
fail('unsupported platform');
225214
}

0 commit comments

Comments
 (0)