feat(Session Replay): Track touches for replay (#4041)

Added a touch tracker to send touch events with replays

Author: brustolin

.swiftlint.yml

@@ -10,7 +10,6 @@ excluded:
   - SentryTestUtils/Dynamic/**
 
 only_rules:
-  #- attributes
   - class_delegate_protocol
   - closing_brace
   - closure_end_indentation
@@ -19,16 +18,13 @@ only_rules:
   - colon
   - comma
   - compiler_protocol_init
-  #- conditional_returns_on_newline
   - control_statement
   - custom_rules
   - cyclomatic_complexity
   - dynamic_inline
-  - empty_count
   - empty_parameters
   - empty_parentheses_with_trailing_closure
   - explicit_init
-  #- file_header
   - file_length
   - first_where
   - force_cast
@@ -45,8 +41,6 @@ only_rules:
   - legacy_nsgeometry_functions
   - line_length
   - mark
-  #- missing_docs
-  #- nesting
   - nimble_operator
   - number_separator
   - object_literal
@@ -65,13 +59,11 @@ only_rules:
   - shorthand_operator
   - sorted_imports
   - statement_position
-  - switch_case_on_newline
   - syntactic_sugar
   - todo
   - trailing_comma
   - trailing_newline
   - trailing_semicolon
-  #- trailing_whitespace
   - type_body_length
   - type_name
   - unused_closure_parameter
@@ -90,12 +82,11 @@ identifier_name:
     - i
     - _
   min_length:
-    - 2
+    - 1
   max_length:
     warning: 50
     error: 50
 
-    
 line_length:
   - 1000 
 type_name:

CHANGELOG.md

@@ -4,9 +4,11 @@
 
 ### Features
 
+- Add a touch tracker for replay (#4041)
 - Add enableMetricKitRawPayload (#4044)
 
 ### Fixes
+
 - `SentryCrashMonitor_CPPException.cpp` compilation using Xcode 16b1 (#4051)
 
 ## 8.28.0

Sentry.xcodeproj/project.pbxproj

@@ -891,6 +891,9 @@
 		D8B76B062808066D000A58C4 /* SentryScreenshotIntegrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8B76B042808060E000A58C4 /* SentryScreenshotIntegrationTests.swift */; };
 		D8B76B0828081461000A58C4 /* TestSentryScreenShot.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8B76B0728081461000A58C4 /* TestSentryScreenShot.swift */; };
 		D8BBD32728FD9FC00011F850 /* SentrySwift.h in Headers */ = {isa = PBXBuildFile; fileRef = D8BBD32628FD9FBF0011F850 /* SentrySwift.h */; settings = {ATTRIBUTES = (Private, ); }; };
+		D8BC28C82BFF5EBB0054DA4D /* SentryTouchTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8BC28C72BFF5EBB0054DA4D /* SentryTouchTracker.swift */; };
+		D8BC28CA2BFF68CA0054DA4D /* NumberExtensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8BC28C92BFF68CA0054DA4D /* NumberExtensions.swift */; };
+		D8BC28CC2BFF78220054DA4D /* SentryRRWebTouchEvent.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8BC28CB2BFF78220054DA4D /* SentryRRWebTouchEvent.swift */; };
 		D8BC28D62C00C6DF0054DA4D /* StringExtensionsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8BC28D42C00C6D30054DA4D /* StringExtensionsTests.swift */; };
 		D8BD2E6829361A0F00D96C6A /* PrivatesHeader.h in Headers */ = {isa = PBXBuildFile; fileRef = D8BD2E67293619F600D96C6A /* PrivatesHeader.h */; settings = {ATTRIBUTES = (Private, ); }; };
 		D8BFE37229A3782F002E73F3 /* SentryTimeToDisplayTracker.h in Headers */ = {isa = PBXBuildFile; fileRef = D8BFE37029A3782F002E73F3 /* SentryTimeToDisplayTracker.h */; };
@@ -912,6 +915,7 @@
 		D8CB742E294B294B00A5F964 /* MockUIScene.m in Sources */ = {isa = PBXBuildFile; fileRef = D8CB742D294B294B00A5F964 /* MockUIScene.m */; };
 		D8CCFC632A1520C900DE232E /* SentryBinaryImageCacheTests.m in Sources */ = {isa = PBXBuildFile; fileRef = D8CCFC622A1520C900DE232E /* SentryBinaryImageCacheTests.m */; };
 		D8CE69BC277E39C700C6EC5C /* SentryFileIOTrackingIntegrationObjCTests.m in Sources */ = {isa = PBXBuildFile; fileRef = D8CE69BB277E39C700C6EC5C /* SentryFileIOTrackingIntegrationObjCTests.m */; };
+		D8DBE0CA2C0E093000FAB1FD /* SentryTouchTrackerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8DBE0C92C0E093000FAB1FD /* SentryTouchTrackerTests.swift */; };
 		D8DBE0D22C0EFFC300FAB1FD /* SentryReplayOptionsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8DBE0D12C0EFFC300FAB1FD /* SentryReplayOptionsTests.swift */; };
 		D8F016B32B9622D6007B9AFB /* SentryId.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8F016B22B9622D6007B9AFB /* SentryId.swift */; };
 		D8F016B62B962548007B9AFB /* StringExtensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8F016B52B962548007B9AFB /* StringExtensions.swift */; };
@@ -1954,6 +1958,9 @@
 		D8B76B042808060E000A58C4 /* SentryScreenshotIntegrationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryScreenshotIntegrationTests.swift; sourceTree = "<group>"; };
 		D8B76B0728081461000A58C4 /* TestSentryScreenShot.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TestSentryScreenShot.swift; sourceTree = "<group>"; };
 		D8BBD32628FD9FBF0011F850 /* SentrySwift.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = SentrySwift.h; path = include/SentrySwift.h; sourceTree = "<group>"; };
+		D8BC28C72BFF5EBB0054DA4D /* SentryTouchTracker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryTouchTracker.swift; sourceTree = "<group>"; };
+		D8BC28C92BFF68CA0054DA4D /* NumberExtensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NumberExtensions.swift; sourceTree = "<group>"; };
+		D8BC28CB2BFF78220054DA4D /* SentryRRWebTouchEvent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryRRWebTouchEvent.swift; sourceTree = "<group>"; };
 		D8BC28D42C00C6D30054DA4D /* StringExtensionsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StringExtensionsTests.swift; sourceTree = "<group>"; };
 		D8BC83BA2AFCF08C00A662B7 /* SentryUIApplication+Private.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "SentryUIApplication+Private.h"; sourceTree = "<group>"; };
 		D8BD2E27292D1F7300D96C6A /* SDK.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = SDK.xcconfig; sourceTree = "<group>"; };
@@ -1978,6 +1985,7 @@
 		D8CB742D294B294B00A5F964 /* MockUIScene.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = MockUIScene.m; sourceTree = "<group>"; };
 		D8CCFC622A1520C900DE232E /* SentryBinaryImageCacheTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SentryBinaryImageCacheTests.m; sourceTree = "<group>"; };
 		D8CE69BB277E39C700C6EC5C /* SentryFileIOTrackingIntegrationObjCTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SentryFileIOTrackingIntegrationObjCTests.m; sourceTree = "<group>"; };
+		D8DBE0C92C0E093000FAB1FD /* SentryTouchTrackerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryTouchTrackerTests.swift; sourceTree = "<group>"; };
 		D8DBE0D12C0EFFC300FAB1FD /* SentryReplayOptionsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryReplayOptionsTests.swift; sourceTree = "<group>"; };
 		D8F016B22B9622D6007B9AFB /* SentryId.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryId.swift; sourceTree = "<group>"; };
 		D8F016B52B962548007B9AFB /* StringExtensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StringExtensions.swift; sourceTree = "<group>"; };
@@ -3593,6 +3601,7 @@
 				D861301B2BB5A267004C0F5E /* SentrySessionReplayTests.swift */,
 				D8AFC0002BD252B900118BE1 /* SentryOnDemandReplayTests.swift */,
 				D82DD1CC2BEEB1A0001AB556 /* SentryBreadcrumbReplayConverterTests.swift */,
+				D8DBE0C92C0E093000FAB1FD /* SentryTouchTrackerTests.swift */,
 				D8DBE0D12C0EFFC300FAB1FD /* SentryReplayOptionsTests.swift */,
 			);
 			path = SessionReplay;
@@ -3632,6 +3641,7 @@
 				D81988C62BEC18E20020E36C /* SentryRRWebVideoEvent.swift */,
 				D8739D132BEE5049007D2F66 /* SentryRRWebSpanEvent.swift */,
 				D81988C82BEC19200020E36C /* SentryRRWebBreadcrumbEvent.swift */,
+				D8BC28CB2BFF78220054DA4D /* SentryRRWebTouchEvent.swift */,
 			);
 			path = RRWeb;
 			sourceTree = "<group>";
@@ -3864,6 +3874,7 @@
 				D8AFC03C2BDA79BF00118BE1 /* SentryReplayVideoMaker.swift */,
 				D8F67B1A2BE9728600C9197B /* SentryReplayBreadcrumbConverter.swift */,
 				D81988BF2BEBFFF70020E36C /* SentryReplayRecording.swift */,
+				D8BC28C72BFF5EBB0054DA4D /* SentryTouchTracker.swift */,
 			);
 			path = SessionReplay;
 			sourceTree = "<group>";
@@ -3892,6 +3903,7 @@
 			children = (
 				D8F016B52B962548007B9AFB /* StringExtensions.swift */,
 				62872B5E2BA1B7F300A4FA7D /* NSLock.swift */,
+				D8BC28C92BFF68CA0054DA4D /* NumberExtensions.swift */,
 			);
 			path = Extensions;
 			sourceTree = "<group>";
@@ -4694,6 +4706,7 @@
 				D80CD8D22B751447002F710B /* SentryMXManager.swift in Sources */,
 				03F84D3527DD4191008FE43F /* SentryThreadHandle.cpp in Sources */,
 				0A2D8DA9289BC905008720F6 /* SentryViewHierarchy.m in Sources */,
+				D8BC28CC2BFF78220054DA4D /* SentryRRWebTouchEvent.swift in Sources */,
 				8EA1ED0B2668F8C400E62B98 /* SentryUIViewControllerSwizzling.m in Sources */,
 				7B98D7CF25FB650F00C5A389 /* SentryWatchdogTerminationTrackingIntegration.m in Sources */,
 				8E5D38DD261D4A3E000D363D /* SentryPerformanceTrackingIntegration.m in Sources */,
@@ -4708,6 +4721,7 @@
 				63FE711920DA4C1000CDBAE8 /* SentryCrashMachineContext.c in Sources */,
 				63FE711B20DA4C1000CDBAE8 /* SentryCrashString.c in Sources */,
 				7B14089824878F950035403D /* SentryCrashStackEntryMapper.m in Sources */,
+				D8BC28C82BFF5EBB0054DA4D /* SentryTouchTracker.swift in Sources */,
 				63FE711720DA4C1000CDBAE8 /* SentryCrashStackCursor_Backtrace.c in Sources */,
 				63FE70CB20DA4C1000CDBAE8 /* SentryCrashReportFixer.c in Sources */,
 				D8CAC0412BA0984500E38F34 /* SentryIntegrationProtocol.swift in Sources */,
@@ -4753,6 +4767,7 @@
 				0A2D8D5B289815C0008720F6 /* SentryBaseIntegration.m in Sources */,
 				62262B912BA1C520004DA3DD /* CounterMetric.swift in Sources */,
 				639FCF991EBC7B9700778193 /* SentryEvent.m in Sources */,
+				D8BC28CA2BFF68CA0054DA4D /* NumberExtensions.swift in Sources */,
 				D820CDB72BB1895F00BA339D /* SentrySessionReplayIntegration.m in Sources */,
 				632F43521F581D5400A18A36 /* SentryCrashExceptionApplication.m in Sources */,
 				62A2F4422BA9AE12000C9FDD /* SetMetric.swift in Sources */,
@@ -4782,6 +4797,7 @@
 				63FE722420DA66EC00CDBAE8 /* SentryCrashMonitor_NSException_Tests.m in Sources */,
 				7B5AB65D27E48E5200F1D1BA /* TestThreadInspector.swift in Sources */,
 				7BF9EF742722A85B00B5BBEF /* SentryClassRegistrator.m in Sources */,
+				D8DBE0CA2C0E093000FAB1FD /* SentryTouchTrackerTests.swift in Sources */,
 				D8F67AF42BE10F9600C9197B /* UIRedactBuilderTests.swift in Sources */,
 				63B819141EC352A7002FDF4C /* SentryInterfacesTests.m in Sources */,
 				62F05D2B2C0DB1F100916E3F /* SentryLogTestHelper.m in Sources */,

SentryTestUtils/TestCurrentDateProvider.swift

@@ -7,6 +7,7 @@ public class TestCurrentDateProvider: SentryCurrentDateProvider {
     private var internalSystemTime: UInt64 = 0
     public var driftTimeForEveryRead = false
     public var driftTimeInterval = 0.1
+    private var _systemUptime: TimeInterval = 0
 
     public override init() {
 
@@ -57,4 +58,11 @@ public class TestCurrentDateProvider: SentryCurrentDateProvider {
     public override func systemTime() -> UInt64 {
         return internalSystemTime
     }
+    
+    override public func systemUptime() -> TimeInterval {
+        _systemUptime
+    }
+    public func setSystemUptime(_ uptime: TimeInterval) {
+        _systemUptime = uptime
+    }
 }

Sources/Sentry/SentryScope.m

@@ -47,12 +47,13 @@
 
 @property (atomic) NSUInteger maxBreadcrumbs;
 @property (atomic) NSUInteger currentBreadcrumbIndex;
-@property (atomic, strong) NSMutableArray<SentryBreadcrumb *> *breadcrumbArray;
 
 @property (atomic, strong) NSMutableArray<SentryAttachment *> *attachmentArray;
 
 @property (nonatomic, retain) NSMutableArray<id<SentryScopeObserver>> *observers;
 
+@property (atomic, strong) NSMutableArray<SentryBreadcrumb *> *breadcrumbArray;
+
 @end
 
 @implementation SentryScope {

Sources/Sentry/SentrySessionReplay.m

@@ -43,13 +43,15 @@ @implementation SentrySessionReplay {
     int _currentSegmentId;
     BOOL _processingScreenshot;
     BOOL _reachedMaximumDuration;
+    SentryTouchTracker *_touchTracker;
     SentryReplayBreadcrumbConverter *_breadcrumbConverter;
 }
 
 - (instancetype)initWithSettings:(SentryReplayOptions *)replayOptions
                 replayFolderPath:(NSURL *)folderPath
               screenshotProvider:(id<SentryViewScreenshotProvider>)screenshotProvider
                      replayMaker:(id<SentryReplayVideoMaker>)replayMaker
+                    touchTracker:(SentryTouchTracker *)touchTracker
                     dateProvider:(SentryCurrentDateProvider *)dateProvider
                           random:(id<SentryRandom>)random
               displayLinkWrapper:(SentryDisplayLinkWrapper *)displayLinkWrapper;
@@ -64,6 +66,7 @@ - (instancetype)initWithSettings:(SentryReplayOptions *)replayOptions
         _urlToCache = folderPath;
         _replayMaker = replayMaker;
         _reachedMaximumDuration = NO;
+        _touchTracker = touchTracker;
         _breadcrumbConverter = [[SentryReplayBreadcrumbConverter alloc] init];
     }
     return self;
@@ -289,13 +292,18 @@ - (void)captureSegment:(NSInteger)segment
     replayEvent.segmentId = segment;
     replayEvent.timestamp = videoInfo.end;
 
-    __block NSArray<SentryRRWebEvent *> *events;
+    __block NSArray<SentryBreadcrumb *> *breadcrumbs;
+    [SentrySDK.currentHub
+        configureScope:^(SentryScope *_Nonnull scope) { breadcrumbs = scope.breadcrumbs; }];
+    NSMutableArray<SentryRRWebEvent *> *events = [NSMutableArray array];
 
-    [SentrySDK.currentHub configureScope:^(SentryScope *_Nonnull scope) {
-        events = [self->_breadcrumbConverter convertWithBreadcrumbs:scope.breadcrumbs
-                                                               from:videoInfo.start
-                                                              until:videoInfo.end];
-    }];
+    [events addObjectsFromArray:[self->_breadcrumbConverter convertWithBreadcrumbs:breadcrumbs
+                                                                              from:videoInfo.start
+                                                                             until:videoInfo.end]];
+
+    [events addObjectsFromArray:[_touchTracker replayEventsFrom:videoInfo.start
+                                                          until:videoInfo.end]];
+    [_touchTracker flushFinishedEvents];
 
     SentryReplayRecording *recording =
         [[SentryReplayRecording alloc] initWithSegmentId:replayEvent.segmentId

Sources/Sentry/SentrySessionReplayIntegration.m

@@ -14,13 +14,21 @@
 #    import "SentrySDK+Private.h"
 #    import "SentrySessionReplay.h"
 #    import "SentrySwift.h"
+#    import "SentrySwizzle.h"
 #    import "SentryUIApplication.h"
 #    import <UIKit/UIKit.h>
 
 NS_ASSUME_NONNULL_BEGIN
 
 static NSString *SENTRY_REPLAY_FOLDER = @"replay";
 
+/**
+ * We need to use this from the swizzled block
+ * and using an instance property would hold reference
+ * and leak memory.
+ */
+static SentryTouchTracker *_touchTracker;
+
 API_AVAILABLE(ios(16.0), tvos(16.0))
 @interface
 SentrySessionReplayIntegration ()
@@ -48,6 +56,13 @@ - (BOOL)installWithOptions:(nonnull SentryOptions *)options
             return NO;
         }
 
+        if (options.enableSwizzling) {
+            _touchTracker = [[SentryTouchTracker alloc]
+                initWithDateProvider:SentryDependencyContainer.sharedInstance.dateProvider
+                               scale:options.experimental.sessionReplay.sizeScale];
+            [self swizzleApplicationTouch];
+        }
+
         if (SentryDependencyContainer.sharedInstance.application.windows.count > 0) {
             // If a window its already available start replay right away
             [self startWithOptions:_replayOptions fullSession:_startedAsFullSession];
@@ -100,9 +115,9 @@ - (void)startWithOptions:(SentryReplayOptions *)replayOptions
               replayFolderPath:docs
             screenshotProvider:SentryViewPhotographer.shared
                    replayMaker:replayMaker
+                  touchTracker:_touchTracker
                   dateProvider:SentryDependencyContainer.sharedInstance.dateProvider
                         random:SentryDependencyContainer.sharedInstance.random
-
             displayLinkWrapper:[[SentryDisplayLinkWrapper alloc] init]];
 
         [self.sessionReplay
@@ -139,6 +154,7 @@ - (SentryIntegrationOption)integrationOptions
 
 - (void)uninstall
 {
+    _touchTracker = nil;
     [self stop];
 }
 
@@ -147,6 +163,27 @@ - (BOOL)shouldReplayFullSession:(CGFloat)rate
     return [SentryDependencyContainer.sharedInstance.random nextNumber] < rate;
 }
 
+- (void)swizzleApplicationTouch
+{
+#    pragma clang diagnostic push
+#    pragma clang diagnostic ignored "-Wshadow"
+    SEL selector = NSSelectorFromString(@"sendEvent:");
+    SentrySwizzleInstanceMethod([UIApplication class], selector, SentrySWReturnType(void),
+        SentrySWArguments(UIEvent * event), SentrySWReplacement({
+            [_touchTracker trackTouchFromEvent:event];
+            SentrySWCallOriginal(event);
+        }),
+        SentrySwizzleModeOncePerClass, (void *)selector);
+#    pragma clang diagnostic pop
+}
+
+#    if TEST || TESTCI
+- (SentryTouchTracker *)getTouchTracker
+{
+    return _touchTracker;
+}
+#    endif
+
 @end
 NS_ASSUME_NONNULL_END
 

Sources/Sentry/include/SentryDisplayLinkWrapper.h

@@ -21,4 +21,4 @@ NS_ASSUME_NONNULL_BEGIN
 
 NS_ASSUME_NONNULL_END
 
-#endif //
+#endif

Sources/Sentry/include/SentrySessionReplay.h

@@ -10,6 +10,7 @@
 @class SentryDisplayLinkWrapper;
 @class SentryVideoInfo;
 @class SentryId;
+@class SentryTouchTracker;
 
 @protocol SentryRandom;
 @protocol SentryRedactOptions;
@@ -26,6 +27,7 @@ NS_ASSUME_NONNULL_BEGIN
                 replayFolderPath:(NSURL *)folderPath
               screenshotProvider:(id<SentryViewScreenshotProvider>)photographer
                      replayMaker:(id<SentryReplayVideoMaker>)replayMaker
+                    touchTracker:(SentryTouchTracker *)touchTracker
                     dateProvider:(SentryCurrentDateProvider *)dateProvider
                           random:(id<SentryRandom>)random
               displayLinkWrapper:(SentryDisplayLinkWrapper *)displayLinkWrapper;

Sources/SentryCrash/Recording/Monitors/SentryCrashMonitor_CPPException.cpp

@@ -35,6 +35,7 @@
 
 #include <cxxabi.h>
 #include <dlfcn.h>
+#include <exception>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>

Sources/Swift/Extensions/NumberExtensions.swift

@@ -0,0 +1,12 @@
+import Foundation
+
+extension FixedWidthInteger {
+    
+    @inlinable
+    @discardableResult
+    postfix static func ++ (lhs: inout Self) -> Self {
+        defer { lhs += 1 }
+        return lhs
+    }
+    
+}