Offline storage to file system on failed upload

Agent.xcodeproj/project.pbxproj

@@ -1046,6 +1046,8 @@
 		F8A455492AFBE31E0057B1E0 /* NRMAURLSessionHeaderTrackingTestsOldEventSystem.m in Sources */ = {isa = PBXBuildFile; fileRef = F8A455472AFBE31E0057B1E0 /* NRMAURLSessionHeaderTrackingTestsOldEventSystem.m */; };
 		F8AC3E932938FD6C002B4AA8 /* NRMAFakeDataHelper.m in Sources */ = {isa = PBXBuildFile; fileRef = F8AC3E922938FD6C002B4AA8 /* NRMAFakeDataHelper.m */; };
 		F8AC3E942938FD6C002B4AA8 /* NRMAFakeDataHelper.m in Sources */ = {isa = PBXBuildFile; fileRef = F8AC3E922938FD6C002B4AA8 /* NRMAFakeDataHelper.m */; };
+		F8E202DE2B07BA61008E0B7B /* NRMAOfflineStorage.m in Sources */ = {isa = PBXBuildFile; fileRef = F8E202DD2B07BA61008E0B7B /* NRMAOfflineStorage.m */; };
+		F8E202DF2B07BA61008E0B7B /* NRMAOfflineStorage.m in Sources */ = {isa = PBXBuildFile; fileRef = F8E202DD2B07BA61008E0B7B /* NRMAOfflineStorage.m */; };
 		F8FBFA512A71ACB200CDC8C5 /* NRMARequestEvent.m in Sources */ = {isa = PBXBuildFile; fileRef = F8FBFA502A71ACB200CDC8C5 /* NRMARequestEvent.m */; };
 		F8FBFA522A71ACB200CDC8C5 /* NRMARequestEvent.m in Sources */ = {isa = PBXBuildFile; fileRef = F8FBFA502A71ACB200CDC8C5 /* NRMARequestEvent.m */; };
 		F8FBFA552A71BA2900CDC8C5 /* NRMAPayload.m in Sources */ = {isa = PBXBuildFile; fileRef = F8FBFA542A71BA2900CDC8C5 /* NRMAPayload.m */; };
@@ -1966,6 +1968,8 @@
 		F8A455472AFBE31E0057B1E0 /* NRMAURLSessionHeaderTrackingTestsOldEventSystem.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = NRMAURLSessionHeaderTrackingTestsOldEventSystem.m; sourceTree = "<group>"; };
 		F8AC3E922938FD6C002B4AA8 /* NRMAFakeDataHelper.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = NRMAFakeDataHelper.m; sourceTree = "<group>"; };
 		F8AC3EA72938FDDB002B4AA8 /* NRMAFakeDataHelper.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NRMAFakeDataHelper.h; sourceTree = "<group>"; };
+		F8E202DD2B07BA61008E0B7B /* NRMAOfflineStorage.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = NRMAOfflineStorage.m; sourceTree = "<group>"; };
+		F8E202F32B07BA6E008E0B7B /* NRMAOfflineStorage.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NRMAOfflineStorage.h; sourceTree = "<group>"; };
 		F8FBFA502A71ACB200CDC8C5 /* NRMARequestEvent.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = NRMARequestEvent.m; sourceTree = "<group>"; };
 		F8FBFA532A71ACC000CDC8C5 /* NRMARequestEvent.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NRMARequestEvent.h; sourceTree = "<group>"; };
 		F8FBFA542A71BA2900CDC8C5 /* NRMAPayload.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = NRMAPayload.m; sourceTree = "<group>"; };
@@ -3105,6 +3109,8 @@
 				02FF4AB324DC653000115469 /* Debugging */,
 				02FF4A8924DC652D00115469 /* NewRelicInternalUtils.h */,
 				02FF4A9024DC652D00115469 /* NewRelicInternalUtils.m */,
+				F8E202F32B07BA6E008E0B7B /* NRMAOfflineStorage.h */,
+				F8E202DD2B07BA61008E0B7B /* NRMAOfflineStorage.m */,
 				02FF4A7E24DC652B00115469 /* NRLogger.m */,
 				02FF4A8224DC652C00115469 /* NRMAAppToken.h */,
 				02FF4A8E24DC652D00115469 /* NRMAAppToken.m */,
@@ -4492,6 +4498,7 @@
 				02FF4A2324DC648600115469 /* NRMAMeasurementConsumer.m in Sources */,
 				02FF4AA824DC652E00115469 /* NRMAUDIDManager.m in Sources */,
 				02FF48DC24DC622700115469 /* NRMACrashDataUploader.m in Sources */,
+				F8E202DE2B07BA61008E0B7B /* NRMAOfflineStorage.m in Sources */,
 				02FF49E824DC644900115469 /* NRMAKeyAttributes.m in Sources */,
 				02FF49EF24DC645B00115469 /* NRMAAppInstallMetricGenerator.m in Sources */,
 				02FF4A7724DC64FC00115469 /* NRTimer.m in Sources */,
@@ -4725,6 +4732,7 @@
 				2B33E5D32AA9160E00AEB7B4 /* NRMASessionEvent.m in Sources */,
 				F8FBFA632A78416300CDC8C5 /* NRMAMobileEvent.m in Sources */,
 				02FF4C0524E3201400115469 /* NRMAMetric.m in Sources */,
+				F8E202DF2B07BA61008E0B7B /* NRMAOfflineStorage.m in Sources */,
 				02FF4C0624E3201400115469 /* NRMAInteractionHistory.c in Sources */,
 				02FF4C0724E3201400115469 /* NRMAActivityNameGenerator.m in Sources */,
 				02FF4C0824E3201400115469 /* NRMAScopedMeasurement.m in Sources */,

Agent/Analytics/Constants.h

@@ -86,6 +86,7 @@ extern NSString *const kNRMA_Val_errorType_Network;
 extern NSString *const kNRMA_Attrib_file;
 extern NSString *const kNRMA_Attrib_file_private;
 
+extern NSString *const kNRMA_Offline_file;
 
 // Integer Analytics Constants
 static int kNRMA_Attrib_Max_Name_Length = 256;

Agent/Analytics/Constants.m

@@ -91,3 +91,5 @@
 
 NSString * const kNRMA_Attrib_file       = @"attributes.txt";
 NSString * const kNRMA_Attrib_file_private    = @"privateAttributes.txt";
+
+NSString * const kNRMA_Offline_file           = @"offlineStorage";

Agent/Harvester/NRMAHarvester.mm

@@ -333,13 +333,25 @@ - (void) connected
     //TODO: add addition collector response processing.
     if (response.isError) {
         // failure
-        [self fireOnHarvestFailure];
+        if(response.error.code == NSURLErrorNotConnectedToInternet || response.error.code == NSURLErrorTimedOut) {
+            NSError* error = nil;
+            NSData* jsonData = [NRMAJSON dataWithJSONABLEObject:self.harvestData options:0 error:&error];
+            if (error) {
+                NRLOG_ERROR(@"Failed to generate JSON");
+                goto continues;
+            }
+            [connection.offlineStorage persistDataToDisk:jsonData];
+            [self.harvestData clear];
+        } else {
+            [self fireOnHarvestFailure];
+        }
     } else {
         // success
         [self.harvestData clear];
+        [connection sendOfflineStorage];
     }
     //Supportability/MobileAgent/Collector/Harvest
-    
+continues:
     [harvestTimer stopTimer];
 #ifndef  DISABLE_NRMA_EXCEPTION_WRAPPER
     @try {

Agent/Harvester/NRMAHarvesterConnection.h

@@ -11,6 +11,7 @@
 #import "NRLogger.h"
 #import "NRMAJSON.h"
 #import "NRMAConnection.h"
+#import "NRMAOfflineStorage.h"
 
 #define kCOLLECTOR_CONNECT_URI         @"/mobile/v4/connect"
 #define kCOLLECTOR_DATA_URL            @"/mobile/v3/data"
@@ -23,6 +24,7 @@
 @property(assign) long long             serverTimestamp;
 @property(strong) NRMAConnectInformation* connectionInformation;
 @property(strong) NSURLSession* harvestSession;
+@property(strong) NRMAOfflineStorage* offlineStorage;
 
 - (id) init;
 - (NSURLRequest*) createPostWithURI:(NSString*)uri message:(NSString*)message;
@@ -31,4 +33,5 @@
 - (NRMAHarvestResponse*) sendData:(NRMAHarvestable*)harvestable;
 - (NSURLRequest*) createConnectPost:(NSString*)message;
 - (NSURLRequest*) createDataPost:(NSString*)message;
+- (void) sendOfflineStorage;
 @end

Agent/Harvester/NRMAHarvesterConnection.m

@@ -22,9 +22,31 @@ - (id) init
     self = [super init];
     if (self) {
         self.harvestSession = [NSURLSession sessionWithConfiguration:[NSURLSessionConfiguration defaultSessionConfiguration]];
+        self.offlineStorage = [[NRMAOfflineStorage alloc] initWithEndpoint:@"data"];
     }
     return self;
 }
+
+-(void) sendOfflineStorage {
+    NSArray<NSData *> * offlineData = [self.offlineStorage getAllOfflineData];
+    if(offlineData.count == 0){
+        return;
+    }
+    NRLOG_VERBOSE(@"Number of offline data posts: %lu", (unsigned long)offlineData.count);
+
+    [offlineData enumerateObjectsUsingBlock:^(id obj, NSUInteger idx, BOOL *stop) {
+        NSData *jsonData = (NSData *)obj;
+
+        NSURLRequest* post = [self createDataPost:[[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding]];
+        if (post == nil) {
+            NRLOG_ERROR(@"Failed to create data POST");
+            return;
+        }
+
+        [self send:post];
+    }];
+}
+
 - (NSURLRequest*) createPostWithURI:(NSString*)url message:(NSString*)message
 {
     NSMutableURLRequest * postRequest = [super newPostWithURI:url];
@@ -169,6 +191,7 @@ - (NRMAHarvestResponse*) sendData:(NRMAHarvestable *)harvestable
         NRLOG_ERROR(@"Failed to generate JSON");
         return nil;
     }
+
     NSURLRequest* post = [self createDataPost:[[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding]];
     if (post == nil) {
         NRLOG_ERROR(@"Failed to create data POST");

Agent/Utilities/NRMAOfflineStorage.h

@@ -0,0 +1,16 @@
+//
+//  NRMAOfflineStorage.h
+//  Agent
+//
+//  Created by Mike Bruin on 11/17/23.
+//  Copyright © 2023 New Relic. All rights reserved.
+//
+
+@interface NRMAOfflineStorage : NSObject
+
+
+- (id)initWithEndpoint:(NSString*) name;
+- (BOOL)persistDataToDisk:(NSData*) data;
+- (NSArray<NSData *> *) getAllOfflineData;
+
+@end

Agent/Utilities/NRMAOfflineStorage.m

@@ -0,0 +1,90 @@
+//
+//  NRMAOfflineStorage.m
+//  Agent
+//
+//  Created by Mike Bruin on 11/17/23.
+//  Copyright © 2023 New Relic. All rights reserved.
+//
+
+#import <Foundation/Foundation.h>
+#import "NewRelicInternalUtils.h"
+#import "NRLogger.h"
+#import "NRMAOfflineStorage.h"
+#import "Constants.h"
+#import "NRMAJSON.h"
+
+@implementation NRMAOfflineStorage {
+}
+static NSString* _name;
+
+
+- (id)initWithEndpoint:(NSString*) name {
+    self = [super init];
+    if (self) {
+        _name = name;
+    }
+    return self;
+}
+
+- (void) createDirectory {
+    BOOL fileExists = [[NSFileManager defaultManager] fileExistsAtPath:[NRMAOfflineStorage offlineDirectoryPath] isDirectory:nil];
+    if(!fileExists){
+        NSError *error = nil;
+        if(![[NSFileManager defaultManager] createDirectoryAtPath:[NRMAOfflineStorage offlineDirectoryPath] withIntermediateDirectories:YES attributes:nil error:&error]) {
+            NSLog(@"Failed to create directory \"%@\". Error: %@", [NRMAOfflineStorage offlineDirectoryPath], error);
+        }
+    }
+}
+
+- (BOOL) persistDataToDisk:(NSData*) data {
+    @synchronized (_name) {
+        [self createDirectory];
+
+        NSError *error = nil;
+        if (data) {
+            if ([data writeToFile:[NRMAOfflineStorage newOfflineFilePath] options:NSDataWritingAtomic error:&error]) {
+                return YES;
+            }
+        }
+        NRLOG_ERROR(@"Failed to persist data to disk %@", error.description);
+
+        return NO;
+    }
+}
+
+- (NSArray<NSData *> *) getAllOfflineData {
+    @synchronized (_name) {
+        NSMutableArray<NSData *> *combinedPosts = [NSMutableArray array];
+
+        NSArray* dirs = [[NSFileManager defaultManager] contentsOfDirectoryAtPath:[NRMAOfflineStorage offlineDirectoryPath]
+                                                                            error:NULL];
+        [dirs enumerateObjectsUsingBlock:^(id obj, NSUInteger idx, BOOL *stop) {
+            NSString *filename = (NSString *)obj;
+            NSData * data = [NSData dataWithContentsOfFile:[NSString stringWithFormat:@"%@/%@",[NRMAOfflineStorage offlineDirectoryPath],filename]];
+
+            [combinedPosts addObject:data];
+        }];
+
+        [self clearAllOfflineFiles];
+
+        return [combinedPosts copy];
+    }
+}
+
+- (BOOL) clearAllOfflineFiles {
+    return [[NSFileManager defaultManager] removeItemAtPath:[NRMAOfflineStorage offlineDirectoryPath] error:NULL];
+}
+
++ (NSString*)offlineDirectoryPath {
+    return [NSString stringWithFormat:@"%@/%@/%@",[NewRelicInternalUtils getStorePath],kNRMA_Offline_file,_name];
+}
+
++ (NSString*)newOfflineFilePath {
+    NSDateFormatter *dateFormatter = [[NSDateFormatter alloc] init];
+    [dateFormatter setDateFormat:@"yyyy-MM-dd-HH-mm-ss"];
+    NSString *date = [dateFormatter stringFromDate:[NSDate date]];
+
+    return [NSString stringWithFormat:@"%@/%@%@",[NRMAOfflineStorage offlineDirectoryPath],date,@".txt"];
+}
+
+@end

Test Harness/NRTestApp/NRTestApp/AppDelegate.swift

@@ -31,7 +31,7 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
 #endif
         NewRelic.addHTTPHeaderTracking(for: ["Test"])
         NewRelic.enableFeatures([NRMAFeatureFlags.NRFeatureFlag_SwiftAsyncURLSessionSupport,
-                                 NRMAFeatureFlags.NRFeatureFlag_SwiftInteractionTracing])
+                                 NRMAFeatureFlags.NRFeatureFlag_NewEventSystem])
 
 //        NewRelic.enableFeatures([NRMAFeatureFlags.NRFeatureFlag_NewEventSystem])
 

Test Harness/NRTestApp/NRTestApp/ViewModels/UtilViewModel.swift

@@ -22,6 +22,8 @@ class UtilViewModel {
     var badAttribute = false
     var attributes = ""
     var events = 0
+
+    var timer:Timer?
 
     var uniqueInteractionTraceIdentifier: String? =  nil
 
@@ -37,14 +39,26 @@ class UtilViewModel {
         options.append(UtilOption(title: "Record Error", handler: { [self] in makeError()}))
         options.append(UtilOption(title: "Record Handled Exception", handler: { triggerException.testing()}))
         options.append(UtilOption(title: "Set UserID", handler: { [self] in changeUserID()}))
-        options.append(UtilOption(title: "Make 100 events", handler: { [self] in make100Events()}))
+        options.append(UtilOption(title: "Make 100 events every 10 seconds", handler: { [self] in startCustomEventTimer()}))
+        options.append(UtilOption(title: "Stop 100 events every 10 seconds", handler: { [self] in stopCustomEventTimer()}))
         options.append(UtilOption(title: "Start Interaction Trace", handler: { [self] in startInteractionTrace()}))
         options.append(UtilOption(title: "End Interaction Trace", handler: { [self] in stopInteractionTrace()}))
         options.append(UtilOption(title: "Notice Network Request", handler: { [self] in noticeNWRequest()}))
         options.append(UtilOption(title: "Notice Network Failure", handler: { [self] in noticeFailedNWRequest()}))
         options.append(UtilOption(title: "URLSession dataTask", handler: { [self] in doDataTask()}))
         options.append(UtilOption(title: "Shut down New Relic Agent", handler: { [self] in shutDown()}))
     }
+
+    func startCustomEventTimer(){
+        NewRelic.logInfo("Staring custom event timer")
+        timer = Timer.scheduledTimer(timeInterval: 10.0, target: self, selector: #selector(make100Events), userInfo: nil, repeats: true)
+    }
+
+    func stopCustomEventTimer(){
+        NewRelic.logInfo("Stopping custom event timer")
+        timer?.invalidate()
+        timer = nil
+    }
 
     func crash() {
         // This will cause a crash to test the crash uploader, crash files may not get recorded if the debugger is running.
@@ -101,7 +115,7 @@ class UtilViewModel {
         }
     }
 
-    func make100Events() {
+    @objc func make100Events() {
         for _ in 0...100 {
             NewRelic.recordCustomEvent("ButtonPress")
         }