Added a feature flag for offline storage and a maximum amount of stor…

…age allowed

Agent.xcodeproj/project.pbxproj

@@ -1034,6 +1034,8 @@
 		F824A43129AEAD63000886A6 /* NRViewModifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = F824A43029AEAD63000886A6 /* NRViewModifier.swift */; };
 		F824A43229AEAD63000886A6 /* NRViewModifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = F824A43029AEAD63000886A6 /* NRViewModifier.swift */; };
 		F825FAE329C8F52800E1C1DC /* NRMAWKFakeNavigationAction.m in Sources */ = {isa = PBXBuildFile; fileRef = F825FAE229C8F52800E1C1DC /* NRMAWKFakeNavigationAction.m */; };
+		F83A77B02B48810900B3D180 /* NRMAOfflineStorageTests.m in Sources */ = {isa = PBXBuildFile; fileRef = F83A77AF2B48810900B3D180 /* NRMAOfflineStorageTests.m */; };
+		F83A77B12B48810900B3D180 /* NRMAOfflineStorageTests.m in Sources */ = {isa = PBXBuildFile; fileRef = F83A77AF2B48810900B3D180 /* NRMAOfflineStorageTests.m */; };
 		F848CDC02AA133EA0082052F /* NRMAInteractionEvent.m in Sources */ = {isa = PBXBuildFile; fileRef = F848CDBF2AA133EA0082052F /* NRMAInteractionEvent.m */; };
 		F848CDC12AA133EA0082052F /* NRMAInteractionEvent.m in Sources */ = {isa = PBXBuildFile; fileRef = F848CDBF2AA133EA0082052F /* NRMAInteractionEvent.m */; };
 		F858F3602AE04B0C00CF9EB5 /* NRMAURLSessionHeaderTrackingTests.m in Sources */ = {isa = PBXBuildFile; fileRef = F858F35F2AE04B0C00CF9EB5 /* NRMAURLSessionHeaderTrackingTests.m */; };
@@ -1959,6 +1961,7 @@
 		F824A43029AEAD63000886A6 /* NRViewModifier.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NRViewModifier.swift; sourceTree = "<group>"; };
 		F825FAE229C8F52800E1C1DC /* NRMAWKFakeNavigationAction.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = NRMAWKFakeNavigationAction.m; sourceTree = "<group>"; };
 		F825FAF729C8F5BA00E1C1DC /* NRMAWKFakeNavigationAction.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NRMAWKFakeNavigationAction.h; sourceTree = "<group>"; };
+		F83A77AF2B48810900B3D180 /* NRMAOfflineStorageTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = NRMAOfflineStorageTests.m; sourceTree = "<group>"; };
 		F848CDBF2AA133EA0082052F /* NRMAInteractionEvent.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = NRMAInteractionEvent.m; sourceTree = "<group>"; };
 		F848CDD52AA133FB0082052F /* NRMAInteractionEvent.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NRMAInteractionEvent.h; sourceTree = "<group>"; };
 		F858F35F2AE04B0C00CF9EB5 /* NRMAURLSessionHeaderTrackingTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = NRMAURLSessionHeaderTrackingTests.m; sourceTree = "<group>"; };
@@ -2380,6 +2383,7 @@
 				F8AC3E922938FD6C002B4AA8 /* NRMAFakeDataHelper.m */,
 				F8AC3EA72938FDDB002B4AA8 /* NRMAFakeDataHelper.h */,
 				2B4226E529DB99740068BB8A /* NRMAHTTPUtilitiesTests.m */,
+				F83A77AF2B48810900B3D180 /* NRMAOfflineStorageTests.m */,
 			);
 			path = Uncategorized;
 			sourceTree = "<group>";
@@ -4340,6 +4344,7 @@
 				0209AC1224E7071500E45C90 /* NRMAAPIHelperTests.m in Sources */,
 				0209AC4524E7078000E45C90 /* NRMAFeatureFlagsTests.m in Sources */,
 				2B02DB86293E630B001A59D6 /* NRMAURLSessionAsyncTests.swift in Sources */,
+				F83A77B02B48810900B3D180 /* NRMAOfflineStorageTests.m in Sources */,
 				0209AC3224E7073800E45C90 /* NRMAHarvestableAnalyticsTest.m in Sources */,
 				0209AC9924E7393300E45C90 /* NSURLConnectionTests.m in Sources */,
 				0209ABFE24E7070900E45C90 /* NRURLSessionDelegateTests.m in Sources */,
@@ -4590,6 +4595,7 @@
 				025657F924EB19BF00FE3125 /* NSURLSessionOverrideTest.m in Sources */,
 				F8AC3E942938FD6C002B4AA8 /* NRMAFakeDataHelper.m in Sources */,
 				025657FA24EB19BF00FE3125 /* NRActivityTracesTest.m in Sources */,
+				F83A77B12B48810900B3D180 /* NRMAOfflineStorageTests.m in Sources */,
 				025657FB24EB19BF00FE3125 /* NRMeasurementTest.m in Sources */,
 				2B40F4FF28BEB95E00EAD248 /* NRMAStartTimerTests.m in Sources */,
 				025657FC24EB19BF00FE3125 /* TestUITableViewInstrumentation.m in Sources */,

Agent/FeatureFlags/NRMAFlags.h

@@ -49,6 +49,8 @@
 
 + (BOOL) shouldEnableLogReporting;
 
++ (BOOL) shouldEnableOfflineStorage;
+
 + (BOOL) shouldEnableNewEventSystem;
 
 + (NSArray<NSString*>*) namesForFlags:(NRMAFeatureFlags)flags;

Agent/FeatureFlags/NRMAFlags.m

@@ -161,6 +161,11 @@ + (BOOL) shouldEnableFedRampSupport {
 + (BOOL) shouldEnableSwiftAsyncURLSessionSupport {
     return ([NRMAFlags featureFlags] & NRFeatureFlag_SwiftAsyncURLSessionSupport) != 0;
 }
+
++ (BOOL) shouldEnableOfflineStorage {
+    return ([NRMAFlags featureFlags] & NRFeatureFlag_OfflineStorage) != 0;
+}
+
 + (BOOL) shouldEnableLogReporting {
     return NO;
     //return ([NRMAFlags featureFlags] & NRFeatureFlag_LogReporting) != 0;
@@ -218,6 +223,9 @@ + (BOOL) shouldEnableNewEventSystem {
     if ((flags & NRFeatureFlag_SwiftAsyncURLSessionSupport) == NRFeatureFlag_SwiftAsyncURLSessionSupport) {
         [retArray addObject:@"SwiftAsyncURLSessionSupport"];
     }
+    if ((flags & NRFeatureFlag_OfflineStorage) == NRFeatureFlag_OfflineStorage) {
+        [retArray addObject:@"OfflineStorage"];
+    }
 
     // NOTE: Temporarily removed NRFeatureFlag_LogReporting
 //    if ((flags & NRFeatureFlag_LogReporting) == NRFeatureFlag_LogReporting) {

Agent/Harvester/DataStore/NRMAAgentConfiguration.h

@@ -44,4 +44,7 @@
 + (void) setMaxEventBufferTime:(NSUInteger)seconds;
 + (NSUInteger) getMaxEventBufferTime;
 
++ (void) setMaxOfflineStorageSize:(NSUInteger)megaBytes;
++ (NSUInteger) getMaxOfflineStorageSize;
+
 @end

Agent/Harvester/DataStore/NRMAAgentConfiguration.m

@@ -22,6 +22,7 @@
 
 // Default max event buffer time is 10 minutes (600 seconds).
 static NSUInteger __NRMA__maxEventBufferTime = 600;
+static NSUInteger __NRMA__maxOfflineStorageSize = 1000000;
 
 @implementation NRMAAgentConfiguration
 
@@ -49,6 +50,14 @@ + (NSUInteger) getMaxEventBufferTime {
     return __NRMA__maxEventBufferTime;
 }
 
++ (void) setMaxOfflineStorageSize:(NSUInteger)megaBytes {
+    __NRMA__maxOfflineStorageSize = megaBytes;
+}
+
++ (NSUInteger) getMaxOfflineStorageSize {
+    return __NRMA__maxOfflineStorageSize;
+}
+
 - (id) initWithAppToken:(NRMAAppToken*)token
        collectorAddress:(NSString*)collectorHost
            crashAddress:(NSString*)crashHost {

Agent/Harvester/NRMAHarvestController.h

@@ -40,6 +40,8 @@
 
 + (BOOL) shouldNotCollectTraces;
 
++ (void) setMaxOfflineStorageSize:(NSUInteger) size;
+
 #pragma mark - HarvestController interface
 
 + (NRMAHarvesterConfiguration*) configuration;

Agent/Harvester/NRMAHarvestController.m

@@ -317,6 +317,10 @@ + (void) addHarvestableActivity:(NRMAHarvestableActivity*)activity
         [harvestData.activityTraces addActivityTraces:activity];
     }
 }
+
++ (void) setMaxOfflineStorageSize:(NSUInteger) size {
+    [[[NRMAHarvestController harvestController] harvester] setMaxOfflineStorageSize:size];
+}
 @end
 
 #ifdef __cplusplus

Agent/Harvester/NRMAHarvester.h

@@ -67,6 +67,8 @@ typedef enum {
 - (void) removeHarvestAwareObject:(id<NRMAHarvestAware>)harvestAware;
 - (void) fireOnHarvestStart;
 - (void) stop;
+
+- (void) setMaxOfflineStorageSize:(NSUInteger) size;
 @end
 
 #ifdef __cplusplus

Agent/Harvester/NRMAHarvester.mm

@@ -365,6 +365,9 @@ - (void) connected
 }
 
 - (BOOL) checkOfflineAndPersist:(NRMAHarvestResponse*) response {
+    if (![NRMAFlags shouldEnableOfflineStorage]) {
+        return false;
+    }
     if([NRMAOfflineStorage checkErrorToPersist:response.error]) {
         NSMutableDictionary *tempAnalyticsAttributes = [[NSMutableDictionary alloc] initWithDictionary:self.harvestData.analyticsAttributes];
         [tempAnalyticsAttributes setValue:[NSNumber numberWithBool:YES] forKey:@"offline"];
@@ -721,4 +724,8 @@ - (void) stop
     }
 }
 
+- (void) setMaxOfflineStorageSize:(NSUInteger) size {
+    [connection setMaxOfflineStorageSize:size];
+}
+
 @end

Agent/Harvester/NRMAHarvesterConnection.h

@@ -35,4 +35,5 @@
 - (NSURLRequest*) createDataPost:(NSString*)message;
 - (NSArray<NSData *> *) getOfflineData;
 - (void) sendOfflineStorage;
+- (void) setMaxOfflineStorageSize:(NSUInteger) size;
 @end

Agent/Harvester/NRMAHarvesterConnection.m

@@ -14,6 +14,7 @@
 #import <time.h>
 #import "NRMAHarvesterConnection+GZip.h"
 #import "NRMASupportMetricHelper.h"
+#import "NRMAFlags.h"
 
 @implementation NRMAHarvesterConnection
 @synthesize connectionInformation = _connectionInformation;
@@ -23,6 +24,7 @@ - (id) init
     if (self) {
         self.harvestSession = [NSURLSession sessionWithConfiguration:[NSURLSessionConfiguration defaultSessionConfiguration]];
         self.offlineStorage = [[NRMAOfflineStorage alloc] initWithEndpoint:@"data"];
+        [_offlineStorage setMaxOfflineStorageSize:[NRMAAgentConfiguration getMaxOfflineStorageSize]];
     }
     return self;
 }
@@ -32,6 +34,9 @@ - (id) init
 }
 
 -(void) sendOfflineStorage {
+    if (![NRMAFlags shouldEnableOfflineStorage]) {
+        return;
+    }
     NSArray<NSData *> * offlineData = [self.offlineStorage getAllOfflineData:YES];
     if(offlineData.count == 0){
         return;
@@ -51,7 +56,7 @@ -(void) sendOfflineStorage {
         NRMAHarvestResponse* response = [self send:post];
 
         if([NRMAOfflineStorage checkErrorToPersist:response.error]) {
-         //   [_offlineStorage persistDataToDisk:jsonData]; Re-save if failed to send again?
+            [_offlineStorage persistDataToDisk:jsonData];
         }
     }];
 }
@@ -237,4 +242,8 @@ - (NSString*) collectorHostURL:(NSString*)resource
     return [NSString stringWithFormat:@"%@%@%@",protocol,self.collectorHost,resource];
 }
 
+- (void) setMaxOfflineStorageSize:(NSUInteger) size {
+    [_offlineStorage setMaxOfflineStorageSize:size];
+}
+
 @end

Agent/Public/NewRelic.h

@@ -693,6 +693,15 @@ extern "C" {
  */
 + (void) setMaxEventPoolSize:(unsigned int)size;
 
+/*!
+ Change the maximum size in mega bytes that the agent will store for offline storage.
+ 
+ @param megaBytes the maximum size in mega bytes of offline storage that can be stored in the local file system
+ 
+ By default the SDK will store up to 100 MB worth of offline payloads in the local file system.
+ */
+
++ (void) setMaxOfflineStorageSize:(unsigned int)megaBytes;
 
 #pragma mark - Tracking global attributes
 

Agent/Public/NewRelic.m

@@ -626,6 +626,16 @@ + (void) setMaxEventPoolSize:(unsigned int)size {
     [[NewRelicAgentInternal sharedInstance].analyticsController setMaxEventBufferSize:size];
 }
 
+/*
+ * this method sets the maximum size for offline storage that the can be collected in the agent.
+ * this means: once the maximum size has been met the agent will stop storing offline payloads until
+ * more room is made.
+ */
++ (void) setMaxOfflineStorageSize:(unsigned int)megaBytes {
+    [NRMAAgentConfiguration setMaxOfflineStorageSize:megaBytes];
+
+    [NRMAHarvestController setMaxOfflineStorageSize:megaBytes];
+}
 #pragma mark - Hidden APIs
 
 

Agent/Public/NewRelicFeatureFlags.h

@@ -99,4 +99,5 @@ typedef NS_OPTIONS(unsigned long long, NRMAFeatureFlags){
 
     // NOTE: Temporarily removed NRFeatureFlag_LogReporting
     NRFeatureFlag_NewEventSystem                        = 1 << 20, // Disabled by default
+    NRFeatureFlag_OfflineStorage                        = 1 << 21, // Disabled by default
 };

Agent/Utilities/NRMAOfflineStorage.h

@@ -10,8 +10,11 @@
 
 
 - (id)initWithEndpoint:(NSString*) name;
-- (BOOL)persistDataToDisk:(NSData*) data;
+- (BOOL) persistDataToDisk:(NSData*) data;
 - (NSArray<NSData *> *) getAllOfflineData:(BOOL) clear;
-+ (BOOL)checkErrorToPersist:(NSError*) error;
++ (BOOL) checkErrorToPersist:(NSError*) error;
+- (BOOL) clearAllOfflineFiles;
+- (void) setMaxOfflineStorageSize:(NSUInteger) size;
+- (NSString*) offlineDirectoryPath;
 
 @end

Agent/Utilities/NRMAOfflineStorage.m

@@ -14,15 +14,18 @@
 #import "NRMAJSON.h"
 #import "NRMASupportMetricHelper.h"
 
+#define kNRMAOfflineStorageCurrentSize @"com.newrelic.offlineStorageCurrentSize"
+
 @implementation NRMAOfflineStorage {
+    NSUInteger maxOfflineStorageSize;
 }
 static NSString* _name;
 
-
 - (id)initWithEndpoint:(NSString*) name {
     self = [super init];
     if (self) {
         _name = name;
+        maxOfflineStorageSize = 1000000;
     }
     return self;
 }
@@ -41,10 +44,18 @@ - (BOOL) persistDataToDisk:(NSData*) data {
     @synchronized (self) {
         [self createDirectory];
 
+        NSUInteger currentOfflineStorageSize = [[NSUserDefaults standardUserDefaults] integerForKey:kNRMAOfflineStorageCurrentSize];
+        currentOfflineStorageSize += data.length;
+        if(currentOfflineStorageSize > maxOfflineStorageSize){
+            NRLOG_WARNING(@"Not saving to offline storage because max storage size has been reached.");
+            return NO;
+        }
+
         NSError *error = nil;
         if (data) {
             if ([data writeToFile:[self newOfflineFilePath] options:NSDataWritingAtomic error:&error]) {
-                NRLOG_VERBOSE(@"Successfully persisted failed upload data to disk for offline storage.");
+                [[NSUserDefaults standardUserDefaults] setInteger:currentOfflineStorageSize forKey:kNRMAOfflineStorageCurrentSize]; // If we successfully save the data save the new current total size
+                NRLOG_VERBOSE(@"Successfully persisted failed upload data to disk for offline storage. Current offline storage: %lu", (unsigned long)currentOfflineStorageSize);
                 return YES;
             }
         }
@@ -77,21 +88,29 @@ - (BOOL) persistDataToDisk:(NSData*) data {
 }
 
 - (BOOL) clearAllOfflineFiles {
-    return [[NSFileManager defaultManager] removeItemAtPath:[self offlineDirectoryPath] error:NULL];
+    if ([[NSFileManager defaultManager] removeItemAtPath:[self offlineDirectoryPath] error:NULL]) {
+        [[NSUserDefaults standardUserDefaults] setInteger:0 forKey:kNRMAOfflineStorageCurrentSize];
+        return true;
+    }
+    return false;
 }
 
-- (NSString*)offlineDirectoryPath {
+- (NSString*) offlineDirectoryPath {
     return [NSString stringWithFormat:@"%@/%@/%@",[NewRelicInternalUtils getStorePath],kNRMA_Offline_file,_name];
 }
 
-- (NSString*)newOfflineFilePath {
+- (NSString*) newOfflineFilePath {
     NSDateFormatter *dateFormatter = [[NSDateFormatter alloc] init];
     [dateFormatter setDateFormat:@"yyyy-MM-dd-HH-mm-ss"];
     NSString *date = [dateFormatter stringFromDate:[NSDate date]];
 
     return [NSString stringWithFormat:@"%@/%@%@",[self offlineDirectoryPath],date,@".txt"];
 }
 
+- (void) setMaxOfflineStorageSize:(NSUInteger) size {
+    maxOfflineStorageSize = (size * 1000000);
+}
+
 + (BOOL)checkErrorToPersist:(NSError*) error {
     return (error.code == NSURLErrorNotConnectedToInternet || error.code == NSURLErrorTimedOut || error.code == NSURLErrorCannotFindHost || error.code == NSURLErrorNetworkConnectionLost || error.code == NSURLErrorCannotConnectToHost);
 }

Test Harness/NRTestApp/NRTestApp.xcodeproj/project.pbxproj

@@ -85,6 +85,7 @@
 		F86429442979CE31002ABA01 /* NewRelic.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = F8302B6429705766003EC291 /* NewRelic.framework */; };
 		F86429452979CE31002ABA01 /* NewRelic.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = F8302B6429705766003EC291 /* NewRelic.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
 		F8C8812929845B8C001C15B9 /* NRTestAppNavigationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F8C8812829845B8C001C15B9 /* NRTestAppNavigationTests.swift */; };
+		F8D7C6052B45C38700170F79 /* AppDelegate+UITest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2BF1FA3B2AE32C9500E9019C /* AppDelegate+UITest.swift */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -862,6 +863,7 @@
 				F86429342979CDFA002ABA01 /* plistHelper.swift in Sources */,
 				F86429352979CDFA002ABA01 /* triggerException.m in Sources */,
 				F86429382979CDFA002ABA01 /* AppDelegate.swift in Sources */,
+				F8D7C6052B45C38700170F79 /* AppDelegate+UITest.swift in Sources */,
 				F86429392979CDFA002ABA01 /* SceneDelegate.swift in Sources */,
 				F864293A2979CDFA002ABA01 /* ViewControllerProvider.swift in Sources */,
 				F864293B2979CDFA002ABA01 /* Coordinator.swift in Sources */,

Test Harness/NRTestApp/NRTestApp/AppDelegate.swift

@@ -31,9 +31,8 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
 #endif
         NewRelic.addHTTPHeaderTracking(for: ["Test"])
         NewRelic.enableFeatures([NRMAFeatureFlags.NRFeatureFlag_SwiftAsyncURLSessionSupport,
-                                 NRMAFeatureFlags.NRFeatureFlag_NewEventSystem])
-
-//        NewRelic.enableFeatures([NRMAFeatureFlags.NRFeatureFlag_NewEventSystem])
+                                 NRMAFeatureFlags.NRFeatureFlag_NewEventSystem,
+                                 NRMAFeatureFlags.NRFeatureFlag_OfflineStorage])
 
         NewRelic.replaceDeviceIdentifier("myDeviceId")
 
@@ -68,7 +67,7 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
         }
 
         NewRelic.setMaxEventPoolSize(5000)
-        NewRelic.setMaxEventBufferTime(60)
+        NewRelic.setMaxEventBufferTime(600)
 
         NewRelic.logVerbose("NewRelic.start was called.")
         return true

Tests/Unit-Tests/NewRelicAgentTests/API-Tests/NewRelicTests.m

@@ -135,14 +135,24 @@ - (void) testEnableNewEventSystem {
     XCTAssertFalse(flags, @"flags should be empty");
 
     XCTAssertFalse([NRMAFlags shouldEnableNewEventSystem], @"flags should be empty");
-    //XCTAssertNil([NewRelic startInteractionWithName:@"test"], @"should be nil when Interaction Tracing is disabled");
 
     [NewRelic enableFeatures:NRFeatureFlag_NewEventSystem];
     flags = [NRMAFlags featureFlags];
     XCTAssertTrue(flags & NRFeatureFlag_NewEventSystem, @"flags should have New Event System enabled");
     XCTAssertFalse(flags & ~NRFeatureFlag_NewEventSystem , @"flags shouldn't have any other bit enabled.");
+}
+
+- (void) testEnableOfflineStorage {
+    [NRMAFlags setFeatureFlags:0];
+    NRMAFeatureFlags flags = [NRMAFlags featureFlags];
+    XCTAssertFalse(flags, @"flags should be empty");
+
+    XCTAssertFalse([NRMAFlags shouldEnableOfflineStorage], @"flags should be empty");
 
-   // XCTAssertNotNil([NewRelic startInteractionWithName:@"test"]);
+    [NewRelic enableFeatures:NRFeatureFlag_OfflineStorage];
+    flags = [NRMAFlags featureFlags];
+    XCTAssertTrue(flags & NRFeatureFlag_OfflineStorage, @"flags should have offline storage enabled");
+    XCTAssertFalse(flags & ~NRFeatureFlag_OfflineStorage , @"flags shouldn't have any other bit enabled.");
 }
 
 - (void) testRecordMetricsConsistency