Merge pull request #3616 from DataDog/aleksandr-gringauz/RUM-16305/fix-limits

RUM-16305: Update several hardcoded data limits

Author: aleksandr-gringauz

dd-sdk-android-core/api/apiSurface

@@ -200,7 +200,6 @@ interface com.datadog.android.api.net.RequestFactory
 interface com.datadog.android.api.storage.DataWriter<T>
   fun write(EventBatchWriter, T, EventType): Boolean
 interface com.datadog.android.api.storage.EventBatchWriter
-  fun currentMetadata(): ByteArray?
   fun write(RawBatchEvent, ByteArray?, EventType): Boolean
 enum com.datadog.android.api.storage.EventType
   - DEFAULT

dd-sdk-android-core/api/dd-sdk-android-core.api

@@ -577,7 +577,6 @@ public abstract interface class com/datadog/android/api/storage/DataWriter {
 }
 
 public abstract interface class com/datadog/android/api/storage/EventBatchWriter {
-	public abstract fun currentMetadata ()[B
 	public abstract fun write (Lcom/datadog/android/api/storage/RawBatchEvent;[BLcom/datadog/android/api/storage/EventType;)Z
 }
 

dd-sdk-android-core/src/main/kotlin/com/datadog/android/api/storage/EventBatchWriter.kt

@@ -9,16 +9,10 @@ package com.datadog.android.api.storage
 import androidx.annotation.WorkerThread
 
 /**
- * Writer allowing [FeatureScope] to write events in the storage exposing current batch metadata.
+ * Writer allowing [FeatureScope] to write events in the storage.
  */
 interface EventBatchWriter {
 
-    /**
-     * @return the metadata of the current writeable batch
-     */
-    @WorkerThread
-    fun currentMetadata(): ByteArray?
-
     /**
      * Writes the content of the event to the current available batch.
      * @param event the event to write (content + metadata)

dd-sdk-android-core/src/main/kotlin/com/datadog/android/api/storage/FeatureStorageConfiguration.kt

@@ -24,17 +24,17 @@ data class FeatureStorageConfiguration(
 
         /**
          * Default storage configuration with the following parameters:
-         * max item size = 512 KB,
-         * max items per batch = 500,
-         * max batch size = 4 MB,
+         * max item size = 1 MB,
+         * max items per batch = 1000,
+         * max batch size = 5 MB,
          * old batch threshold = 18 hours.
          */
         val DEFAULT: FeatureStorageConfiguration = FeatureStorageConfiguration(
-            // 512 KB
-            maxItemSize = 512L * 1024,
-            maxItemsPerBatch = 500,
-            // 4 MB
-            maxBatchSize = 4L * 1024 * 1024,
+            // 1 MB
+            maxItemSize = 1024L * 1024,
+            maxItemsPerBatch = 1000,
+            // 5 MB
+            maxBatchSize = 5L * 1024 * 1024,
             // 18 hours
             oldBatchThreshold = 18L * 60L * 60L * 1000L
         )

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/CoreFeature.kt

@@ -40,7 +40,6 @@ import com.datadog.android.core.internal.net.info.NetworkInfoProvider
 import com.datadog.android.core.internal.net.info.NoOpNetworkInfoProvider
 import com.datadog.android.core.internal.persistence.JsonObjectDeserializer
 import com.datadog.android.core.internal.persistence.file.FilePersistenceConfig
-import com.datadog.android.core.internal.persistence.file.FileWriter
 import com.datadog.android.core.internal.persistence.file.batch.BatchFileReaderWriter
 import com.datadog.android.core.internal.persistence.file.deleteSafe
 import com.datadog.android.core.internal.persistence.file.existsSafe
@@ -231,7 +230,7 @@ internal class CoreFeature(
 
     @get:WorkerThread
     private val lastViewEventFile: File by lazy { File(storageDir, LAST_RUM_VIEW_EVENT_FILE_NAME) }
-    private val lastViewEventFileWriter: FileWriter<RawBatchEvent> by lazy {
+    private val lastViewEventFileWriter: BatchFileReaderWriter by lazy {
         BatchFileReaderWriter.create(
             internalLogger = internalLogger,
             encryption = localDataEncryption
@@ -387,7 +386,8 @@ internal class CoreFeature(
 
     @WorkerThread
     internal fun writeLastViewEvent(data: ByteArray) {
-        lastViewEventFileWriter.writeData(lastViewEventFile, RawBatchEvent(data), false)
+        val serialized = lastViewEventFileWriter.serializeToBytes(RawBatchEvent(data = data)) ?: return
+        lastViewEventFileWriter.writeBinaryData(lastViewEventFile, serialized, false)
     }
 
     @WorkerThread

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/persistence/AbstractStorage.kt

@@ -67,11 +67,6 @@ internal class AbstractStorage(
     ): EventWriteScope {
         val strategy = resolvePersistenceStrategy(datadogContext)
         val writer = object : EventBatchWriter {
-            @WorkerThread
-            override fun currentMetadata(): ByteArray? {
-                return strategy.currentMetadata()
-            }
-
             @WorkerThread
             override fun write(event: RawBatchEvent, batchMetadata: ByteArray?, eventType: EventType): Boolean {
                 return strategy.write(event, batchMetadata, eventType)

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/persistence/FileEventBatchWriter.kt

@@ -14,51 +14,39 @@ import com.datadog.android.api.storage.RawBatchEvent
 import com.datadog.android.core.internal.persistence.file.FileOrchestrator
 import com.datadog.android.core.internal.persistence.file.FilePersistenceConfig
 import com.datadog.android.core.internal.persistence.file.FileReaderWriter
-import com.datadog.android.core.internal.persistence.file.FileWriter
-import com.datadog.android.core.internal.persistence.file.existsSafe
+import com.datadog.android.core.internal.persistence.file.batch.BatchFileReaderWriter
 import java.io.File
 import java.util.Locale
 
 internal class FileEventBatchWriter(
     private val fileOrchestrator: FileOrchestrator,
-    private val eventsWriter: FileWriter<RawBatchEvent>,
+    private val eventsWriter: BatchFileReaderWriter,
     private val metadataReaderWriter: FileReaderWriter,
     private val filePersistenceConfig: FilePersistenceConfig,
     private val batchWriteEventListener: BatchWriteEventListener,
     private val internalLogger: InternalLogger
 ) : EventBatchWriter {
 
-    @get:WorkerThread
-    private val batchFile: File? by lazy {
-        @Suppress("ThreadSafety") // called in the worker context
-        fileOrchestrator.getWritableFile()
-    }
-
-    @get:WorkerThread
-    private val metadataFile: File?
-        get() = batchFile?.let {
-            @Suppress("ThreadSafety") // called in the worker context
-            fileOrchestrator.getMetadataFile(it)
-        }
-
-    @WorkerThread
-    override fun currentMetadata(): ByteArray? {
-        return with(metadataFile) {
-            if (this == null || !existsSafe(internalLogger)) {
-                null
-            } else {
-                metadataReaderWriter.readData(this)
-            }
-        }
-    }
-
     @WorkerThread
+    @Suppress("ReturnCount")
     override fun write(
         event: RawBatchEvent,
         batchMetadata: ByteArray?,
         eventType: EventType
     ): Boolean {
-        val (batchFile, metadataFile) = batchFile to metadataFile
+        // prevent useless operation for empty event
+        if (event.data.isEmpty()) {
+            return true
+        }
+        if (!checkEventSize(event.data.size)) {
+            return false
+        }
+
+        // Serialize once (TLV-wrapped, encrypted if configured) so the exact on-disk size is known
+        // before asking the orchestrator for a file with enough room to hold it.
+        val serializedEvent = eventsWriter.serializeToBytes(event) ?: return false
+
+        val batchFile = fileOrchestrator.getWritableFile(serializedEvent.size.toLong())
         if (batchFile == null) {
             internalLogger.log(
                 InternalLogger.Level.ERROR,
@@ -68,12 +56,9 @@ internal class FileEventBatchWriter(
             return false
         }
 
-        // prevent useless operation for empty event
-        return if (event.data.isEmpty()) {
-            true
-        } else if (!checkEventSize(event.data.size)) {
-            false
-        } else if (eventsWriter.writeData(batchFile, event, true)) {
+        val metadataFile = fileOrchestrator.getMetadataFile(batchFile)
+
+        return if (eventsWriter.writeBinaryData(batchFile, serializedEvent, true)) {
             batchWriteEventListener.onWriteEvent(event.data.size.toLong())
             if (batchMetadata?.isNotEmpty() == true && metadataFile != null) {
                 writeBatchMetadata(metadataFile, batchMetadata)

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/persistence/NoOpEventBatchWriter.kt

@@ -12,10 +12,6 @@ import com.datadog.android.api.storage.RawBatchEvent
 
 internal class NoOpEventBatchWriter : EventBatchWriter {
 
-    override fun currentMetadata(): ByteArray? {
-        return null
-    }
-
     override fun write(
         event: RawBatchEvent,
         batchMetadata: ByteArray?,

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/persistence/file/FileOrchestrator.kt

@@ -21,11 +21,14 @@ import java.io.File
 internal interface FileOrchestrator {
 
     /**
+     * @param eventSize the size (in bytes) of the event about to be written, used to decide
+     * whether an existing batch file has enough room to accommodate it without exceeding the
+     * maximum batch size.
      * @return a File with enough space to write data, or null if no space is available
      * or the disk can't be written to.
      */
     @WorkerThread
-    fun getWritableFile(): File?
+    fun getWritableFile(eventSize: Long): File?
 
     /**
      * @param excludeFiles a set of files to exclude from the readable files

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/persistence/file/FilePersistenceConfig.kt

@@ -19,11 +19,11 @@ internal data class FilePersistenceConfig(
     val cleanupFrequencyThreshold: Long = CLEANUP_FREQUENCY_THRESHOLD_MS
 ) {
     companion object {
-        internal const val MAX_BATCH_SIZE: Long = 4L * 1024 * 1024 // 4 MB
-        internal const val MAX_ITEMS_PER_BATCH: Int = 500
-        internal const val MAX_ITEM_SIZE: Long = 512L * 1024 // 512 KB
+        internal const val MAX_BATCH_SIZE: Long = 5L * 1024 * 1024 // 5 MB
+        internal const val MAX_ITEMS_PER_BATCH: Int = 1000
+        internal const val MAX_ITEM_SIZE: Long = 1024L * 1024 // 1 MB
         internal const val OLD_FILE_THRESHOLD: Long = 18L * 60L * 60L * 1000L // 18 hours
-        internal const val MAX_DISK_SPACE: Long = 128 * MAX_BATCH_SIZE // 512 MB
+        internal const val MAX_DISK_SPACE: Long = 512L * 1024 * 1024 // 512 MB
         internal const val MAX_DELAY_BETWEEN_MESSAGES_MS = 5000L
         internal const val CLEANUP_FREQUENCY_THRESHOLD_MS = 5000L // 5s
     }