Use only files and byte arrays for object storage uploads

There are various problems with using InputStream as the default API for
uploading files to object storage. One is that many object storage
providers won't calculate a checksum for streaming uploads, increasing
our chances for data corruption. Another is that many "S3-compatible"
object storage providers don't properly support streaming uploads,
causing either upload errors or worse: silent data corruption. We've
actually experienced silent data corruption when using streaming uploads
to Google Cloud Storage via their XML API which is supposed to be
S3-compatible. Transparent headers in the PutObject request were
_silently being appended_ to the object data, causing logic errors when
we attempted to read and use those objects later (in our case, when
publishing apps).

Overall it seems to be a much better choice to use local files and
memory buffers as upload sources for the best object storage provider
compatibility and to prevent insidious bugs like the formerly described
data corruption issue.

Author: lberrymage

console/src/main/kotlin/app/accrescent/parcelo/console/routes/Drafts.kt

@@ -219,16 +219,8 @@ fun Route.createDraftRoute() {
                             null
                         }
 
-                        val iconFileId = iconData.inputStream()
-                            .use {
-                                runBlocking {
-                                    storageService.saveObject(it, iconData.size.toLong())
-                                }
-                            }
-                        val appFileId = tempApkSet.inputStream()
-                            .use {
-                                runBlocking { storageService.saveObject(it, tempApkSet.size()) }
-                            }
+                        val iconFileId = runBlocking { storageService.uploadBytes(iconData) }
+                        val appFileId = runBlocking { storageService.uploadFile(tempApkSet.path) }
                         val icon = Icon.new { fileId = iconFileId }
                         Draft.new {
                             this.label = label

console/src/main/kotlin/app/accrescent/parcelo/console/routes/Updates.kt

@@ -169,8 +169,7 @@ fun Route.createUpdateRoute() {
                     return@post
                 }
 
-                val apkSetFileId = tempApkSet.inputStream()
-                    .use { storageService.saveObject(it, tempApkSet.size()) }
+                val apkSetFileId = storageService.uploadFile(tempApkSet.path)
 
                 // There exists:
                 //

console/src/main/kotlin/app/accrescent/parcelo/console/storage/ObjectStorageService.kt

@@ -6,17 +6,27 @@ package app.accrescent.parcelo.console.storage
 
 import org.jetbrains.exposed.dao.id.EntityID
 import java.io.InputStream
+import java.nio.file.Path
 
 /**
  * Abstraction of object storage
  */
 interface ObjectStorageService {
     /**
-     * Save a file to the file storage service
+     * Upload an object from a file to the object storage service
      *
-     * @return the database ID of the new file
+     * @param path the path of the file to upload
+     * @return the database ID of the new object
      */
-    suspend fun saveObject(inputStream: InputStream, size: Long): EntityID<Int>
+    suspend fun uploadFile(path: Path): EntityID<Int>
+
+    /**
+     * Upload an object from memory to the object storage service
+     *
+     * @param bytes the object data to upload
+     * @return the database ID of the new object
+     */
+    suspend fun uploadBytes(bytes: ByteArray): EntityID<Int>
 
     /**
      * Marks the given file deleted

console/src/main/kotlin/app/accrescent/parcelo/console/storage/S3ObjectStorageService.kt

@@ -16,6 +16,7 @@ import aws.sdk.kotlin.services.s3.model.DeleteObjectsRequest
 import aws.sdk.kotlin.services.s3.model.GetObjectRequest
 import aws.sdk.kotlin.services.s3.model.ObjectIdentifier
 import aws.sdk.kotlin.services.s3.model.PutObjectRequest
+import aws.smithy.kotlin.runtime.content.ByteStream
 import aws.smithy.kotlin.runtime.content.asByteStream
 import aws.smithy.kotlin.runtime.content.toInputStream
 import aws.smithy.kotlin.runtime.net.url.Url
@@ -27,6 +28,7 @@ import org.jetbrains.exposed.sql.transactions.transaction
 import java.io.FileNotFoundException
 import java.io.IOException
 import java.io.InputStream
+import java.nio.file.Path
 import java.util.UUID
 
 /**
@@ -39,7 +41,7 @@ class S3ObjectStorageService(
     private val s3AccessKeyId: String,
     private val s3SecretAccessKey: String,
 ) : ObjectStorageService {
-    override suspend fun saveObject(inputStream: InputStream, size: Long): EntityID<Int> {
+    override suspend fun uploadFile(path: Path): EntityID<Int> {
         S3Client {
             endpointUrl = s3EndpointUrl
             region = s3Region
@@ -53,7 +55,31 @@ class S3ObjectStorageService(
             val req = PutObjectRequest {
                 bucket = s3Bucket
                 key = objectKey
-                body = inputStream.asByteStream(size)
+                body = path.asByteStream()
+            }
+            s3Client.putObject(req)
+
+            val fileId = transaction { File.new { s3ObjectKey = objectKey }.id }
+
+            return fileId
+        }
+    }
+
+    override suspend fun uploadBytes(bytes: ByteArray): EntityID<Int> {
+        S3Client {
+            endpointUrl = s3EndpointUrl
+            region = s3Region
+            credentialsProvider = StaticCredentialsProvider {
+                accessKeyId = s3AccessKeyId
+                secretAccessKey = s3SecretAccessKey
+            }
+        }.use { s3Client ->
+            val objectKey = UUID.randomUUID().toString()
+
+            val req = PutObjectRequest {
+                bucket = s3Bucket
+                key = objectKey
+                body = ByteStream.fromBytes(bytes)
             }
             s3Client.putObject(req)