From 8ed62330ad514ed3561ed6d2322198d56a07df9e Mon Sep 17 00:00:00 2001 From: lberrymage Date: Tue, 17 Sep 2024 15:54:58 -0700 Subject: [PATCH] Specify size of uploaded S3 objects Not all third-party "S3-compatible" storage implementations support the x-amz-checksum-algorithm header, so we should instead simply specify the content length of uploaded objects for wider provider support. --- .../app/accrescent/parcelo/console/routes/Drafts.kt | 8 ++++++-- .../app/accrescent/parcelo/console/routes/Updates.kt | 3 ++- .../parcelo/console/storage/FileStorageService.kt | 2 +- .../parcelo/console/storage/S3FileStorageService.kt | 7 ++----- .../app/accrescent/parcelo/console/util/TempFile.kt | 7 +++++++ 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/console/src/main/kotlin/app/accrescent/parcelo/console/routes/Drafts.kt b/console/src/main/kotlin/app/accrescent/parcelo/console/routes/Drafts.kt index 7cfdb195..a1e10561 100644 --- a/console/src/main/kotlin/app/accrescent/parcelo/console/routes/Drafts.kt +++ b/console/src/main/kotlin/app/accrescent/parcelo/console/routes/Drafts.kt @@ -219,9 +219,13 @@ fun Route.createDraftRoute() { } val iconFileId = iconData.inputStream() - .use { runBlocking { storageService.saveFile(it) } } + .use { + runBlocking { + storageService.saveFile(it, iconData.size.toLong()) + } + } val appFileId = tempApkSet.inputStream() - .use { runBlocking { storageService.saveFile(it) } } + .use { runBlocking { storageService.saveFile(it, tempApkSet.size()) } } val icon = Icon.new { fileId = iconFileId } Draft.new { this.label = label diff --git a/console/src/main/kotlin/app/accrescent/parcelo/console/routes/Updates.kt b/console/src/main/kotlin/app/accrescent/parcelo/console/routes/Updates.kt index db2e2be8..3894fe3e 100644 --- a/console/src/main/kotlin/app/accrescent/parcelo/console/routes/Updates.kt +++ b/console/src/main/kotlin/app/accrescent/parcelo/console/routes/Updates.kt @@ -169,7 +169,8 @@ fun Route.createUpdateRoute() { return@post } - val apkSetFileId = tempApkSet.inputStream().use { storageService.saveFile(it) } + val apkSetFileId = + tempApkSet.inputStream().use { storageService.saveFile(it, tempApkSet.size()) } // There exists: // diff --git a/console/src/main/kotlin/app/accrescent/parcelo/console/storage/FileStorageService.kt b/console/src/main/kotlin/app/accrescent/parcelo/console/storage/FileStorageService.kt index 9f26ef47..5db904c0 100644 --- a/console/src/main/kotlin/app/accrescent/parcelo/console/storage/FileStorageService.kt +++ b/console/src/main/kotlin/app/accrescent/parcelo/console/storage/FileStorageService.kt @@ -16,7 +16,7 @@ interface FileStorageService { * * @return the database ID of the new file */ - suspend fun saveFile(inputStream: InputStream): EntityID + suspend fun saveFile(inputStream: InputStream, size: Long): EntityID /** * Marks the given file deleted diff --git a/console/src/main/kotlin/app/accrescent/parcelo/console/storage/S3FileStorageService.kt b/console/src/main/kotlin/app/accrescent/parcelo/console/storage/S3FileStorageService.kt index 57985233..b8195cf8 100644 --- a/console/src/main/kotlin/app/accrescent/parcelo/console/storage/S3FileStorageService.kt +++ b/console/src/main/kotlin/app/accrescent/parcelo/console/storage/S3FileStorageService.kt @@ -10,7 +10,6 @@ import app.accrescent.parcelo.console.data.Files.deleted import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider.Companion.invoke import aws.sdk.kotlin.services.s3.S3Client -import aws.sdk.kotlin.services.s3.model.ChecksumAlgorithm import aws.sdk.kotlin.services.s3.model.Delete import aws.sdk.kotlin.services.s3.model.DeleteObjectRequest import aws.sdk.kotlin.services.s3.model.DeleteObjectsRequest @@ -40,7 +39,7 @@ class S3FileStorageService( private val s3AccessKeyId: String, private val s3SecretAccessKey: String, ) : FileStorageService { - override suspend fun saveFile(inputStream: InputStream): EntityID { + override suspend fun saveFile(inputStream: InputStream, size: Long): EntityID { S3Client { endpointUrl = s3EndpointUrl region = s3Region @@ -54,9 +53,7 @@ class S3FileStorageService( val req = PutObjectRequest { bucket = s3Bucket key = objectKey - body = inputStream.asByteStream() - // Necessary to specify so the S3 library can calculate a body hash from a stream - checksumAlgorithm = ChecksumAlgorithm.Sha256 + body = inputStream.asByteStream(size) } s3Client.putObject(req) diff --git a/console/src/main/kotlin/app/accrescent/parcelo/console/util/TempFile.kt b/console/src/main/kotlin/app/accrescent/parcelo/console/util/TempFile.kt index ec1fbc26..4604f271 100644 --- a/console/src/main/kotlin/app/accrescent/parcelo/console/util/TempFile.kt +++ b/console/src/main/kotlin/app/accrescent/parcelo/console/util/TempFile.kt @@ -45,6 +45,13 @@ class TempFile : AutoCloseable { return path.toFile().outputStream() } + /** + * Returns the size of this file in bytes + */ + fun size(): Long { + return path.toFile().length() + } + /** * Deletes the underlying file */