refactored all functions (#145)

* refactored stat_object

refactored select_object_content

refactor get_presigned_object_url

refactor get_presigned_policy_form_data

refactored upload-part-copy

* fixed object.unwrap

* update region

* made client Arc

* made client Arc

* update client

* update tests

* update segmented_bytes

* bench updated

* cleanup version handling

* cleanup of headers: multimap

* added inner in Client

* updated clients: added Into<String>in API

* Separated http_client and shared client items in Client

Author: HJLebbink

.github/workflows/rust.yml

@@ -23,7 +23,7 @@ jobs:
         cargo clippy --all-targets --all-features
         cargo build --bins --examples --tests --benches --verbose
 
-    - name: Run tests
+    - name: Run tests S3
       run: |
         ./tests/start-server.sh
         export SERVER_ENDPOINT=localhost:9000

Cargo.toml

@@ -48,7 +48,7 @@ ring = { version = "0.17.14", optional = true, default-features = false, feature
 serde = { version = "1.0.219", features = ["derive"] }
 serde_json = "1.0.140"
 sha2 = { version = "0.10.8", optional = true }
-tokio = { version = "1.44.1", features = ["full"] }
+tokio = { version = "1.44.2", features = ["full"] }
 tokio-stream = "0.1.17"
 tokio-util = { version = "0.7.14", features = ["io"] }
 urlencoding = "2.1.3"
@@ -59,7 +59,7 @@ http = "1.3.1"
 [dev-dependencies]
 minio_common = { path = "./common" }
 async-std = { version = "1.13.1", features = ["attributes", "tokio1"] }
-clap = { version = "4.5.34", features = ["derive"] }
+clap = { version = "4.5.35", features = ["derive"] }
 quickcheck = "1.0.3"
 criterion = "0.5.1"
 
@@ -76,6 +76,9 @@ name = "file_downloader"
 [[example]]
 name = "object_prompt"
 
+[[example]]
+name = "append_object"
+
 [[bench]]
 name = "s3-api"
 path = "benches/s3/api_benchmarks.rs"

benches/s3/api_benchmarks.rs

@@ -21,14 +21,15 @@ mod bench_bucket_replication;
 mod bench_bucket_tags;
 mod bench_bucket_versioning;
 mod bench_list_bucket;
+mod bench_object_append;
+mod bench_object_copy;
 mod bench_object_legal_hold;
 mod bench_object_lock_config;
+mod bench_object_put;
 mod bench_object_retention;
 mod bench_object_tags;
 mod common_benches;
 
-mod bench_object_copy;
-
 use criterion::{Criterion, criterion_group, criterion_main};
 use std::time::Duration;
 
@@ -43,9 +44,12 @@ use crate::bench_bucket_tags::*;
 use crate::bench_bucket_versioning::*;
 use crate::bench_list_bucket::*;
 #[allow(unused_imports)]
+use crate::bench_object_append::bench_object_append;
+#[allow(unused_imports)]
 use crate::bench_object_copy::*;
 use crate::bench_object_legal_hold::*;
 use crate::bench_object_lock_config::*;
+use crate::bench_object_put::bench_object_put;
 use crate::bench_object_retention::*;
 use crate::bench_object_tags::*;
 
@@ -54,9 +58,9 @@ criterion_group!(
     config = Criterion::default()
         .configure_from_args()
         .warm_up_time(Duration::from_secs_f32(0.01))
-        .sample_size(100)
+        .sample_size(1000)
         .nresamples(1001)
-        .measurement_time(Duration::from_secs_f32(0.5));
+        .measurement_time(Duration::from_secs_f32(10.0));
     targets =
         bench_bucket_exists,
         bench_set_bucket_lifecycle,
@@ -83,7 +87,9 @@ criterion_group!(
         bench_get_bucket_versioning,
         //
         bench_list_buckets,
-        //bench_object_copy, //TODO first refactor object_copy
+        bench_object_copy_internal,
+        bench_object_append,
+        bench_object_put,
         //
         bench_enable_object_legal_hold,
         bench_disable_object_legal_hold,

benches/s3/bench_bucket_exists.rs

@@ -23,6 +23,6 @@ pub(crate) fn bench_bucket_exists(criterion: &mut Criterion) {
         "bucket_exists",
         criterion,
         || async { Ctx2::new().await },
-        |ctx| BucketExists::new(&ctx.bucket).client(&ctx.client),
+        |ctx| BucketExists::new(ctx.client.clone(), ctx.bucket.clone()),
     );
 }

benches/s3/bench_bucket_lifecycle.rs

@@ -27,8 +27,7 @@ pub(crate) fn bench_set_bucket_lifecycle(criterion: &mut Criterion) {
         || async { Ctx2::new().await },
         |ctx| {
             let config = create_bucket_lifecycle_config_examples();
-            SetBucketLifecycle::new(&ctx.bucket)
-                .client(&ctx.client)
+            SetBucketLifecycle::new(ctx.client.clone(), ctx.bucket.clone())
                 .life_cycle_config(config)
         },
     )
@@ -40,22 +39,22 @@ pub(crate) fn bench_get_bucket_lifecycle(criterion: &mut Criterion) {
         || async {
             let ctx = Ctx2::new().await;
             let config = create_bucket_lifecycle_config_examples();
-            SetBucketLifecycle::new(&ctx.bucket)
-                .client(&ctx.client)
+            ctx.client
+                .set_bucket_lifecycle(&ctx.bucket)
                 .life_cycle_config(config)
                 .send()
                 .await
                 .unwrap();
             ctx
         },
-        |ctx| GetBucketLifecycle::new(&ctx.bucket).client(&ctx.client),
+        |ctx| GetBucketLifecycle::new(ctx.client.clone(), ctx.bucket.clone()),
     )
 }
 pub(crate) fn bench_delete_bucket_lifecycle(criterion: &mut Criterion) {
     benchmark_s3_api(
         "delete_bucket_lifecycle",
         criterion,
         || async { Ctx2::new().await },
-        |ctx| DeleteBucketLifecycle::new(&ctx.bucket).client(&ctx.client),
+        |ctx| DeleteBucketLifecycle::new(ctx.client.clone(), ctx.bucket.clone()),
     )
 }

benches/s3/bench_bucket_notification.rs

@@ -28,8 +28,7 @@ pub(crate) fn bench_set_bucket_notification(criterion: &mut Criterion) {
         || async { Ctx2::new().await },
         |ctx| {
             let config = create_bucket_notification_config_example();
-            SetBucketNotification::new(&ctx.bucket)
-                .client(&ctx.client)
+            SetBucketNotification::new(ctx.client.clone(), ctx.bucket.clone())
                 .notification_config(config)
         },
     )
@@ -42,15 +41,15 @@ pub(crate) fn bench_get_bucket_notification(criterion: &mut Criterion) {
         || async {
             let ctx = Ctx2::new().await;
             let config = create_bucket_notification_config_example();
-            SetBucketNotification::new(&ctx.bucket)
-                .client(&ctx.client)
+            ctx.client
+                .set_bucket_notification(&ctx.bucket)
                 .notification_config(config)
                 .send()
                 .await
                 .unwrap();
             ctx
         },
-        |ctx| GetBucketNotification::new(&ctx.bucket).client(&ctx.client),
+        |ctx| GetBucketNotification::new(ctx.client.clone(), ctx.bucket.clone()),
     )
 }
 #[allow(dead_code)]
@@ -59,6 +58,6 @@ pub(crate) fn bench_delete_bucket_notification(criterion: &mut Criterion) {
         "delete_bucket_notification",
         criterion,
         || async { Ctx2::new().await },
-        |ctx| DeleteBucketNotification::new(&ctx.bucket).client(&ctx.client),
+        |ctx| DeleteBucketNotification::new(ctx.client.clone(), ctx.bucket.clone()),
     )
 }

benches/s3/bench_bucket_policy.rs

@@ -27,9 +27,7 @@ pub(crate) fn bench_set_bucket_policy(criterion: &mut Criterion) {
         || async { Ctx2::new().await },
         |ctx| {
             let config = create_bucket_policy_config_example(&ctx.bucket);
-            SetBucketPolicy::new(&ctx.bucket)
-                .client(&ctx.client)
-                .config(config)
+            SetBucketPolicy::new(ctx.client.clone(), ctx.bucket.clone()).config(config)
         },
     )
 }
@@ -40,22 +38,21 @@ pub(crate) fn bench_get_bucket_policy(criterion: &mut Criterion) {
         || async {
             let ctx = Ctx2::new().await;
             let config = create_bucket_policy_config_example(&ctx.bucket);
-            SetBucketPolicy::new(&ctx.bucket)
-                .client(&ctx.client)
+            SetBucketPolicy::new(ctx.client.clone(), ctx.bucket.clone())
                 .config(config)
                 .send()
                 .await
                 .unwrap();
             ctx
         },
-        |ctx| GetBucketPolicy::new(&ctx.bucket).client(&ctx.client),
+        |ctx| GetBucketPolicy::new(ctx.client.clone(), ctx.bucket.clone()),
     )
 }
 pub(crate) fn bench_delete_bucket_policy(criterion: &mut Criterion) {
     benchmark_s3_api(
         "delete_bucket_policy",
         criterion,
         || async { Ctx2::new().await },
-        |ctx| DeleteBucketPolicy::new(&ctx.bucket).client(&ctx.client),
+        |ctx| DeleteBucketPolicy::new(ctx.client.clone(), ctx.bucket.clone()),
     )
 }

benches/s3/bench_bucket_replication.rs

@@ -53,8 +53,7 @@ pub(crate) fn bench_set_bucket_replication(criterion: &mut Criterion) {
         |ctx| {
             let config =
                 create_bucket_replication_config_example(ctx.aux_bucket.clone().unwrap().as_str());
-            SetBucketReplication::new(&ctx.bucket)
-                .client(&ctx.client)
+            SetBucketReplication::new(ctx.client.clone(), ctx.bucket.clone())
                 .replication_config(config)
         },
     )
@@ -86,7 +85,7 @@ pub(crate) fn bench_get_bucket_replication(criterion: &mut Criterion) {
 
             ctx
         },
-        |ctx| GetBucketReplication::new(&ctx.bucket).client(&ctx.client),
+        |ctx| GetBucketReplication::new(ctx.client.clone(), ctx.bucket.clone()),
     )
 }
 #[allow(dead_code)]
@@ -116,6 +115,6 @@ pub(crate) fn bench_delete_bucket_replication(criterion: &mut Criterion) {
 
             ctx
         },
-        |ctx| DeleteBucketReplication::new(&ctx.bucket).client(&ctx.client),
+        |ctx| DeleteBucketReplication::new(ctx.client.clone(), ctx.bucket.clone()),
     )
 }

benches/s3/bench_bucket_tags.rs

@@ -13,49 +13,54 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common_benches::{Ctx2, benchmark_s3_api};
+use crate::common_benches::{Ctx2, benchmark_s3_api, skip_express_mode};
 
 use criterion::Criterion;
 use minio::s3::builders::{DeleteBucketTags, GetBucketTags, SetBucketTags};
-use minio::s3::response::SetBucketTagsResponse;
 use minio::s3::types::S3Api;
 use minio_common::example::create_tags_example;
 
 pub(crate) fn bench_set_bucket_tags(criterion: &mut Criterion) {
+    if skip_express_mode("bench_set_bucket_tags") {
+        return;
+    }
     benchmark_s3_api(
         "set_bucket_tags",
         criterion,
         || async { Ctx2::new().await },
         |ctx| {
-            SetBucketTags::new(&ctx.bucket)
-                .client(&ctx.client)
-                .tags(create_tags_example())
+            SetBucketTags::new(ctx.client.clone(), ctx.bucket.clone()).tags(create_tags_example())
         },
     )
 }
 pub(crate) fn bench_get_bucket_tags(criterion: &mut Criterion) {
+    if skip_express_mode("bench_get_bucket_tags") {
+        return;
+    }
     benchmark_s3_api(
         "get_bucket_tags",
         criterion,
         || async {
             let ctx = Ctx2::new().await;
-            let _resp: SetBucketTagsResponse = ctx
-                .client
+            ctx.client
                 .set_bucket_tags(&ctx.bucket)
                 .tags(create_tags_example())
                 .send()
                 .await
                 .unwrap();
             ctx
         },
-        |ctx| GetBucketTags::new(&ctx.bucket).client(&ctx.client),
+        |ctx| GetBucketTags::new(ctx.client.clone(), ctx.bucket.clone()),
     )
 }
 pub(crate) fn bench_delete_bucket_tags(criterion: &mut Criterion) {
+    if skip_express_mode("bench_delete_bucket_tags") {
+        return;
+    }
     benchmark_s3_api(
         "delete_bucket_tags",
         criterion,
         || async { Ctx2::new().await },
-        |ctx| DeleteBucketTags::new(&ctx.bucket).client(&ctx.client),
+        |ctx| DeleteBucketTags::new(ctx.client.clone(), ctx.bucket.clone()),
     )
 }

benches/s3/bench_bucket_versioning.rs

@@ -13,29 +13,33 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common_benches::{Ctx2, benchmark_s3_api};
+use crate::common_benches::{Ctx2, benchmark_s3_api, skip_express_mode};
 
 use criterion::Criterion;
 use minio::s3::builders::{GetBucketVersioning, SetBucketVersioning, VersioningStatus};
 
 pub(crate) fn bench_get_bucket_versioning(criterion: &mut Criterion) {
+    if skip_express_mode("bench_get_bucket_versioning") {
+        return;
+    }
     benchmark_s3_api(
         "get_bucket_versioning",
         criterion,
         || async { Ctx2::new().await },
-        |ctx| GetBucketVersioning::new(&ctx.bucket).client(&ctx.client),
+        |ctx| GetBucketVersioning::new(ctx.client.clone(), ctx.bucket.clone()),
     )
 }
 pub(crate) fn bench_set_bucket_versioning(criterion: &mut Criterion) {
+    if skip_express_mode("bench_set_bucket_versioning") {
+        return;
+    }
     benchmark_s3_api(
         "set_bucket_versioning",
         criterion,
         || async { Ctx2::new().await },
         |ctx| {
-            let status = VersioningStatus::Enabled;
-            SetBucketVersioning::new(&ctx.bucket)
-                .client(&ctx.client)
-                .versioning_status(status)
+            SetBucketVersioning::new(ctx.client.clone(), ctx.bucket.clone())
+                .versioning_status(VersioningStatus::Enabled)
         },
     )
 }