update tests

Author: HJLebbink

Cargo.toml

@@ -62,7 +62,6 @@ async-std = { version = "1.13.1", features = ["attributes", "tokio1"] }
 clap = { version = "4.5.35", features = ["derive"] }
 quickcheck = "1.0.3"
 criterion = "0.5.1"
-test-tag = "0.1.4"
 
 [lib]
 name = "minio"

common/src/cleanup_guard.rs

@@ -58,7 +58,9 @@ impl Drop for CleanupGuard {
                         Ok(_) => {
                             //println!("Bucket {} removed successfully", bucket_name),
                         }
-                        Err(e) => println!("Error removing bucket {}: {:?}", bucket_name, e),
+                        Err(_e) => {
+                            //println!("Error removing bucket {}: {:?}", bucket_name, e)
+                        }
                     },
                     Err(_) => println!("Timeout after 60s while removing bucket {}", bucket_name),
                 }

common/src/example.rs

@@ -16,9 +16,10 @@
 use chrono::{DateTime, Utc};
 use minio::s3::builders::PostPolicy;
 use minio::s3::types::{
-    AndOperator, Destination, Filter, LifecycleConfig, LifecycleRule, NotificationConfig,
-    ObjectLockConfig, PrefixFilterRule, QueueConfig, ReplicationConfig, ReplicationRule,
-    RetentionMode, SuffixFilterRule,
+    AndOperator, CsvInputSerialization, CsvOutputSerialization, Destination, FileHeaderInfo,
+    Filter, LifecycleConfig, LifecycleRule, NotificationConfig, ObjectLockConfig, PrefixFilterRule,
+    QueueConfig, QuoteFields, ReplicationConfig, ReplicationRule, RetentionMode, SelectRequest,
+    SuffixFilterRule,
 };
 use minio::s3::utils::utc_now;
 use std::collections::HashMap;
@@ -195,3 +196,38 @@ pub fn create_post_policy_example(bucket_name: &str, object_name: &str) -> PostP
         .unwrap();
     policy
 }
+/// return (body, data)
+pub fn create_select_content_data() -> (String, String) {
+    let mut data = String::new();
+    data.push_str("1997,Ford,E350,\"ac, abs, moon\",3000.00\n");
+    data.push_str("1999,Chevy,\"Venture \"\"Extended Edition\"\"\",,4900.00\n");
+    data.push_str("1999,Chevy,\"Venture \"\"Extended Edition, Very Large\"\"\",,5000.00\n");
+    data.push_str("1996,Jeep,Grand Cherokee,\"MUST SELL!\n");
+    data.push_str("air, moon roof, loaded\",4799.00\n");
+    let body = String::from("Year,Make,Model,Description,Price\n") + &data;
+    (body, data)
+}
+pub fn create_select_content_request() -> SelectRequest {
+    let request = SelectRequest::new_csv_input_output(
+        "select * from S3Object",
+        CsvInputSerialization {
+            compression_type: None,
+            allow_quoted_record_delimiter: false,
+            comments: None,
+            field_delimiter: None,
+            file_header_info: Some(FileHeaderInfo::USE),
+            quote_character: None,
+            quote_escape_character: None,
+            record_delimiter: None,
+        },
+        CsvOutputSerialization {
+            field_delimiter: None,
+            quote_character: None,
+            quote_escape_character: None,
+            quote_fields: Some(QuoteFields::ASNEEDED),
+            record_delimiter: None,
+        },
+    )
+    .unwrap();
+    request
+}

src/s3/builders/append_object.rs

@@ -227,7 +227,7 @@ impl AppendObjectContent {
             .stat_object(&self.bucket, &self.object)
             .send()
             .await?;
-        println!("statObjectResponse={:#?}", resp);
+        //println!("statObjectResponse={:#?}", resp);
 
         let current_file_size = resp.size;
 

src/s3/client.rs

@@ -19,7 +19,7 @@ use std::fs::File;
 use std::io::prelude::*;
 use std::mem;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
 
 use crate::s3::creds::Provider;
 use crate::s3::error::{Error, ErrorResponse};
@@ -28,12 +28,15 @@ use crate::s3::response::*;
 use crate::s3::signer::sign_v4_s3;
 use crate::s3::utils::{EMPTY_SHA256, Multimap, sha256_hash_sb, to_amz_date, utc_now};
 
-use crate::s3::builders::ComposeSource;
+use crate::s3::builders::{BucketExists, ComposeSource};
 use crate::s3::segmented_bytes::SegmentedBytes;
 use bytes::Bytes;
 use dashmap::DashMap;
 use hyper::http::Method;
+use rand::Rng;
+use rand::distributions::Alphanumeric;
 use reqwest::Body;
+use tokio::task;
 
 mod append_object;
 mod bucket_exists;
@@ -186,7 +189,7 @@ impl ClientBuilder {
             client: builder.build()?,
             base_url: self.base_url,
             provider: self.provider,
-            region_map: Arc::default(),
+            ..Default::default()
         })
     }
 }
@@ -200,7 +203,8 @@ pub struct Client {
     client: reqwest::Client,
     pub(crate) base_url: BaseUrl,
     pub(crate) provider: Option<Arc<Box<(dyn Provider + Send + Sync + 'static)>>>,
-    pub(crate) region_map: Arc<DashMap<String, String>>,
+    pub(crate) region_map: DashMap<String, String>,
+    express: OnceLock<bool>,
 }
 
 impl Client {
@@ -234,14 +238,53 @@ impl Client {
             .build()
     }
 
+    /// Returns whether is client uses an AWS host.
     pub fn is_aws_host(&self) -> bool {
         self.base_url.is_aws_host()
     }
 
+    /// Returns whether this client is configured to use HTTPS.
     pub fn is_secure(&self) -> bool {
         self.base_url.https
     }
 
+    /// Returns whether this client is configured to use the express endpoint and is minio enterprise.
+    pub fn is_minio_express(self: &Arc<Self>) -> bool {
+        if self.express.get().is_some() {
+            self.express.get().unwrap().clone()
+        } else {
+            task::block_in_place(|| match tokio::runtime::Runtime::new() {
+                Ok(rt) => {
+                    let bucket_name: String = rand::thread_rng()
+                        .sample_iter(&Alphanumeric)
+                        .take(20)
+                        .map(char::from)
+                        .collect::<String>()
+                        .to_lowercase();
+
+                    let express = rt.block_on(async {
+                        match BucketExists::new(self, bucket_name).send().await {
+                            Ok(v) => {
+                                if let Some(server) = v.headers.get("server") {
+                                    if let Ok(s) = server.to_str() {
+                                        return s == "MinIO Enterprise/S3express";
+                                    }
+                                }
+                            }
+                            Err(e) => {
+                                println!("is_express_internal: error: {e}\nassume false");
+                            }
+                        }
+                        false
+                    });
+                    self.express.set(express).unwrap_or_default();
+                    express
+                }
+                Err(_) => false,
+            })
+        }
+    }
+
     fn handle_redirect_response(
         &self,
         status_code: u16,
@@ -297,7 +340,7 @@ impl Client {
             return match headers.get("Content-Type") {
                 Some(v) => match v.to_str() {
                     Ok(s) => match s.to_lowercase().contains("application/xml") {
-                        true => match ErrorResponse::parse(body) {
+                        true => match ErrorResponse::parse(body, headers) {
                             Ok(v) => Error::S3Error(v),
                             Err(e) => e,
                         },
@@ -349,15 +392,15 @@ impl Client {
             _ => return Error::ServerError(status_code),
         };
 
-        let request_id = match headers.get("x-amz-request-id") {
+        let request_id: String = match headers.get("x-amz-request-id") {
             Some(v) => match v.to_str() {
                 Ok(s) => s.to_string(),
                 Err(e) => return Error::StrError(e),
             },
             _ => String::new(),
         };
 
-        let host_id = match headers.get("x-amz-id-2") {
+        let host_id: String = match headers.get("x-amz-id-2") {
             Some(v) => match v.to_str() {
                 Ok(s) => s.to_string(),
                 Err(e) => return Error::StrError(e),
@@ -366,6 +409,7 @@ impl Client {
         };
 
         Error::S3Error(ErrorResponse {
+            headers,
             code,
             message,
             resource: resource.to_string(),
@@ -376,7 +420,7 @@ impl Client {
         })
     }
 
-    pub async fn do_execute(
+    async fn execute_internal(
         &self,
         method: &Method,
         region: &str,
@@ -447,6 +491,31 @@ impl Client {
             }
         }
 
+        if false {
+            let mut header_strings: Vec<String> = headers
+                .iter_all()
+                .map(|(k, v)| format!("{}: {}", k, v.join(",")))
+                .collect();
+
+            // Sort headers alphabetically by name
+            header_strings.sort();
+
+            let body_str: String = String::from_utf8(
+                body.clone()
+                    .unwrap_or(&SegmentedBytes::new())
+                    .to_bytes()
+                    .to_vec(),
+            )?;
+
+            println!(
+                "S3 request: {} url={:?}; headers={:?}; body={}\n",
+                method,
+                url.path,
+                header_strings.join("; "),
+                body_str
+            );
+        }
+
         if *method == Method::PUT || *method == Method::POST {
             let mut bytes_vec = vec![];
             if let Some(body) = body {
@@ -461,15 +530,17 @@ impl Client {
         }
 
         let resp = req.send().await?;
+
         if resp.status().is_success() {
             return Ok(resp);
         }
 
         let mut resp = resp;
         let status_code = resp.status().as_u16();
         let headers: reqwest::header::HeaderMap = mem::take(resp.headers_mut());
+
         let body: Bytes = resp.bytes().await?;
-        let e = self.get_error_response(
+        let e: Error = self.get_error_response(
             body,
             status_code,
             headers,
@@ -504,8 +575,8 @@ impl Client {
         object_name: &Option<&str>,
         data: Option<&SegmentedBytes>,
     ) -> Result<reqwest::Response, Error> {
-        let res = self
-            .do_execute(
+        let resp: Result<reqwest::Response, Error> = self
+            .execute_internal(
                 &method,
                 region,
                 headers,
@@ -516,7 +587,7 @@ impl Client {
                 true,
             )
             .await;
-        match res {
+        match resp {
             Ok(r) => return Ok(r),
             Err(e) => match e {
                 Error::S3Error(ref er) => {
@@ -529,7 +600,7 @@ impl Client {
         };
 
         // Retry only once on RetryHead error.
-        self.do_execute(
+        self.execute_internal(
             &method,
             region,
             headers,