ilm feature add

ecstore/src/client/admin_handler_utils.rs

@@ -0,0 +1,33 @@
+use http::status::StatusCode;
+use std::fmt::{self, Display, Formatter};
+
+#[derive(Default, thiserror::Error, Debug, PartialEq)]
+pub struct AdminError {
+    pub code: &'static str,
+    pub message: &'static str,
+    pub status_code: StatusCode,
+}
+
+impl Display for AdminError {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "{}", self.message)
+    }
+}
+
+impl AdminError {
+    pub fn new(code: &'static str, message: &'static str, status_code: StatusCode) -> Self {
+        Self {
+            code,
+            message,
+            status_code,
+        }
+    }
+
+    pub fn msg(message: &'static str) -> Self {
+        Self {
+            code: "InternalError",
+            message,
+            status_code: StatusCode::INTERNAL_SERVER_ERROR,
+        }
+    }
+}

ecstore/src/client/api_bucket_policy.rs

@@ -0,0 +1,113 @@
+#![allow(clippy::map_entry)]
+use std::collections::HashMap;
+use http::{HeaderMap, StatusCode};
+use bytes::Bytes;
+
+use crate::client::{
+    api_error_response::http_resp_to_error_response,
+    transition_api::{RequestMetadata, TransitionClient, ReaderImpl}
+};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+
+impl TransitionClient {
+    pub async fn set_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
+        if policy == "" {
+            return self.remove_bucket_policy(bucket_name).await;
+        }
+
+        self.put_bucket_policy(bucket_name, policy).await
+    }
+
+    pub async fn put_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("policy".to_string(), "".to_string());
+
+        let mut req_metadata = RequestMetadata {
+            bucket_name:    bucket_name.to_string(),
+            query_values:   url_values,
+            content_body:   ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())),
+            content_length: policy.len() as i64,
+            object_name: "".to_string(),
+            custom_header: HeaderMap::new(),
+            content_md5_base64: "".to_string(),
+            content_sha256_hex: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
+        //defer closeResponse(resp)
+        //if resp != nil {
+            if resp.status() != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK {
+                return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
+            }
+        //}
+        Ok(())
+    }
+
+    pub async fn remove_bucket_policy(&self, bucket_name: &str) -> Result<(), std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("policy".to_string(), "".to_string());
+
+        let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata {
+            bucket_name:       bucket_name.to_string(),
+            query_values:      url_values,
+            content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+            object_name: "".to_string(),
+            custom_header: HeaderMap::new(),
+            content_body:  ReaderImpl::Body(Bytes::new()),
+            content_length: 0,
+            content_md5_base64: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        }).await?;
+        //defer closeResponse(resp)
+
+        if resp.status() != StatusCode::NO_CONTENT {
+            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
+        }
+
+        Ok(())
+    }
+
+    pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        let bucket_policy = self.get_bucket_policy_inner(bucket_name).await?;
+        Ok(bucket_policy)
+    }
+
+    pub async fn get_bucket_policy_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("policy".to_string(), "".to_string());
+
+        let resp = self.execute_method(http::Method::GET, &mut RequestMetadata {
+            bucket_name:       bucket_name.to_string(),
+            query_values:      url_values,
+            content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+            object_name: "".to_string(),
+            custom_header: HeaderMap::new(),
+            content_body:  ReaderImpl::Body(Bytes::new()),
+            content_length: 0,
+            content_md5_base64: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        }).await?;
+
+        let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string();
+        Ok(policy)
+    }
+}

ecstore/src/client/api_error_response.rs

@@ -0,0 +1,252 @@
+#![allow(clippy::map_entry)]
+use std::fmt::Display;
+use http::StatusCode;
+use serde::{Serialize, Deserialize};
+use serde::{ser::Serializer, de::Deserializer};
+
+use s3s::S3ErrorCode;
+use s3s::Body;
+
+const REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues.";
+
+#[derive(Serialize, Deserialize)]
+#[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct ErrorResponse {
+    #[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")]
+    pub code:        S3ErrorCode,
+    pub message:     String,
+    pub bucket_name: String,
+    pub key:         String,
+    pub resource:    String,
+    pub request_id:  String,
+    pub host_id:     String,
+    pub region: String,
+    pub server: String,
+    #[serde(skip)]
+    pub status_code: StatusCode,
+}
+
+fn serialize_code<S>(data: &S3ErrorCode, s: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer
+{
+    s.serialize_str("")
+}
+
+fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
+where
+    D: Deserializer<'de>
+{
+    Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
+}
+
+impl Default for ErrorResponse {
+    fn default() -> Self {
+        ErrorResponse {
+            code: S3ErrorCode::Custom("".into()),
+            message:     Default::default(),
+            bucket_name: Default::default(),
+            key:         Default::default(),
+            resource:    Default::default(),
+            request_id:  Default::default(),
+            host_id:     Default::default(),
+            region:      Default::default(),
+            server:      Default::default(),
+            status_code: Default::default(),
+        }
+    }
+}
+
+impl Display for ErrorResponse {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.message)
+    }
+}
+
+pub fn to_error_response(err: &std::io::Error) -> ErrorResponse {
+    if let Some(err) = err.get_ref() {
+        if err.is::<ErrorResponse>() {
+            err.downcast_ref::<ErrorResponse>().expect("err!").clone()
+        } else {
+            ErrorResponse::default()
+        }
+    } else {
+        ErrorResponse::default()
+    }
+}
+
+pub fn http_resp_to_error_response(resp: http::Response<Body>, b: Vec<u8>, bucket_name: &str, object_name: &str) -> ErrorResponse {
+    let err_body = String::from_utf8(b).unwrap();
+    //let err_body = xml_decode_and_body(resp.body, &err_resp);
+    let err_resp_ = serde_xml_rs::from_str::<ErrorResponse>(&err_body);
+    let mut err_resp = ErrorResponse::default();
+    if err_resp_.is_err() {
+        match resp.status() {
+            StatusCode::NOT_FOUND => {
+                if object_name == "" {
+                    err_resp = ErrorResponse {
+                        status_code: resp.status(),
+                        code:        S3ErrorCode::NoSuchBucket,
+                        message:     "The specified bucket does not exist.".to_string(),
+                        bucket_name: bucket_name.to_string(),
+                        ..Default::default()
+                    };
+                } else {
+                    err_resp = ErrorResponse {
+                        status_code: resp.status(),
+                        code:        S3ErrorCode::NoSuchKey,
+                        message:     "The specified key does not exist.".to_string(),
+                        bucket_name: bucket_name.to_string(),
+                        key:         object_name.to_string(),
+                        ..Default::default()
+                    };
+                }
+            }
+            StatusCode::FORBIDDEN => {
+                err_resp = ErrorResponse {
+                    status_code: resp.status(),
+                    code:        S3ErrorCode::AccessDenied,
+                    message:     "Access Denied.".to_string(),
+                    bucket_name: bucket_name.to_string(),
+                    key:         object_name.to_string(),
+                    ..Default::default()
+                };
+            }
+            StatusCode::CONFLICT => {
+                err_resp = ErrorResponse {
+                    status_code: resp.status(),
+                    code:        S3ErrorCode::BucketNotEmpty,
+                    message:     "Bucket not empty.".to_string(),
+                    bucket_name: bucket_name.to_string(),
+                    ..Default::default()
+                };
+            }
+            StatusCode::PRECONDITION_FAILED => {
+                err_resp = ErrorResponse {
+                    status_code: resp.status(),
+                    code:        S3ErrorCode::PreconditionFailed,
+                    message:     "Pre condition failed.".to_string(),
+                    bucket_name: bucket_name.to_string(),
+                    key:         object_name.to_string(),
+                    ..Default::default()
+                };
+            }
+            _ => {
+                let mut msg = resp.status().to_string();
+                if err_body.len() > 0 {
+                    msg = err_body;
+                }
+                err_resp = ErrorResponse{
+                    status_code: resp.status(),
+                    code:       S3ErrorCode::Custom(resp.status().to_string().into()),
+                    message:    msg,
+                    bucket_name: bucket_name.to_string(),
+                    ..Default::default()
+                };
+            }
+        }
+    } else {
+        err_resp = err_resp_.unwrap();
+    }
+    err_resp.status_code = resp.status();
+    if let Some(server_name) = resp.headers().get("Server") {
+        err_resp.server = server_name.to_str().expect("err").to_string();
+    }
+
+    let code = resp.headers().get("x-minio-error-code");
+    if code.is_some() {
+        err_resp.code = S3ErrorCode::Custom(code.expect("err").to_str().expect("err").into());
+    }
+    let desc = resp.headers().get("x-minio-error-desc");
+    if desc.is_some() {
+        err_resp.message = desc.expect("err").to_str().expect("err").trim_matches('"').to_string();
+    }
+
+    if err_resp.request_id == "" {
+        if let Some(x_amz_request_id) = resp.headers().get("x-amz-request-id") {
+            err_resp.request_id = x_amz_request_id.to_str().expect("err").to_string();
+        }
+    }
+    if err_resp.host_id == "" {
+        if let Some(x_amz_id_2) = resp.headers().get("x-amz-id-2") {
+            err_resp.host_id = x_amz_id_2.to_str().expect("err").to_string();
+        }
+    }
+    if err_resp.region == "" {
+        if let Some(x_amz_bucket_region) = resp.headers().get("x-amz-bucket-region") {
+            err_resp.region = x_amz_bucket_region.to_str().expect("err").to_string();
+        }
+    }
+    if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" {
+        err_resp.message = format!("Region does not match, expecting region ‘{}’.", err_resp.region);
+    }
+
+    err_resp
+}
+
+pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code:        S3ErrorCode::InvalidArgument,
+        message:     "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’.".to_string(),
+        bucket_name: bucket_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
+    let msg = format!("Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.", total_size, max_object_size);
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code:        S3ErrorCode::EntityTooLarge,
+        message:     msg,
+        bucket_name: bucket_name.to_string(),
+        key:         object_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
+    let msg = format!("Your proposed upload size ‘{}’ is below the minimum allowed object size ‘0B’ for single PUT operation.", total_size);
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code:        S3ErrorCode::EntityTooSmall,
+        message:     msg,
+        bucket_name: bucket_name.to_string(),
+        key:         object_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
+    let msg = format!("Data read ‘{}’ is not equal to the size ‘{}’ of the input Reader.", total_read, total_size);
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code:        S3ErrorCode::Custom("UnexpectedEOF".into()),
+        message:     msg,
+        bucket_name: bucket_name.to_string(),
+        key:         object_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_invalid_argument(message: &str) -> ErrorResponse {
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code:        S3ErrorCode::InvalidArgument,
+        message:     message.to_string(),
+        request_id:  "rustfs".to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_api_not_supported(message: &str) -> ErrorResponse {
+    ErrorResponse {
+        status_code: StatusCode::NOT_IMPLEMENTED,
+        code:        S3ErrorCode::Custom("APINotSupported".into()),
+        message:     message.to_string(),
+        request_id:  "rustfs".to_string(),
+        ..Default::default()
+    }
+}

ecstore/src/client/api_get_object.rs

@@ -0,0 +1,198 @@
+#![allow(clippy::map_entry)]
+use bytes::Bytes;
+use http::HeaderMap;
+use tokio::io::BufReader;
+use std::io::Cursor;
+
+use crate::client::{
+    transition_api::{ObjectInfo, to_object_info, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient},
+    api_error_response::err_invalid_argument,
+    api_get_options::GetObjectOptions,
+};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+
+impl TransitionClient {
+    pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
+        todo!();
+    }
+
+    pub async fn get_object_inner(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
+        let resp = self.execute_method(http::Method::GET, &mut RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            object_name: object_name.to_string(),
+            query_values:      opts.to_query_values(),
+            custom_header:     opts.header(),
+            content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+            content_body:  ReaderImpl::Body(Bytes::new()),
+            content_length: 0,
+            content_md5_base64: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        }).await?;
+
+        let resp = &resp;
+        let object_stat = to_object_info(bucket_name, object_name, resp.headers())?;
+
+        let b = resp.body().bytes().expect("err").to_vec();
+        Ok((object_stat, resp.headers().clone(), BufReader::new(Cursor::new(b))))
+    }
+}
+
+#[derive(Default)]
+struct GetRequest {
+    pub buffer:              Vec<u8>,
+    pub offset:              i64,
+    pub did_offset_change:   bool,
+    pub been_read:           bool,
+    pub is_read_at:           bool,
+    pub is_read_op:           bool,
+    pub is_first_req:         bool,
+    pub setting_object_info: bool,
+}
+
+struct GetResponse {
+    pub size:        i64,
+    //pub error:       error,
+    pub did_read:    bool,
+    pub object_info: ObjectInfo,
+}
+
+#[derive(Default)]
+struct Object {
+    //pub reqch:      chan<- getRequest,
+    //pub resch:      <-chan getResponse,
+    //pub cancel:     context.CancelFunc,
+    pub curr_offset: i64,
+    pub object_info: ObjectInfo,
+    pub seek_data: bool,
+    pub is_closed: bool,
+    pub is_started: bool,
+    //pub prev_err: error,
+    pub been_read: bool,
+    pub object_info_set: bool,
+}
+
+impl Object {
+    pub fn new() -> Object {
+        Self {
+          ..Default::default()
+        }
+    }
+
+    fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
+        todo!()
+    }
+
+    fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
+        self.curr_offset += bytes_read;
+
+        Ok(())
+    }
+
+    fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
+        let mut read_req = GetRequest {
+            is_read_op: true,
+            been_read: self.been_read,
+            buffer: b.to_vec(),
+            ..Default::default()
+        };
+
+        if !self.is_started {
+            read_req.is_first_req = true;
+        }
+
+        read_req.did_offset_change = self.seek_data;
+        read_req.offset = self.curr_offset;
+
+        let response = self.do_get_request(&read_req)?;
+
+        let bytes_read = response.size;
+
+        let oerr = self.set_offset(bytes_read);
+
+        Ok(response.size)
+    }
+
+    fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
+        if !self.is_started || !self.object_info_set {
+            let _ = self.do_get_request(&GetRequest {
+                is_first_req:        !self.is_started,
+                setting_object_info: !self.object_info_set,
+                ..Default::default()
+            })?;
+        }
+
+        Ok(self.object_info.clone())
+    }
+
+    fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
+        self.curr_offset = offset;
+
+        let mut read_at_req = GetRequest {
+            is_read_op:        true,
+            is_read_at:        true,
+            did_offset_change: true,
+            been_read:         self.been_read,
+            offset,
+            buffer:            b.to_vec(),
+            ..Default::default()
+        };
+
+        if !self.is_started {
+            read_at_req.is_first_req = true;
+        }
+
+        let response = self.do_get_request(&read_at_req)?;
+        let bytes_read = response.size;
+        if !self.object_info_set {
+            self.curr_offset += bytes_read;
+        } else {
+            let oerr = self.set_offset(bytes_read);
+        }
+        Ok(response.size)
+    }
+
+    fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
+        if !self.is_started || !self.object_info_set {
+            let seek_req = GetRequest {
+                is_read_op:   false,
+                offset:     offset,
+                is_first_req: true,
+                ..Default::default()
+            };
+            let _ = self.do_get_request(&seek_req);
+        }
+
+        let mut new_offset = self.curr_offset;
+
+        match whence {
+            0 => {
+                new_offset = offset;
+            }
+            1 => {
+                new_offset += offset;
+            }
+            2 => {
+                new_offset = self.object_info.size as i64 + offset as i64;
+            }
+            _ => {
+                return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid whence {}", whence))));
+            }
+        }
+
+        self.seek_data = (new_offset != self.curr_offset) || self.seek_data;
+        self.curr_offset = new_offset;
+
+        Ok(self.curr_offset)
+    }
+
+    fn close(&mut self) -> Result<(), std::io::Error> {
+        self.is_closed = true;
+        Ok(())
+    }
+}

ecstore/src/client/api_get_options.rs

@@ -0,0 +1,127 @@
+#![allow(clippy::map_entry)]
+use std::collections::HashMap;
+use http::{HeaderMap, HeaderName, HeaderValue};
+use time::OffsetDateTime;
+use tracing::warn;
+
+use crate::client::api_error_response::err_invalid_argument;
+
+#[derive(Default)]
+pub struct AdvancedGetOptions {
+    replication_deletemarker:              bool,
+    is_replication_ready_for_deletemarker: bool,
+    replication_proxy_request:             String,
+}
+
+pub struct GetObjectOptions {
+    pub headers: HashMap<String, String>,
+    pub req_params: HashMap<String, String>,
+    //pub server_side_encryption: encrypt.ServerSide,
+    pub version_id: String,
+    pub part_number: i64,
+    pub checksum: bool,
+    pub internal: AdvancedGetOptions,
+}
+
+type StatObjectOptions = GetObjectOptions;
+
+impl Default for GetObjectOptions {
+    fn default() -> Self {
+        Self {
+            headers: HashMap::new(),
+            req_params: HashMap::new(),
+            //server_side_encryption: encrypt.ServerSide::default(),
+            version_id: "".to_string(),
+            part_number: 0,
+            checksum: false,
+            internal: AdvancedGetOptions::default(),
+        }
+    }
+}
+
+impl GetObjectOptions {
+    pub fn header(&self) -> HeaderMap {
+        let mut headers: HeaderMap = HeaderMap::with_capacity(self.headers.len());
+        for (k, v) in &self.headers {
+            if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
+                headers.insert(header_name, v.parse().expect("err"));
+            } else {
+                warn!("Invalid header name: {}", k);
+            }
+        }
+        if self.checksum {
+           headers.insert("x-amz-checksum-mode", "ENABLED".parse().expect("err"));
+        }
+        headers
+    }
+
+    pub fn set(&self, key: &str, value: &str) {
+        //self.headers[http.CanonicalHeaderKey(key)] = value;
+    }
+
+    pub fn set_req_param(&mut self, key: &str, value: &str) {
+        self.req_params.insert(key.to_string(), value.to_string());
+    }
+
+    pub fn add_req_param(&mut self, key: &str, value: &str) {
+        self.req_params.insert(key.to_string(), value.to_string());
+    }
+
+    pub fn set_match_etag(&mut self, etag: &str) -> Result<(), std::io::Error> {
+        self.set("If-Match", &format!("\"{etag}\""));
+        Ok(())
+    }
+
+    pub fn set_match_etag_except(&mut self, etag: &str) -> Result<(), std::io::Error> {
+        self.set("If-None-Match", &format!("\"{etag}\""));
+        Ok(())
+    }
+
+    pub fn set_unmodified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> {
+        if mod_time.unix_timestamp() == 0 {
+            return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
+        }
+        self.set("If-Unmodified-Since", &mod_time.to_string());
+        Ok(())
+    }
+
+    pub fn set_modified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> {
+        if mod_time.unix_timestamp() == 0 {
+            return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
+        }
+        self.set("If-Modified-Since", &mod_time.to_string());
+        Ok(())
+    }
+
+    pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> {
+        if start == 0 && end < 0 {
+            self.set("Range", &format!("bytes={}", end));
+        }
+        else if 0 < start && end == 0 {
+            self.set("Range", &format!("bytes={}-", start));
+        }
+        else if 0 <= start && start <= end {
+            self.set("Range", &format!("bytes={}-{}", start, end));
+        }
+        else {
+            return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid range specified: start={} end={}", start, end))));
+        }
+        Ok(())
+    }
+
+    pub fn to_query_values(&self) -> HashMap<String, String> {
+        let mut url_values = HashMap::new();
+        if self.version_id != "" {
+            url_values.insert("versionId".to_string(), self.version_id.clone());
+        }
+        if self.part_number > 0 {
+            url_values.insert("partNumber".to_string(), self.part_number.to_string());
+        }
+
+        for (key, value) in self.req_params.iter() {
+            url_values.insert(key.to_string(), value.to_string());
+        }
+
+        url_values
+    }
+}

ecstore/src/client/api_list.rs

@@ -0,0 +1,229 @@
+#![allow(clippy::map_entry)]
+use std::collections::HashMap;
+use bytes::Bytes;
+use http::{HeaderMap, StatusCode};
+
+use crate::client::{
+    api_error_response::http_resp_to_error_response,
+    credentials,
+    api_s3_datatypes::{ListBucketV2Result, ListMultipartUploadsResult, ListBucketResult, ListObjectPartsResult, ListVersionsResult, ObjectPart},
+    transition_api::{ReaderImpl, TransitionClient, RequestMetadata,},
+};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+use crate::store_api::BucketInfo;
+
+impl TransitionClient {
+    pub fn list_buckets(&self) -> Result<Vec<BucketInfo>, std::io::Error> {
+        todo!();
+    }
+
+    pub async fn list_objects_v2_query(&self, bucket_name: &str, object_prefix: &str, continuation_token: &str, fetch_owner: bool, metadata: bool, delimiter: &str, start_after: &str, max_keys: i64, headers: HeaderMap) -> Result<ListBucketV2Result, std::io::Error> {
+        let mut url_values = HashMap::new();
+
+        url_values.insert("list-type".to_string(), "2".to_string());
+        if metadata {
+            url_values.insert("metadata".to_string(), "true".to_string());
+        }
+        if start_after != "" {
+            url_values.insert("start-after".to_string(), start_after.to_string());
+        }
+        url_values.insert("encoding-type".to_string(), "url".to_string());
+        url_values.insert("prefix".to_string(), object_prefix.to_string());
+        url_values.insert("delimiter".to_string(), delimiter.to_string());
+
+        if continuation_token != "" {
+            url_values.insert("continuation-token".to_string(), continuation_token.to_string());
+        }
+
+        if fetch_owner {
+            url_values.insert("fetch-owner".to_string(), "true".to_string());
+        }
+
+        if max_keys > 0 {
+            url_values.insert("max-keys".to_string(), max_keys.to_string());
+        }
+
+        let mut resp = self.execute_method(http::Method::GET, &mut RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            object_name: "".to_string(),
+            query_values:       url_values,
+            content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+            custom_header:     headers,
+            content_body:      ReaderImpl::Body(Bytes::new()),
+            content_length:    0,
+            content_md5_base64: "".to_string(),
+            stream_sha256:     false,
+            trailer:           HeaderMap::new(),
+            pre_sign_url:      Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location:   Default::default(),
+            expires:           Default::default(),
+        }).await?;
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
+        }
+
+        //let mut list_bucket_result = ListBucketV2Result::default();
+        let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
+        let mut list_bucket_result = match serde_xml_rs::from_str::<ListBucketV2Result>(&String::from_utf8(b).unwrap()) {
+            Ok(result) => result,
+            Err(err) => {
+                return Err(std::io::Error::other(err.to_string()));
+            }
+        };
+        //println!("list_bucket_result: {:?}", list_bucket_result);
+
+        if list_bucket_result.is_truncated && list_bucket_result.next_continuation_token == "" {
+            return Err(std::io::Error::other(credentials::ErrorResponse {
+              sts_error: credentials::STSError {
+                  r#type:  "".to_string(),
+                  code:    "NotImplemented".to_string(),
+                  message: "Truncated response should have continuation token set".to_string(),
+              },
+              request_id: "".to_string(),
+            }));
+        }
+
+        for (i, obj) in list_bucket_result.contents.iter_mut().enumerate() {
+            obj.name = decode_s3_name(&obj.name, &list_bucket_result.encoding_type)?;
+            //list_bucket_result.contents[i].mod_time = list_bucket_result.contents[i].mod_time.Truncate(time.Millisecond);
+        }
+
+        for (i, obj) in list_bucket_result.common_prefixes.iter_mut().enumerate() {
+            obj.prefix = decode_s3_name(&obj.prefix, &list_bucket_result.encoding_type)?;
+        }
+
+        Ok(list_bucket_result)
+    }
+
+    pub fn list_object_versions_query(&self, bucket_name: &str, opts: &ListObjectsOptions, key_marker: &str, version_id_marker: &str, delimiter: &str) -> Result<ListVersionsResult, std::io::Error> {
+        /*if err := s3utils.CheckValidBucketName(bucketName); err != nil {
+          return ListVersionsResult{}, err
+        }
+        if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
+          return ListVersionsResult{}, err
+        }
+        urlValues := make(url.Values)
+
+        urlValues.Set("versions", "")
+
+        urlValues.Set("prefix", opts.Prefix)
+
+        urlValues.Set("delimiter", delimiter)
+
+        if keyMarker != "" {
+          urlValues.Set("key-marker", keyMarker)
+        }
+
+        if opts.max_keys > 0 {
+          urlValues.Set("max-keys", fmt.Sprintf("%d", opts.max_keys))
+        }
+
+        if versionIDMarker != "" {
+          urlValues.Set("version-id-marker", versionIDMarker)
+        }
+
+        if opts.WithMetadata {
+          urlValues.Set("metadata", "true")
+        }
+
+        urlValues.Set("encoding-type", "url")
+
+        let resp = self.executeMethod(http::Method::GET, &mut RequestMetadata{
+            bucketName:       bucketName,
+            queryValues:      urlValues,
+            contentSHA256Hex: emptySHA256Hex,
+            customHeader:     opts.headers,
+        }).await?;
+        defer closeResponse(resp)
+        if err != nil {
+          return ListVersionsResult{}, err
+        }
+        if resp != nil {
+          if resp.StatusCode != http.StatusOK {
+            return ListVersionsResult{}, httpRespToErrorResponse(resp, bucketName, "")
+          }
+        }
+
+        listObjectVersionsOutput := ListVersionsResult{}
+        err = xml_decoder(resp.Body, &listObjectVersionsOutput)
+        if err != nil {
+          return ListVersionsResult{}, err
+        }
+
+        for i, obj := range listObjectVersionsOutput.Versions {
+          listObjectVersionsOutput.Versions[i].Key, err = decode_s3_name(obj.Key, listObjectVersionsOutput.EncodingType)
+          if err != nil {
+            return listObjectVersionsOutput, err
+          }
+        }
+
+        for i, obj := range listObjectVersionsOutput.CommonPrefixes {
+          listObjectVersionsOutput.CommonPrefixes[i].Prefix, err = decode_s3_name(obj.Prefix, listObjectVersionsOutput.EncodingType)
+          if err != nil {
+            return listObjectVersionsOutput, err
+          }
+        }
+
+        if listObjectVersionsOutput.NextKeyMarker != "" {
+          listObjectVersionsOutput.NextKeyMarker, err = decode_s3_name(listObjectVersionsOutput.NextKeyMarker, listObjectVersionsOutput.EncodingType)
+          if err != nil {
+            return listObjectVersionsOutput, err
+          }
+        }
+
+        Ok(listObjectVersionsOutput)*/
+        todo!();
+    }
+
+    pub fn list_objects_query(&self, bucket_name: &str, object_prefix: &str, object_marker: &str, delimiter: &str, max_keys: i64, headers: HeaderMap) -> Result<ListBucketResult, std::io::Error> {
+        todo!();
+    }
+
+    pub fn list_multipart_uploads_query(&self, bucket_name: &str, key_marker: &str, upload_id_marker: &str, prefix: &str, delimiter: &str, max_uploads: i64) -> Result<ListMultipartUploadsResult, std::io::Error> {
+        todo!();
+    }
+
+    pub fn list_object_parts(&self, bucket_name: &str, object_name: &str, upload_id: &str) -> Result<HashMap<i64, ObjectPart>, std::io::Error> {
+        todo!();
+    }
+
+    pub fn find_upload_ids(&self, bucket_name: &str, object_name: &str) -> Result<Vec<String>, std::io::Error> {
+        todo!();
+    }
+
+    pub async fn list_object_parts_query(&self, bucket_name: &str, object_name: &str, upload_id: &str, part_number_marker: i64, max_parts: i64) -> Result<ListObjectPartsResult, std::io::Error> {
+        todo!();
+    }
+}
+
+pub struct ListObjectsOptions {
+    reverse_versions: bool,
+    with_versions: bool,
+    with_metadata: bool,
+    prefix: String,
+    recursive: bool,
+    max_keys: i64,
+    start_after: String,
+    use_v1: bool,
+    headers: HeaderMap,
+}
+
+impl ListObjectsOptions {
+    pub fn set(&mut self, key: &str, value: &str) {
+        todo!();
+    }
+}
+
+fn decode_s3_name(name: &str, encoding_type: &str) -> Result<String, std::io::Error> {
+    match encoding_type {
+        "url" => {
+            //return url::QueryUnescape(name);
+            return Ok(name.to_string());
+        }
+        _ => {
+            return Ok(name.to_string());
+        }
+    }
+}

ecstore/src/client/api_put_object.rs

@@ -0,0 +1,382 @@
+#![allow(clippy::map_entry)]
+use std::{collections::HashMap, sync::Arc};
+use bytes::Bytes;
+use time::{OffsetDateTime, macros::format_description, Duration};
+use http::{HeaderMap, HeaderName, HeaderValue};
+use tracing::{error, info, warn};
+
+use s3s::dto::{
+    ObjectLockRetentionMode, ObjectLockLegalHoldStatus,
+    ReplicationStatus,
+};
+use s3s::header::{
+    X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_OBJECT_LOCK_LEGAL_HOLD,
+    X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION, X_AMZ_REPLICATION_STATUS,
+};
+use reader::hasher::Hasher;
+//use crate::disk::{BufferReader, Reader};
+use rustfs_utils::{
+    crypto::base64_encode,
+    net::{is_amz_header, is_standard_header, is_storageclass_header, is_rustfs_header, is_minio_header},
+};
+use crate::checksum::ChecksumMode;
+use crate::client::{
+    api_s3_datatypes::{CompletePart, ObjectPart, CompleteMultipartUpload},
+    api_put_object_common::optimal_part_info,
+    transition_api::{TransitionClient, UploadInfo, ReaderImpl},
+    api_error_response::{err_invalid_argument, err_entity_too_large},
+    api_put_object_multipart::UploadPartParams,
+    credentials::SignatureType,
+    constants::{MAX_MULTIPART_PUT_OBJECT_SIZE, TOTAL_WORKERS, MIN_PART_SIZE, ISO8601_DATEFORMAT,},
+};
+
+#[derive(Debug, Clone)]
+pub struct AdvancedPutOptions {
+    pub source_version_id: String,
+    pub source_etag: String,
+    pub replication_status: ReplicationStatus,
+    pub source_mtime: OffsetDateTime,
+    pub replication_request: bool,
+    pub retention_timestamp: OffsetDateTime,
+    pub tagging_timestamp: OffsetDateTime,
+    pub legalhold_timestamp: OffsetDateTime,
+    pub replication_validity_check: bool,
+}
+
+impl Default for AdvancedPutOptions {
+    fn default() -> Self {
+        Self {
+            source_version_id: "".to_string(),
+            source_etag: "".to_string(),
+            replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
+            source_mtime: OffsetDateTime::now_utc(),
+            replication_request: false,
+            retention_timestamp: OffsetDateTime::now_utc(),
+            tagging_timestamp: OffsetDateTime::now_utc(),
+            legalhold_timestamp: OffsetDateTime::now_utc(),
+            replication_validity_check: false,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct PutObjectOptions {
+    pub user_metadata: HashMap<String, String>,
+    pub user_tags: HashMap<String, String>,
+    //pub progress: ReaderImpl,
+    pub content_type: String,
+    pub content_encoding: String,
+    pub content_disposition: String,
+    pub content_language: String,
+    pub cache_control: String,
+    pub expires: OffsetDateTime,
+    pub mode: ObjectLockRetentionMode,
+    pub retain_until_date: OffsetDateTime,
+    //pub server_side_encryption: encrypt.ServerSide,
+    pub num_threads: u64,
+    pub storage_class: String,
+    pub website_redirect_location: String,
+    pub part_size: u64,
+    pub legalhold: ObjectLockLegalHoldStatus,
+    pub send_content_md5: bool,
+    pub disable_content_sha256: bool,
+    pub disable_multipart: bool,
+    pub auto_checksum: ChecksumMode,
+    pub checksum: ChecksumMode,
+    pub concurrent_stream_parts: bool,
+    pub internal: AdvancedPutOptions,
+    pub custom_header: HeaderMap,
+}
+
+impl Default for PutObjectOptions {
+    fn default() -> Self {
+        Self {
+            user_metadata: HashMap::new(),
+            user_tags: HashMap::new(),
+            //progress: ReaderImpl::Body(Bytes::new()),
+            content_type: "".to_string(),
+            content_encoding: "".to_string(),
+            content_disposition: "".to_string(),
+            content_language: "".to_string(),
+            cache_control: "".to_string(),
+            expires: OffsetDateTime::UNIX_EPOCH,
+            mode: ObjectLockRetentionMode::from_static(""),
+            retain_until_date: OffsetDateTime::UNIX_EPOCH,
+            //server_side_encryption: encrypt.ServerSide::default(),
+            num_threads: 0,
+            storage_class: "".to_string(),
+            website_redirect_location: "".to_string(),
+            part_size: 0,
+            legalhold: ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF),
+            send_content_md5: false,
+            disable_content_sha256: false,
+            disable_multipart: false,
+            auto_checksum: ChecksumMode::ChecksumNone,
+            checksum: ChecksumMode::ChecksumNone,
+            concurrent_stream_parts: false,
+            internal: AdvancedPutOptions::default(),
+            custom_header: HeaderMap::new(),
+        }
+    }
+}
+
+impl PutObjectOptions {
+    fn set_matche_tag(&mut self, etag: &str) {
+        if etag == "*" {
+            self.custom_header.insert("If-Match", HeaderValue::from_str("*").expect("err"));
+        } else {
+            self.custom_header.insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err"));
+        }
+    }
+
+    fn set_matche_tag_except(&mut self, etag: &str) {
+        if etag == "*" {
+            self.custom_header.insert("If-None-Match", HeaderValue::from_str("*").expect("err"));
+        } else {
+            self.custom_header.insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err"));
+        }
+    }
+
+    pub fn header(&self) -> HeaderMap {
+        let mut header = HeaderMap::new();
+
+        let mut content_type = self.content_type.clone();
+        if content_type == "" {
+            content_type = "application/octet-stream".to_string();
+        }
+        header.insert("Content-Type", HeaderValue::from_str(&content_type).expect("err"));
+
+        if self.content_encoding != "" {
+            header.insert("Content-Encoding", HeaderValue::from_str(&self.content_encoding).expect("err"));
+        }
+        if self.content_disposition != "" {
+            header.insert("Content-Disposition", HeaderValue::from_str(&self.content_disposition).expect("err"));
+        }
+        if self.content_language != "" {
+            header.insert("Content-Language", HeaderValue::from_str(&self.content_language).expect("err"));
+        }
+        if self.cache_control != "" {
+            header.insert("Cache-Control", HeaderValue::from_str(&self.cache_control).expect("err"));
+        }
+
+        if self.expires.unix_timestamp() != 0 {
+            header.insert("Expires", HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err"));  //rustfs invalid heade
+        }
+
+        if self.mode.as_str() != "" {
+            header.insert(X_AMZ_OBJECT_LOCK_MODE, HeaderValue::from_str(self.mode.as_str()).expect("err"));
+        }
+
+        if self.retain_until_date.unix_timestamp() != 0 {
+            header.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err"));
+        }
+
+        if self.legalhold.as_str() != "" {
+            header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, HeaderValue::from_str(self.legalhold.as_str()).expect("err"));
+        }
+
+        if self.storage_class != "" {
+            header.insert(X_AMZ_STORAGE_CLASS, HeaderValue::from_str(&self.storage_class).expect("err"));
+        }
+
+        if self.website_redirect_location != "" {
+            header.insert(X_AMZ_WEBSITE_REDIRECT_LOCATION, HeaderValue::from_str(&self.website_redirect_location).expect("err"));
+        }
+
+        if !self.internal.replication_status.as_str().is_empty() {
+            header.insert(X_AMZ_REPLICATION_STATUS, HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err"));
+        }
+
+        for (k, v) in &self.user_metadata {
+            if is_amz_header(k) || is_standard_header(k) || is_storageclass_header(k) || is_rustfs_header(k) || is_minio_header(k) {
+                if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
+                    header.insert(header_name, HeaderValue::from_str(&v).unwrap());
+                }
+            } else {
+                if let Ok(header_name) = HeaderName::from_bytes(format!("x-amz-meta-{}", k).as_bytes()) {
+                    header.insert(header_name, HeaderValue::from_str(&v).unwrap());
+                }
+            }
+        }
+
+        for (k, v) in self.custom_header.iter() {
+            header.insert(k.clone(), v.clone());
+        }
+
+        header
+    }
+
+    fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
+        //if self.checksum.is_set() {
+            /*if !self.trailing_header_support {
+                return Err(Error::from(err_invalid_argument("Checksum requires Client with TrailingHeaders enabled")));
+            }*/
+            /*else if self.override_signer_type == SignatureType::SignatureV2 {
+                return Err(Error::from(err_invalid_argument("Checksum cannot be used with v2 signatures")));
+            }*/
+        //}
+
+        Ok(())
+    }
+}
+
+impl TransitionClient {
+    pub async fn put_object(self: Arc<Self>, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, object_size: i64,
+        opts: &PutObjectOptions
+    ) -> Result<UploadInfo, std::io::Error> {
+        if object_size < 0 && opts.disable_multipart {
+            return Err(std::io::Error::other("object size must be provided with disable multipart upload"));
+        }
+
+        self.put_object_common(bucket_name, object_name, reader, object_size, opts).await
+    }
+
+    pub async fn put_object_common(self: Arc<Self>, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
+        if size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+            return Err(std::io::Error::other(err_entity_too_large(size, MAX_MULTIPART_PUT_OBJECT_SIZE, bucket_name, object_name)));
+        }
+        let mut opts = opts.clone();
+        opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
+
+        let mut part_size = opts.part_size as i64;
+        if opts.part_size == 0 {
+            part_size = MIN_PART_SIZE;
+        }
+
+        if SignatureType::SignatureV2 == self.override_signer_type {
+            if size >= 0 && size < part_size || opts.disable_multipart {
+                return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
+            }
+            return self.put_object_multipart(bucket_name, object_name, reader, size, &opts).await;
+        }
+
+        if size < 0 {
+            if opts.disable_multipart {
+                return Err(std::io::Error::other("no length provided and multipart disabled"));
+            }
+            if opts.concurrent_stream_parts && opts.num_threads > 1 {
+                return self.put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts).await;
+            }
+            return self.put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts).await;
+        }
+
+        if size <= part_size || opts.disable_multipart {
+            return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
+        }
+
+        self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts).await
+    }
+
+    pub async fn put_object_multipart_stream_no_length(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
+        let mut total_uploaded_size: i64 = 0;
+
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let (total_parts_count, part_size, _) = optimal_part_info(-1, opts.part_size)?;
+
+        let mut opts = opts.clone();
+
+        if opts.checksum.is_set() {
+            opts.send_content_md5 = false;
+            opts.auto_checksum = opts.checksum.clone();
+        }
+        if !opts.send_content_md5 {
+            //add_auto_checksum_headers(&mut opts);
+        }
+
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut part_number = 1;
+        let mut parts_info = HashMap::<i64, ObjectPart>::new();
+        let mut buf = Vec::<u8>::with_capacity(part_size as usize);
+
+        let mut custom_header = HeaderMap::new();
+
+        while part_number <= total_parts_count {
+            buf = match &mut reader {
+                ReaderImpl::Body(content_body) => {
+                    content_body.to_vec()
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    content_body.read_all().await?
+                }
+            };
+            let length = buf.len();
+
+            let mut md5_base64: String = "".to_string();
+            if opts.send_content_md5 {
+                let mut md5_hasher = self.md5_hasher.lock().unwrap();
+                let mut hash = md5_hasher.as_mut().expect("err");
+                hash.write(&buf[..length]);
+                md5_base64 = base64_encode(hash.sum().as_bytes());
+            } else {
+                let csum;
+                {
+                    let mut crc = opts.auto_checksum.hasher()?;
+                    crc.reset();
+                    crc.write(&buf[..length]);
+                    csum = crc.sum();
+                }
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
+                    custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().unwrap());
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key());
+                }
+            }
+
+            //let rd = newHook(bytes.NewReader(buf[..length]), opts.progress);
+            let rd = ReaderImpl::Body(Bytes::from(buf));
+
+            let mut p = UploadPartParams {
+                bucket_name: bucket_name.to_string(),
+                object_name: object_name.to_string(),
+                upload_id: upload_id.clone(),
+                reader: rd,
+                part_number,
+                md5_base64,
+                size: length as i64,
+                //sse: opts.server_side_encryption,
+                stream_sha256: !opts.disable_content_sha256,
+                custom_header: custom_header.clone(),
+                sha256_hex: Default::default(),
+                trailer: Default::default(),
+            };
+            let obj_part = self.upload_part(&mut p).await?;
+
+            parts_info.entry(part_number).or_insert(obj_part);
+            total_uploaded_size += length as i64;
+            part_number += 1;
+        }
+
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
+        for i in 1..part_number {
+            let part = parts_info[&i].clone();
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag:                part.etag,
+                part_num:            part.part_num,
+                checksum_crc32:      part.checksum_crc32,
+                checksum_crc32c:     part.checksum_crc32c,
+                checksum_sha1:       part.checksum_sha1,
+                checksum_sha256:     part.checksum_sha256,
+                checksum_crc64nvme:  part.checksum_crc64nvme,
+                ..Default::default()
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+
+        let mut opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum: opts.auto_checksum,
+            ..Default::default()
+        };
+        //apply_auto_checksum(&mut opts, all_parts);
+
+        let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+}

ecstore/src/client/api_put_object_common.rs

@@ -0,0 +1,76 @@
+#![allow(clippy::map_entry)]
+use crate::client::{
+    api_put_object::PutObjectOptions,
+    constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PARTS_COUNT, MAX_PART_SIZE, MIN_PART_SIZE},
+    transition_api::TransitionClient,
+    transition_api::ReaderImpl,
+    api_error_response::{err_entity_too_large, err_invalid_argument},
+};
+
+const NULL_VERSION_ID: &str = "null";
+
+pub fn is_object(reader: &ReaderImpl) -> bool {
+    todo!();
+}
+
+pub fn is_read_at(reader: ReaderImpl) -> bool {
+    todo!();
+}
+
+pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
+    let unknown_size;
+    let mut object_size = object_size;
+    if object_size == -1 {
+        unknown_size = true;
+        object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
+    } else {
+        unknown_size = false;
+    }
+
+    if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+        return Err(std::io::Error::other(err_entity_too_large(object_size, MAX_MULTIPART_PUT_OBJECT_SIZE, "", "")));
+    }
+
+    let mut part_size_flt: f64;
+    if configured_part_size > 0 {
+        if configured_part_size as i64 > object_size {
+            return Err(std::io::Error::other(err_entity_too_large(configured_part_size as i64, object_size, "", "")));
+        }
+
+        if !unknown_size {
+            if object_size > (configured_part_size as i64 * MAX_PARTS_COUNT) {
+                return Err(std::io::Error::other(err_invalid_argument("Part size * max_parts(10000) is lesser than input objectSize.")));
+            }
+        }
+
+        if (configured_part_size as i64) < ABS_MIN_PART_SIZE {
+            return Err(std::io::Error::other(err_invalid_argument("Input part size is smaller than allowed minimum of 5MiB.")));
+        }
+
+        if configured_part_size as i64 > MAX_PART_SIZE {
+            return Err(std::io::Error::other(err_invalid_argument("Input part size is bigger than allowed maximum of 5GiB.")));
+        }
+
+        part_size_flt = configured_part_size as f64;
+        if unknown_size {
+            object_size = configured_part_size as i64 * MAX_PARTS_COUNT;
+        }
+    } else {
+        let mut configured_part_size = configured_part_size;
+        configured_part_size = MIN_PART_SIZE as u64;
+        part_size_flt = (object_size / MAX_PARTS_COUNT) as f64;
+        part_size_flt = (part_size_flt / configured_part_size as f64) * configured_part_size as f64;
+    }
+
+    let total_parts_count = (object_size as f64 / part_size_flt).ceil() as i64;
+    let part_size = part_size_flt.ceil() as i64;
+    let last_part_size = object_size - (total_parts_count-1) * part_size;
+    Ok((total_parts_count, part_size, last_part_size))
+}
+
+impl TransitionClient {
+    pub async fn new_upload_id(&self ,bucket_name: &str, object_name: &str, opts: &PutObjectOptions) -> Result<String, std::io::Error> {
+        let init_multipart_upload_result = self.initiate_multipart_upload(bucket_name, object_name, opts).await?;
+        Ok(init_multipart_upload_result.upload_id)
+    }
+}

ecstore/src/client/api_put_object_multipart.rs

@@ -0,0 +1,348 @@
+#![allow(clippy::map_entry)]
+use std::io::Read;
+use std::{collections::HashMap, sync::Arc};
+use bytes::Bytes;
+use s3s::S3ErrorCode;
+use time::{format_description, OffsetDateTime};
+use uuid::Uuid;
+use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
+use url::form_urlencoded::Serializer;
+use tokio_util::sync::CancellationToken;
+use tracing::warn;
+use tracing::{error, info};
+
+use s3s::{dto::StreamingBlob, Body};
+use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
+use reader::hasher::Hasher;
+//use crate::disk::{Reader, BufferReader};
+use crate::client::{
+    transition_api::{RequestMetadata, TransitionClient, UploadInfo, ReaderImpl,},
+    api_error_response::{err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response},
+    api_put_object::PutObjectOptions,
+    api_put_object_common::optimal_part_info,
+    api_s3_datatypes::{CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart},
+    constants::{ABS_MIN_PART_SIZE, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE, ISO8601_DATEFORMAT, },
+};
+use rustfs_utils::{
+    path::trim_etag,
+    crypto::base64_encode,
+};
+use crate::{
+    disk::DiskAPI,
+    store_api::{
+        GetObjectReader, StorageAPI,
+    },
+    checksum::ChecksumMode,
+};
+
+impl TransitionClient {
+    pub async fn put_object_multipart(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64,
+        opts: &PutObjectOptions
+    ) -> Result<UploadInfo, std::io::Error> {
+        let info = self.put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts).await;
+        if let Err(err) = &info {
+            let err_resp = to_error_response(err);
+            if err_resp.code == S3ErrorCode::AccessDenied && err_resp.message.contains("Access Denied") {
+                if size > MAX_SINGLE_PUT_OBJECT_SIZE {
+                    return Err(std::io::Error::other(err_entity_too_large(size, MAX_SINGLE_PUT_OBJECT_SIZE, bucket_name, object_name)));
+                }
+                return self.put_object_gcs(bucket_name, object_name, reader, size, opts).await;
+            }
+        }
+        Ok(info?)
+    }
+
+    pub async fn put_object_multipart_no_stream(&self, bucket_name: &str, object_name: &str, reader: &mut ReaderImpl, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
+        let mut total_uploaded_size: i64 = 0;
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let ret = optimal_part_info(-1, opts.part_size)?;
+        let (total_parts_count, part_size, _) = ret;
+
+        let (mut hash_algos, mut hash_sums) = self.hash_materials(opts.send_content_md5, !opts.disable_content_sha256);
+        let upload_id = self.new_upload_id(bucket_name, object_name, opts).await?;
+        let mut opts = opts.clone();
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut part_number = 1;
+        let mut parts_info = HashMap::<i64, ObjectPart>::new();
+        let mut buf = Vec::<u8>::with_capacity(part_size as usize);
+        let mut custom_header = HeaderMap::new();
+        while part_number <= total_parts_count {
+            match reader {
+                ReaderImpl::Body(content_body) => {
+                    buf = content_body.to_vec();
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    buf = content_body.read_all().await?;
+                }
+            }
+            let length = buf.len();
+
+            for (k, v) in hash_algos.iter_mut() {
+                v.write(&buf[..length]);
+                hash_sums.insert(k.to_string(), Vec::try_from(v.sum().as_bytes()).unwrap());
+            }
+
+            //let rd = newHook(bytes.NewReader(buf[..length]), opts.progress);
+            let rd = Bytes::from(buf.clone());
+
+            let mut md5_base64: String;
+            let mut sha256_hex: String;
+
+            //if hash_sums["md5"] != nil {
+                md5_base64 = base64_encode(&hash_sums["md5"]);
+            //}
+            //if hash_sums["sha256"] != nil {
+                sha256_hex = hex_simd::encode_to_string(hash_sums["sha256"].clone(), hex_simd::AsciiCase::Lower);
+            //}
+            if hash_sums.len() == 0 {
+                let csum;
+                {
+                    let mut crc = opts.auto_checksum.hasher()?;
+                    crc.reset();
+                    crc.write(&buf[..length]);
+                    csum = crc.sum();
+                }
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
+                    custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().expect("err"));
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key());
+                }
+            }
+
+            let mut p = UploadPartParams {
+                bucket_name: bucket_name.to_string(),
+                object_name: object_name.to_string(),
+                upload_id: upload_id.clone(),
+                reader: ReaderImpl::Body(rd),
+                part_number,
+                md5_base64,
+                sha256_hex,
+                size: length as i64,
+                //sse: opts.server_side_encryption,
+                stream_sha256: !opts.disable_content_sha256,
+                custom_header: custom_header.clone(),
+                trailer: HeaderMap::new(),
+            };
+            let obj_part = self.upload_part(&mut p).await?;
+
+            parts_info.insert(part_number, obj_part);
+            total_uploaded_size += length as i64;
+            part_number += 1;
+        }
+
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
+        for i in 1..part_number {
+            let part = parts_info[&i].clone();
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag:               part.etag,
+                part_num:           part.part_num,
+                checksum_crc32:     part.checksum_crc32,
+                checksum_crc32c:    part.checksum_crc32c,
+                checksum_sha1:      part.checksum_sha1,
+                checksum_sha256:    part.checksum_sha256,
+                checksum_crc64nvme: part.checksum_crc64nvme,
+                ..Default::default()
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+        let mut opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum:          opts.auto_checksum,
+            ..Default::default()
+        };
+        //apply_auto_checksum(&mut opts, all_parts);
+
+        let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+
+    pub async fn initiate_multipart_upload(&self, bucket_name: &str, object_name: &str, opts: &PutObjectOptions) -> Result<InitiateMultipartUploadResult, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("uploads".to_string(), "".to_string());
+
+        if opts.internal.source_version_id != "" {
+            if !opts.internal.source_version_id.is_empty() {
+                if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) {
+                    return Err(std::io::Error::other(err_invalid_argument(&err.to_string())));
+                }
+            }
+            url_values.insert("versionId".to_string(), opts.internal.source_version_id.clone());
+        }
+
+        let mut custom_header = opts.header();
+
+        let mut req_metadata = RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            object_name: object_name.to_string(),
+            query_values:  url_values,
+            custom_header,
+            content_body:  ReaderImpl::Body(Bytes::new()),
+            content_length: 0,
+            content_md5_base64: "".to_string(),
+            content_sha256_hex: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
+        //if resp.is_none() {
+            if resp.status() != StatusCode::OK {
+                return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
+            }
+        //}
+        let initiate_multipart_upload_result = InitiateMultipartUploadResult::default();
+        Ok(initiate_multipart_upload_result)
+    }
+
+    pub async fn upload_part(&self, p: &mut UploadPartParams) -> Result<ObjectPart, std::io::Error> {
+        if p.size > MAX_PART_SIZE {
+            return Err(std::io::Error::other(err_entity_too_large(p.size, MAX_PART_SIZE, &p.bucket_name, &p.object_name)));
+        }
+        if p.size <= -1 {
+            return Err(std::io::Error::other(err_entity_too_small(p.size, &p.bucket_name, &p.object_name)));
+        }
+        if p.part_number <= 0 {
+            return Err(std::io::Error::other(err_invalid_argument("Part number cannot be negative or equal to zero.")));
+        }
+        if p.upload_id == "" {
+            return Err(std::io::Error::other(err_invalid_argument("UploadID cannot be empty.")));
+        }
+
+        let mut url_values = HashMap::new();
+        url_values.insert("partNumber".to_string(), p.part_number.to_string());
+        url_values.insert("uploadId".to_string(), p.upload_id.clone());
+
+        let buf = match &mut p.reader {
+            ReaderImpl::Body(content_body) => {
+                content_body.to_vec()
+            }
+            ReaderImpl::ObjectBody(content_body) => {
+                content_body.read_all().await?
+            }
+        };
+        let mut req_metadata = RequestMetadata {
+            bucket_name:       p.bucket_name.clone(),
+            object_name:       p.object_name.clone(),
+            query_values:      url_values,
+            custom_header:     p.custom_header.clone(),
+            content_body:      ReaderImpl::Body(Bytes::from(buf)),
+            content_length:    p.size,
+            content_md5_base64: p.md5_base64.clone(),
+            content_sha256_hex: p.sha256_hex.clone(),
+            stream_sha256:     p.stream_sha256,
+            trailer:           p.trailer.clone(),
+            pre_sign_url:      Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location:   Default::default(),
+            expires:           Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
+        //defer closeResponse(resp)
+        //if resp.is_none() {
+            if resp.status() != StatusCode::OK {
+                return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], &p.bucket_name.clone(), &p.object_name)));
+            }
+        //}
+        let h = resp.headers();
+        let mut obj_part = ObjectPart {
+            checksum_crc32:     if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { h_checksum_crc32.to_str().expect("err").to_string() } else { "".to_string() },
+            checksum_crc32c:    if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { h_checksum_crc32c.to_str().expect("err").to_string() } else { "".to_string() },
+            checksum_sha1:      if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { h_checksum_sha1.to_str().expect("err").to_string() } else { "".to_string() },
+            checksum_sha256:    if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { h_checksum_sha256.to_str().expect("err").to_string() } else { "".to_string() },
+            checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { h_checksum_crc64nvme.to_str().expect("err").to_string() } else { "".to_string() },
+            ..Default::default()
+        };
+        obj_part.size = p.size;
+        obj_part.part_num = p.part_number;
+        obj_part.etag = if let Some(h_etag) = h.get("ETag") { h_etag.to_str().expect("err").trim_matches('"').to_string() } else { "".to_string() };
+        Ok(obj_part)
+    }
+
+    pub async fn complete_multipart_upload(&self, bucket_name: &str, object_name: &str, upload_id: &str,
+        complete: CompleteMultipartUpload, opts: &PutObjectOptions
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("uploadId".to_string(), upload_id.to_string());
+        let complete_multipart_upload_bytes = complete.marshal_msg()?.as_bytes().to_vec();
+
+        let mut headers = opts.header();
+
+        let complete_multipart_upload_buffer = Bytes::from(complete_multipart_upload_bytes);
+        let mut req_metadata = RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            object_name: object_name.to_string(),
+            query_values:      url_values,
+            content_body:      ReaderImpl::Body(complete_multipart_upload_buffer),
+            content_length:    100,//complete_multipart_upload_bytes.len(),
+            content_sha256_hex: "".to_string(),//hex_simd::encode_to_string(complete_multipart_upload_bytes, hex_simd::AsciiCase::Lower),
+            custom_header:     headers,
+            stream_sha256:     Default::default(),
+            trailer:           Default::default(),
+            content_md5_base64: "".to_string(),
+            pre_sign_url:      Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location:   Default::default(),
+            expires:           Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
+
+        let b = resp.body().bytes().expect("err").to_vec();
+        let complete_multipart_upload_result: CompleteMultipartUploadResult = CompleteMultipartUploadResult::default();
+
+        let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
+            (
+                OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
+                "".to_string()
+            )
+        } else {
+            (OffsetDateTime::now_utc(), "".to_string())
+        };
+
+        let h = resp.headers();
+        Ok(UploadInfo {
+            bucket:           complete_multipart_upload_result.bucket,
+            key:              complete_multipart_upload_result.key,
+            etag:             trim_etag(&complete_multipart_upload_result.etag),
+            version_id:       if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { h_x_amz_version_id.to_str().expect("err").to_string() } else { "".to_string() },
+            location:         complete_multipart_upload_result.location,
+            expiration:       exp_time,
+            expiration_rule_id: rule_id,
+            checksum_sha256:    complete_multipart_upload_result.checksum_sha256,
+            checksum_sha1:      complete_multipart_upload_result.checksum_sha1,
+            checksum_crc32:     complete_multipart_upload_result.checksum_crc32,
+            checksum_crc32c:    complete_multipart_upload_result.checksum_crc32c,
+            checksum_crc64nvme: complete_multipart_upload_result.checksum_crc64nvme,
+            ..Default::default()
+        })
+    }
+}
+
+pub struct UploadPartParams {
+    pub bucket_name: String,
+    pub object_name: String,
+    pub upload_id: String,
+    pub reader: ReaderImpl,
+    pub part_number: i64,
+    pub md5_base64: String,
+    pub sha256_hex: String,
+    pub size: i64,
+    //pub sse: encrypt.ServerSide,
+    pub stream_sha256: bool,
+    pub custom_header: HeaderMap,
+    pub trailer: HeaderMap,
+}

ecstore/src/client/api_put_object_streaming.rs

@@ -0,0 +1,447 @@
+#![allow(clippy::map_entry)]
+use std::sync::RwLock;
+use std::{collections::HashMap, sync::Arc};
+use bytes::Bytes;
+use futures::future::join_all;
+use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
+use time::{format_description, OffsetDateTime};
+use tokio::{select, sync::mpsc};
+use tokio_util::sync::CancellationToken;
+use tracing::warn;
+use uuid::Uuid;
+
+use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
+use reader::hasher::Hasher;
+use crate::client::{
+    constants::ISO8601_DATEFORMAT,
+    api_put_object::PutObjectOptions,
+    api_put_object_multipart::UploadPartParams,
+    api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
+    transition_api::{TransitionClient, RequestMetadata, UploadInfo, ReaderImpl},
+    api_put_object_common::{is_object, optimal_part_info,},
+    api_error_response::{err_invalid_argument, http_resp_to_error_response, err_unexpected_eof},
+};
+use rustfs_utils::{crypto::base64_encode, path::trim_etag};
+use crate::checksum::{add_auto_checksum_headers, apply_auto_checksum, ChecksumMode};
+
+pub struct UploadedPartRes {
+    pub error: std::io::Error,
+    pub part_num: i64,
+    pub size: i64,
+    pub part: ObjectPart,
+}
+
+pub struct UploadPartReq {
+    pub part_num: i64,
+    pub part: ObjectPart,
+}
+
+impl TransitionClient {
+    pub async fn put_object_multipart_stream(self: Arc<Self>, bucket_name: &str, object_name: &str,
+        mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions
+    ) -> Result<UploadInfo, std::io::Error> {
+        let info: UploadInfo;
+        if opts.concurrent_stream_parts && opts.num_threads > 1 {
+            info = self.put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts).await?;
+        } else if !is_object(&reader) && !opts.send_content_md5 {
+            info = self.put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts).await?;
+        } else {
+            info = self.put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts).await?;
+        }
+        
+        Ok(info)
+    }
+
+    pub async fn put_object_multipart_stream_from_readat(&self, bucket_name: &str, object_name: &str,
+        mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions
+    ) -> Result<UploadInfo, std::io::Error> {
+        let ret = optimal_part_info(size, opts.part_size)?;
+        let (total_parts_count, part_size, lastpart_size) = ret;
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.auto_checksum = opts.checksum.clone();
+        }
+        let with_checksum = self.trailing_header_support;
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        todo!();
+    }
+
+    pub async fn put_object_multipart_stream_optional_checksum(&self, bucket_name: &str, object_name: &str,
+        mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.auto_checksum = opts.checksum.clone();
+            opts.send_content_md5 = false;
+        }
+
+        if !opts.send_content_md5 {
+            add_auto_checksum_headers(&mut opts);
+        }
+
+        let ret = optimal_part_info(size, opts.part_size)?;
+        let (total_parts_count, mut part_size, lastpart_size) = ret;
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut custom_header = opts.header().clone();
+
+        let mut total_uploaded_size: i64 = 0;
+
+        let mut parts_info = HashMap::<i64, ObjectPart>::new();
+        let mut buf = Vec::<u8>::with_capacity(part_size as usize);
+
+        let mut md5_base64: String = "".to_string();
+        for part_number in 1..=total_parts_count {
+            if part_number == total_parts_count {
+                part_size = lastpart_size;
+            }
+
+            match &mut reader {
+                ReaderImpl::Body(content_body) => {
+                    buf = content_body.to_vec();
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    buf = content_body.read_all().await?;
+                }
+            }
+            let length = buf.len();
+
+            if opts.send_content_md5 {
+                let mut md5_hasher = self.md5_hasher.lock().unwrap();
+                let mut md5_hash = md5_hasher.as_mut().expect("err");
+                md5_hash.reset();
+                md5_hash.write(&buf[..length]);
+                md5_base64 = base64_encode(md5_hash.sum().as_bytes());
+            } else {
+                let csum;
+                {
+                    let mut crc = opts.auto_checksum.hasher()?;
+                    crc.reset();
+                    crc.write(&buf[..length]);
+                    csum = crc.sum();
+                }
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key_capitalized().as_bytes()) {
+                    custom_header.insert(header_name, HeaderValue::from_str(&base64_encode(csum.as_bytes())).expect("err"));
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key_capitalized());
+                }
+            }
+
+            let hooked = ReaderImpl::Body(Bytes::from(buf));//newHook(BufferReader::new(buf), opts.progress);
+            let mut p = UploadPartParams {
+                bucket_name: bucket_name.to_string(),
+                object_name: object_name.to_string(),
+                upload_id: upload_id.clone(),
+                reader: hooked,
+                part_number,
+                md5_base64: md5_base64.clone(),
+                size: part_size,
+                //sse: opts.server_side_encryption,
+                stream_sha256: !opts.disable_content_sha256,
+                custom_header: custom_header.clone(),
+                sha256_hex: "".to_string(),
+                trailer: HeaderMap::new(),
+            };
+            let obj_part = self.upload_part(&mut p).await?;
+
+            parts_info.entry(part_number).or_insert(obj_part);
+
+            total_uploaded_size += part_size as i64;
+        }
+
+        if size > 0 {
+            if total_uploaded_size != size {
+                return Err(std::io::Error::other(err_unexpected_eof(total_uploaded_size, size, bucket_name, object_name)));
+            }
+        }
+
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
+        let part_number = total_parts_count;
+        for i in 1..part_number {
+            let part = parts_info[&i].clone();
+
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag:               part.etag,
+                part_num:           part.part_num,
+                checksum_crc32:     part.checksum_crc32,
+                checksum_crc32c:    part.checksum_crc32c,
+                checksum_sha1:      part.checksum_sha1,
+                checksum_sha256:    part.checksum_sha256,
+                checksum_crc64nvme: part.checksum_crc64nvme,
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+
+        let mut opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum:          opts.auto_checksum,
+            ..Default::default()
+        };
+        apply_auto_checksum(&mut opts, &mut all_parts);
+        let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+
+    pub async fn put_object_multipart_stream_parallel(self: Arc<Self>, bucket_name: &str, object_name: &str,
+        mut reader: ReaderImpl/*GetObjectReader*/, opts: &PutObjectOptions
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.send_content_md5 = false;
+            opts.auto_checksum = opts.checksum.clone();
+        }
+        if !opts.send_content_md5 {
+            add_auto_checksum_headers(&mut opts);
+        }
+
+        let ret = optimal_part_info(-1, opts.part_size)?;
+        let (total_parts_count, part_size, _) = ret;
+
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut total_uploaded_size: i64 = 0;
+        let mut parts_info = Arc::new(RwLock::new(HashMap::<i64, ObjectPart>::new()));
+
+        let n_buffers = opts.num_threads;
+        let (bufs_tx, mut bufs_rx) = mpsc::channel(n_buffers as usize);
+        //let all = Vec::<u8>::with_capacity(n_buffers as usize * part_size as usize);
+        for i in 0..n_buffers {
+            //bufs_tx.send(&all[i * part_size..i * part_size + part_size]);
+            bufs_tx.send(Vec::<u8>::with_capacity(part_size as usize));
+        }
+
+        let mut futures = Vec::with_capacity(total_parts_count as usize);
+        let (err_tx, mut err_rx) = mpsc::channel(opts.num_threads as usize);
+        let cancel_token = CancellationToken::new();
+
+        //reader = newHook(reader, opts.progress);
+
+        for part_number in 1..=total_parts_count {
+            let mut buf = Vec::<u8>::new();
+            select! {
+                buf = bufs_rx.recv() => {}
+                err = err_rx.recv() => {
+                    //cancel_token.cancel();
+                    //wg.Wait()
+                    return Err(err.expect("err"));
+                }
+                else => (),
+            }
+
+            if buf.len() != part_size as usize {
+                return Err(std::io::Error::other(format!("read buffer < {} than expected partSize: {}", buf.len(), part_size)));
+            }
+
+            match &mut reader {
+                ReaderImpl::Body(content_body) => {
+                    buf = content_body.to_vec();
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    buf = content_body.read_all().await?;
+                }
+            }
+            let length = buf.len();
+
+            let mut custom_header = HeaderMap::new();
+            if !opts.send_content_md5 {
+                let csum;
+                {
+                    let mut crc = opts.auto_checksum.hasher()?;
+                    crc.reset();
+                    crc.write(&buf[..length]);
+                    csum = crc.sum();
+                }
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
+                    if let Ok(header_value) = HeaderValue::from_str(&base64_encode(csum.as_bytes())) {
+                        custom_header.insert(header_name, header_value);
+                    }
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key());
+                }
+            }
+
+            let clone_bufs_tx = bufs_tx.clone();
+            let clone_parts_info = parts_info.clone();
+            let clone_upload_id = upload_id.clone();
+            let clone_self = self.clone();
+            futures.push(async move {
+                let mut md5_base64: String = "".to_string();
+
+                if opts.send_content_md5 {
+                    let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
+                    let mut md5_hash = md5_hasher.as_mut().expect("err");
+                    md5_hash.write(&buf[..length]);
+                    md5_base64 = base64_encode(md5_hash.sum().as_bytes());
+                }
+
+                //defer wg.Done()
+                let mut p = UploadPartParams {
+                    bucket_name:  bucket_name.to_string(),
+                    object_name:  object_name.to_string(),
+                    upload_id: clone_upload_id,
+                    reader:        ReaderImpl::Body(Bytes::from(buf.clone())),
+                    part_number,
+                    md5_base64,
+                    size:          length as i64,
+                    //sse:           opts.server_side_encryption,
+                    stream_sha256: !opts.disable_content_sha256,
+                    custom_header,
+                    sha256_hex:    "".to_string(),
+                    trailer:       HeaderMap::new(),
+                };
+                let obj_part = clone_self.upload_part(&mut p).await.expect("err");
+                
+                let mut clone_parts_info = clone_parts_info.write().unwrap();
+                clone_parts_info.entry(part_number).or_insert(obj_part);
+
+                clone_bufs_tx.send(buf);
+            });
+
+            total_uploaded_size += length as i64;
+        }
+
+        let results = join_all(futures).await;
+
+        select! {
+            err = err_rx.recv() => {
+                return Err(err.expect("err"));
+            }
+            else => (),
+        }
+
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let mut part_number: i64 = total_parts_count;
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.read().unwrap().len());
+        for i in 1..part_number {
+            let part = parts_info.read().unwrap()[&i].clone();
+
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag:               part.etag,
+                part_num:           part.part_num,
+                checksum_crc32:     part.checksum_crc32,
+                checksum_crc32c:    part.checksum_crc32c,
+                checksum_sha1:      part.checksum_sha1,
+                checksum_sha256:    part.checksum_sha256,
+                checksum_crc64nvme: part.checksum_crc64nvme,
+                ..Default::default()
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+
+        let mut opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum:          opts.auto_checksum,
+            ..Default::default()
+        };
+        apply_auto_checksum(&mut opts, &mut all_parts);
+
+        let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+
+    pub async fn put_object_gcs(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.send_content_md5 = false;
+        }
+
+        let mut md5_base64: String = "".to_string();
+        let progress_reader = reader;//newHook(reader, opts.progress);
+
+        self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts).await
+    }
+
+    pub async fn put_object_do(&self, bucket_name: &str, object_name: &str, reader: ReaderImpl, md5_base64: &str, sha256_hex: &str, size: i64, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
+        let custom_header = opts.header();
+
+        let mut req_metadata = RequestMetadata {
+            bucket_name:      bucket_name.to_string(),
+            object_name:      object_name.to_string(),
+            custom_header,
+            content_body:      reader,
+            content_length:    size,
+            content_md5_base64: md5_base64.to_string(),
+            content_sha256_hex: sha256_hex.to_string(),
+            stream_sha256:     !opts.disable_content_sha256,
+            add_crc: Default::default(),
+            bucket_location: Default::default(),
+            pre_sign_url: Default::default(),
+            query_values: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            expires: Default::default(),
+            trailer: Default::default(),
+        };
+        let mut add_crc = false;//self.trailing_header_support && md5_base64 == "" && !s3utils.IsGoogleEndpoint(self.endpoint_url) && (opts.disable_content_sha256 || self.secure);
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            req_metadata.add_crc = opts.checksum;
+        } else if add_crc {
+            for (k, _) in opts.user_metadata {
+                if k.to_lowercase().starts_with("x-amz-checksum-") {
+                    add_crc = false;
+                }
+            }
+            if add_crc {
+                opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
+                req_metadata.add_crc = opts.auto_checksum;
+            }
+        }
+
+        if opts.internal.source_version_id != "" {
+            if !opts.internal.source_version_id.is_empty() {
+                if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) {
+                    return Err(std::io::Error::other(err_invalid_argument(&err.to_string())));
+                }
+            }
+            let mut url_values = HashMap::new();
+            url_values.insert("versionId".to_string(), opts.internal.source_version_id);
+            req_metadata.query_values = url_values;
+        }
+
+        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
+
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
+        }
+
+        let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
+            (
+                OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
+                "".to_string()
+            )
+        } else {
+            (OffsetDateTime::now_utc(), "".to_string())
+        };
+        let h = resp.headers();
+        Ok(UploadInfo {
+            bucket:           bucket_name.to_string(),
+            key:              object_name.to_string(),
+            etag:             trim_etag(h.get("ETag").expect("err").to_str().expect("err")),
+            version_id:       if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { h_x_amz_version_id.to_str().expect("err").to_string() } else { "".to_string() },
+            size:             size,
+            expiration:       exp_time,
+            expiration_rule_id: rule_id,
+            checksum_crc32:     if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { h_checksum_crc32.to_str().expect("err").to_string() } else { "".to_string() },
+            checksum_crc32c:    if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { h_checksum_crc32c.to_str().expect("err").to_string() } else { "".to_string() },
+            checksum_sha1:      if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { h_checksum_sha1.to_str().expect("err").to_string() } else { "".to_string() },
+            checksum_sha256:    if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { h_checksum_sha256.to_str().expect("err").to_string() } else { "".to_string() },
+            checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { h_checksum_crc64nvme.to_str().expect("err").to_string() } else { "".to_string() },
+            ..Default::default()
+        })
+    }
+}

ecstore/src/client/api_remove.rs

@@ -0,0 +1,387 @@
+#![allow(clippy::map_entry)]
+use std::fmt::Display;
+use std::{collections::HashMap, sync::Arc};
+use bytes::Bytes;
+use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
+use s3s::S3ErrorCode;
+use time::OffsetDateTime;
+use s3s::dto::ReplicationStatus;
+use tokio::sync::mpsc::{self, Receiver, Sender};
+use http::{HeaderMap, HeaderValue, Method, StatusCode};
+
+use reader::hasher::{sum_sha256_hex, sum_md5_base64};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+use crate::{
+    disk::DiskAPI,
+    store_api::{
+        GetObjectReader, ObjectInfo, StorageAPI,
+    },
+};
+use crate::client::{
+    transition_api::{TransitionClient, RequestMetadata, ReaderImpl},
+    api_error_response::{http_resp_to_error_response, to_error_response, ErrorResponse,},
+};
+
+struct RemoveBucketOptions {
+    forced_elete: bool,
+}
+
+#[derive(Debug)]
+pub struct AdvancedRemoveOptions {
+    replication_delete_marker: bool,
+    replication_status: ReplicationStatus,
+    replication_mtime: OffsetDateTime,
+    replication_request: bool,
+    replication_validity_check: bool,
+}
+
+impl Default for AdvancedRemoveOptions {
+    fn default() -> Self {
+        Self {
+            replication_delete_marker: false,
+            replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
+            replication_mtime: OffsetDateTime::now_utc(),
+            replication_request: false,
+            replication_validity_check: false,
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct RemoveObjectOptions {
+    pub force_delete: bool,
+    pub governance_bypass: bool,
+    pub version_id: String,
+    pub internal: AdvancedRemoveOptions,
+}
+
+impl TransitionClient {
+    pub async fn remove_bucket_with_options(&self, bucket_name: &str, opts: &RemoveBucketOptions) -> Result<(), std::io::Error> {
+        let mut headers = HeaderMap::new();
+        /*if opts.force_delete {
+            headers.insert(rustFSForceDelete, "true");
+        }*/
+
+        let resp = self.execute_method(Method::DELETE, &mut RequestMetadata {
+            bucket_name:        bucket_name.to_string(),
+            content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+            custom_header:      headers,
+            object_name: "".to_string(),
+            query_values:  Default::default(),
+            content_body:  ReaderImpl::Body(Bytes::new()),
+            content_length: 0,
+            content_md5_base64: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        }).await?;
+
+        {
+            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
+            bucket_loc_cache.delete(bucket_name);
+        }
+        Ok(())
+    }
+
+    pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> {
+        let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata {
+            bucket_name:        bucket_name.to_string(),
+            content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+            custom_header: Default::default(),
+            object_name: "".to_string(),
+            query_values:  Default::default(),
+            content_body:  ReaderImpl::Body(Bytes::new()),
+            content_length: 0,
+            content_md5_base64: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        }).await?;
+
+        {
+            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
+            bucket_loc_cache.delete(bucket_name);
+        }
+
+        Ok(())
+    }
+
+    pub async fn remove_object(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Option<std::io::Error> {
+        let res = self.remove_object_inner(bucket_name, object_name, opts).await.expect("err");
+        res.err
+    }
+
+    pub async fn remove_object_inner(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Result<RemoveObjectResult, std::io::Error> {
+        let mut url_values = HashMap::new();
+
+        if opts.version_id != "" {
+            url_values.insert("versionId".to_string(), opts.version_id.clone());
+        }
+
+        let mut headers = HeaderMap::new();
+
+        if opts.governance_bypass {
+            headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));//amzBypassGovernance
+        }
+
+        let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata {
+            bucket_name:       bucket_name.to_string(),
+            object_name:       object_name.to_string(),
+            content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+            query_values:      url_values,
+            custom_header:     headers,
+            content_body:  ReaderImpl::Body(Bytes::new()),
+            content_length: 0,
+            content_md5_base64: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        }).await?;
+
+        Ok(RemoveObjectResult {
+            object_name: object_name.to_string(),
+            object_version_id:        opts.version_id,
+            delete_marker:            resp.headers().get("x-amz-delete-marker").expect("err") == "true",
+            delete_marker_version_id: resp.headers().get("x-amz-version-id").expect("err").to_str().expect("err").to_string(),
+            ..Default::default()
+        })
+    }
+
+    pub async fn remove_objects_with_result(self: Arc<Self>, bucket_name: &str, objects_rx: Receiver<ObjectInfo>, opts: RemoveObjectsOptions) -> Receiver<RemoveObjectResult> {
+        let (result_tx, mut result_rx) = mpsc::channel(1);
+
+        let self_clone = Arc::clone(&self);
+        let bucket_name_owned = bucket_name.to_string();
+
+        tokio::spawn(async move {
+            self_clone.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts).await;
+        });
+        result_rx
+    }
+
+    pub async fn remove_objects(self: Arc<Self>, bucket_name: &str, objects_rx: Receiver<ObjectInfo>, opts: RemoveObjectsOptions) -> Receiver<RemoveObjectError> {
+        let (error_tx, mut error_rx) = mpsc::channel(1);
+
+        let self_clone = Arc::clone(&self);
+        let bucket_name_owned = bucket_name.to_string();
+
+        let (result_tx, mut result_rx) = mpsc::channel(1);
+        tokio::spawn(async move {
+            self_clone.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts).await;
+        });
+        tokio::spawn(async move {
+            while let Some(res) = result_rx.recv().await {
+                if res.err.is_none() {
+                    continue;
+                }
+                error_tx.send(RemoveObjectError {
+                    object_name: res.object_name,
+                    version_id:  res.object_version_id,
+                    err:         res.err,
+                    ..Default::default()
+                }).await;
+            }
+        });
+
+        error_rx
+    }
+
+    pub async fn remove_objects_inner(&self, bucket_name: &str, mut objects_rx: Receiver<ObjectInfo>, result_tx: &Sender<RemoveObjectResult>, opts: RemoveObjectsOptions) -> Result<(), std::io::Error> {
+        let max_entries = 1000;
+        let mut finish = false;
+        let mut url_values = HashMap::new();
+        url_values.insert("delete".to_string(), "".to_string());
+
+        loop {
+            if finish {
+                break;
+            }
+            let mut count = 0;
+            let mut batch = Vec::<ObjectInfo>::new();
+
+            while let Some(object) = objects_rx.recv().await {
+                if has_invalid_xml_char(&object.name) {
+                    let remove_result = self.remove_object_inner(bucket_name, &object.name, RemoveObjectOptions {
+                        version_id:        object.version_id.expect("err").to_string(),
+                        governance_bypass: opts.governance_bypass,
+                        ..Default::default()
+                    }).await?;
+                    let remove_result_clone = remove_result.clone();
+                    if !remove_result.err.is_none() {
+                        match to_error_response(&remove_result.err.expect("err")).code {
+                            S3ErrorCode::InvalidArgument | S3ErrorCode::NoSuchVersion => {
+                                continue;
+                            }
+                            _ => (),
+                        }
+                        result_tx.send(remove_result_clone.clone()).await;
+                    }
+
+                    result_tx.send(remove_result_clone).await;
+                    continue;
+                }
+
+                batch.push(object);
+                count += 1;
+                if count >= max_entries {
+                    break;
+                }
+            }
+            if count == 0 {
+                break;
+            } 
+            if count < max_entries {
+                finish = true;
+            }
+
+            let mut headers = HeaderMap::new();
+            if opts.governance_bypass {
+                headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));
+            }
+
+            let remove_bytes = generate_remove_multi_objects_request(&batch);
+            let resp = self.execute_method(http::Method::POST, &mut RequestMetadata {
+                bucket_name:       bucket_name.to_string(),
+                query_values:      url_values.clone(),
+                content_body:      ReaderImpl::Body(Bytes::from(remove_bytes.clone())),
+                content_length:    remove_bytes.len() as i64,
+                content_md5_base64: sum_md5_base64(&remove_bytes),
+                content_sha256_hex: sum_sha256_hex(&remove_bytes),
+                custom_header:     headers,
+                object_name: "".to_string(),
+                stream_sha256: false,
+                trailer: HeaderMap::new(),
+                pre_sign_url: Default::default(),
+                add_crc: Default::default(),
+                extra_pre_sign_header: Default::default(),
+                bucket_location: Default::default(),
+                expires: Default::default(),
+            }).await?;
+
+            let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec();
+            process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_bytes)), result_tx.clone());
+        }
+        Ok(())
+    }
+
+    pub async fn remove_incomplete_upload(&self, bucket_name: &str, object_name: &str) -> Result<(), std::io::Error> {
+        let upload_ids = self.find_upload_ids(bucket_name, object_name)?;
+        for upload_id in upload_ids {
+            self.abort_multipart_upload(bucket_name, object_name, &upload_id).await?;
+        }
+
+        Ok(())
+    }
+
+    pub async fn abort_multipart_upload(&self, bucket_name: &str, object_name: &str, upload_id: &str) -> Result<(), std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("uploadId".to_string(), upload_id.to_string());
+
+        let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata {
+            bucket_name:       bucket_name.to_string(),
+            object_name:       object_name.to_string(),
+            query_values:       url_values,
+            content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+            custom_header: HeaderMap::new(),
+            content_body:  ReaderImpl::Body(Bytes::new()),
+            content_length: 0,
+            content_md5_base64: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        }).await?;
+        //if resp.is_some() {
+            if resp.status() != StatusCode::NO_CONTENT {
+                let error_response: ErrorResponse;
+                match resp.status() {
+                    StatusCode::NOT_FOUND => {
+                        error_response = ErrorResponse {
+                            code:        S3ErrorCode::NoSuchUpload,
+                            message:     "The specified multipart upload does not exist.".to_string(),
+                            bucket_name: bucket_name.to_string(),
+                            key:         object_name.to_string(),
+                            request_id:  resp.headers().get("x-amz-request-id").expect("err").to_str().expect("err").to_string(),
+                            host_id:     resp.headers().get("x-amz-id-2").expect("err").to_str().expect("err").to_string(),
+                            region:      resp.headers().get("x-amz-bucket-region").expect("err").to_str().expect("err").to_string(),
+                            ..Default::default()
+                        };
+                    }
+                    _ => {
+                        return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
+                    }
+                }
+                return Err(std::io::Error::other(error_response));
+            }
+        //}
+        Ok(())
+    }
+}
+
+#[derive(Debug, Default)]
+struct RemoveObjectError {
+    object_name: String,
+    version_id:  String,
+    err:        Option<std::io::Error>,
+}
+
+impl Display for RemoveObjectError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        if self.err.is_none() {
+            return write!(f, "unexpected remove object error result");
+        }
+        write!(f, "{}", self.err.as_ref().expect("err").to_string())
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct RemoveObjectResult {
+    pub object_name: String,
+    pub object_version_id: String,
+    pub delete_marker: bool,
+    pub delete_marker_version_id: String,
+    pub err: Option<std::io::Error>,
+}
+
+impl Clone for RemoveObjectResult {
+    fn clone(&self) -> Self {
+        Self {
+            object_name: self.object_name.clone(),
+            object_version_id: self.object_version_id.clone(),
+            delete_marker: self.delete_marker,
+            delete_marker_version_id: self.delete_marker_version_id.clone(),
+            err: None, //err
+        }
+    }    
+}
+
+pub struct RemoveObjectsOptions {
+    pub governance_bypass: bool,
+}
+
+pub fn generate_remove_multi_objects_request(objects: &[ObjectInfo]) -> Vec<u8> {
+    todo!();
+}
+
+pub fn process_remove_multi_objects_response(body: ReaderImpl, result_tx: Sender<RemoveObjectResult>) {
+    todo!();
+}
+
+fn has_invalid_xml_char(str: &str) -> bool {
+    false
+}

ecstore/src/client/api_s3_datatypes.rs

@@ -0,0 +1,336 @@
+#![allow(clippy::map_entry)]
+use std::collections::HashMap;
+use s3s::dto::Owner;
+use serde::{Serialize, Deserialize};
+use time::OffsetDateTime;
+
+use rustfs_utils::crypto::base64_decode;
+use crate::checksum::ChecksumMode;
+use crate::client::transition_api::ObjectMultipartInfo;
+
+use super::transition_api;
+
+pub struct ListAllMyBucketsResult {
+    pub owner: Owner,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct CommonPrefix {
+    pub prefix: String,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct ListBucketV2Result {
+    pub common_prefixes: Vec<CommonPrefix>,
+    pub contents: Vec<transition_api::ObjectInfo>,
+    pub delimiter: String,
+    pub encoding_type: String,
+    pub is_truncated: bool,
+    pub max_keys: i64,
+    pub name: String,
+    pub next_continuation_token: String,
+    pub continuation_token: String,
+    pub prefix:             String,
+    pub fetch_owner: String,
+    pub start_after: String,
+}
+
+pub struct Version {
+    etag:         String,
+    is_latest:    bool,
+    key:          String,
+    last_modified: OffsetDateTime,
+    owner: Owner,
+    size: i64,
+    storage_class: String,
+    version_id: String,
+    user_metadata: HashMap<String, String>,
+    user_tags: HashMap<String, String>,
+    is_delete_marker: bool,
+}
+
+pub struct ListVersionsResult {
+    versions: Vec<Version>,
+    common_prefixes: Vec<CommonPrefix>,
+    name:                String,
+    prefix:              String,
+    delimiter:           String,
+    max_keys:            i64,
+    encoding_type:       String,
+    is_truncated:        bool,
+    key_marker:          String,
+    version_id_marker:      String,
+    next_key_marker:        String,
+    next_version_id_marker: String,
+}
+
+impl ListVersionsResult {
+    fn unmarshal_xml() -> Result<(), std::io::Error> {
+        todo!();
+    }
+}
+
+pub struct ListBucketResult {
+    common_prefixes: Vec<CommonPrefix>,
+    contents: Vec<transition_api::ObjectInfo>,
+    delimiter: String,
+    encoding_type: String,
+    is_truncated: bool,
+    marker:       String,
+    max_keys:     i64,
+    name:         String,
+    next_marker:  String,
+    prefix:       String,
+}
+
+pub struct ListMultipartUploadsResult {
+    bucket:                String,
+    key_marker:            String,
+    upload_id_marker:      String,
+    next_key_marker:       String,
+    next_upload_id_marker: String,
+    encoding_type:         String,
+    max_uploads:           i64,
+    is_truncated:          bool,
+    uploads:            Vec<ObjectMultipartInfo>,
+    prefix:             String,
+    delimiter:          String,
+    common_prefixes:    Vec<CommonPrefix>,
+}
+
+pub struct Initiator {
+    id:           String,
+    display_name: String,
+}
+
+pub struct CopyObjectResult {
+    pub etag:          String,
+    pub last_modified: OffsetDateTime,
+}
+
+#[derive(Debug, Clone)]
+pub struct ObjectPart {
+    pub etag: String,
+    pub part_num: i64,
+    pub last_modified: OffsetDateTime,
+    pub size: i64,
+    pub checksum_crc32:     String,
+    pub checksum_crc32c:    String,
+    pub checksum_sha1:      String,
+    pub checksum_sha256:    String,
+    pub checksum_crc64nvme: String,
+}
+
+impl Default for ObjectPart {
+    fn default() -> Self {
+        ObjectPart {
+            etag: Default::default(),
+            part_num: 0,
+            last_modified: OffsetDateTime::now_utc(),
+            size: 0,
+            checksum_crc32:     Default::default(),
+            checksum_crc32c:    Default::default(),
+            checksum_sha1:      Default::default(),
+            checksum_sha256:    Default::default(),
+            checksum_crc64nvme: Default::default(),
+        }
+    }
+    
+}
+
+impl ObjectPart {
+    fn checksum(&self, t: &ChecksumMode) -> String {
+        match t {
+            ChecksumMode::ChecksumCRC32C => {
+                return self.checksum_crc32c.clone();
+            }
+            ChecksumMode::ChecksumCRC32 => {
+                return self.checksum_crc32.clone();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return self.checksum_sha1.clone();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return self.checksum_sha256.clone();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return self.checksum_crc64nvme.clone();
+            }
+            _ => {
+                return "".to_string();
+            }
+        }
+    }
+
+    pub fn checksum_raw(&self, t: &ChecksumMode) -> Result<Vec<u8>, std::io::Error> {
+        let b = self.checksum(t);
+        if b == "" {
+            return Err(std::io::Error::other("no checksum set"));
+        }
+        let decoded = match base64_decode(b.as_bytes()) {
+            Ok(b) => b,
+            Err(e) => return Err(std::io::Error::other(e)),
+        };
+        if decoded.len() != t.raw_byte_len() as usize {
+            return Err(std::io::Error::other("checksum length mismatch"));
+        }
+        Ok(decoded)
+    }
+}
+
+pub struct ListObjectPartsResult {
+    pub bucket: String,
+    pub key: String,
+    pub upload_id: String,
+    pub initiator: Initiator,
+    pub owner: Owner,
+    pub storage_class: String,
+    pub part_number_marker: i32,
+    pub next_part_number_marker: i32,
+    pub max_parts: i32,
+    pub checksum_algorithm: String,
+    pub checksum_type: String,
+    pub is_truncated: bool,
+    pub object_parts: Vec<ObjectPart>,
+    pub encoding_type: String,
+}
+
+#[derive(Debug, Default)]
+pub struct InitiateMultipartUploadResult {
+    pub bucket: String,
+    pub key: String,
+    pub upload_id: String,
+}
+
+#[derive(Debug, Default)]
+pub struct CompleteMultipartUploadResult {
+    pub location: String,
+    pub bucket: String,
+    pub key: String,
+    pub etag: String,
+    pub checksum_crc32: String,
+    pub checksum_crc32c: String,
+    pub checksum_sha1: String,
+    pub checksum_sha256: String,
+    pub checksum_crc64nvme: String,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(serde::Serialize)]
+pub struct CompletePart { //api has
+    pub etag: String,
+    pub part_num: i64,
+    pub checksum_crc32:     String,
+    pub checksum_crc32c:    String,
+    pub checksum_sha1:      String,
+    pub checksum_sha256:    String,
+    pub checksum_crc64nvme: String,
+}
+
+impl CompletePart {
+    fn checksum(&self, t: &ChecksumMode) -> String {
+        match t {
+            ChecksumMode::ChecksumCRC32C => {
+                return self.checksum_crc32c.clone();
+            }
+            ChecksumMode::ChecksumCRC32 => {
+                return self.checksum_crc32.clone();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return self.checksum_sha1.clone();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return self.checksum_sha256.clone();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return self.checksum_crc64nvme.clone();
+            }
+            _ => {
+                return "".to_string();
+            }
+        }
+    }
+}
+
+pub struct CopyObjectPartResult {
+    pub etag: String,
+    pub last_modified: OffsetDateTime,
+}
+
+#[derive(Debug, Default)]
+#[derive(serde::Serialize)]
+pub struct CompleteMultipartUpload {
+    pub parts: Vec<CompletePart>,
+}
+
+impl CompleteMultipartUpload {
+    pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
+        //let buf = serde_json::to_string(self)?;
+        let buf = match serde_xml_rs::to_string(self) {
+            Ok(buf) => buf,
+            Err(e) => {
+                return Err(std::io::Error::other(e));
+            }
+        };
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
+        todo!();
+    }
+}
+
+pub struct CreateBucketConfiguration {
+    pub location: String,
+}
+
+#[derive(serde::Serialize)]
+pub struct DeleteObject {  //api has
+    pub key: String,
+    pub version_id: String,
+}
+
+pub struct DeletedObject {  //s3s has
+    pub key: String,
+    pub version_id: String,
+    pub deletemarker: bool,
+    pub deletemarker_version_id: String,
+}
+
+pub struct NonDeletedObject {
+    pub key:        String,
+    pub code:       String,
+    pub message:    String,
+    pub version_id: String,
+}
+
+#[derive(serde::Serialize)]
+pub struct DeleteMultiObjects {
+    pub quiet:   bool,
+    pub objects: Vec<DeleteObject>,
+}
+
+impl DeleteMultiObjects {
+    pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
+        //let buf = serde_json::to_string(self)?;
+        let buf = match serde_xml_rs::to_string(self) {
+            Ok(buf) => buf,
+            Err(e) => {
+                return Err(std::io::Error::other(e));
+            }
+        };
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
+        todo!();
+    }
+}
+
+pub struct DeleteMultiObjectsResult {
+    pub deleted_objects:   Vec<DeletedObject>,
+    pub undeleted_objects: Vec<NonDeletedObject>,
+}

ecstore/src/client/bucket_cache.rs

@@ -0,0 +1,224 @@
+#![allow(clippy::map_entry)]
+use http::Request;
+use hyper::body::Incoming;
+use std::{collections::HashMap, sync::Arc};
+use tracing::warn;
+use tracing::{error, info, debug};
+use hyper::StatusCode;
+
+use reader::hasher::{Hasher, Sha256};
+use s3s::S3ErrorCode;
+use s3s::Body;
+use crate::client::{
+    api_error_response::{http_resp_to_error_response, to_error_response},
+    transition_api::{TransitionClient, Document},
+};
+use crate::signer;
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+
+use super::constants::UNSIGNED_PAYLOAD;
+use super::credentials::SignatureType;
+
+pub struct BucketLocationCache {
+    items: HashMap<String, String>,
+}
+
+impl BucketLocationCache {
+    pub fn new() -> BucketLocationCache {
+        BucketLocationCache{
+            items: HashMap::new(),
+        }
+    }
+
+    pub fn get(&self, bucket_name: &str) -> Option<String> {
+        self.items.get(bucket_name).map(|s| s.clone())
+    }
+
+    pub fn set(&mut self, bucket_name: &str, location: &str) {
+        self.items.insert(bucket_name.to_string(), location.to_string());
+    }
+
+    pub fn delete(&mut self, bucket_name: &str) {
+        self.items.remove(bucket_name);
+    }
+}
+
+impl TransitionClient {
+    pub async fn get_bucket_location(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        Ok(self.get_bucket_location_inner(bucket_name).await?)
+    }
+
+    async fn get_bucket_location_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        if self.region != "" {
+            return Ok(self.region.clone())
+        }
+
+        let mut location;
+        {
+            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
+            let ret = bucket_loc_cache.get(bucket_name);
+            if let Some(location) = ret {
+                return Ok(location);
+            }
+            //location = ret?;
+        }
+
+        let req = self.get_bucket_location_request(bucket_name)?;
+
+        let mut resp = self.doit(req).await?;
+        location = process_bucket_location_response(resp, bucket_name).await?;
+        {
+            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
+            bucket_loc_cache.set(bucket_name, &location);
+        }
+        Ok(location)
+    }
+
+    fn get_bucket_location_request(&self, bucket_name: &str) -> Result<http::Request<Body>, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("location".to_string(), "".to_string());
+
+        let mut target_url = self.endpoint_url.clone();
+        let scheme = self.endpoint_url.scheme();
+        let h = target_url.host().expect("host is none.");
+        let default_port = if scheme == "https" {
+            443
+        } else {
+            80
+        };
+        let p = target_url.port().unwrap_or(default_port);
+
+        let is_virtual_style = self.is_virtual_host_style_request(&target_url, bucket_name);
+
+        let mut url_str: String = "".to_string();
+
+        if is_virtual_style {
+            url_str = scheme.to_string();
+            url_str.push_str("://");
+            url_str.push_str(bucket_name);
+            url_str.push_str(".");
+            url_str.push_str(target_url.host_str().expect("err"));
+            url_str.push_str("/?location");
+        } else {
+            let mut path = bucket_name.to_string();
+            path.push_str("/");
+            target_url.set_path(&path);
+            {
+                let mut q = target_url.query_pairs_mut();
+                for (k, v) in url_values {
+                    q.append_pair(&k, &urlencoding::encode(&v));
+                }
+            }
+            url_str = target_url.to_string();
+        }
+
+        let mut req_builder = Request::builder().method(http::Method::GET).uri(url_str);
+
+        self.set_user_agent(&mut req_builder);
+
+        let value;
+        {
+            let mut creds_provider = self.creds_provider.lock().unwrap();
+            value = match creds_provider.get_with_context(Some(self.cred_context())) {
+                Ok(v) => v,
+                Err(err) => {
+                    return Err(std::io::Error::other(err));
+                }
+            };
+        }
+
+        let mut signer_type = value.signer_type.clone();
+        let mut access_key_id      = value.access_key_id;
+        let mut secret_access_key  = value.secret_access_key;
+        let mut session_token      = value.session_token;
+
+        if self.override_signer_type != SignatureType::SignatureDefault {
+            signer_type = self.override_signer_type.clone();
+        }
+
+        if value.signer_type == SignatureType::SignatureAnonymous {
+            signer_type = SignatureType::SignatureAnonymous
+        }
+
+        if signer_type == SignatureType::SignatureAnonymous {
+            let req = match req_builder.body(Body::empty()) {
+                Ok(req) => return Ok(req),
+                Err(err) => {
+                    return Err(std::io::Error::other(err));
+                }
+            };
+        }
+
+        if signer_type == SignatureType::SignatureV2 {
+            let req_builder = signer::sign_v2(req_builder, 0, &access_key_id, &secret_access_key, is_virtual_style);
+            let req = match req_builder.body(Body::empty()) {
+                Ok(req) => return Ok(req),
+                Err(err) => {
+                    return Err(std::io::Error::other(err));
+                }
+            };
+        }
+
+        let mut content_sha256 = EMPTY_STRING_SHA256_HASH.to_string();
+        if self.secure {
+            content_sha256 = UNSIGNED_PAYLOAD.to_string();
+        }
+
+        req_builder.headers_mut().expect("err").insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
+        let req_builder = signer::sign_v4(req_builder, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
+        let req = match req_builder.body(Body::empty()) {
+            Ok(req) => return Ok(req),
+            Err(err) => {
+                return Err(std::io::Error::other(err));
+            }
+        };
+    }
+}
+
+async fn process_bucket_location_response(mut resp: http::Response<Body>, bucket_name: &str) -> Result<String, std::io::Error> {
+    //if resp != nil {
+        if resp.status() != StatusCode::OK {
+            let err_resp = http_resp_to_error_response(resp, vec![], bucket_name, "");
+            match err_resp.code {
+                S3ErrorCode::NotImplemented => {
+                    match err_resp.server.as_str() {
+                        "AmazonSnowball" => {
+                            return Ok("snowball".to_string());
+                        }
+                        "cloudflare" => {
+                            return Ok("us-east-1".to_string());
+                        }
+                        _ => {
+                            return Err(std::io::Error::other(err_resp));
+                        }
+                    }
+                }
+                S3ErrorCode::AuthorizationHeaderMalformed |
+                //S3ErrorCode::InvalidRegion |
+                S3ErrorCode::AccessDenied => {
+                    if err_resp.region == "" {
+                      return Ok("us-east-1".to_string());
+                    }
+                    return Ok(err_resp.region);
+                }
+                _ => {
+                    return Err(std::io::Error::other(err_resp));
+                }
+            }
+        }
+    //}
+
+    let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
+    let Document(location_constraint) = serde_xml_rs::from_str::<Document>(&String::from_utf8(b).unwrap()).unwrap();
+
+    let  mut location = location_constraint;
+    if location == "" {
+        location = "us-east-1".to_string();
+    }
+
+    if location == "EU" {
+        location = "eu-west-1".to_string();
+    }
+
+    Ok(location)
+}

ecstore/src/client/constants.rs

@@ -0,0 +1,24 @@
+#![allow(clippy::map_entry)]
+use std::{collections::HashMap, sync::Arc};
+use time::{macros::format_description, format_description::FormatItem};
+use lazy_static::lazy_static;
+
+pub const ABS_MIN_PART_SIZE: i64 = 1024 * 1024 * 5;
+pub const MAX_PARTS_COUNT: i64 = 10000;
+pub const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+pub const MIN_PART_SIZE: i64 = 1024 * 1024 * 16;
+
+pub const MAX_SINGLE_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+
+pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
+pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
+
+pub const TOTAL_WORKERS: i64 = 4;
+
+pub const SIGN_V4_ALGORITHM: &str   = "AWS4-HMAC-SHA256";
+pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] = format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
+
+const GetObjectAttributesTags: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts";
+const GetObjectAttributesMaxParts: i64 = 1000;
+const RUSTFS_BUCKET_SOURCE_MTIME: &str = "X-RustFs-Source-Mtime";

ecstore/src/client/credentials.rs

@@ -0,0 +1,166 @@
+use std::fmt::{Display, Formatter};
+
+use time::OffsetDateTime;
+
+#[derive(Debug, Default, Clone, Eq, PartialEq)]
+pub enum SignatureType {
+    #[default]
+    SignatureDefault,
+    SignatureV4,
+    SignatureV2,
+    SignatureV4Streaming,
+    SignatureAnonymous,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct Credentials<P: Provider + Default> {
+    creds: Value,
+    force_refresh: bool,
+    provider: P,
+}
+
+impl<P: Provider + Default> Credentials<P>
+{
+    pub fn new(provider: P) -> Self {
+        Self {
+            provider: provider,
+            force_refresh: true,
+            ..Default::default()
+        }
+    }
+
+    pub fn get(&mut self) -> Result<Value, std::io::Error> {
+        self.get_with_context(None)
+    }
+
+    pub fn get_with_context(&mut self, mut cc: Option<CredContext>) -> Result<Value, std::io::Error> {
+        if self.is_expired() {
+            let creds = self.provider.retrieve_with_cred_context(cc.expect("err"));
+            self.creds = creds;
+            self.force_refresh = false;
+        }
+
+        Ok(self.creds.clone())
+    }
+
+    fn expire(&mut self) {
+        self.force_refresh = true;
+    }
+
+    pub fn is_expired(&self) -> bool {
+        self.force_refresh || self.provider.is_expired()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Value {
+    pub access_key_id: String,
+    pub secret_access_key: String,
+    pub session_token: String,
+    pub expiration: OffsetDateTime,
+    pub signer_type: SignatureType,
+}
+
+impl Default for Value {
+    fn default() -> Self {
+        Self {
+            access_key_id: "".to_string(),
+            secret_access_key: "".to_string(),
+            session_token: "".to_string(),
+            expiration: OffsetDateTime::now_utc(),
+            signer_type: SignatureType::SignatureDefault,
+        }
+    }
+}
+
+pub struct CredContext {
+    //pub client: SendRequest,
+    pub endpoint: String,
+}
+
+trait Provider {
+    fn retrieve(&self) -> Value;
+    fn retrieve_with_cred_context(&self, _: CredContext) -> Value;
+    fn is_expired(&self) -> bool;
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct Static(pub Value);
+
+impl Provider for Static {
+    fn retrieve(&self) -> Value {
+        if self.0.access_key_id == "" || self.0.secret_access_key == "" {
+            return Value {
+                signer_type: SignatureType::SignatureAnonymous,
+                ..Default::default()
+            };
+        }
+        self.0.clone()
+    }
+
+    fn retrieve_with_cred_context(&self, _: CredContext) -> Value {
+        self.retrieve()
+    }
+
+    fn is_expired(&self) -> bool {
+        false
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct STSError {
+    pub r#type:  String,
+    pub code:    String,
+    pub message: String,
+}
+
+#[derive(Debug, Clone, thiserror::Error)]
+pub struct ErrorResponse {
+    pub sts_error: STSError,
+    pub request_id: String,
+}
+
+impl Display for ErrorResponse {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.error())
+    }
+}
+
+impl ErrorResponse {
+    fn error(&self) -> String {
+      if self.sts_error.message == "" {
+          return format!("Error response code {}.", self.sts_error.code);
+      }
+      return self.sts_error.message.clone();
+    }
+}
+
+struct Error {
+    code:       String,
+    message:    String,
+    bucket_name: String,
+    key:        String,
+    resource:   String,
+    request_id:  String,
+    host_id:     String,
+    region: String,
+    server: String,
+    status_code: i64,
+}
+
+impl Display for Error {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        if self.message == "" {
+            return write!(f, "{}", format!("Error response code {}.", self.code));
+        }
+        write!(f, "{}", self.message)
+    }
+}
+
+pub fn xml_decoder<T>(body: &[u8]) -> Result<T, std::io::Error> {
+    todo!();
+}
+
+pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error> {
+    todo!();
+}

ecstore/src/client/hook_reader.rs

@@ -0,0 +1,45 @@
+#![allow(clippy::map_entry)]
+use std::{collections::HashMap, sync::Arc};
+
+use crate::{
+    disk::{
+        error::{is_unformatted_disk, DiskError},
+        format::{DistributionAlgoVersion, FormatV3},
+        new_disk, DiskAPI, DiskInfo, DiskOption, DiskStore,
+    },
+    store_api::{
+        BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec,
+        ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartInfo, MultipartUploadResult,
+        ObjectIO, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI,
+    },
+    credentials::{Credentials, SignatureType,},
+    api_put_object_multipart::UploadPartParams,
+};
+
+use http::HeaderMap;
+use tokio_util::sync::CancellationToken;
+use tracing::warn;
+use tracing::{error, info};
+use url::Url;
+
+struct HookReader {
+    source: GetObjectReader,
+    hook:   GetObjectReader,
+}
+
+impl HookReader {
+    pub fn new(source: GetObjectReader, hook: GetObjectReader) -> HookReader {
+        HookReader {
+            source,
+            hook,
+        }
+    }
+
+    fn seek(&self, offset: i64, whence: i64) -> Result<i64> {
+        todo!();
+    }
+
+    fn read(&self, b: &[u8]) -> Result<i64> {
+        todo!();
+    }
+}

ecstore/src/client/mod.rs

@@ -0,0 +1,18 @@
+pub mod constants;
+pub mod transition_api;
+pub mod api_list;
+pub mod api_error_response;
+pub mod api_s3_datatypes;
+pub mod api_bucket_policy;
+pub mod api_put_object_common;
+pub mod api_get_options;
+pub mod api_get_object;
+pub mod api_put_object;
+pub mod api_put_object_streaming;
+pub mod api_put_object_multipart;
+pub mod api_remove;
+pub mod object_api_utils;
+pub mod object_handlers_common;
+pub mod admin_handler_utils;
+pub mod credentials;
+pub mod bucket_cache;

ecstore/src/client/object_api_utils.rs

@@ -0,0 +1,130 @@
+#![allow(clippy::map_entry)]
+use std::{collections::HashMap, sync::Arc};
+use http::HeaderMap;
+use tokio::io::BufReader;
+use std::io::Cursor;
+
+use s3s::S3ErrorCode;
+use crate::store_api::{
+    GetObjectReader, HTTPRangeSpec,
+    ObjectInfo, ObjectOptions,
+};
+use rustfs_filemeta::fileinfo::ObjectPartInfo;
+use rustfs_rio::HashReader;
+use crate::error::ErrorResponse;
+
+//#[derive(Clone)]
+pub struct PutObjReader {
+    pub reader: HashReader,
+    pub raw_reader: HashReader,
+    //pub sealMD5Fn: SealMD5CurrFn,
+}
+
+impl PutObjReader {
+    pub fn new(raw_reader: HashReader) -> Self {
+        todo!();
+    }
+
+    fn size(&self) -> usize {
+        //self.reader.size()
+        todo!();
+    }
+
+    fn md5_current_hex_string(&self) -> String {
+        todo!();
+    }
+
+    fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
+        self.reader = enc_reader;
+
+        Ok(())
+    }
+}
+
+pub type ObjReaderFn = Arc<dyn Fn(BufReader<Cursor<Vec<u8>>>, HeaderMap) -> GetObjectReader + 'static>;
+
+fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRangeSpec> {
+    if oi.size == 0 || oi.parts.len() == 0 {
+        return None;
+    }
+
+    let mut start: i64 = 0;
+    let mut end: i64 = -1;
+    let mut i = 0;
+    while i < oi.parts.len() && i < part_number {
+        start = end + 1;
+        end = start + oi.parts[i].actual_size as i64 - 1;
+        i += 1;
+    }
+
+    Some(HTTPRangeSpec {start: start as usize, end: Some(end as usize), is_suffix_length: false})
+}
+
+fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) {
+    let mut skip_length: i64 = 0;
+    let mut cumulative_actual_size: i64 = 0;
+    let mut first_part_idx: i64 = 0;
+    let mut compressed_offset: i64 = 0;
+    let mut part_skip: i64 = 0;
+    let mut decrypt_skip: i64 = 0;
+    let mut seq_num: u64 = 0;
+    for (i, part) in oi.parts.iter().enumerate() {
+        cumulative_actual_size += part.actual_size as i64;
+        if cumulative_actual_size <= offset {
+           compressed_offset += part.size as i64;
+        } else {
+            first_part_idx = i as i64;
+            skip_length = cumulative_actual_size - part.actual_size as i64;
+            break;
+        }
+    }
+    skip_length = offset - skip_length;
+
+    let parts: &[ObjectPartInfo] = &oi.parts;
+    if skip_length > 0 && parts.len() > first_part_idx as usize && parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0 {
+        todo!();
+    }
+
+    (compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
+}
+
+pub fn new_getobjectreader(rs: HTTPRangeSpec, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> {
+    //let (_, mut is_encrypted) = crypto.is_encrypted(oi.user_defined)?;
+    let mut is_encrypted = false;
+    let is_compressed = false;//oi.is_compressed_ok();
+
+    let mut get_fn: ObjReaderFn;
+
+        let (off, length) = match rs.get_offset_length(oi.size) {
+            Ok(x) => x,
+            Err(err) => return Err(ErrorResponse {
+                code: S3ErrorCode::InvalidRange,
+                message: err.to_string(),
+                key: None,
+                bucket_name: None,
+                region: None,
+                request_id: None,
+                host_id: "".to_string(),
+            }),
+        };
+        get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| {
+            //Box::pin({
+                /*let r = GetObjectReader {
+                    object_info: oi.clone(),
+                    stream: StreamingBlob::new(HashReader::new(input_reader, 10, None, None, 10)),
+                };
+                r*/
+                todo!();
+            //})
+        });
+
+    Ok((get_fn, off as i64, length as i64))
+}
+
+pub fn extract_etag(metadata: &HashMap<String, String>) -> String {
+	if let Some(etag) = metadata.get("etag") {
+        etag.clone()
+    } else {
+		metadata["md5Sum"].clone()
+	}
+}

ecstore/src/client/object_handlers_common.rs

@@ -0,0 +1,27 @@
+use lock::local_locker::MAX_DELETE_LIST;
+use crate::bucket::versioning::VersioningApi;
+use crate::bucket::versioning_sys::BucketVersioningSys;
+use crate::store::ECStore;
+use crate::store_api::{ObjectOptions, ObjectToDelete,};
+use crate::StorageAPI;
+use crate::bucket::lifecycle::lifecycle;
+
+pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], lc_event: lifecycle::Event) {
+    let mut remaining = to_del;
+    loop {
+        if remaining.len() <= 0 {break};
+        let mut to_del = remaining;
+        if to_del.len() > MAX_DELETE_LIST {
+            remaining = &to_del[MAX_DELETE_LIST..];
+            to_del = &to_del[..MAX_DELETE_LIST];
+        } else {
+            remaining = &[];
+        }
+        let vc = BucketVersioningSys::get(bucket).await.expect("err!");
+        let deleted_objs = api.delete_objects(bucket, to_del.to_vec(), ObjectOptions {
+            //prefix_enabled_fn:  vc.prefix_enabled(""),
+            version_suspended: vc.suspended(),
+            ..Default::default()
+        });
+    }
+}

ecstore/src/client/transition_api.rs

@@ -0,0 +1,888 @@
+#![allow(clippy::map_entry)]
+use std::pin::Pin;
+use bytes::Bytes;
+use futures::Future;
+use http::{HeaderMap, HeaderName};
+use serde::{Serialize, Deserialize};
+use uuid::Uuid;
+use rand::Rng;
+use std::{collections::HashMap, sync::{Arc, Mutex}};
+use std::sync::atomic::{AtomicI32, Ordering};
+use std::task::{Context, Poll};
+use time::Duration;
+use time::OffsetDateTime;
+use hyper_rustls::{ConfigBuilderExt, HttpsConnector};
+use hyper_util::{client::legacy::Client, rt::TokioExecutor, client::legacy::connect::HttpConnector};
+use http::{StatusCode, HeaderValue, request::{Request, Builder}, Response};
+use tracing::{error, debug};
+use url::{form_urlencoded, Url};
+use tokio::io::BufReader;
+use std::io::Cursor;
+
+use s3s::S3ErrorCode;
+use s3s::{dto::Owner, Body};
+use s3s::dto::ReplicationStatus;
+use crate::client::bucket_cache::BucketLocationCache;
+use reader::hasher::{Sha256, MD5,};
+use rustfs_rio::HashReader;
+use rustfs_utils::{
+    net::get_endpoint_url,
+    retry::{new_retry_timer, MAX_RETRY},
+};
+use crate::{
+    store_api::GetObjectReader,
+    checksum::ChecksumMode,
+};
+use crate::signer;
+use crate::client::{
+    constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER},
+    credentials::{Credentials, SignatureType, CredContext, Static,},
+    api_error_response::{to_error_response, http_resp_to_error_response, err_invalid_argument},
+    api_put_object_multipart::UploadPartParams,
+    api_put_object::PutObjectOptions,
+    api_get_options::GetObjectOptions,
+    api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ObjectPart},
+};
+
+const C_USER_AGENT_PREFIX: &str = "RustFS (linux; x86)";
+const C_USER_AGENT: &str        = "RustFS (linux; x86)";
+
+const SUCCESS_STATUS: [StatusCode; 3] = [
+    StatusCode::OK,
+    StatusCode::NO_CONTENT,
+    StatusCode::PARTIAL_CONTENT,
+];
+
+const C_UNKNOWN: i32 = -1;
+const C_OFFLINE: i32 = 0;
+const C_ONLINE: i32  = 1;
+
+//pub type ReaderImpl = Box<dyn Reader + Send + Sync + 'static>;
+pub enum ReaderImpl {
+    Body(Bytes),
+    ObjectBody(GetObjectReader),
+}
+
+pub type ReadCloser = BufReader<Cursor<Vec<u8>>>;
+
+pub struct TransitionClient {
+    pub endpoint_url: Url,
+    pub creds_provider: Arc<Mutex<Credentials<Static>>>,
+    pub override_signer_type: SignatureType,
+    /*app_info: TODO*/
+    pub secure: bool,
+    pub http_client:     Client<HttpsConnector<HttpConnector>, Body>,
+    //pub http_trace:      Httptrace.ClientTrace,
+    pub bucket_loc_cache: Arc<Mutex<BucketLocationCache>>,
+    pub is_trace_enabled: Arc<Mutex<bool>>,
+    pub trace_errors_only: Arc<Mutex<bool>>,
+    //pub trace_output: io.Writer,
+    pub s3_accelerate_endpoint: Arc<Mutex<String>>,
+    pub s3_dual_stack_enabled: Arc<Mutex<bool>>,
+    pub region: String,
+    pub random: u64,
+    pub lookup: BucketLookupType,
+    //pub lookupFn: func(u url.URL, bucketName string) BucketLookupType,
+    pub md5_hasher:    Arc<Mutex<Option<MD5>>>,
+    pub sha256_hasher: Option<Sha256>,
+    pub health_status: AtomicI32,
+    pub trailing_header_support: bool,
+    pub max_retries: i64,
+}
+
+#[derive(Debug, Default)]
+pub struct Options {
+    pub creds:        Credentials<Static>,
+    pub secure:       bool,
+    //pub transport:    http.RoundTripper,
+    //pub trace:        *httptrace.ClientTrace,
+    pub region:        String,
+    pub bucket_lookup: BucketLookupType,
+    //pub custom_region_via_url: func(u url.URL) string,
+    //pub bucket_lookup_via_url: func(u url.URL, bucketName string) BucketLookupType,
+    pub trailing_headers: bool,
+    pub custom_md5:    Option<MD5>,
+    pub custom_sha256: Option<Sha256>,
+    pub max_retries: i64,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+enum BucketLookupType {
+    #[default]
+    BucketLookupAuto,
+    BucketLookupDNS,
+    BucketLookupPath,
+}
+
+impl TransitionClient {
+    pub async fn new(endpoint: &str, opts: Options) -> Result<TransitionClient, std::io::Error> {
+        let clnt = Self::private_new(endpoint, opts).await?;
+
+        Ok(clnt)
+    }
+
+    async fn private_new(endpoint: &str, opts: Options) -> Result<TransitionClient, std::io::Error> {
+        let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
+
+        //let jar = cookiejar.New(cookiejar.Options{PublicSuffixList: publicsuffix.List})?;
+
+        //#[cfg(feature = "ring")]
+        //let _ = rustls::crypto::ring::default_provider().install_default();
+        //#[cfg(feature = "aws-lc-rs")]
+        let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+
+        let scheme = endpoint_url.scheme();
+        let client;
+        //if scheme == "https" {
+        //    client = Client::builder(TokioExecutor::new()).build_http();
+        //} else {
+            let tls = rustls::ClientConfig::builder()
+                .with_native_roots()?
+                .with_no_client_auth();
+            let https = hyper_rustls::HttpsConnectorBuilder::new()
+                .with_tls_config(tls)
+                .https_or_http()
+                .enable_http1()
+                .build();
+            client = Client::builder(TokioExecutor::new()).build(https);
+        //}
+
+        let mut clnt = TransitionClient {
+            endpoint_url,
+            creds_provider: Arc::new(Mutex::new(opts.creds)),
+            override_signer_type: SignatureType::SignatureDefault,
+            secure: opts.secure,
+            http_client: client,
+            bucket_loc_cache: Arc::new(Mutex::new(BucketLocationCache::new())),
+            is_trace_enabled: Arc::new(Mutex::new(false)),
+            trace_errors_only: Arc::new(Mutex::new(false)),
+            s3_accelerate_endpoint: Arc::new(Mutex::new("".to_string())),
+            s3_dual_stack_enabled: Arc::new(Mutex::new(false)),
+            region: opts.region,
+            random: rand::rng().random_range(10..=50),
+            lookup: opts.bucket_lookup,
+            md5_hasher: Arc::new(Mutex::new(opts.custom_md5)),
+            sha256_hasher: opts.custom_sha256,
+            health_status: AtomicI32::new(C_UNKNOWN),
+            trailing_header_support: opts.trailing_headers,
+            max_retries: opts.max_retries,
+        };
+        
+        {
+            let mut md5_hasher = clnt.md5_hasher.lock().unwrap();
+            if md5_hasher.is_none() {
+                *md5_hasher = Some(MD5::new());
+            }
+        }
+        if clnt.sha256_hasher.is_none() {
+            clnt.sha256_hasher = Some(Sha256::new());
+        }
+
+        clnt.trailing_header_support = opts.trailing_headers && clnt.override_signer_type == SignatureType::SignatureV4;
+
+        if opts.max_retries > 0 {
+            clnt.max_retries = opts.max_retries;
+        }
+
+        Ok(clnt)
+    }
+
+    fn endpoint_url(&self) -> Url {
+        self.endpoint_url.clone()
+    }
+
+    fn set_appinfo(&self, app_name: &str, app_version: &str) {
+        /*if app_name != "" && app_version != "" {
+            self.appInfo.app_name = app_name
+            self.appInfo.app_version = app_version
+        }*/
+    }
+
+    fn trace_errors_only_off(&self) {
+        let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
+        *trace_errors_only = false;
+    }
+
+    fn trace_off(&self) {
+        let mut is_trace_enabled = self.is_trace_enabled.lock().unwrap();
+        *is_trace_enabled = false;
+        let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
+        *trace_errors_only = false;
+    }
+
+    fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
+        todo!();
+    }
+
+    fn set_s3_enable_dual_stack(&self, enabled: bool) {
+        todo!();
+    }
+
+    pub fn hash_materials(&self, is_md5_requested: bool, is_sha256_requested: bool) -> (HashMap<String, MD5>, HashMap<String, Vec<u8>>) {
+        todo!();
+    }
+
+    fn is_online(&self) -> bool {
+        !self.is_offline()
+    }
+
+    fn mark_offline(&self) {
+        self.health_status.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst);
+    }
+
+    fn is_offline(&self) -> bool {
+        self.health_status.load(Ordering::SeqCst) == C_OFFLINE
+    }
+
+    fn health_check(hc_duration: Duration) {
+        todo!();
+    }
+
+    fn dump_http(&self, req: &http::Request<Body>, resp: &http::Response<Body>) -> Result<(), std::io::Error> {
+        let mut resp_trace: Vec<u8>;
+
+        //info!("{}{}", self.trace_output, "---------END-HTTP---------");
+
+        Ok(())
+    }
+
+    pub async fn doit(&self, req: http::Request<Body>) -> Result<http::Response<Body>, std::io::Error> {
+        let req_method;
+        let req_uri;
+        let req_headers;
+        let resp;
+        let http_client = self.http_client.clone();
+        {
+            //let mut http_client = http_client.lock().unwrap();
+            req_method = req.method().clone();
+            req_uri = req.uri().clone();
+            req_headers = req.headers().clone();
+
+            debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string());
+            resp = http_client.request(req);
+        }
+        let resp = resp.await/*.map_err(Into::into)*/.map(|res| res.map(Body::from));
+        debug!("http_client url: {} {}", req_method, req_uri);
+        debug!("http_client headers: {:?}", req_headers);
+        if let Err(err) = resp {
+            error!("http_client call error: {:?}", err);
+            return Err(std::io::Error::other(err));
+        }
+
+        let mut resp = resp.unwrap();
+        debug!("http_resp: {:?}", resp);
+
+        //if self.is_trace_enabled && !(self.trace_errors_only && resp.status() == StatusCode::OK) {
+        if resp.status() != StatusCode::OK {
+            //self.dump_http(&cloned_req, &resp)?;
+            let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
+            debug!("err_body: {}", String::from_utf8(b).unwrap());
+        }
+
+        Ok(resp)
+    }
+
+    pub async fn execute_method(&self, method: http::Method, metadata: &mut RequestMetadata) -> Result<http::Response<Body>, std::io::Error> {
+        if self.is_offline() {
+            let mut s = self.endpoint_url.to_string();
+            s.push_str(" is offline.");
+            return Err(std::io::Error::other(s));
+        }
+
+        let mut retryable: bool;
+        //let mut body_seeker: BufferReader;
+        let mut req_retry = self.max_retries;
+        let mut resp: http::Response<Body>;
+
+        //if metadata.content_body != nil {
+            //body_seeker = BufferReader::new(metadata.content_body.read_all().await?);
+            retryable = true;
+            if !retryable {
+                req_retry = 1;
+            }
+        //}
+
+        //let mut retry_timer = RetryTimer::new();
+        //while let Some(v) = retry_timer.next().await {
+        for _ in [1;1]/*new_retry_timer(req_retry, DefaultRetryUnit, DefaultRetryCap, MaxJitter)*/ {
+            let req = self.new_request(method, metadata).await?;
+
+            resp = self.doit(req).await?;
+
+            for http_status in SUCCESS_STATUS {
+                if http_status == resp.status() {
+                    return Ok(resp);
+                }
+            }
+
+            let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
+            let err_response = http_resp_to_error_response(resp, b.clone(), &metadata.bucket_name, &metadata.object_name);
+
+            if self.region == "" {
+                match err_response.code {
+                    S3ErrorCode::AuthorizationHeaderMalformed | S3ErrorCode::InvalidArgument /*S3ErrorCode::InvalidRegion*/ => {
+                        //break;
+                        return Err(std::io::Error::other(err_response));
+                    }
+                    S3ErrorCode::AccessDenied => {
+                        if err_response.region == "" {
+                            return Err(std::io::Error::other(err_response));
+                        }
+                        if metadata.bucket_name != "" {
+                            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
+                            let location = bucket_loc_cache.get(&metadata.bucket_name);
+                            if location.is_some() && location.unwrap() != err_response.region {
+                                bucket_loc_cache.set(&metadata.bucket_name, &err_response.region);
+                                //continue;
+                            }
+                        } else {
+                            if err_response.region != metadata.bucket_location {
+                                metadata.bucket_location = err_response.region.clone();
+                                //continue;
+                            }
+                        }
+                        return Err(std::io::Error::other(err_response));
+                    }
+                    _ => {
+                        return Err(std::io::Error::other(err_response));
+                    }
+                }
+            }
+
+            break;
+        }
+
+        Err(std::io::Error::other("resp err"))
+    }
+
+    async fn new_request(&self, method: http::Method, metadata: &mut RequestMetadata) -> Result<http::Request<Body>, std::io::Error> {
+        let location = metadata.bucket_location.clone();
+        if location == "" {
+            if metadata.bucket_name != "" {
+                let location = self.get_bucket_location(&metadata.bucket_name).await?;
+            }
+        }
+
+        let is_makebucket = metadata.object_name == "" && method == http::Method::PUT && metadata.query_values.len() == 0;
+        let is_virtual_host = self.is_virtual_host_style_request(&self.endpoint_url, &metadata.bucket_name) && !is_makebucket;
+
+        let target_url = self.make_target_url(&metadata.bucket_name, &metadata.object_name, &location,
+            is_virtual_host, &metadata.query_values)?;
+
+        let mut req_builder = Request::builder().method(method).uri(target_url.to_string());
+
+        let value;
+        {
+            let mut creds_provider = self.creds_provider.lock().unwrap();
+            value = creds_provider.get_with_context(Some(self.cred_context()))?;
+        }
+
+        let mut signer_type   = value.signer_type.clone();
+        let access_key_id     = value.access_key_id;
+        let secret_access_key = value.secret_access_key;
+        let session_token     = value.session_token;
+
+        if self.override_signer_type != SignatureType::SignatureDefault {
+            signer_type = self.override_signer_type.clone();
+        }
+
+        if value.signer_type == SignatureType::SignatureAnonymous {
+            signer_type = SignatureType::SignatureAnonymous;
+        }
+
+        if metadata.expires != 0 && metadata.pre_sign_url {
+            if signer_type == SignatureType::SignatureAnonymous {
+                return Err(std::io::Error::other(err_invalid_argument("Presigned URLs cannot be generated with anonymous credentials.")));
+            }
+            if metadata.extra_pre_sign_header.is_some() {
+                if signer_type == SignatureType::SignatureV2 {
+                    return Err(std::io::Error::other(err_invalid_argument("Extra signed headers for Presign with Signature V2 is not supported.")));
+                }
+                for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() {
+                    req_builder = req_builder.header(k, v);
+                }
+            }
+            if signer_type == SignatureType::SignatureV2 {
+                req_builder = signer::pre_sign_v2(req_builder, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host);
+            } else if signer_type == SignatureType::SignatureV4 {
+                req_builder = signer::pre_sign_v4(req_builder, &access_key_id, &secret_access_key, &session_token, &location, metadata.expires, OffsetDateTime::now_utc());
+            }
+            let req = match req_builder.body(Body::empty()) {
+                Ok(req) => req,
+                Err(err) => { return Err(std::io::Error::other(err)); }
+            };
+            return Ok(req);
+        }
+
+        self.set_user_agent(&mut req_builder);
+
+        for (k, v) in metadata.custom_header.clone() {
+            req_builder.headers_mut().expect("err").insert(k.expect("err"), v);
+        }
+
+        //req.content_length = metadata.content_length;
+        if metadata.content_length <= -1 {
+            let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err");
+            req_builder.headers_mut().expect("err").insert(http::header::TRANSFER_ENCODING, chunked_value);
+        }
+
+        if metadata.content_md5_base64.len() > 0 {
+            let md5_value = HeaderValue::from_str(&metadata.content_md5_base64).expect("err");
+            req_builder.headers_mut().expect("err").insert("Content-Md5", md5_value);
+        }
+
+        if signer_type == SignatureType::SignatureAnonymous {
+            let req = match req_builder.body(Body::empty()) {
+                Ok(req) => req,
+                Err(err) => { return Err(std::io::Error::other(err)); }
+            };
+            return Ok(req);
+        }
+
+        if signer_type == SignatureType::SignatureV2 {
+            req_builder = signer::sign_v2(req_builder, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host);
+        }
+        else if metadata.stream_sha256 && !self.secure {
+            if metadata.trailer.len() > 0 {
+                //req.Trailer = metadata.trailer;
+                for (_, v) in &metadata.trailer {
+                    req_builder = req_builder.header(http::header::TRAILER, v.clone());
+                }
+            }
+            //req_builder = signer::streaming_sign_v4(req_builder, &access_key_id,
+            //  &secret_access_key, &session_token, &location, metadata.content_length, OffsetDateTime::now_utc(), self.sha256_hasher());
+        }
+        else {
+            let mut sha_header = UNSIGNED_PAYLOAD.to_string();
+            if metadata.content_sha256_hex != "" {
+                sha_header = metadata.content_sha256_hex.clone();
+                if metadata.trailer.len() > 0 {
+                    return Err(std::io::Error::other("internal error: content_sha256_hex with trailer not supported"));
+                }
+            } else if metadata.trailer.len() > 0 {
+                sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string();
+            }
+            req_builder = req_builder.header::<HeaderName, HeaderValue>("X-Amz-Content-Sha256".parse().unwrap(), sha_header.parse().expect("err"));
+
+            req_builder = signer::sign_v4_trailer(req_builder, &access_key_id, &secret_access_key, &session_token, &location, metadata.trailer.clone());
+        }
+
+        let req;
+        if metadata.content_length == 0 {
+            req = req_builder.body(Body::empty());
+        } else {
+            match &mut metadata.content_body {
+                ReaderImpl::Body(content_body) => {
+                    req = req_builder.body(Body::from(content_body.clone()));
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    req = req_builder.body(Body::from(content_body.read_all().await?));
+                }
+            }
+            //req = req_builder.body(s3s::Body::from(metadata.content_body.read_all().await?));
+        }
+
+        match req { 
+            Ok(req) => Ok(req),
+            Err(err) => {
+                Err(std::io::Error::other(err))   
+            }
+        }
+    }
+
+    pub fn set_user_agent(&self, req: &mut Builder) {
+        let mut headers = req.headers_mut().expect("err");
+        headers.insert("User-Agent", C_USER_AGENT.parse().expect("err"));
+        /*if self.app_info.app_name != "" && self.app_info.app_version != "" {
+            headers.insert("User-Agent", C_USER_AGENT+" "+self.app_info.app_name+"/"+self.app_info.app_version);
+        }*/
+    }
+
+    fn make_target_url(&self, bucket_name: &str, object_name: &str, bucket_location: &str, is_virtual_host_style: bool, query_values: &HashMap<String, String>) -> Result<Url, std::io::Error> {
+        let scheme = self.endpoint_url.scheme();
+        let host = self.endpoint_url.host().unwrap();
+        let default_port = if scheme == "https" {
+            443
+        } else {
+            80
+        };
+        let port = self.endpoint_url.port().unwrap_or(default_port);
+
+        let mut url_str = format!("{scheme}://{host}:{port}/");
+
+        if bucket_name != "" {
+            if is_virtual_host_style {
+                url_str = format!("{scheme}://{bucket_name}.{host}:{port}/");
+                if object_name != "" {
+                    url_str.push_str(object_name);
+                }
+            } else {
+                url_str.push_str(bucket_name);
+                url_str.push_str("/");
+                if object_name != "" {
+                    url_str.push_str(object_name);
+                }
+            }
+        }
+
+        if query_values.len() > 0 {
+            let mut encoded = form_urlencoded::Serializer::new(String::new());
+            for (k, v) in query_values {
+                encoded.append_pair(&k, &v);
+            }
+            url_str.push_str("?");
+            url_str.push_str(&encoded.finish());
+        }
+
+        Url::parse(&url_str).map_err(|e| std::io::Error::other(e.to_string()))
+    }
+
+    pub fn is_virtual_host_style_request(&self, url: &Url, bucket_name: &str) -> bool {
+        if bucket_name == "" {
+            return false;
+        }
+
+        if self.lookup == BucketLookupType::BucketLookupDNS {
+            return true;
+        }
+
+        if self.lookup == BucketLookupType::BucketLookupPath {
+            return false;
+        }
+
+        false
+    }
+
+    pub fn cred_context(&self) -> CredContext {
+        CredContext {
+            //client: http_client,
+            endpoint: self.endpoint_url.to_string(),
+        }
+    }
+}
+
+struct LockedRandSource {
+    src: u64,//rand.Source,
+}
+
+impl LockedRandSource {
+    fn int63(&self) -> i64 {
+        /*let n = self.src.int63();
+        n*/
+        todo!();
+    }
+
+    fn seed(&self, seed: i64) {
+        //self.src.seed(seed);
+        todo!();
+    }
+}
+
+pub struct RequestMetadata {
+    pub pre_sign_url: bool,
+    pub bucket_name: String,
+    pub object_name: String,
+    pub query_values: HashMap<String, String>,
+    pub custom_header: HeaderMap,
+    pub extra_pre_sign_header: Option<HeaderMap>,
+    pub expires: i64,
+    pub bucket_location: String,
+    pub content_body: ReaderImpl,
+    pub content_length: i64,
+    pub content_md5_base64: String,
+    pub content_sha256_hex: String,
+    pub stream_sha256: bool,
+    pub add_crc: ChecksumMode,
+    pub trailer: HeaderMap,
+}
+
+pub struct TransitionCore(pub Arc<TransitionClient>);
+
+impl TransitionCore {
+    pub async fn new(endpoint: &str, opts: Options) -> Result<Self, std::io::Error> {
+        let client = TransitionClient::new(endpoint, opts).await?;
+        Ok(Self(Arc::new(client)))
+    }
+
+    pub fn list_objects(&self, bucket: &str, prefix: &str, marker: &str, delimiter: &str, max_keys: i64) -> Result<ListBucketResult, std::io::Error> {
+        let client = self.0.clone();
+        client.list_objects_query(bucket, prefix, marker, delimiter, max_keys, HeaderMap::new())
+    }
+
+    pub async fn list_objects_v2(&self, bucket_name: &str, object_prefix: &str, start_after: &str, continuation_token: &str, delimiter: &str, max_keys: i64) -> Result<ListBucketV2Result, std::io::Error> {
+        let client = self.0.clone();
+        client.list_objects_v2_query(bucket_name, object_prefix, continuation_token, true, false, delimiter, start_after, max_keys, HeaderMap::new()).await
+    }
+
+    /*pub fn copy_object(&self, source_bucket: &str, source_object: &str, dest_bucket: &str, dest_object: &str, metadata: HashMap<String, String>, src_opts: CopySrcOptions, dst_opts: PutObjectOptions) -> Result<ObjectInfo> {
+        self.0.copy_object_do(source_bucket, source_object, dest_bucket, dest_object, metadata, src_opts, dst_opts)
+    }*/
+
+    pub fn copy_object_part(&self, src_bucket: &str, src_object: &str, dest_bucket: &str, dest_object: &str, upload_id: &str,
+        part_id: i32, start_offset: i32, length: i64, metadata: HashMap<String, String>,
+    ) -> Result<CompletePart, std::io::Error> {
+        //self.0.copy_object_part_do(src_bucket, src_object, dest_bucket, dest_object, upload_id,
+        //    part_id, start_offset, length, metadata)
+        todo!();
+    }
+
+    pub async fn put_object(&self, bucket: &str, object: &str, data: ReaderImpl, size: i64, md5_base64: &str, sha256_hex: &str, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
+        let hook_reader = data;//newHook(data, opts.progress);
+        let client = self.0.clone();
+        client.put_object_do(bucket, object, hook_reader, md5_base64, sha256_hex, size, opts).await
+    }
+
+    pub async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: PutObjectOptions) -> Result<String, std::io::Error> {
+        let client = self.0.clone();
+        let result = client.initiate_multipart_upload(bucket, object, &opts).await?;
+        Ok(result.upload_id)
+    }
+
+    pub fn list_multipart_uploads(&self, bucket: &str, prefix: &str, key_marker: &str, upload_id_marker: &str, delimiter: &str, max_uploads: i64) -> Result<ListMultipartUploadsResult, std::io::Error> {
+        let client = self.0.clone();
+        client.list_multipart_uploads_query(bucket, key_marker, upload_id_marker, prefix, delimiter, max_uploads)
+    }
+
+    pub async fn put_object_part(&self, bucket: &str, object: &str, upload_id: &str, part_id: i64,
+        data: ReaderImpl, size: i64, opts: PutObjectPartOptions
+    ) -> Result<ObjectPart, std::io::Error> {
+        let mut p = UploadPartParams {
+            bucket_name:   bucket.to_string(),
+            object_name:   object.to_string(),
+            upload_id:     upload_id.to_string(),
+            reader:        data,
+            part_number:   part_id,
+            md5_base64:    opts.md5_base64,
+            sha256_hex:    opts.sha256_hex,
+            size:          size,
+            //sse:           opts.sse,
+            stream_sha256: !opts.disable_content_sha256,
+            custom_header: opts.custom_header,
+            trailer:       opts.trailer,
+        };
+        let client = self.0.clone();
+        client.upload_part(&mut p).await
+    }
+
+    pub async fn list_object_parts(&self, bucket: &str, object: &str, upload_id: &str, part_number_marker: i64, max_parts: i64) -> Result<ListObjectPartsResult, std::io::Error> {
+        let client = self.0.clone();
+        client.list_object_parts_query(bucket, object, upload_id, part_number_marker, max_parts).await
+    }
+
+    pub async fn complete_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, parts: &[CompletePart], opts: PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
+        let client = self.0.clone();
+        let res = client.complete_multipart_upload(bucket, object, upload_id, CompleteMultipartUpload {
+            parts: parts.to_vec(),
+        }, &opts).await?;
+        Ok(res)
+    }
+
+    pub async fn abort_multipart_upload(&self, bucket_name: &str, object: &str, upload_id: &str) -> Result<(), std::io::Error> {
+        let client = self.0.clone();
+        client.abort_multipart_upload(bucket_name, object, upload_id).await
+    }
+
+    pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        let client = self.0.clone();
+        client.get_bucket_policy(bucket_name).await
+    }
+
+    pub async fn put_bucket_policy(&self, bucket_name: &str, bucket_policy: &str) -> Result<(), std::io::Error> {
+        let client = self.0.clone();
+        client.put_bucket_policy(bucket_name, bucket_policy).await
+    }
+
+    pub async fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
+        let client = self.0.clone();
+        client.get_object_inner(bucket_name, object_name, opts).await
+    }
+}
+
+pub struct PutObjectPartOptions {
+    pub md5_base64: String,
+    pub sha256_hex: String,
+    //pub sse: encrypt.ServerSide,
+    pub custom_header: HeaderMap,
+    pub trailer: HeaderMap,
+    pub disable_content_sha256: bool,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct ObjectInfo {
+    pub etag:          String,
+    pub name:           String,
+    pub mod_time: OffsetDateTime,
+    pub size: usize,
+    pub content_type: Option<String>,
+    #[serde(skip)]
+    pub metadata: HeaderMap,
+    pub user_metadata: HashMap<String, String>,
+    pub user_tags: String,
+    pub user_tag_count: i64,
+    #[serde(skip)]
+    pub owner: Owner,
+    //pub grant: Vec<Grant>,
+    pub storage_class: String,
+    pub is_latest: bool,
+    pub is_delete_marker: bool,
+    pub version_id: Uuid,
+
+    #[serde(skip, default = "replication_status_default")]
+    pub replication_status: ReplicationStatus,
+    pub replication_ready: bool,
+    pub expiration:    OffsetDateTime,
+    pub expiration_rule_id: String,
+    pub num_versions: usize,
+    
+    pub restore: RestoreInfo,
+
+    pub checksum_crc32:     String,
+    pub checksum_crc32c:    String,
+    pub checksum_sha1:      String,
+    pub checksum_sha256:    String,
+    pub checksum_crc64nvme: String,
+    pub checksum_mode: String,
+}
+
+fn replication_status_default() -> ReplicationStatus {
+    ReplicationStatus::from_static(ReplicationStatus::PENDING)
+}
+
+impl Default for ObjectInfo {
+    fn default() -> Self {
+        Self {
+            etag: "".to_string(),
+            name: "".to_string(),
+            mod_time: OffsetDateTime::now_utc(),
+            size: 0,
+            content_type: None,
+            metadata: HeaderMap::new(),
+            user_metadata: HashMap::new(),
+            user_tags: "".to_string(),
+            user_tag_count: 0,
+            owner: Owner::default(),
+            storage_class: "".to_string(),
+            is_latest: false,
+            is_delete_marker: false,
+            version_id: Uuid::nil(),
+            replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
+            replication_ready: false,
+            expiration:    OffsetDateTime::now_utc(),
+            expiration_rule_id: "".to_string(),
+            num_versions: 0,
+            restore: RestoreInfo::default(),
+            checksum_crc32:     "".to_string(),
+            checksum_crc32c:    "".to_string(),
+            checksum_sha1:      "".to_string(),
+            checksum_sha256:    "".to_string(),
+            checksum_crc64nvme: "".to_string(),
+            checksum_mode: "".to_string(),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[derive(Debug, Clone)]
+pub struct RestoreInfo {
+    ongoing_restore: bool,
+    expiry_time: OffsetDateTime,
+}
+
+impl Default for RestoreInfo {
+    fn default() -> Self {
+        Self {
+            ongoing_restore: false,
+            expiry_time: OffsetDateTime::now_utc(),
+        }
+    }
+}
+
+pub struct ObjectMultipartInfo {
+    pub initiated: OffsetDateTime,
+    //pub initiator: initiator,
+    //pub owner:     owner,
+    pub storage_class: String,
+    pub key: String,
+    pub size: i64,
+    pub upload_id: String,
+    //pub err error,
+}
+
+pub struct UploadInfo {
+    pub bucket:        String,
+    pub key:           String,
+    pub etag:          String,
+    pub size:          i64,
+    pub last_modified: OffsetDateTime,
+    pub location:      String,
+    pub version_id:    String,
+    pub expiration:    OffsetDateTime,
+    pub expiration_rule_id: String,
+    pub checksum_crc32:     String,
+    pub checksum_crc32c:    String,
+    pub checksum_sha1:      String,
+    pub checksum_sha256:    String,
+    pub checksum_crc64nvme: String,
+    pub checksum_mode: String,
+}
+
+impl Default for UploadInfo {
+    fn default() -> Self {
+        Self {
+            bucket: "".to_string(),
+            key: "".to_string(),
+            etag: "".to_string(),
+            size: 0,
+            last_modified: OffsetDateTime::now_utc(),
+            location: "".to_string(),
+            version_id: "".to_string(),
+            expiration: OffsetDateTime::now_utc(),
+            expiration_rule_id: "".to_string(),
+            checksum_crc32: "".to_string(),
+            checksum_crc32c: "".to_string(),
+            checksum_sha1: "".to_string(),
+            checksum_sha256: "".to_string(),
+            checksum_crc64nvme: "".to_string(),
+            checksum_mode: "".to_string(),
+        }
+    }
+}
+
+pub fn to_object_info(bucket_name: &str, object_name: &str, h: &HeaderMap) -> Result<ObjectInfo, std::io::Error> {
+    todo!()
+}
+
+type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
+
+//#[derive(Clone)]
+pub struct SendRequest {
+    inner: hyper::client::conn::http1::SendRequest<Body>,
+}
+
+impl From<hyper::client::conn::http1::SendRequest<Body>> for SendRequest {
+    fn from(inner: hyper::client::conn::http1::SendRequest<Body>) -> Self {
+        Self { inner }
+    }
+}
+
+impl tower::Service<Request<Body>> for SendRequest {
+    type Response = Response<Body>;
+    type Error = std::io::Error;
+    type Future = BoxFuture<'static, Result<Self::Response, Self::Error>>;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        self.inner.poll_ready(cx).map_err(std::io::Error::other)
+    }
+
+    fn call(&mut self, req: Request<Body>) -> Self::Future {
+        //let req = hyper::Request::builder().uri("/").body(http_body_util::Empty::<Bytes>::new()).unwrap();
+        //let req = hyper::Request::builder().uri("/").body(Body::empty()).unwrap();
+
+        let fut = self.inner.send_request(req);
+
+        Box::pin(async move { fut.await.map_err(std::io::Error::other).map(|res| res.map(Body::from)) })
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Document(pub String);