refactor(ecstore): extract the embedded S3 client into rustfs-s3-client (#6627)

The storage engine embedded a ~8.4K-line hand-written S3 HTTP client under crates/ecstore/src/client (rustfs/backlog#1842). That client is a legitimate engine capability — it consumes remote S3-compatible endpoints for ILM tier warm backends and transition targets — but it was misfiled inside the engine, dragging s3s/hyper wire types into ecstore and blocking ARCHITECTURE.md invariant 4.

This PR is the pure-move step: 21 modules move verbatim to the new crates/s3-client crate (rustfs-s3-client), and crates/ecstore/src/client/mod.rs becomes a re-export shim so every in-crate crate::client:: path keeps working. The two server-side modules that were historically misfiled under client/ — object_api_utils.rs and object_handlers_common.rs — stay in ecstore.

Three reverse dependencies from the client into engine internals are severed so the move can be pure:

- transition_api::ReaderImpl::ObjectBody held ecstore's GetObjectReader; the client only ever reads the body, so the variant now holds an ObjectReader newtype over Box<dyn AsyncRead + Send + Sync + Unpin> with the same read_all() surface. The single production construction site (set_disk transition upload) and the two engine-side consumers were adjusted.
- api_list/api_remove used ecstore's storage_api_contracts / object_api types; api_list now imports BucketInfo from rustfs-storage-api directly, and api_remove uses the client's own transition_api::ObjectInfo (only .name/.version_id were read; the error-path bucket name is now threaded as a parameter instead of read from the deleted objects).
- the api_put_object_streaming regression tests built a GetObjectReader by hand; they now wrap the duplex stream in ObjectReader::new.

Guard updates: the s3s footprint ratchet gains an ecstore-scoped counter (42 files, shrink-only, per rustfs/backlog#1842), the ecstore module-lint-blanket register follows the moved files into crates/s3-client so the blanket ratchet keeps covering them, the logging guardrail path pin follows transition_api.rs, and the ::other(format!) baseline is regenerated (moved call sites left ecstore).

Verification: cargo check -p rustfs-s3-client -p rustfs-ecstore; cargo nextest run -p rustfs-s3-client (43 passed) and -p rustfs-ecstore (4515/4523; the 8 failures reproduce identically on pristine origin/main on the same machine); cargo clippy --all-targets; scripts/check_layer_dependencies.sh, check_architecture_migration_rules.sh, check_s3s_footprint.sh, check_logging_guardrails.sh, check_error_other_format_ratchet.sh, check_doc_paths.sh, check_ci_paths_sync.sh all pass.

crates/s3-client/Cargo.toml

@@ -0,0 +1,73 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[package]
+name = "rustfs-s3-client"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+homepage.workspace = true
+description = "S3 client used by RustFS when the storage engine consumes remote S3-compatible endpoints (tier warm backends, transition targets)."
+keywords = ["s3", "client", "tiering", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "network-programming"]
+documentation = "https://docs.rs/rustfs-s3-client/latest/rustfs_s3_client/"
+
+[dependencies]
+rustfs-checksums.workspace = true
+rustfs-config.workspace = true
+rustfs-rio.workspace = true
+rustfs-signer.workspace = true
+rustfs-storage-api.workspace = true
+rustfs-tls-runtime.workspace = true
+rustfs-utils = { workspace = true, features = ["full"] }
+base64-simd.workspace = true
+bytes = { workspace = true, features = ["serde"] }
+hex-simd = { workspace = true }
+enumset = { workspace = true }
+futures.workspace = true
+futures-util.workspace = true
+http.workspace = true
+http-body = { workspace = true }
+http-body-util.workspace = true
+hyper = { workspace = true, features = ["http2", "http1", "server"] }
+hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] }
+hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
+lazy_static.workspace = true
+md-5.workspace = true
+quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
+rand = { workspace = true, features = ["serde"] }
+rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
+rustls-pki-types.workspace = true
+s3s = { workspace = true, features = ["minio"] }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true, features = ["raw_value"] }
+sha1 = { workspace = true }
+sha2 = { workspace = true }
+thiserror.workspace = true
+time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
+tokio = { workspace = true, features = ["io-util", "sync", "fs", "rt-multi-thread"] }
+tokio-util = { workspace = true, features = ["io", "compat"] }
+tower = { workspace = true, features = ["timeout"] }
+tracing.workspace = true
+url.workspace = true
+urlencoding = { workspace = true }
+uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
+
+[lints]
+workspace = true
+
+[lib]
+doctest = false

crates/s3-client/src/admin_handler_utils.rs

@@ -0,0 +1,47 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use http::status::StatusCode;
+use std::fmt::{self, Display, Formatter};
+
+#[derive(Default, thiserror::Error, Debug, Clone, PartialEq)]
+pub struct AdminError {
+    pub code: String,
+    pub message: String,
+    pub status_code: StatusCode,
+}
+
+impl Display for AdminError {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "{}", self.message)
+    }
+}
+
+impl AdminError {
+    pub fn new(code: &str, message: &str, status_code: StatusCode) -> Self {
+        Self {
+            code: code.to_string(),
+            message: message.to_string(),
+            status_code,
+        }
+    }
+
+    pub fn msg(message: &str) -> Self {
+        Self {
+            code: "InternalError".to_string(),
+            message: message.to_string(),
+            status_code: StatusCode::INTERNAL_SERVER_ERROR,
+        }
+    }
+}

crates/s3-client/src/api_error_response.rs

@@ -0,0 +1,328 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use http::{HeaderMap, StatusCode};
+use serde::{Deserialize, Serialize};
+use serde::{de::Deserializer, ser::Serializer};
+use std::fmt::Display;
+
+use s3s::S3ErrorCode;
+
+const _REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues.";
+
+#[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error, PartialEq, Eq)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct ErrorResponse {
+    #[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")]
+    pub code: S3ErrorCode,
+    pub message: String,
+    pub bucket_name: String,
+    pub key: String,
+    pub resource: String,
+    // External S3-style error response contract: keep `RequestId`.
+    #[serde(rename = "RequestId")]
+    pub request_id: String,
+    pub host_id: String,
+    pub region: String,
+    pub server: String,
+    #[serde(skip)]
+    pub status_code: StatusCode,
+}
+
+fn serialize_code<S>(_data: &S3ErrorCode, s: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    s.serialize_str("")
+}
+
+fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
+}
+
+impl Default for ErrorResponse {
+    fn default() -> Self {
+        ErrorResponse {
+            code: S3ErrorCode::Custom("".into()),
+            message: Default::default(),
+            bucket_name: Default::default(),
+            key: Default::default(),
+            resource: Default::default(),
+            request_id: Default::default(),
+            host_id: Default::default(),
+            region: Default::default(),
+            server: Default::default(),
+            status_code: Default::default(),
+        }
+    }
+}
+
+impl Display for ErrorResponse {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.message)
+    }
+}
+
+pub fn to_error_response(err: &std::io::Error) -> ErrorResponse {
+    if let Some(err) = err.get_ref() {
+        if err.is::<ErrorResponse>() {
+            err.downcast_ref::<ErrorResponse>().expect("err!").clone()
+        } else {
+            ErrorResponse::default()
+        }
+    } else {
+        ErrorResponse::default()
+    }
+}
+
+pub fn http_resp_to_error_response(
+    resp_status: StatusCode,
+    h: &HeaderMap,
+    b: Vec<u8>,
+    bucket_name: &str,
+    object_name: &str,
+) -> ErrorResponse {
+    let err_body = String::from_utf8_lossy(&b).to_string();
+    if h.is_empty() || !(resp_status.is_client_error() || resp_status.is_server_error()) {
+        return ErrorResponse {
+            status_code: resp_status,
+            code: S3ErrorCode::ResponseInterrupted,
+            message: "Invalid HTTP response.".to_string(),
+            bucket_name: bucket_name.to_string(),
+            key: object_name.to_string(),
+            ..Default::default()
+        };
+    }
+    let err_resp_ = quick_xml::de::from_str::<ErrorResponse>(&err_body);
+    let mut err_resp = ErrorResponse::default();
+    if err_resp_.is_err() {
+        match resp_status {
+            StatusCode::NOT_FOUND => {
+                if object_name == "" {
+                    err_resp = ErrorResponse {
+                        status_code: resp_status,
+                        code: S3ErrorCode::NoSuchBucket,
+                        message: "The specified bucket does not exist.".to_string(),
+                        bucket_name: bucket_name.to_string(),
+                        ..Default::default()
+                    };
+                } else {
+                    err_resp = ErrorResponse {
+                        status_code: resp_status,
+                        code: S3ErrorCode::NoSuchKey,
+                        message: "The specified key does not exist.".to_string(),
+                        bucket_name: bucket_name.to_string(),
+                        key: object_name.to_string(),
+                        ..Default::default()
+                    };
+                }
+            }
+            StatusCode::FORBIDDEN => {
+                err_resp = ErrorResponse {
+                    status_code: resp_status,
+                    code: S3ErrorCode::AccessDenied,
+                    message: "Access Denied.".to_string(),
+                    bucket_name: bucket_name.to_string(),
+                    key: object_name.to_string(),
+                    ..Default::default()
+                };
+            }
+            StatusCode::CONFLICT => {
+                err_resp = ErrorResponse {
+                    status_code: resp_status,
+                    code: S3ErrorCode::BucketNotEmpty,
+                    message: "Bucket not empty.".to_string(),
+                    bucket_name: bucket_name.to_string(),
+                    ..Default::default()
+                };
+            }
+            StatusCode::PRECONDITION_FAILED => {
+                err_resp = ErrorResponse {
+                    status_code: resp_status,
+                    code: S3ErrorCode::PreconditionFailed,
+                    message: "Pre condition failed.".to_string(),
+                    bucket_name: bucket_name.to_string(),
+                    key: object_name.to_string(),
+                    ..Default::default()
+                };
+            }
+            _ => {
+                let mut msg = resp_status.to_string();
+                if err_body.len() > 0 {
+                    msg = err_body;
+                }
+                err_resp = ErrorResponse {
+                    status_code: resp_status,
+                    code: S3ErrorCode::Custom(resp_status.to_string().into()),
+                    message: msg,
+                    bucket_name: bucket_name.to_string(),
+                    ..Default::default()
+                };
+            }
+        }
+    } else if let Ok(parsed_resp) = err_resp_ {
+        err_resp = parsed_resp;
+    }
+    err_resp.status_code = resp_status;
+    if let Some(server_name) = h.get("Server") {
+        if let Ok(server_str) = server_name.to_str() {
+            err_resp.server = server_str.to_string();
+        }
+    }
+
+    if let Some(code) = h.get("x-minio-error-code") {
+        if let Ok(code_str) = code.to_str() {
+            err_resp.code = S3ErrorCode::Custom(code_str.into());
+        }
+    }
+    if let Some(desc) = h.get("x-minio-error-desc") {
+        if let Ok(desc_str) = desc.to_str() {
+            err_resp.message = desc_str.trim_matches('"').to_string();
+        }
+    }
+
+    if err_resp.request_id == "" {
+        if let Some(x_amz_request_id) = h.get("x-amz-request-id") {
+            if let Ok(request_id_str) = x_amz_request_id.to_str() {
+                err_resp.request_id = request_id_str.to_string();
+            }
+        }
+    }
+    if err_resp.host_id == "" {
+        if let Some(x_amz_id_2) = h.get("x-amz-id-2") {
+            if let Ok(host_id_str) = x_amz_id_2.to_str() {
+                err_resp.host_id = host_id_str.to_string();
+            }
+        }
+    }
+    if err_resp.region == "" {
+        if let Some(x_amz_bucket_region) = h.get("x-amz-bucket-region") {
+            if let Ok(region_str) = x_amz_bucket_region.to_str() {
+                err_resp.region = region_str.to_string();
+            }
+        }
+    }
+    if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" {
+        err_resp.message = format!("Region does not match, expecting region ‘{}’.", err_resp.region);
+    }
+
+    err_resp
+}
+
+pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
+    let msg = format!(
+        "Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
+        total_size, max_object_size
+    );
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::EntityTooLarge,
+        message: msg,
+        bucket_name: bucket_name.to_string(),
+        key: object_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
+    let msg = format!(
+        "Your proposed upload size ‘{}’ is below the minimum allowed object size ‘0B’ for single PUT operation.",
+        total_size
+    );
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::EntityTooSmall,
+        message: msg,
+        bucket_name: bucket_name.to_string(),
+        key: object_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
+    let msg = format!(
+        "Data read ‘{}’ is not equal to the size ‘{}’ of the input Reader.",
+        total_read, total_size
+    );
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::Custom("UnexpectedEOF".into()),
+        message: msg,
+        bucket_name: bucket_name.to_string(),
+        key: object_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_invalid_argument(message: &str) -> ErrorResponse {
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::InvalidArgument,
+        message: message.to_string(),
+        request_id: "rustfs".to_string(),
+        ..Default::default()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::Value;
+
+    #[test]
+    fn error_response_serializes_request_id_as_pascal_case_contract() {
+        let response = ErrorResponse {
+            code: S3ErrorCode::InvalidArgument,
+            message: "bad request".to_string(),
+            bucket_name: "bucket".to_string(),
+            key: "key".to_string(),
+            resource: "/bucket/key".to_string(),
+            request_id: "req-xml-123".to_string(),
+            host_id: "host-1".to_string(),
+            region: "us-east-1".to_string(),
+            server: "rustfs".to_string(),
+            status_code: StatusCode::BAD_REQUEST,
+        };
+
+        let value = serde_json::to_value(response).expect("error response should serialize");
+        assert_eq!(value["RequestId"], Value::String("req-xml-123".to_string()));
+        assert!(value.get("request_id").is_none(), "external error contract must not expose request_id");
+    }
+
+    #[test]
+    fn parses_s3_error_code_from_client_error_response() {
+        let mut headers = HeaderMap::new();
+        headers.insert("x-amz-request-id", "request-id".parse().expect("request ID header should parse"));
+
+        let response = http_resp_to_error_response(
+            StatusCode::NOT_FOUND,
+            &headers,
+            b"<Error><Code>NoSuchVersion</Code><Message>remote detail</Message></Error>".to_vec(),
+            "bucket",
+            "object",
+        );
+
+        assert_eq!(response.code, S3ErrorCode::NoSuchVersion);
+        assert_eq!(response.status_code, StatusCode::NOT_FOUND);
+    }
+}

crates/s3-client/src/api_get_object.rs

@@ -0,0 +1,288 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(clippy::map_entry)]
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use futures_util::ready;
+use http::HeaderMap;
+use std::io::{Cursor, Error as IoError, ErrorKind as IoErrorKind, Read};
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::io::BufReader;
+use tokio_util::io::StreamReader;
+
+use crate::{
+    api_error_response::err_invalid_argument,
+    api_get_options::GetObjectOptions,
+    transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info_for_provider},
+};
+use futures_util::StreamExt;
+use http_body_util::BodyExt;
+use hyper::body::Body;
+use hyper::body::Bytes;
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+use tokio_util::io::ReaderStream;
+
+impl TransitionClient {
+    pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
+        let _ = opts;
+        Err(std::io::Error::new(
+            IoErrorKind::Unsupported,
+            format!("get_object is not implemented for {bucket_name}/{object_name}"),
+        ))
+    }
+
+    pub async fn get_object_inner(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        opts: &GetObjectOptions,
+    ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
+        let resp = self
+            .execute_method(
+                http::Method::GET,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: object_name.to_string(),
+                    query_values: opts.to_query_values(),
+                    custom_header: opts.header(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        let object_stat =
+            to_object_info_for_provider(bucket_name, object_name, resp.headers(), self.provider_version_capabilities())?;
+
+        let h = resp.headers().clone();
+
+        let mut body_vec = Vec::new();
+        let mut body = resp.into_body();
+        while let Some(frame) = body.frame().await {
+            let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+            if let Some(data) = frame.data_ref() {
+                body_vec.extend_from_slice(data);
+            }
+        }
+        Ok((object_stat, h, BufReader::new(Cursor::new(body_vec))))
+    }
+}
+
+#[derive(Default)]
+pub struct GetRequest {
+    pub buffer: Vec<u8>,
+    pub offset: i64,
+    pub did_offset_change: bool,
+    pub been_read: bool,
+    pub is_read_at: bool,
+    pub is_read_op: bool,
+    pub is_first_req: bool,
+    pub setting_object_info: bool,
+}
+
+pub struct GetResponse {
+    pub size: i64,
+    //pub error:       error,
+    #[allow(dead_code, reason = "written but never read back (backlog#1823)")]
+    pub did_read: bool,
+    #[allow(dead_code, reason = "written but never read back (backlog#1823)")]
+    pub object_info: ObjectInfo,
+}
+
+#[derive(Default)]
+pub struct Object {
+    //pub reqch:      chan<- getRequest,
+    //pub resch:      <-chan getResponse,
+    //pub cancel:     context.CancelFunc,
+    pub curr_offset: i64,
+    pub object_info: ObjectInfo,
+    pub seek_data: bool,
+    pub is_closed: bool,
+    pub is_started: bool,
+    //pub prev_err: error,
+    pub been_read: bool,
+    pub object_info_set: bool,
+}
+
+impl Object {
+    pub fn new() -> Object {
+        Self { ..Default::default() }
+    }
+
+    #[allow(
+        dead_code,
+        reason = "MinIO-parity reader surface with no caller in this port (backlog#1823)"
+    )]
+    fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
+        let _ = request.did_offset_change;
+        let _ = request.offset;
+        let _ = request.is_first_req;
+        let _ = request.is_read_at;
+        let _ = request.setting_object_info;
+        let _ = request.is_read_op;
+        let _ = request.been_read;
+        let _ = request.buffer.len();
+        Err(std::io::Error::new(
+            IoErrorKind::Unsupported,
+            "read-path for Object in api_get_object is not implemented",
+        ))
+    }
+
+    #[allow(
+        dead_code,
+        reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
+    )]
+    fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
+        self.curr_offset += bytes_read;
+
+        Ok(())
+    }
+
+    #[allow(
+        dead_code,
+        reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
+    )]
+    fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
+        let mut read_req = GetRequest {
+            is_read_op: true,
+            been_read: self.been_read,
+            buffer: b.to_vec(),
+            ..Default::default()
+        };
+
+        if !self.is_started {
+            read_req.is_first_req = true;
+        }
+
+        read_req.did_offset_change = self.seek_data;
+        read_req.offset = self.curr_offset;
+
+        let response = self.do_get_request(&read_req)?;
+
+        let bytes_read = response.size;
+
+        let oerr = self.set_offset(bytes_read);
+
+        Ok(response.size)
+    }
+
+    #[allow(
+        dead_code,
+        reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
+    )]
+    fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
+        if !self.is_started || !self.object_info_set {
+            let _ = self.do_get_request(&GetRequest {
+                is_first_req: !self.is_started,
+                setting_object_info: !self.object_info_set,
+                ..Default::default()
+            })?;
+        }
+
+        Ok(self.object_info.clone())
+    }
+
+    #[allow(
+        dead_code,
+        reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
+    )]
+    fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
+        self.curr_offset = offset;
+
+        let mut read_at_req = GetRequest {
+            is_read_op: true,
+            is_read_at: true,
+            did_offset_change: true,
+            been_read: self.been_read,
+            offset,
+            buffer: b.to_vec(),
+            ..Default::default()
+        };
+
+        if !self.is_started {
+            read_at_req.is_first_req = true;
+        }
+
+        let response = self.do_get_request(&read_at_req)?;
+        let bytes_read = response.size;
+        if !self.object_info_set {
+            self.curr_offset += bytes_read;
+        } else {
+            let oerr = self.set_offset(bytes_read);
+        }
+        Ok(response.size)
+    }
+
+    #[allow(
+        dead_code,
+        reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
+    )]
+    fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
+        if !self.is_started || !self.object_info_set {
+            let seek_req = GetRequest {
+                is_read_op: false,
+                offset,
+                is_first_req: true,
+                ..Default::default()
+            };
+            let _ = self.do_get_request(&seek_req);
+        }
+
+        let mut new_offset = self.curr_offset;
+
+        match whence {
+            0 => {
+                new_offset = offset;
+            }
+            1 => {
+                new_offset += offset;
+            }
+            2 => {
+                new_offset = self.object_info.size as i64 + offset as i64;
+            }
+            _ => {
+                return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid whence {}", whence))));
+            }
+        }
+
+        self.seek_data = (new_offset != self.curr_offset) || self.seek_data;
+        self.curr_offset = new_offset;
+
+        Ok(self.curr_offset)
+    }
+
+    #[allow(
+        dead_code,
+        reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
+    )]
+    fn close(&mut self) -> Result<(), std::io::Error> {
+        self.is_closed = true;
+        Ok(())
+    }
+}

crates/s3-client/src/api_get_options.rs

@@ -0,0 +1,197 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use http::{HeaderMap, HeaderName, HeaderValue};
+use rustfs_utils::http::headers::AMZ_CHECKSUM_MODE;
+use std::collections::HashMap;
+use time::OffsetDateTime;
+use tracing::warn;
+
+use crate::api_error_response::err_invalid_argument;
+
+#[derive(Default)]
+pub struct AdvancedGetOptions {
+    pub replication_delete_marker: bool,
+    pub is_replication_ready_for_delete_marker: bool,
+    pub replication_proxy_request: String,
+}
+
+pub struct GetObjectOptions {
+    pub headers: HashMap<String, String>,
+    pub req_params: HashMap<String, String>,
+    //pub server_side_encryption: encrypt.ServerSide,
+    pub version_id: String,
+    pub part_number: i64,
+    pub checksum: bool,
+    pub internal: AdvancedGetOptions,
+}
+
+pub type StatObjectOptions = GetObjectOptions;
+
+impl Default for GetObjectOptions {
+    fn default() -> Self {
+        Self {
+            headers: HashMap::new(),
+            req_params: HashMap::new(),
+            //server_side_encryption: encrypt.ServerSide::default(),
+            version_id: "".to_string(),
+            part_number: 0,
+            checksum: false,
+            internal: AdvancedGetOptions::default(),
+        }
+    }
+}
+
+impl GetObjectOptions {
+    pub fn header(&self) -> HeaderMap {
+        let mut headers: HeaderMap = HeaderMap::with_capacity(self.headers.len());
+        for (k, v) in &self.headers {
+            match (HeaderName::from_bytes(k.as_bytes()), HeaderValue::from_str(v)) {
+                (Ok(header_name), Ok(header_value)) => {
+                    headers.insert(header_name, header_value);
+                }
+                (Err(_), _) => {
+                    warn!("Invalid header name: {}", k);
+                }
+                (_, Err(_)) => {
+                    warn!("Invalid header value for {}: {:?}", k, v);
+                }
+            }
+        }
+        if self.checksum {
+            headers.insert(HeaderName::from_static(AMZ_CHECKSUM_MODE), HeaderValue::from_static("ENABLED"));
+        }
+        headers
+    }
+
+    pub fn set(&mut self, key: &str, value: &str) -> Result<(), std::io::Error> {
+        let header_name = HeaderName::from_bytes(key.as_bytes())
+            .map_err(|err| std::io::Error::other(err_invalid_argument(&format!("Invalid header name {key}: {err}"))))?;
+        HeaderValue::from_str(value)
+            .map_err(|err| std::io::Error::other(err_invalid_argument(&format!("Invalid header value for {key}: {err}"))))?;
+
+        self.headers.insert(header_name.as_str().to_string(), value.to_string());
+        Ok(())
+    }
+
+    pub fn set_req_param(&mut self, key: &str, value: &str) {
+        self.req_params.insert(key.to_string(), value.to_string());
+    }
+
+    pub fn add_req_param(&mut self, key: &str, value: &str) {
+        self.req_params.insert(key.to_string(), value.to_string());
+    }
+
+    pub fn set_match_etag(&mut self, etag: &str) -> Result<(), std::io::Error> {
+        self.set("If-Match", &format!("\"{etag}\""))?;
+        Ok(())
+    }
+
+    pub fn set_match_etag_except(&mut self, etag: &str) -> Result<(), std::io::Error> {
+        self.set("If-None-Match", &format!("\"{etag}\""))?;
+        Ok(())
+    }
+
+    pub fn set_unmodified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> {
+        if mod_time.unix_timestamp() == 0 {
+            return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
+        }
+        self.set("If-Unmodified-Since", &mod_time.to_string())?;
+        Ok(())
+    }
+
+    pub fn set_modified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> {
+        if mod_time.unix_timestamp() == 0 {
+            return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
+        }
+        self.set("If-Modified-Since", &mod_time.to_string())?;
+        Ok(())
+    }
+
+    pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> {
+        if start == 0 && end < 0 {
+            self.set("Range", &format!("bytes={}", end))?;
+        } else if 0 < start && end == 0 {
+            self.set("Range", &format!("bytes={}-", start))?;
+        } else if 0 <= start && start <= end {
+            self.set("Range", &format!("bytes={}-{}", start, end))?;
+        } else {
+            return Err(std::io::Error::other(err_invalid_argument(&format!(
+                "Invalid range specified: start={} end={}",
+                start, end
+            ))));
+        }
+        Ok(())
+    }
+
+    pub fn to_query_values(&self) -> HashMap<String, String> {
+        let mut url_values = HashMap::new();
+        if self.version_id != "" {
+            url_values.insert("versionId".to_string(), self.version_id.clone());
+        }
+        if self.part_number > 0 {
+            url_values.insert("partNumber".to_string(), self.part_number.to_string());
+        }
+
+        for (key, value) in self.req_params.iter() {
+            url_values.insert(key.to_string(), value.to_string());
+        }
+
+        url_values
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::GetObjectOptions;
+
+    #[test]
+    fn set_range_populates_range_header() {
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(5, 9).expect("valid range should succeed");
+
+        let headers = opts.header();
+        let range = headers.get("range").expect("range header should be present");
+        assert_eq!(range.to_str().expect("range header must be valid ascii"), "bytes=5-9");
+    }
+
+    #[test]
+    fn set_rejects_invalid_header_value() {
+        let mut opts = GetObjectOptions::default();
+
+        let err = opts
+            .set("Range", "bytes=5-\n9")
+            .expect_err("invalid header value should fail");
+
+        assert!(err.to_string().contains("Invalid header value"));
+        assert!(opts.headers.is_empty(), "invalid headers must not be stored");
+    }
+
+    #[test]
+    fn header_skips_invalid_prepopulated_header_value() {
+        let mut opts = GetObjectOptions::default();
+        opts.headers.insert("Range".to_string(), "bytes=5-\n9".to_string());
+
+        let headers = opts.header();
+
+        assert!(headers.get("range").is_none(), "invalid stored header values should be ignored");
+    }
+}

crates/s3-client/src/api_list.rs

@@ -0,0 +1,476 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use crate::{
+    api_error_response::http_resp_to_error_response,
+    api_s3_datatypes::{
+        ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart,
+    },
+    credentials,
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient, collect_response_body},
+};
+use http::{HeaderMap, StatusCode};
+use http_body_util::BodyExt;
+use hyper::body::Body;
+use hyper::body::Bytes;
+use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
+use rustfs_storage_api::BucketInfo;
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+use std::collections::HashMap;
+use std::io::ErrorKind;
+
+impl TransitionClient {
+    pub fn list_buckets(&self) -> Result<Vec<BucketInfo>, std::io::Error> {
+        Err(std::io::Error::new(
+            ErrorKind::Unsupported,
+            credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: "The list_buckets API is not implemented in this build.".to_string(),
+                },
+                request_id: "".to_string(),
+            },
+        ))
+    }
+
+    pub async fn list_objects_v2_query(
+        &self,
+        bucket_name: &str,
+        object_prefix: &str,
+        continuation_token: &str,
+        fetch_owner: bool,
+        metadata: bool,
+        delimiter: &str,
+        start_after: &str,
+        max_keys: i64,
+        headers: HeaderMap,
+    ) -> Result<ListBucketV2Result, std::io::Error> {
+        let mut url_values = HashMap::new();
+
+        url_values.insert("list-type".to_string(), "2".to_string());
+        if metadata {
+            url_values.insert("metadata".to_string(), "true".to_string());
+        }
+        if start_after != "" {
+            url_values.insert("start-after".to_string(), start_after.to_string());
+        }
+        url_values.insert("encoding-type".to_string(), "url".to_string());
+        url_values.insert("prefix".to_string(), object_prefix.to_string());
+        url_values.insert("delimiter".to_string(), delimiter.to_string());
+
+        if continuation_token != "" {
+            url_values.insert("continuation-token".to_string(), continuation_token.to_string());
+        }
+
+        if fetch_owner {
+            url_values.insert("fetch-owner".to_string(), "true".to_string());
+        }
+
+        if max_keys > 0 {
+            url_values.insert("max-keys".to_string(), max_keys.to_string());
+        }
+
+        let resp = self
+            .execute_method(
+                http::Method::GET,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: "".to_string(),
+                    query_values: url_values,
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: headers,
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                resp_status,
+                &h,
+                vec![],
+                bucket_name,
+                "",
+            )));
+        }
+
+        //let mut list_bucket_result = ListBucketV2Result::default();
+        let mut body_vec = Vec::new();
+        let mut body = resp.into_body();
+        while let Some(frame) = body.frame().await {
+            let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+            if let Some(data) = frame.data_ref() {
+                body_vec.extend_from_slice(data);
+            }
+        }
+        let mut list_bucket_result = match quick_xml::de::from_str::<ListBucketV2Result>(&String::from_utf8_lossy(&body_vec)) {
+            Ok(result) => result,
+            Err(err) => {
+                return Err(std::io::Error::other(err.to_string()));
+            }
+        };
+        //println!("list_bucket_result: {:?}", list_bucket_result);
+
+        if list_bucket_result.is_truncated && list_bucket_result.next_continuation_token == "" {
+            return Err(std::io::Error::other(credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: "Truncated response should have continuation token set".to_string(),
+                },
+                request_id: "".to_string(),
+            }));
+        }
+
+        for (i, obj) in list_bucket_result.contents.iter_mut().enumerate() {
+            obj.name = decode_s3_name(&obj.name, &list_bucket_result.encoding_type)?;
+            //list_bucket_result.contents[i].mod_time = list_bucket_result.contents[i].mod_time.Truncate(time.Millisecond);
+        }
+
+        for (i, obj) in list_bucket_result.common_prefixes.iter_mut().enumerate() {
+            obj.prefix = decode_s3_name(&obj.prefix, &list_bucket_result.encoding_type)?;
+        }
+
+        Ok(list_bucket_result)
+    }
+
+    pub async fn list_object_versions_query(
+        &self,
+        bucket_name: &str,
+        opts: &ListObjectsOptions,
+        key_marker: &str,
+        version_id_marker: &str,
+        delimiter: &str,
+    ) -> Result<ListVersionsResult, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("versions".to_string(), "".to_string());
+        url_values.insert("prefix".to_string(), opts.prefix.clone());
+        url_values.insert("delimiter".to_string(), delimiter.to_string());
+        url_values.insert("encoding-type".to_string(), "url".to_string());
+
+        if !key_marker.is_empty() {
+            url_values.insert("key-marker".to_string(), key_marker.to_string());
+        }
+        if opts.max_keys > 0 {
+            url_values.insert("max-keys".to_string(), opts.max_keys.to_string());
+        }
+        if !version_id_marker.is_empty() {
+            url_values.insert("version-id-marker".to_string(), version_id_marker.to_string());
+        }
+        if opts.with_metadata {
+            url_values.insert("metadata".to_string(), "true".to_string());
+        }
+
+        let mut resp = self
+            .execute_method(
+                http::Method::GET,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: "".to_string(),
+                    query_values: url_values,
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: opts.headers.clone(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        let resp_status = resp.status();
+        let headers = resp.headers().clone();
+        let body = collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE).await?;
+        if resp_status != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                resp_status,
+                &headers,
+                body,
+                bucket_name,
+                "",
+            )));
+        }
+
+        let mut versions = quick_xml::de::from_reader::<_, ListVersionsResult>(body.as_slice())
+            .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?;
+        for version in &mut versions.versions {
+            version.key = decode_s3_name(&version.key, &versions.encoding_type)?;
+        }
+        for marker in &mut versions.delete_markers {
+            marker.key = decode_s3_name(&marker.key, &versions.encoding_type)?;
+        }
+        for prefix in &mut versions.common_prefixes {
+            prefix.prefix = decode_s3_name(&prefix.prefix, &versions.encoding_type)?;
+        }
+        if !versions.next_key_marker.is_empty() {
+            versions.next_key_marker = decode_s3_name(&versions.next_key_marker, &versions.encoding_type)?;
+        }
+
+        if versions.is_truncated && versions.next_key_marker.is_empty() {
+            return Err(std::io::Error::other(credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: "Truncated ListObjectVersions response should have next key marker set".to_string(),
+                },
+                request_id: "".to_string(),
+            }));
+        }
+
+        Ok(versions)
+    }
+
+    pub fn list_objects_query(
+        &self,
+        bucket_name: &str,
+        object_prefix: &str,
+        object_marker: &str,
+        delimiter: &str,
+        max_keys: i64,
+        headers: HeaderMap,
+    ) -> Result<ListBucketResult, std::io::Error> {
+        Err(std::io::Error::new(
+            ErrorKind::Unsupported,
+            credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: format!("list_objects_query is not implemented for bucket {bucket_name}"),
+                },
+                request_id: "".to_string(),
+            },
+        ))
+    }
+
+    pub fn list_multipart_uploads_query(
+        &self,
+        bucket_name: &str,
+        key_marker: &str,
+        upload_id_marker: &str,
+        prefix: &str,
+        delimiter: &str,
+        max_uploads: i64,
+    ) -> Result<ListMultipartUploadsResult, std::io::Error> {
+        Err(std::io::Error::new(
+            ErrorKind::Unsupported,
+            credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: format!("list_multipart_uploads_query is not implemented for bucket {bucket_name}"),
+                },
+                request_id: "".to_string(),
+            },
+        ))
+    }
+
+    pub fn list_object_parts(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        upload_id: &str,
+    ) -> Result<HashMap<i64, ObjectPart>, std::io::Error> {
+        Err(std::io::Error::new(
+            ErrorKind::Unsupported,
+            credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: format!(
+                        "list_object_parts is not implemented for bucket {bucket_name}, object {object_name}, upload_id {upload_id}"
+                    ),
+                },
+                request_id: "".to_string(),
+            },
+        ))
+    }
+
+    pub fn find_upload_ids(&self, bucket_name: &str, object_name: &str) -> Result<Vec<String>, std::io::Error> {
+        Err(std::io::Error::new(
+            ErrorKind::Unsupported,
+            credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: format!("find_upload_ids is not implemented for bucket {bucket_name}, object {object_name}"),
+                },
+                request_id: "".to_string(),
+            },
+        ))
+    }
+
+    pub async fn list_object_parts_query(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        upload_id: &str,
+        part_number_marker: i64,
+        max_parts: i64,
+    ) -> Result<ListObjectPartsResult, std::io::Error> {
+        Err(std::io::Error::new(
+            ErrorKind::Unsupported,
+            credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: format!(
+                        "list_object_parts_query is not implemented for bucket {bucket_name}, object {object_name}, upload_id {upload_id}"
+                    ),
+                },
+                request_id: "".to_string(),
+            },
+        ))
+    }
+}
+
+#[derive(Default)]
+pub struct ListObjectsOptions {
+    reverse_versions: bool,
+    with_versions: bool,
+    with_metadata: bool,
+    prefix: String,
+    recursive: bool,
+    max_keys: i64,
+    start_after: String,
+    use_v1: bool,
+    headers: HeaderMap,
+}
+
+impl ListObjectsOptions {
+    pub fn set(&mut self, key: &str, value: &str) {
+        match key {
+            "prefix" => {
+                self.prefix = value.to_string();
+            }
+            "start-after" => {
+                self.start_after = value.to_string();
+            }
+            "max-keys" => {
+                if let Ok(v) = value.parse::<i64>() {
+                    self.max_keys = v;
+                }
+            }
+            "delimiter" => {
+                // delimiter is currently kept in request only; this option structure does not persist it yet.
+            }
+            "reverse" | "versions" | "metadata" | "recursive" | "use-v1" => {
+                if let Some(v) = value.strip_prefix("v").or_else(|| value.strip_prefix("V")) {
+                    let v = v.eq_ignore_ascii_case("true");
+                    match key {
+                        "reverse" => self.reverse_versions = v,
+                        "versions" => self.with_versions = v,
+                        "metadata" => self.with_metadata = v,
+                        "recursive" => self.recursive = v,
+                        _ => self.use_v1 = v,
+                    }
+                } else {
+                    let v = value.eq_ignore_ascii_case("true");
+                    match key {
+                        "reverse" => self.reverse_versions = v,
+                        "versions" => self.with_versions = v,
+                        "metadata" => self.with_metadata = v,
+                        "recursive" => self.recursive = v,
+                        _ => self.use_v1 = v,
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+}
+
+fn decode_s3_name(name: &str, encoding_type: &str) -> Result<String, std::io::Error> {
+    match encoding_type {
+        "url" => {
+            //return url::QueryUnescape(name);
+            return Ok(name.to_string());
+        }
+        _ => {
+            return Ok(name.to_string());
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn list_versions_xml_preserves_versions_and_delete_markers() {
+        let xml = br#"
+            <ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+                <Name>tier-bucket</Name>
+                <Prefix>archive/object</Prefix>
+                <KeyMarker></KeyMarker>
+                <VersionIdMarker></VersionIdMarker>
+                <MaxKeys>2</MaxKeys>
+                <IsTruncated>true</IsTruncated>
+                <NextKeyMarker>archive/object</NextKeyMarker>
+                <NextVersionIdMarker>version-a</NextVersionIdMarker>
+                <Version>
+                    <Key>archive/object</Key>
+                    <VersionId>version-a</VersionId>
+                    <IsLatest>true</IsLatest>
+                    <LastModified>2026-07-22T00:00:00Z</LastModified>
+                    <ETag>&quot;etag-a&quot;</ETag>
+                    <Size>5</Size>
+                    <StorageClass>STANDARD</StorageClass>
+                </Version>
+                <DeleteMarker>
+                    <Key>archive/object</Key>
+                    <VersionId>marker-a</VersionId>
+                    <IsLatest>false</IsLatest>
+                    <LastModified>2026-07-22T00:00:01Z</LastModified>
+                </DeleteMarker>
+            </ListVersionsResult>
+        "#;
+
+        let parsed =
+            quick_xml::de::from_reader::<_, ListVersionsResult>(xml.as_slice()).expect("ListObjectVersions XML should parse");
+
+        assert!(parsed.is_truncated);
+        assert_eq!(parsed.next_key_marker, "archive/object");
+        assert_eq!(parsed.next_version_id_marker, "version-a");
+        assert_eq!(parsed.versions.len(), 1);
+        assert_eq!(parsed.versions[0].key, "archive/object");
+        assert_eq!(parsed.versions[0].version_id, "version-a");
+        assert_eq!(parsed.delete_markers.len(), 1);
+        assert_eq!(parsed.delete_markers[0].version_id, "marker-a");
+    }
+}

crates/s3-client/src/api_put_object.rs

@@ -0,0 +1,465 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use bytes::Bytes;
+use http::{HeaderMap, HeaderName, HeaderValue};
+use std::{collections::HashMap, sync::Arc};
+use time::{Duration, OffsetDateTime, macros::format_description};
+use tracing::{error, info, warn};
+
+use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus};
+use s3s::header::{
+    X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
+    X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION,
+};
+//use crate::disk::{BufferReader, Reader};
+use crate::checksum::ChecksumMode;
+use crate::utils::base64_encode;
+use crate::{
+    api_error_response::{err_entity_too_large, err_invalid_argument},
+    api_put_object_common::optimal_part_info,
+    api_put_object_multipart::UploadPartParams,
+    api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
+    constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE},
+    credentials::SignatureType,
+    transition_api::{ReaderImpl, TransitionClient, UploadInfo},
+    utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
+};
+
+#[derive(Debug, Clone)]
+pub struct AdvancedPutOptions {
+    pub source_version_id: String,
+    pub source_etag: String,
+    pub replication_status: ReplicationStatus,
+    pub source_mtime: OffsetDateTime,
+    pub replication_request: bool,
+    pub retention_timestamp: OffsetDateTime,
+    pub tagging_timestamp: OffsetDateTime,
+    pub legalhold_timestamp: OffsetDateTime,
+    pub replication_validity_check: bool,
+}
+
+impl Default for AdvancedPutOptions {
+    fn default() -> Self {
+        Self {
+            source_version_id: "".to_string(),
+            source_etag: "".to_string(),
+            replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
+            source_mtime: OffsetDateTime::now_utc(),
+            replication_request: false,
+            retention_timestamp: OffsetDateTime::now_utc(),
+            tagging_timestamp: OffsetDateTime::now_utc(),
+            legalhold_timestamp: OffsetDateTime::now_utc(),
+            replication_validity_check: false,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct PutObjectOptions {
+    pub user_metadata: HashMap<String, String>,
+    pub user_tags: HashMap<String, String>,
+    //pub progress: ReaderImpl,
+    pub content_type: String,
+    pub content_encoding: String,
+    pub content_disposition: String,
+    pub content_language: String,
+    pub cache_control: String,
+    pub expires: OffsetDateTime,
+    pub mode: ObjectLockRetentionMode,
+    pub retain_until_date: OffsetDateTime,
+    //pub server_side_encryption: encrypt::ServerSide,
+    pub num_threads: u64,
+    pub storage_class: String,
+    pub website_redirect_location: String,
+    pub part_size: u64,
+    pub legalhold: ObjectLockLegalHoldStatus,
+    pub send_content_md5: bool,
+    pub disable_content_sha256: bool,
+    pub disable_multipart: bool,
+    pub auto_checksum: ChecksumMode,
+    pub checksum: ChecksumMode,
+    pub concurrent_stream_parts: bool,
+    pub internal: AdvancedPutOptions,
+    pub custom_header: HeaderMap,
+}
+
+impl Default for PutObjectOptions {
+    fn default() -> Self {
+        Self {
+            user_metadata: HashMap::new(),
+            user_tags: HashMap::new(),
+            //progress: ReaderImpl::Body(Bytes::new()),
+            content_type: "".to_string(),
+            content_encoding: "".to_string(),
+            content_disposition: "".to_string(),
+            content_language: "".to_string(),
+            cache_control: "".to_string(),
+            expires: OffsetDateTime::UNIX_EPOCH,
+            mode: ObjectLockRetentionMode::from_static(""),
+            retain_until_date: OffsetDateTime::UNIX_EPOCH,
+            //server_side_encryption: encrypt.ServerSide::default(),
+            num_threads: 0,
+            storage_class: "".to_string(),
+            website_redirect_location: "".to_string(),
+            part_size: 0,
+            // Empty, not OFF: `header()` emits x-amz-object-lock-legal-hold for
+            // any non-empty status, and CompleteMultipartUpload rejects requests
+            // that carry object-lock headers, breaking multipart transitions
+            // (rustfs/rustfs#4811). Only send the header when a status is set.
+            legalhold: ObjectLockLegalHoldStatus::from_static(""),
+            send_content_md5: false,
+            disable_content_sha256: false,
+            disable_multipart: false,
+            auto_checksum: ChecksumMode::ChecksumNone,
+            checksum: ChecksumMode::ChecksumNone,
+            concurrent_stream_parts: false,
+            internal: AdvancedPutOptions::default(),
+            custom_header: HeaderMap::new(),
+        }
+    }
+}
+
+impl PutObjectOptions {
+    #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
+    fn set_match_etag(&mut self, etag: &str) {
+        if etag == "*" {
+            self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
+        } else {
+            if let Ok(etag_value) = HeaderValue::from_str(&format!("\"{}\"", etag)) {
+                self.custom_header.insert("If-Match", etag_value);
+            }
+        }
+    }
+
+    #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
+    fn set_match_etag_except(&mut self, etag: &str) {
+        if etag == "*" {
+            self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
+        } else {
+            if let Ok(etag_value) = HeaderValue::from_str(&format!("\"{etag}\"")) {
+                self.custom_header.insert("If-None-Match", etag_value);
+            }
+        }
+    }
+
+    pub fn header(&self) -> HeaderMap {
+        let mut header = HeaderMap::new();
+
+        let mut content_type = self.content_type.clone();
+        if content_type == "" {
+            content_type = "application/octet-stream".to_string();
+        }
+        if let Ok(content_type_value) = HeaderValue::from_str(&content_type) {
+            header.insert("Content-Type", content_type_value);
+        }
+
+        if self.content_encoding != "" {
+            if let Ok(encoding_value) = HeaderValue::from_str(&self.content_encoding) {
+                header.insert("Content-Encoding", encoding_value);
+            }
+        }
+        if self.content_disposition != "" {
+            if let Ok(disposition_value) = HeaderValue::from_str(&self.content_disposition) {
+                header.insert("Content-Disposition", disposition_value);
+            }
+        }
+        if self.content_language != "" {
+            if let Ok(language_value) = HeaderValue::from_str(&self.content_language) {
+                header.insert("Content-Language", language_value);
+            }
+        }
+        if self.cache_control != "" {
+            if let Ok(cache_value) = HeaderValue::from_str(&self.cache_control) {
+                header.insert("Cache-Control", cache_value);
+            }
+        }
+
+        if self.expires.unix_timestamp() != 0 {
+            if let Ok(expires_str) = self.expires.format(ISO8601_DATEFORMAT) {
+                if let Ok(expires_value) = HeaderValue::from_str(&expires_str) {
+                    header.insert("Expires", expires_value);
+                }
+            }
+        }
+
+        if self.mode.as_str() != "" {
+            if let Ok(mode_value) = HeaderValue::from_str(self.mode.as_str()) {
+                header.insert(X_AMZ_OBJECT_LOCK_MODE, mode_value);
+            }
+        }
+
+        if self.retain_until_date.unix_timestamp() != 0 {
+            if let Ok(retain_str) = self.retain_until_date.format(ISO8601_DATEFORMAT) {
+                if let Ok(retain_value) = HeaderValue::from_str(&retain_str) {
+                    header.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, retain_value);
+                }
+            }
+        }
+
+        if self.legalhold.as_str() != "" {
+            if let Ok(legalhold_value) = HeaderValue::from_str(self.legalhold.as_str()) {
+                header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, legalhold_value);
+            }
+        }
+
+        if self.storage_class != "" {
+            if let Ok(storage_class_value) = HeaderValue::from_str(&self.storage_class) {
+                header.insert(X_AMZ_STORAGE_CLASS, storage_class_value);
+            }
+        }
+
+        if self.website_redirect_location != "" {
+            if let Ok(redirect_value) = HeaderValue::from_str(&self.website_redirect_location) {
+                header.insert(X_AMZ_WEBSITE_REDIRECT_LOCATION, redirect_value);
+            }
+        }
+
+        if !self.internal.replication_status.as_str().is_empty() {
+            if let Ok(replication_status_value) = HeaderValue::from_str(self.internal.replication_status.as_str()) {
+                header.insert(X_AMZ_REPLICATION_STATUS, replication_status_value);
+            }
+        }
+
+        for (k, v) in &self.user_metadata {
+            let Ok(header_value) = HeaderValue::from_str(v) else {
+                warn!("skipping user metadata header with invalid value: {}", k);
+                continue;
+            };
+            if is_amz_header(k) || is_standard_header(k) || is_storageclass_header(k) || is_rustfs_header(k) || is_minio_header(k)
+            {
+                if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
+                    header.insert(header_name, header_value);
+                }
+            } else if let Ok(header_name) = HeaderName::from_bytes(format!("x-amz-meta-{}", k).as_bytes()) {
+                header.insert(header_name, header_value);
+            }
+        }
+
+        for (k, v) in self.custom_header.iter() {
+            header.insert(k.clone(), v.clone());
+        }
+
+        header
+    }
+
+    #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
+    fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
+        //if self.checksum.is_set() {
+        /*if !self.trailing_header_support {
+            return Err(Error::from(err_invalid_argument("Checksum requires Client with TrailingHeaders enabled")));
+        }*/
+        /*else if self.override_signer_type == SignatureType::SignatureV2 {
+            return Err(Error::from(err_invalid_argument("Checksum cannot be used with v2 signatures")));
+        }*/
+        //}
+
+        Ok(())
+    }
+}
+
+impl TransitionClient {
+    pub async fn put_object(
+        self: Arc<Self>,
+        bucket_name: &str,
+        object_name: &str,
+        reader: ReaderImpl,
+        object_size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        if object_size < 0 && opts.disable_multipart {
+            return Err(std::io::Error::other("object size must be provided with disable multipart upload"));
+        }
+
+        self.put_object_common(bucket_name, object_name, reader, object_size, opts)
+            .await
+    }
+
+    pub async fn put_object_common(
+        self: Arc<Self>,
+        bucket_name: &str,
+        object_name: &str,
+        reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        if size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+            return Err(std::io::Error::other(err_entity_too_large(
+                size,
+                MAX_MULTIPART_PUT_OBJECT_SIZE,
+                bucket_name,
+                object_name,
+            )));
+        }
+        let mut opts = opts.clone();
+        opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
+
+        let mut part_size = opts.part_size as i64;
+        if opts.part_size == 0 {
+            part_size = MIN_PART_SIZE;
+        }
+
+        if SignatureType::SignatureV2 == self.override_signer_type {
+            if size >= 0 && size < part_size || opts.disable_multipart {
+                return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
+            }
+            return self.put_object_multipart(bucket_name, object_name, reader, size, &opts).await;
+        }
+
+        if size < 0 {
+            if opts.disable_multipart {
+                return Err(std::io::Error::other("no length provided and multipart disabled"));
+            }
+            if opts.concurrent_stream_parts && opts.num_threads > 1 {
+                return self
+                    .put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts)
+                    .await;
+            }
+            return self
+                .put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts)
+                .await;
+        }
+
+        if size <= part_size || opts.disable_multipart {
+            return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
+        }
+
+        self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts)
+            .await
+    }
+
+    pub async fn put_object_multipart_stream_no_length(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut total_uploaded_size: i64 = 0;
+
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let (total_parts_count, part_size, _) = optimal_part_info(-1, opts.part_size)?;
+
+        let mut opts = opts.clone();
+
+        if opts.checksum.is_set() {
+            opts.send_content_md5 = false;
+            opts.auto_checksum = opts.checksum.clone();
+        }
+        if !opts.send_content_md5 {
+            //add_auto_checksum_headers(&mut opts);
+        }
+
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut part_number = 1;
+        let mut parts_info = HashMap::<i64, ObjectPart>::new();
+        let mut buf = Vec::<u8>::with_capacity(part_size as usize);
+
+        let mut custom_header = HeaderMap::new();
+
+        while part_number <= total_parts_count {
+            buf = match &mut reader {
+                ReaderImpl::Body(content_body) => content_body.to_vec(),
+                ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?,
+            };
+            let length = buf.len();
+
+            let mut md5_base64: String = "".to_string();
+            if opts.send_content_md5 {
+                if let Some(mut md5_hasher) = self.md5_hasher.lock().expect("operation should succeed").as_mut() {
+                    let hash = md5_hasher.hash_encode(&buf[..length]);
+                    md5_base64 = base64_encode(hash.as_ref());
+                }
+            } else {
+                let mut crc = opts.auto_checksum.hasher()?;
+                crc.update(&buf[..length]);
+                let csum = crc.finalize();
+
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key());
+                }
+            }
+
+            //let rd = newHook(bytes.NewReader(buf[..length]), opts.progress);
+            let rd = ReaderImpl::Body(Bytes::from(buf));
+
+            let mut p = UploadPartParams {
+                bucket_name: bucket_name.to_string(),
+                object_name: object_name.to_string(),
+                upload_id: upload_id.clone(),
+                reader: rd,
+                part_number,
+                md5_base64,
+                size: length as i64,
+                //sse: opts.server_side_encryption,
+                stream_sha256: !opts.disable_content_sha256,
+                custom_header: custom_header.clone(),
+                sha256_hex: Default::default(),
+                trailer: Default::default(),
+            };
+            let obj_part = self.upload_part(&mut p).await?;
+
+            parts_info.entry(part_number).or_insert(obj_part);
+            total_uploaded_size += length as i64;
+            part_number += 1;
+        }
+
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
+        for i in 1..part_number {
+            let part = parts_info[&i].clone();
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag: part.etag,
+                part_num: part.part_num,
+                checksum_crc32: part.checksum_crc32,
+                checksum_crc32c: part.checksum_crc32c,
+                checksum_sha1: part.checksum_sha1,
+                checksum_sha256: part.checksum_sha256,
+                checksum_crc64nvme: part.checksum_crc64nvme,
+                ..Default::default()
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+
+        let opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum: opts.auto_checksum,
+            ..Default::default()
+        };
+        //apply_auto_checksum(&mut opts, all_parts);
+
+        let mut upload_info = self
+            .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
+            .await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+}

crates/s3-client/src/api_put_object_common.rs

@@ -0,0 +1,108 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use crate::{
+    api_error_response::{err_entity_too_large, err_invalid_argument},
+    api_put_object::PutObjectOptions,
+    constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PART_SIZE, MAX_PARTS_COUNT, MIN_PART_SIZE},
+    transition_api::ReaderImpl,
+    transition_api::TransitionClient,
+};
+
+pub fn is_object(reader: &ReaderImpl) -> bool {
+    matches!(reader, ReaderImpl::ObjectBody(_))
+}
+
+pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
+    let unknown_size;
+    let mut object_size = object_size;
+    if object_size == -1 {
+        unknown_size = true;
+        object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
+    } else {
+        unknown_size = false;
+    }
+
+    if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+        return Err(std::io::Error::other(err_entity_too_large(
+            object_size,
+            MAX_MULTIPART_PUT_OBJECT_SIZE,
+            "",
+            "",
+        )));
+    }
+
+    let mut part_size_flt: f64;
+    if configured_part_size > 0 {
+        if configured_part_size as i64 > object_size {
+            return Err(std::io::Error::other(err_entity_too_large(
+                configured_part_size as i64,
+                object_size,
+                "",
+                "",
+            )));
+        }
+
+        if !unknown_size && object_size > (configured_part_size as i64 * MAX_PARTS_COUNT) {
+            return Err(std::io::Error::other(err_invalid_argument(
+                "Part size * max_parts(10000) is lesser than input objectSize.",
+            )));
+        }
+
+        if (configured_part_size as i64) < ABS_MIN_PART_SIZE {
+            return Err(std::io::Error::other(err_invalid_argument(
+                "Input part size is smaller than allowed minimum of 5MiB.",
+            )));
+        }
+
+        if configured_part_size as i64 > MAX_PART_SIZE {
+            return Err(std::io::Error::other(err_invalid_argument(
+                "Input part size is bigger than allowed maximum of 5GiB.",
+            )));
+        }
+
+        part_size_flt = configured_part_size as f64;
+        if unknown_size {
+            object_size = configured_part_size as i64 * MAX_PARTS_COUNT;
+        }
+    } else {
+        let min_part = MIN_PART_SIZE as f64;
+        part_size_flt = (object_size as f64 / MAX_PARTS_COUNT as f64).ceil();
+        part_size_flt = part_size_flt.max(min_part);
+        part_size_flt = (part_size_flt / min_part).ceil() * min_part;
+    }
+
+    let total_parts_count = (object_size as f64 / part_size_flt).ceil() as i64;
+    let part_size = part_size_flt as i64;
+    let last_part_size = object_size - (total_parts_count - 1) * part_size;
+    Ok((total_parts_count, part_size, last_part_size))
+}
+
+impl TransitionClient {
+    pub async fn new_upload_id(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        opts: &PutObjectOptions,
+    ) -> Result<String, std::io::Error> {
+        let init_multipart_upload_result = self.initiate_multipart_upload(bucket_name, object_name, opts).await?;
+        Ok(init_multipart_upload_result.upload_id)
+    }
+}

crates/s3-client/src/api_put_object_multipart.rs

@@ -0,0 +1,500 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use http::{HeaderMap, HeaderName, StatusCode};
+use http_body_util::BodyExt;
+use hyper::body::Bytes;
+use s3s::S3ErrorCode;
+use std::collections::HashMap;
+use time::OffsetDateTime;
+use tracing::warn;
+use uuid::Uuid;
+
+use crate::checksum::ChecksumMode;
+use crate::utils::base64_encode;
+use crate::{
+    api_error_response::{
+        err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response,
+    },
+    api_put_object::PutObjectOptions,
+    api_put_object_common::optimal_part_info,
+    api_s3_datatypes::{
+        CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart,
+    },
+    constants::{ISO8601_DATEFORMAT, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE},
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
+};
+use rustfs_utils::path::trim_etag;
+use s3s::header::X_AMZ_EXPIRATION;
+
+impl TransitionClient {
+    pub async fn put_object_multipart(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let info = self
+            .put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts)
+            .await;
+        if let Err(err) = &info {
+            let err_resp = to_error_response(err);
+            if err_resp.code == S3ErrorCode::AccessDenied && err_resp.message.contains("Access Denied") {
+                if size > MAX_SINGLE_PUT_OBJECT_SIZE {
+                    return Err(std::io::Error::other(err_entity_too_large(
+                        size,
+                        MAX_SINGLE_PUT_OBJECT_SIZE,
+                        bucket_name,
+                        object_name,
+                    )));
+                }
+                return self.put_object_gcs(bucket_name, object_name, reader, size, opts).await;
+            }
+        }
+        Ok(info?)
+    }
+
+    pub async fn put_object_multipart_no_stream(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        reader: &mut ReaderImpl,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut total_uploaded_size: i64 = 0;
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let ret = optimal_part_info(-1, opts.part_size)?;
+        let (total_parts_count, part_size, _) = ret;
+
+        let (mut hash_algos, mut hash_sums) = self.hash_materials(opts.send_content_md5, !opts.disable_content_sha256);
+        let upload_id = self.new_upload_id(bucket_name, object_name, opts).await?;
+        let mut opts = opts.clone();
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut part_number = 1;
+        let mut parts_info = HashMap::<i64, ObjectPart>::new();
+        let mut buf = Vec::<u8>::with_capacity(part_size as usize);
+        let mut custom_header = HeaderMap::new();
+        while part_number <= total_parts_count {
+            match reader {
+                ReaderImpl::Body(content_body) => {
+                    buf = content_body.to_vec();
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    buf = content_body.read_all().await?;
+                }
+            }
+            let length = buf.len();
+
+            for (k, v) in hash_algos.iter_mut() {
+                let hash = v.hash_encode(&buf[..length]);
+                hash_sums.insert(k.to_string(), hash.as_ref().to_vec());
+            }
+
+            //let rd = newHook(bytes.NewReader(buf[..length]), opts.progress);
+            let rd = Bytes::from(buf.clone());
+
+            let md5_base64: String;
+            let sha256_hex: String;
+
+            //if hash_sums["md5"] != nil {
+            md5_base64 = base64_encode(&hash_sums["md5"]);
+            //}
+            //if hash_sums["sha256"] != nil {
+            sha256_hex = hex_simd::encode_to_string(hash_sums["sha256"].clone(), hex_simd::AsciiCase::Lower);
+            //}
+            if hash_sums.len() == 0 {
+                let mut crc = opts.auto_checksum.hasher()?;
+                crc.update(&buf[..length]);
+                let csum = crc.finalize();
+
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key());
+                }
+            }
+
+            let mut p = UploadPartParams {
+                bucket_name: bucket_name.to_string(),
+                object_name: object_name.to_string(),
+                upload_id: upload_id.clone(),
+                reader: ReaderImpl::Body(rd),
+                part_number,
+                md5_base64,
+                sha256_hex,
+                size: length as i64,
+                //sse: opts.server_side_encryption,
+                stream_sha256: !opts.disable_content_sha256,
+                custom_header: custom_header.clone(),
+                trailer: HeaderMap::new(),
+            };
+            let obj_part = self.upload_part(&mut p).await?;
+
+            parts_info.insert(part_number, obj_part);
+            total_uploaded_size += length as i64;
+            part_number += 1;
+        }
+
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
+        for i in 1..part_number {
+            let part = parts_info[&i].clone();
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag: part.etag,
+                part_num: part.part_num,
+                checksum_crc32: part.checksum_crc32,
+                checksum_crc32c: part.checksum_crc32c,
+                checksum_sha1: part.checksum_sha1,
+                checksum_sha256: part.checksum_sha256,
+                checksum_crc64nvme: part.checksum_crc64nvme,
+                ..Default::default()
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+        let opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum: opts.auto_checksum,
+            ..Default::default()
+        };
+        //apply_auto_checksum(&mut opts, all_parts);
+
+        let mut upload_info = self
+            .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
+            .await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+
+    pub async fn initiate_multipart_upload(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        opts: &PutObjectOptions,
+    ) -> Result<InitiateMultipartUploadResult, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("uploads".to_string(), "".to_string());
+
+        if opts.internal.source_version_id != "" {
+            if !opts.internal.source_version_id.is_empty() {
+                if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) {
+                    return Err(std::io::Error::other(err_invalid_argument(&err.to_string())));
+                }
+            }
+            url_values.insert("versionId".to_string(), opts.internal.source_version_id.clone());
+        }
+
+        let custom_header = opts.header();
+
+        let mut req_metadata = RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            object_name: object_name.to_string(),
+            query_values: url_values,
+            custom_header,
+            content_body: ReaderImpl::Body(Bytes::new()),
+            content_length: 0,
+            content_md5_base64: "".to_string(),
+            content_sha256_hex: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
+
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
+        //if resp.is_none() {
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                resp_status,
+                &h,
+                vec![],
+                bucket_name,
+                object_name,
+            )));
+        }
+        //}
+        // Parse the CreateMultipartUpload response for the UploadId. Returning a
+        // default (empty) result here made every multipart transition fail at the
+        // first UploadPart with "UploadID cannot be empty" (rustfs/rustfs#4811).
+        let mut body_vec = Vec::new();
+        let mut body = resp.into_body();
+        while let Some(frame) = body.frame().await {
+            let frame = frame.map_err(|e| std::io::Error::other(e.to_string()))?;
+            if let Some(data) = frame.data_ref() {
+                body_vec.extend_from_slice(data);
+            }
+        }
+        let initiate_multipart_upload_result =
+            quick_xml::de::from_str::<InitiateMultipartUploadResult>(&String::from_utf8_lossy(&body_vec))
+                .map_err(|e| std::io::Error::other(format!("failed to parse CreateMultipartUpload response: {e}")))?;
+        if initiate_multipart_upload_result.upload_id.is_empty() {
+            return Err(std::io::Error::other("CreateMultipartUpload response missing UploadId"));
+        }
+        Ok(initiate_multipart_upload_result)
+    }
+
+    pub async fn upload_part(&self, p: &mut UploadPartParams) -> Result<ObjectPart, std::io::Error> {
+        if p.size > MAX_PART_SIZE {
+            return Err(std::io::Error::other(err_entity_too_large(
+                p.size,
+                MAX_PART_SIZE,
+                &p.bucket_name,
+                &p.object_name,
+            )));
+        }
+        if p.size <= -1 {
+            return Err(std::io::Error::other(err_entity_too_small(p.size, &p.bucket_name, &p.object_name)));
+        }
+        if p.part_number <= 0 {
+            return Err(std::io::Error::other(err_invalid_argument(
+                "Part number cannot be negative or equal to zero.",
+            )));
+        }
+        if p.upload_id == "" {
+            return Err(std::io::Error::other(err_invalid_argument("UploadID cannot be empty.")));
+        }
+
+        let mut url_values = HashMap::new();
+        url_values.insert("partNumber".to_string(), p.part_number.to_string());
+        url_values.insert("uploadId".to_string(), p.upload_id.clone());
+
+        let buf = match &mut p.reader {
+            ReaderImpl::Body(content_body) => content_body.to_vec(),
+            ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?,
+        };
+        let mut req_metadata = RequestMetadata {
+            bucket_name: p.bucket_name.clone(),
+            object_name: p.object_name.clone(),
+            query_values: url_values,
+            custom_header: p.custom_header.clone(),
+            content_body: ReaderImpl::Body(Bytes::from(buf)),
+            content_length: p.size,
+            content_md5_base64: p.md5_base64.clone(),
+            content_sha256_hex: p.sha256_hex.clone(),
+            stream_sha256: p.stream_sha256,
+            trailer: p.trailer.clone(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
+
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                resp_status,
+                &h,
+                vec![],
+                &p.bucket_name.clone(),
+                &p.object_name,
+            )));
+        }
+        //}
+        let h = resp.headers();
+        let mut obj_part = ObjectPart {
+            checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
+                h_checksum_crc32.to_str().unwrap_or("").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
+                h_checksum_crc32c.to_str().unwrap_or("").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
+                h_checksum_sha1.to_str().unwrap_or("").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
+                h_checksum_sha256.to_str().unwrap_or("").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
+                h_checksum_crc64nvme.to_str().unwrap_or("").to_string()
+            } else {
+                "".to_string()
+            },
+            ..Default::default()
+        };
+        obj_part.size = p.size;
+        obj_part.part_num = p.part_number;
+        obj_part.etag = if let Some(h_etag) = h.get("ETag") {
+            h_etag.to_str().unwrap_or("").trim_matches('"').to_string()
+        } else {
+            "".to_string()
+        };
+        Ok(obj_part)
+    }
+
+    pub async fn complete_multipart_upload(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        upload_id: &str,
+        complete: CompleteMultipartUpload,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("uploadId".to_string(), upload_id.to_string());
+        let complete_multipart_upload_bytes = complete.marshal_msg()?.as_bytes().to_vec();
+
+        let headers = opts.header();
+
+        let complete_multipart_upload_buffer = Bytes::from(complete_multipart_upload_bytes);
+        let mut req_metadata = RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            object_name: object_name.to_string(),
+            query_values: url_values,
+            custom_header: headers,
+            content_body: ReaderImpl::Body(complete_multipart_upload_buffer),
+            content_length: 100,                //complete_multipart_upload_bytes.len(),
+            content_sha256_hex: "".to_string(), //hex_simd::encode_to_string(complete_multipart_upload_bytes, hex_simd::AsciiCase::Lower),
+            content_md5_base64: "".to_string(),
+            stream_sha256: Default::default(),
+            trailer: Default::default(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
+
+        let h = resp.headers().clone();
+
+        let complete_multipart_upload_result: CompleteMultipartUploadResult = CompleteMultipartUploadResult::default();
+
+        let exp_time = resp
+            .headers()
+            .get(X_AMZ_EXPIRATION)
+            .and_then(|v| v.to_str().ok())
+            .and_then(|s| OffsetDateTime::parse(s, ISO8601_DATEFORMAT).ok())
+            .unwrap_or_else(OffsetDateTime::now_utc);
+        let rule_id = "".to_string();
+
+        Ok(UploadInfo {
+            bucket: complete_multipart_upload_result.bucket,
+            key: complete_multipart_upload_result.key,
+            etag: trim_etag(&complete_multipart_upload_result.etag),
+            version_id: self.legacy_remote_version_id(&h)?,
+            location: complete_multipart_upload_result.location,
+            expiration: exp_time,
+            expiration_rule_id: rule_id,
+            checksum_sha256: complete_multipart_upload_result.checksum_sha256,
+            checksum_sha1: complete_multipart_upload_result.checksum_sha1,
+            checksum_crc32: complete_multipart_upload_result.checksum_crc32,
+            checksum_crc32c: complete_multipart_upload_result.checksum_crc32c,
+            checksum_crc64nvme: complete_multipart_upload_result.checksum_crc64nvme,
+            ..Default::default()
+        })
+    }
+}
+
+pub struct UploadPartParams {
+    pub bucket_name: String,
+    pub object_name: String,
+    pub upload_id: String,
+    pub reader: ReaderImpl,
+    pub part_number: i64,
+    pub md5_base64: String,
+    pub sha256_hex: String,
+    pub size: i64,
+    //pub sse: encrypt.ServerSide,
+    pub stream_sha256: bool,
+    pub custom_header: HeaderMap,
+    pub trailer: HeaderMap,
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::api_s3_datatypes::{CompleteMultipartUpload, CompletePart, InitiateMultipartUploadResult};
+
+    #[test]
+    fn complete_multipart_upload_serializes_s3_part_elements() {
+        // Regression for rustfs/rustfs#4811: without serde renames quick-xml emits
+        // <parts>/<part_num>/<etag>, so the remote parses zero <Part> elements and
+        // completes a 0-byte object (while still returning 200). The body must use
+        // S3 element names, and MD5-only transitions must not emit empty checksum
+        // elements.
+        let complete = CompleteMultipartUpload {
+            parts: vec![
+                CompletePart {
+                    part_num: 1,
+                    etag: "etag-one".to_string(),
+                    ..Default::default()
+                },
+                CompletePart {
+                    part_num: 2,
+                    etag: "etag-two".to_string(),
+                    ..Default::default()
+                },
+            ],
+        };
+        let xml = complete.marshal_msg().expect("marshal");
+        assert!(xml.contains("<Part>"), "missing <Part>: {xml}");
+        assert!(xml.contains("<PartNumber>1</PartNumber>"), "missing PartNumber: {xml}");
+        assert!(xml.contains("<ETag>etag-one</ETag>"), "missing ETag: {xml}");
+        assert!(xml.contains("<PartNumber>2</PartNumber>"), "missing part 2: {xml}");
+        assert!(!xml.contains("part_num"), "leaked rust field name: {xml}");
+        assert!(!xml.contains("<ChecksumCRC32>"), "emitted empty checksum: {xml}");
+    }
+
+    #[test]
+    fn parses_create_multipart_upload_response() {
+        // Regression for rustfs/rustfs#4811: `initiate_multipart_upload` used to
+        // return a default (empty) result, so every multipart transition failed
+        // the first UploadPart with "UploadID cannot be empty". The UploadId must
+        // be parsed out of the S3 XML response.
+        let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
+<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Bucket>bar</Bucket>
+  <Key>foo/payload.bin</Key>
+  <UploadId>a1b2c3-d4e5-f6</UploadId>
+</InitiateMultipartUploadResult>"#;
+        let parsed: InitiateMultipartUploadResult = quick_xml::de::from_str(xml).expect("parse");
+        assert_eq!(parsed.upload_id, "a1b2c3-d4e5-f6");
+        assert_eq!(parsed.bucket, "bar");
+        assert_eq!(parsed.key, "foo/payload.bin");
+    }
+}

crates/s3-client/src/api_put_object_streaming.rs

@@ -0,0 +1,741 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use bytes::Bytes;
+use futures::future::join_all;
+use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
+use std::io::Error;
+use std::sync::{Mutex, MutexGuard, RwLock};
+use std::{collections::HashMap, sync::Arc};
+use time::{OffsetDateTime, format_description};
+use tokio::io::AsyncReadExt;
+use tokio::{select, sync::mpsc};
+use tokio_util::sync::CancellationToken;
+use tracing::warn;
+use uuid::Uuid;
+
+use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
+use crate::{
+    api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response},
+    api_put_object::PutObjectOptions,
+    api_put_object_common::{is_object, optimal_part_info},
+    api_put_object_multipart::UploadPartParams,
+    api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
+    constants::ISO8601_DATEFORMAT,
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
+};
+
+use crate::utils::base64_encode;
+use rustfs_utils::path::trim_etag;
+use s3s::header::X_AMZ_EXPIRATION;
+
+fn lock_md5_hasher(
+    md5_hasher: &Mutex<Option<rustfs_utils::hash::HashAlgorithm>>,
+) -> Result<MutexGuard<'_, Option<rustfs_utils::hash::HashAlgorithm>>, std::io::Error> {
+    md5_hasher
+        .lock()
+        .map_err(|_| std::io::Error::other("MD5 hasher state is unavailable"))
+}
+
+/// Read exactly `want` bytes for a single multipart part, or fewer if the reader
+/// reaches EOF first. Advances the reader so the next call returns the following
+/// part. Replaces the previous per-part `read_all()`/`to_vec()`, which drained
+/// the entire source into the first part and left later parts empty
+/// (rustfs/rustfs#4811).
+async fn read_multipart_part(reader: &mut ReaderImpl, want: usize) -> Result<Vec<u8>, std::io::Error> {
+    match reader {
+        ReaderImpl::Body(content_body) => {
+            let take = content_body.len().min(want);
+            Ok(content_body.split_to(take).to_vec())
+        }
+        ReaderImpl::ObjectBody(content_body) => {
+            let mut buf = vec![0u8; want];
+            let mut filled = 0;
+            while filled < want {
+                let n = content_body.read(&mut buf[filled..]).await?;
+                if n == 0 {
+                    break;
+                }
+                filled += n;
+            }
+            buf.truncate(filled);
+            Ok(buf)
+        }
+    }
+}
+
+impl TransitionClient {
+    pub async fn put_object_multipart_stream(
+        self: Arc<Self>,
+        bucket_name: &str,
+        object_name: &str,
+        reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let info: UploadInfo;
+        if opts.concurrent_stream_parts && opts.num_threads > 1 {
+            info = self
+                .put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts)
+                .await?;
+        } else if !is_object(&reader) && !opts.send_content_md5 {
+            info = self
+                .put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts)
+                .await?;
+        } else {
+            info = self
+                .put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts)
+                .await?;
+        }
+
+        Ok(info)
+    }
+
+    pub async fn put_object_multipart_stream_from_readat(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let ret = optimal_part_info(size, opts.part_size)?;
+        let (total_parts_count, part_size, lastpart_size) = ret;
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.auto_checksum = opts.checksum.clone();
+        }
+
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        self.put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, &opts)
+            .await
+    }
+
+    pub async fn put_object_multipart_stream_optional_checksum(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.auto_checksum = opts.checksum.clone();
+            opts.send_content_md5 = false;
+        }
+
+        if !opts.send_content_md5 {
+            add_auto_checksum_headers(&mut opts);
+        }
+
+        let ret = optimal_part_info(size, opts.part_size)?;
+        let (total_parts_count, mut part_size, lastpart_size) = ret;
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut custom_header = opts.header().clone();
+
+        let mut total_uploaded_size: i64 = 0;
+
+        let mut parts_info = HashMap::<i64, ObjectPart>::new();
+
+        let mut md5_base64: String = "".to_string();
+        for part_number in 1..=total_parts_count {
+            if part_number == total_parts_count {
+                part_size = lastpart_size;
+            }
+
+            // Read exactly this part's bytes. Using `read_all()`/`to_vec()` here
+            // drained the whole source into the first part and left every later
+            // part empty, silently corrupting any multipart upload of a streamed
+            // (`ObjectBody`) source — e.g. ILM transitions of >128 MiB objects,
+            // which split into 128 MiB parts (rustfs/rustfs#4811).
+            let buf = read_multipart_part(&mut reader, part_size as usize).await?;
+            let length = buf.len();
+
+            if opts.send_content_md5 {
+                let mut md5_hasher = lock_md5_hasher(&self.md5_hasher)?;
+                let md5_hash = match md5_hasher.as_mut() {
+                    Some(hasher) => hasher,
+                    None => return Err(std::io::Error::other("MD5 hasher not initialized")),
+                };
+                let hash = md5_hash.hash_encode(&buf[..length]);
+                md5_base64 = base64_encode(hash.as_ref());
+            } else if opts.auto_checksum.is_set() {
+                let mut crc = opts.auto_checksum.hasher()?;
+                crc.update(&buf[..length]);
+                let csum = crc.finalize();
+
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key());
+                }
+            }
+            // else: neither MD5 nor a concrete additional checksum was requested,
+            // so upload the part without a per-part checksum header. Guarding the
+            // branch on `is_set()` avoids calling `hasher()` on `ChecksumNone`,
+            // which errors with "unsupported checksum type" (rustfs/rustfs#4811).
+
+            let hooked = ReaderImpl::Body(Bytes::from(buf)); //newHook(BufferReader::new(buf), opts.progress);
+            let mut p = UploadPartParams {
+                bucket_name: bucket_name.to_string(),
+                object_name: object_name.to_string(),
+                upload_id: upload_id.clone(),
+                reader: hooked,
+                part_number,
+                md5_base64: md5_base64.clone(),
+                // Use the bytes actually read, not the planned part_size, so the
+                // uploaded Content-Length matches the body even on a short read.
+                size: length as i64,
+                //sse: opts.server_side_encryption,
+                stream_sha256: !opts.disable_content_sha256,
+                custom_header: custom_header.clone(),
+                sha256_hex: "".to_string(),
+                trailer: HeaderMap::new(),
+            };
+            let obj_part = self.upload_part(&mut p).await?;
+
+            parts_info.entry(part_number).or_insert(obj_part);
+
+            total_uploaded_size += length as i64;
+        }
+
+        if size > 0 && total_uploaded_size != size {
+            return Err(std::io::Error::other(err_unexpected_eof(
+                total_uploaded_size,
+                size,
+                bucket_name,
+                object_name,
+            )));
+        }
+
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        // Parts are keyed 1..=total_parts_count during upload; every one — including the last —
+        // must be collected. The previous exclusive `1..total_parts_count` bound dropped the final
+        // part, silently truncating the completed object (and produced zero parts for a single-part
+        // upload).
+        let mut all_parts = collect_complete_parts(&parts_info, total_parts_count)?;
+        for part in &all_parts {
+            compl_multipart_upload.parts.push(CompletePart {
+                etag: part.etag.clone(),
+                part_num: part.part_num,
+                checksum_crc32: part.checksum_crc32.clone(),
+                checksum_crc32c: part.checksum_crc32c.clone(),
+                checksum_sha1: part.checksum_sha1.clone(),
+                checksum_sha256: part.checksum_sha256.clone(),
+                checksum_crc64nvme: part.checksum_crc64nvme.clone(),
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+
+        let mut opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum: opts.auto_checksum,
+            ..Default::default()
+        };
+        apply_auto_checksum(&mut opts, &mut all_parts);
+        let mut upload_info = self
+            .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
+            .await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+
+    pub async fn put_object_multipart_stream_parallel(
+        self: Arc<Self>,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl, /*GetObjectReader*/
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.send_content_md5 = false;
+            opts.auto_checksum = opts.checksum.clone();
+        }
+        if !opts.send_content_md5 {
+            add_auto_checksum_headers(&mut opts);
+        }
+
+        let ret = optimal_part_info(-1, opts.part_size)?;
+        let (total_parts_count, part_size, _) = ret;
+
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut total_uploaded_size: i64 = 0;
+        let parts_info = Arc::new(RwLock::new(HashMap::<i64, ObjectPart>::new()));
+
+        let n_buffers = opts.num_threads;
+        let (bufs_tx, mut bufs_rx) = mpsc::channel(n_buffers as usize);
+        //let all = Vec::<u8>::with_capacity(n_buffers as usize * part_size as usize);
+        for i in 0..n_buffers {
+            //bufs_tx.send(&all[i * part_size..i * part_size + part_size]);
+            bufs_tx.send(Vec::<u8>::with_capacity(part_size as usize));
+        }
+
+        let mut futures = Vec::with_capacity(total_parts_count as usize);
+        let (err_tx, mut err_rx) = mpsc::channel(opts.num_threads as usize);
+        let cancel_token = CancellationToken::new();
+
+        //reader = newHook(reader, opts.progress);
+
+        for part_number in 1..=total_parts_count {
+            let mut buf = Vec::<u8>::new();
+            select! {
+                buf1 = bufs_rx.recv() => {
+                    if let Some(buf1) = buf1 {
+                        buf = buf1;
+                    }
+                }
+                err = err_rx.recv() => {
+                    //cancel_token.cancel();
+                    return Err(err.unwrap_or_else(|| std::io::Error::other("Unknown error received from channel")));
+                }
+                else => (),
+            }
+
+            if buf.len() != part_size as usize {
+                return Err(std::io::Error::other(format!(
+                    "read buffer < {} than expected partSize: {}",
+                    buf.len(),
+                    part_size
+                )));
+            }
+
+            match &mut reader {
+                ReaderImpl::Body(content_body) => {
+                    buf = content_body.to_vec();
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    buf = content_body.read_all().await?;
+                }
+            }
+            let length = buf.len();
+
+            let mut custom_header = HeaderMap::new();
+            if !opts.send_content_md5 {
+                let mut crc = opts.auto_checksum.hasher()?;
+                crc.update(&buf[..length]);
+                let csum = crc.finalize();
+
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key());
+                }
+            }
+
+            let clone_bufs_tx = bufs_tx.clone();
+            let clone_parts_info = parts_info.clone();
+            let clone_upload_id = upload_id.clone();
+            let clone_self = self.clone();
+            let err_tx_clone = err_tx.clone();
+            futures.push(async move {
+                let mut md5_base64: String = "".to_string();
+
+                if opts.send_content_md5 {
+                    let mut md5_hasher = lock_md5_hasher(&clone_self.md5_hasher)?;
+                    let md5_hash = match md5_hasher.as_mut() {
+                        Some(hasher) => hasher,
+                        None => {
+                            //let _ = err_tx_clone.send(std::io::Error::other("MD5 hasher not initialized")).await;
+                            return Ok::<(), Error>(());
+                        }
+                    };
+                    let hash = md5_hash.hash_encode(&buf[..length]);
+                    md5_base64 = base64_encode(hash.as_ref());
+                }
+
+                //defer wg.Done()
+                let mut p = UploadPartParams {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: object_name.to_string(),
+                    upload_id: clone_upload_id,
+                    reader: ReaderImpl::Body(Bytes::from(buf.clone())),
+                    part_number,
+                    md5_base64,
+                    size: length as i64,
+                    //sse:           opts.server_side_encryption,
+                    stream_sha256: !opts.disable_content_sha256,
+                    custom_header,
+                    sha256_hex: "".to_string(),
+                    trailer: HeaderMap::new(),
+                };
+                let obj_part = match clone_self.upload_part(&mut p).await {
+                    Ok(part) => part,
+                    Err(err) => {
+                        let _ = err_tx_clone.send(std::io::Error::other(err.to_string())).await;
+                        return Err::<(), Error>(err);
+                    }
+                };
+
+                {
+                    let mut clone_parts_info = clone_parts_info.write().unwrap();
+                    clone_parts_info.entry(part_number).or_insert(obj_part);
+                }
+
+                let _ = clone_bufs_tx.send(buf).await;
+                Ok::<(), Error>(())
+            });
+
+            total_uploaded_size += length as i64;
+        }
+
+        let results = join_all(futures).await;
+        for result in results {
+            result?;
+        }
+
+        select! {
+            err = err_rx.recv() => {
+                return Err(err.unwrap_or_else(|| std::io::Error::other("Unknown error received from channel")));
+            }
+            else => (),
+        }
+
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        // Same inclusive collection as the serial path: parts are keyed 1..=total_parts_count, so
+        // the exclusive `1..total_parts_count` bound dropped the final part (and produced zero
+        // parts for a single-part upload), silently truncating the object.
+        let parts_snapshot = parts_info.read().unwrap().clone();
+        let mut all_parts = collect_complete_parts(&parts_snapshot, total_parts_count)?;
+        for part in &all_parts {
+            compl_multipart_upload.parts.push(CompletePart {
+                etag: part.etag.clone(),
+                part_num: part.part_num,
+                checksum_crc32: part.checksum_crc32.clone(),
+                checksum_crc32c: part.checksum_crc32c.clone(),
+                checksum_sha1: part.checksum_sha1.clone(),
+                checksum_sha256: part.checksum_sha256.clone(),
+                checksum_crc64nvme: part.checksum_crc64nvme.clone(),
+                ..Default::default()
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+
+        let mut opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum: opts.auto_checksum,
+            ..Default::default()
+        };
+        apply_auto_checksum(&mut opts, &mut all_parts);
+
+        let mut upload_info = self
+            .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
+            .await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+
+    pub async fn put_object_gcs(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.send_content_md5 = false;
+        }
+
+        let md5_base64: String = "".to_string();
+        let progress_reader = reader; //newHook(reader, opts.progress);
+
+        self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts)
+            .await
+    }
+
+    pub async fn put_object_do(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        reader: ReaderImpl,
+        md5_base64: &str,
+        sha256_hex: &str,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let custom_header = opts.header();
+
+        let mut req_metadata = RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            object_name: object_name.to_string(),
+            custom_header,
+            content_body: reader,
+            content_length: size,
+            content_md5_base64: md5_base64.to_string(),
+            content_sha256_hex: sha256_hex.to_string(),
+            stream_sha256: !opts.disable_content_sha256,
+            add_crc: Default::default(),
+            bucket_location: Default::default(),
+            pre_sign_url: Default::default(),
+            query_values: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            expires: Default::default(),
+            trailer: Default::default(),
+        };
+        let mut add_crc = false; //self.trailing_header_support && md5_base64 == "" && !s3utils.IsGoogleEndpoint(self.endpoint_url) && (opts.disable_content_sha256 || self.secure);
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            req_metadata.add_crc = opts.checksum;
+        } else if add_crc {
+            for (k, _) in opts.user_metadata {
+                if k.to_lowercase().starts_with("x-amz-checksum-") {
+                    add_crc = false;
+                }
+            }
+            if add_crc {
+                opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
+                req_metadata.add_crc = opts.auto_checksum;
+            }
+        }
+
+        if opts.internal.source_version_id != "" {
+            if !opts.internal.source_version_id.is_empty() {
+                if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) {
+                    return Err(std::io::Error::other(err_invalid_argument(&err.to_string())));
+                }
+            }
+            let mut url_values = HashMap::new();
+            url_values.insert("versionId".to_string(), opts.internal.source_version_id);
+            req_metadata.query_values = url_values;
+        }
+
+        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
+
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                resp_status,
+                &h,
+                vec![],
+                bucket_name,
+                object_name,
+            )));
+        }
+
+        let exp_time = resp
+            .headers()
+            .get(X_AMZ_EXPIRATION)
+            .and_then(|v| v.to_str().ok())
+            .and_then(|s| OffsetDateTime::parse(s, ISO8601_DATEFORMAT).ok())
+            .unwrap_or_else(OffsetDateTime::now_utc);
+        let rule_id = "".to_string();
+        let h = resp.headers();
+        Ok(UploadInfo {
+            bucket: bucket_name.to_string(),
+            key: object_name.to_string(),
+            etag: trim_etag(h.get("ETag").and_then(|v| v.to_str().ok()).unwrap_or("")),
+
+            version_id: self.legacy_remote_version_id(h)?,
+            size,
+            expiration: exp_time,
+            expiration_rule_id: rule_id,
+            checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
+                h_checksum_crc32.to_str().unwrap_or("").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
+                h_checksum_crc32c.to_str().unwrap_or("").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
+                h_checksum_sha1.to_str().unwrap_or("").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
+                h_checksum_sha256.to_str().unwrap_or("").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
+                h_checksum_crc64nvme.to_str().unwrap_or("").to_string()
+            } else {
+                "".to_string()
+            },
+            ..Default::default()
+        })
+    }
+}
+
+/// Collect the uploaded parts for CompleteMultipartUpload in ascending part order.
+///
+/// Parts are keyed `1..=total_parts_count` during upload (see the upload loop that inserts each
+/// part), so every one — including the final part — must be collected. The previous exclusive
+/// `1..total_parts_count` bound dropped the last part, silently truncating the completed object,
+/// and collected zero parts for a single-part upload.
+fn collect_complete_parts(parts_info: &HashMap<i64, ObjectPart>, total_parts_count: i64) -> Result<Vec<ObjectPart>, Error> {
+    let mut all_parts = Vec::with_capacity(parts_info.len());
+    for i in 1..=total_parts_count {
+        let part = parts_info
+            .get(&i)
+            .ok_or_else(|| Error::other(format!("missing uploaded part {i} of {total_parts_count}")))?;
+        all_parts.push(part.clone());
+    }
+    Ok(all_parts)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{ObjectPart, ReaderImpl, collect_complete_parts, lock_md5_hasher, read_multipart_part};
+    use crate::transition_api::ObjectReader;
+    use bytes::Bytes;
+    use rustfs_utils::hash::HashAlgorithm;
+    use std::collections::HashMap;
+    use std::sync::{Arc, Mutex};
+
+    // Drive a reader through the same per-part loop the multipart stream uses and
+    // collect the size of every part. Regression for rustfs/rustfs#4811: the old
+    // `read_all()` per part drained the whole source into part 1.
+    async fn collect_part_sizes(mut reader: ReaderImpl, total: usize, part_size: usize, last_part_size: usize) -> Vec<usize> {
+        let parts = total.div_ceil(part_size);
+        let mut sizes = Vec::new();
+        for part_number in 1..=parts {
+            let want = if part_number == parts { last_part_size } else { part_size };
+            let buf = read_multipart_part(&mut reader, want).await.unwrap();
+            sizes.push(buf.len());
+        }
+        // Nothing must remain after the planned parts are consumed.
+        assert!(read_multipart_part(&mut reader, part_size).await.unwrap().is_empty());
+        sizes
+    }
+
+    #[tokio::test]
+    async fn read_multipart_part_splits_streamed_object_body_evenly() {
+        // 250 bytes at part_size 100 -> parts [100, 100, 50], mirroring the
+        // >128 MiB / 128 MiB split from the bug report on a small deterministic
+        // stream.
+        let total = 250usize;
+        let (mut w, r) = tokio::io::duplex(64);
+        tokio::spawn(async move {
+            use tokio::io::AsyncWriteExt;
+            let data: Vec<u8> = (0..total).map(|i| i as u8).collect();
+            w.write_all(&data).await.unwrap();
+        });
+        let reader = ReaderImpl::ObjectBody(ObjectReader::new(r));
+
+        let sizes = collect_part_sizes(reader, total, 100, 50).await;
+        assert_eq!(sizes, vec![100, 100, 50]);
+    }
+
+    #[tokio::test]
+    async fn read_multipart_part_splits_in_memory_body_evenly() {
+        let total = 250usize;
+        let data: Vec<u8> = (0..total).map(|i| i as u8).collect();
+        let reader = ReaderImpl::Body(Bytes::from(data));
+
+        let sizes = collect_part_sizes(reader, total, 100, 50).await;
+        assert_eq!(sizes, vec![100, 100, 50]);
+    }
+
+    #[tokio::test]
+    async fn read_multipart_part_stops_at_eof_without_overrun() {
+        // Reader shorter than the requested part size must return only what is
+        // available, not block or pad.
+        let (mut w, r) = tokio::io::duplex(64);
+        tokio::spawn(async move {
+            use tokio::io::AsyncWriteExt;
+            w.write_all(&[1u8; 30]).await.unwrap();
+        });
+        let mut reader = ReaderImpl::ObjectBody(ObjectReader::new(r));
+        let buf = read_multipart_part(&mut reader, 100).await.unwrap();
+        assert_eq!(buf.len(), 30);
+    }
+
+    fn parts_map(n: i64) -> HashMap<i64, ObjectPart> {
+        let mut m = HashMap::new();
+        for i in 1..=n {
+            m.insert(
+                i,
+                ObjectPart {
+                    part_num: i,
+                    ..Default::default()
+                },
+            );
+        }
+        m
+    }
+
+    #[test]
+    fn collects_every_part_including_the_last() {
+        let collected: Vec<i64> = collect_complete_parts(&parts_map(3), 3)
+            .expect("all parts present")
+            .iter()
+            .map(|p| p.part_num)
+            .collect();
+        assert_eq!(collected, vec![1, 2, 3], "CompleteMultipartUpload must include the final part");
+    }
+
+    #[test]
+    fn single_part_upload_submits_one_part() {
+        let collected = collect_complete_parts(&parts_map(1), 1).expect("single part present");
+        assert_eq!(collected.len(), 1, "a single-part object must submit exactly one part, not zero");
+        assert_eq!(collected[0].part_num, 1);
+    }
+
+    #[test]
+    fn missing_part_is_an_error_not_a_panic() {
+        let mut m = parts_map(3);
+        m.remove(&2);
+        assert!(
+            collect_complete_parts(&m, 3).is_err(),
+            "a gap in the parts map must be an error, not a panic"
+        );
+    }
+
+    #[test]
+    fn poisoned_md5_state_fails_closed() {
+        let hasher = Arc::new(Mutex::new(Some(HashAlgorithm::Md5)));
+        let poison_target = Arc::clone(&hasher);
+        let _ = std::thread::spawn(move || {
+            let _guard = poison_target.lock().expect("fresh mutex should lock");
+            panic!("poison MD5 state");
+        })
+        .join();
+
+        let error = lock_md5_hasher(&hasher).expect_err("poisoned hash state must not be reused");
+        assert_eq!(error.kind(), std::io::ErrorKind::Other);
+    }
+}

crates/s3-client/src/api_remove.rs

@@ -0,0 +1,866 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use http::{HeaderMap, HeaderValue, Method, StatusCode};
+use http_body_util::BodyExt;
+use hyper::body::Body;
+use hyper::body::Bytes;
+use rustfs_utils::HashAlgorithm;
+use s3s::S3ErrorCode;
+use s3s::dto::ReplicationStatus;
+use s3s::header::{X_AMZ_BYPASS_GOVERNANCE_RETENTION, X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
+use serde::Deserialize;
+use std::fmt::Display;
+use std::{
+    collections::{HashMap, HashSet},
+    sync::Arc,
+};
+use time::OffsetDateTime;
+use tokio::sync::mpsc::{self, Receiver, Sender};
+use tracing::Instrument;
+
+use crate::transition_api::ObjectInfo;
+use crate::utils::base64_encode;
+use crate::{
+    api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
+    api_s3_datatypes::{DeleteMultiObjects, DeleteObject},
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
+};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+
+pub struct RemoveBucketOptions {
+    _forced_delete: bool,
+}
+
+const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024;
+
+#[derive(Debug)]
+pub struct AdvancedRemoveOptions {
+    pub replication_delete_marker: bool,
+    pub replication_status: ReplicationStatus,
+    pub replication_mtime: Option<OffsetDateTime>,
+    pub replication_request: bool,
+    pub replication_validity_check: bool,
+}
+
+impl Default for AdvancedRemoveOptions {
+    fn default() -> Self {
+        Self {
+            replication_delete_marker: false,
+            replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
+            replication_mtime: None,
+            replication_request: false,
+            replication_validity_check: false,
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct RemoveObjectOptions {
+    pub force_delete: bool,
+    pub governance_bypass: bool,
+    pub version_id: String,
+    pub internal: AdvancedRemoveOptions,
+}
+
+impl TransitionClient {
+    pub async fn remove_bucket_with_options(&self, bucket_name: &str, opts: &RemoveBucketOptions) -> Result<(), std::io::Error> {
+        let headers = HeaderMap::new();
+
+        let resp = self
+            .execute_method(
+                Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: headers,
+                    object_name: "".to_string(),
+                    query_values: Default::default(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        {
+            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                bucket_loc_cache.delete(bucket_name);
+            }
+        }
+        Ok(())
+    }
+
+    pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> {
+        let resp = self
+            .execute_method(
+                http::Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: Default::default(),
+                    object_name: "".to_string(),
+                    query_values: Default::default(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        {
+            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                bucket_loc_cache.delete(bucket_name);
+            }
+        }
+
+        Ok(())
+    }
+
+    pub async fn remove_object(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Option<std::io::Error> {
+        self.remove_object_inner(bucket_name, object_name, opts).await.err()
+    }
+
+    pub async fn remove_object_inner(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        opts: RemoveObjectOptions,
+    ) -> Result<RemoveObjectResult, std::io::Error> {
+        let mut url_values = HashMap::new();
+
+        if opts.version_id != "" {
+            url_values.insert("versionId".to_string(), opts.version_id.clone());
+        }
+
+        let mut headers = HeaderMap::new();
+
+        if opts.governance_bypass {
+            headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, HeaderValue::from_static("true")); //amzBypassGovernance
+        }
+
+        let resp = self
+            .execute_method(
+                http::Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: object_name.to_string(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    query_values: url_values,
+                    custom_header: headers,
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        Ok(RemoveObjectResult {
+            object_name: object_name.to_string(),
+            object_version_id: opts.version_id,
+            delete_marker: resp.headers().get(X_AMZ_DELETE_MARKER).map_or(false, |v| v == "true"),
+            delete_marker_version_id: self.legacy_remote_version_id(resp.headers())?,
+            ..Default::default()
+        })
+    }
+
+    pub async fn remove_objects_with_result(
+        self: Arc<Self>,
+        bucket_name: &str,
+        objects_rx: Receiver<ObjectInfo>,
+        opts: RemoveObjectsOptions,
+    ) -> Receiver<RemoveObjectResult> {
+        let (result_tx, result_rx) = mpsc::channel(1);
+
+        let self_clone = Arc::clone(&self);
+        let bucket_name_owned = bucket_name.to_string();
+
+        tokio::spawn(
+            async move {
+                self_clone
+                    .remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts)
+                    .await;
+            }
+            .instrument(tracing::Span::current()),
+        );
+        result_rx
+    }
+
+    pub async fn remove_objects(
+        self: Arc<Self>,
+        bucket_name: &str,
+        objects_rx: Receiver<ObjectInfo>,
+        opts: RemoveObjectsOptions,
+    ) -> Receiver<RemoveObjectError> {
+        let (error_tx, error_rx) = mpsc::channel(1);
+
+        let self_clone = Arc::clone(&self);
+        let bucket_name_owned = bucket_name.to_string();
+
+        let (result_tx, mut result_rx) = mpsc::channel(1);
+        tokio::spawn(
+            async move {
+                self_clone
+                    .remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts)
+                    .await;
+            }
+            .instrument(tracing::Span::current()),
+        );
+        tokio::spawn(
+            async move {
+                while let Some(res) = result_rx.recv().await {
+                    if res.err.is_none() {
+                        continue;
+                    }
+                    error_tx
+                        .send(RemoveObjectError {
+                            object_name: res.object_name,
+                            version_id: res.object_version_id,
+                            err: res.err,
+                            ..Default::default()
+                        })
+                        .await;
+                }
+            }
+            .instrument(tracing::Span::current()),
+        );
+
+        error_rx
+    }
+
+    pub async fn remove_objects_inner(
+        &self,
+        bucket_name: &str,
+        mut objects_rx: Receiver<ObjectInfo>,
+        result_tx: &Sender<RemoveObjectResult>,
+        opts: RemoveObjectsOptions,
+    ) -> Result<(), std::io::Error> {
+        let max_entries = 1000;
+        let mut finish = false;
+        let mut url_values = HashMap::new();
+        url_values.insert("delete".to_string(), "".to_string());
+
+        loop {
+            if finish {
+                break;
+            }
+            let mut count = 0;
+            let mut batch = Vec::<ObjectInfo>::new();
+
+            while let Some(object) = objects_rx.recv().await {
+                if has_invalid_xml_char(&object.name) {
+                    let remove_result = self
+                        .remove_object_inner(
+                            bucket_name,
+                            &object.name,
+                            RemoveObjectOptions {
+                                version_id: object.version_id.map(|id| id.to_string()).unwrap_or_default(),
+                                governance_bypass: opts.governance_bypass,
+                                ..Default::default()
+                            },
+                        )
+                        .await?;
+                    let remove_result_clone = remove_result.clone();
+                    if let Some(err) = &remove_result.err {
+                        match to_error_response(err).code {
+                            S3ErrorCode::InvalidArgument | S3ErrorCode::NoSuchVersion => {
+                                continue;
+                            }
+                            _ => (),
+                        }
+                        result_tx.send(remove_result_clone.clone()).await;
+                    }
+
+                    result_tx.send(remove_result_clone).await;
+                    continue;
+                }
+
+                batch.push(object);
+                count += 1;
+                if count >= max_entries {
+                    break;
+                }
+            }
+            if count == 0 {
+                break;
+            }
+            if count < max_entries {
+                finish = true;
+            }
+
+            let mut headers = HeaderMap::new();
+            if opts.governance_bypass {
+                headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, HeaderValue::from_static("true"));
+            }
+
+            let remove_bytes = generate_remove_multi_objects_request(&batch);
+            let resp = self
+                .execute_method(
+                    http::Method::POST,
+                    &mut RequestMetadata {
+                        bucket_name: bucket_name.to_string(),
+                        query_values: url_values.clone(),
+                        content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())),
+                        content_length: remove_bytes.len() as i64,
+                        content_md5_base64: base64_encode(&HashAlgorithm::Md5.hash_encode(&remove_bytes).as_ref()),
+                        content_sha256_hex: rustfs_utils::hex(HashAlgorithm::SHA256.hash_encode(&remove_bytes)),
+                        custom_header: headers,
+                        object_name: "".to_string(),
+                        stream_sha256: false,
+                        trailer: HeaderMap::new(),
+                        pre_sign_url: Default::default(),
+                        add_crc: Default::default(),
+                        extra_pre_sign_header: Default::default(),
+                        bucket_location: Default::default(),
+                        expires: Default::default(),
+                    },
+                )
+                .await?;
+
+            let mut body_vec = Vec::new();
+            let mut body = resp.into_body();
+            while let Some(frame) = body.frame().await {
+                let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+                if let Some(data) = frame.data_ref() {
+                    body_vec.extend_from_slice(data);
+                }
+            }
+            process_remove_multi_objects_response(
+                ReaderImpl::Body(Bytes::from(body_vec)),
+                bucket_name,
+                &batch,
+                result_tx.clone(),
+            )
+            .await;
+        }
+        Ok(())
+    }
+
+    pub async fn remove_incomplete_upload(&self, bucket_name: &str, object_name: &str) -> Result<(), std::io::Error> {
+        let upload_ids = self.find_upload_ids(bucket_name, object_name)?;
+        for upload_id in upload_ids {
+            self.abort_multipart_upload(bucket_name, object_name, &upload_id).await?;
+        }
+
+        Ok(())
+    }
+
+    pub async fn abort_multipart_upload(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        upload_id: &str,
+    ) -> Result<(), std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("uploadId".to_string(), upload_id.to_string());
+
+        let resp = self
+            .execute_method(
+                http::Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: object_name.to_string(),
+                    query_values: url_values,
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: HeaderMap::new(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
+        //if resp.is_some() {
+        if resp.status() != StatusCode::NO_CONTENT {
+            let error_response: ErrorResponse;
+            match resp.status() {
+                StatusCode::NOT_FOUND => {
+                    error_response = ErrorResponse {
+                        code: S3ErrorCode::NoSuchUpload,
+                        message: "The specified multipart upload does not exist.".to_string(),
+                        bucket_name: bucket_name.to_string(),
+                        key: object_name.to_string(),
+                        request_id: resp
+                            .headers()
+                            .get("x-amz-request-id")
+                            .and_then(|v| v.to_str().ok())
+                            .unwrap_or_default()
+                            .to_string(),
+                        host_id: resp
+                            .headers()
+                            .get("x-amz-id-2")
+                            .and_then(|v| v.to_str().ok())
+                            .unwrap_or_default()
+                            .to_string(),
+                        region: resp
+                            .headers()
+                            .get("x-amz-bucket-region")
+                            .and_then(|v| v.to_str().ok())
+                            .unwrap_or_default()
+                            .to_string(),
+                        ..Default::default()
+                    };
+                }
+                _ => {
+                    return Err(std::io::Error::other(http_resp_to_error_response(
+                        resp_status,
+                        &h,
+                        vec![],
+                        bucket_name,
+                        object_name,
+                    )));
+                }
+            }
+            return Err(std::io::Error::other(error_response));
+        }
+        //}
+        Ok(())
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct RemoveObjectError {
+    #[allow(dead_code, reason = "written but never read back (backlog#1823)")]
+    object_name: String,
+    #[allow(dead_code, reason = "written but never read back (backlog#1823)")]
+    version_id: String,
+    err: Option<std::io::Error>,
+}
+
+impl Display for RemoveObjectError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        if let Some(err) = &self.err {
+            write!(f, "{}", err.to_string())
+        } else {
+            write!(f, "unexpected remove object error result")
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct RemoveObjectResult {
+    pub object_name: String,
+    pub object_version_id: String,
+    pub delete_marker: bool,
+    pub delete_marker_version_id: String,
+    pub err: Option<std::io::Error>,
+}
+
+impl Clone for RemoveObjectResult {
+    fn clone(&self) -> Self {
+        Self {
+            object_name: self.object_name.clone(),
+            object_version_id: self.object_version_id.clone(),
+            delete_marker: self.delete_marker,
+            delete_marker_version_id: self.delete_marker_version_id.clone(),
+            err: None, //err
+        }
+    }
+}
+
+pub struct RemoveObjectsOptions {
+    pub governance_bypass: bool,
+}
+
+pub fn generate_remove_multi_objects_request(objects: &[ObjectInfo]) -> Vec<u8> {
+    let escape_xml = |value: &str| -> String {
+        value
+            .replace('&', "&amp;")
+            .replace('<', "&lt;")
+            .replace('>', "&gt;")
+            .replace('\"', "&quot;")
+            .replace('\'', "&apos;")
+    };
+
+    let request: DeleteMultiObjects = DeleteMultiObjects {
+        quiet: false,
+        objects: objects
+            .iter()
+            .map(|object| DeleteObject {
+                key: object.name.clone(),
+                version_id: object.version_id.map(|v| v.to_string()).unwrap_or_default(),
+            })
+            .collect(),
+    };
+
+    match request.marshal_msg() {
+        Ok(body) => body.into_bytes(),
+        Err(_) => {
+            let mut body = String::new();
+            body.push_str("<Delete><Quiet>false</Quiet>");
+            for object in objects {
+                body.push_str("<Object>");
+                body.push_str("<Key>");
+                body.push_str(&escape_xml(&object.name));
+                body.push_str("</Key>");
+                if object.version_id.is_some() {
+                    body.push_str("<VersionId>");
+                    body.push_str(&escape_xml(&object.version_id.as_ref().map(|v| v.to_string()).unwrap_or_default()));
+                    body.push_str("</VersionId>");
+                }
+                body.push_str("</Object>");
+            }
+            body.push_str("</Delete>");
+            body.into_bytes()
+        }
+    }
+}
+
+pub async fn process_remove_multi_objects_response(
+    body: ReaderImpl,
+    bucket_name: &str,
+    objects: &[ObjectInfo],
+    result_tx: Sender<RemoveObjectResult>,
+) {
+    let mut body_vec = Vec::new();
+    match body {
+        ReaderImpl::Body(content_body) => {
+            body_vec = content_body.to_vec();
+        }
+        ReaderImpl::ObjectBody(mut object_body) => match object_body.read_all().await {
+            Ok(content) => {
+                body_vec = content;
+            }
+            Err(err) => {
+                for object in objects {
+                    let version_id = object.version_id.as_ref().map(|v| v.to_string()).unwrap_or_default();
+                    let _ = result_tx
+                        .send(RemoveObjectResult {
+                            object_name: object.name.clone(),
+                            object_version_id: version_id,
+                            err: Some(std::io::Error::other(ErrorResponse {
+                                code: S3ErrorCode::Custom("ReadDeleteResponseFailed".into()),
+                                message: format!("read multi remove response failed: {err}"),
+                                bucket_name: bucket_name.to_string(),
+                                key: object.name.clone(),
+                                resource: "".to_string(),
+                                request_id: "".to_string(),
+                                host_id: "".to_string(),
+                                region: "".to_string(),
+                                server: "".to_string(),
+                                status_code: StatusCode::OK,
+                            })),
+                            ..Default::default()
+                        })
+                        .await;
+                }
+                return;
+            }
+        },
+    }
+
+    #[derive(Debug, Deserialize)]
+    #[serde(rename = "DeleteResult")]
+    struct Deleted {
+        #[serde(rename = "Deleted", default)]
+        deleted: Vec<DeleteResultDeleted>,
+        #[serde(rename = "Error", default)]
+        error: Vec<DeleteResultError>,
+    }
+
+    #[derive(Debug, Deserialize)]
+    struct DeleteResultDeleted {
+        #[serde(rename = "Key")]
+        key: String,
+        #[serde(rename = "VersionId", default)]
+        version_id: String,
+        #[serde(rename = "DeleteMarker")]
+        deletemarker: bool,
+        #[serde(rename = "DeleteMarkerVersionId", default)]
+        deletemarker_version_id: String,
+    }
+
+    #[derive(Debug, Deserialize)]
+    struct DeleteResultError {
+        #[serde(rename = "Key")]
+        key: String,
+        #[serde(rename = "VersionId", default)]
+        version_id: String,
+        #[serde(rename = "Code")]
+        code: String,
+        #[serde(rename = "Message")]
+        message: String,
+    }
+
+    let mut pending = HashSet::with_capacity(objects.len());
+    for object in objects {
+        pending.insert((object.name.clone(), object.version_id.as_ref().map(|v| v.to_string()).unwrap_or_default()));
+    }
+
+    let body = String::from_utf8_lossy(&body_vec).into_owned();
+    let parsed: Deleted = match quick_xml::de::from_str(&body) {
+        Ok(parsed) => parsed,
+        Err(err) => {
+            for object in objects {
+                let version_id = object.version_id.as_ref().map(|v| v.to_string()).unwrap_or_default();
+                let _ = result_tx
+                    .send(RemoveObjectResult {
+                        object_name: object.name.clone(),
+                        object_version_id: version_id,
+                        err: Some(std::io::Error::other(ErrorResponse {
+                            code: S3ErrorCode::Custom("UnmarshalDeleteResponseFailed".into()),
+                            message: format!(
+                                "unmarshal multi remove response failed: {err}; response_body={}",
+                                body.chars().take(DELETE_RESPONSE_PREVIEW_LEN).collect::<String>()
+                            ),
+                            bucket_name: bucket_name.to_string(),
+                            key: object.name.clone(),
+                            resource: "".to_string(),
+                            request_id: "".to_string(),
+                            host_id: "".to_string(),
+                            region: "".to_string(),
+                            server: "".to_string(),
+                            status_code: StatusCode::OK,
+                        })),
+                        ..Default::default()
+                    })
+                    .await;
+            }
+            return;
+        }
+    };
+
+    for deleted in parsed.deleted {
+        if !pending.remove(&(deleted.key.clone(), deleted.version_id.clone())) {
+            continue;
+        }
+
+        let _ = result_tx
+            .send(RemoveObjectResult {
+                object_name: deleted.key,
+                object_version_id: deleted.version_id,
+                delete_marker: deleted.deletemarker,
+                delete_marker_version_id: deleted.deletemarker_version_id,
+                err: None,
+            })
+            .await;
+    }
+
+    for removed in parsed.error {
+        if !pending.remove(&(removed.key.clone(), removed.version_id.clone())) {
+            continue;
+        }
+
+        let _ = result_tx
+            .send(RemoveObjectResult {
+                object_name: removed.key.clone(),
+                object_version_id: removed.version_id,
+                err: Some(std::io::Error::other(ErrorResponse {
+                    code: S3ErrorCode::Custom(removed.code.into()),
+                    message: removed.message,
+                    bucket_name: "".to_string(),
+                    key: removed.key,
+                    resource: "".to_string(),
+                    request_id: "".to_string(),
+                    host_id: "".to_string(),
+                    region: "".to_string(),
+                    server: "".to_string(),
+                    status_code: StatusCode::OK,
+                })),
+                ..Default::default()
+            })
+            .await;
+    }
+
+    for (object_name, object_version_id) in pending {
+        let bucket_name = bucket_name.to_string();
+        let object_name = object_name;
+        let object_version_id = object_version_id;
+        let error_message = format!(
+            "remove response did not contain an entry for object {} with version {}",
+            object_name, object_version_id
+        );
+
+        let _ = result_tx
+            .send(RemoveObjectResult {
+                object_name: object_name.clone(),
+                object_version_id: object_version_id.clone(),
+                err: Some(std::io::Error::other(ErrorResponse {
+                    code: S3ErrorCode::Custom("UnmatchedDeleteResponseEntry".into()),
+                    message: error_message,
+                    bucket_name,
+                    key: object_name,
+                    resource: "".to_string(),
+                    request_id: "".to_string(),
+                    host_id: "".to_string(),
+                    region: "".to_string(),
+                    server: "".to_string(),
+                    status_code: StatusCode::OK,
+                })),
+                ..Default::default()
+            })
+            .await;
+    }
+}
+
+fn has_invalid_xml_char(str: &str) -> bool {
+    false
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        credentials::{Credentials, SignatureType, Static, Value},
+        transition_api::{BucketLookupType, Options},
+    };
+    use tokio::{
+        io::{AsyncReadExt, AsyncWriteExt},
+        net::TcpListener,
+    };
+
+    async fn capture_delete_objects_sha256_header() -> Option<(String, tokio::task::JoinHandle<String>)> {
+        let listener = match TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let task = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.unwrap();
+            let mut request = Vec::new();
+            let mut buffer = [0; 1024];
+            loop {
+                let read = stream.read(&mut buffer).await.unwrap();
+                assert_ne!(read, 0, "connection closed before request headers were received");
+                request.extend_from_slice(&buffer[..read]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
+            }
+
+            let request = String::from_utf8_lossy(&request);
+            let sha256_header = request
+                .lines()
+                .find_map(|line| {
+                    let (name, value) = line.split_once(':')?;
+                    name.eq_ignore_ascii_case("x-amz-content-sha256")
+                        .then(|| value.trim().to_string())
+                })
+                .expect("delete objects request should include X-Amz-Content-Sha256");
+
+            let response_body = r#"<?xml version="1.0" encoding="UTF-8"?><DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Deleted><Key>object.txt</Key></Deleted></DeleteResult>"#;
+            let response = format!(
+                "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
+                response_body.len(),
+                response_body
+            );
+            stream.write_all(response.as_bytes()).await.unwrap();
+            sha256_header
+        });
+
+        Some((endpoint, task))
+    }
+
+    #[tokio::test]
+    async fn multi_object_delete_request_uses_lowercase_hex_sha256_header() {
+        let objects = vec![ObjectInfo {
+            name: "object.txt".to_string(),
+            ..Default::default()
+        }];
+        let body = generate_remove_multi_objects_request(&objects);
+        let expected = rustfs_utils::hex(HashAlgorithm::SHA256.hash_encode(&body));
+        let Some((endpoint, header_task)) = capture_delete_objects_sha256_header().await else {
+            return;
+        };
+        let client = TransitionClient::new(
+            &endpoint,
+            Options {
+                creds: Credentials::new(Static(Value {
+                    access_key_id: "access-key".to_string(),
+                    secret_access_key: "secret-key".to_string(),
+                    signer_type: SignatureType::SignatureV4,
+                    ..Default::default()
+                })),
+                region: "us-east-1".to_string(),
+                bucket_lookup: BucketLookupType::BucketLookupPath,
+                max_retries: 1,
+                ..Default::default()
+            },
+            "",
+        )
+        .await
+        .unwrap();
+        let (objects_tx, objects_rx) = mpsc::channel(1);
+        let (result_tx, mut result_rx) = mpsc::channel(1);
+
+        objects_tx.send(objects[0].clone()).await.unwrap();
+        drop(objects_tx);
+
+        client
+            .remove_objects_inner(
+                "bucket",
+                objects_rx,
+                &result_tx,
+                RemoveObjectsOptions {
+                    governance_bypass: false,
+                },
+            )
+            .await
+            .unwrap();
+        drop(result_tx);
+
+        let header = header_task.await.unwrap();
+
+        assert_eq!(header, expected);
+        assert_eq!(header.len(), 64);
+        assert!(
+            header
+                .bytes()
+                .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+        );
+        assert_ne!(header, base64_encode(&HashAlgorithm::SHA256.hash_encode(&body).as_ref()));
+        assert!(result_rx.recv().await.is_some());
+    }
+}

crates/s3-client/src/api_s3_datatypes.rs

@@ -0,0 +1,423 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use s3s::dto::Owner;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use time::OffsetDateTime;
+
+use crate::checksum::ChecksumMode;
+use crate::transition_api::ObjectMultipartInfo;
+use crate::utils::base64_decode;
+
+use super::transition_api;
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct CommonPrefix {
+    pub prefix: String,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct ListBucketV2Result {
+    pub common_prefixes: Vec<CommonPrefix>,
+    pub contents: Vec<transition_api::ObjectInfo>,
+    pub delimiter: String,
+    pub encoding_type: String,
+    pub is_truncated: bool,
+    pub max_keys: i64,
+    pub name: String,
+    pub next_continuation_token: String,
+    pub continuation_token: String,
+    pub prefix: String,
+    pub fetch_owner: String,
+    pub start_after: String,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct Version {
+    #[serde(rename = "ETag")]
+    pub etag: String,
+    pub is_latest: bool,
+    pub key: String,
+    pub size: i64,
+    pub storage_class: String,
+    pub version_id: String,
+    pub user_metadata: HashMap<String, String>,
+    pub user_tags: HashMap<String, String>,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct ListVersionsResult {
+    #[serde(rename = "Version")]
+    pub versions: Vec<Version>,
+    #[serde(rename = "DeleteMarker")]
+    pub delete_markers: Vec<Version>,
+    pub common_prefixes: Vec<CommonPrefix>,
+    pub name: String,
+    pub prefix: String,
+    pub delimiter: String,
+    pub max_keys: i64,
+    pub encoding_type: String,
+    pub is_truncated: bool,
+    pub key_marker: String,
+    pub version_id_marker: String,
+    pub next_key_marker: String,
+    pub next_version_id_marker: String,
+}
+
+#[allow(
+    dead_code,
+    reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
+)]
+pub struct ListBucketResult {
+    common_prefixes: Vec<CommonPrefix>,
+    contents: Vec<transition_api::ObjectInfo>,
+    delimiter: String,
+    encoding_type: String,
+    is_truncated: bool,
+    marker: String,
+    max_keys: i64,
+    name: String,
+    next_marker: String,
+    prefix: String,
+}
+
+#[allow(
+    dead_code,
+    reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
+)]
+pub struct ListMultipartUploadsResult {
+    bucket: String,
+    key_marker: String,
+    upload_id_marker: String,
+    next_key_marker: String,
+    next_upload_id_marker: String,
+    encoding_type: String,
+    max_uploads: i64,
+    is_truncated: bool,
+    uploads: Vec<ObjectMultipartInfo>,
+    prefix: String,
+    delimiter: String,
+    common_prefixes: Vec<CommonPrefix>,
+}
+
+#[allow(
+    dead_code,
+    reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
+)]
+pub struct Initiator {
+    id: String,
+    display_name: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct ObjectPart {
+    pub etag: String,
+    pub part_num: i64,
+    pub last_modified: OffsetDateTime,
+    pub size: i64,
+    pub checksum_crc32: String,
+    pub checksum_crc32c: String,
+    pub checksum_sha1: String,
+    pub checksum_sha256: String,
+    pub checksum_crc64nvme: String,
+}
+
+impl Default for ObjectPart {
+    fn default() -> Self {
+        ObjectPart {
+            etag: Default::default(),
+            part_num: 0,
+            last_modified: OffsetDateTime::now_utc(),
+            size: 0,
+            checksum_crc32: Default::default(),
+            checksum_crc32c: Default::default(),
+            checksum_sha1: Default::default(),
+            checksum_sha256: Default::default(),
+            checksum_crc64nvme: Default::default(),
+        }
+    }
+}
+
+impl ObjectPart {
+    fn checksum(&self, t: &ChecksumMode) -> String {
+        match t {
+            ChecksumMode::ChecksumCRC32C => {
+                return self.checksum_crc32c.clone();
+            }
+            ChecksumMode::ChecksumCRC32 => {
+                return self.checksum_crc32.clone();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return self.checksum_sha1.clone();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return self.checksum_sha256.clone();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return self.checksum_crc64nvme.clone();
+            }
+            _ => {
+                return "".to_string();
+            }
+        }
+    }
+
+    pub fn checksum_raw(&self, t: &ChecksumMode) -> Result<Vec<u8>, std::io::Error> {
+        let b = self.checksum(t);
+        if b == "" {
+            return Err(std::io::Error::other("no checksum set"));
+        }
+        let decoded = match base64_decode(b.as_bytes()) {
+            Ok(b) => b,
+            Err(e) => return Err(std::io::Error::other(e)),
+        };
+        if decoded.len() != t.raw_byte_len() as usize {
+            return Err(std::io::Error::other("checksum length mismatch"));
+        }
+        Ok(decoded)
+    }
+}
+
+pub struct ListObjectPartsResult {
+    pub bucket: String,
+    pub key: String,
+    pub upload_id: String,
+    pub initiator: Initiator,
+    pub owner: Owner,
+    pub storage_class: String,
+    pub part_number_marker: i32,
+    pub next_part_number_marker: i32,
+    pub max_parts: i32,
+    pub checksum_algorithm: String,
+    pub checksum_type: String,
+    pub is_truncated: bool,
+    pub object_parts: Vec<ObjectPart>,
+    pub encoding_type: String,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct InitiateMultipartUploadResult {
+    pub bucket: String,
+    pub key: String,
+    pub upload_id: String,
+}
+
+#[derive(Debug, Default)]
+pub struct CompleteMultipartUploadResult {
+    pub location: String,
+    pub bucket: String,
+    pub key: String,
+    pub etag: String,
+    pub checksum_crc32: String,
+    pub checksum_crc32c: String,
+    pub checksum_sha1: String,
+    pub checksum_sha256: String,
+    pub checksum_crc64nvme: String,
+}
+
+// Field renames drive the CompleteMultipartUpload XML body sent to the remote.
+// Without them quick-xml emits the Rust field names (<part_num>/<etag>), so the
+// remote parses zero <Part> elements and completes a 0-byte object while still
+// returning 200 — the multipart transition silently produces an empty object
+// (rustfs/rustfs#4811). Empty checksum fields are skipped so an MD5-only
+// transition does not emit blank <ChecksumCRC32/> elements.
+#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
+pub struct CompletePart {
+    //api has
+    #[serde(rename = "PartNumber")]
+    pub part_num: i64,
+    #[serde(rename = "ETag")]
+    pub etag: String,
+    #[serde(rename = "ChecksumCRC32", skip_serializing_if = "String::is_empty")]
+    pub checksum_crc32: String,
+    #[serde(rename = "ChecksumCRC32C", skip_serializing_if = "String::is_empty")]
+    pub checksum_crc32c: String,
+    #[serde(rename = "ChecksumSHA1", skip_serializing_if = "String::is_empty")]
+    pub checksum_sha1: String,
+    #[serde(rename = "ChecksumSHA256", skip_serializing_if = "String::is_empty")]
+    pub checksum_sha256: String,
+    #[serde(rename = "ChecksumCRC64NVME", skip_serializing_if = "String::is_empty")]
+    pub checksum_crc64nvme: String,
+}
+
+impl CompletePart {
+    #[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
+    fn checksum(&self, t: &ChecksumMode) -> String {
+        match t {
+            ChecksumMode::ChecksumCRC32C => {
+                return self.checksum_crc32c.clone();
+            }
+            ChecksumMode::ChecksumCRC32 => {
+                return self.checksum_crc32.clone();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return self.checksum_sha1.clone();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return self.checksum_sha256.clone();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return self.checksum_crc64nvme.clone();
+            }
+            _ => {
+                return "".to_string();
+            }
+        }
+    }
+}
+
+#[derive(Debug, Default, serde::Serialize)]
+#[serde(rename = "CompleteMultipartUpload")]
+pub struct CompleteMultipartUpload {
+    #[serde(rename = "Part")]
+    pub parts: Vec<CompletePart>,
+}
+
+impl CompleteMultipartUpload {
+    pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
+        //let buf = serde_json::to_string(self)?;
+        let buf = match quick_xml::se::to_string(self) {
+            Ok(buf) => buf,
+            Err(e) => {
+                return Err(std::io::Error::other(e));
+            }
+        };
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
+        #[derive(Debug, Deserialize)]
+        struct WirePart {
+            #[serde(rename = "ETag")]
+            etag: String,
+            #[serde(rename = "PartNumber")]
+            part_num: i64,
+            #[serde(rename = "ChecksumCRC32")]
+            checksum_crc32: String,
+            #[serde(rename = "ChecksumCRC32C")]
+            checksum_crc32c: String,
+            #[serde(rename = "ChecksumSHA1")]
+            checksum_sha1: String,
+            #[serde(rename = "ChecksumSHA256")]
+            checksum_sha256: String,
+            #[serde(rename = "ChecksumCRC64NVME")]
+            checksum_crc64nvme: String,
+        }
+
+        #[derive(Debug, Deserialize)]
+        #[serde(rename = "CompleteMultipartUpload")]
+        struct WireCompleteMultipartUpload {
+            #[serde(rename = "Part", default)]
+            parts: Vec<WirePart>,
+        }
+
+        let body = String::from_utf8_lossy(buf);
+        let wire: WireCompleteMultipartUpload = quick_xml::de::from_str(&body)
+            .map_err(|err| std::io::Error::other(format!("failed to parse CompleteMultipartUpload XML: {err}; body: {body}")))?;
+
+        Ok(Self {
+            parts: wire
+                .parts
+                .into_iter()
+                .map(|p| CompletePart {
+                    etag: p.etag,
+                    part_num: p.part_num,
+                    checksum_crc32: p.checksum_crc32,
+                    checksum_crc32c: p.checksum_crc32c,
+                    checksum_sha1: p.checksum_sha1,
+                    checksum_sha256: p.checksum_sha256,
+                    checksum_crc64nvme: p.checksum_crc64nvme,
+                })
+                .collect(),
+        })
+    }
+}
+
+#[allow(
+    dead_code,
+    reason = "live via quick_xml::de::from_str in bucket_cache.rs; serde deserialization is not a construction (backlog#1823)"
+)]
+#[derive(serde::Serialize)]
+pub struct DeleteObject {
+    //api has
+    pub key: String,
+    pub version_id: String,
+}
+
+#[derive(serde::Serialize)]
+pub struct DeleteMultiObjects {
+    pub quiet: bool,
+    pub objects: Vec<DeleteObject>,
+}
+
+impl DeleteMultiObjects {
+    pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
+        //let buf = serde_json::to_string(self)?;
+        let buf = match quick_xml::se::to_string(self) {
+            Ok(buf) => buf,
+            Err(e) => {
+                return Err(std::io::Error::other(e));
+            }
+        };
+
+        Ok(buf)
+    }
+
+    #[allow(dead_code, reason = "MinIO-parity XML helper with no caller in this port (backlog#1823)")]
+    pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
+        #[derive(Debug, Deserialize)]
+        struct WireDeleteObject {
+            #[serde(rename = "Key")]
+            key: String,
+            #[serde(rename = "VersionId")]
+            version_id: String,
+        }
+
+        #[derive(Debug, Deserialize)]
+        #[serde(rename = "Delete")]
+        struct WireDeleteMultiObjects {
+            #[serde(rename = "Quiet", default)]
+            quiet: bool,
+            #[serde(rename = "Object", default)]
+            objects: Vec<WireDeleteObject>,
+        }
+
+        let body = String::from_utf8_lossy(buf);
+        let wire: WireDeleteMultiObjects = quick_xml::de::from_str(&body).map_err(|err| std::io::Error::other(err))?;
+
+        Ok(Self {
+            quiet: wire.quiet,
+            objects: wire
+                .objects
+                .into_iter()
+                .map(|o| DeleteObject {
+                    key: o.key,
+                    version_id: o.version_id,
+                })
+                .collect(),
+        })
+    }
+}

crates/s3-client/src/api_stat.rs

@@ -0,0 +1,344 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use http::{HeaderMap, HeaderValue, StatusCode};
+use http_body_util::BodyExt;
+use hyper::body::Body;
+use hyper::body::Bytes;
+use rustfs_utils::EMPTY_STRING_SHA256_HASH;
+use std::{collections::HashMap, str::FromStr};
+use tokio::io::BufReader;
+use tracing::warn;
+use uuid::Uuid;
+
+use crate::{
+    api_error_response::{ErrorResponse, err_invalid_argument, http_resp_to_error_response},
+    api_get_options::GetObjectOptions,
+    transition_api::{
+        ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, collect_response_body, to_object_info_for_provider,
+    },
+};
+use s3s::{
+    dto::{BucketVersioningStatus, MFADelete, VersioningConfiguration},
+    header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID},
+};
+
+const S3_XML_NAMESPACE: &str = "http://s3.amazonaws.com/doc/2006-03-01/";
+
+#[derive(serde::Deserialize)]
+#[serde(deny_unknown_fields)]
+struct StrictVersioningConfiguration {
+    #[serde(rename = "@xmlns")]
+    namespace: Option<String>,
+    #[serde(rename = "MfaDelete")]
+    mfa_delete: Option<String>,
+    #[serde(rename = "Status")]
+    status: Option<String>,
+}
+
+#[derive(serde::Deserialize)]
+enum StrictVersioningResponse {
+    VersioningConfiguration(StrictVersioningConfiguration),
+}
+
+fn parse_bucket_versioning_response(
+    status: StatusCode,
+    headers: &HeaderMap,
+    body: Vec<u8>,
+    bucket_name: &str,
+) -> Result<VersioningConfiguration, std::io::Error> {
+    if status != StatusCode::OK {
+        return Err(std::io::Error::other(http_resp_to_error_response(status, headers, body, bucket_name, "")));
+    }
+
+    let StrictVersioningResponse::VersioningConfiguration(parsed) =
+        quick_xml::de::from_reader(body.as_slice()).map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?;
+    if parsed
+        .namespace
+        .as_deref()
+        .is_some_and(|namespace| namespace != S3_XML_NAMESPACE)
+    {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidData,
+            "unexpected GetBucketVersioning XML namespace",
+        ));
+    }
+    Ok(VersioningConfiguration {
+        mfa_delete: parsed.mfa_delete.map(MFADelete::from),
+        status: parsed.status.map(BucketVersioningStatus::from),
+        ..Default::default()
+    })
+}
+
+impl TransitionClient {
+    pub async fn bucket_exists(&self, bucket_name: &str) -> Result<bool, std::io::Error> {
+        let resp = self
+            .execute_method(
+                http::Method::HEAD,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: "".to_string(),
+                    query_values: HashMap::new(),
+                    custom_header: HeaderMap::new(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    content_md5_base64: "".to_string(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await;
+
+        if let Ok(resp) = resp {
+            if resp.status() != http::StatusCode::OK {
+                return Ok(false);
+            }
+
+            let resp_status = resp.status();
+            let h = resp.headers().clone();
+
+            let mut body_vec = Vec::new();
+            let mut body = resp.into_body();
+            while let Some(frame) = body.frame().await {
+                let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+                if let Some(data) = frame.data_ref() {
+                    body_vec.extend_from_slice(data);
+                }
+            }
+            let resperr = http_resp_to_error_response(resp_status, &h, body_vec, bucket_name, "");
+
+            warn!("bucket exists, resperr: {:?}", resperr);
+            /*if to_error_response(resperr).code == "NoSuchBucket" {
+                return Ok(false);
+            }
+            if resp.status_code() != http::StatusCode::OK {
+                return Ok(false);
+            }*/
+        }
+        Ok(true)
+    }
+
+    pub async fn get_bucket_versioning(&self, bucket_name: &str) -> Result<VersioningConfiguration, std::io::Error> {
+        let mut query_values = HashMap::new();
+        query_values.insert("versioning".to_string(), "".to_string());
+        let resp = self
+            .execute_method(
+                http::Method::GET,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: "".to_string(),
+                    query_values,
+                    custom_header: HeaderMap::new(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    content_md5_base64: "".to_string(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await;
+
+        match resp {
+            Ok(resp) => {
+                let resp_status = resp.status();
+                let h = resp.headers().clone();
+
+                let body_vec = collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE).await?;
+                parse_bucket_versioning_response(resp_status, &h, body_vec, bucket_name)
+            }
+
+            Err(err) => Err(std::io::Error::other(err)),
+        }
+    }
+
+    pub async fn stat_object(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        opts: &GetObjectOptions,
+    ) -> Result<ObjectInfo, std::io::Error> {
+        let mut headers = opts.header();
+        if opts.internal.replication_delete_marker {
+            headers.insert("X-Source-DeleteMarker", HeaderValue::from_str("true").expect("operation should succeed"));
+        }
+        if opts.internal.is_replication_ready_for_delete_marker {
+            headers.insert(
+                "X-Check-Replication-Ready",
+                HeaderValue::from_str("true").expect("operation should succeed"),
+            );
+        }
+
+        let resp = self
+            .execute_method(
+                http::Method::HEAD,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: object_name.to_string(),
+                    query_values: opts.to_query_values(),
+                    custom_header: headers,
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    content_md5_base64: "".to_string(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await;
+
+        match resp {
+            Ok(resp) => {
+                let h = resp.headers();
+                let delete_marker = if let Some(x_amz_delete_marker) = h.get(X_AMZ_DELETE_MARKER.as_str()) {
+                    x_amz_delete_marker.to_str().expect("operation should succeed") == "true"
+                } else {
+                    false
+                };
+                let replication_ready = if let Some(x_amz_delete_marker) = h.get("X-Replication-Ready") {
+                    x_amz_delete_marker.to_str().expect("operation should succeed") == "true"
+                } else {
+                    false
+                };
+                if resp.status() != http::StatusCode::OK && resp.status() != http::StatusCode::PARTIAL_CONTENT {
+                    if resp.status() == http::StatusCode::METHOD_NOT_ALLOWED && opts.version_id != "" && delete_marker {
+                        let err_resp = ErrorResponse {
+                            status_code: resp.status(),
+                            code: s3s::S3ErrorCode::MethodNotAllowed,
+                            message: "the specified method is not allowed against this resource.".to_string(),
+                            bucket_name: bucket_name.to_string(),
+                            key: object_name.to_string(),
+                            ..Default::default()
+                        };
+                        return Ok(ObjectInfo {
+                            version_id: self
+                                .raw_version_id(h)?
+                                .and_then(|s| Uuid::from_str(s).ok())
+                                .filter(|v| !v.is_nil()),
+                            is_delete_marker: delete_marker,
+                            ..Default::default()
+                        });
+                        //err_resp
+                    }
+                    return Ok(ObjectInfo {
+                        version_id: self
+                            .raw_version_id(h)?
+                            .and_then(|s| Uuid::from_str(s).ok())
+                            .filter(|v| !v.is_nil()),
+                        is_delete_marker: delete_marker,
+                        replication_ready: replication_ready,
+                        ..Default::default()
+                    });
+                    //http_resp_to_error_response(resp, bucket_name, object_name)
+                }
+
+                to_object_info_for_provider(bucket_name, object_name, h, self.provider_version_capabilities())
+            }
+            Err(err) => {
+                return Err(std::io::Error::other(err));
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::parse_bucket_versioning_response;
+    use http::{HeaderMap, StatusCode};
+    use s3s::dto::BucketVersioningStatus;
+
+    #[test]
+    fn parses_bucket_versioning_statuses_mfa_delete_and_unversioned_state() {
+        for (xml, expected) in [
+            (
+                br#"<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>"#
+                    .as_slice(),
+                Some(BucketVersioningStatus::ENABLED),
+            ),
+            (
+                br#"<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>"#
+                    .as_slice(),
+                Some(BucketVersioningStatus::SUSPENDED),
+            ),
+            (
+                br#"<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>"#.as_slice(),
+                None,
+            ),
+        ] {
+            let config = parse_bucket_versioning_response(StatusCode::OK, &HeaderMap::new(), xml.to_vec(), "tier-bucket")
+                .expect("valid GetBucketVersioning response should parse");
+            assert_eq!(config.status.as_ref().map(|status| status.as_str()), expected);
+        }
+
+        let config = parse_bucket_versioning_response(
+            StatusCode::OK,
+            &HeaderMap::new(),
+            br#"<VersioningConfiguration><MfaDelete>Enabled</MfaDelete></VersioningConfiguration>"#.to_vec(),
+            "tier-bucket",
+        )
+        .expect("valid MfaDelete should parse");
+        assert_eq!(config.mfa_delete.as_ref().map(|status| status.as_str()), Some("Enabled"));
+    }
+
+    #[test]
+    fn rejects_non_ok_and_malformed_bucket_versioning_responses() {
+        let valid = br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#.to_vec();
+        for status in [StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT] {
+            let status_err = parse_bucket_versioning_response(status, &HeaderMap::new(), valid.clone(), "tier-bucket")
+                .expect_err("GetBucketVersioning must return exactly HTTP 200");
+            assert_eq!(status_err.kind(), std::io::ErrorKind::Other);
+        }
+
+        let parse_err = parse_bucket_versioning_response(
+            StatusCode::OK,
+            &HeaderMap::new(),
+            b"<VersioningConfiguration>".to_vec(),
+            "tier-bucket",
+        )
+        .expect_err("malformed GetBucketVersioning XML must fail closed");
+        assert_eq!(parse_err.kind(), std::io::ErrorKind::InvalidData);
+
+        for xml in [
+            b"<VersioningConfiguration><Statuz>Enabled</Statuz></VersioningConfiguration>".as_slice(),
+            b"<WrongRoot><Status>Enabled</Status></WrongRoot>".as_slice(),
+            b"<VersioningConfiguration xmlns=\"https://example.invalid\"/>".as_slice(),
+        ] {
+            let strict_err = parse_bucket_versioning_response(StatusCode::OK, &HeaderMap::new(), xml.to_vec(), "tier-bucket")
+                .expect_err("unknown GetBucketVersioning XML must fail closed");
+            assert_eq!(strict_err.kind(), std::io::ErrorKind::InvalidData);
+        }
+    }
+}

crates/s3-client/src/bucket_cache.rs

@@ -0,0 +1,273 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use super::constants::UNSIGNED_PAYLOAD;
+use super::credentials::SignatureType;
+use crate::{
+    api_error_response::http_resp_to_error_response,
+    signer_error,
+    transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient},
+};
+use http::Request;
+use http_body_util::BodyExt;
+use hyper::StatusCode;
+use hyper::body::Body;
+use hyper::body::Bytes;
+use hyper::body::Incoming;
+use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+use s3s::S3ErrorCode;
+use std::collections::HashMap;
+
+fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> std::io::Error {
+    signer_error::signer_error_to_io_error(scope, error)
+}
+
+#[derive(Debug, Clone)]
+pub struct BucketLocationCache {
+    items: HashMap<String, String>,
+}
+
+impl BucketLocationCache {
+    pub fn new() -> BucketLocationCache {
+        BucketLocationCache { items: HashMap::new() }
+    }
+
+    pub fn get(&self, bucket_name: &str) -> Option<String> {
+        self.items.get(bucket_name).map(|s| s.clone())
+    }
+
+    pub fn set(&mut self, bucket_name: &str, location: &str) {
+        self.items.insert(bucket_name.to_string(), location.to_string());
+    }
+
+    pub fn delete(&mut self, bucket_name: &str) {
+        self.items.remove(bucket_name);
+    }
+}
+
+impl TransitionClient {
+    pub async fn get_bucket_location(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        Ok(self.get_bucket_location_inner(bucket_name).await?)
+    }
+
+    async fn get_bucket_location_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        if self.region != "" {
+            return Ok(self.region.clone());
+        }
+
+        let mut location;
+        {
+            if let Ok(bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                if let Some(location) = bucket_loc_cache.get(bucket_name) {
+                    return Ok(location);
+                }
+            }
+            //location = ret?;
+        }
+
+        let req = self.get_bucket_location_request(bucket_name)?;
+
+        let mut resp = self.doit(req).await?;
+        location = process_bucket_location_response(resp, bucket_name, &self.tier_type).await?;
+        {
+            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                bucket_loc_cache.set(bucket_name, &location);
+            }
+        }
+        Ok(location)
+    }
+
+    fn get_bucket_location_request(&self, bucket_name: &str) -> Result<http::Request<s3s::Body>, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("location".to_string(), "".to_string());
+
+        let mut target_url = self.endpoint_url.clone();
+        let scheme = self.endpoint_url.scheme();
+        let h = target_url.host().expect("host is none.");
+        let default_port = if scheme == "https" { 443 } else { 80 };
+        let p = target_url.port().unwrap_or(default_port);
+
+        let is_virtual_style = self.is_virtual_host_style_request(&target_url, bucket_name);
+
+        let mut url_str: String = "".to_string();
+
+        if is_virtual_style {
+            url_str = scheme.to_string();
+            url_str.push_str("://");
+            url_str.push_str(bucket_name);
+            url_str.push_str(".");
+            url_str.push_str(
+                target_url
+                    .host_str()
+                    .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "host is none"))?,
+            );
+            url_str.push_str("/?location");
+        } else {
+            let mut path = bucket_name.to_string();
+            path.push_str("/");
+            target_url.set_path(&path);
+            {
+                let mut q = target_url.query_pairs_mut();
+                for (k, v) in url_values {
+                    q.append_pair(&k, &urlencoding::encode(&v));
+                }
+            }
+            url_str = target_url.to_string();
+        }
+
+        let Ok(mut req) = Request::builder()
+            .method(http::Method::GET)
+            .uri(url_str)
+            .body(s3s::Body::empty())
+        else {
+            return Err(std::io::Error::other("create request error"));
+        };
+
+        self.set_user_agent(&mut req);
+
+        let value;
+        {
+            if let Ok(mut creds_provider) = self.creds_provider.lock() {
+                value = match creds_provider.get_with_context(Some(self.cred_context())) {
+                    Ok(v) => v,
+                    Err(err) => {
+                        return Err(std::io::Error::other(err));
+                    }
+                };
+            } else {
+                return Err(std::io::Error::other("Failed to acquire credentials provider lock"));
+            }
+        }
+
+        let mut signer_type = value.signer_type.clone();
+        let mut access_key_id = value.access_key_id;
+        let mut secret_access_key = value.secret_access_key;
+        let mut session_token = value.session_token;
+
+        if self.override_signer_type != SignatureType::SignatureDefault {
+            signer_type = self.override_signer_type.clone();
+        }
+
+        if value.signer_type == SignatureType::SignatureAnonymous {
+            signer_type = SignatureType::SignatureAnonymous
+        }
+
+        if signer_type == SignatureType::SignatureAnonymous {
+            return Ok(req);
+        }
+
+        if signer_type == SignatureType::SignatureV2 {
+            let req = rustfs_signer::sign_v2(req, 0, &access_key_id, &secret_access_key, is_virtual_style);
+            return Ok(req);
+        }
+
+        let mut content_sha256 = EMPTY_STRING_SHA256_HASH.to_string();
+        if self.secure {
+            content_sha256 = UNSIGNED_PAYLOAD.to_string();
+        }
+
+        let content_sha256_value = content_sha256.parse().map_err(|err| {
+            std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                format!("invalid X-Amz-Content-Sha256 header value: {err}"),
+            )
+        })?;
+        req.headers_mut().insert("X-Amz-Content-Sha256", content_sha256_value);
+        let req = rustfs_signer::try_sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1")
+            .map_err(|err| signer_error_to_io_error("failed to sign bucket location request", err))?;
+        Ok(req)
+    }
+}
+
+async fn process_bucket_location_response(
+    mut resp: http::Response<Incoming>,
+    bucket_name: &str,
+    tier_type: &str,
+) -> Result<String, std::io::Error> {
+    //if resp != nil {
+    if resp.status() != StatusCode::OK {
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
+        let err_resp = http_resp_to_error_response(resp_status, &h, vec![], bucket_name, "");
+        match err_resp.code {
+                S3ErrorCode::NotImplemented => {
+                    match err_resp.server.as_str() {
+                        "AmazonSnowball" => {
+                            return Ok("snowball".to_string());
+                        }
+                        "cloudflare" => {
+                            return Ok("us-east-1".to_string());
+                        }
+                        _ => {
+                            return Err(std::io::Error::other(err_resp));
+                        }
+                    }
+                }
+                S3ErrorCode::AuthorizationHeaderMalformed |
+                //S3ErrorCode::InvalidRegion |
+                S3ErrorCode::AccessDenied => {
+                    if err_resp.region == "" {
+                      return Ok("us-east-1".to_string());
+                    }
+                    return Ok(err_resp.region);
+                }
+                _ => {
+                    return Err(std::io::Error::other(err_resp));
+                }
+            }
+    }
+    //}
+
+    let mut body_vec = Vec::new();
+    let mut body = resp.into_body();
+    while let Some(frame) = body.frame().await {
+        let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+        if let Some(data) = frame.data_ref() {
+            body_vec.extend_from_slice(data);
+        }
+    }
+    let mut location = "".to_string();
+    if tier_type == "huaweicloud" {
+        if let Ok(body_str) = String::from_utf8(body_vec) {
+            if let Ok(d) = quick_xml::de::from_str::<CreateBucketConfiguration>(&body_str) {
+                location = d.location_constraint;
+            }
+        }
+    } else {
+        if let Ok(body_str) = String::from_utf8(body_vec) {
+            if let Ok(LocationConstraint { field }) = quick_xml::de::from_str::<LocationConstraint>(&body_str) {
+                location = field;
+            }
+        }
+    }
+    //debug!("location: {}", location);
+
+    if location == "" {
+        location = "us-east-1".to_string();
+    }
+
+    if location == "EU" {
+        location = "eu-west-1".to_string();
+    }
+
+    Ok(location)
+}

crates/s3-client/src/checksum.rs

@@ -0,0 +1,451 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use lazy_static::lazy_static;
+use rustfs_checksums::ChecksumAlgorithm;
+use std::collections::HashMap;
+
+use crate::utils::base64_decode;
+use crate::utils::base64_encode;
+use crate::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
+// s3s::header has no CRC64NVME constant yet; the canonical RustFS copy lives
+// in rustfs-utils' headers module.
+use rustfs_utils::http::headers::AMZ_CHECKSUM_CRC64NVME;
+use s3s::header::{
+    X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
+};
+
+use enumset::{EnumSet, EnumSetType, enum_set};
+
+/// One of three deliberately separate checksum registries (backlog#1833):
+/// this enum is the MinIO-port client's wire vocabulary and stops at the
+/// standard S3 set (CRC64NVME is its newest member; the RustFS extensions do
+/// not exist on this client path). The streaming-hash registry lives in
+/// `rustfs_checksums::ChecksumAlgorithm` (crates/checksums/src/lib.rs) and
+/// the on-disk xl.meta bitset in `rustfs_rio::ChecksumType`
+/// (crates/rio/src/checksum.rs, varint bits are append-only). When adding an
+/// algorithm, extend all three (or record why not) — they do not derive from
+/// each other.
+#[derive(Debug, EnumSetType, Default)]
+#[enumset(repr = "u8")]
+pub enum ChecksumMode {
+    #[default]
+    ChecksumNone,
+    ChecksumSHA256,
+    ChecksumSHA1,
+    ChecksumCRC32,
+    ChecksumCRC32C,
+    ChecksumCRC64NVME,
+    ChecksumFullObject,
+}
+
+lazy_static! {
+    static ref C_ChecksumMask: EnumSet<ChecksumMode> = {
+        let mut s = EnumSet::all();
+        s.remove(ChecksumMode::ChecksumFullObject);
+        s
+    };
+    static ref C_ChecksumFullObjectCRC32: EnumSet<ChecksumMode> =
+        enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject);
+    static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> =
+        enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject);
+}
+impl ChecksumMode {
+    //pub const CRC64_NVME_POLYNOMIAL: i64 = 0xad93d23594c93659;
+
+    pub fn base(&self) -> ChecksumMode {
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        match s.as_u8() {
+            1_u8 => ChecksumMode::ChecksumNone,
+            2_u8 => ChecksumMode::ChecksumSHA256,
+            4_u8 => ChecksumMode::ChecksumSHA1,
+            8_u8 => ChecksumMode::ChecksumCRC32,
+            16_u8 => ChecksumMode::ChecksumCRC32C,
+            32_u8 => ChecksumMode::ChecksumCRC64NVME,
+            // Fail closed: any mode without a concrete base algorithm (e.g. a
+            // bare ChecksumFullObject flag) is treated as "no checksum" rather
+            // than panicking. Callers already gate real work behind
+            // is_set()/can_composite()/hasher(), so this only removes a crash.
+            _ => ChecksumMode::ChecksumNone,
+        }
+    }
+
+    pub fn is(&self, t: ChecksumMode) -> bool {
+        *self & t == t
+    }
+
+    pub fn key(&self) -> String {
+        //match c & checksumMask {
+        match self {
+            ChecksumMode::ChecksumCRC32 => {
+                return X_AMZ_CHECKSUM_CRC32.to_string();
+            }
+            ChecksumMode::ChecksumCRC32C => {
+                return X_AMZ_CHECKSUM_CRC32C.to_string();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return X_AMZ_CHECKSUM_SHA1.to_string();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return X_AMZ_CHECKSUM_SHA256.to_string();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return AMZ_CHECKSUM_CRC64NVME.to_string();
+            }
+            _ => {
+                return "".to_string();
+            }
+        }
+    }
+
+    pub fn can_composite(&self) -> bool {
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        match s.as_u8() {
+            2_u8 => true,
+            4_u8 => true,
+            8_u8 => true,
+            16_u8 => true,
+            _ => false,
+        }
+    }
+
+    pub fn can_merge_crc(&self) -> bool {
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        match s.as_u8() {
+            8_u8 => true,
+            16_u8 => true,
+            32_u8 => true,
+            _ => false,
+        }
+    }
+
+    pub fn full_object_requested(&self) -> bool {
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        match s.as_u8() {
+            //C_ChecksumFullObjectCRC32 as u8 => true,
+            //C_ChecksumFullObjectCRC32C as u8 => true,
+            32_u8 => true,
+            _ => false,
+        }
+    }
+
+    pub fn key_capitalized(&self) -> String {
+        self.key()
+    }
+
+    pub fn raw_byte_len(&self) -> usize {
+        let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8();
+        if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 {
+            4
+        } else if u == ChecksumMode::ChecksumSHA1 as u8 {
+            use sha1::Digest;
+            sha1::Sha1::output_size() as usize
+        } else if u == ChecksumMode::ChecksumSHA256 as u8 {
+            use sha2::Digest;
+            sha2::Sha256::output_size() as usize
+        } else if u == ChecksumMode::ChecksumCRC64NVME as u8 {
+            8
+        } else {
+            0
+        }
+    }
+
+    pub fn hasher(&self) -> Result<Box<dyn rustfs_checksums::http::HttpChecksum>, std::io::Error> {
+        match /*C_ChecksumMask & **/self {
+            ChecksumMode::ChecksumCRC32 => {
+                return Ok(ChecksumAlgorithm::Crc32.into_impl());
+            }
+            ChecksumMode::ChecksumCRC32C => {
+                return Ok(ChecksumAlgorithm::Crc32c.into_impl());
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return Ok(ChecksumAlgorithm::Sha1.into_impl());
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return Ok(ChecksumAlgorithm::Sha256.into_impl());
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return Ok(ChecksumAlgorithm::Crc64Nvme.into_impl());
+            }
+            _ => return Err(std::io::Error::other("unsupported checksum type")),
+        }
+    }
+
+    pub fn is_set(&self) -> bool {
+        // `ChecksumNone` is the zeroth enum variant, so it occupies bit 0 of the
+        // `EnumSet` repr and a naive `len() == 1` check reports "no checksum" as a
+        // configured checksum. A checksum is only "set" when a concrete algorithm
+        // (one with a real hasher) is selected; the bare `ChecksumFullObject` flag
+        // has no base algorithm and is likewise not set. Treating `ChecksumNone`
+        // as set made ILM transitions of >128 MiB objects fail with
+        // "unsupported checksum type" (rustfs/rustfs#4811): the multipart put path
+        // took the checksum branch and called `ChecksumNone.hasher()`.
+        if matches!(self, ChecksumMode::ChecksumNone) {
+            return false;
+        }
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        s.len() == 1
+    }
+
+    pub fn set_default(&mut self, t: ChecksumMode) {
+        if !self.is_set() {
+            *self = t;
+        }
+    }
+
+    pub fn encode_to_string(&self, b: &[u8]) -> Result<String, std::io::Error> {
+        if !self.is_set() {
+            return Ok("".to_string());
+        }
+        let mut h = self.hasher()?;
+        h.update(b);
+        let hash = h.finalize();
+        Ok(base64_encode(hash.as_ref()))
+    }
+
+    pub fn to_string(&self) -> String {
+        //match c & checksumMask {
+        match self {
+            ChecksumMode::ChecksumCRC32 => {
+                return "CRC32".to_string();
+            }
+            ChecksumMode::ChecksumCRC32C => {
+                return "CRC32C".to_string();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return "SHA1".to_string();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return "SHA256".to_string();
+            }
+            ChecksumMode::ChecksumNone => {
+                return "".to_string();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return "CRC64NVME".to_string();
+            }
+            _ => {
+                return "<invalid>".to_string();
+            }
+        }
+    }
+
+    // pub fn check_sum_reader(&self, r: GetObjectReader) -> Result<Checksum, std::io::Error> {
+    //     let mut h = self.hasher()?;
+    //     Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
+    // }
+
+    // pub fn check_sum_bytes(&self, b: &[u8]) -> Result<Checksum, std::io::Error> {
+    //     let mut h = self.hasher()?;
+    //     Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
+    // }
+
+    pub fn composite_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
+        if !self.can_composite() {
+            return Err(std::io::Error::other("cannot do composite checksum"));
+        }
+        p.sort_by(|i, j| {
+            if i.part_num < j.part_num {
+                std::cmp::Ordering::Less
+            } else if i.part_num > j.part_num {
+                std::cmp::Ordering::Greater
+            } else {
+                std::cmp::Ordering::Equal
+            }
+        });
+        let c = self.base();
+        let mut crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize);
+        let mut h = self.hasher()?;
+        for part in p.iter() {
+            let part_checksum = part.checksum_raw(&c)?;
+            crc_bytes.extend(part_checksum);
+        }
+        h.update(crc_bytes.as_ref());
+        let hash = h.finalize();
+        Ok(Checksum {
+            checksum_type: self.clone(),
+            r: hash.as_ref().to_vec(),
+            computed: false,
+        })
+    }
+
+    pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
+        if !self.can_merge_crc() {
+            return Err(std::io::Error::other("cannot do full-object checksum"));
+        }
+
+        self.composite_checksum(p)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::ChecksumMode;
+
+    #[test]
+    fn test_base_is_fail_closed_and_never_panics() {
+        // Every mode must resolve to a concrete base without panicking. The bare
+        // ChecksumFullObject flag has no base algorithm and must fall back to
+        // ChecksumNone instead of crashing (previously `panic!("enum err.")`).
+        assert_eq!(ChecksumMode::ChecksumFullObject.base(), ChecksumMode::ChecksumNone);
+        assert_eq!(ChecksumMode::ChecksumNone.base(), ChecksumMode::ChecksumNone);
+        assert_eq!(ChecksumMode::ChecksumCRC32.base(), ChecksumMode::ChecksumCRC32);
+        assert_eq!(ChecksumMode::ChecksumCRC32C.base(), ChecksumMode::ChecksumCRC32C);
+        assert_eq!(ChecksumMode::ChecksumSHA1.base(), ChecksumMode::ChecksumSHA1);
+        assert_eq!(ChecksumMode::ChecksumSHA256.base(), ChecksumMode::ChecksumSHA256);
+        assert_eq!(ChecksumMode::ChecksumCRC64NVME.base(), ChecksumMode::ChecksumCRC64NVME);
+    }
+
+    #[test]
+    fn test_hasher_fails_closed_for_unsupported_mode() {
+        // Modes without a real hasher must return an error, not panic.
+        assert!(ChecksumMode::ChecksumNone.hasher().is_err());
+        assert!(ChecksumMode::ChecksumFullObject.hasher().is_err());
+        assert!(ChecksumMode::ChecksumCRC32.hasher().is_ok());
+    }
+
+    #[test]
+    fn test_is_set_is_false_for_none_and_bare_full_object() {
+        // Regression for rustfs/rustfs#4811: `ChecksumNone` must NOT be reported as
+        // a configured checksum. It is the zeroth enum variant (bit 0 of the
+        // EnumSet repr), so the old `len() == 1` check treated it as set and drove
+        // the multipart put path into `ChecksumNone.hasher()` → "unsupported
+        // checksum type". Every mode reported as set must also have a real hasher.
+        assert!(!ChecksumMode::ChecksumNone.is_set());
+        assert!(!ChecksumMode::ChecksumFullObject.is_set());
+
+        for mode in [
+            ChecksumMode::ChecksumCRC32,
+            ChecksumMode::ChecksumCRC32C,
+            ChecksumMode::ChecksumSHA1,
+            ChecksumMode::ChecksumSHA256,
+            ChecksumMode::ChecksumCRC64NVME,
+        ] {
+            assert!(mode.is_set(), "{mode:?} should be set");
+            assert!(mode.hasher().is_ok(), "{mode:?} reported set but has no hasher");
+        }
+    }
+
+    #[test]
+    fn test_set_default_upgrades_none() {
+        // With `is_set()` fixed, `set_default` must upgrade an unset mode to the
+        // provided default (previously `ChecksumNone` was seen as set and never
+        // upgraded).
+        let mut mode = ChecksumMode::ChecksumNone;
+        mode.set_default(ChecksumMode::ChecksumCRC32C);
+        assert_eq!(mode, ChecksumMode::ChecksumCRC32C);
+
+        // An already-set mode is left untouched.
+        let mut existing = ChecksumMode::ChecksumSHA256;
+        existing.set_default(ChecksumMode::ChecksumCRC32C);
+        assert_eq!(existing, ChecksumMode::ChecksumSHA256);
+    }
+}
+
+#[derive(Default)]
+pub struct Checksum {
+    checksum_type: ChecksumMode,
+    r: Vec<u8>,
+    #[allow(
+        dead_code,
+        reason = "checksum bookkeeping field kept beside the value it guards (backlog#1823)"
+    )]
+    computed: bool,
+}
+
+impl Checksum {
+    #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
+    fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
+        if t.is_set() && b.len() == t.raw_byte_len() {
+            return Checksum {
+                checksum_type: t,
+                r: b.to_vec(),
+                computed: false,
+            };
+        }
+        Checksum::default()
+    }
+
+    #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
+    fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
+        let b = match base64_decode(s.as_bytes()) {
+            Ok(b) => b,
+            Err(err) => return Err(std::io::Error::other(err.to_string())),
+        };
+        if t.is_set() && b.len() == t.raw_byte_len() {
+            return Ok(Checksum {
+                checksum_type: t,
+                r: b,
+                computed: false,
+            });
+        }
+        Ok(Checksum::default())
+    }
+
+    fn is_set(&self) -> bool {
+        self.checksum_type.is_set() && self.r.len() == self.checksum_type.raw_byte_len()
+    }
+
+    fn encoded(&self) -> String {
+        if !self.is_set() {
+            return "".to_string();
+        }
+        base64_encode(&self.r)
+    }
+
+    #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
+    fn raw(&self) -> Option<Vec<u8>> {
+        if !self.is_set() {
+            return None;
+        }
+        Some(self.r.clone())
+    }
+}
+
+pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) {
+    opts.user_metadata
+        .insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string());
+    if opts.auto_checksum.full_object_requested() {
+        opts.user_metadata
+            .insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
+    }
+}
+
+pub fn apply_auto_checksum(opts: &mut PutObjectOptions, all_parts: &mut [ObjectPart]) -> Result<(), std::io::Error> {
+    if opts.auto_checksum.can_composite() && !opts.auto_checksum.is(ChecksumMode::ChecksumFullObject) {
+        let crc = opts.auto_checksum.composite_checksum(all_parts)?;
+        opts.user_metadata = {
+            let mut hm = HashMap::new();
+            hm.insert(opts.auto_checksum.key(), crc.encoded());
+            hm
+        }
+    } else if opts.auto_checksum.can_merge_crc() {
+        let crc = opts.auto_checksum.full_object_checksum(all_parts)?;
+        opts.user_metadata = {
+            let mut hm = HashMap::new();
+            hm.insert(opts.auto_checksum.key_capitalized(), crc.encoded());
+            hm.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
+            hm
+        }
+    }
+
+    Ok(())
+}

crates/s3-client/src/constants.rs

@@ -0,0 +1,36 @@
+#![allow(unused_imports)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+
+use lazy_static::lazy_static;
+use std::{collections::HashMap, sync::Arc};
+use time::{format_description::FormatItem, macros::format_description};
+
+pub const ABS_MIN_PART_SIZE: i64 = 1024 * 1024 * 5;
+pub const MAX_PARTS_COUNT: i64 = 10000;
+pub const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+pub const MIN_PART_SIZE: i64 = 1024 * 1024 * 16;
+
+pub const MAX_SINGLE_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+
+pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
+pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
+
+pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
+    format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");

crates/s3-client/src/credentials.rs

@@ -0,0 +1,172 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use quick_xml;
+use serde::de::Deserialize;
+use std::fmt::{Display, Formatter};
+use std::io::{Error, ErrorKind};
+
+use time::OffsetDateTime;
+
+#[derive(Debug, Default, Clone, Eq, PartialEq)]
+pub enum SignatureType {
+    #[default]
+    SignatureDefault,
+    SignatureV4,
+    SignatureV2,
+    SignatureV4Streaming,
+    SignatureAnonymous,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct Credentials<P: Provider + Default> {
+    creds: Value,
+    force_refresh: bool,
+    provider: P,
+}
+
+impl<P: Provider + Default> Credentials<P> {
+    pub fn new(provider: P) -> Self {
+        Self {
+            provider,
+            force_refresh: true,
+            ..Default::default()
+        }
+    }
+
+    pub fn get(&mut self) -> Result<Value, std::io::Error> {
+        self.get_with_context(None)
+    }
+
+    pub fn get_with_context(&mut self, mut cc: Option<CredContext>) -> Result<Value, std::io::Error> {
+        if self.is_expired() {
+            let creds = self.provider.retrieve_with_cred_context(cc.unwrap_or(CredContext {
+                endpoint: "".to_string(),
+            }));
+            self.creds = creds;
+            self.force_refresh = false;
+        }
+
+        Ok(self.creds.clone())
+    }
+
+    #[allow(
+        dead_code,
+        reason = "MinIO-parity credential surface with no caller in this port (backlog#1823)"
+    )]
+    fn expire(&mut self) {
+        self.force_refresh = true;
+    }
+
+    pub fn is_expired(&self) -> bool {
+        self.force_refresh || self.provider.is_expired()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Value {
+    pub access_key_id: String,
+    pub secret_access_key: String,
+    pub session_token: String,
+    pub expiration: OffsetDateTime,
+    pub signer_type: SignatureType,
+}
+
+impl Default for Value {
+    fn default() -> Self {
+        Self {
+            access_key_id: "".to_string(),
+            secret_access_key: "".to_string(),
+            session_token: "".to_string(),
+            expiration: OffsetDateTime::now_utc(),
+            signer_type: SignatureType::SignatureDefault,
+        }
+    }
+}
+
+pub struct CredContext {
+    //pub client: SendRequest,
+    pub endpoint: String,
+}
+
+pub trait Provider {
+    fn retrieve(&self) -> Value;
+    fn retrieve_with_cred_context(&self, _: CredContext) -> Value;
+    fn is_expired(&self) -> bool;
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct Static(pub Value);
+
+impl Provider for Static {
+    fn retrieve(&self) -> Value {
+        if self.0.access_key_id == "" || self.0.secret_access_key == "" {
+            return Value {
+                signer_type: SignatureType::SignatureAnonymous,
+                ..Default::default()
+            };
+        }
+        self.0.clone()
+    }
+
+    fn retrieve_with_cred_context(&self, _: CredContext) -> Value {
+        self.retrieve()
+    }
+
+    fn is_expired(&self) -> bool {
+        false
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct STSError {
+    #[allow(
+        dead_code,
+        reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
+    )]
+    pub r#type: String,
+    pub code: String,
+    pub message: String,
+}
+
+#[derive(Debug, Clone, thiserror::Error)]
+pub struct ErrorResponse {
+    pub sts_error: STSError,
+    #[allow(
+        dead_code,
+        reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
+    )]
+    pub request_id: String,
+}
+
+impl Display for ErrorResponse {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.error())
+    }
+}
+
+impl ErrorResponse {
+    fn error(&self) -> String {
+        if self.sts_error.message == "" {
+            return format!("Error response code {}.", self.sts_error.code);
+        }
+        return self.sts_error.message.clone();
+    }
+}

crates/s3-client/src/lib.rs

@@ -0,0 +1,41 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! S3 client used by the storage engine when it *consumes* remote S3-compatible
+//! endpoints (ILM tier warm backends, transition targets). Extracted from
+//! `crates/ecstore/src/client` (rustfs/backlog#1842) so the engine no longer
+//! embeds an S3 HTTP client; ecstore re-exports these modules during the
+//! migration window.
+
+pub mod admin_handler_utils;
+pub mod api_error_response;
+pub mod api_get_object;
+pub mod api_get_options;
+pub mod api_list;
+pub mod api_put_object;
+pub mod api_put_object_common;
+pub mod api_put_object_multipart;
+pub mod api_put_object_streaming;
+pub mod api_remove;
+pub mod api_s3_datatypes;
+pub mod api_stat;
+pub mod bucket_cache;
+pub mod checksum;
+pub mod constants;
+pub mod credentials;
+pub mod provider_versions;
+pub mod runtime_sources;
+pub mod signer_error;
+pub mod transition_api;
+pub mod utils;

crates/s3-client/src/provider_versions.rs

@@ -0,0 +1,356 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::io::{Error, ErrorKind};
+
+use http::HeaderMap;
+
+const X_AMZ_VERSION_ID: &str = "x-amz-version-id";
+const X_OSS_VERSION_ID: &str = "x-oss-version-id";
+const X_COS_VERSION_ID: &str = "x-cos-version-id";
+const X_OBS_VERSION_ID: &str = "x-obs-version-id";
+const MAX_REMOTE_VERSION_ID_LEN: usize = 1024;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[allow(dead_code, reason = "bucket versioning states kept as a complete vocabulary (backlog#1823)")]
+pub enum BucketVersioningState {
+    Unknown,
+    Disabled,
+    Suspended,
+    Enabled,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum RemoteVersion {
+    Unknown,
+    Disabled,
+    SuspendedNull,
+    Exact(String),
+}
+
+impl RemoteVersion {
+    pub fn exact_id(&self) -> Option<&str> {
+        match self {
+            Self::SuspendedNull => Some("null"),
+            Self::Exact(version_id) => Some(version_id),
+            Self::Unknown | Self::Disabled => None,
+        }
+    }
+
+    #[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
+    pub fn exact_request_id(&self) -> Result<Option<&str>, Error> {
+        match self {
+            Self::Unknown => Err(Error::new(
+                ErrorKind::InvalidData,
+                "remote object version is unknown; exact version routing is unsafe",
+            )),
+            Self::Disabled => Ok(None),
+            Self::SuspendedNull => Ok(Some("null")),
+            Self::Exact(version_id) => Ok(Some(version_id)),
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum ConditionalCreateCapability {
+    Unsupported,
+    IfNoneMatchStar,
+    GenerationMatchZero,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct ProviderVersionCapabilities {
+    raw_version_header: Option<&'static str>,
+    pub bucket_versioning_state: bool,
+    pub list_object_versions: bool,
+    pub conditional_create: ConditionalCreateCapability,
+    pub exact_get_delete: bool,
+}
+
+impl ProviderVersionCapabilities {
+    pub fn for_tier_type(tier_type: &str) -> Self {
+        if tier_type.eq_ignore_ascii_case("s3")
+            || tier_type.eq_ignore_ascii_case("rustfs")
+            || tier_type.eq_ignore_ascii_case("minio")
+            || tier_type.eq_ignore_ascii_case("r2")
+            || tier_type.eq_ignore_ascii_case("wasabi")
+        {
+            let list_object_versions = tier_type.eq_ignore_ascii_case("s3")
+                || tier_type.eq_ignore_ascii_case("rustfs")
+                || tier_type.eq_ignore_ascii_case("minio")
+                || tier_type.eq_ignore_ascii_case("r2");
+            Self {
+                raw_version_header: Some(X_AMZ_VERSION_ID),
+                bucket_versioning_state: list_object_versions,
+                list_object_versions,
+                conditional_create: if tier_type.eq_ignore_ascii_case("s3") || tier_type.eq_ignore_ascii_case("r2") {
+                    ConditionalCreateCapability::IfNoneMatchStar
+                } else {
+                    ConditionalCreateCapability::Unsupported
+                },
+                exact_get_delete: true,
+            }
+        } else if tier_type.eq_ignore_ascii_case("aliyun") {
+            Self {
+                raw_version_header: Some(X_OSS_VERSION_ID),
+                bucket_versioning_state: false,
+                list_object_versions: false,
+                conditional_create: ConditionalCreateCapability::Unsupported,
+                exact_get_delete: true,
+            }
+        } else if tier_type.eq_ignore_ascii_case("tencent") {
+            Self {
+                raw_version_header: Some(X_COS_VERSION_ID),
+                bucket_versioning_state: false,
+                list_object_versions: false,
+                conditional_create: ConditionalCreateCapability::Unsupported,
+                exact_get_delete: true,
+            }
+        } else if tier_type.eq_ignore_ascii_case("huaweicloud") {
+            Self {
+                raw_version_header: Some(X_OBS_VERSION_ID),
+                bucket_versioning_state: false,
+                list_object_versions: false,
+                conditional_create: ConditionalCreateCapability::Unsupported,
+                exact_get_delete: true,
+            }
+        } else if tier_type.eq_ignore_ascii_case("gcs") {
+            Self {
+                raw_version_header: None,
+                bucket_versioning_state: false,
+                list_object_versions: false,
+                conditional_create: ConditionalCreateCapability::GenerationMatchZero,
+                exact_get_delete: false,
+            }
+        } else {
+            Self {
+                raw_version_header: None,
+                bucket_versioning_state: false,
+                list_object_versions: false,
+                conditional_create: ConditionalCreateCapability::Unsupported,
+                exact_get_delete: false,
+            }
+        }
+    }
+
+    pub fn raw_version_id(self, headers: &HeaderMap) -> Result<Option<&str>, Error> {
+        let Some(header_name) = self.raw_version_header else {
+            return Ok(None);
+        };
+        let Some(value) = headers.get(header_name) else {
+            return Ok(None);
+        };
+        let value = value
+            .to_str()
+            .map_err(|_| Error::new(ErrorKind::InvalidData, "remote object version id is not valid ASCII"))?;
+        validate_remote_version_id(value)?;
+        Ok(Some(value))
+    }
+
+    pub fn remote_version(self, headers: &HeaderMap, versioning: BucketVersioningState) -> Result<RemoteVersion, Error> {
+        let Some(value) = self.raw_version_id(headers)? else {
+            return Ok(match versioning {
+                BucketVersioningState::Disabled => RemoteVersion::Disabled,
+                BucketVersioningState::Unknown | BucketVersioningState::Suspended | BucketVersioningState::Enabled => {
+                    RemoteVersion::Unknown
+                }
+            });
+        };
+        if value == "null" {
+            return Ok(RemoteVersion::SuspendedNull);
+        }
+        Ok(RemoteVersion::Exact(value.to_string()))
+    }
+}
+
+pub fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
+    if version_id.is_empty() {
+        return Err(Error::new(
+            ErrorKind::InvalidData,
+            "remote tier returned an empty object version id header",
+        ));
+    }
+    if version_id.len() > MAX_REMOTE_VERSION_ID_LEN {
+        return Err(Error::new(
+            ErrorKind::InvalidData,
+            "remote tier returned an oversized object version id header",
+        ));
+    }
+    if version_id.chars().any(char::is_control) {
+        return Err(Error::new(
+            ErrorKind::InvalidData,
+            "remote tier returned an object version id containing control characters",
+        ));
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{BucketVersioningState, ConditionalCreateCapability, ProviderVersionCapabilities, RemoteVersion};
+    use http::{HeaderMap, HeaderValue};
+
+    #[test]
+    fn provider_version_header_matrix_preserves_opaque_versions() {
+        for (tier_type, header_name) in [
+            ("s3", "x-amz-version-id"),
+            ("S3", "x-amz-version-id"),
+            ("rustfs", "x-amz-version-id"),
+            ("RustFS", "x-amz-version-id"),
+            ("minio", "x-amz-version-id"),
+            ("MinIO", "x-amz-version-id"),
+            ("r2", "x-amz-version-id"),
+            ("R2", "x-amz-version-id"),
+            ("wasabi", "x-amz-version-id"),
+            ("Wasabi", "x-amz-version-id"),
+            ("aliyun", "x-oss-version-id"),
+            ("Aliyun", "x-oss-version-id"),
+            ("tencent", "x-cos-version-id"),
+            ("Tencent", "x-cos-version-id"),
+            ("huaweicloud", "x-obs-version-id"),
+            ("Huaweicloud", "x-obs-version-id"),
+        ] {
+            let mut headers = HeaderMap::new();
+            headers.insert(header_name, HeaderValue::from_static("opaque.version_01"));
+            let capabilities = ProviderVersionCapabilities::for_tier_type(tier_type);
+
+            assert_eq!(capabilities.raw_version_id(&headers).expect("raw version"), Some("opaque.version_01"));
+            assert_eq!(
+                capabilities
+                    .remote_version(&headers, BucketVersioningState::Enabled)
+                    .expect("remote version"),
+                RemoteVersion::Exact("opaque.version_01".to_string())
+            );
+        }
+    }
+
+    #[test]
+    fn provider_version_header_matrix_does_not_cross_read_sibling_headers() {
+        let mut headers = HeaderMap::new();
+        headers.insert("x-amz-version-id", HeaderValue::from_static("aws-version"));
+        headers.insert("x-cos-version-id", HeaderValue::from_static("cos-version"));
+
+        assert_eq!(
+            ProviderVersionCapabilities::for_tier_type("s3")
+                .raw_version_id(&headers)
+                .expect("aws raw version"),
+            Some("aws-version")
+        );
+        assert_eq!(
+            ProviderVersionCapabilities::for_tier_type("tencent")
+                .raw_version_id(&headers)
+                .expect("cos raw version"),
+            Some("cos-version")
+        );
+    }
+
+    #[test]
+    fn provider_version_missing_header_is_unknown_until_bucket_state_is_known() {
+        let headers = HeaderMap::new();
+        let capabilities = ProviderVersionCapabilities::for_tier_type("aliyun");
+
+        assert_eq!(
+            capabilities
+                .remote_version(&headers, BucketVersioningState::Unknown)
+                .expect("unknown versioning"),
+            RemoteVersion::Unknown
+        );
+        assert_eq!(
+            capabilities
+                .remote_version(&headers, BucketVersioningState::Disabled)
+                .expect("disabled versioning"),
+            RemoteVersion::Disabled
+        );
+    }
+
+    #[test]
+    fn provider_capability_matrix_is_conservative_and_provider_specific() {
+        for (tier_type, state, list, conditional_create, exact_get_delete) in [
+            ("s3", true, true, ConditionalCreateCapability::IfNoneMatchStar, true),
+            ("rustfs", true, true, ConditionalCreateCapability::Unsupported, true),
+            ("minio", true, true, ConditionalCreateCapability::Unsupported, true),
+            ("r2", true, true, ConditionalCreateCapability::IfNoneMatchStar, true),
+            ("wasabi", false, false, ConditionalCreateCapability::Unsupported, true),
+            ("aliyun", false, false, ConditionalCreateCapability::Unsupported, true),
+            ("tencent", false, false, ConditionalCreateCapability::Unsupported, true),
+            ("huaweicloud", false, false, ConditionalCreateCapability::Unsupported, true),
+            ("gcs", false, false, ConditionalCreateCapability::GenerationMatchZero, false),
+            ("azure", false, false, ConditionalCreateCapability::Unsupported, false),
+            ("unsupported", false, false, ConditionalCreateCapability::Unsupported, false),
+        ] {
+            let capabilities = ProviderVersionCapabilities::for_tier_type(tier_type);
+            assert_eq!(capabilities.bucket_versioning_state, state, "{tier_type} versioning state");
+            assert_eq!(capabilities.list_object_versions, list, "{tier_type} version listing");
+            assert_eq!(capabilities.conditional_create, conditional_create, "{tier_type} conditional create");
+            assert_eq!(capabilities.exact_get_delete, exact_get_delete, "{tier_type} exact routing");
+        }
+    }
+
+    #[test]
+    fn remote_version_states_preserve_unknown_disabled_suspended_and_exact() {
+        let capabilities = ProviderVersionCapabilities::for_tier_type("s3");
+        let empty = HeaderMap::new();
+        let mut null = HeaderMap::new();
+        null.insert("x-amz-version-id", HeaderValue::from_static("null"));
+        let mut exact = HeaderMap::new();
+        exact.insert("x-amz-version-id", HeaderValue::from_static("opaque.generation-7"));
+
+        for (headers, state, expected) in [
+            (&empty, BucketVersioningState::Unknown, RemoteVersion::Unknown),
+            (&empty, BucketVersioningState::Disabled, RemoteVersion::Disabled),
+            (&empty, BucketVersioningState::Suspended, RemoteVersion::Unknown),
+            (&empty, BucketVersioningState::Enabled, RemoteVersion::Unknown),
+            (&null, BucketVersioningState::Suspended, RemoteVersion::SuspendedNull),
+            (
+                &exact,
+                BucketVersioningState::Enabled,
+                RemoteVersion::Exact("opaque.generation-7".to_string()),
+            ),
+        ] {
+            assert_eq!(
+                capabilities
+                    .remote_version(headers, state)
+                    .expect("version state should normalize"),
+                expected
+            );
+        }
+    }
+
+    #[test]
+    fn exact_request_routing_fails_closed_for_unknown_versions() {
+        for (version, expected) in [
+            (RemoteVersion::Disabled, None),
+            (RemoteVersion::SuspendedNull, Some("null")),
+            (RemoteVersion::Exact("opaque-v1".to_string()), Some("opaque-v1")),
+        ] {
+            assert_eq!(version.exact_request_id().expect("known version state"), expected);
+        }
+        assert!(RemoteVersion::Unknown.exact_request_id().is_err());
+    }
+
+    #[test]
+    fn provider_version_rejects_empty_or_oversized_headers() {
+        let oversized = "v".repeat(1025);
+        for bad in ["", oversized.as_str()] {
+            let mut headers = HeaderMap::new();
+            headers.insert("x-oss-version-id", HeaderValue::from_str(bad).expect("test header value"));
+
+            assert!(
+                ProviderVersionCapabilities::for_tier_type("aliyun")
+                    .raw_version_id(&headers)
+                    .is_err()
+            );
+        }
+    }
+}

crates/s3-client/src/runtime_sources.rs

@@ -0,0 +1,25 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, load_global_outbound_tls_state, record_tls_generation};
+
+const ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER: &str = "ecstore_transition_client";
+
+pub async fn transition_client_outbound_tls_state() -> GlobalPublishedOutboundTlsState {
+    load_global_outbound_tls_state().await
+}
+
+pub fn record_transition_client_tls_generation(generation: u64) {
+    record_tls_generation(ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER, generation);
+}

crates/s3-client/src/signer_error.rs

@@ -0,0 +1,105 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::error::Error as StdError;
+use std::fmt::{Display, Formatter};
+use std::io::{Error, ErrorKind};
+
+pub const SIGNER_HEADER_ERROR_MARKER: &str = "rustfs_signer_header_error";
+
+#[derive(Debug)]
+struct SignerHeaderError {
+    scope: String,
+    header_name: String,
+}
+
+impl SignerHeaderError {
+    fn new(scope: &str, header_name: &str) -> Self {
+        Self {
+            scope: scope.to_string(),
+            header_name: header_name.to_string(),
+        }
+    }
+}
+
+impl Display for SignerHeaderError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}: invalid UTF-8 header value for `{}` [{}]",
+            self.scope, self.header_name, SIGNER_HEADER_ERROR_MARKER
+        )
+    }
+}
+
+impl StdError for SignerHeaderError {}
+
+pub fn invalid_utf8_header_error(scope: &str, header_name: &str) -> Error {
+    Error::new(ErrorKind::InvalidInput, SignerHeaderError::new(scope, header_name))
+}
+
+pub fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> Error {
+    match error {
+        rustfs_signer::SignV4Error::InvalidHeaderValue { name } => invalid_utf8_header_error(scope, &name),
+        other => Error::other(format!("{scope}: {other}")),
+    }
+}
+
+pub fn error_chain_contains_signer_header_marker(err: &(dyn StdError + 'static)) -> bool {
+    let mut current = Some(err);
+    while let Some(source) = current {
+        if source.downcast_ref::<SignerHeaderError>().is_some() {
+            return true;
+        }
+
+        if source.to_string().contains(SIGNER_HEADER_ERROR_MARKER) {
+            return true;
+        }
+
+        current = source.source();
+    }
+
+    false
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{error_chain_contains_signer_header_marker, invalid_utf8_header_error, signer_error_to_io_error};
+
+    #[test]
+    fn invalid_utf8_header_error_is_detected_through_error_chain() {
+        let err = invalid_utf8_header_error("failed to sign request", "x-amz-meta-invalid");
+
+        assert!(error_chain_contains_signer_header_marker(&err));
+    }
+
+    #[test]
+    fn mapped_signer_header_error_is_detected_through_error_chain() {
+        let err = signer_error_to_io_error(
+            "failed to sign request",
+            rustfs_signer::SignV4Error::InvalidHeaderValue {
+                name: "x-amz-meta-invalid".to_string(),
+            },
+        );
+
+        assert!(error_chain_contains_signer_header_marker(&err));
+    }
+
+    #[test]
+    fn generic_io_errors_do_not_match_signer_header_marker() {
+        let err = std::io::Error::other("unrelated failure");
+
+        assert!(!error_chain_contains_signer_header_marker(&err));
+    }
+}

crates/s3-client/src/utils.rs

@@ -0,0 +1,113 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use lazy_static::lazy_static;
+use std::collections::HashMap;
+
+use s3s::header::X_AMZ_STORAGE_CLASS;
+
+lazy_static! {
+    static ref SUPPORTED_QUERY_VALUES: HashMap<String, bool> = {
+        let mut m = HashMap::new();
+        m.insert("attributes".to_string(), true);
+        m.insert("partNumber".to_string(), true);
+        m.insert("versionId".to_string(), true);
+        m.insert("response-cache-control".to_string(), true);
+        m.insert("response-content-disposition".to_string(), true);
+        m.insert("response-content-encoding".to_string(), true);
+        m.insert("response-content-language".to_string(), true);
+        m.insert("response-content-type".to_string(), true);
+        m.insert("response-expires".to_string(), true);
+        m
+    };
+    static ref SUPPORTED_HEADERS: HashMap<String, bool> = {
+        let mut m = HashMap::new();
+        m.insert("content-type".to_string(), true);
+        m.insert("cache-control".to_string(), true);
+        m.insert("content-encoding".to_string(), true);
+        m.insert("content-disposition".to_string(), true);
+        m.insert("content-language".to_string(), true);
+        m.insert("x-amz-website-redirect-location".to_string(), true);
+        m.insert("x-amz-object-lock-mode".to_string(), true);
+        m.insert("x-amz-metadata-directive".to_string(), true);
+        m.insert("x-amz-object-lock-retain-until-date".to_string(), true);
+        m.insert("expires".to_string(), true);
+        m.insert("x-amz-replication-status".to_string(), true);
+        m
+    };
+}
+
+pub fn is_storageclass_header(header_key: &str) -> bool {
+    header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
+}
+
+pub fn is_standard_header(header_key: &str) -> bool {
+    *SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
+}
+
+pub fn is_amz_header(header_key: &str) -> bool {
+    let key = header_key.to_lowercase();
+    key.starts_with("x-amz-meta-")
+        || key.starts_with("x-amz-grant-")
+        || key == "x-amz-acl"
+        || rustfs_utils::http::is_sse_header(header_key)
+        || key.starts_with("x-amz-checksum-")
+}
+
+pub fn is_rustfs_header(header_key: &str) -> bool {
+    header_key.to_lowercase().starts_with("x-rustfs-")
+}
+
+pub fn is_minio_header(header_key: &str) -> bool {
+    header_key.to_lowercase().starts_with("x-minio-")
+}
+
+/// Standard base64 (with `+`/`/` and `=` padding). Every base64 value this
+/// transition client emits or parses — `Content-MD5`, `x-amz-checksum-*`, and
+/// checksum digests in request/response bodies — is S3 wire format, which is
+/// standard base64. The URL-safe, unpadded alphabet used previously made remotes
+/// reject `Content-MD5` with "Invalid content MD5: Base64Error" and could not
+/// even decode a padded checksum coming back from the peer (rustfs/rustfs#4811).
+pub fn base64_encode(input: &[u8]) -> String {
+    base64_simd::STANDARD.encode_to_string(input)
+}
+
+pub fn base64_decode(input: &[u8]) -> Result<Vec<u8>, base64_simd::Error> {
+    base64_simd::STANDARD.decode_to_vec(input)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{base64_decode, base64_encode};
+
+    #[test]
+    fn base64_encode_is_standard_s3_wire_format() {
+        // S3 reads Content-MD5 / checksum values with a standard base64 decoder,
+        // so the encoder must emit '+'/'/' and '=' padding and round-trip through
+        // one. Regression for rustfs/rustfs#4811 ("Invalid content MD5:
+        // Base64Error"). 16-byte MD5-length input chosen to force '=' padding.
+        let digest: [u8; 16] = [
+            0xfb, 0xff, 0xff, 0xef, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0,
+        ];
+        let encoded = base64_encode(&digest);
+        assert!(encoded.ends_with('='), "16-byte input must be padded: {encoded}");
+        assert!(!encoded.contains(['-', '_']), "must use the standard alphabet: {encoded}");
+        let via_standard = base64_simd::STANDARD
+            .decode_to_vec(encoded.as_bytes())
+            .expect("standard decode");
+        assert_eq!(via_standard, digest);
+        // Our own decoder must accept the same wire format it produces.
+        assert_eq!(base64_decode(encoded.as_bytes()).expect("round-trip"), digest);
+    }
+}