refactor: Restructure project layout and clean up dependencies (#30)

This commit introduces a significant reorganization of the project structure to improve maintainability and clarity.

Key changes include:
- Adjusted the directory layout for a more logical module organization.
- Removed unused crate dependencies, reducing the overall project size and potentially speeding up build times.
- Updated import paths and configuration files to reflect the structural changes.

crates/ecstore/src/admin_server_info.rs

@@ -0,0 +1,396 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use crate::{
+    disk::endpoint::Endpoint,
+    global::{GLOBAL_BOOT_TIME, GLOBAL_Endpoints},
+    heal::{
+        data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend},
+        data_usage_cache::DataUsageCache,
+        heal_commands::{DRIVE_STATE_OK, DRIVE_STATE_UNFORMATTED},
+    },
+    new_object_layer_fn,
+    notification_sys::get_global_notification_sys,
+    store_api::StorageAPI,
+};
+use rustfs_common::{
+    // error::{Error, Result},
+    globals::GLOBAL_Local_Node_Name,
+};
+use rustfs_madmin::{
+    BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, InfoMessage, ServerProperties,
+};
+use rustfs_protos::{
+    models::{PingBody, PingBodyBuilder},
+    node_service_time_out_client,
+    proto_gen::node_service::{PingRequest, PingResponse},
+};
+use std::{
+    collections::{HashMap, HashSet},
+    time::SystemTime,
+};
+use time::OffsetDateTime;
+use tonic::Request;
+use tracing::warn;
+
+use shadow_rs::shadow;
+
+shadow!(build);
+
+// pub const ITEM_OFFLINE: &str = "offline";
+// pub const ITEM_INITIALIZING: &str = "initializing";
+// pub const ITEM_ONLINE: &str = "online";
+
+// #[derive(Debug, Default, Serialize, Deserialize)]
+// pub struct MemStats {
+//     alloc: u64,
+//     total_alloc: u64,
+//     mallocs: u64,
+//     frees: u64,
+//     heap_alloc: u64,
+// }
+
+// #[derive(Debug, Default, Serialize, Deserialize)]
+// pub struct ServerProperties {
+//     pub state: String,
+//     pub endpoint: String,
+//     pub scheme: String,
+//     pub uptime: u64,
+//     pub version: String,
+//     pub commit_id: String,
+//     pub network: HashMap<String, String>,
+//     pub disks: Vec<madmin::Disk>,
+//     pub pool_number: i32,
+//     pub pool_numbers: Vec<i32>,
+//     pub mem_stats: MemStats,
+//     pub max_procs: u64,
+//     pub num_cpu: u64,
+//     pub runtime_version: String,
+//     pub rustfs_env_vars: HashMap<String, String>,
+// }
+
+async fn is_server_resolvable(endpoint: &Endpoint) -> Result<()> {
+    let addr = format!(
+        "{}://{}:{}",
+        endpoint.url.scheme(),
+        endpoint.url.host_str().unwrap(),
+        endpoint.url.port().unwrap()
+    );
+    let mut fbb = flatbuffers::FlatBufferBuilder::new();
+    let payload = fbb.create_vector(b"hello world");
+
+    let mut builder = PingBodyBuilder::new(&mut fbb);
+    builder.add_payload(payload);
+    let root = builder.finish();
+    fbb.finish(root, None);
+
+    let finished_data = fbb.finished_data();
+
+    let decoded_payload = flatbuffers::root::<PingBody>(finished_data);
+    assert!(decoded_payload.is_ok());
+
+    // 创建客户端
+    let mut client = node_service_time_out_client(&addr)
+        .await
+        .map_err(|err| Error::other(err.to_string()))?;
+
+    // 构造 PingRequest
+    let request = Request::new(PingRequest {
+        version: 1,
+        body: bytes::Bytes::copy_from_slice(finished_data),
+    });
+
+    // 发送请求并获取响应
+    let response: PingResponse = client.ping(request).await?.into_inner();
+
+    // 打印响应
+    let ping_response_body = flatbuffers::root::<PingBody>(&response.body);
+    if let Err(e) = ping_response_body {
+        eprintln!("{e}");
+    } else {
+        println!("ping_resp:body(flatbuffer): {ping_response_body:?}");
+    }
+
+    Ok(())
+}
+
+pub async fn get_local_server_property() -> ServerProperties {
+    let addr = GLOBAL_Local_Node_Name.read().await.clone();
+    let mut pool_numbers = HashSet::new();
+    let mut network = HashMap::new();
+
+    let endpoints = match GLOBAL_Endpoints.get() {
+        Some(eps) => eps,
+        None => return ServerProperties::default(),
+    };
+    for ep in endpoints.as_ref().iter() {
+        for endpoint in ep.endpoints.as_ref().iter() {
+            let node_name = match endpoint.url.host_str() {
+                Some(s) => s.to_string(),
+                None => addr.clone(),
+            };
+            if endpoint.is_local {
+                pool_numbers.insert(endpoint.pool_idx + 1);
+                network.insert(node_name, ITEM_ONLINE.to_string());
+                continue;
+            }
+            if let std::collections::hash_map::Entry::Vacant(e) = network.entry(node_name) {
+                if is_server_resolvable(endpoint).await.is_err() {
+                    e.insert(ITEM_OFFLINE.to_string());
+                } else {
+                    e.insert(ITEM_ONLINE.to_string());
+                }
+            }
+        }
+    }
+
+    // todo: mem collect
+    // let mem_stats =
+
+    let mut props = ServerProperties {
+        endpoint: addr,
+        uptime: SystemTime::now()
+            .duration_since(*GLOBAL_BOOT_TIME.get().unwrap())
+            .unwrap_or_default()
+            .as_secs(),
+        network,
+        version: get_commit_id(),
+        ..Default::default()
+    };
+
+    for pool_num in pool_numbers.iter() {
+        props.pool_numbers.push(*pool_num);
+    }
+    props.pool_numbers.sort();
+    props.pool_number = if props.pool_numbers.len() == 1 {
+        props.pool_numbers[0]
+    } else {
+        i32::MAX
+    };
+
+    // let mut sensitive = HashSet::new();
+    // sensitive.insert(ENV_ACCESS_KEY.to_string());
+    // sensitive.insert(ENV_SECRET_KEY.to_string());
+    // sensitive.insert(ENV_ROOT_USER.to_string());
+    // sensitive.insert(ENV_ROOT_PASSWORD.to_string());
+
+    if let Some(store) = new_object_layer_fn() {
+        let storage_info = store.local_storage_info().await;
+        props.state = ITEM_ONLINE.to_string();
+        props.disks = storage_info.disks;
+    } else {
+        props.state = ITEM_INITIALIZING.to_string();
+    };
+
+    props
+}
+
+pub async fn get_server_info(get_pools: bool) -> InfoMessage {
+    let nowt: OffsetDateTime = OffsetDateTime::now_utc();
+
+    warn!("get_server_info start {:?}", nowt);
+
+    let local = get_local_server_property().await;
+
+    let after1 = OffsetDateTime::now_utc();
+
+    warn!("get_local_server_property end {:?}", after1 - nowt);
+
+    let mut servers = {
+        if let Some(sys) = get_global_notification_sys() {
+            sys.server_info().await
+        } else {
+            vec![]
+        }
+    };
+
+    let after2 = OffsetDateTime::now_utc();
+
+    warn!("server_info end {:?}", after2 - after1);
+    servers.push(local);
+
+    let mut buckets = rustfs_madmin::Buckets::default();
+    let mut objects = rustfs_madmin::Objects::default();
+    let mut versions = rustfs_madmin::Versions::default();
+    let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
+    let mut usage = rustfs_madmin::Usage::default();
+    let mut mode = ITEM_INITIALIZING;
+    let mut backend = rustfs_madmin::ErasureBackend::default();
+    let mut pools: HashMap<i32, HashMap<i32, ErasureSetInfo>> = HashMap::new();
+
+    if let Some(store) = new_object_layer_fn() {
+        mode = ITEM_ONLINE;
+        match load_data_usage_from_backend(store.clone()).await {
+            Ok(res) => {
+                buckets.count = res.buckets_count;
+                objects.count = res.objects_total_count;
+                versions.count = res.versions_total_count;
+                delete_markers.count = res.delete_markers_total_count;
+                usage.size = res.objects_total_size;
+            }
+            Err(err) => {
+                buckets.error = Some(err.to_string());
+                objects.error = Some(err.to_string());
+                versions.error = Some(err.to_string());
+                delete_markers.error = Some(err.to_string());
+                usage.error = Some(err.to_string());
+            }
+        }
+
+        let after3 = OffsetDateTime::now_utc();
+
+        warn!("load_data_usage_from_backend end {:?}", after3 - after2);
+
+        let backen_info = store.clone().backend_info().await;
+
+        let after4 = OffsetDateTime::now_utc();
+
+        warn!("backend_info end {:?}", after4 - after3);
+
+        let mut all_disks: Vec<Disk> = Vec::new();
+        for server in servers.iter() {
+            all_disks.extend(server.disks.clone());
+        }
+        let (online_disks, offline_disks) = get_online_offline_disks_stats(&all_disks);
+
+        let after5 = OffsetDateTime::now_utc();
+
+        warn!("get_online_offline_disks_stats end {:?}", after5 - after4);
+        backend = rustfs_madmin::ErasureBackend {
+            backend_type: rustfs_madmin::BackendType::ErasureType,
+            online_disks: online_disks.sum(),
+            offline_disks: offline_disks.sum(),
+            standard_sc_parity: backen_info.standard_sc_parity,
+            rr_sc_parity: backen_info.rr_sc_parity,
+            total_sets: backen_info.total_sets,
+            drives_per_set: backen_info.drives_per_set,
+        };
+        if get_pools {
+            pools = get_pools_info(&all_disks).await.unwrap_or_default();
+            let after6 = OffsetDateTime::now_utc();
+
+            warn!("get_pools_info end {:?}", after6 - after5);
+        }
+    }
+
+    let services = rustfs_madmin::Services::default();
+
+    InfoMessage {
+        mode: Some(mode.to_string()),
+        domain: None,
+        region: None,
+        sqs_arn: None,
+        deployment_id: None,
+        buckets: Some(buckets),
+        objects: Some(objects),
+        versions: Some(versions),
+        delete_markers: Some(delete_markers),
+        usage: Some(usage),
+        backend: Some(backend),
+        services: Some(services),
+        servers: Some(servers),
+        pools: Some(pools),
+    }
+}
+
+fn get_online_offline_disks_stats(disks_info: &[Disk]) -> (BackendDisks, BackendDisks) {
+    let mut online_disks: HashMap<String, usize> = HashMap::new();
+    let mut offline_disks: HashMap<String, usize> = HashMap::new();
+
+    for disk in disks_info {
+        let ep = &disk.endpoint;
+        offline_disks.entry(ep.clone()).or_insert(0);
+        online_disks.entry(ep.clone()).or_insert(0);
+    }
+
+    for disk in disks_info {
+        let ep = &disk.endpoint;
+        let state = &disk.state;
+        if *state != DRIVE_STATE_OK && *state != DRIVE_STATE_UNFORMATTED {
+            *offline_disks.get_mut(ep).unwrap() += 1;
+            continue;
+        }
+        *online_disks.get_mut(ep).unwrap() += 1;
+    }
+
+    let mut root_disk_count = 0;
+    for di in disks_info {
+        if di.root_disk {
+            root_disk_count += 1;
+        }
+    }
+
+    if disks_info.len() == (root_disk_count + offline_disks.values().sum::<usize>()) {
+        return (BackendDisks(online_disks), BackendDisks(offline_disks));
+    }
+
+    for disk in disks_info {
+        let ep = &disk.endpoint;
+        if disk.root_disk {
+            *offline_disks.get_mut(ep).unwrap() += 1;
+            *online_disks.get_mut(ep).unwrap() -= 1;
+        }
+    }
+
+    (BackendDisks(online_disks), BackendDisks(offline_disks))
+}
+
+async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32, ErasureSetInfo>>> {
+    let Some(store) = new_object_layer_fn() else {
+        return Err(Error::other("ServerNotInitialized"));
+    };
+
+    let mut pools_info: HashMap<i32, HashMap<i32, ErasureSetInfo>> = HashMap::new();
+    for d in all_disks {
+        let pool_info = pools_info.entry(d.pool_index).or_default();
+        let erasure_set = pool_info.entry(d.set_index).or_default();
+
+        if erasure_set.id == 0 {
+            erasure_set.id = d.set_index;
+            if let Ok(cache) = DataUsageCache::load(
+                &store.pools[d.pool_index as usize].disk_set[d.set_index as usize].clone(),
+                DATA_USAGE_CACHE_NAME,
+            )
+            .await
+            {
+                let data_usage_info = cache.dui(DATA_USAGE_ROOT, &[]);
+                erasure_set.objects_count = data_usage_info.objects_total_count;
+                erasure_set.versions_count = data_usage_info.versions_total_count;
+                erasure_set.delete_markers_count = data_usage_info.delete_markers_total_count;
+                erasure_set.usage = data_usage_info.objects_total_size;
+            };
+        }
+
+        erasure_set.raw_capacity += d.total_space;
+        erasure_set.raw_usage += d.used_space;
+        if d.healing {
+            erasure_set.heal_disks = 1;
+        }
+    }
+    Ok(pools_info)
+}
+
+#[allow(clippy::const_is_empty)]
+pub fn get_commit_id() -> String {
+    let ver = if !build::TAG.is_empty() {
+        build::TAG.to_string()
+    } else if !build::SHORT_COMMIT.is_empty() {
+        build::SHORT_COMMIT.to_string()
+    } else {
+        build::PKG_VERSION.to_string()
+    };
+
+    format!("{}@{}", build::COMMIT_DATE_3339, ver)
+}

crates/ecstore/src/bitrot.rs

@@ -0,0 +1,188 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::disk::error::DiskError;
+use crate::disk::{self, DiskAPI as _, DiskStore};
+use crate::erasure_coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
+use rustfs_utils::HashAlgorithm;
+use std::io::Cursor;
+use tokio::io::AsyncRead;
+
+/// Create a BitrotReader from either inline data or disk file stream
+///
+/// # Parameters
+/// * `inline_data` - Optional inline data, if present, will use Cursor to read from memory
+/// * `disk` - Optional disk reference for file stream reading  
+/// * `bucket` - Bucket name for file path
+/// * `path` - File path within the bucket
+/// * `offset` - Starting offset for reading
+/// * `length` - Length to read
+/// * `shard_size` - Shard size for erasure coding
+/// * `checksum_algo` - Hash algorithm for bitrot verification
+#[allow(clippy::too_many_arguments)]
+pub async fn create_bitrot_reader(
+    inline_data: Option<&[u8]>,
+    disk: Option<&DiskStore>,
+    bucket: &str,
+    path: &str,
+    offset: usize,
+    length: usize,
+    shard_size: usize,
+    checksum_algo: HashAlgorithm,
+) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
+    // Calculate the total length to read, including the checksum overhead
+    let length = length.div_ceil(shard_size) * checksum_algo.size() + length;
+
+    if let Some(data) = inline_data {
+        // Use inline data
+        let rd = Cursor::new(data.to_vec());
+        let reader = BitrotReader::new(Box::new(rd) as Box<dyn AsyncRead + Send + Sync + Unpin>, shard_size, checksum_algo);
+        Ok(Some(reader))
+    } else if let Some(disk) = disk {
+        // Read from disk
+        match disk.read_file_stream(bucket, path, offset, length).await {
+            Ok(rd) => {
+                let reader = BitrotReader::new(rd, shard_size, checksum_algo);
+                Ok(Some(reader))
+            }
+            Err(e) => Err(e),
+        }
+    } else {
+        // Neither inline data nor disk available
+        Ok(None)
+    }
+}
+
+/// Create a new BitrotWriterWrapper based on the provided parameters
+///
+/// # Parameters
+/// - `is_inline_buffer`: If true, creates an in-memory buffer writer; if false, uses disk storage
+/// - `disk`: Optional disk instance for file creation (used when is_inline_buffer is false)
+/// - `shard_size`: Size of each shard for bitrot calculation
+/// - `checksum_algo`: Hash algorithm to use for bitrot verification
+/// - `volume`: Volume/bucket name for disk storage
+/// - `path`: File path for disk storage
+/// - `length`: Expected file length for disk storage
+///
+/// # Returns
+/// A Result containing the BitrotWriterWrapper or an error
+pub async fn create_bitrot_writer(
+    is_inline_buffer: bool,
+    disk: Option<&DiskStore>,
+    volume: &str,
+    path: &str,
+    length: i64,
+    shard_size: usize,
+    checksum_algo: HashAlgorithm,
+) -> disk::error::Result<BitrotWriterWrapper> {
+    let writer = if is_inline_buffer {
+        CustomWriter::new_inline_buffer()
+    } else if let Some(disk) = disk {
+        let length = if length > 0 {
+            let length = length as usize;
+            (length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
+        } else {
+            0
+        };
+
+        let file = disk.create_file("", volume, path, length).await?;
+        CustomWriter::new_tokio_writer(file)
+    } else {
+        return Err(DiskError::DiskNotFound);
+    };
+
+    Ok(BitrotWriterWrapper::new(writer, shard_size, checksum_algo))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_create_bitrot_reader_with_inline_data() {
+        let test_data = b"hello world test data";
+        let shard_size = 16;
+        let checksum_algo = HashAlgorithm::HighwayHash256;
+
+        let result =
+            create_bitrot_reader(Some(test_data), None, "test-bucket", "test-path", 0, 0, shard_size, checksum_algo).await;
+
+        assert!(result.is_ok());
+        assert!(result.unwrap().is_some());
+    }
+
+    #[tokio::test]
+    async fn test_create_bitrot_reader_without_data_or_disk() {
+        let shard_size = 16;
+        let checksum_algo = HashAlgorithm::HighwayHash256;
+
+        let result = create_bitrot_reader(None, None, "test-bucket", "test-path", 0, 1024, shard_size, checksum_algo).await;
+
+        assert!(result.is_ok());
+        assert!(result.unwrap().is_none());
+    }
+
+    #[tokio::test]
+    async fn test_create_bitrot_writer_inline() {
+        use rustfs_utils::HashAlgorithm;
+
+        let wrapper = create_bitrot_writer(
+            true, // is_inline_buffer
+            None, // disk not needed for inline buffer
+            "test-volume",
+            "test-path",
+            1024, // length
+            1024, // shard_size
+            HashAlgorithm::HighwayHash256,
+        )
+        .await;
+
+        assert!(wrapper.is_ok());
+        let mut wrapper = wrapper.unwrap();
+
+        // Test writing some data
+        let test_data = b"hello world";
+        let result = wrapper.write(test_data).await;
+        assert!(result.is_ok());
+
+        // Test getting inline data
+        let inline_data = wrapper.into_inline_data();
+        assert!(inline_data.is_some());
+        // The inline data should contain both hash and data
+        let data = inline_data.unwrap();
+        assert!(!data.is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_create_bitrot_writer_disk_without_disk() {
+        use rustfs_utils::HashAlgorithm;
+
+        // Test error case: trying to create disk writer without providing disk instance
+        let wrapper = create_bitrot_writer(
+            false, // is_inline_buffer = false, so needs disk
+            None,  // disk = None, should cause error
+            "test-volume",
+            "test-path",
+            1024, // length
+            1024, // shard_size
+            HashAlgorithm::HighwayHash256,
+        )
+        .await;
+
+        assert!(wrapper.is_err());
+        let error = wrapper.unwrap_err();
+        println!("error: {error:?}");
+        assert_eq!(error, DiskError::DiskNotFound);
+    }
+}

crates/ecstore/src/bucket/error.rs

@@ -0,0 +1,106 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::Error;
+
+#[derive(Debug, thiserror::Error)]
+pub enum BucketMetadataError {
+    #[error("tagging not found")]
+    TaggingNotFound,
+    #[error("bucket policy not found")]
+    BucketPolicyNotFound,
+    #[error("bucket object lock not found")]
+    BucketObjectLockConfigNotFound,
+    #[error("bucket lifecycle not found")]
+    BucketLifecycleNotFound,
+    #[error("bucket SSE config not found")]
+    BucketSSEConfigNotFound,
+    #[error("bucket quota config not found")]
+    BucketQuotaConfigNotFound,
+    #[error("bucket replication config not found")]
+    BucketReplicationConfigNotFound,
+    #[error("bucket remote target not found")]
+    BucketRemoteTargetNotFound,
+
+    #[error("Io error: {0}")]
+    Io(std::io::Error),
+}
+
+impl BucketMetadataError {
+    pub fn other<E>(error: E) -> Self
+    where
+        E: Into<Box<dyn std::error::Error + Send + Sync>>,
+    {
+        BucketMetadataError::Io(std::io::Error::other(error))
+    }
+}
+
+impl From<Error> for BucketMetadataError {
+    fn from(e: Error) -> Self {
+        match e {
+            Error::Io(e) => e.into(),
+            _ => BucketMetadataError::other(e),
+        }
+    }
+}
+
+impl From<std::io::Error> for BucketMetadataError {
+    fn from(e: std::io::Error) -> Self {
+        e.downcast::<BucketMetadataError>().unwrap_or_else(BucketMetadataError::other)
+    }
+}
+
+impl PartialEq for BucketMetadataError {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (BucketMetadataError::Io(e1), BucketMetadataError::Io(e2)) => {
+                e1.kind() == e2.kind() && e1.to_string() == e2.to_string()
+            }
+            (e1, e2) => e1.to_u32() == e2.to_u32(),
+        }
+    }
+}
+
+impl Eq for BucketMetadataError {}
+
+impl BucketMetadataError {
+    pub fn to_u32(&self) -> u32 {
+        match self {
+            BucketMetadataError::TaggingNotFound => 0x01,
+            BucketMetadataError::BucketPolicyNotFound => 0x02,
+            BucketMetadataError::BucketObjectLockConfigNotFound => 0x03,
+            BucketMetadataError::BucketLifecycleNotFound => 0x04,
+            BucketMetadataError::BucketSSEConfigNotFound => 0x05,
+            BucketMetadataError::BucketQuotaConfigNotFound => 0x06,
+            BucketMetadataError::BucketReplicationConfigNotFound => 0x07,
+            BucketMetadataError::BucketRemoteTargetNotFound => 0x08,
+            BucketMetadataError::Io(_) => 0x09,
+        }
+    }
+
+    pub fn from_u32(error: u32) -> Option<Self> {
+        match error {
+            0x01 => Some(BucketMetadataError::TaggingNotFound),
+            0x02 => Some(BucketMetadataError::BucketPolicyNotFound),
+            0x03 => Some(BucketMetadataError::BucketObjectLockConfigNotFound),
+            0x04 => Some(BucketMetadataError::BucketLifecycleNotFound),
+            0x05 => Some(BucketMetadataError::BucketSSEConfigNotFound),
+            0x06 => Some(BucketMetadataError::BucketQuotaConfigNotFound),
+            0x07 => Some(BucketMetadataError::BucketReplicationConfigNotFound),
+            0x08 => Some(BucketMetadataError::BucketRemoteTargetNotFound),
+            0x09 => Some(BucketMetadataError::Io(std::io::Error::other("Io error"))),
+            _ => None,
+        }
+    }
+}

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs

@@ -0,0 +1,43 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::lifecycle;
+
+#[derive(Debug, Clone, Default)]
+pub enum LcEventSrc {
+    #[default]
+    None,
+    Heal,
+    Scanner,
+    Decom,
+    Rebal,
+    S3HeadObject,
+    S3GetObject,
+    S3ListObjects,
+    S3PutObject,
+    S3CopyObject,
+    S3CompleteMultipartUpload,
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct LcAuditEvent {
+    pub event: lifecycle::Event,
+    pub source: LcEventSrc,
+}
+
+impl LcAuditEvent {
+    pub fn new(event: lifecycle::Event, source: LcEventSrc) -> Self {
+        Self { event, source }
+    }
+}

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -0,0 +1,844 @@
+#![allow(unused_imports)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
+use futures::Future;
+use http::HeaderMap;
+use lazy_static::lazy_static;
+use s3s::Body;
+use sha2::{Digest, Sha256};
+use std::any::Any;
+use std::collections::HashMap;
+use std::env;
+use std::io::Write;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicI64, Ordering};
+use std::sync::{Arc, Mutex};
+use tokio::select;
+use tokio::sync::mpsc::{Receiver, Sender};
+use tokio::sync::{RwLock, mpsc};
+use tracing::{error, info};
+use uuid::Uuid;
+use xxhash_rust::xxh64;
+
+//use rustfs_notify::{BucketNotificationConfig, Event, EventName, LogLevel, NotificationError, init_logger};
+//use rustfs_notify::{initialize, notification_system};
+use super::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc};
+use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions};
+use super::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
+use super::tier_sweeper::{Jentry, delete_object_from_remote_tier};
+use crate::bucket::{metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys};
+use crate::client::object_api_utils::new_getobjectreader;
+use crate::error::Error;
+use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down};
+use crate::event::name::EventName;
+use crate::event_notification::{EventArgs, send_event};
+use crate::global::GLOBAL_LocalNodeName;
+use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
+use crate::heal::{
+    data_scanner::{apply_expiry_on_non_transitioned_objects, apply_expiry_on_transitioned_object},
+    data_scanner_metric::ScannerMetrics,
+    data_usage_cache::TierStats,
+};
+use crate::store::ECStore;
+use crate::store_api::StorageAPI;
+use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete};
+use crate::tier::warm_backend::WarmBackendGetOpts;
+use s3s::dto::BucketLifecycleConfiguration;
+
+pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
+pub type TraceFn =
+    Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
+pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
+
+static XXHASH_SEED: u64 = 0;
+
+const _DISABLED: &str = "Disabled";
+
+//pub const ERR_INVALID_STORAGECLASS: &str = "invalid storage class.";
+pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
+
+lazy_static! {
+    pub static ref GLOBAL_ExpiryState: Arc<RwLock<ExpiryState>> = ExpiryState::new();
+    pub static ref GLOBAL_TransitionState: Arc<TransitionState> = TransitionState::new();
+}
+
+pub struct LifecycleSys;
+
+impl LifecycleSys {
+    pub fn new() -> Arc<Self> {
+        Arc::new(Self)
+    }
+
+    pub async fn get(&self, bucket: &str) -> Option<BucketLifecycleConfiguration> {
+        let lc = get_lifecycle_config(bucket).await.expect("get_lifecycle_config err!").0;
+        Some(lc)
+    }
+
+    pub fn trace(_oi: &ObjectInfo) -> TraceFn {
+        todo!();
+    }
+}
+
+struct ExpiryTask {
+    obj_info: ObjectInfo,
+    event: lifecycle::Event,
+    src: LcEventSrc,
+}
+
+impl ExpiryOp for ExpiryTask {
+    fn op_hash(&self) -> u64 {
+        let mut hasher = Sha256::new();
+        let _ = hasher.write(format!("{}", self.obj_info.bucket).as_bytes());
+        let _ = hasher.write(format!("{}", self.obj_info.name).as_bytes());
+        hasher.flush();
+        xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+struct ExpiryStats {
+    missed_expiry_tasks: AtomicI64,
+    missed_freevers_tasks: AtomicI64,
+    missed_tier_journal_tasks: AtomicI64,
+    workers: AtomicI64,
+}
+
+#[allow(dead_code)]
+impl ExpiryStats {
+    pub fn missed_tasks(&self) -> i64 {
+        self.missed_expiry_tasks.load(Ordering::SeqCst)
+    }
+
+    fn missed_free_vers_tasks(&self) -> i64 {
+        self.missed_freevers_tasks.load(Ordering::SeqCst)
+    }
+
+    fn missed_tier_journal_tasks(&self) -> i64 {
+        self.missed_tier_journal_tasks.load(Ordering::SeqCst)
+    }
+
+    fn num_workers(&self) -> i64 {
+        self.workers.load(Ordering::SeqCst)
+    }
+}
+
+pub trait ExpiryOp: 'static {
+    fn op_hash(&self) -> u64;
+    fn as_any(&self) -> &dyn Any;
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct TransitionedObject {
+    pub name: String,
+    pub version_id: String,
+    pub tier: String,
+    pub free_version: bool,
+    pub status: String,
+}
+
+struct FreeVersionTask(ObjectInfo);
+
+impl ExpiryOp for FreeVersionTask {
+    fn op_hash(&self) -> u64 {
+        let mut hasher = Sha256::new();
+        let _ = hasher.write(format!("{}", self.0.transitioned_object.tier).as_bytes());
+        let _ = hasher.write(format!("{}", self.0.transitioned_object.name).as_bytes());
+        hasher.flush();
+        xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+struct NewerNoncurrentTask {
+    bucket: String,
+    versions: Vec<ObjectToDelete>,
+    event: lifecycle::Event,
+}
+
+impl ExpiryOp for NewerNoncurrentTask {
+    fn op_hash(&self) -> u64 {
+        let mut hasher = Sha256::new();
+        let _ = hasher.write(format!("{}", self.bucket).as_bytes());
+        let _ = hasher.write(format!("{}", self.versions[0].object_name).as_bytes());
+        hasher.flush();
+        xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+pub struct ExpiryState {
+    tasks_tx: Vec<Sender<Option<ExpiryOpType>>>,
+    tasks_rx: Vec<Arc<tokio::sync::Mutex<Receiver<Option<ExpiryOpType>>>>>,
+    stats: Option<ExpiryStats>,
+}
+
+impl ExpiryState {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new() -> Arc<RwLock<Self>> {
+        Arc::new(RwLock::new(Self {
+            tasks_tx: vec![],
+            tasks_rx: vec![],
+            stats: Some(ExpiryStats {
+                missed_expiry_tasks: AtomicI64::new(0),
+                missed_freevers_tasks: AtomicI64::new(0),
+                missed_tier_journal_tasks: AtomicI64::new(0),
+                workers: AtomicI64::new(0),
+            }),
+        }))
+    }
+
+    pub async fn pending_tasks(&self) -> usize {
+        let rxs = &self.tasks_rx;
+        if rxs.len() == 0 {
+            return 0;
+        }
+        let mut tasks = 0;
+        for rx in rxs.iter() {
+            tasks += rx.lock().await.len();
+        }
+        tasks
+    }
+
+    pub async fn enqueue_tier_journal_entry(&mut self, je: &Jentry) -> Result<(), std::io::Error> {
+        let wrkr = self.get_worker_ch(je.op_hash());
+        if wrkr.is_none() {
+            *self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
+        }
+        let wrkr = wrkr.expect("err");
+        select! {
+            //_ -> GlobalContext.Done() => ()
+            _ = wrkr.send(Some(Box::new(je.clone()))) => (),
+            else => {
+                *self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
+            }
+        }
+        return Ok(());
+    }
+
+    pub async fn enqueue_free_version(&mut self, oi: ObjectInfo) {
+        let task = FreeVersionTask(oi);
+        let wrkr = self.get_worker_ch(task.op_hash());
+        if wrkr.is_none() {
+            *self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
+            return;
+        }
+        let wrkr = wrkr.expect("err!");
+        select! {
+            //_ -> GlobalContext.Done() => {}
+            _ = wrkr.send(Some(Box::new(task))) => (),
+            else => {
+                *self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
+            }
+        }
+    }
+
+    pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
+        let task = ExpiryTask {
+            obj_info: oi.clone(),
+            event: event.clone(),
+            src: src.clone(),
+        };
+        let wrkr = self.get_worker_ch(task.op_hash());
+        if wrkr.is_none() {
+            *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
+            return;
+        }
+        let wrkr = wrkr.expect("err!");
+        select! {
+            //_ -> GlobalContext.Done() => {}
+            _ = wrkr.send(Some(Box::new(task))) => (),
+            else => {
+                *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
+            }
+        }
+    }
+
+    pub async fn enqueue_by_newer_noncurrent(&mut self, bucket: &str, versions: Vec<ObjectToDelete>, lc_event: lifecycle::Event) {
+        if versions.len() == 0 {
+            return;
+        }
+
+        let task = NewerNoncurrentTask {
+            bucket: String::from(bucket),
+            versions,
+            event: lc_event,
+        };
+        let wrkr = self.get_worker_ch(task.op_hash());
+        if wrkr.is_none() {
+            *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
+            return;
+        }
+        let wrkr = wrkr.expect("err!");
+        select! {
+            //_ -> GlobalContext.Done() => {}
+            _ = wrkr.send(Some(Box::new(task))) => (),
+            else => {
+                *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
+            }
+        }
+    }
+
+    pub fn get_worker_ch(&self, h: u64) -> Option<Sender<Option<ExpiryOpType>>> {
+        if self.tasks_tx.len() == 0 {
+            return None;
+        }
+        Some(self.tasks_tx[h as usize % self.tasks_tx.len()].clone())
+    }
+
+    pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
+        if n == GLOBAL_ExpiryState.read().await.tasks_tx.len() || n < 1 {
+            return;
+        }
+
+        let mut state = GLOBAL_ExpiryState.write().await;
+
+        while state.tasks_tx.len() < n {
+            let (tx, rx) = mpsc::channel(10000);
+            let api = api.clone();
+            let rx = Arc::new(tokio::sync::Mutex::new(rx));
+            state.tasks_tx.push(tx);
+            state.tasks_rx.push(rx.clone());
+            *state.stats.as_mut().expect("err").workers.get_mut() += 1;
+            tokio::spawn(async move {
+                let mut rx = rx.lock().await;
+                //let mut expiry_state = GLOBAL_ExpiryState.read().await;
+                ExpiryState::worker(&mut *rx, api).await;
+            });
+        }
+
+        let mut l = state.tasks_tx.len();
+        while l > n {
+            let worker = state.tasks_tx[l - 1].clone();
+            worker.send(None).await.unwrap_or(());
+            state.tasks_tx.remove(l - 1);
+            state.tasks_rx.remove(l - 1);
+            *state.stats.as_mut().expect("err").workers.get_mut() -= 1;
+            l -= 1;
+        }
+    }
+
+    pub async fn worker(rx: &mut Receiver<Option<ExpiryOpType>>, api: Arc<ECStore>) {
+        loop {
+            select! {
+                _ = tokio::signal::ctrl_c() => {
+                    info!("got ctrl+c, exits");
+                    break;
+                }
+                v = rx.recv() => {
+                    if v.is_none() {
+                        break;
+                    }
+                    let v = v.expect("err!");
+                    if v.is_none() {
+                        //rx.close();
+                        //drop(rx);
+                        let _ = rx;
+                        return;
+                    }
+                    let v = v.expect("err!");
+                    if v.as_any().is::<ExpiryTask>() {
+                        let v = v.as_any().downcast_ref::<ExpiryTask>().expect("err!");
+                        if v.obj_info.transitioned_object.status != "" {
+                            apply_expiry_on_transitioned_object(api.clone(), &v.obj_info, &v.event, &v.src).await;
+                        } else {
+                            apply_expiry_on_non_transitioned_objects(api.clone(), &v.obj_info, &v.event, &v.src).await;
+                        }
+                    }
+                    else if v.as_any().is::<NewerNoncurrentTask>() {
+                        let _v = v.as_any().downcast_ref::<NewerNoncurrentTask>().expect("err!");
+                        //delete_object_versions(api, &v.bucket, &v.versions, v.event).await;
+                    }
+                    else if v.as_any().is::<Jentry>() {
+                        //transitionLogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
+                    }
+                    else if v.as_any().is::<FreeVersionTask>() {
+                        let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("err!");
+                        let _oi = v.0.clone();
+
+                    }
+                    else {
+                        //info!("Invalid work type - {:?}", v);
+                        todo!();
+                    }
+                }
+            }
+        }
+    }
+}
+
+struct TransitionTask {
+    obj_info: ObjectInfo,
+    src: LcEventSrc,
+    event: lifecycle::Event,
+}
+
+impl ExpiryOp for TransitionTask {
+    fn op_hash(&self) -> u64 {
+        let mut hasher = Sha256::new();
+        let _ = hasher.write(format!("{}", self.obj_info.bucket).as_bytes());
+        //let _ = hasher.write(format!("{}", self.obj_info.versions[0].object_name).as_bytes());
+        hasher.flush();
+        xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+pub struct TransitionState {
+    transition_tx: A_Sender<Option<TransitionTask>>,
+    transition_rx: A_Receiver<Option<TransitionTask>>,
+    pub num_workers: AtomicI64,
+    kill_tx: A_Sender<()>,
+    kill_rx: A_Receiver<()>,
+    active_tasks: AtomicI64,
+    missed_immediate_tasks: AtomicI64,
+    last_day_stats: Arc<Mutex<HashMap<String, LastDayTierStats>>>,
+}
+
+impl TransitionState {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new() -> Arc<Self> {
+        let (tx1, rx1) = bounded(100000);
+        let (tx2, rx2) = bounded(1);
+        Arc::new(Self {
+            transition_tx: tx1,
+            transition_rx: rx1,
+            num_workers: AtomicI64::new(0),
+            kill_tx: tx2,
+            kill_rx: rx2,
+            active_tasks: AtomicI64::new(0),
+            missed_immediate_tasks: AtomicI64::new(0),
+            last_day_stats: Arc::new(Mutex::new(HashMap::new())),
+        })
+    }
+
+    pub async fn queue_transition_task(&self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
+        let task = TransitionTask {
+            obj_info: oi.clone(),
+            src: src.clone(),
+            event: event.clone(),
+        };
+        select! {
+            //_ -> t.ctx.Done() => (),
+            _ = self.transition_tx.send(Some(task)) => (),
+            else => {
+                match src {
+                    LcEventSrc::S3PutObject | LcEventSrc::S3CopyObject | LcEventSrc::S3CompleteMultipartUpload => {
+                        self.missed_immediate_tasks.fetch_add(1, Ordering::SeqCst);
+                    }
+                    _ => ()
+                }
+            },
+        }
+    }
+
+    pub async fn init(api: Arc<ECStore>) {
+        let mut n = 10; //globalAPIConfig.getTransitionWorkers();
+        let tw = 10; //globalILMConfig.getTransitionWorkers(); 
+        if tw > 0 {
+            n = tw;
+        }
+
+        //let mut transition_state = GLOBAL_TransitionState.write().await;
+        //self.objAPI = objAPI
+        Self::update_workers(api, n).await;
+    }
+
+    pub fn pending_tasks(&self) -> usize {
+        //let transition_rx = GLOBAL_TransitionState.transition_rx.lock().unwrap();
+        let transition_rx = &GLOBAL_TransitionState.transition_rx;
+        transition_rx.len()
+    }
+
+    pub fn active_tasks(&self) -> i64 {
+        self.active_tasks.load(Ordering::SeqCst)
+    }
+
+    pub fn missed_immediate_tasks(&self) -> i64 {
+        self.missed_immediate_tasks.load(Ordering::SeqCst)
+    }
+
+    pub async fn worker(api: Arc<ECStore>) {
+        loop {
+            select! {
+                _ = GLOBAL_TransitionState.kill_rx.recv() => {
+                    return;
+                }
+                task = GLOBAL_TransitionState.transition_rx.recv() => {
+                    if task.is_err() {
+                        break;
+                    }
+                    let task = task.expect("err!");
+                    if task.is_none() {
+                        //self.transition_rx.close();
+                        //drop(self.transition_rx);
+                        return;
+                    }
+                    let task = task.expect("err!");
+                    if task.as_any().is::<TransitionTask>() {
+                        let task = task.as_any().downcast_ref::<TransitionTask>().expect("err!");
+
+                        GLOBAL_TransitionState.active_tasks.fetch_add(1, Ordering::SeqCst);
+                        if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await {
+                            if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
+                                error!("Transition to {} failed for {}/{} version:{} with {}",
+                                    task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.expect("err"), err.to_string());
+                            }
+                        } else {
+                            let mut ts = TierStats {
+                                total_size: task.obj_info.size as u64,
+                                num_versions: 1,
+                                ..Default::default()
+                            };
+                            if task.obj_info.is_latest {
+                                ts.num_objects = 1;
+                            }
+                            GLOBAL_TransitionState.add_lastday_stats(&task.event.storage_class, ts);
+                        }
+                        GLOBAL_TransitionState.active_tasks.fetch_add(-1, Ordering::SeqCst);
+                    }
+                }
+                else => ()
+            }
+        }
+    }
+
+    pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) {
+        let mut tier_stats = self.last_day_stats.lock().unwrap();
+        tier_stats
+            .entry(tier.to_string())
+            .and_modify(|e| e.add_stats(ts))
+            .or_insert(LastDayTierStats::default());
+    }
+
+    pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats {
+        let tier_stats = self.last_day_stats.lock().unwrap();
+        let mut res = DailyAllTierStats::with_capacity(tier_stats.len());
+        for (tier, st) in tier_stats.iter() {
+            res.insert(tier.clone(), st.clone());
+        }
+        res
+    }
+
+    pub async fn update_workers(api: Arc<ECStore>, n: i64) {
+        Self::update_workers_inner(api, n).await;
+    }
+
+    pub async fn update_workers_inner(api: Arc<ECStore>, n: i64) {
+        let mut n = n;
+        if n == 0 {
+            n = 100;
+        }
+
+        let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
+        while num_workers < n {
+            let clone_api = api.clone();
+            tokio::spawn(async move {
+                TransitionState::worker(clone_api).await;
+            });
+            num_workers = num_workers + 1;
+            GLOBAL_TransitionState.num_workers.fetch_add(1, Ordering::SeqCst);
+        }
+
+        let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
+        while num_workers > n {
+            let worker = GLOBAL_TransitionState.kill_tx.clone();
+            worker.send(()).await;
+            num_workers = num_workers - 1;
+            GLOBAL_TransitionState.num_workers.fetch_add(-1, Ordering::SeqCst);
+        }
+    }
+}
+
+pub async fn init_background_expiry(api: Arc<ECStore>) {
+    let mut workers = num_cpus::get() / 2;
+    //globalILMConfig.getExpirationWorkers()
+    if let Ok(env_expiration_workers) = env::var("_RUSTFS_EXPIRATION_WORKERS") {
+        if let Ok(num_expirations) = env_expiration_workers.parse::<usize>() {
+            workers = num_expirations;
+        }
+    }
+
+    if workers == 0 {
+        workers = 100;
+    }
+
+    //let expiry_state = GLOBAL_ExpiryStSate.write().await;
+    ExpiryState::resize_workers(workers, api).await;
+}
+
+pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Result<(), std::io::Error> {
+    for rule in &lc.rules {
+        if let Some(transitions) = &rule.transitions {
+            for transition in transitions {
+                if let Some(storage_class) = &transition.storage_class {
+                    if storage_class.as_str() != "" {
+                        let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
+                        if !valid {
+                            return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
+                        }
+                    }
+                }
+            }
+        }
+        if let Some(noncurrent_version_transitions) = &rule.noncurrent_version_transitions {
+            for noncurrent_version_transition in noncurrent_version_transitions {
+                if let Some(storage_class) = &noncurrent_version_transition.storage_class {
+                    if storage_class.as_str() != "" {
+                        let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
+                        if !valid {
+                            return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
+                        }
+                    }
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
+    let lc = GLOBAL_LifecycleSys.get(&oi.bucket).await;
+    if !lc.is_none() {
+        let event = lc.expect("err").eval(&oi.to_lifecycle_opts()).await;
+        match event.action {
+            lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => {
+                if oi.delete_marker || oi.is_dir {
+                    return;
+                }
+                GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await;
+            }
+            _ => (),
+        }
+    }
+}
+
+pub async fn expire_transitioned_object(
+    api: Arc<ECStore>,
+    oi: &ObjectInfo,
+    lc_event: &lifecycle::Event,
+    _src: &LcEventSrc,
+) -> Result<ObjectInfo, std::io::Error> {
+    //let traceFn = GLOBAL_LifecycleSys.trace(oi);
+    let mut opts = ObjectOptions {
+        versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
+        expiration: ExpirationOptions { expire: true },
+        ..Default::default()
+    };
+    if lc_event.action == IlmAction::DeleteVersionAction {
+        opts.version_id = oi.version_id.map(|id| id.to_string());
+    }
+    //let tags = LcAuditEvent::new(src, lcEvent).Tags();
+    if lc_event.action == IlmAction::DeleteRestoredAction {
+        opts.transition.expire_restored = true;
+        match api.delete_object(&oi.bucket, &oi.name, opts).await {
+            Ok(dobj) => {
+                //audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn);
+                return Ok(dobj);
+            }
+            Err(err) => return Err(std::io::Error::other(err)),
+        }
+    }
+
+    let ret = delete_object_from_remote_tier(
+        &oi.transitioned_object.name,
+        &oi.transitioned_object.version_id,
+        &oi.transitioned_object.tier,
+    )
+    .await;
+    if ret.is_ok() {
+        opts.skip_decommissioned = true;
+    } else {
+        //transitionLogIf(ctx, err);
+    }
+
+    let dobj = api.delete_object(&oi.bucket, &oi.name, opts).await?;
+
+    //defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
+
+    let mut event_name = EventName::ObjectRemovedDelete;
+    if oi.delete_marker {
+        event_name = EventName::ObjectRemovedDeleteMarkerCreated;
+    }
+    let obj_info = ObjectInfo {
+        name: oi.name.clone(),
+        version_id: oi.version_id,
+        delete_marker: oi.delete_marker,
+        ..Default::default()
+    };
+    send_event(EventArgs {
+        event_name: event_name.as_ref().to_string(),
+        bucket_name: obj_info.bucket.clone(),
+        object: obj_info,
+        user_agent: "Internal: [ILM-Expiry]".to_string(),
+        host: GLOBAL_LocalNodeName.to_string(),
+        ..Default::default()
+    });
+    /*let system = match notification_system() {
+        Some(sys) => sys,
+        None => {
+            let config = Config::new();
+            initialize(config).await?;
+            notification_system().expect("Failed to initialize notification system")
+        }
+    };
+    let event = Arc::new(Event::new_test_event("my-bucket", "document.pdf", EventName::ObjectCreatedPut));
+    system.send_event(event).await;*/
+
+    Ok(dobj)
+}
+
+pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
+    let us = Uuid::new_v4().to_string();
+    let mut hasher = Sha256::new();
+    let _ = hasher.write(format!("{}/{}", get_global_deployment_id().unwrap_or_default(), bucket).as_bytes());
+    hasher.flush();
+    let hash = rustfs_utils::crypto::hex(hasher.clone().finalize().as_slice());
+    let obj = format!("{}/{}/{}/{}", &hash[0..16], &us[0..2], &us[2..4], &us);
+    Ok(obj)
+}
+
+pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> {
+    let time_ilm = ScannerMetrics::time_ilm(lae.event.action);
+
+    let opts = ObjectOptions {
+        transition: TransitionOptions {
+            status: lifecycle::TRANSITION_PENDING.to_string(),
+            tier: lae.event.storage_class,
+            etag: oi.etag.clone().expect("err").to_string(),
+            ..Default::default()
+        },
+        //lifecycle_audit_event: lae,
+        version_id: Some(oi.version_id.expect("err").to_string()),
+        versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
+        version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await,
+        mod_time: oi.mod_time,
+        ..Default::default()
+    };
+    time_ilm(1);
+    api.transition_object(&oi.bucket, &oi.name, &opts).await
+}
+
+pub fn audit_tier_actions(_api: ECStore, _tier: &str, _bytes: i64) -> TimeFn {
+    todo!();
+}
+
+pub async fn get_transitioned_object_reader(
+    bucket: &str,
+    object: &str,
+    rs: HTTPRangeSpec,
+    h: HeaderMap,
+    oi: ObjectInfo,
+    opts: &ObjectOptions,
+) -> Result<GetObjectReader, std::io::Error> {
+    let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
+    let tgt_client = match tier_config_mgr.get_driver(&oi.transitioned_object.tier).await {
+        Ok(d) => d,
+        Err(err) => return Err(std::io::Error::other(err)),
+    };
+
+    let ret = new_getobjectreader(rs, &oi, opts, &h);
+    if let Err(err) = ret {
+        return Err(error_resp_to_object_err(err, vec![bucket, object]));
+    }
+    let (get_fn, off, length) = ret.expect("err");
+    let mut gopts = WarmBackendGetOpts::default();
+
+    if off >= 0 && length >= 0 {
+        gopts.start_offset = off;
+        gopts.length = length;
+    }
+
+    //return Ok(HttpFileReader::new(rs, &oi, opts, &h));
+    //timeTierAction := auditTierActions(oi.transitioned_object.Tier, length)
+    let reader = tgt_client
+        .get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts)
+        .await?;
+    Ok(get_fn(reader, h))
+}
+
+pub fn post_restore_opts(_r: http::Request<Body>, _bucket: &str, _object: &str) -> Result<ObjectOptions, std::io::Error> {
+    todo!();
+}
+
+pub fn put_restore_opts(_bucket: &str, _object: &str, _rreq: &RestoreObjectRequest, _oi: &ObjectInfo) -> ObjectOptions {
+    todo!();
+}
+
+pub trait LifecycleOps {
+    fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts;
+}
+
+impl LifecycleOps for ObjectInfo {
+    fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts {
+        lifecycle::ObjectOpts {
+            name: self.name.clone(),
+            user_tags: self.user_tags.clone(),
+            version_id: self.version_id.expect("err").to_string(),
+            mod_time: self.mod_time,
+            size: self.size as usize,
+            is_latest: self.is_latest,
+            num_versions: self.num_versions,
+            delete_marker: self.delete_marker,
+            successor_mod_time: self.successor_mod_time,
+            //restore_ongoing:    self.restore_ongoing,
+            //restore_expires:    self.restore_expires,
+            transition_status: self.transitioned_object.status.clone(),
+            ..Default::default()
+        }
+    }
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct S3Location {
+    pub bucketname: String,
+    //pub encryption:    Encryption,
+    pub prefix: String,
+    pub storage_class: String,
+    //pub tagging:       Tags,
+    pub user_metadata: HashMap<String, String>,
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct OutputLocation(pub S3Location);
+
+#[derive(Debug, Default, Clone)]
+pub struct RestoreObjectRequest {
+    pub days: i64,
+    pub ror_type: String,
+    pub tier: String,
+    pub description: String,
+    //pub select_parameters: SelectParameters,
+    pub output_location: OutputLocation,
+}
+
+const _MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20;

crates/ecstore/src/bucket/lifecycle/lifecycle.rs

@@ -0,0 +1,729 @@
+#![allow(unused_imports)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use s3s::dto::{
+    BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionTransition,
+    ObjectLockConfiguration, ObjectLockEnabled, Transition,
+};
+use std::cmp::Ordering;
+use std::env;
+use std::fmt::Display;
+use time::macros::{datetime, offset};
+use time::{self, Duration, OffsetDateTime};
+
+use crate::bucket::lifecycle::rule::TransitionOps;
+
+use super::bucket_lifecycle_ops::RestoreObjectRequest;
+
+pub const TRANSITION_COMPLETE: &str = "complete";
+pub const TRANSITION_PENDING: &str = "pending";
+
+const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration allows a maximum of 1000 rules";
+const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
+const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
+const _ERR_XML_NOT_WELL_FORMED: &str =
+    "The XML you provided was not well-formed or did not validate against our published schema";
+const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
+    "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum IlmAction {
+    NoneAction = 0,
+    DeleteAction,
+    DeleteVersionAction,
+    TransitionAction,
+    TransitionVersionAction,
+    DeleteRestoredAction,
+    DeleteRestoredVersionAction,
+    DeleteAllVersionsAction,
+    DelMarkerDeleteAllVersionsAction,
+    ActionCount,
+}
+
+impl IlmAction {
+    pub fn delete_restored(&self) -> bool {
+        *self == Self::DeleteRestoredAction || *self == Self::DeleteRestoredVersionAction
+    }
+
+    pub fn delete_versioned(&self) -> bool {
+        *self == Self::DeleteVersionAction || *self == Self::DeleteRestoredVersionAction
+    }
+
+    pub fn delete_all(&self) -> bool {
+        *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction
+    }
+
+    pub fn delete(&self) -> bool {
+        if self.delete_restored() {
+            return true;
+        }
+        *self == Self::DeleteVersionAction
+            || *self == Self::DeleteAction
+            || *self == Self::DeleteAllVersionsAction
+            || *self == Self::DelMarkerDeleteAllVersionsAction
+    }
+}
+
+impl Display for IlmAction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+#[async_trait::async_trait]
+pub trait RuleValidate {
+    fn validate(&self) -> Result<(), std::io::Error>;
+}
+
+#[async_trait::async_trait]
+impl RuleValidate for LifecycleRule {
+    /*fn validate_id(&self) -> Result<()> {
+        if self.id.len() > 255 {
+            return errInvalidRuleID;
+        }
+        Ok(())
+    }
+
+    fn validate_status(&self) -> Result<()> {
+        if self.Status.len() == 0 {
+            return errEmptyRuleStatus;
+        }
+
+        if self.Status != Enabled && self.Status != Disabled {
+            return errInvalidRuleStatus;
+        }
+        Ok(())
+    }
+
+    fn validate_expiration(&self) -> Result<()> {
+        self.Expiration.Validate();
+    }
+
+    fn validate_noncurrent_expiration(&self) -> Result<()> {
+        self.NoncurrentVersionExpiration.Validate()
+    }
+
+    fn validate_prefix_and_filter(&self) -> Result<()> {
+        if !self.Prefix.set && self.Filter.IsEmpty() || self.Prefix.set && !self.Filter.IsEmpty() {
+            return errXMLNotWellFormed;
+        }
+        if !self.Prefix.set {
+            return self.Filter.Validate();
+        }
+        Ok(())
+    }
+
+    fn validate_transition(&self) -> Result<()> {
+        self.Transition.Validate()
+    }
+
+    fn validate_noncurrent_transition(&self) -> Result<()> {
+        self.NoncurrentVersionTransition.Validate()
+    }
+
+    fn get_prefix(&self) -> String {
+        if p := self.Prefix.String(); p != "" {
+          return p
+        }
+        if p := self.Filter.Prefix.String(); p != "" {
+          return p
+        }
+        if p := self.Filter.And.Prefix.String(); p != "" {
+          return p
+        }
+        "".to_string()
+    }*/
+
+    fn validate(&self) -> Result<(), std::io::Error> {
+        /*self.validate_id()?;
+        self.validate_status()?;
+        self.validate_expiration()?;
+        self.validate_noncurrent_expiration()?;
+        self.validate_prefix_and_filter()?;
+        self.validate_transition()?;
+        self.validate_noncurrent_transition()?;
+        if (!self.Filter.Tag.IsEmpty() || len(self.Filter.And.Tags) != 0) && !self.delmarker_expiration.Empty() {
+          return errInvalidRuleDelMarkerExpiration
+        }
+        if !self.expiration.set && !self.transition.set && !self.noncurrent_version_expiration.set && !self.noncurrent_version_transitions.unwrap()[0].set && self.delmarker_expiration.Empty() {
+          return errXMLNotWellFormed
+        }*/
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+pub trait Lifecycle {
+    async fn has_transition(&self) -> bool;
+    fn has_expiry(&self) -> bool;
+    async fn has_active_rules(&self, prefix: &str) -> bool;
+    async fn validate(&self, lr_retention: bool) -> Result<(), std::io::Error>;
+    async fn filter_rules(&self, obj: &ObjectOpts) -> Option<Vec<LifecycleRule>>;
+    async fn eval(&self, obj: &ObjectOpts) -> Event;
+    async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event;
+    //fn set_prediction_headers(&self, w: http.ResponseWriter, obj: ObjectOpts);
+    async fn noncurrent_versions_expiration_limit(&self, obj: &ObjectOpts) -> Event;
+}
+
+#[async_trait::async_trait]
+impl Lifecycle for BucketLifecycleConfiguration {
+    async fn has_transition(&self) -> bool {
+        for rule in self.rules.iter() {
+            if !rule.transitions.is_none() {
+                return true;
+            }
+        }
+        false
+    }
+
+    fn has_expiry(&self) -> bool {
+        for rule in self.rules.iter() {
+            if !rule.expiration.is_none() || !rule.noncurrent_version_expiration.is_none() {
+                return true;
+            }
+        }
+        false
+    }
+
+    async fn has_active_rules(&self, prefix: &str) -> bool {
+        if self.rules.len() == 0 {
+            return false;
+        }
+        for rule in self.rules.iter() {
+            if rule.status.as_str() == ExpirationStatus::DISABLED {
+                continue;
+            }
+
+            let rule_prefix = rule.prefix.as_ref().expect("err!");
+            if prefix.len() > 0 && rule_prefix.len() > 0 && !prefix.starts_with(rule_prefix) && !rule_prefix.starts_with(&prefix)
+            {
+                continue;
+            }
+
+            let rule_noncurrent_version_expiration = rule.noncurrent_version_expiration.as_ref().expect("err!");
+            if rule_noncurrent_version_expiration.noncurrent_days.expect("err!") > 0 {
+                return true;
+            }
+            if rule_noncurrent_version_expiration.newer_noncurrent_versions.expect("err!") > 0 {
+                return true;
+            }
+            if !rule.noncurrent_version_transitions.is_none() {
+                return true;
+            }
+            let rule_expiration = rule.expiration.as_ref().expect("err!");
+            if !rule_expiration.date.is_none()
+                && OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp()
+                    < OffsetDateTime::now_utc().unix_timestamp()
+            {
+                return true;
+            }
+            if !rule_expiration.date.is_none() {
+                return true;
+            }
+            if rule_expiration.expired_object_delete_marker.expect("err!") {
+                return true;
+            }
+            let rule_transitions: &[Transition] = &rule.transitions.as_ref().expect("err!");
+            let rule_transitions_0 = rule_transitions[0].clone();
+            if !rule_transitions_0.date.is_none()
+                && OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp()
+                    < OffsetDateTime::now_utc().unix_timestamp()
+            {
+                return true;
+            }
+            if !rule.transitions.is_none() {
+                return true;
+            }
+        }
+        false
+    }
+
+    async fn validate(&self, lr_retention: bool) -> Result<(), std::io::Error> {
+        if self.rules.len() > 1000 {
+            return Err(std::io::Error::other(ERR_LIFECYCLE_TOO_MANY_RULES));
+        }
+        if self.rules.len() == 0 {
+            return Err(std::io::Error::other(ERR_LIFECYCLE_NO_RULE));
+        }
+
+        for r in &self.rules {
+            r.validate()?;
+            if let Some(expiration) = r.expiration.as_ref() {
+                if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
+                    if lr_retention && (!expired_object_delete_marker) {
+                        return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED));
+                    }
+                }
+            }
+        }
+        for (i, _) in self.rules.iter().enumerate() {
+            if i == self.rules.len() - 1 {
+                break;
+            }
+            let other_rules = &self.rules[i + 1..];
+            for other_rule in other_rules {
+                if self.rules[i].id == other_rule.id {
+                    return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID));
+                }
+            }
+        }
+        Ok(())
+    }
+
+    async fn filter_rules(&self, obj: &ObjectOpts) -> Option<Vec<LifecycleRule>> {
+        if obj.name == "" {
+            return None;
+        }
+        let mut rules = Vec::<LifecycleRule>::new();
+        for rule in self.rules.iter() {
+            if rule.status.as_str() == ExpirationStatus::DISABLED {
+                continue;
+            }
+            if let Some(prefix) = rule.prefix.clone() {
+                if !obj.name.starts_with(prefix.as_str()) {
+                    continue;
+                }
+            }
+            /*if !rule.filter.test_tags(obj.user_tags) {
+                continue;
+            }*/
+            //if !obj.delete_marker && !rule.filter.BySize(obj.size) {
+            if !obj.delete_marker && false {
+                continue;
+            }
+            rules.push(rule.clone());
+        }
+        Some(rules)
+    }
+
+    async fn eval(&self, obj: &ObjectOpts) -> Event {
+        self.eval_inner(obj, OffsetDateTime::now_utc()).await
+    }
+
+    async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event {
+        let mut events = Vec::<Event>::new();
+        if obj.mod_time.expect("err").unix_timestamp() == 0 {
+            return Event::default();
+        }
+
+        if let Some(restore_expires) = obj.restore_expires {
+            if !restore_expires.unix_timestamp() == 0 && now.unix_timestamp() > restore_expires.unix_timestamp() {
+                let mut action = IlmAction::DeleteRestoredAction;
+                if !obj.is_latest {
+                    action = IlmAction::DeleteRestoredVersionAction;
+                }
+
+                events.push(Event {
+                    action,
+                    due: Some(now),
+                    rule_id: "".into(),
+                    noncurrent_days: 0,
+                    newer_noncurrent_versions: 0,
+                    storage_class: "".into(),
+                });
+            }
+        }
+
+        if let Some(ref lc_rules) = self.filter_rules(obj).await {
+            for rule in lc_rules.iter() {
+                if obj.expired_object_deletemarker() {
+                    if let Some(expiration) = rule.expiration.as_ref() {
+                        if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
+                            events.push(Event {
+                                action: IlmAction::DeleteVersionAction,
+                                rule_id: rule.id.clone().expect("err!"),
+                                due: Some(now),
+                                noncurrent_days: 0,
+                                newer_noncurrent_versions: 0,
+                                storage_class: "".into(),
+                            });
+                            break;
+                        }
+                    }
+
+                    if let Some(expiration) = rule.expiration.as_ref() {
+                        if let Some(days) = expiration.days {
+                            let expected_expiry = expected_expiry_time(obj.mod_time.expect("err!"), days /*, date*/);
+                            if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
+                                events.push(Event {
+                                    action: IlmAction::DeleteVersionAction,
+                                    rule_id: rule.id.clone().expect("err!"),
+                                    due: Some(expected_expiry),
+                                    noncurrent_days: 0,
+                                    newer_noncurrent_versions: 0,
+                                    storage_class: "".into(),
+                                });
+                                break;
+                            }
+                        }
+                    }
+                }
+
+                if obj.is_latest {
+                    if let Some(ref expiration) = rule.expiration {
+                        if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
+                            if obj.delete_marker && expired_object_delete_marker {
+                                let due = expiration.next_due(obj);
+                                if let Some(due) = due {
+                                    if now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp() {
+                                        events.push(Event {
+                                            action: IlmAction::DelMarkerDeleteAllVersionsAction,
+                                            rule_id: rule.id.clone().expect("err!"),
+                                            due: Some(due),
+                                            noncurrent_days: 0,
+                                            newer_noncurrent_versions: 0,
+                                            storage_class: "".into(),
+                                        });
+                                    }
+                                }
+                                continue;
+                            }
+                        }
+                    }
+                }
+
+                if !obj.is_latest {
+                    if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
+                        if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions {
+                            if newer_noncurrent_versions > 0 {
+                                continue;
+                            }
+                        }
+                    }
+                }
+
+                if !obj.is_latest {
+                    if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
+                        if let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days {
+                            if noncurrent_days != 0 {
+                                if let Some(successor_mod_time) = obj.successor_mod_time {
+                                    let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
+                                    if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
+                                        events.push(Event {
+                                            action: IlmAction::DeleteVersionAction,
+                                            rule_id: rule.id.clone().expect("err!"),
+                                            due: Some(expected_expiry),
+                                            noncurrent_days: 0,
+                                            newer_noncurrent_versions: 0,
+                                            storage_class: "".into(),
+                                        });
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                if !obj.is_latest {
+                    if let Some(ref noncurrent_version_transitions) = rule.noncurrent_version_transitions {
+                        if let Some(ref storage_class) = noncurrent_version_transitions[0].storage_class {
+                            if storage_class.as_str() != "" && !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE
+                            {
+                                let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj);
+                                if due.is_some()
+                                    && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp())
+                                {
+                                    events.push(Event {
+                                        action: IlmAction::TransitionVersionAction,
+                                        rule_id: rule.id.clone().expect("err!"),
+                                        due,
+                                        storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0]
+                                            .storage_class
+                                            .clone()
+                                            .unwrap()
+                                            .as_str()
+                                            .to_string(),
+                                        ..Default::default()
+                                    });
+                                }
+                            }
+                        }
+                    }
+                }
+
+                if obj.is_latest && !obj.delete_marker {
+                    if let Some(ref expiration) = rule.expiration {
+                        if let Some(ref date) = expiration.date {
+                            let date0 = OffsetDateTime::from(date.clone());
+                            if date0.unix_timestamp() != 0
+                                && (now.unix_timestamp() == 0 || now.unix_timestamp() > date0.unix_timestamp())
+                            {
+                                events.push(Event {
+                                    action: IlmAction::DeleteAction,
+                                    rule_id: rule.id.clone().expect("err!"),
+                                    due: Some(date0),
+                                    noncurrent_days: 0,
+                                    newer_noncurrent_versions: 0,
+                                    storage_class: "".into(),
+                                });
+                            }
+                        } else if let Some(days) = expiration.days {
+                            if days != 0 {
+                                let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.expect("err!"), days);
+                                if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
+                                    let mut event = Event {
+                                        action: IlmAction::DeleteAction,
+                                        rule_id: rule.id.clone().expect("err!"),
+                                        due: Some(expected_expiry),
+                                        noncurrent_days: 0,
+                                        newer_noncurrent_versions: 0,
+                                        storage_class: "".into(),
+                                    };
+                                    /*if rule.expiration.expect("err!").delete_all.val {
+                                        event.action = IlmAction::DeleteAllVersionsAction
+                                    }*/
+                                    events.push(event);
+                                }
+                            }
+                        }
+                    }
+
+                    if obj.transition_status != TRANSITION_COMPLETE {
+                        if let Some(ref transitions) = rule.transitions {
+                            let due = transitions[0].next_due(obj);
+                            if let Some(due) = due {
+                                if due.unix_timestamp() > 0
+                                    && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp())
+                                {
+                                    events.push(Event {
+                                        action: IlmAction::TransitionAction,
+                                        rule_id: rule.id.clone().expect("err!"),
+                                        due: Some(due),
+                                        storage_class: transitions[0].storage_class.clone().expect("err!").as_str().to_string(),
+                                        noncurrent_days: 0,
+                                        newer_noncurrent_versions: 0,
+                                    });
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        if events.len() > 0 {
+            events.sort_by(|a, b| {
+                if now.unix_timestamp() > a.due.expect("err!").unix_timestamp()
+                    && now.unix_timestamp() > b.due.expect("err").unix_timestamp()
+                    || a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp()
+                {
+                    match a.action {
+                        IlmAction::DeleteAllVersionsAction
+                        | IlmAction::DelMarkerDeleteAllVersionsAction
+                        | IlmAction::DeleteAction
+                        | IlmAction::DeleteVersionAction => {
+                            return Ordering::Less;
+                        }
+                        _ => (),
+                    }
+                    match b.action {
+                        IlmAction::DeleteAllVersionsAction
+                        | IlmAction::DelMarkerDeleteAllVersionsAction
+                        | IlmAction::DeleteAction
+                        | IlmAction::DeleteVersionAction => {
+                            return Ordering::Greater;
+                        }
+                        _ => (),
+                    }
+                    return Ordering::Less;
+                }
+
+                if a.due.expect("err").unix_timestamp() < b.due.expect("err").unix_timestamp() {
+                    return Ordering::Less;
+                }
+                return Ordering::Greater;
+            });
+            return events[0].clone();
+        }
+
+        Event::default()
+    }
+
+    async fn noncurrent_versions_expiration_limit(&self, obj: &ObjectOpts) -> Event {
+        if let Some(filter_rules) = self.filter_rules(obj).await {
+            for rule in filter_rules.iter() {
+                if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
+                    if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions {
+                        if newer_noncurrent_versions == 0 {
+                            continue;
+                        }
+                        return Event {
+                            action: IlmAction::DeleteVersionAction,
+                            rule_id: rule.id.clone().expect("err"),
+                            noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
+                            newer_noncurrent_versions: newer_noncurrent_versions as usize,
+                            due: Some(OffsetDateTime::UNIX_EPOCH),
+                            storage_class: "".into(),
+                        };
+                    } else {
+                        return Event {
+                            action: IlmAction::DeleteVersionAction,
+                            rule_id: rule.id.clone().expect("err"),
+                            noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
+                            newer_noncurrent_versions: 0,
+                            due: Some(OffsetDateTime::UNIX_EPOCH),
+                            storage_class: "".into(),
+                        };
+                    }
+                }
+            }
+        }
+        Event::default()
+    }
+}
+
+#[async_trait::async_trait]
+pub trait LifecycleCalculate {
+    fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime>;
+}
+
+#[async_trait::async_trait]
+impl LifecycleCalculate for LifecycleExpiration {
+    fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
+        if !obj.is_latest || !obj.delete_marker {
+            return None;
+        }
+
+        Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap()))
+    }
+}
+
+#[async_trait::async_trait]
+impl LifecycleCalculate for NoncurrentVersionTransition {
+    fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
+        if obj.is_latest || self.storage_class.is_none() {
+            return None;
+        }
+        if self.noncurrent_days.is_none() {
+            return obj.successor_mod_time;
+        }
+        Some(expected_expiry_time(obj.successor_mod_time.unwrap(), self.noncurrent_days.unwrap()))
+    }
+}
+
+#[async_trait::async_trait]
+impl LifecycleCalculate for Transition {
+    fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
+        if !obj.is_latest || self.days.is_none() {
+            return None;
+        }
+
+        if let Some(date) = self.date.clone() {
+            return Some(date.into());
+        }
+
+        if self.days.is_none() {
+            return obj.mod_time;
+        }
+        Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap()))
+    }
+}
+
+pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTime {
+    if days == 0 {
+        return mod_time;
+    }
+    let t = mod_time
+        .to_offset(offset!(-0:00:00))
+        .saturating_add(Duration::days(0 /*days as i64*/)); //debug
+    let mut hour = 3600;
+    if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_HOUR") {
+        if let Ok(num_hour) = env_ilm_hour.parse::<usize>() {
+            hour = num_hour;
+        }
+    }
+    //t.Truncate(24 * hour)
+    t
+}
+
+#[derive(Default)]
+pub struct ObjectOpts {
+    pub name: String,
+    pub user_tags: String,
+    pub mod_time: Option<OffsetDateTime>,
+    pub size: usize,
+    pub version_id: String,
+    pub is_latest: bool,
+    pub delete_marker: bool,
+    pub num_versions: usize,
+    pub successor_mod_time: Option<OffsetDateTime>,
+    pub transition_status: String,
+    pub restore_ongoing: bool,
+    pub restore_expires: Option<OffsetDateTime>,
+    pub versioned: bool,
+    pub version_suspended: bool,
+}
+
+impl ObjectOpts {
+    pub fn expired_object_deletemarker(&self) -> bool {
+        self.delete_marker && self.num_versions == 1
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Event {
+    pub action: IlmAction,
+    pub rule_id: String,
+    pub due: Option<OffsetDateTime>,
+    pub noncurrent_days: u32,
+    pub newer_noncurrent_versions: usize,
+    pub storage_class: String,
+}
+
+impl Default for Event {
+    fn default() -> Self {
+        Self {
+            action: IlmAction::NoneAction,
+            rule_id: "".into(),
+            due: Some(OffsetDateTime::UNIX_EPOCH),
+            noncurrent_days: 0,
+            newer_noncurrent_versions: 0,
+            storage_class: "".into(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct ExpirationOptions {
+    pub expire: bool,
+}
+
+#[derive(Debug, Clone)]
+pub struct TransitionOptions {
+    pub status: String,
+    pub tier: String,
+    pub etag: String,
+    pub restore_request: RestoreObjectRequest,
+    pub restore_expiry: OffsetDateTime,
+    pub expire_restored: bool,
+}
+
+impl Default for TransitionOptions {
+    fn default() -> Self {
+        Self {
+            status: Default::default(),
+            tier: Default::default(),
+            etag: Default::default(),
+            restore_request: Default::default(),
+            restore_expiry: OffsetDateTime::now_utc(),
+            expire_restored: Default::default(),
+        }
+    }
+}

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -0,0 +1,20 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod bucket_lifecycle_audit;
+pub mod bucket_lifecycle_ops;
+pub mod lifecycle;
+pub mod rule;
+pub mod tier_last_day_stats;
+pub mod tier_sweeper;

crates/ecstore/src/bucket/lifecycle/rule.rs

@@ -0,0 +1,69 @@
+#![allow(unused_imports)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use s3s::dto::{LifecycleRuleFilter, Transition};
+
+const _ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition";
+const _ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format";
+const ERR_TRANSITION_INVALID: &str =
+    "Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition.";
+const _ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT";
+
+pub trait Filter {
+    fn test_tags(&self, user_tags: &str) -> bool;
+    fn by_size(&self, sz: i64) -> bool;
+}
+
+impl Filter for LifecycleRuleFilter {
+    fn test_tags(&self, user_tags: &str) -> bool {
+        true
+    }
+
+    fn by_size(&self, sz: i64) -> bool {
+        true
+    }
+}
+
+pub trait TransitionOps {
+    fn validate(&self) -> Result<(), std::io::Error>;
+}
+
+impl TransitionOps for Transition {
+    fn validate(&self) -> Result<(), std::io::Error> {
+        if !self.date.is_none() && self.days.expect("err!") > 0 {
+            return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
+        }
+
+        if self.storage_class.is_none() {
+            return Err(std::io::Error::other("ERR_XML_NOT_WELL_FORMED"));
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_rule() {
+        //assert!(skip_access_checks(p.to_str().unwrap()));
+    }
+}

crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs

@@ -0,0 +1,104 @@
+#![allow(unused_imports)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use sha2::Sha256;
+
+use std::collections::HashMap;
+use std::ops::Sub;
+use time::OffsetDateTime;
+use tracing::{error, warn};
+
+use crate::heal::data_usage_cache::TierStats;
+
+pub type DailyAllTierStats = HashMap<String, LastDayTierStats>;
+
+#[derive(Clone)]
+pub struct LastDayTierStats {
+    bins: [TierStats; 24],
+    updated_at: OffsetDateTime,
+}
+
+impl Default for LastDayTierStats {
+    fn default() -> Self {
+        Self {
+            bins: Default::default(),
+            updated_at: OffsetDateTime::now_utc(),
+        }
+    }
+}
+
+impl LastDayTierStats {
+    pub fn add_stats(&mut self, ts: TierStats) {
+        let mut now = OffsetDateTime::now_utc();
+        self.forward_to(&mut now);
+
+        let now_idx = now.hour() as usize;
+        self.bins[now_idx] = self.bins[now_idx].add(&ts);
+    }
+
+    fn forward_to(&mut self, t: &mut OffsetDateTime) {
+        if t.unix_timestamp() == 0 {
+            *t = OffsetDateTime::now_utc();
+        }
+
+        let since = t.sub(self.updated_at).whole_hours();
+        if since < 1 {
+            return;
+        }
+
+        let (idx, mut last_idx) = (t.hour(), self.updated_at.hour());
+
+        self.updated_at = *t;
+
+        if since >= 24 {
+            self.bins = [TierStats::default(); 24];
+            return;
+        }
+
+        while last_idx != idx {
+            last_idx = (last_idx + 1) % 24;
+            self.bins[last_idx as usize] = TierStats::default();
+        }
+    }
+
+    #[allow(dead_code)]
+    fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
+        let mut cl = self.clone();
+        let mut cm = m.clone();
+        let mut merged = LastDayTierStats::default();
+
+        if cl.updated_at.unix_timestamp() > cm.updated_at.unix_timestamp() {
+            cm.forward_to(&mut cl.updated_at);
+            merged.updated_at = cl.updated_at;
+        } else {
+            cl.forward_to(&mut cm.updated_at);
+            merged.updated_at = cm.updated_at;
+        }
+
+        for (i, _) in cl.bins.iter().enumerate() {
+            merged.bins[i] = cl.bins[i].add(&cm.bins[i]);
+        }
+
+        merged
+    }
+}
+
+#[cfg(test)]
+mod test {}

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -0,0 +1,152 @@
+#![allow(unused_imports)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use sha2::{Digest, Sha256};
+use std::any::Any;
+use std::io::{Cursor, Write};
+use xxhash_rust::xxh64;
+
+use super::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject};
+use super::lifecycle::{self, ObjectOpts};
+use crate::global::GLOBAL_TierConfigMgr;
+
+static XXHASH_SEED: u64 = 0;
+
+#[derive(Default)]
+#[allow(dead_code)]
+struct ObjSweeper {
+    object: String,
+    bucket: String,
+    version_id: String,
+    versioned: bool,
+    suspended: bool,
+    transition_status: String,
+    transition_tier: String,
+    transition_version_id: String,
+    remote_object: String,
+}
+
+#[allow(dead_code)]
+impl ObjSweeper {
+    #[allow(clippy::new_ret_no_self)]
+    pub async fn new(bucket: &str, object: &str) -> Result<Self, std::io::Error> {
+        Ok(Self {
+            object: object.into(),
+            bucket: bucket.into(),
+            ..Default::default()
+        })
+    }
+
+    pub fn with_version(&mut self, vid: String) -> &Self {
+        self.version_id = vid;
+        self
+    }
+
+    pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self {
+        self.versioned = versioned;
+        self.suspended = suspended;
+        self
+    }
+
+    pub fn get_opts(&self) -> lifecycle::ObjectOpts {
+        let mut opts = ObjectOpts {
+            version_id: self.version_id.clone(),
+            versioned: self.versioned,
+            version_suspended: self.suspended,
+            ..Default::default()
+        };
+        if self.suspended && self.version_id == "" {
+            opts.version_id = String::from("");
+        }
+        opts
+    }
+
+    pub fn set_transition_state(&mut self, info: TransitionedObject) {
+        self.transition_tier = info.tier;
+        self.transition_status = info.status;
+        self.remote_object = info.name;
+        self.transition_version_id = info.version_id;
+    }
+
+    pub fn should_remove_remote_object(&self) -> Option<Jentry> {
+        if self.transition_status != lifecycle::TRANSITION_COMPLETE {
+            return None;
+        }
+
+        let mut del_tier = false;
+        if !self.versioned || self.suspended {
+            // 1, 2.a, 2.b
+            del_tier = true;
+        } else if self.versioned && self.version_id != "" {
+            // 3.a
+            del_tier = true;
+        }
+        if del_tier {
+            return Some(Jentry {
+                obj_name: self.remote_object.clone(),
+                version_id: self.transition_version_id.clone(),
+                tier_name: self.transition_tier.clone(),
+            });
+        }
+        None
+    }
+
+    pub async fn sweep(&self) {
+        let je = self.should_remove_remote_object();
+        if !je.is_none() {
+            let mut expiry_state = GLOBAL_ExpiryState.write().await;
+            expiry_state.enqueue_tier_journal_entry(&je.expect("err!"));
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+#[allow(unused_assignments)]
+pub struct Jentry {
+    obj_name: String,
+    version_id: String,
+    tier_name: String,
+}
+
+impl ExpiryOp for Jentry {
+    fn op_hash(&self) -> u64 {
+        let mut hasher = Sha256::new();
+        let _ = hasher.write(format!("{}", self.tier_name).as_bytes());
+        let _ = hasher.write(format!("{}", self.obj_name).as_bytes());
+        hasher.flush();
+        xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
+    let mut config_mgr = GLOBAL_TierConfigMgr.write().await;
+    let w = match config_mgr.get_driver(tier_name).await {
+        Ok(w) => w,
+        Err(e) => return Err(std::io::Error::other(e)),
+    };
+    w.remove(obj_name, rv_id).await
+}
+
+#[cfg(test)]
+mod test {}

crates/ecstore/src/bucket/metadata.rs

@@ -0,0 +1,454 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{quota::BucketQuota, target::BucketTargets};
+
+use super::object_lock::ObjectLockApi;
+use super::versioning::VersioningApi;
+use byteorder::{BigEndian, ByteOrder, LittleEndian};
+use rmp_serde::Serializer as rmpSerializer;
+use rustfs_policy::policy::BucketPolicy;
+use s3s::dto::{
+    BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration,
+    ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
+};
+use serde::Serializer;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::sync::Arc;
+use time::OffsetDateTime;
+use tracing::error;
+
+use crate::bucket::target::BucketTarget;
+use crate::bucket::utils::deserialize;
+use crate::config::com::{read_config, save_config};
+use crate::error::{Error, Result};
+use crate::new_object_layer_fn;
+
+use crate::disk::BUCKET_META_PREFIX;
+use crate::store::ECStore;
+
+pub const BUCKET_METADATA_FILE: &str = ".metadata.bin";
+pub const BUCKET_METADATA_FORMAT: u16 = 1;
+pub const BUCKET_METADATA_VERSION: u16 = 1;
+
+pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
+pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
+pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
+pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
+pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
+pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
+pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
+pub const BUCKET_VERSIONING_CONFIG: &str = "versioning.xml";
+pub const BUCKET_REPLICATION_CONFIG: &str = "replication.xml";
+pub const BUCKET_TARGETS_FILE: &str = "bucket-targets.json";
+
+#[derive(Debug, Deserialize, Serialize, Clone)]
+#[serde(rename_all = "PascalCase", default)]
+pub struct BucketMetadata {
+    pub name: String,
+    pub created: OffsetDateTime,
+    pub lock_enabled: bool, // While marked as unused, it may need to be retained
+    pub policy_config_json: Vec<u8>,
+    pub notification_config_xml: Vec<u8>,
+    pub lifecycle_config_xml: Vec<u8>,
+    pub object_lock_config_xml: Vec<u8>,
+    pub versioning_config_xml: Vec<u8>,
+    pub encryption_config_xml: Vec<u8>,
+    pub tagging_config_xml: Vec<u8>,
+    pub quota_config_json: Vec<u8>,
+    pub replication_config_xml: Vec<u8>,
+    pub bucket_targets_config_json: Vec<u8>,
+    pub bucket_targets_config_meta_json: Vec<u8>,
+
+    pub policy_config_updated_at: OffsetDateTime,
+    pub object_lock_config_updated_at: OffsetDateTime,
+    pub encryption_config_updated_at: OffsetDateTime,
+    pub tagging_config_updated_at: OffsetDateTime,
+    pub quota_config_updated_at: OffsetDateTime,
+    pub replication_config_updated_at: OffsetDateTime,
+    pub versioning_config_updated_at: OffsetDateTime,
+    pub lifecycle_config_updated_at: OffsetDateTime,
+    pub notification_config_updated_at: OffsetDateTime,
+    pub bucket_targets_config_updated_at: OffsetDateTime,
+    pub bucket_targets_config_meta_updated_at: OffsetDateTime,
+
+    #[serde(skip)]
+    pub new_field_updated_at: OffsetDateTime,
+
+    #[serde(skip)]
+    pub policy_config: Option<BucketPolicy>,
+    #[serde(skip)]
+    pub notification_config: Option<NotificationConfiguration>,
+    #[serde(skip)]
+    pub lifecycle_config: Option<BucketLifecycleConfiguration>,
+    #[serde(skip)]
+    pub object_lock_config: Option<ObjectLockConfiguration>,
+    #[serde(skip)]
+    pub versioning_config: Option<VersioningConfiguration>,
+    #[serde(skip)]
+    pub sse_config: Option<ServerSideEncryptionConfiguration>,
+    #[serde(skip)]
+    pub tagging_config: Option<Tagging>,
+    #[serde(skip)]
+    pub quota_config: Option<BucketQuota>,
+    #[serde(skip)]
+    pub replication_config: Option<ReplicationConfiguration>,
+    #[serde(skip)]
+    pub bucket_target_config: Option<BucketTargets>,
+    #[serde(skip)]
+    pub bucket_target_config_meta: Option<HashMap<String, String>>,
+}
+
+impl Default for BucketMetadata {
+    fn default() -> Self {
+        Self {
+            name: Default::default(),
+            created: OffsetDateTime::UNIX_EPOCH,
+            lock_enabled: Default::default(),
+            policy_config_json: Default::default(),
+            notification_config_xml: Default::default(),
+            lifecycle_config_xml: Default::default(),
+            object_lock_config_xml: Default::default(),
+            versioning_config_xml: Default::default(),
+            encryption_config_xml: Default::default(),
+            tagging_config_xml: Default::default(),
+            quota_config_json: Default::default(),
+            replication_config_xml: Default::default(),
+            bucket_targets_config_json: Default::default(),
+            bucket_targets_config_meta_json: Default::default(),
+            policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            tagging_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            quota_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            replication_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            versioning_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            lifecycle_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            notification_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            bucket_targets_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            bucket_targets_config_meta_updated_at: OffsetDateTime::UNIX_EPOCH,
+            new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
+            policy_config: Default::default(),
+            notification_config: Default::default(),
+            lifecycle_config: Default::default(),
+            object_lock_config: Default::default(),
+            versioning_config: Default::default(),
+            sse_config: Default::default(),
+            tagging_config: Default::default(),
+            quota_config: Default::default(),
+            replication_config: Default::default(),
+            bucket_target_config: Default::default(),
+            bucket_target_config_meta: Default::default(),
+        }
+    }
+}
+
+impl BucketMetadata {
+    pub fn new(name: &str) -> Self {
+        BucketMetadata {
+            name: name.to_string(),
+            ..Default::default()
+        }
+    }
+
+    pub fn save_file_path(&self) -> String {
+        format!("{}/{}/{}", BUCKET_META_PREFIX, self.name.as_str(), BUCKET_METADATA_FILE)
+    }
+
+    pub fn versioning(&self) -> bool {
+        self.lock_enabled
+            || (self.object_lock_config.as_ref().is_some_and(|v| v.enabled())
+                || self.versioning_config.as_ref().is_some_and(|v| v.enabled()))
+    }
+
+    pub fn object_locking(&self) -> bool {
+        self.lock_enabled || (self.versioning_config.as_ref().is_some_and(|v| v.enabled()))
+    }
+
+    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
+        let mut buf = Vec::new();
+
+        self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self> {
+        let t: BucketMetadata = rmp_serde::from_slice(buf)?;
+        Ok(t)
+    }
+
+    pub fn check_header(buf: &[u8]) -> Result<()> {
+        if buf.len() <= 4 {
+            return Err(Error::other("read_bucket_metadata: data invalid"));
+        }
+
+        let format = LittleEndian::read_u16(&buf[0..2]);
+        let version = LittleEndian::read_u16(&buf[2..4]);
+
+        match format {
+            BUCKET_METADATA_FORMAT => {}
+            _ => return Err(Error::other("read_bucket_metadata: format invalid")),
+        }
+
+        match version {
+            BUCKET_METADATA_VERSION => {}
+            _ => return Err(Error::other("read_bucket_metadata: version invalid")),
+        }
+
+        Ok(())
+    }
+
+    fn default_timestamps(&mut self) {
+        if self.policy_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.policy_config_updated_at = self.created
+        }
+        if self.encryption_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.encryption_config_updated_at = self.created
+        }
+
+        if self.tagging_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.tagging_config_updated_at = self.created
+        }
+        if self.object_lock_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.object_lock_config_updated_at = self.created
+        }
+        if self.quota_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.quota_config_updated_at = self.created
+        }
+
+        if self.replication_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.replication_config_updated_at = self.created
+        }
+
+        if self.versioning_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.versioning_config_updated_at = self.created
+        }
+
+        if self.lifecycle_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.lifecycle_config_updated_at = self.created
+        }
+        if self.notification_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.notification_config_updated_at = self.created
+        }
+
+        if self.bucket_targets_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.bucket_targets_config_updated_at = self.created
+        }
+        if self.bucket_targets_config_meta_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.bucket_targets_config_meta_updated_at = self.created
+        }
+    }
+
+    pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
+        let updated = OffsetDateTime::now_utc();
+
+        match config_file {
+            BUCKET_POLICY_CONFIG => {
+                self.policy_config_json = data;
+                self.policy_config_updated_at = updated;
+            }
+            BUCKET_NOTIFICATION_CONFIG => {
+                self.notification_config_xml = data;
+                self.notification_config_updated_at = updated;
+            }
+            BUCKET_LIFECYCLE_CONFIG => {
+                self.lifecycle_config_xml = data;
+                self.lifecycle_config_updated_at = updated;
+            }
+            BUCKET_SSECONFIG => {
+                self.encryption_config_xml = data;
+                self.encryption_config_updated_at = updated;
+            }
+            BUCKET_TAGGING_CONFIG => {
+                self.tagging_config_xml = data;
+                self.tagging_config_updated_at = updated;
+            }
+            BUCKET_QUOTA_CONFIG_FILE => {
+                self.quota_config_json = data;
+                self.quota_config_updated_at = updated;
+            }
+            OBJECT_LOCK_CONFIG => {
+                self.object_lock_config_xml = data;
+                self.object_lock_config_updated_at = updated;
+            }
+            BUCKET_VERSIONING_CONFIG => {
+                self.versioning_config_xml = data;
+                self.versioning_config_updated_at = updated;
+            }
+            BUCKET_REPLICATION_CONFIG => {
+                self.replication_config_xml = data;
+                self.replication_config_updated_at = updated;
+            }
+            BUCKET_TARGETS_FILE => {
+                // let x = data.clone();
+                // let str = std::str::from_utf8(&x).expect("Invalid UTF-8");
+                // println!("update config:{}", str);
+                self.bucket_targets_config_json = data.clone();
+                self.bucket_targets_config_updated_at = updated;
+            }
+            _ => return Err(Error::other(format!("config file not found : {config_file}"))),
+        }
+
+        Ok(updated)
+    }
+
+    pub fn set_created(&mut self, created: Option<OffsetDateTime>) {
+        self.created = created.unwrap_or_else(OffsetDateTime::now_utc)
+    }
+
+    pub async fn save(&mut self) -> Result<()> {
+        let Some(store) = new_object_layer_fn() else {
+            return Err(Error::other("errServerNotInitialized"));
+        };
+
+        self.parse_all_configs(store.clone())?;
+
+        let mut buf: Vec<u8> = vec![0; 4];
+
+        LittleEndian::write_u16(&mut buf[0..2], BUCKET_METADATA_FORMAT);
+
+        LittleEndian::write_u16(&mut buf[2..4], BUCKET_METADATA_VERSION);
+
+        let data = self.marshal_msg()?;
+
+        buf.extend_from_slice(&data);
+
+        save_config(store, self.save_file_path().as_str(), buf).await?;
+
+        Ok(())
+    }
+
+    fn parse_all_configs(&mut self, _api: Arc<ECStore>) -> Result<()> {
+        if !self.policy_config_json.is_empty() {
+            self.policy_config = Some(serde_json::from_slice(&self.policy_config_json)?);
+        }
+        if !self.notification_config_xml.is_empty() {
+            self.notification_config = Some(deserialize::<NotificationConfiguration>(&self.notification_config_xml)?);
+        }
+        if !self.lifecycle_config_xml.is_empty() {
+            self.lifecycle_config = Some(deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml)?);
+        }
+
+        if !self.object_lock_config_xml.is_empty() {
+            self.object_lock_config = Some(deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml)?);
+        }
+        if !self.versioning_config_xml.is_empty() {
+            self.versioning_config = Some(deserialize::<VersioningConfiguration>(&self.versioning_config_xml)?);
+        }
+        if !self.encryption_config_xml.is_empty() {
+            self.sse_config = Some(deserialize::<ServerSideEncryptionConfiguration>(&self.encryption_config_xml)?);
+        }
+        if !self.tagging_config_xml.is_empty() {
+            self.tagging_config = Some(deserialize::<Tagging>(&self.tagging_config_xml)?);
+        }
+        if !self.quota_config_json.is_empty() {
+            self.quota_config = Some(BucketQuota::unmarshal(&self.quota_config_json)?);
+        }
+        if !self.replication_config_xml.is_empty() {
+            self.replication_config = Some(deserialize::<ReplicationConfiguration>(&self.replication_config_xml)?);
+        }
+        //let temp = self.bucket_targets_config_json.clone();
+        if !self.bucket_targets_config_json.is_empty() {
+            let arr: Vec<BucketTarget> = serde_json::from_slice(&self.bucket_targets_config_json)?;
+            self.bucket_target_config = Some(BucketTargets { targets: arr });
+        } else {
+            self.bucket_target_config = Some(BucketTargets::default())
+        }
+
+        Ok(())
+    }
+}
+
+pub async fn load_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<BucketMetadata> {
+    load_bucket_metadata_parse(api, bucket, true).await
+}
+
+pub async fn load_bucket_metadata_parse(api: Arc<ECStore>, bucket: &str, parse: bool) -> Result<BucketMetadata> {
+    let mut bm = match read_bucket_metadata(api.clone(), bucket).await {
+        Ok(res) => res,
+        Err(err) => {
+            if err != Error::ConfigNotFound {
+                return Err(err);
+            }
+
+            // info!("bucketmeta {} not found with err {:?}, start to init ", bucket, &err);
+
+            BucketMetadata::new(bucket)
+        }
+    };
+
+    bm.default_timestamps();
+
+    if parse {
+        bm.parse_all_configs(api)?;
+    }
+
+    // TODO: parse_all_configs
+
+    Ok(bm)
+}
+
+async fn read_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<BucketMetadata> {
+    if bucket.is_empty() {
+        error!("bucket name empty");
+        return Err(Error::other("invalid argument"));
+    }
+
+    let bm = BucketMetadata::new(bucket);
+    let file_path = bm.save_file_path();
+
+    let data = read_config(api, &file_path).await?;
+
+    BucketMetadata::check_header(&data)?;
+
+    let bm = BucketMetadata::unmarshal(&data[4..])?;
+
+    Ok(bm)
+}
+
+fn _write_time<S>(t: &OffsetDateTime, s: S) -> std::result::Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut buf = vec![0x0; 15];
+
+    let sec = t.unix_timestamp() - 62135596800;
+    let nsec = t.nanosecond();
+    buf[0] = 0xc7; // mext8
+    buf[1] = 0x0c; // 长度
+    buf[2] = 0x05; // 时间扩展类型
+    BigEndian::write_u64(&mut buf[3..], sec as u64);
+    BigEndian::write_u32(&mut buf[11..], nsec);
+    s.serialize_bytes(&buf)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[tokio::test]
+    async fn marshal_msg() {
+        // write_time(OffsetDateTime::UNIX_EPOCH).unwrap();
+
+        let bm = BucketMetadata::new("dada");
+
+        let buf = bm.marshal_msg().unwrap();
+
+        let new = BucketMetadata::unmarshal(&buf).unwrap();
+
+        assert_eq!(bm.name, new.name);
+    }
+}

crates/ecstore/src/bucket/metadata_sys.rs

@@ -0,0 +1,551 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::StorageAPI;
+use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, load_bucket_metadata_parse};
+use crate::bucket::utils::{deserialize, is_meta_bucketname};
+use crate::cmd::bucket_targets;
+use crate::error::{Error, Result, is_err_bucket_not_found};
+use crate::global::{GLOBAL_Endpoints, is_dist_erasure, is_erasure, new_object_layer_fn};
+use crate::heal::heal_commands::HealOpts;
+use crate::store::ECStore;
+use futures::future::join_all;
+use rustfs_policy::policy::BucketPolicy;
+use s3s::dto::{
+    BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration,
+    ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
+};
+use std::collections::HashSet;
+use std::sync::OnceLock;
+use std::time::Duration;
+use std::{collections::HashMap, sync::Arc};
+use time::OffsetDateTime;
+use tokio::sync::RwLock;
+use tokio::time::sleep;
+use tracing::error;
+
+use super::metadata::{BucketMetadata, load_bucket_metadata};
+use super::quota::BucketQuota;
+use super::target::BucketTargets;
+
+use lazy_static::lazy_static;
+
+lazy_static! {
+    pub static ref GLOBAL_BucketMetadataSys: OnceLock<Arc<RwLock<BucketMetadataSys>>> = OnceLock::new();
+}
+
+pub async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
+    let mut sys = BucketMetadataSys::new(api);
+    sys.init(buckets).await;
+
+    let sys = Arc::new(RwLock::new(sys));
+
+    GLOBAL_BucketMetadataSys.set(sys).unwrap();
+}
+
+// panic if not init
+pub(super) fn get_bucket_metadata_sys() -> Result<Arc<RwLock<BucketMetadataSys>>> {
+    if let Some(sys) = GLOBAL_BucketMetadataSys.get() {
+        Ok(sys.clone())
+    } else {
+        Err(Error::other("GLOBAL_BucketMetadataSys not init"))
+    }
+}
+
+pub async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> Result<()> {
+    let sys = get_bucket_metadata_sys()?;
+    let lock = sys.write().await;
+    lock.set(bucket, Arc::new(bm)).await;
+    Ok(())
+}
+
+pub async fn get(bucket: &str) -> Result<Arc<BucketMetadata>> {
+    let sys = get_bucket_metadata_sys()?;
+    let lock = sys.read().await;
+    lock.get(bucket).await
+}
+
+pub async fn update(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
+
+    bucket_meta_sys.update(bucket, config_file, data).await
+}
+
+pub async fn delete(bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
+
+    bucket_meta_sys.delete(bucket, config_file).await
+}
+
+pub async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_bucket_policy(bucket).await
+}
+
+pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_quota_config(bucket).await
+}
+
+pub async fn get_bucket_targets_config(bucket: &str) -> Result<BucketTargets> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_bucket_targets_config(bucket).await
+}
+
+pub async fn get_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_tagging_config(bucket).await
+}
+
+pub async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_lifecycle_config(bucket).await
+}
+
+pub async fn get_sse_config(bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_sse_config(bucket).await
+}
+
+pub async fn get_object_lock_config(bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_object_lock_config(bucket).await
+}
+
+pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_replication_config(bucket).await
+}
+
+pub async fn get_notification_config(bucket: &str) -> Result<Option<NotificationConfiguration>> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_notification_config(bucket).await
+}
+
+pub async fn get_versioning_config(bucket: &str) -> Result<(VersioningConfiguration, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_versioning_config(bucket).await
+}
+
+pub async fn get_config_from_disk(bucket: &str) -> Result<BucketMetadata> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.get_config_from_disk(bucket).await
+}
+
+pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.created_at(bucket).await
+}
+
+#[derive(Debug)]
+pub struct BucketMetadataSys {
+    metadata_map: RwLock<HashMap<String, Arc<BucketMetadata>>>,
+    api: Arc<ECStore>,
+    initialized: RwLock<bool>,
+}
+
+impl BucketMetadataSys {
+    pub fn new(api: Arc<ECStore>) -> Self {
+        Self {
+            metadata_map: RwLock::new(HashMap::new()),
+            api,
+            initialized: RwLock::new(false),
+        }
+    }
+
+    pub async fn init(&mut self, buckets: Vec<String>) {
+        let _ = self.init_internal(buckets).await;
+    }
+    async fn init_internal(&self, buckets: Vec<String>) -> Result<()> {
+        let count = {
+            if let Some(endpoints) = GLOBAL_Endpoints.get() {
+                endpoints.es_count() * 10
+            } else {
+                return Err(Error::other("GLOBAL_Endpoints not init"));
+            }
+        };
+
+        let mut failed_buckets: HashSet<String> = HashSet::new();
+        let mut buckets = buckets.as_slice();
+
+        loop {
+            if buckets.len() < count {
+                self.concurrent_load(buckets, &mut failed_buckets).await;
+                break;
+            }
+
+            self.concurrent_load(&buckets[..count], &mut failed_buckets).await;
+
+            buckets = &buckets[count..]
+        }
+
+        let mut initialized = self.initialized.write().await;
+        *initialized = true;
+
+        if is_dist_erasure().await {
+            // TODO: refresh_buckets_metadata_loop
+        }
+
+        Ok(())
+    }
+
+    async fn concurrent_load(&self, buckets: &[String], failed_buckets: &mut HashSet<String>) {
+        let mut futures = Vec::new();
+
+        for bucket in buckets.iter() {
+            // TODO: HealBucket
+            let api = self.api.clone();
+            let bucket = bucket.clone();
+            futures.push(async move {
+                sleep(Duration::from_millis(30)).await;
+                let _ = api
+                    .heal_bucket(
+                        &bucket,
+                        &HealOpts {
+                            recreate: true,
+                            ..Default::default()
+                        },
+                    )
+                    .await;
+                load_bucket_metadata(self.api.clone(), bucket.as_str()).await
+            });
+        }
+
+        let results = join_all(futures).await;
+
+        let mut idx = 0;
+
+        let mut mp = self.metadata_map.write().await;
+
+        // TODO:EventNotifier,BucketTargetSys
+        for res in results {
+            match res {
+                Ok(res) => {
+                    if let Some(bucket) = buckets.get(idx) {
+                        let x = Arc::new(res);
+                        mp.insert(bucket.clone(), x.clone());
+                        bucket_targets::init_bucket_targets(bucket, x.clone()).await;
+                    }
+                }
+                Err(e) => {
+                    error!("Unable to load bucket metadata, will be retried: {:?}", e);
+                    if let Some(bucket) = buckets.get(idx) {
+                        failed_buckets.insert(bucket.clone());
+                    }
+                }
+            }
+
+            idx += 1;
+        }
+    }
+
+    pub async fn get(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
+        if is_meta_bucketname(bucket) {
+            return Err(Error::ConfigNotFound);
+        }
+
+        let map = self.metadata_map.read().await;
+        if let Some(bm) = map.get(bucket) {
+            Ok(bm.clone())
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
+    pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
+        if !is_meta_bucketname(&bucket) {
+            let mut map = self.metadata_map.write().await;
+            map.insert(bucket, bm);
+        }
+    }
+
+    async fn _reset(&mut self) {
+        let mut map = self.metadata_map.write().await;
+        map.clear();
+    }
+
+    pub async fn update(&mut self, bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
+        self.update_and_parse(bucket, config_file, data, true).await
+    }
+
+    pub async fn delete(&mut self, bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
+        if config_file == BUCKET_LIFECYCLE_CONFIG {
+            let meta = match self.get_config_from_disk(bucket).await {
+                Ok(res) => res,
+                Err(err) => {
+                    if err != Error::ConfigNotFound {
+                        return Err(err);
+                    } else {
+                        BucketMetadata::new(bucket)
+                    }
+                }
+            };
+
+            if !meta.lifecycle_config_xml.is_empty() {
+                let cfg = deserialize::<BucketLifecycleConfiguration>(&meta.lifecycle_config_xml)?;
+                // TODO: FIXME:
+                // for _v in cfg.rules.iter() {
+                //     break;
+                // }
+                if let Some(_v) = cfg.rules.first() {}
+            }
+
+            // TODO: other lifecycle handle
+        }
+
+        self.update_and_parse(bucket, config_file, Vec::new(), false).await
+    }
+
+    async fn update_and_parse(&mut self, bucket: &str, config_file: &str, data: Vec<u8>, parse: bool) -> Result<OffsetDateTime> {
+        let Some(store) = new_object_layer_fn() else {
+            return Err(Error::other("errServerNotInitialized"));
+        };
+
+        if is_meta_bucketname(bucket) {
+            return Err(Error::other("errInvalidArgument"));
+        }
+
+        let mut bm = match load_bucket_metadata_parse(store, bucket, parse).await {
+            Ok(res) => res,
+            Err(err) => {
+                if !is_erasure().await && !is_dist_erasure().await && is_err_bucket_not_found(&err) {
+                    BucketMetadata::new(bucket)
+                } else {
+                    return Err(err);
+                }
+            }
+        };
+
+        let updated = bm.update_config(config_file, data)?;
+
+        self.save(bm).await?;
+
+        Ok(updated)
+    }
+
+    async fn save(&self, bm: BucketMetadata) -> Result<()> {
+        if is_meta_bucketname(&bm.name) {
+            return Err(Error::other("errInvalidArgument"));
+        }
+
+        let mut bm = bm;
+
+        bm.save().await?;
+
+        self.set(bm.name.clone(), Arc::new(bm)).await;
+
+        Ok(())
+    }
+
+    pub async fn get_config_from_disk(&self, bucket: &str) -> Result<BucketMetadata> {
+        if is_meta_bucketname(bucket) {
+            return Err(Error::other("errInvalidArgument"));
+        }
+
+        load_bucket_metadata(self.api.clone(), bucket).await
+    }
+
+    pub async fn get_config(&self, bucket: &str) -> Result<(Arc<BucketMetadata>, bool)> {
+        let has_bm = {
+            let map = self.metadata_map.read().await;
+            map.get(&bucket.to_string()).cloned()
+        };
+
+        if let Some(bm) = has_bm {
+            Ok((bm, false))
+        } else {
+            let bm = match load_bucket_metadata(self.api.clone(), bucket).await {
+                Ok(res) => res,
+                Err(err) => {
+                    return if *self.initialized.read().await {
+                        Err(Error::other("errBucketMetadataNotInitialized"))
+                    } else {
+                        Err(err)
+                    };
+                }
+            };
+
+            let mut map = self.metadata_map.write().await;
+
+            let bm = Arc::new(bm);
+            map.insert(bucket.to_string(), bm.clone());
+
+            Ok((bm, true))
+        }
+    }
+
+    pub async fn get_versioning_config(&self, bucket: &str) -> Result<(VersioningConfiguration, OffsetDateTime)> {
+        let bm = match self.get_config(bucket).await {
+            Ok((res, _)) => res,
+            Err(err) => {
+                return if err == Error::ConfigNotFound {
+                    Ok((VersioningConfiguration::default(), OffsetDateTime::UNIX_EPOCH))
+                } else {
+                    Err(err)
+                };
+            }
+        };
+
+        if let Some(config) = &bm.versioning_config {
+            Ok((config.clone(), bm.versioning_config_updated_at))
+        } else {
+            Ok((VersioningConfiguration::default(), bm.versioning_config_updated_at))
+        }
+    }
+
+    pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
+        let (bm, _) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.policy_config {
+            Ok((config.clone(), bm.policy_config_updated_at))
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
+    pub async fn get_tagging_config(&self, bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
+        let (bm, _) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.tagging_config {
+            Ok((config.clone(), bm.tagging_config_updated_at))
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
+    pub async fn get_object_lock_config(&self, bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> {
+        let (bm, _) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.object_lock_config {
+            Ok((config.clone(), bm.object_lock_config_updated_at))
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
+    pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
+        let (bm, _) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.lifecycle_config {
+            if config.rules.is_empty() {
+                Err(Error::ConfigNotFound)
+            } else {
+                Ok((config.clone(), bm.lifecycle_config_updated_at))
+            }
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
+    pub async fn get_notification_config(&self, bucket: &str) -> Result<Option<NotificationConfiguration>> {
+        let bm = match self.get_config(bucket).await {
+            Ok((bm, _)) => bm.notification_config.clone(),
+            Err(err) => {
+                if err == Error::ConfigNotFound {
+                    None
+                } else {
+                    return Err(err);
+                }
+            }
+        };
+
+        Ok(bm)
+    }
+
+    pub async fn get_sse_config(&self, bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> {
+        let (bm, _) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.sse_config {
+            Ok((config.clone(), bm.encryption_config_updated_at))
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
+    pub async fn created_at(&self, bucket: &str) -> Result<OffsetDateTime> {
+        let bm = match self.get_config(bucket).await {
+            Ok((bm, _)) => bm.created,
+            Err(err) => {
+                return Err(err);
+            }
+        };
+
+        Ok(bm)
+    }
+
+    pub async fn get_quota_config(&self, bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
+        let (bm, _) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.quota_config {
+            Ok((config.clone(), bm.quota_config_updated_at))
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
+    pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
+        let (bm, reload) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.replication_config {
+            if reload {
+                // TODO: globalBucketTargetSys
+            }
+            //println!("549 {:?}", config.clone());
+            Ok((config.clone(), bm.replication_config_updated_at))
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+
+    pub async fn get_bucket_targets_config(&self, bucket: &str) -> Result<BucketTargets> {
+        let (bm, reload) = self.get_config(bucket).await?;
+
+        if let Some(config) = &bm.bucket_target_config {
+            if reload {
+                // TODO: globalBucketTargetSys
+                //config.
+            }
+
+            Ok(config.clone())
+        } else {
+            Err(Error::ConfigNotFound)
+        }
+    }
+}

crates/ecstore/src/bucket/mod.rs

@@ -0,0 +1,27 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod error;
+pub mod lifecycle;
+pub mod metadata;
+pub mod metadata_sys;
+pub mod object_lock;
+pub mod policy_sys;
+pub mod quota;
+pub mod replication;
+pub mod tagging;
+pub mod target;
+pub mod utils;
+pub mod versioning;
+pub mod versioning_sys;

crates/ecstore/src/bucket/msgpkg.md

@@ -0,0 +1,31 @@
+
+BucketMetadata::new("dada")
+
+```go
+func WriteTime(t time.Time) error {
+    t = t.UTC()
+    o :=0
+    mw.buf[o] =  0xc7 //mext8 // 0xc7
+    mw.buf[o+1] = 12 // 0c
+    mw.buf[o+2] = 0x05 TimeExtension // 05
+    putUnix(mw.buf[o+3:], t.Unix(), int32(t.Nanosecond()))
+    return nil
+}
+
+// 0001-01-01 00:00:00 +0000 UTC == -62135596800 0 (sec() - 62135596800) -62135596800   c70c0b fffffff1886e090000000000 c70c05
+// 2024-10-01 00:00:00 +0000 UTC == 1727740800 0 0 (sec() - 62135596800)
+func putUnix(b []byte, sec int64, nsec int32) {
+    binary.BigEndian.PutUint64(b, uint64(sec))
+    binary.BigEndian.PutUint32(b[8:], uint32(nsec))
+}
+```
+
+# go
+
+de0019a44e616d65a464616461a743726561746564c70c05fffffff1886e090000000000ab4c6f636b456e61626c6564c2b0506f6c696379436f6e6669674a534f4ec400b54e6f74696669636174696f6e436f6e666967584d4cc400b24c6966656379636c65436f6e666967584d4cc400b34f626a6563744c6f636b436f6e666967584d4cc400b356657273696f6e696e67436f6e666967584d4cc400b3456e6372797074696f6e436f6e666967584d4cc400b054616767696e67436f6e666967584d4cc400af51756f7461436f6e6669674a534f4ec400b45265706c69636174696f6e436f6e666967584d4cc400b74275636b657454617267657473436f6e6669674a534f4ec400bb4275636b657454617267657473436f6e6669674d6574614a534f4ec400b5506f6c696379436f6e666967557064617465644174c70c05fffffff1886e090000000000b94f626a6563744c6f636b436f6e666967557064617465644174c70c05fffffff1886e090000000000b9456e6372797074696f6e436f6e666967557064617465644174c70c05fffffff1886e090000000000b654616767696e67436f6e666967557064617465644174c70c05fffffff1886e090000000000b451756f7461436f6e666967557064617465644174c70c05fffffff1886e090000000000ba5265706c69636174696f6e436f6e666967557064617465644174c70c05fffffff1886e090000000000b956657273696f6e696e67436f6e666967557064617465644174c70c05fffffff1886e090000000000b84c6966656379636c65436f6e666967557064617465644174c70c05fffffff1886e090000000000bb4e6f74696669636174696f6e436f6e666967557064617465644174c70c05fffffff1886e090000000000bc4275636b657454617267657473436f6e666967557064617465644174c70c05fffffff1886e090000000000d9204275636b657454617267657473436f6e6669674d657461557064617465644174c70c05fffffff1886e090000000000
+
+de0019a44e616d65a464616461a743726561746564 c40f c70c05fffffff1886e090000000000ab4c6f636b456e61626c6564c2b0506f6c696379436f6e6669674a736f6e90b54e6f74696669636174696f6e436f6e666967586d6c90b24c6966656379636c65436f6e666967586d6c90b34f626a6563744c6f636b436f6e666967586d6c90b356657273696f6e696e67436f6e666967586d6c90b3456e6372797074696f6e436f6e666967586d6c90b054616767696e67436f6e666967586d6c90af51756f7461436f6e6669674a736f6e90b45265706c69636174696f6e436f6e666967586d6c90b74275636b657454617267657473436f6e6669674a736f6e90bb4275636b657454617267657473436f6e6669674d6574614a736f6e90b5506f6c696379436f6e666967557064617465644174c40fc70c05fffffff1886e090000000000b94f626a6563744c6f636b436f6e666967557064617465644174c40fc70c05fffffff1886e090000000000b9456e6372797074696f6e436f6e666967557064617465644174c40fc70c05fffffff1886e090000000000b654616767696e67436f6e666967557064617465644174c40fc70c05fffffff1886e090000000000b451756f7461436f6e666967557064617465644174c40fc70c05fffffff1886e090000000000ba5265706c69636174696f6e436f6e666967557064617465644174c40fc70c05fffffff1886e090000000000b956657273696f6e696e67436f6e666967557064617465644174c40fc70c05fffffff1886e090000000000b84c6966656379636c65436f6e666967557064617465644174c40fc70c05fffffff1886e090000000000bb4e6f74696669636174696f6e436f6e666967557064617465644174c40fc70c05fffffff1886e090000000000bc4275636b657454617267657473436f6e666967557064617465644174c40fc70c05fffffff1886e090000000000d9204275636b657454617267657473436f6e6669674d657461557064617465644174c40fc70c05fffffff1886e090000000000
+
+# rust
+
+de0019a44e616d65 a464616461 a743726561746564 c0 ab4c6f636b456e61626c6564 c2b0506f6c696379436f6e6669674a736f6e90b54e6f74696669636174696f6e436f6e666967586d6c90b24c6966656379636c65436f6e666967586d6c90b34f626a6563744c6f636b436f6e666967586d6c90b356657273696f6e696e67436f6e666967586d6c90b3456e6372797074696f6e436f6e666967586d6c90b054616767696e67436f6e666967586d6c90af51756f7461436f6e6669674a736f6e90b45265706c69636174696f6e436f6e666967586d6c90b74275636b657454617267657473436f6e6669674a736f6e90bb4275636b657454617267657473436f6e6669674d6574614a736f6e90b5506f6c696379436f6e666967557064617465644174c0b94f626a6563744c6f636b436f6e666967557064617465644174c0b9456e6372797074696f6e436f6e666967557064617465644174c0b654616767696e67436f6e666967557064617465644174c0b451756f7461436f6e666967557064617465644174c0ba5265706c69636174696f6e436f6e666967557064617465644174c0b956657273696f6e696e67436f6e666967557064617465644174c0b84c6966656379636c65436f6e666967557064617465644174c0bb4e6f74696669636174696f6e436f6e666967557064617465644174c0bc4275636b657454617267657473436f6e666967557064617465644174c0d9204275636b657454617267657473436f6e6669674d657461557064617465644174c0

crates/ecstore/src/bucket/object_lock/mod.rs

@@ -0,0 +1,30 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod objectlock;
+pub mod objectlock_sys;
+
+use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
+
+pub trait ObjectLockApi {
+    fn enabled(&self) -> bool;
+}
+
+impl ObjectLockApi for ObjectLockConfiguration {
+    fn enabled(&self) -> bool {
+        self.object_lock_enabled
+            .as_ref()
+            .is_some_and(|v| v.as_str() == ObjectLockEnabled::ENABLED)
+    }
+}

crates/ecstore/src/bucket/object_lock/objectlock.rs

@@ -0,0 +1,94 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use time::{OffsetDateTime, format_description};
+
+use s3s::dto::{Date, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, ObjectLockRetentionMode};
+use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
+
+const _ERR_MALFORMED_BUCKET_OBJECT_CONFIG: &str = "invalid bucket object lock config";
+const _ERR_INVALID_RETENTION_DATE: &str = "date must be provided in ISO 8601 format";
+const _ERR_PAST_OBJECTLOCK_RETAIN_DATE: &str = "the retain until date must be in the future";
+const _ERR_UNKNOWN_WORMMODE_DIRECTIVE: &str = "unknown WORM mode directive";
+const _ERR_OBJECTLOCK_MISSING_CONTENT_MD5: &str =
+    "content-MD5 HTTP header is required for Put Object requests with Object Lock parameters";
+const _ERR_OBJECTLOCK_INVALID_HEADERS: &str =
+    "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied";
+const _ERR_MALFORMED_XML: &str = "the XML you provided was not well-formed or did not validate against our published schema";
+
+pub fn utc_now_ntp() -> OffsetDateTime {
+    OffsetDateTime::now_utc()
+}
+
+pub fn get_object_retention_meta(meta: HashMap<String, String>) -> ObjectLockRetention {
+    let mut retain_until_date: Date = Date::from(OffsetDateTime::UNIX_EPOCH);
+
+    let mut mode_str = meta.get(X_AMZ_OBJECT_LOCK_MODE.as_str().to_lowercase().as_str());
+    if mode_str.is_none() {
+        mode_str = Some(&meta[X_AMZ_OBJECT_LOCK_MODE.as_str()]);
+    }
+    let mode = if let Some(mode_str) = mode_str {
+        parse_ret_mode(mode_str.as_str())
+    } else {
+        return ObjectLockRetention {
+            mode: None,
+            retain_until_date: None,
+        };
+    };
+
+    let mut till_str = meta.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_lowercase().as_str());
+    if till_str.is_none() {
+        till_str = Some(&meta[X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()]);
+    }
+    if let Some(till_str) = till_str {
+        let t = OffsetDateTime::parse(till_str, &format_description::well_known::Iso8601::DEFAULT);
+        if t.is_err() {
+            retain_until_date = Date::from(t.expect("err")); //TODO: utc
+        }
+    }
+    ObjectLockRetention {
+        mode: Some(mode),
+        retain_until_date: Some(retain_until_date),
+    }
+}
+
+pub fn get_object_legalhold_meta(meta: HashMap<String, String>) -> ObjectLockLegalHold {
+    let mut hold_str = meta.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_lowercase().as_str());
+    if hold_str.is_none() {
+        hold_str = Some(&meta[X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()]);
+    }
+    if let Some(hold_str) = hold_str {
+        return ObjectLockLegalHold {
+            status: Some(parse_legalhold_status(hold_str)),
+        };
+    }
+    ObjectLockLegalHold { status: None }
+}
+
+pub fn parse_ret_mode(mode_str: &str) -> ObjectLockRetentionMode {
+    match mode_str.to_uppercase().as_str() {
+        "GOVERNANCE" => ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE),
+        "COMPLIANCE" => ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE),
+        _ => unreachable!(),
+    }
+}
+
+pub fn parse_legalhold_status(hold_str: &str) -> ObjectLockLegalHoldStatus {
+    match hold_str {
+        "ON" => ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON),
+        "OFF" => ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF),
+        _ => unreachable!(),
+    }
+}

crates/ecstore/src/bucket/object_lock/objectlock_sys.rs

@@ -0,0 +1,67 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+use time::OffsetDateTime;
+
+use s3s::dto::{DefaultRetention, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
+
+use crate::bucket::metadata_sys::get_object_lock_config;
+use crate::store_api::ObjectInfo;
+
+use super::objectlock;
+
+pub struct BucketObjectLockSys {}
+
+impl BucketObjectLockSys {
+    #[allow(clippy::new_ret_no_self)]
+    pub async fn new() -> Arc<Self> {
+        Arc::new(Self {})
+    }
+
+    pub async fn get(bucket: &str) -> Option<DefaultRetention> {
+        if let Ok(object_lock_config) = get_object_lock_config(bucket).await {
+            if let Some(object_lock_rule) = object_lock_config.0.rule {
+                return object_lock_rule.default_retention;
+            }
+        }
+        None
+    }
+}
+
+pub fn enforce_retention_for_deletion(obj_info: &ObjectInfo) -> bool {
+    if obj_info.delete_marker {
+        return false;
+    }
+
+    let lhold = objectlock::get_object_legalhold_meta(obj_info.user_defined.clone());
+    match lhold.status {
+        Some(st) if st.as_str() == ObjectLockLegalHoldStatus::ON => {
+            return true;
+        }
+        _ => (),
+    }
+
+    let ret = objectlock::get_object_retention_meta(obj_info.user_defined.clone());
+    match ret.mode {
+        Some(r) if (r.as_str() == ObjectLockRetentionMode::COMPLIANCE || r.as_str() == ObjectLockRetentionMode::GOVERNANCE) => {
+            let t = objectlock::utc_now_ntp();
+            if OffsetDateTime::from(ret.retain_until_date.expect("err!")).unix_timestamp() > t.unix_timestamp() {
+                return true;
+            }
+        }
+        _ => (),
+    }
+    false
+}

crates/ecstore/src/bucket/policy_sys.rs

@@ -0,0 +1,44 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{error::BucketMetadataError, metadata_sys::get_bucket_metadata_sys};
+use crate::error::Result;
+use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
+use tracing::warn;
+
+pub struct PolicySys {}
+
+impl PolicySys {
+    pub async fn is_allowed(args: &BucketPolicyArgs<'_>) -> bool {
+        match Self::get(args.bucket).await {
+            Ok(cfg) => return cfg.is_allowed(args),
+            Err(err) => {
+                let berr: BucketMetadataError = err.into();
+                if berr != BucketMetadataError::BucketPolicyNotFound {
+                    warn!("config get err {:?}", berr);
+                }
+            }
+        }
+
+        args.is_owner
+    }
+    pub async fn get(bucket: &str) -> Result<BucketPolicy> {
+        let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+        let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+        let (cfg, _) = bucket_meta_sys.get_bucket_policy(bucket).await?;
+
+        Ok(cfg)
+    }
+}

crates/ecstore/src/bucket/quota/mod.rs

@@ -0,0 +1,52 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::Result;
+use rmp_serde::Serializer as rmpSerializer;
+use serde::{Deserialize, Serialize};
+
+// 定义 QuotaType 枚举类型
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum QuotaType {
+    Hard,
+}
+
+// 定义 BucketQuota 结构体
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub struct BucketQuota {
+    quota: Option<u64>, // 使用 Option 来表示可能不存在的字段
+
+    size: u64,
+
+    rate: u64,
+
+    requests: u64,
+
+    quota_type: Option<QuotaType>,
+}
+
+impl BucketQuota {
+    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
+        let mut buf = Vec::new();
+
+        self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self> {
+        let t: BucketQuota = rmp_serde::from_slice(buf)?;
+        Ok(t)
+    }
+}

crates/ecstore/src/bucket/replication/datatypes.rs

@@ -0,0 +1,41 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Replication status type for x-amz-replication-status header
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum StatusType {
+    Pending,
+    Completed,
+    CompletedLegacy,
+    Failed,
+    Replica,
+}
+
+impl StatusType {
+    // Converts the enum variant to its string representation
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            StatusType::Pending => "PENDING",
+            StatusType::Completed => "COMPLETED",
+            StatusType::CompletedLegacy => "COMPLETE",
+            StatusType::Failed => "FAILED",
+            StatusType::Replica => "REPLICA",
+        }
+    }
+
+    // Checks if the status is empty (not set)
+    pub fn is_empty(&self) -> bool {
+        matches!(self, StatusType::Pending) // Adjust this as needed
+    }
+}

crates/ecstore/src/bucket/replication/mod.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod datatypes;

crates/ecstore/src/bucket/tagging/mod.rs

@@ -0,0 +1,48 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use s3s::dto::Tag;
+use url::form_urlencoded;
+
+pub fn decode_tags(tags: &str) -> Vec<Tag> {
+    let values = form_urlencoded::parse(tags.as_bytes());
+
+    let mut list = Vec::new();
+
+    for (k, v) in values {
+        if k.is_empty() || v.is_empty() {
+            continue;
+        }
+
+        list.push(Tag {
+            key: Some(k.to_string()),
+            value: Some(v.to_string()),
+        });
+    }
+
+    list
+}
+
+pub fn encode_tags(tags: Vec<Tag>) -> String {
+    let mut encoded = form_urlencoded::Serializer::new(String::new());
+
+    for tag in tags.iter() {
+        if let (Some(k), Some(v)) = (tag.key.as_ref(), tag.value.as_ref()) {
+            //encoded.append_pair(k.as_ref().unwrap().as_str(), v.as_ref().unwrap().as_str());
+            encoded.append_pair(k.as_str(), v.as_str());
+        }
+    }
+
+    encoded.finish()
+}

crates/ecstore/src/bucket/target/mod.rs

@@ -0,0 +1,135 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::Result;
+use rmp_serde::Serializer as rmpSerializer;
+use serde::{Deserialize, Serialize};
+use time::OffsetDateTime;
+
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub struct Credentials {
+    #[serde(rename = "accessKey")]
+    pub access_key: String,
+    #[serde(rename = "secretKey")]
+    pub secret_key: String,
+    pub session_token: Option<String>,
+    pub expiration: Option<chrono::DateTime<chrono::Utc>>,
+}
+
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub enum ServiceType {
+    #[default]
+    Replication,
+}
+
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub struct LatencyStat {
+    curr: u64, // 当前延迟
+    avg: u64,  // 平均延迟
+    max: u64,  // 最大延迟
+}
+
+// 定义 BucketTarget 结构体
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub struct BucketTarget {
+    #[serde(rename = "sourcebucket")]
+    pub source_bucket: String,
+
+    pub endpoint: String,
+
+    pub credentials: Option<Credentials>,
+    #[serde(rename = "targetbucket")]
+    pub target_bucket: String,
+
+    secure: bool,
+    pub path: Option<String>,
+
+    api: Option<String>,
+
+    pub arn: Option<String>,
+    #[serde(rename = "type")]
+    pub type_: Option<String>,
+
+    pub region: Option<String>,
+
+    bandwidth_limit: Option<i64>,
+
+    #[serde(rename = "replicationSync")]
+    replication_sync: bool,
+
+    storage_class: Option<String>,
+    #[serde(rename = "healthCheckDuration")]
+    health_check_duration: u64,
+    #[serde(rename = "disableProxy")]
+    disable_proxy: bool,
+
+    #[serde(rename = "resetBeforeDate")]
+    reset_before_date: String,
+    reset_id: Option<String>,
+    #[serde(rename = "totalDowntime")]
+    total_downtime: u64,
+
+    last_online: Option<OffsetDateTime>,
+    #[serde(rename = "isOnline")]
+    online: bool,
+
+    latency: Option<LatencyStat>,
+
+    deployment_id: Option<String>,
+
+    edge: bool,
+    #[serde(rename = "edgeSyncBeforeExpiry")]
+    edge_sync_before_expiry: bool,
+}
+
+impl BucketTarget {
+    pub fn is_empty(self) -> bool {
+        //self.target_bucket.is_empty() && self.endpoint.is_empty() && self.arn.is_empty()
+        self.target_bucket.is_empty() && self.endpoint.is_empty() && self.arn.is_none()
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub struct BucketTargets {
+    pub targets: Vec<BucketTarget>,
+}
+
+impl BucketTargets {
+    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
+        let mut buf = Vec::new();
+
+        self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self> {
+        let t: BucketTargets = rmp_serde::from_slice(buf)?;
+        Ok(t)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        if self.targets.is_empty() {
+            return true;
+        }
+
+        for target in &self.targets {
+            if !target.clone().is_empty() {
+                return false;
+            }
+        }
+
+        true
+    }
+}

crates/ecstore/src/bucket/utils.rs

@@ -0,0 +1,115 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::disk::RUSTFS_META_BUCKET;
+use crate::error::{Error, Result};
+use s3s::xml;
+
+pub fn is_meta_bucketname(name: &str) -> bool {
+    name.starts_with(RUSTFS_META_BUCKET)
+}
+
+use regex::Regex;
+
+lazy_static::lazy_static! {
+    static ref VALID_BUCKET_NAME: Regex = Regex::new(r"^[A-Za-z0-9][A-Za-z0-9\.\-\_\:]{1,61}[A-Za-z0-9]$").unwrap();
+    static ref VALID_BUCKET_NAME_STRICT: Regex = Regex::new(r"^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$").unwrap();
+    static ref IP_ADDRESS: Regex = Regex::new(r"^(\d+\.){3}\d+$").unwrap();
+}
+
+pub fn check_bucket_name_common(bucket_name: &str, strict: bool) -> Result<()> {
+    let bucket_name_trimmed = bucket_name.trim();
+
+    if bucket_name_trimmed.is_empty() {
+        return Err(Error::other("Bucket name cannot be empty"));
+    }
+    if bucket_name_trimmed.len() < 3 {
+        return Err(Error::other("Bucket name cannot be shorter than 3 characters"));
+    }
+    if bucket_name_trimmed.len() > 63 {
+        return Err(Error::other("Bucket name cannot be longer than 63 characters"));
+    }
+
+    if bucket_name_trimmed == "rustfs" {
+        return Err(Error::other("Bucket name cannot be rustfs"));
+    }
+
+    if IP_ADDRESS.is_match(bucket_name_trimmed) {
+        return Err(Error::other("Bucket name cannot be an IP address"));
+    }
+    if bucket_name_trimmed.contains("..") || bucket_name_trimmed.contains(".-") || bucket_name_trimmed.contains("-.") {
+        return Err(Error::other("Bucket name contains invalid characters"));
+    }
+    if strict {
+        if !VALID_BUCKET_NAME_STRICT.is_match(bucket_name_trimmed) {
+            return Err(Error::other("Bucket name contains invalid characters"));
+        }
+    } else if !VALID_BUCKET_NAME.is_match(bucket_name_trimmed) {
+        return Err(Error::other("Bucket name contains invalid characters"));
+    }
+    Ok(())
+}
+
+pub fn check_valid_bucket_name(bucket_name: &str) -> Result<()> {
+    check_bucket_name_common(bucket_name, false)
+}
+
+pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
+    check_bucket_name_common(bucket_name, true)
+}
+
+pub fn check_valid_object_name_prefix(object_name: &str) -> Result<()> {
+    if object_name.len() > 1024 {
+        return Err(Error::other("Object name cannot be longer than 1024 characters"));
+    }
+    if !object_name.is_ascii() {
+        return Err(Error::other("Object name with non-UTF-8 strings are not supported"));
+    }
+    Ok(())
+}
+
+pub fn check_valid_object_name(object_name: &str) -> Result<()> {
+    if object_name.trim().is_empty() {
+        return Err(Error::other("Object name cannot be empty"));
+    }
+    check_valid_object_name_prefix(object_name)
+}
+
+pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
+where
+    T: for<'xml> xml::Deserialize<'xml>,
+{
+    let mut d = xml::Deserializer::new(input);
+    let ans = T::deserialize(&mut d)?;
+    d.expect_eof()?;
+    Ok(ans)
+}
+
+pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
+    let mut buf = Vec::with_capacity(256);
+    {
+        let mut ser = xml::Serializer::new(&mut buf);
+        val.serialize_content(&mut ser)?;
+    }
+    Ok(String::from_utf8(buf).unwrap())
+}
+
+pub fn serialize<T: xml::Serialize>(val: &T) -> xml::SerResult<Vec<u8>> {
+    let mut buf = Vec::with_capacity(256);
+    {
+        let mut ser = xml::Serializer::new(&mut buf);
+        val.serialize(&mut ser)?;
+    }
+    Ok(buf)
+}

crates/ecstore/src/bucket/versioning/mod.rs

@@ -0,0 +1,96 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
+
+use rustfs_utils::string::match_simple;
+
+pub trait VersioningApi {
+    fn enabled(&self) -> bool;
+    fn prefix_enabled(&self, prefix: &str) -> bool;
+    fn prefix_suspended(&self, prefix: &str) -> bool;
+    fn versioned(&self, prefix: &str) -> bool;
+    fn suspended(&self) -> bool;
+}
+
+impl VersioningApi for VersioningConfiguration {
+    fn enabled(&self) -> bool {
+        self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED))
+    }
+    fn prefix_enabled(&self, prefix: &str) -> bool {
+        if self.status != Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) {
+            return false;
+        }
+
+        if prefix.is_empty() {
+            return true;
+        }
+
+        if let Some(exclude_folders) = self.exclude_folders {
+            if exclude_folders && prefix.ends_with('/') {
+                return false;
+            }
+        }
+
+        if let Some(ref excluded_prefixes) = self.excluded_prefixes {
+            for p in excluded_prefixes.iter() {
+                if let Some(ref sprefix) = p.prefix {
+                    let pattern = format!("{sprefix}*");
+                    if match_simple(&pattern, prefix) {
+                        return false;
+                    }
+                }
+            }
+        }
+
+        true
+    }
+
+    fn prefix_suspended(&self, prefix: &str) -> bool {
+        if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)) {
+            return true;
+        }
+
+        if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) {
+            if prefix.is_empty() {
+                return false;
+            }
+
+            if let Some(exclude_folders) = self.exclude_folders {
+                if exclude_folders && prefix.ends_with('/') {
+                    return true;
+                }
+            }
+
+            if let Some(ref excluded_prefixes) = self.excluded_prefixes {
+                for p in excluded_prefixes.iter() {
+                    if let Some(ref sprefix) = p.prefix {
+                        let pattern = format!("{sprefix}*");
+                        if match_simple(&pattern, prefix) {
+                            return true;
+                        }
+                    }
+                }
+            }
+        }
+
+        false
+    }
+    fn versioned(&self, prefix: &str) -> bool {
+        self.prefix_enabled(prefix) || self.prefix_suspended(prefix)
+    }
+    fn suspended(&self) -> bool {
+        self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED))
+    }
+}

crates/ecstore/src/bucket/versioning_sys.rs

@@ -0,0 +1,85 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{metadata_sys::get_bucket_metadata_sys, versioning::VersioningApi};
+use crate::disk::RUSTFS_META_BUCKET;
+use crate::error::Result;
+use s3s::dto::VersioningConfiguration;
+use tracing::warn;
+
+pub struct BucketVersioningSys {}
+
+impl Default for BucketVersioningSys {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl BucketVersioningSys {
+    pub fn new() -> Self {
+        Self {}
+    }
+    pub async fn enabled(bucket: &str) -> bool {
+        match Self::get(bucket).await {
+            Ok(res) => res.enabled(),
+            Err(err) => {
+                warn!("{:?}", err);
+                false
+            }
+        }
+    }
+
+    pub async fn prefix_enabled(bucket: &str, prefix: &str) -> bool {
+        match Self::get(bucket).await {
+            Ok(res) => res.prefix_enabled(prefix),
+            Err(err) => {
+                warn!("{:?}", err);
+                false
+            }
+        }
+    }
+
+    pub async fn suspended(bucket: &str) -> bool {
+        match Self::get(bucket).await {
+            Ok(res) => res.suspended(),
+            Err(err) => {
+                warn!("{:?}", err);
+                false
+            }
+        }
+    }
+
+    pub async fn prefix_suspended(bucket: &str, prefix: &str) -> bool {
+        match Self::get(bucket).await {
+            Ok(res) => res.prefix_suspended(prefix),
+            Err(err) => {
+                warn!("{:?}", err);
+                false
+            }
+        }
+    }
+
+    pub async fn get(bucket: &str) -> Result<VersioningConfiguration> {
+        if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
+            return Ok(VersioningConfiguration::default());
+        }
+
+        let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+        let bucket_meta_sys = bucket_meta_sys_lock.write().await;
+
+        let (cfg, _) = bucket_meta_sys.get_versioning_config(bucket).await?;
+
+        Ok(cfg)
+    }
+}

crates/ecstore/src/cache_value/cache.rs

@@ -0,0 +1,137 @@
+#![allow(unsafe_code)] // TODO: audit unsafe code
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+use std::{
+    fmt::Debug,
+    future::Future,
+    pin::Pin,
+    ptr,
+    sync::{
+        Arc,
+        atomic::{AtomicPtr, AtomicU64, Ordering},
+    },
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use tokio::{spawn, sync::Mutex};
+
+use std::io::Result;
+
+pub type UpdateFn<T> = Box<dyn Fn() -> Pin<Box<dyn Future<Output = Result<T>> + Send>> + Send + Sync + 'static>;
+
+#[derive(Clone, Debug, Default)]
+pub struct Opts {
+    return_last_good: bool,
+    no_wait: bool,
+}
+
+pub struct Cache<T: Clone + Debug + Send> {
+    update_fn: UpdateFn<T>,
+    ttl: Duration,
+    opts: Opts,
+    val: AtomicPtr<T>,
+    last_update_ms: AtomicU64,
+    updating: Arc<Mutex<bool>>,
+}
+
+impl<T: Clone + Debug + Send + 'static> Cache<T> {
+    pub fn new(update_fn: UpdateFn<T>, ttl: Duration, opts: Opts) -> Self {
+        let val = AtomicPtr::new(ptr::null_mut());
+        Self {
+            update_fn,
+            ttl,
+            opts,
+            val,
+            last_update_ms: AtomicU64::new(0),
+            updating: Arc::new(Mutex::new(false)),
+        }
+    }
+
+    pub async fn get(self: Arc<Self>) -> Result<T> {
+        let v_ptr = self.val.load(Ordering::SeqCst);
+        let v = if v_ptr.is_null() {
+            None
+        } else {
+            Some(unsafe { (*v_ptr).clone() })
+        };
+
+        let now = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("Time went backwards")
+            .as_secs();
+        if now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() {
+            if let Some(v) = v {
+                return Ok(v);
+            }
+        }
+
+        if self.opts.no_wait && v.is_some() && now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() * 2 {
+            if self.updating.try_lock().is_ok() {
+                let this = Arc::clone(&self);
+                spawn(async move {
+                    let _ = this.update().await;
+                });
+            }
+
+            return Ok(v.unwrap());
+        }
+
+        let _ = self.updating.lock().await;
+
+        if let Ok(duration) =
+            SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(Ordering::SeqCst)))
+        {
+            if duration < self.ttl {
+                return Ok(v.unwrap());
+            }
+        }
+
+        match self.update().await {
+            Ok(_) => {
+                let v_ptr = self.val.load(Ordering::SeqCst);
+                let v = if v_ptr.is_null() {
+                    None
+                } else {
+                    Some(unsafe { (*v_ptr).clone() })
+                };
+                Ok(v.unwrap())
+            }
+            Err(err) => Err(err),
+        }
+    }
+
+    async fn update(&self) -> Result<()> {
+        match (self.update_fn)().await {
+            Ok(val) => {
+                self.val.store(Box::into_raw(Box::new(val)), Ordering::SeqCst);
+                let now = SystemTime::now()
+                    .duration_since(UNIX_EPOCH)
+                    .expect("Time went backwards")
+                    .as_secs();
+                self.last_update_ms.store(now, Ordering::SeqCst);
+                Ok(())
+            }
+            Err(err) => {
+                let v_ptr = self.val.load(Ordering::SeqCst);
+                if self.opts.return_last_good && !v_ptr.is_null() {
+                    return Ok(());
+                }
+
+                Err(err)
+            }
+        }
+    }
+}

crates/ecstore/src/cache_value/metacache_set.rs

@@ -0,0 +1,364 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::disk::error::DiskError;
+use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
+use futures::future::join_all;
+use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
+use std::{future::Future, pin::Pin, sync::Arc};
+use tokio::{spawn, sync::broadcast::Receiver as B_Receiver};
+use tracing::error;
+
+pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
+pub type PartialFn =
+    Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
+type FinishedFn = Box<dyn Fn(&[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
+
+#[derive(Default)]
+pub struct ListPathRawOptions {
+    pub disks: Vec<Option<DiskStore>>,
+    pub fallback_disks: Vec<Option<DiskStore>>,
+    pub bucket: String,
+    pub path: String,
+    pub recursice: bool,
+    pub filter_prefix: Option<String>,
+    pub forward_to: Option<String>,
+    pub min_disks: usize,
+    pub report_not_found: bool,
+    pub per_disk_limit: i32,
+    pub agreed: Option<AgreedFn>,
+    pub partial: Option<PartialFn>,
+    pub finished: Option<FinishedFn>,
+    // pub agreed: Option<Arc<dyn Fn(MetaCacheEntry) + Send + Sync>>,
+    // pub partial: Option<Arc<dyn Fn(MetaCacheEntries, &[Option<Error>]) + Send + Sync>>,
+    // pub finished: Option<Arc<dyn Fn(&[Option<Error>]) + Send + Sync>>,
+}
+
+impl Clone for ListPathRawOptions {
+    fn clone(&self) -> Self {
+        Self {
+            disks: self.disks.clone(),
+            fallback_disks: self.fallback_disks.clone(),
+            bucket: self.bucket.clone(),
+            path: self.path.clone(),
+            recursice: self.recursice,
+            filter_prefix: self.filter_prefix.clone(),
+            forward_to: self.forward_to.clone(),
+            min_disks: self.min_disks,
+            report_not_found: self.report_not_found,
+            per_disk_limit: self.per_disk_limit,
+            ..Default::default()
+        }
+    }
+}
+
+pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -> disk::error::Result<()> {
+    if opts.disks.is_empty() {
+        return Err(DiskError::other("list_path_raw: 0 drives provided"));
+    }
+
+    let mut jobs: Vec<tokio::task::JoinHandle<std::result::Result<(), DiskError>>> = Vec::new();
+    let mut readers = Vec::with_capacity(opts.disks.len());
+    let fds = Arc::new(opts.fallback_disks.clone());
+
+    let (cancel_tx, cancel_rx) = tokio::sync::broadcast::channel::<bool>(1);
+
+    for disk in opts.disks.iter() {
+        let opdisk = disk.clone();
+        let opts_clone = opts.clone();
+        let fds_clone = fds.clone();
+        let mut cancel_rx_clone = cancel_rx.resubscribe();
+        let (rd, mut wr) = tokio::io::duplex(64);
+        readers.push(MetacacheReader::new(rd));
+        jobs.push(spawn(async move {
+            let wakl_opts = WalkDirOptions {
+                bucket: opts_clone.bucket.clone(),
+                base_dir: opts_clone.path.clone(),
+                recursive: opts_clone.recursice,
+                report_notfound: opts_clone.report_not_found,
+                filter_prefix: opts_clone.filter_prefix.clone(),
+                forward_to: opts_clone.forward_to.clone(),
+                limit: opts_clone.per_disk_limit,
+                ..Default::default()
+            };
+
+            let mut need_fallback = false;
+            if let Some(disk) = opdisk {
+                match disk.walk_dir(wakl_opts, &mut wr).await {
+                    Ok(_res) => {}
+                    Err(err) => {
+                        error!("walk dir err {:?}", &err);
+                        need_fallback = true;
+                    }
+                }
+            } else {
+                need_fallback = true;
+            }
+
+            if cancel_rx_clone.try_recv().is_ok() {
+                // warn!("list_path_raw: cancel_rx_clone.try_recv().await.is_ok()");
+                return Ok(());
+            }
+
+            while need_fallback {
+                // warn!("list_path_raw: while need_fallback start");
+                let disk = match fds_clone.iter().find(|d| d.is_some()) {
+                    Some(d) => {
+                        if let Some(disk) = d.clone() {
+                            disk
+                        } else {
+                            break;
+                        }
+                    }
+                    None => break,
+                };
+                match disk
+                    .as_ref()
+                    .walk_dir(
+                        WalkDirOptions {
+                            bucket: opts_clone.bucket.clone(),
+                            base_dir: opts_clone.path.clone(),
+                            recursive: opts_clone.recursice,
+                            report_notfound: opts_clone.report_not_found,
+                            filter_prefix: opts_clone.filter_prefix.clone(),
+                            forward_to: opts_clone.forward_to.clone(),
+                            limit: opts_clone.per_disk_limit,
+                            ..Default::default()
+                        },
+                        &mut wr,
+                    )
+                    .await
+                {
+                    Ok(_r) => {
+                        need_fallback = false;
+                    }
+                    Err(err) => {
+                        error!("walk dir2 err {:?}", &err);
+                        break;
+                    }
+                }
+            }
+
+            // warn!("list_path_raw: while need_fallback done");
+            Ok(())
+        }));
+    }
+
+    let revjob = spawn(async move {
+        let mut errs: Vec<Option<DiskError>> = Vec::with_capacity(readers.len());
+        for _ in 0..readers.len() {
+            errs.push(None);
+        }
+
+        loop {
+            let mut current = MetaCacheEntry::default();
+
+            // warn!(
+            //     "list_path_raw: loop start, bucket: {}, path: {}, current: {:?}",
+            //     opts.bucket, opts.path, &current.name
+            // );
+
+            if rx.try_recv().is_ok() {
+                return Err(DiskError::other("canceled"));
+            }
+
+            let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
+
+            let mut at_eof = 0;
+            let mut fnf = 0;
+            let mut vnf = 0;
+            let mut has_err = 0;
+            let mut agree = 0;
+
+            for (i, r) in readers.iter_mut().enumerate() {
+                if errs[i].is_some() {
+                    has_err += 1;
+                    continue;
+                }
+
+                let entry = match r.peek().await {
+                    Ok(res) => {
+                        if let Some(entry) = res {
+                            // info!("read entry disk: {}, name: {}", i, entry.name);
+                            entry
+                        } else {
+                            // eof
+                            at_eof += 1;
+                            // warn!("list_path_raw: peek eof, disk: {}", i);
+                            continue;
+                        }
+                    }
+                    Err(err) => {
+                        if err == rustfs_filemeta::Error::Unexpected {
+                            at_eof += 1;
+                            // warn!("list_path_raw: peek err eof, disk: {}", i);
+                            continue;
+                        }
+
+                        // warn!("list_path_raw: peek err00, err: {:?}", err);
+
+                        if is_io_eof(&err) {
+                            at_eof += 1;
+                            // warn!("list_path_raw: peek eof, disk: {}", i);
+                            continue;
+                        }
+
+                        if err == rustfs_filemeta::Error::FileNotFound {
+                            at_eof += 1;
+                            fnf += 1;
+                            // warn!("list_path_raw: peek fnf, disk: {}", i);
+                            continue;
+                        } else if err == rustfs_filemeta::Error::VolumeNotFound {
+                            at_eof += 1;
+                            fnf += 1;
+                            vnf += 1;
+                            // warn!("list_path_raw: peek vnf, disk: {}", i);
+                            continue;
+                        } else {
+                            has_err += 1;
+                            errs[i] = Some(err.into());
+                            // warn!("list_path_raw: peek err, disk: {}", i);
+                            continue;
+                        }
+                    }
+                };
+
+                // warn!("list_path_raw: loop entry: {:?}, disk: {}", &entry.name, i);
+
+                // If no current, add it.
+                if current.name.is_empty() {
+                    top_entries[i] = Some(entry.clone());
+                    current = entry;
+                    agree += 1;
+
+                    continue;
+                }
+                // If exact match, we agree.
+                if let (_, true) = current.matches(Some(&entry), true) {
+                    top_entries[i] = Some(entry);
+                    agree += 1;
+
+                    continue;
+                }
+                // If only the name matches we didn't agree, but add it for resolution.
+                if entry.name == current.name {
+                    top_entries[i] = Some(entry);
+                    continue;
+                }
+                // We got different entries
+                if entry.name > current.name {
+                    continue;
+                }
+
+                for item in top_entries.iter_mut().take(i) {
+                    *item = None;
+                }
+
+                agree = 1;
+                top_entries[i] = Some(entry.clone());
+                current = entry;
+            }
+
+            if vnf > 0 && vnf >= (readers.len() - opts.min_disks) {
+                // warn!("list_path_raw: vnf > 0 && vnf >= (readers.len() - opts.min_disks) break");
+                return Err(DiskError::VolumeNotFound);
+            }
+
+            if fnf > 0 && fnf >= (readers.len() - opts.min_disks) {
+                // warn!("list_path_raw: fnf > 0 && fnf >= (readers.len() - opts.min_disks) break");
+                return Err(DiskError::FileNotFound);
+            }
+
+            if has_err > 0 && has_err > opts.disks.len() - opts.min_disks {
+                if let Some(finished_fn) = opts.finished.as_ref() {
+                    finished_fn(&errs).await;
+                }
+                let mut combined_err = Vec::new();
+                errs.iter().zip(opts.disks.iter()).for_each(|(err, disk)| match (err, disk) {
+                    (Some(err), Some(disk)) => {
+                        combined_err.push(format!("drive {} returned: {}", disk.to_string(), err));
+                    }
+                    (Some(err), None) => {
+                        combined_err.push(err.to_string());
+                    }
+                    _ => {}
+                });
+
+                error!(
+                    "list_path_raw: has_err > 0 && has_err > opts.disks.len() - opts.min_disks break, err: {:?}",
+                    &combined_err.join(", ")
+                );
+                return Err(DiskError::other(combined_err.join(", ")));
+            }
+
+            // Break if all at EOF or error.
+            if at_eof + has_err == readers.len() {
+                if has_err > 0 {
+                    if let Some(finished_fn) = opts.finished.as_ref() {
+                        if has_err > 0 {
+                            finished_fn(&errs).await;
+                        }
+                    }
+                }
+
+                // error!("list_path_raw: at_eof + has_err == readers.len() break {:?}", &errs);
+                break;
+            }
+
+            if agree == readers.len() {
+                for r in readers.iter_mut() {
+                    let _ = r.skip(1).await;
+                }
+
+                if let Some(agreed_fn) = opts.agreed.as_ref() {
+                    // warn!("list_path_raw: agreed_fn start, current: {:?}", &current.name);
+                    agreed_fn(current).await;
+                    // warn!("list_path_raw: agreed_fn done");
+                }
+
+                continue;
+            }
+
+            // warn!("list_path_raw: skip start, current: {:?}", &current.name);
+
+            for (i, r) in readers.iter_mut().enumerate() {
+                if top_entries[i].is_some() {
+                    let _ = r.skip(1).await;
+                }
+            }
+
+            if let Some(partial_fn) = opts.partial.as_ref() {
+                partial_fn(MetaCacheEntries(top_entries), &errs).await;
+            }
+        }
+        Ok(())
+    });
+
+    if let Err(err) = revjob.await.map_err(std::io::Error::other)? {
+        error!("list_path_raw: revjob err {:?}", err);
+        let _ = cancel_tx.send(true);
+
+        return Err(err);
+    }
+
+    let results = join_all(jobs).await;
+    for result in results {
+        if let Err(err) = result {
+            error!("list_path_raw err {:?}", err);
+        }
+    }
+
+    // warn!("list_path_raw: done");
+    Ok(())
+}

crates/ecstore/src/cache_value/mod.rs

@@ -0,0 +1,16 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// pub mod cache;
+pub mod metacache_set;

crates/ecstore/src/checksum.rs

@@ -0,0 +1,327 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use lazy_static::lazy_static;
+use std::collections::HashMap;
+
+use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
+use crate::{disk::DiskAPI, store_api::GetObjectReader};
+use rustfs_utils::crypto::{base64_decode, base64_encode};
+use rustfs_utils::hasher::{Hasher, Sha256};
+use s3s::header::{
+    X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
+};
+
+use enumset::{EnumSet, EnumSetType, enum_set};
+
+#[derive(Debug, EnumSetType, Default)]
+#[enumset(repr = "u8")]
+pub enum ChecksumMode {
+    #[default]
+    ChecksumNone,
+    ChecksumSHA256,
+    ChecksumSHA1,
+    ChecksumCRC32,
+    ChecksumCRC32C,
+    ChecksumCRC64NVME,
+    ChecksumFullObject,
+}
+
+lazy_static! {
+    static ref C_ChecksumMask: EnumSet<ChecksumMode> = {
+        let mut s = EnumSet::all();
+        s.remove(ChecksumMode::ChecksumFullObject);
+        s
+    };
+    static ref C_ChecksumFullObjectCRC32: EnumSet<ChecksumMode> =
+        enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject);
+    static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> =
+        enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject);
+}
+const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme";
+
+impl ChecksumMode {
+    //pub const CRC64_NVME_POLYNOMIAL: i64 = 0xad93d23594c93659;
+
+    pub fn base(&self) -> ChecksumMode {
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        match s.as_u8() {
+            1_u8 => ChecksumMode::ChecksumNone,
+            2_u8 => ChecksumMode::ChecksumSHA256,
+            4_u8 => ChecksumMode::ChecksumSHA1,
+            8_u8 => ChecksumMode::ChecksumCRC32,
+            16_u8 => ChecksumMode::ChecksumCRC32C,
+            32_u8 => ChecksumMode::ChecksumCRC64NVME,
+            _ => panic!("enum err."),
+        }
+    }
+
+    pub fn is(&self, t: ChecksumMode) -> bool {
+        *self & t == t
+    }
+
+    pub fn key(&self) -> String {
+        //match c & checksumMask {
+        match self {
+            ChecksumMode::ChecksumCRC32 => {
+                return X_AMZ_CHECKSUM_CRC32.to_string();
+            }
+            ChecksumMode::ChecksumCRC32C => {
+                return X_AMZ_CHECKSUM_CRC32C.to_string();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return X_AMZ_CHECKSUM_SHA1.to_string();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return X_AMZ_CHECKSUM_SHA256.to_string();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return AMZ_CHECKSUM_CRC64NVME.to_string();
+            }
+            _ => {
+                return "".to_string();
+            }
+        }
+    }
+
+    pub fn can_composite(&self) -> bool {
+        todo!();
+    }
+
+    pub fn can_merge_crc(&self) -> bool {
+        todo!();
+    }
+
+    pub fn full_object_requested(&self) -> bool {
+        todo!();
+    }
+
+    pub fn key_capitalized(&self) -> String {
+        self.key()
+    }
+
+    pub fn raw_byte_len(&self) -> usize {
+        let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8();
+        if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 {
+            4
+        } else if u == ChecksumMode::ChecksumSHA1 as u8 {
+            4 //sha1.size
+        } else if u == ChecksumMode::ChecksumSHA256 as u8 {
+            4 //sha256.size
+        } else if u == ChecksumMode::ChecksumCRC64NVME as u8 {
+            4 //crc64.size
+        } else {
+            0
+        }
+    }
+
+    pub fn hasher(&self) -> Result<Box<dyn Hasher>, std::io::Error> {
+        match /*C_ChecksumMask & **/self {
+            /*ChecksumMode::ChecksumCRC32 => {
+                return Ok(Box::new(crc32fast::Hasher::new()));
+            }*/
+            /*ChecksumMode::ChecksumCRC32C => {
+                return Ok(Box::new(crc32::new(crc32.MakeTable(crc32.Castagnoli))));
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return Ok(Box::new(sha1::new()));
+            }*/
+            ChecksumMode::ChecksumSHA256 => {
+                return Ok(Box::new(Sha256::new()));
+            }
+            /*ChecksumMode::ChecksumCRC64NVME => {
+                return Ok(Box::new(crc64nvme.New());
+            }*/
+            _ => return Err(std::io::Error::other("unsupported checksum type")),
+        }
+    }
+
+    pub fn is_set(&self) -> bool {
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        s.len() == 1
+    }
+
+    pub fn set_default(&mut self, t: ChecksumMode) {
+        if !self.is_set() {
+            *self = t;
+        }
+    }
+
+    pub fn encode_to_string(&self, b: &[u8]) -> Result<String, std::io::Error> {
+        if !self.is_set() {
+            return Ok("".to_string());
+        }
+        let mut h = self.hasher()?;
+        h.write(b);
+        Ok(base64_encode(h.sum().as_bytes()))
+    }
+
+    pub fn to_string(&self) -> String {
+        //match c & checksumMask {
+        match self {
+            ChecksumMode::ChecksumCRC32 => {
+                return "CRC32".to_string();
+            }
+            ChecksumMode::ChecksumCRC32C => {
+                return "CRC32C".to_string();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return "SHA1".to_string();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return "SHA256".to_string();
+            }
+            ChecksumMode::ChecksumNone => {
+                return "".to_string();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return "CRC64NVME".to_string();
+            }
+            _ => {
+                return "<invalid>".to_string();
+            }
+        }
+    }
+
+    pub fn check_sum_reader(&self, r: GetObjectReader) -> Result<Checksum, std::io::Error> {
+        let mut h = self.hasher()?;
+        Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
+    }
+
+    pub fn check_sum_bytes(&self, b: &[u8]) -> Result<Checksum, std::io::Error> {
+        let mut h = self.hasher()?;
+        Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
+    }
+
+    pub fn composite_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
+        if !self.can_composite() {
+            return Err(std::io::Error::other("cannot do composite checksum"));
+        }
+        p.sort_by(|i, j| {
+            if i.part_num < j.part_num {
+                std::cmp::Ordering::Less
+            } else if i.part_num > j.part_num {
+                std::cmp::Ordering::Greater
+            } else {
+                std::cmp::Ordering::Equal
+            }
+        });
+        let c = self.base();
+        let crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize);
+        let mut h = self.hasher()?;
+        h.write(&crc_bytes);
+        Ok(Checksum {
+            checksum_type: self.clone(),
+            r: h.sum().as_bytes().to_vec(),
+            computed: false,
+        })
+    }
+
+    pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
+        todo!();
+    }
+}
+
+#[derive(Default)]
+pub struct Checksum {
+    checksum_type: ChecksumMode,
+    r: Vec<u8>,
+    computed: bool,
+}
+
+#[allow(dead_code)]
+impl Checksum {
+    fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
+        if t.is_set() && b.len() == t.raw_byte_len() {
+            return Checksum {
+                checksum_type: t,
+                r: b.to_vec(),
+                computed: false,
+            };
+        }
+        Checksum::default()
+    }
+
+    #[allow(dead_code)]
+    fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
+        let b = match base64_decode(s.as_bytes()) {
+            Ok(b) => b,
+            Err(err) => return Err(std::io::Error::other(err.to_string())),
+        };
+        if t.is_set() && b.len() == t.raw_byte_len() {
+            return Ok(Checksum {
+                checksum_type: t,
+                r: b,
+                computed: false,
+            });
+        }
+        Ok(Checksum::default())
+    }
+
+    fn is_set(&self) -> bool {
+        self.checksum_type.is_set() && self.r.len() == self.checksum_type.raw_byte_len()
+    }
+
+    fn encoded(&self) -> String {
+        if !self.is_set() {
+            return "".to_string();
+        }
+        base64_encode(&self.r)
+    }
+
+    #[allow(dead_code)]
+    fn raw(&self) -> Option<Vec<u8>> {
+        if !self.is_set() {
+            return None;
+        }
+        Some(self.r.clone())
+    }
+}
+
+pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) {
+    opts.user_metadata
+        .insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string());
+    if opts.auto_checksum.full_object_requested() {
+        opts.user_metadata
+            .insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
+    }
+}
+
+pub fn apply_auto_checksum(opts: &mut PutObjectOptions, all_parts: &mut [ObjectPart]) -> Result<(), std::io::Error> {
+    if opts.auto_checksum.can_composite() && !opts.auto_checksum.is(ChecksumMode::ChecksumFullObject) {
+        let crc = opts.auto_checksum.composite_checksum(all_parts)?;
+        opts.user_metadata = {
+            let mut hm = HashMap::new();
+            hm.insert(opts.auto_checksum.key(), crc.encoded());
+            hm
+        }
+    } else if opts.auto_checksum.can_merge_crc() {
+        let crc = opts.auto_checksum.full_object_checksum(all_parts)?;
+        opts.user_metadata = {
+            let mut hm = HashMap::new();
+            hm.insert(opts.auto_checksum.key_capitalized(), crc.encoded());
+            hm.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
+            hm
+        }
+    }
+
+    Ok(())
+}

crates/ecstore/src/chunk_stream.rs

@@ -0,0 +1,270 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// use crate::error::StdError;
+// use bytes::Bytes;
+// use futures::pin_mut;
+// use futures::stream::{Stream, StreamExt};
+// use std::future::Future;
+// use std::pin::Pin;
+// use std::task::{Context, Poll};
+// use transform_stream::AsyncTryStream;
+
+// pub type SyncBoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + Sync + 'a>>;
+
+// pub struct ChunkedStream<'a> {
+//     /// inner
+//     inner: AsyncTryStream<Bytes, StdError, SyncBoxFuture<'a, Result<(), StdError>>>,
+
+//     remaining_length: usize,
+// }
+
+// impl<'a> ChunkedStream<'a> {
+//     pub fn new<S>(body: S, content_length: usize, chunk_size: usize, need_padding: bool) -> Self
+//     where
+//         S: Stream<Item = Result<Bytes, StdError>> + Send + Sync + 'a,
+//     {
+//         let inner = AsyncTryStream::<_, _, SyncBoxFuture<'a, Result<(), StdError>>>::new(|mut y| {
+//             #[allow(clippy::shadow_same)] // necessary for `pin_mut!`
+//             Box::pin(async move {
+//                 pin_mut!(body);
+//                 // 上一次没用完的数据
+//                 let mut prev_bytes = Bytes::new();
+//                 let mut readed_size = 0;
+
+//                 loop {
+//                     let data: Vec<Bytes> = {
+//                         // 读固定大小的数据
+//                         match Self::read_data(body.as_mut(), prev_bytes, chunk_size).await {
+//                             None => break,
+//                             Some(Err(e)) => return Err(e),
+//                             Some(Ok((data, remaining_bytes))) => {
+//                                 // debug!(
+//                                 //     "content_length:{},readed_size:{}, read_data data:{}, remaining_bytes: {} ",
+//                                 //     content_length,
+//                                 //     readed_size,
+//                                 //     data.len(),
+//                                 //     remaining_bytes.len()
+//                                 // );
+
+//                                 prev_bytes = remaining_bytes;
+//                                 data
+//                             }
+//                         }
+//                     };
+
+//                     for bytes in data {
+//                         readed_size += bytes.len();
+//                         // debug!("readed_size {}, content_length {}", readed_size, content_length,);
+//                         y.yield_ok(bytes).await;
+//                     }
+
+//                     if readed_size + prev_bytes.len() >= content_length {
+//                         // debug!(
+//                         //     "读完了 readed_size:{} + prev_bytes.len({}) == content_length {}",
+//                         //     readed_size,
+//                         //     prev_bytes.len(),
+//                         //     content_length,
+//                         // );
+
+//                         // 填充 0？
+//                         if !need_padding {
+//                             y.yield_ok(prev_bytes).await;
+//                             break;
+//                         }
+
+//                         let mut bytes = vec![0u8; chunk_size];
+//                         let (left, _) = bytes.split_at_mut(prev_bytes.len());
+//                         left.copy_from_slice(&prev_bytes);
+
+//                         y.yield_ok(Bytes::from(bytes)).await;
+
+//                         break;
+//                     }
+//                 }
+
+//                 // debug!("chunked stream exit");
+
+//                 Ok(())
+//             })
+//         });
+//         Self {
+//             inner,
+//             remaining_length: content_length,
+//         }
+//     }
+//     /// read data and return remaining bytes
+//     async fn read_data<S>(
+//         mut body: Pin<&mut S>,
+//         prev_bytes: Bytes,
+//         data_size: usize,
+//     ) -> Option<Result<(Vec<Bytes>, Bytes), StdError>>
+//     where
+//         S: Stream<Item = Result<Bytes, StdError>> + Send,
+//     {
+//         let mut bytes_buffer = Vec::new();
+
+//         // 只执行一次
+//         let mut push_data_bytes = |mut bytes: Bytes| {
+//             // debug!("read from body {} split per {}, prev_bytes: {}", bytes.len(), data_size, prev_bytes.len());
+
+//             if bytes.is_empty() {
+//                 return None;
+//             }
+
+//             if data_size == 0 {
+//                 return Some(bytes);
+//             }
+
+//             // 合并上一次数据
+//             if !prev_bytes.is_empty() {
+//                 let need_size = data_size.wrapping_sub(prev_bytes.len());
+//                 // debug!(
+//                 //     " 上一次有剩余{},从这一次中取{},共：{}",
+//                 //     prev_bytes.len(),
+//                 //     need_size,
+//                 //     prev_bytes.len() + need_size
+//                 // );
+//                 if bytes.len() >= need_size {
+//                     let data = bytes.split_to(need_size);
+//                     let mut combined = Vec::new();
+//                     combined.extend_from_slice(&prev_bytes);
+//                     combined.extend_from_slice(&data);
+
+//                     // debug!(
+//                     //     "取到的长度大于所需，取出需要的长度：{},与上一次合并得到：{}，bytes 剩余：{}",
+//                     //     need_size,
+//                     //     combined.len(),
+//                     //     bytes.len(),
+//                     // );
+
+//                     bytes_buffer.push(Bytes::from(combined));
+//                 } else {
+//                     let mut combined = Vec::new();
+//                     combined.extend_from_slice(&prev_bytes);
+//                     combined.extend_from_slice(&bytes);
+
+//                     // debug!(
+//                     //     "取到的长度小于所需，取出需要的长度：{},与上一次合并得到：{}，bytes 剩余：{}，直接返回",
+//                     //     need_size,
+//                     //     combined.len(),
+//                     //     bytes.len(),
+//                     // );
+
+//                     return Some(Bytes::from(combined));
+//                 }
+//             }
+
+//             // 取到的数据比需要的块大，从 bytes 中截取需要的块大小
+//             if data_size <= bytes.len() {
+//                 let n = bytes.len() / data_size;
+
+//                 for _ in 0..n {
+//                     let data = bytes.split_to(data_size);
+
+//                     // println!("bytes_buffer.push: {}，剩余：{}", data.len(), bytes.len());
+//                     bytes_buffer.push(data);
+//                 }
+
+//                 Some(bytes)
+//             } else {
+//                 // 不够
+//                 Some(bytes)
+//             }
+//         };
+
+//         // 剩余数据
+//         let remaining_bytes = 'outer: {
+//             // // 如果上一次数据足够，跳出
+//             // if let Some(remaining_bytes) = push_data_bytes(prev_bytes) {
+//             //     println!("从剩下的取");
+//             //     break 'outer remaining_bytes;
+//             // }
+
+//             loop {
+//                 match body.next().await? {
+//                     Err(e) => return Some(Err(e)),
+//                     Ok(bytes) => {
+//                         if let Some(remaining_bytes) = push_data_bytes(bytes) {
+//                             break 'outer remaining_bytes;
+//                         }
+//                     }
+//                 }
+//             }
+//         };
+
+//         Some(Ok((bytes_buffer, remaining_bytes)))
+//     }
+
+//     fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Result<Bytes, StdError>>> {
+//         let ans = Pin::new(&mut self.inner).poll_next(cx);
+//         if let Poll::Ready(Some(Ok(ref bytes))) = ans {
+//             self.remaining_length = self.remaining_length.saturating_sub(bytes.len());
+//         }
+//         ans
+//     }
+
+//     // pub fn exact_remaining_length(&self) -> usize {
+//     //     self.remaining_length
+//     // }
+// }
+
+// impl Stream for ChunkedStream<'_> {
+//     type Item = Result<Bytes, StdError>;
+
+//     fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+//         self.poll(cx)
+//     }
+
+//     fn size_hint(&self) -> (usize, Option<usize>) {
+//         (0, None)
+//     }
+// }
+
+// #[cfg(test)]
+// mod test {
+
+//     use super::*;
+
+//     #[tokio::test]
+//     async fn test_chunked_stream() {
+//         let chunk_size = 4;
+
+//         let data1 = vec![1u8; 7777]; // 65536
+//         let data2 = vec![1u8; 7777]; // 65536
+
+//         let content_length = data1.len() + data2.len();
+
+//         let chunk1 = Bytes::from(data1);
+//         let chunk2 = Bytes::from(data2);
+
+//         let chunk_results: Vec<Result<Bytes, _>> = vec![Ok(chunk1), Ok(chunk2)];
+
+//         let stream = futures::stream::iter(chunk_results);
+
+//         let mut chunked_stream = ChunkedStream::new(stream, content_length, chunk_size, true);
+
+//         loop {
+//             let ans1 = chunked_stream.next().await;
+//             if ans1.is_none() {
+//                 break;
+//             }
+
+//             let bytes = ans1.unwrap().unwrap();
+//             assert!(bytes.len() == chunk_size)
+//         }
+
+//         // assert_eq!(ans1.unwrap(), chunk1_data.as_slice());
+//     }
+// }

crates/ecstore/src/client/admin_handler_utils.rs

@@ -0,0 +1,47 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use http::status::StatusCode;
+use std::fmt::{self, Display, Formatter};
+
+#[derive(Default, thiserror::Error, Debug, Clone, PartialEq)]
+pub struct AdminError {
+    pub code: String,
+    pub message: String,
+    pub status_code: StatusCode,
+}
+
+impl Display for AdminError {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "{}", self.message)
+    }
+}
+
+impl AdminError {
+    pub fn new(code: &str, message: &str, status_code: StatusCode) -> Self {
+        Self {
+            code: code.to_string(),
+            message: message.to_string(),
+            status_code,
+        }
+    }
+
+    pub fn msg(message: &str) -> Self {
+        Self {
+            code: "InternalError".to_string(),
+            message: message.to_string(),
+            status_code: StatusCode::INTERNAL_SERVER_ERROR,
+        }
+    }
+}

crates/ecstore/src/client/api_bucket_policy.rs

@@ -0,0 +1,143 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use bytes::Bytes;
+use http::{HeaderMap, StatusCode};
+use std::collections::HashMap;
+
+use crate::client::{
+    api_error_response::http_resp_to_error_response,
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
+};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+
+impl TransitionClient {
+    pub async fn set_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
+        if policy == "" {
+            return self.remove_bucket_policy(bucket_name).await;
+        }
+
+        self.put_bucket_policy(bucket_name, policy).await
+    }
+
+    pub async fn put_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("policy".to_string(), "".to_string());
+
+        let mut req_metadata = RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            query_values: url_values,
+            content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())),
+            content_length: policy.len() as i64,
+            object_name: "".to_string(),
+            custom_header: HeaderMap::new(),
+            content_md5_base64: "".to_string(),
+            content_sha256_hex: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
+        //defer closeResponse(resp)
+        //if resp != nil {
+        if resp.status() != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
+        }
+        //}
+        Ok(())
+    }
+
+    pub async fn remove_bucket_policy(&self, bucket_name: &str) -> Result<(), std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("policy".to_string(), "".to_string());
+
+        let resp = self
+            .execute_method(
+                http::Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    query_values: url_values,
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    object_name: "".to_string(),
+                    custom_header: HeaderMap::new(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+        //defer closeResponse(resp)
+
+        if resp.status() != StatusCode::NO_CONTENT {
+            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
+        }
+
+        Ok(())
+    }
+
+    pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        let bucket_policy = self.get_bucket_policy_inner(bucket_name).await?;
+        Ok(bucket_policy)
+    }
+
+    pub async fn get_bucket_policy_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("policy".to_string(), "".to_string());
+
+        let resp = self
+            .execute_method(
+                http::Method::GET,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    query_values: url_values,
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    object_name: "".to_string(),
+                    custom_header: HeaderMap::new(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string();
+        Ok(policy)
+    }
+}

crates/ecstore/src/client/api_error_response.rs

@@ -0,0 +1,285 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use http::StatusCode;
+use serde::{Deserialize, Serialize};
+use serde::{de::Deserializer, ser::Serializer};
+use std::fmt::Display;
+
+use s3s::Body;
+use s3s::S3ErrorCode;
+
+const _REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues.";
+
+#[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error, PartialEq, Eq)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct ErrorResponse {
+    #[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")]
+    pub code: S3ErrorCode,
+    pub message: String,
+    pub bucket_name: String,
+    pub key: String,
+    pub resource: String,
+    pub request_id: String,
+    pub host_id: String,
+    pub region: String,
+    pub server: String,
+    #[serde(skip)]
+    pub status_code: StatusCode,
+}
+
+fn serialize_code<S>(_data: &S3ErrorCode, s: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    s.serialize_str("")
+}
+
+fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
+}
+
+impl Default for ErrorResponse {
+    fn default() -> Self {
+        ErrorResponse {
+            code: S3ErrorCode::Custom("".into()),
+            message: Default::default(),
+            bucket_name: Default::default(),
+            key: Default::default(),
+            resource: Default::default(),
+            request_id: Default::default(),
+            host_id: Default::default(),
+            region: Default::default(),
+            server: Default::default(),
+            status_code: Default::default(),
+        }
+    }
+}
+
+impl Display for ErrorResponse {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.message)
+    }
+}
+
+pub fn to_error_response(err: &std::io::Error) -> ErrorResponse {
+    if let Some(err) = err.get_ref() {
+        if err.is::<ErrorResponse>() {
+            err.downcast_ref::<ErrorResponse>().expect("err!").clone()
+        } else {
+            ErrorResponse::default()
+        }
+    } else {
+        ErrorResponse::default()
+    }
+}
+
+pub fn http_resp_to_error_response(
+    resp: http::Response<Body>,
+    b: Vec<u8>,
+    bucket_name: &str,
+    object_name: &str,
+) -> ErrorResponse {
+    let err_body = String::from_utf8(b).unwrap();
+    let err_resp_ = serde_xml_rs::from_str::<ErrorResponse>(&err_body);
+    let mut err_resp = ErrorResponse::default();
+    if err_resp_.is_err() {
+        match resp.status() {
+            StatusCode::NOT_FOUND => {
+                if object_name == "" {
+                    err_resp = ErrorResponse {
+                        status_code: resp.status(),
+                        code: S3ErrorCode::NoSuchBucket,
+                        message: "The specified bucket does not exist.".to_string(),
+                        bucket_name: bucket_name.to_string(),
+                        ..Default::default()
+                    };
+                } else {
+                    err_resp = ErrorResponse {
+                        status_code: resp.status(),
+                        code: S3ErrorCode::NoSuchKey,
+                        message: "The specified key does not exist.".to_string(),
+                        bucket_name: bucket_name.to_string(),
+                        key: object_name.to_string(),
+                        ..Default::default()
+                    };
+                }
+            }
+            StatusCode::FORBIDDEN => {
+                err_resp = ErrorResponse {
+                    status_code: resp.status(),
+                    code: S3ErrorCode::AccessDenied,
+                    message: "Access Denied.".to_string(),
+                    bucket_name: bucket_name.to_string(),
+                    key: object_name.to_string(),
+                    ..Default::default()
+                };
+            }
+            StatusCode::CONFLICT => {
+                err_resp = ErrorResponse {
+                    status_code: resp.status(),
+                    code: S3ErrorCode::BucketNotEmpty,
+                    message: "Bucket not empty.".to_string(),
+                    bucket_name: bucket_name.to_string(),
+                    ..Default::default()
+                };
+            }
+            StatusCode::PRECONDITION_FAILED => {
+                err_resp = ErrorResponse {
+                    status_code: resp.status(),
+                    code: S3ErrorCode::PreconditionFailed,
+                    message: "Pre condition failed.".to_string(),
+                    bucket_name: bucket_name.to_string(),
+                    key: object_name.to_string(),
+                    ..Default::default()
+                };
+            }
+            _ => {
+                let mut msg = resp.status().to_string();
+                if err_body.len() > 0 {
+                    msg = err_body;
+                }
+                err_resp = ErrorResponse {
+                    status_code: resp.status(),
+                    code: S3ErrorCode::Custom(resp.status().to_string().into()),
+                    message: msg,
+                    bucket_name: bucket_name.to_string(),
+                    ..Default::default()
+                };
+            }
+        }
+    } else {
+        err_resp = err_resp_.unwrap();
+    }
+    err_resp.status_code = resp.status();
+    if let Some(server_name) = resp.headers().get("Server") {
+        err_resp.server = server_name.to_str().expect("err").to_string();
+    }
+
+    let code = resp.headers().get("x-minio-error-code");
+    if code.is_some() {
+        err_resp.code = S3ErrorCode::Custom(code.expect("err").to_str().expect("err").into());
+    }
+    let desc = resp.headers().get("x-minio-error-desc");
+    if desc.is_some() {
+        err_resp.message = desc.expect("err").to_str().expect("err").trim_matches('"').to_string();
+    }
+
+    if err_resp.request_id == "" {
+        if let Some(x_amz_request_id) = resp.headers().get("x-amz-request-id") {
+            err_resp.request_id = x_amz_request_id.to_str().expect("err").to_string();
+        }
+    }
+    if err_resp.host_id == "" {
+        if let Some(x_amz_id_2) = resp.headers().get("x-amz-id-2") {
+            err_resp.host_id = x_amz_id_2.to_str().expect("err").to_string();
+        }
+    }
+    if err_resp.region == "" {
+        if let Some(x_amz_bucket_region) = resp.headers().get("x-amz-bucket-region") {
+            err_resp.region = x_amz_bucket_region.to_str().expect("err").to_string();
+        }
+    }
+    if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" {
+        err_resp.message = format!("Region does not match, expecting region ‘{}’.", err_resp.region);
+    }
+
+    err_resp
+}
+
+pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::InvalidArgument,
+        message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’."
+            .to_string(),
+        bucket_name: bucket_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
+    let msg = format!(
+        "Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
+        total_size, max_object_size
+    );
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::EntityTooLarge,
+        message: msg,
+        bucket_name: bucket_name.to_string(),
+        key: object_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
+    let msg = format!(
+        "Your proposed upload size ‘{}’ is below the minimum allowed object size ‘0B’ for single PUT operation.",
+        total_size
+    );
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::EntityTooSmall,
+        message: msg,
+        bucket_name: bucket_name.to_string(),
+        key: object_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
+    let msg = format!(
+        "Data read ‘{}’ is not equal to the size ‘{}’ of the input Reader.",
+        total_read, total_size
+    );
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::Custom("UnexpectedEOF".into()),
+        message: msg,
+        bucket_name: bucket_name.to_string(),
+        key: object_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_invalid_argument(message: &str) -> ErrorResponse {
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::InvalidArgument,
+        message: message.to_string(),
+        request_id: "rustfs".to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_api_not_supported(message: &str) -> ErrorResponse {
+    ErrorResponse {
+        status_code: StatusCode::NOT_IMPLEMENTED,
+        code: S3ErrorCode::Custom("APINotSupported".into()),
+        message: message.to_string(),
+        request_id: "rustfs".to_string(),
+        ..Default::default()
+    }
+}

crates/ecstore/src/client/api_get_object.rs

@@ -0,0 +1,228 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use bytes::Bytes;
+use http::HeaderMap;
+use std::io::Cursor;
+use tokio::io::BufReader;
+
+use crate::client::{
+    api_error_response::err_invalid_argument,
+    api_get_options::GetObjectOptions,
+    transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
+};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+
+impl TransitionClient {
+    pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
+        todo!();
+    }
+
+    pub async fn get_object_inner(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        opts: &GetObjectOptions,
+    ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
+        let resp = self
+            .execute_method(
+                http::Method::GET,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: object_name.to_string(),
+                    query_values: opts.to_query_values(),
+                    custom_header: opts.header(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        let resp = &resp;
+        let object_stat = to_object_info(bucket_name, object_name, resp.headers())?;
+
+        let b = resp.body().bytes().expect("err").to_vec();
+        Ok((object_stat, resp.headers().clone(), BufReader::new(Cursor::new(b))))
+    }
+}
+
+#[derive(Default)]
+#[allow(dead_code)]
+pub struct GetRequest {
+    pub buffer: Vec<u8>,
+    pub offset: i64,
+    pub did_offset_change: bool,
+    pub been_read: bool,
+    pub is_read_at: bool,
+    pub is_read_op: bool,
+    pub is_first_req: bool,
+    pub setting_object_info: bool,
+}
+
+#[allow(dead_code)]
+pub struct GetResponse {
+    pub size: i64,
+    //pub error:       error,
+    pub did_read: bool,
+    pub object_info: ObjectInfo,
+}
+
+#[derive(Default)]
+pub struct Object {
+    //pub reqch:      chan<- getRequest,
+    //pub resch:      <-chan getResponse,
+    //pub cancel:     context.CancelFunc,
+    pub curr_offset: i64,
+    pub object_info: ObjectInfo,
+    pub seek_data: bool,
+    pub is_closed: bool,
+    pub is_started: bool,
+    //pub prev_err: error,
+    pub been_read: bool,
+    pub object_info_set: bool,
+}
+
+impl Object {
+    pub fn new() -> Object {
+        Self { ..Default::default() }
+    }
+
+    fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
+        todo!()
+    }
+
+    fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
+        self.curr_offset += bytes_read;
+
+        Ok(())
+    }
+
+    fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
+        let mut read_req = GetRequest {
+            is_read_op: true,
+            been_read: self.been_read,
+            buffer: b.to_vec(),
+            ..Default::default()
+        };
+
+        if !self.is_started {
+            read_req.is_first_req = true;
+        }
+
+        read_req.did_offset_change = self.seek_data;
+        read_req.offset = self.curr_offset;
+
+        let response = self.do_get_request(&read_req)?;
+
+        let bytes_read = response.size;
+
+        let oerr = self.set_offset(bytes_read);
+
+        Ok(response.size)
+    }
+
+    fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
+        if !self.is_started || !self.object_info_set {
+            let _ = self.do_get_request(&GetRequest {
+                is_first_req: !self.is_started,
+                setting_object_info: !self.object_info_set,
+                ..Default::default()
+            })?;
+        }
+
+        Ok(self.object_info.clone())
+    }
+
+    fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
+        self.curr_offset = offset;
+
+        let mut read_at_req = GetRequest {
+            is_read_op: true,
+            is_read_at: true,
+            did_offset_change: true,
+            been_read: self.been_read,
+            offset,
+            buffer: b.to_vec(),
+            ..Default::default()
+        };
+
+        if !self.is_started {
+            read_at_req.is_first_req = true;
+        }
+
+        let response = self.do_get_request(&read_at_req)?;
+        let bytes_read = response.size;
+        if !self.object_info_set {
+            self.curr_offset += bytes_read;
+        } else {
+            let oerr = self.set_offset(bytes_read);
+        }
+        Ok(response.size)
+    }
+
+    fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
+        if !self.is_started || !self.object_info_set {
+            let seek_req = GetRequest {
+                is_read_op: false,
+                offset,
+                is_first_req: true,
+                ..Default::default()
+            };
+            let _ = self.do_get_request(&seek_req);
+        }
+
+        let mut new_offset = self.curr_offset;
+
+        match whence {
+            0 => {
+                new_offset = offset;
+            }
+            1 => {
+                new_offset += offset;
+            }
+            2 => {
+                new_offset = self.object_info.size as i64 + offset as i64;
+            }
+            _ => {
+                return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid whence {}", whence))));
+            }
+        }
+
+        self.seek_data = (new_offset != self.curr_offset) || self.seek_data;
+        self.curr_offset = new_offset;
+
+        Ok(self.curr_offset)
+    }
+
+    fn close(&mut self) -> Result<(), std::io::Error> {
+        self.is_closed = true;
+        Ok(())
+    }
+}

crates/ecstore/src/client/api_get_options.rs

@@ -0,0 +1,146 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use http::{HeaderMap, HeaderName, HeaderValue};
+use std::collections::HashMap;
+use time::OffsetDateTime;
+use tracing::warn;
+
+use crate::client::api_error_response::err_invalid_argument;
+
+#[derive(Default)]
+#[allow(dead_code)]
+pub struct AdvancedGetOptions {
+    replication_deletemarker: bool,
+    is_replication_ready_for_deletemarker: bool,
+    replication_proxy_request: String,
+}
+
+pub struct GetObjectOptions {
+    pub headers: HashMap<String, String>,
+    pub req_params: HashMap<String, String>,
+    //pub server_side_encryption: encrypt.ServerSide,
+    pub version_id: String,
+    pub part_number: i64,
+    pub checksum: bool,
+    pub internal: AdvancedGetOptions,
+}
+
+impl Default for GetObjectOptions {
+    fn default() -> Self {
+        Self {
+            headers: HashMap::new(),
+            req_params: HashMap::new(),
+            //server_side_encryption: encrypt.ServerSide::default(),
+            version_id: "".to_string(),
+            part_number: 0,
+            checksum: false,
+            internal: AdvancedGetOptions::default(),
+        }
+    }
+}
+
+impl GetObjectOptions {
+    pub fn header(&self) -> HeaderMap {
+        let mut headers: HeaderMap = HeaderMap::with_capacity(self.headers.len());
+        for (k, v) in &self.headers {
+            if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
+                headers.insert(header_name, v.parse().expect("err"));
+            } else {
+                warn!("Invalid header name: {}", k);
+            }
+        }
+        if self.checksum {
+            headers.insert("x-amz-checksum-mode", "ENABLED".parse().expect("err"));
+        }
+        headers
+    }
+
+    pub fn set(&self, key: &str, value: &str) {
+        //self.headers[http.CanonicalHeaderKey(key)] = value;
+    }
+
+    pub fn set_req_param(&mut self, key: &str, value: &str) {
+        self.req_params.insert(key.to_string(), value.to_string());
+    }
+
+    pub fn add_req_param(&mut self, key: &str, value: &str) {
+        self.req_params.insert(key.to_string(), value.to_string());
+    }
+
+    pub fn set_match_etag(&mut self, etag: &str) -> Result<(), std::io::Error> {
+        self.set("If-Match", &format!("\"{etag}\""));
+        Ok(())
+    }
+
+    pub fn set_match_etag_except(&mut self, etag: &str) -> Result<(), std::io::Error> {
+        self.set("If-None-Match", &format!("\"{etag}\""));
+        Ok(())
+    }
+
+    pub fn set_unmodified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> {
+        if mod_time.unix_timestamp() == 0 {
+            return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
+        }
+        self.set("If-Unmodified-Since", &mod_time.to_string());
+        Ok(())
+    }
+
+    pub fn set_modified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> {
+        if mod_time.unix_timestamp() == 0 {
+            return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
+        }
+        self.set("If-Modified-Since", &mod_time.to_string());
+        Ok(())
+    }
+
+    pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> {
+        if start == 0 && end < 0 {
+            self.set("Range", &format!("bytes={}", end));
+        } else if 0 < start && end == 0 {
+            self.set("Range", &format!("bytes={}-", start));
+        } else if 0 <= start && start <= end {
+            self.set("Range", &format!("bytes={}-{}", start, end));
+        } else {
+            return Err(std::io::Error::other(err_invalid_argument(&format!(
+                "Invalid range specified: start={} end={}",
+                start, end
+            ))));
+        }
+        Ok(())
+    }
+
+    pub fn to_query_values(&self) -> HashMap<String, String> {
+        let mut url_values = HashMap::new();
+        if self.version_id != "" {
+            url_values.insert("versionId".to_string(), self.version_id.clone());
+        }
+        if self.part_number > 0 {
+            url_values.insert("partNumber".to_string(), self.part_number.to_string());
+        }
+
+        for (key, value) in self.req_params.iter() {
+            url_values.insert(key.to_string(), value.to_string());
+        }
+
+        url_values
+    }
+}

crates/ecstore/src/client/api_list.rs

@@ -0,0 +1,303 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use bytes::Bytes;
+use http::{HeaderMap, StatusCode};
+use std::collections::HashMap;
+
+use crate::client::{
+    api_error_response::http_resp_to_error_response,
+    api_s3_datatypes::{
+        ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart,
+    },
+    credentials,
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
+};
+use crate::store_api::BucketInfo;
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+
+impl TransitionClient {
+    pub fn list_buckets(&self) -> Result<Vec<BucketInfo>, std::io::Error> {
+        todo!();
+    }
+
+    pub async fn list_objects_v2_query(
+        &self,
+        bucket_name: &str,
+        object_prefix: &str,
+        continuation_token: &str,
+        fetch_owner: bool,
+        metadata: bool,
+        delimiter: &str,
+        start_after: &str,
+        max_keys: i64,
+        headers: HeaderMap,
+    ) -> Result<ListBucketV2Result, std::io::Error> {
+        let mut url_values = HashMap::new();
+
+        url_values.insert("list-type".to_string(), "2".to_string());
+        if metadata {
+            url_values.insert("metadata".to_string(), "true".to_string());
+        }
+        if start_after != "" {
+            url_values.insert("start-after".to_string(), start_after.to_string());
+        }
+        url_values.insert("encoding-type".to_string(), "url".to_string());
+        url_values.insert("prefix".to_string(), object_prefix.to_string());
+        url_values.insert("delimiter".to_string(), delimiter.to_string());
+
+        if continuation_token != "" {
+            url_values.insert("continuation-token".to_string(), continuation_token.to_string());
+        }
+
+        if fetch_owner {
+            url_values.insert("fetch-owner".to_string(), "true".to_string());
+        }
+
+        if max_keys > 0 {
+            url_values.insert("max-keys".to_string(), max_keys.to_string());
+        }
+
+        let mut resp = self
+            .execute_method(
+                http::Method::GET,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: "".to_string(),
+                    query_values: url_values,
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: headers,
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
+        }
+
+        //let mut list_bucket_result = ListBucketV2Result::default();
+        let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
+        let mut list_bucket_result = match serde_xml_rs::from_str::<ListBucketV2Result>(&String::from_utf8(b).unwrap()) {
+            Ok(result) => result,
+            Err(err) => {
+                return Err(std::io::Error::other(err.to_string()));
+            }
+        };
+        //println!("list_bucket_result: {:?}", list_bucket_result);
+
+        if list_bucket_result.is_truncated && list_bucket_result.next_continuation_token == "" {
+            return Err(std::io::Error::other(credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: "Truncated response should have continuation token set".to_string(),
+                },
+                request_id: "".to_string(),
+            }));
+        }
+
+        for (i, obj) in list_bucket_result.contents.iter_mut().enumerate() {
+            obj.name = decode_s3_name(&obj.name, &list_bucket_result.encoding_type)?;
+            //list_bucket_result.contents[i].mod_time = list_bucket_result.contents[i].mod_time.Truncate(time.Millisecond);
+        }
+
+        for (i, obj) in list_bucket_result.common_prefixes.iter_mut().enumerate() {
+            obj.prefix = decode_s3_name(&obj.prefix, &list_bucket_result.encoding_type)?;
+        }
+
+        Ok(list_bucket_result)
+    }
+
+    pub fn list_object_versions_query(
+        &self,
+        bucket_name: &str,
+        opts: &ListObjectsOptions,
+        key_marker: &str,
+        version_id_marker: &str,
+        delimiter: &str,
+    ) -> Result<ListVersionsResult, std::io::Error> {
+        /*if err := s3utils.CheckValidBucketName(bucketName); err != nil {
+          return ListVersionsResult{}, err
+        }
+        if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
+          return ListVersionsResult{}, err
+        }
+        urlValues := make(url.Values)
+
+        urlValues.Set("versions", "")
+
+        urlValues.Set("prefix", opts.Prefix)
+
+        urlValues.Set("delimiter", delimiter)
+
+        if keyMarker != "" {
+          urlValues.Set("key-marker", keyMarker)
+        }
+
+        if opts.max_keys > 0 {
+          urlValues.Set("max-keys", fmt.Sprintf("%d", opts.max_keys))
+        }
+
+        if versionIDMarker != "" {
+          urlValues.Set("version-id-marker", versionIDMarker)
+        }
+
+        if opts.WithMetadata {
+          urlValues.Set("metadata", "true")
+        }
+
+        urlValues.Set("encoding-type", "url")
+
+        let resp = self.executeMethod(http::Method::GET, &mut RequestMetadata{
+            bucketName:       bucketName,
+            queryValues:      urlValues,
+            contentSHA256Hex: emptySHA256Hex,
+            customHeader:     opts.headers,
+        }).await?;
+        defer closeResponse(resp)
+        if err != nil {
+          return ListVersionsResult{}, err
+        }
+        if resp != nil {
+          if resp.StatusCode != http.StatusOK {
+            return ListVersionsResult{}, httpRespToErrorResponse(resp, bucketName, "")
+          }
+        }
+
+        listObjectVersionsOutput := ListVersionsResult{}
+        err = xml_decoder(resp.Body, &listObjectVersionsOutput)
+        if err != nil {
+          return ListVersionsResult{}, err
+        }
+
+        for i, obj := range listObjectVersionsOutput.Versions {
+          listObjectVersionsOutput.Versions[i].Key, err = decode_s3_name(obj.Key, listObjectVersionsOutput.EncodingType)
+          if err != nil {
+            return listObjectVersionsOutput, err
+          }
+        }
+
+        for i, obj := range listObjectVersionsOutput.CommonPrefixes {
+          listObjectVersionsOutput.CommonPrefixes[i].Prefix, err = decode_s3_name(obj.Prefix, listObjectVersionsOutput.EncodingType)
+          if err != nil {
+            return listObjectVersionsOutput, err
+          }
+        }
+
+        if listObjectVersionsOutput.NextKeyMarker != "" {
+          listObjectVersionsOutput.NextKeyMarker, err = decode_s3_name(listObjectVersionsOutput.NextKeyMarker, listObjectVersionsOutput.EncodingType)
+          if err != nil {
+            return listObjectVersionsOutput, err
+          }
+        }
+
+        Ok(listObjectVersionsOutput)*/
+        todo!();
+    }
+
+    pub fn list_objects_query(
+        &self,
+        bucket_name: &str,
+        object_prefix: &str,
+        object_marker: &str,
+        delimiter: &str,
+        max_keys: i64,
+        headers: HeaderMap,
+    ) -> Result<ListBucketResult, std::io::Error> {
+        todo!();
+    }
+
+    pub fn list_multipart_uploads_query(
+        &self,
+        bucket_name: &str,
+        key_marker: &str,
+        upload_id_marker: &str,
+        prefix: &str,
+        delimiter: &str,
+        max_uploads: i64,
+    ) -> Result<ListMultipartUploadsResult, std::io::Error> {
+        todo!();
+    }
+
+    pub fn list_object_parts(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        upload_id: &str,
+    ) -> Result<HashMap<i64, ObjectPart>, std::io::Error> {
+        todo!();
+    }
+
+    pub fn find_upload_ids(&self, bucket_name: &str, object_name: &str) -> Result<Vec<String>, std::io::Error> {
+        todo!();
+    }
+
+    pub async fn list_object_parts_query(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        upload_id: &str,
+        part_number_marker: i64,
+        max_parts: i64,
+    ) -> Result<ListObjectPartsResult, std::io::Error> {
+        todo!();
+    }
+}
+
+#[allow(dead_code)]
+pub struct ListObjectsOptions {
+    reverse_versions: bool,
+    with_versions: bool,
+    with_metadata: bool,
+    prefix: String,
+    recursive: bool,
+    max_keys: i64,
+    start_after: String,
+    use_v1: bool,
+    headers: HeaderMap,
+}
+
+impl ListObjectsOptions {
+    pub fn set(&mut self, key: &str, value: &str) {
+        todo!();
+    }
+}
+
+fn decode_s3_name(name: &str, encoding_type: &str) -> Result<String, std::io::Error> {
+    match encoding_type {
+        "url" => {
+            //return url::QueryUnescape(name);
+            return Ok(name.to_string());
+        }
+        _ => {
+            return Ok(name.to_string());
+        }
+    }
+}

crates/ecstore/src/client/api_put_object.rs

@@ -0,0 +1,440 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use bytes::Bytes;
+use http::{HeaderMap, HeaderName, HeaderValue};
+use std::{collections::HashMap, sync::Arc};
+use time::{Duration, OffsetDateTime, macros::format_description};
+use tracing::{error, info, warn};
+
+use rustfs_utils::hasher::Hasher;
+use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus};
+use s3s::header::{
+    X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
+    X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION,
+};
+//use crate::disk::{BufferReader, Reader};
+use crate::checksum::ChecksumMode;
+use crate::client::{
+    api_error_response::{err_entity_too_large, err_invalid_argument},
+    api_put_object_common::optimal_part_info,
+    api_put_object_multipart::UploadPartParams,
+    api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
+    constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS},
+    credentials::SignatureType,
+    transition_api::{ReaderImpl, TransitionClient, UploadInfo},
+    utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
+};
+use rustfs_utils::crypto::base64_encode;
+
+#[derive(Debug, Clone)]
+pub struct AdvancedPutOptions {
+    pub source_version_id: String,
+    pub source_etag: String,
+    pub replication_status: ReplicationStatus,
+    pub source_mtime: OffsetDateTime,
+    pub replication_request: bool,
+    pub retention_timestamp: OffsetDateTime,
+    pub tagging_timestamp: OffsetDateTime,
+    pub legalhold_timestamp: OffsetDateTime,
+    pub replication_validity_check: bool,
+}
+
+impl Default for AdvancedPutOptions {
+    fn default() -> Self {
+        Self {
+            source_version_id: "".to_string(),
+            source_etag: "".to_string(),
+            replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
+            source_mtime: OffsetDateTime::now_utc(),
+            replication_request: false,
+            retention_timestamp: OffsetDateTime::now_utc(),
+            tagging_timestamp: OffsetDateTime::now_utc(),
+            legalhold_timestamp: OffsetDateTime::now_utc(),
+            replication_validity_check: false,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct PutObjectOptions {
+    pub user_metadata: HashMap<String, String>,
+    pub user_tags: HashMap<String, String>,
+    //pub progress: ReaderImpl,
+    pub content_type: String,
+    pub content_encoding: String,
+    pub content_disposition: String,
+    pub content_language: String,
+    pub cache_control: String,
+    pub expires: OffsetDateTime,
+    pub mode: ObjectLockRetentionMode,
+    pub retain_until_date: OffsetDateTime,
+    //pub server_side_encryption: encrypt.ServerSide,
+    pub num_threads: u64,
+    pub storage_class: String,
+    pub website_redirect_location: String,
+    pub part_size: u64,
+    pub legalhold: ObjectLockLegalHoldStatus,
+    pub send_content_md5: bool,
+    pub disable_content_sha256: bool,
+    pub disable_multipart: bool,
+    pub auto_checksum: ChecksumMode,
+    pub checksum: ChecksumMode,
+    pub concurrent_stream_parts: bool,
+    pub internal: AdvancedPutOptions,
+    pub custom_header: HeaderMap,
+}
+
+impl Default for PutObjectOptions {
+    fn default() -> Self {
+        Self {
+            user_metadata: HashMap::new(),
+            user_tags: HashMap::new(),
+            //progress: ReaderImpl::Body(Bytes::new()),
+            content_type: "".to_string(),
+            content_encoding: "".to_string(),
+            content_disposition: "".to_string(),
+            content_language: "".to_string(),
+            cache_control: "".to_string(),
+            expires: OffsetDateTime::UNIX_EPOCH,
+            mode: ObjectLockRetentionMode::from_static(""),
+            retain_until_date: OffsetDateTime::UNIX_EPOCH,
+            //server_side_encryption: encrypt.ServerSide::default(),
+            num_threads: 0,
+            storage_class: "".to_string(),
+            website_redirect_location: "".to_string(),
+            part_size: 0,
+            legalhold: ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF),
+            send_content_md5: false,
+            disable_content_sha256: false,
+            disable_multipart: false,
+            auto_checksum: ChecksumMode::ChecksumNone,
+            checksum: ChecksumMode::ChecksumNone,
+            concurrent_stream_parts: false,
+            internal: AdvancedPutOptions::default(),
+            custom_header: HeaderMap::new(),
+        }
+    }
+}
+
+#[allow(dead_code)]
+impl PutObjectOptions {
+    fn set_matche_tag(&mut self, etag: &str) {
+        if etag == "*" {
+            self.custom_header
+                .insert("If-Match", HeaderValue::from_str("*").expect("err"));
+        } else {
+            self.custom_header
+                .insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err"));
+        }
+    }
+
+    fn set_matche_tag_except(&mut self, etag: &str) {
+        if etag == "*" {
+            self.custom_header
+                .insert("If-None-Match", HeaderValue::from_str("*").expect("err"));
+        } else {
+            self.custom_header
+                .insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err"));
+        }
+    }
+
+    pub fn header(&self) -> HeaderMap {
+        let mut header = HeaderMap::new();
+
+        let mut content_type = self.content_type.clone();
+        if content_type == "" {
+            content_type = "application/octet-stream".to_string();
+        }
+        header.insert("Content-Type", HeaderValue::from_str(&content_type).expect("err"));
+
+        if self.content_encoding != "" {
+            header.insert("Content-Encoding", HeaderValue::from_str(&self.content_encoding).expect("err"));
+        }
+        if self.content_disposition != "" {
+            header.insert("Content-Disposition", HeaderValue::from_str(&self.content_disposition).expect("err"));
+        }
+        if self.content_language != "" {
+            header.insert("Content-Language", HeaderValue::from_str(&self.content_language).expect("err"));
+        }
+        if self.cache_control != "" {
+            header.insert("Cache-Control", HeaderValue::from_str(&self.cache_control).expect("err"));
+        }
+
+        if self.expires.unix_timestamp() != 0 {
+            header.insert(
+                "Expires",
+                HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
+            ); //rustfs invalid heade
+        }
+
+        if self.mode.as_str() != "" {
+            header.insert(X_AMZ_OBJECT_LOCK_MODE, HeaderValue::from_str(self.mode.as_str()).expect("err"));
+        }
+
+        if self.retain_until_date.unix_timestamp() != 0 {
+            header.insert(
+                X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
+                HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
+            );
+        }
+
+        if self.legalhold.as_str() != "" {
+            header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, HeaderValue::from_str(self.legalhold.as_str()).expect("err"));
+        }
+
+        if self.storage_class != "" {
+            header.insert(X_AMZ_STORAGE_CLASS, HeaderValue::from_str(&self.storage_class).expect("err"));
+        }
+
+        if self.website_redirect_location != "" {
+            header.insert(
+                X_AMZ_WEBSITE_REDIRECT_LOCATION,
+                HeaderValue::from_str(&self.website_redirect_location).expect("err"),
+            );
+        }
+
+        if !self.internal.replication_status.as_str().is_empty() {
+            header.insert(
+                X_AMZ_REPLICATION_STATUS,
+                HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err"),
+            );
+        }
+
+        for (k, v) in &self.user_metadata {
+            if is_amz_header(k) || is_standard_header(k) || is_storageclass_header(k) || is_rustfs_header(k) || is_minio_header(k)
+            {
+                if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
+                    header.insert(header_name, HeaderValue::from_str(&v).unwrap());
+                }
+            } else if let Ok(header_name) = HeaderName::from_bytes(format!("x-amz-meta-{}", k).as_bytes()) {
+                header.insert(header_name, HeaderValue::from_str(&v).unwrap());
+            }
+        }
+
+        for (k, v) in self.custom_header.iter() {
+            header.insert(k.clone(), v.clone());
+        }
+
+        header
+    }
+
+    fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
+        //if self.checksum.is_set() {
+        /*if !self.trailing_header_support {
+            return Err(Error::from(err_invalid_argument("Checksum requires Client with TrailingHeaders enabled")));
+        }*/
+        /*else if self.override_signer_type == SignatureType::SignatureV2 {
+            return Err(Error::from(err_invalid_argument("Checksum cannot be used with v2 signatures")));
+        }*/
+        //}
+
+        Ok(())
+    }
+}
+
+impl TransitionClient {
+    pub async fn put_object(
+        self: Arc<Self>,
+        bucket_name: &str,
+        object_name: &str,
+        reader: ReaderImpl,
+        object_size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        if object_size < 0 && opts.disable_multipart {
+            return Err(std::io::Error::other("object size must be provided with disable multipart upload"));
+        }
+
+        self.put_object_common(bucket_name, object_name, reader, object_size, opts)
+            .await
+    }
+
+    pub async fn put_object_common(
+        self: Arc<Self>,
+        bucket_name: &str,
+        object_name: &str,
+        reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        if size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+            return Err(std::io::Error::other(err_entity_too_large(
+                size,
+                MAX_MULTIPART_PUT_OBJECT_SIZE,
+                bucket_name,
+                object_name,
+            )));
+        }
+        let mut opts = opts.clone();
+        opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
+
+        let mut part_size = opts.part_size as i64;
+        if opts.part_size == 0 {
+            part_size = MIN_PART_SIZE;
+        }
+
+        if SignatureType::SignatureV2 == self.override_signer_type {
+            if size >= 0 && size < part_size || opts.disable_multipart {
+                return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
+            }
+            return self.put_object_multipart(bucket_name, object_name, reader, size, &opts).await;
+        }
+
+        if size < 0 {
+            if opts.disable_multipart {
+                return Err(std::io::Error::other("no length provided and multipart disabled"));
+            }
+            if opts.concurrent_stream_parts && opts.num_threads > 1 {
+                return self
+                    .put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts)
+                    .await;
+            }
+            return self
+                .put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts)
+                .await;
+        }
+
+        if size <= part_size || opts.disable_multipart {
+            return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
+        }
+
+        self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts)
+            .await
+    }
+
+    pub async fn put_object_multipart_stream_no_length(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut total_uploaded_size: i64 = 0;
+
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let (total_parts_count, part_size, _) = optimal_part_info(-1, opts.part_size)?;
+
+        let mut opts = opts.clone();
+
+        if opts.checksum.is_set() {
+            opts.send_content_md5 = false;
+            opts.auto_checksum = opts.checksum.clone();
+        }
+        if !opts.send_content_md5 {
+            //add_auto_checksum_headers(&mut opts);
+        }
+
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut part_number = 1;
+        let mut parts_info = HashMap::<i64, ObjectPart>::new();
+        let mut buf = Vec::<u8>::with_capacity(part_size as usize);
+
+        let mut custom_header = HeaderMap::new();
+
+        while part_number <= total_parts_count {
+            buf = match &mut reader {
+                ReaderImpl::Body(content_body) => content_body.to_vec(),
+                ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?,
+            };
+            let length = buf.len();
+
+            let mut md5_base64: String = "".to_string();
+            if opts.send_content_md5 {
+                let mut md5_hasher = self.md5_hasher.lock().unwrap();
+                let hash = md5_hasher.as_mut().expect("err");
+                hash.write(&buf[..length]);
+                md5_base64 = base64_encode(hash.sum().as_bytes());
+            } else {
+                let csum;
+                {
+                    let mut crc = opts.auto_checksum.hasher()?;
+                    crc.reset();
+                    crc.write(&buf[..length]);
+                    csum = crc.sum();
+                }
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
+                    custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().unwrap());
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key());
+                }
+            }
+
+            //let rd = newHook(bytes.NewReader(buf[..length]), opts.progress);
+            let rd = ReaderImpl::Body(Bytes::from(buf));
+
+            let mut p = UploadPartParams {
+                bucket_name: bucket_name.to_string(),
+                object_name: object_name.to_string(),
+                upload_id: upload_id.clone(),
+                reader: rd,
+                part_number,
+                md5_base64,
+                size: length as i64,
+                //sse: opts.server_side_encryption,
+                stream_sha256: !opts.disable_content_sha256,
+                custom_header: custom_header.clone(),
+                sha256_hex: Default::default(),
+                trailer: Default::default(),
+            };
+            let obj_part = self.upload_part(&mut p).await?;
+
+            parts_info.entry(part_number).or_insert(obj_part);
+            total_uploaded_size += length as i64;
+            part_number += 1;
+        }
+
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
+        for i in 1..part_number {
+            let part = parts_info[&i].clone();
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag: part.etag,
+                part_num: part.part_num,
+                checksum_crc32: part.checksum_crc32,
+                checksum_crc32c: part.checksum_crc32c,
+                checksum_sha1: part.checksum_sha1,
+                checksum_sha256: part.checksum_sha256,
+                checksum_crc64nvme: part.checksum_crc64nvme,
+                ..Default::default()
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+
+        let opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum: opts.auto_checksum,
+            ..Default::default()
+        };
+        //apply_auto_checksum(&mut opts, all_parts);
+
+        let mut upload_info = self
+            .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
+            .await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+}

crates/ecstore/src/client/api_put_object_common.rs

@@ -0,0 +1,113 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use crate::client::{
+    api_error_response::{err_entity_too_large, err_invalid_argument},
+    api_put_object::PutObjectOptions,
+    constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PART_SIZE, MAX_PARTS_COUNT, MIN_PART_SIZE},
+    transition_api::ReaderImpl,
+    transition_api::TransitionClient,
+};
+
+pub fn is_object(reader: &ReaderImpl) -> bool {
+    todo!();
+}
+
+pub fn is_read_at(reader: ReaderImpl) -> bool {
+    todo!();
+}
+
+pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
+    let unknown_size;
+    let mut object_size = object_size;
+    if object_size == -1 {
+        unknown_size = true;
+        object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
+    } else {
+        unknown_size = false;
+    }
+
+    if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+        return Err(std::io::Error::other(err_entity_too_large(
+            object_size,
+            MAX_MULTIPART_PUT_OBJECT_SIZE,
+            "",
+            "",
+        )));
+    }
+
+    let mut part_size_flt: f64;
+    if configured_part_size > 0 {
+        if configured_part_size as i64 > object_size {
+            return Err(std::io::Error::other(err_entity_too_large(
+                configured_part_size as i64,
+                object_size,
+                "",
+                "",
+            )));
+        }
+
+        if !unknown_size && object_size > (configured_part_size as i64 * MAX_PARTS_COUNT) {
+            return Err(std::io::Error::other(err_invalid_argument(
+                "Part size * max_parts(10000) is lesser than input objectSize.",
+            )));
+        }
+
+        if (configured_part_size as i64) < ABS_MIN_PART_SIZE {
+            return Err(std::io::Error::other(err_invalid_argument(
+                "Input part size is smaller than allowed minimum of 5MiB.",
+            )));
+        }
+
+        if configured_part_size as i64 > MAX_PART_SIZE {
+            return Err(std::io::Error::other(err_invalid_argument(
+                "Input part size is bigger than allowed maximum of 5GiB.",
+            )));
+        }
+
+        part_size_flt = configured_part_size as f64;
+        if unknown_size {
+            object_size = configured_part_size as i64 * MAX_PARTS_COUNT;
+        }
+    } else {
+        let mut configured_part_size = configured_part_size;
+        configured_part_size = MIN_PART_SIZE as u64;
+        part_size_flt = (object_size / MAX_PARTS_COUNT) as f64;
+        part_size_flt = (part_size_flt / configured_part_size as f64) * configured_part_size as f64;
+    }
+
+    let total_parts_count = (object_size as f64 / part_size_flt).ceil() as i64;
+    let part_size = part_size_flt.ceil() as i64;
+    let last_part_size = object_size - (total_parts_count - 1) * part_size;
+    Ok((total_parts_count, part_size, last_part_size))
+}
+
+impl TransitionClient {
+    pub async fn new_upload_id(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        opts: &PutObjectOptions,
+    ) -> Result<String, std::io::Error> {
+        let init_multipart_upload_result = self.initiate_multipart_upload(bucket_name, object_name, opts).await?;
+        Ok(init_multipart_upload_result.upload_id)
+    }
+}

crates/ecstore/src/client/api_put_object_multipart.rs

@@ -0,0 +1,433 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use bytes::Bytes;
+use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
+use s3s::S3ErrorCode;
+use std::io::Read;
+use std::{collections::HashMap, sync::Arc};
+use time::{OffsetDateTime, format_description};
+use tokio_util::sync::CancellationToken;
+use tracing::warn;
+use tracing::{error, info};
+use url::form_urlencoded::Serializer;
+use uuid::Uuid;
+
+use rustfs_utils::hasher::Hasher;
+use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
+use s3s::{Body, dto::StreamingBlob};
+//use crate::disk::{Reader, BufferReader};
+use crate::client::{
+    api_error_response::{
+        err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response,
+    },
+    api_put_object::PutObjectOptions,
+    api_put_object_common::optimal_part_info,
+    api_s3_datatypes::{
+        CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart,
+    },
+    constants::{ABS_MIN_PART_SIZE, ISO8601_DATEFORMAT, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE},
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
+};
+use crate::{
+    checksum::ChecksumMode,
+    disk::DiskAPI,
+    store_api::{GetObjectReader, StorageAPI},
+};
+use rustfs_utils::{crypto::base64_encode, path::trim_etag};
+
+impl TransitionClient {
+    pub async fn put_object_multipart(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let info = self
+            .put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts)
+            .await;
+        if let Err(err) = &info {
+            let err_resp = to_error_response(err);
+            if err_resp.code == S3ErrorCode::AccessDenied && err_resp.message.contains("Access Denied") {
+                if size > MAX_SINGLE_PUT_OBJECT_SIZE {
+                    return Err(std::io::Error::other(err_entity_too_large(
+                        size,
+                        MAX_SINGLE_PUT_OBJECT_SIZE,
+                        bucket_name,
+                        object_name,
+                    )));
+                }
+                return self.put_object_gcs(bucket_name, object_name, reader, size, opts).await;
+            }
+        }
+        Ok(info?)
+    }
+
+    pub async fn put_object_multipart_no_stream(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        reader: &mut ReaderImpl,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut total_uploaded_size: i64 = 0;
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let ret = optimal_part_info(-1, opts.part_size)?;
+        let (total_parts_count, part_size, _) = ret;
+
+        let (mut hash_algos, mut hash_sums) = self.hash_materials(opts.send_content_md5, !opts.disable_content_sha256);
+        let upload_id = self.new_upload_id(bucket_name, object_name, opts).await?;
+        let mut opts = opts.clone();
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut part_number = 1;
+        let mut parts_info = HashMap::<i64, ObjectPart>::new();
+        let mut buf = Vec::<u8>::with_capacity(part_size as usize);
+        let mut custom_header = HeaderMap::new();
+        while part_number <= total_parts_count {
+            match reader {
+                ReaderImpl::Body(content_body) => {
+                    buf = content_body.to_vec();
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    buf = content_body.read_all().await?;
+                }
+            }
+            let length = buf.len();
+
+            for (k, v) in hash_algos.iter_mut() {
+                v.write(&buf[..length]);
+                hash_sums.insert(k.to_string(), Vec::try_from(v.sum().as_bytes()).unwrap());
+            }
+
+            //let rd = newHook(bytes.NewReader(buf[..length]), opts.progress);
+            let rd = Bytes::from(buf.clone());
+
+            let md5_base64: String;
+            let sha256_hex: String;
+
+            //if hash_sums["md5"] != nil {
+            md5_base64 = base64_encode(&hash_sums["md5"]);
+            //}
+            //if hash_sums["sha256"] != nil {
+            sha256_hex = hex_simd::encode_to_string(hash_sums["sha256"].clone(), hex_simd::AsciiCase::Lower);
+            //}
+            if hash_sums.len() == 0 {
+                let csum;
+                {
+                    let mut crc = opts.auto_checksum.hasher()?;
+                    crc.reset();
+                    crc.write(&buf[..length]);
+                    csum = crc.sum();
+                }
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
+                    custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().expect("err"));
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key());
+                }
+            }
+
+            let mut p = UploadPartParams {
+                bucket_name: bucket_name.to_string(),
+                object_name: object_name.to_string(),
+                upload_id: upload_id.clone(),
+                reader: ReaderImpl::Body(rd),
+                part_number,
+                md5_base64,
+                sha256_hex,
+                size: length as i64,
+                //sse: opts.server_side_encryption,
+                stream_sha256: !opts.disable_content_sha256,
+                custom_header: custom_header.clone(),
+                trailer: HeaderMap::new(),
+            };
+            let obj_part = self.upload_part(&mut p).await?;
+
+            parts_info.insert(part_number, obj_part);
+            total_uploaded_size += length as i64;
+            part_number += 1;
+        }
+
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
+        for i in 1..part_number {
+            let part = parts_info[&i].clone();
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag: part.etag,
+                part_num: part.part_num,
+                checksum_crc32: part.checksum_crc32,
+                checksum_crc32c: part.checksum_crc32c,
+                checksum_sha1: part.checksum_sha1,
+                checksum_sha256: part.checksum_sha256,
+                checksum_crc64nvme: part.checksum_crc64nvme,
+                ..Default::default()
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+        let opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum: opts.auto_checksum,
+            ..Default::default()
+        };
+        //apply_auto_checksum(&mut opts, all_parts);
+
+        let mut upload_info = self
+            .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
+            .await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+
+    pub async fn initiate_multipart_upload(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        opts: &PutObjectOptions,
+    ) -> Result<InitiateMultipartUploadResult, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("uploads".to_string(), "".to_string());
+
+        if opts.internal.source_version_id != "" {
+            if !opts.internal.source_version_id.is_empty() {
+                if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) {
+                    return Err(std::io::Error::other(err_invalid_argument(&err.to_string())));
+                }
+            }
+            url_values.insert("versionId".to_string(), opts.internal.source_version_id.clone());
+        }
+
+        let custom_header = opts.header();
+
+        let mut req_metadata = RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            object_name: object_name.to_string(),
+            query_values: url_values,
+            custom_header,
+            content_body: ReaderImpl::Body(Bytes::new()),
+            content_length: 0,
+            content_md5_base64: "".to_string(),
+            content_sha256_hex: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
+        //if resp.is_none() {
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
+        }
+        //}
+        let initiate_multipart_upload_result = InitiateMultipartUploadResult::default();
+        Ok(initiate_multipart_upload_result)
+    }
+
+    pub async fn upload_part(&self, p: &mut UploadPartParams) -> Result<ObjectPart, std::io::Error> {
+        if p.size > MAX_PART_SIZE {
+            return Err(std::io::Error::other(err_entity_too_large(
+                p.size,
+                MAX_PART_SIZE,
+                &p.bucket_name,
+                &p.object_name,
+            )));
+        }
+        if p.size <= -1 {
+            return Err(std::io::Error::other(err_entity_too_small(p.size, &p.bucket_name, &p.object_name)));
+        }
+        if p.part_number <= 0 {
+            return Err(std::io::Error::other(err_invalid_argument(
+                "Part number cannot be negative or equal to zero.",
+            )));
+        }
+        if p.upload_id == "" {
+            return Err(std::io::Error::other(err_invalid_argument("UploadID cannot be empty.")));
+        }
+
+        let mut url_values = HashMap::new();
+        url_values.insert("partNumber".to_string(), p.part_number.to_string());
+        url_values.insert("uploadId".to_string(), p.upload_id.clone());
+
+        let buf = match &mut p.reader {
+            ReaderImpl::Body(content_body) => content_body.to_vec(),
+            ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?,
+        };
+        let mut req_metadata = RequestMetadata {
+            bucket_name: p.bucket_name.clone(),
+            object_name: p.object_name.clone(),
+            query_values: url_values,
+            custom_header: p.custom_header.clone(),
+            content_body: ReaderImpl::Body(Bytes::from(buf)),
+            content_length: p.size,
+            content_md5_base64: p.md5_base64.clone(),
+            content_sha256_hex: p.sha256_hex.clone(),
+            stream_sha256: p.stream_sha256,
+            trailer: p.trailer.clone(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
+        //defer closeResponse(resp)
+        //if resp.is_none() {
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                resp,
+                vec![],
+                &p.bucket_name.clone(),
+                &p.object_name,
+            )));
+        }
+        //}
+        let h = resp.headers();
+        let mut obj_part = ObjectPart {
+            checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
+                h_checksum_crc32.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
+                h_checksum_crc32c.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
+                h_checksum_sha1.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
+                h_checksum_sha256.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
+                h_checksum_crc64nvme.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            ..Default::default()
+        };
+        obj_part.size = p.size;
+        obj_part.part_num = p.part_number;
+        obj_part.etag = if let Some(h_etag) = h.get("ETag") {
+            h_etag.to_str().expect("err").trim_matches('"').to_string()
+        } else {
+            "".to_string()
+        };
+        Ok(obj_part)
+    }
+
+    pub async fn complete_multipart_upload(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        upload_id: &str,
+        complete: CompleteMultipartUpload,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("uploadId".to_string(), upload_id.to_string());
+        let complete_multipart_upload_bytes = complete.marshal_msg()?.as_bytes().to_vec();
+
+        let headers = opts.header();
+
+        let complete_multipart_upload_buffer = Bytes::from(complete_multipart_upload_bytes);
+        let mut req_metadata = RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            object_name: object_name.to_string(),
+            query_values: url_values,
+            content_body: ReaderImpl::Body(complete_multipart_upload_buffer),
+            content_length: 100,                //complete_multipart_upload_bytes.len(),
+            content_sha256_hex: "".to_string(), //hex_simd::encode_to_string(complete_multipart_upload_bytes, hex_simd::AsciiCase::Lower),
+            custom_header: headers,
+            stream_sha256: Default::default(),
+            trailer: Default::default(),
+            content_md5_base64: "".to_string(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
+
+        let b = resp.body().bytes().expect("err").to_vec();
+        let complete_multipart_upload_result: CompleteMultipartUploadResult = CompleteMultipartUploadResult::default();
+
+        let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
+            (
+                OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
+                "".to_string(),
+            )
+        } else {
+            (OffsetDateTime::now_utc(), "".to_string())
+        };
+
+        let h = resp.headers();
+        Ok(UploadInfo {
+            bucket: complete_multipart_upload_result.bucket,
+            key: complete_multipart_upload_result.key,
+            etag: trim_etag(&complete_multipart_upload_result.etag),
+            version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
+                h_x_amz_version_id.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            location: complete_multipart_upload_result.location,
+            expiration: exp_time,
+            expiration_rule_id: rule_id,
+            checksum_sha256: complete_multipart_upload_result.checksum_sha256,
+            checksum_sha1: complete_multipart_upload_result.checksum_sha1,
+            checksum_crc32: complete_multipart_upload_result.checksum_crc32,
+            checksum_crc32c: complete_multipart_upload_result.checksum_crc32c,
+            checksum_crc64nvme: complete_multipart_upload_result.checksum_crc64nvme,
+            ..Default::default()
+        })
+    }
+}
+
+pub struct UploadPartParams {
+    pub bucket_name: String,
+    pub object_name: String,
+    pub upload_id: String,
+    pub reader: ReaderImpl,
+    pub part_number: i64,
+    pub md5_base64: String,
+    pub sha256_hex: String,
+    pub size: i64,
+    //pub sse: encrypt.ServerSide,
+    pub stream_sha256: bool,
+    pub custom_header: HeaderMap,
+    pub trailer: HeaderMap,
+}

crates/ecstore/src/client/api_put_object_streaming.rs

@@ -0,0 +1,544 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use bytes::Bytes;
+use futures::future::join_all;
+use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
+use std::sync::RwLock;
+use std::{collections::HashMap, sync::Arc};
+use time::{OffsetDateTime, format_description};
+use tokio::{select, sync::mpsc};
+use tokio_util::sync::CancellationToken;
+use tracing::warn;
+use uuid::Uuid;
+
+use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
+use crate::client::{
+    api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response},
+    api_put_object::PutObjectOptions,
+    api_put_object_common::{is_object, optimal_part_info},
+    api_put_object_multipart::UploadPartParams,
+    api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
+    constants::ISO8601_DATEFORMAT,
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
+};
+use rustfs_utils::hasher::Hasher;
+use rustfs_utils::{crypto::base64_encode, path::trim_etag};
+use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
+
+pub struct UploadedPartRes {
+    pub error: std::io::Error,
+    pub part_num: i64,
+    pub size: i64,
+    pub part: ObjectPart,
+}
+
+pub struct UploadPartReq {
+    pub part_num: i64,
+    pub part: ObjectPart,
+}
+
+impl TransitionClient {
+    pub async fn put_object_multipart_stream(
+        self: Arc<Self>,
+        bucket_name: &str,
+        object_name: &str,
+        reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let info: UploadInfo;
+        if opts.concurrent_stream_parts && opts.num_threads > 1 {
+            info = self
+                .put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts)
+                .await?;
+        } else if !is_object(&reader) && !opts.send_content_md5 {
+            info = self
+                .put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts)
+                .await?;
+        } else {
+            info = self
+                .put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts)
+                .await?;
+        }
+
+        Ok(info)
+    }
+
+    pub async fn put_object_multipart_stream_from_readat(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let ret = optimal_part_info(size, opts.part_size)?;
+        let (total_parts_count, part_size, lastpart_size) = ret;
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.auto_checksum = opts.checksum.clone();
+        }
+        let with_checksum = self.trailing_header_support;
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        todo!();
+    }
+
+    pub async fn put_object_multipart_stream_optional_checksum(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.auto_checksum = opts.checksum.clone();
+            opts.send_content_md5 = false;
+        }
+
+        if !opts.send_content_md5 {
+            add_auto_checksum_headers(&mut opts);
+        }
+
+        let ret = optimal_part_info(size, opts.part_size)?;
+        let (total_parts_count, mut part_size, lastpart_size) = ret;
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut custom_header = opts.header().clone();
+
+        let mut total_uploaded_size: i64 = 0;
+
+        let mut parts_info = HashMap::<i64, ObjectPart>::new();
+        let mut buf = Vec::<u8>::with_capacity(part_size as usize);
+
+        let mut md5_base64: String = "".to_string();
+        for part_number in 1..=total_parts_count {
+            if part_number == total_parts_count {
+                part_size = lastpart_size;
+            }
+
+            match &mut reader {
+                ReaderImpl::Body(content_body) => {
+                    buf = content_body.to_vec();
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    buf = content_body.read_all().await?;
+                }
+            }
+            let length = buf.len();
+
+            if opts.send_content_md5 {
+                let mut md5_hasher = self.md5_hasher.lock().unwrap();
+                let md5_hash = md5_hasher.as_mut().expect("err");
+                md5_hash.reset();
+                md5_hash.write(&buf[..length]);
+                md5_base64 = base64_encode(md5_hash.sum().as_bytes());
+            } else {
+                let csum;
+                {
+                    let mut crc = opts.auto_checksum.hasher()?;
+                    crc.reset();
+                    crc.write(&buf[..length]);
+                    csum = crc.sum();
+                }
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key_capitalized().as_bytes()) {
+                    custom_header.insert(header_name, HeaderValue::from_str(&base64_encode(csum.as_bytes())).expect("err"));
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key_capitalized());
+                }
+            }
+
+            let hooked = ReaderImpl::Body(Bytes::from(buf)); //newHook(BufferReader::new(buf), opts.progress);
+            let mut p = UploadPartParams {
+                bucket_name: bucket_name.to_string(),
+                object_name: object_name.to_string(),
+                upload_id: upload_id.clone(),
+                reader: hooked,
+                part_number,
+                md5_base64: md5_base64.clone(),
+                size: part_size,
+                //sse: opts.server_side_encryption,
+                stream_sha256: !opts.disable_content_sha256,
+                custom_header: custom_header.clone(),
+                sha256_hex: "".to_string(),
+                trailer: HeaderMap::new(),
+            };
+            let obj_part = self.upload_part(&mut p).await?;
+
+            parts_info.entry(part_number).or_insert(obj_part);
+
+            total_uploaded_size += part_size as i64;
+        }
+
+        if size > 0 && total_uploaded_size != size {
+            return Err(std::io::Error::other(err_unexpected_eof(
+                total_uploaded_size,
+                size,
+                bucket_name,
+                object_name,
+            )));
+        }
+
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
+        let part_number = total_parts_count;
+        for i in 1..part_number {
+            let part = parts_info[&i].clone();
+
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag: part.etag,
+                part_num: part.part_num,
+                checksum_crc32: part.checksum_crc32,
+                checksum_crc32c: part.checksum_crc32c,
+                checksum_sha1: part.checksum_sha1,
+                checksum_sha256: part.checksum_sha256,
+                checksum_crc64nvme: part.checksum_crc64nvme,
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+
+        let mut opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum: opts.auto_checksum,
+            ..Default::default()
+        };
+        apply_auto_checksum(&mut opts, &mut all_parts);
+        let mut upload_info = self
+            .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
+            .await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+
+    pub async fn put_object_multipart_stream_parallel(
+        self: Arc<Self>,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl, /*GetObjectReader*/
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.send_content_md5 = false;
+            opts.auto_checksum = opts.checksum.clone();
+        }
+        if !opts.send_content_md5 {
+            add_auto_checksum_headers(&mut opts);
+        }
+
+        let ret = optimal_part_info(-1, opts.part_size)?;
+        let (total_parts_count, part_size, _) = ret;
+
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut total_uploaded_size: i64 = 0;
+        let parts_info = Arc::new(RwLock::new(HashMap::<i64, ObjectPart>::new()));
+
+        let n_buffers = opts.num_threads;
+        let (bufs_tx, mut bufs_rx) = mpsc::channel(n_buffers as usize);
+        //let all = Vec::<u8>::with_capacity(n_buffers as usize * part_size as usize);
+        for i in 0..n_buffers {
+            //bufs_tx.send(&all[i * part_size..i * part_size + part_size]);
+            bufs_tx.send(Vec::<u8>::with_capacity(part_size as usize));
+        }
+
+        let mut futures = Vec::with_capacity(total_parts_count as usize);
+        let (err_tx, mut err_rx) = mpsc::channel(opts.num_threads as usize);
+        let cancel_token = CancellationToken::new();
+
+        //reader = newHook(reader, opts.progress);
+
+        for part_number in 1..=total_parts_count {
+            let mut buf = Vec::<u8>::new();
+            select! {
+                buf = bufs_rx.recv() => {}
+                err = err_rx.recv() => {
+                    //cancel_token.cancel();
+                    //wg.Wait()
+                    return Err(err.expect("err"));
+                }
+                else => (),
+            }
+
+            if buf.len() != part_size as usize {
+                return Err(std::io::Error::other(format!(
+                    "read buffer < {} than expected partSize: {}",
+                    buf.len(),
+                    part_size
+                )));
+            }
+
+            match &mut reader {
+                ReaderImpl::Body(content_body) => {
+                    buf = content_body.to_vec();
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    buf = content_body.read_all().await?;
+                }
+            }
+            let length = buf.len();
+
+            let mut custom_header = HeaderMap::new();
+            if !opts.send_content_md5 {
+                let csum;
+                {
+                    let mut crc = opts.auto_checksum.hasher()?;
+                    crc.reset();
+                    crc.write(&buf[..length]);
+                    csum = crc.sum();
+                }
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
+                    if let Ok(header_value) = HeaderValue::from_str(&base64_encode(csum.as_bytes())) {
+                        custom_header.insert(header_name, header_value);
+                    }
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key());
+                }
+            }
+
+            let clone_bufs_tx = bufs_tx.clone();
+            let clone_parts_info = parts_info.clone();
+            let clone_upload_id = upload_id.clone();
+            let clone_self = self.clone();
+            futures.push(async move {
+                let mut md5_base64: String = "".to_string();
+
+                if opts.send_content_md5 {
+                    let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
+                    let md5_hash = md5_hasher.as_mut().expect("err");
+                    md5_hash.write(&buf[..length]);
+                    md5_base64 = base64_encode(md5_hash.sum().as_bytes());
+                }
+
+                //defer wg.Done()
+                let mut p = UploadPartParams {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: object_name.to_string(),
+                    upload_id: clone_upload_id,
+                    reader: ReaderImpl::Body(Bytes::from(buf.clone())),
+                    part_number,
+                    md5_base64,
+                    size: length as i64,
+                    //sse:           opts.server_side_encryption,
+                    stream_sha256: !opts.disable_content_sha256,
+                    custom_header,
+                    sha256_hex: "".to_string(),
+                    trailer: HeaderMap::new(),
+                };
+                let obj_part = clone_self.upload_part(&mut p).await.expect("err");
+
+                let mut clone_parts_info = clone_parts_info.write().unwrap();
+                clone_parts_info.entry(part_number).or_insert(obj_part);
+
+                clone_bufs_tx.send(buf);
+            });
+
+            total_uploaded_size += length as i64;
+        }
+
+        let results = join_all(futures).await;
+
+        select! {
+            err = err_rx.recv() => {
+                return Err(err.expect("err"));
+            }
+            else => (),
+        }
+
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let part_number: i64 = total_parts_count;
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.read().unwrap().len());
+        for i in 1..part_number {
+            let part = parts_info.read().unwrap()[&i].clone();
+
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag: part.etag,
+                part_num: part.part_num,
+                checksum_crc32: part.checksum_crc32,
+                checksum_crc32c: part.checksum_crc32c,
+                checksum_sha1: part.checksum_sha1,
+                checksum_sha256: part.checksum_sha256,
+                checksum_crc64nvme: part.checksum_crc64nvme,
+                ..Default::default()
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+
+        let mut opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum: opts.auto_checksum,
+            ..Default::default()
+        };
+        apply_auto_checksum(&mut opts, &mut all_parts);
+
+        let mut upload_info = self
+            .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
+            .await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+
+    pub async fn put_object_gcs(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.send_content_md5 = false;
+        }
+
+        let md5_base64: String = "".to_string();
+        let progress_reader = reader; //newHook(reader, opts.progress);
+
+        self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts)
+            .await
+    }
+
+    pub async fn put_object_do(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        reader: ReaderImpl,
+        md5_base64: &str,
+        sha256_hex: &str,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let custom_header = opts.header();
+
+        let mut req_metadata = RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            object_name: object_name.to_string(),
+            custom_header,
+            content_body: reader,
+            content_length: size,
+            content_md5_base64: md5_base64.to_string(),
+            content_sha256_hex: sha256_hex.to_string(),
+            stream_sha256: !opts.disable_content_sha256,
+            add_crc: Default::default(),
+            bucket_location: Default::default(),
+            pre_sign_url: Default::default(),
+            query_values: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            expires: Default::default(),
+            trailer: Default::default(),
+        };
+        let mut add_crc = false; //self.trailing_header_support && md5_base64 == "" && !s3utils.IsGoogleEndpoint(self.endpoint_url) && (opts.disable_content_sha256 || self.secure);
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            req_metadata.add_crc = opts.checksum;
+        } else if add_crc {
+            for (k, _) in opts.user_metadata {
+                if k.to_lowercase().starts_with("x-amz-checksum-") {
+                    add_crc = false;
+                }
+            }
+            if add_crc {
+                opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
+                req_metadata.add_crc = opts.auto_checksum;
+            }
+        }
+
+        if opts.internal.source_version_id != "" {
+            if !opts.internal.source_version_id.is_empty() {
+                if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) {
+                    return Err(std::io::Error::other(err_invalid_argument(&err.to_string())));
+                }
+            }
+            let mut url_values = HashMap::new();
+            url_values.insert("versionId".to_string(), opts.internal.source_version_id);
+            req_metadata.query_values = url_values;
+        }
+
+        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
+
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
+        }
+
+        let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
+            (
+                OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
+                "".to_string(),
+            )
+        } else {
+            (OffsetDateTime::now_utc(), "".to_string())
+        };
+        let h = resp.headers();
+        Ok(UploadInfo {
+            bucket: bucket_name.to_string(),
+            key: object_name.to_string(),
+            etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")),
+            version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
+                h_x_amz_version_id.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            size,
+            expiration: exp_time,
+            expiration_rule_id: rule_id,
+            checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
+                h_checksum_crc32.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
+                h_checksum_crc32c.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
+                h_checksum_sha1.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
+                h_checksum_sha256.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
+                h_checksum_crc64nvme.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            ..Default::default()
+        })
+    }
+}

crates/ecstore/src/client/api_remove.rs

@@ -0,0 +1,492 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use bytes::Bytes;
+use http::{HeaderMap, HeaderValue, Method, StatusCode};
+use s3s::S3ErrorCode;
+use s3s::dto::ReplicationStatus;
+use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
+use std::fmt::Display;
+use std::{collections::HashMap, sync::Arc};
+use time::OffsetDateTime;
+use tokio::sync::mpsc::{self, Receiver, Sender};
+
+use crate::client::{
+    api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
+};
+use crate::{
+    disk::DiskAPI,
+    store_api::{GetObjectReader, ObjectInfo, StorageAPI},
+};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+use rustfs_utils::hasher::{sum_md5_base64, sum_sha256_hex};
+
+pub struct RemoveBucketOptions {
+    _forced_elete: bool,
+}
+
+#[derive(Debug)]
+#[allow(dead_code)]
+pub struct AdvancedRemoveOptions {
+    replication_delete_marker: bool,
+    replication_status: ReplicationStatus,
+    replication_mtime: OffsetDateTime,
+    replication_request: bool,
+    replication_validity_check: bool,
+}
+
+impl Default for AdvancedRemoveOptions {
+    fn default() -> Self {
+        Self {
+            replication_delete_marker: false,
+            replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
+            replication_mtime: OffsetDateTime::now_utc(),
+            replication_request: false,
+            replication_validity_check: false,
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct RemoveObjectOptions {
+    pub force_delete: bool,
+    pub governance_bypass: bool,
+    pub version_id: String,
+    pub internal: AdvancedRemoveOptions,
+}
+
+impl TransitionClient {
+    pub async fn remove_bucket_with_options(&self, bucket_name: &str, opts: &RemoveBucketOptions) -> Result<(), std::io::Error> {
+        let headers = HeaderMap::new();
+
+        let resp = self
+            .execute_method(
+                Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: headers,
+                    object_name: "".to_string(),
+                    query_values: Default::default(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        {
+            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
+            bucket_loc_cache.delete(bucket_name);
+        }
+        Ok(())
+    }
+
+    pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> {
+        let resp = self
+            .execute_method(
+                http::Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: Default::default(),
+                    object_name: "".to_string(),
+                    query_values: Default::default(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        {
+            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
+            bucket_loc_cache.delete(bucket_name);
+        }
+
+        Ok(())
+    }
+
+    pub async fn remove_object(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Option<std::io::Error> {
+        let res = self.remove_object_inner(bucket_name, object_name, opts).await.expect("err");
+        res.err
+    }
+
+    pub async fn remove_object_inner(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        opts: RemoveObjectOptions,
+    ) -> Result<RemoveObjectResult, std::io::Error> {
+        let mut url_values = HashMap::new();
+
+        if opts.version_id != "" {
+            url_values.insert("versionId".to_string(), opts.version_id.clone());
+        }
+
+        let mut headers = HeaderMap::new();
+
+        if opts.governance_bypass {
+            headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); //amzBypassGovernance
+        }
+
+        let resp = self
+            .execute_method(
+                http::Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: object_name.to_string(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    query_values: url_values,
+                    custom_header: headers,
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        Ok(RemoveObjectResult {
+            object_name: object_name.to_string(),
+            object_version_id: opts.version_id,
+            delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true",
+            delete_marker_version_id: resp
+                .headers()
+                .get("x-amz-version-id")
+                .expect("err")
+                .to_str()
+                .expect("err")
+                .to_string(),
+            ..Default::default()
+        })
+    }
+
+    pub async fn remove_objects_with_result(
+        self: Arc<Self>,
+        bucket_name: &str,
+        objects_rx: Receiver<ObjectInfo>,
+        opts: RemoveObjectsOptions,
+    ) -> Receiver<RemoveObjectResult> {
+        let (result_tx, result_rx) = mpsc::channel(1);
+
+        let self_clone = Arc::clone(&self);
+        let bucket_name_owned = bucket_name.to_string();
+
+        tokio::spawn(async move {
+            self_clone
+                .remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts)
+                .await;
+        });
+        result_rx
+    }
+
+    pub async fn remove_objects(
+        self: Arc<Self>,
+        bucket_name: &str,
+        objects_rx: Receiver<ObjectInfo>,
+        opts: RemoveObjectsOptions,
+    ) -> Receiver<RemoveObjectError> {
+        let (error_tx, error_rx) = mpsc::channel(1);
+
+        let self_clone = Arc::clone(&self);
+        let bucket_name_owned = bucket_name.to_string();
+
+        let (result_tx, mut result_rx) = mpsc::channel(1);
+        tokio::spawn(async move {
+            self_clone
+                .remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts)
+                .await;
+        });
+        tokio::spawn(async move {
+            while let Some(res) = result_rx.recv().await {
+                if res.err.is_none() {
+                    continue;
+                }
+                error_tx
+                    .send(RemoveObjectError {
+                        object_name: res.object_name,
+                        version_id: res.object_version_id,
+                        err: res.err,
+                        ..Default::default()
+                    })
+                    .await;
+            }
+        });
+
+        error_rx
+    }
+
+    pub async fn remove_objects_inner(
+        &self,
+        bucket_name: &str,
+        mut objects_rx: Receiver<ObjectInfo>,
+        result_tx: &Sender<RemoveObjectResult>,
+        opts: RemoveObjectsOptions,
+    ) -> Result<(), std::io::Error> {
+        let max_entries = 1000;
+        let mut finish = false;
+        let mut url_values = HashMap::new();
+        url_values.insert("delete".to_string(), "".to_string());
+
+        loop {
+            if finish {
+                break;
+            }
+            let mut count = 0;
+            let mut batch = Vec::<ObjectInfo>::new();
+
+            while let Some(object) = objects_rx.recv().await {
+                if has_invalid_xml_char(&object.name) {
+                    let remove_result = self
+                        .remove_object_inner(
+                            bucket_name,
+                            &object.name,
+                            RemoveObjectOptions {
+                                version_id: object.version_id.expect("err").to_string(),
+                                governance_bypass: opts.governance_bypass,
+                                ..Default::default()
+                            },
+                        )
+                        .await?;
+                    let remove_result_clone = remove_result.clone();
+                    if !remove_result.err.is_none() {
+                        match to_error_response(&remove_result.err.expect("err")).code {
+                            S3ErrorCode::InvalidArgument | S3ErrorCode::NoSuchVersion => {
+                                continue;
+                            }
+                            _ => (),
+                        }
+                        result_tx.send(remove_result_clone.clone()).await;
+                    }
+
+                    result_tx.send(remove_result_clone).await;
+                    continue;
+                }
+
+                batch.push(object);
+                count += 1;
+                if count >= max_entries {
+                    break;
+                }
+            }
+            if count == 0 {
+                break;
+            }
+            if count < max_entries {
+                finish = true;
+            }
+
+            let mut headers = HeaderMap::new();
+            if opts.governance_bypass {
+                headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));
+            }
+
+            let remove_bytes = generate_remove_multi_objects_request(&batch);
+            let resp = self
+                .execute_method(
+                    http::Method::POST,
+                    &mut RequestMetadata {
+                        bucket_name: bucket_name.to_string(),
+                        query_values: url_values.clone(),
+                        content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())),
+                        content_length: remove_bytes.len() as i64,
+                        content_md5_base64: sum_md5_base64(&remove_bytes),
+                        content_sha256_hex: sum_sha256_hex(&remove_bytes),
+                        custom_header: headers,
+                        object_name: "".to_string(),
+                        stream_sha256: false,
+                        trailer: HeaderMap::new(),
+                        pre_sign_url: Default::default(),
+                        add_crc: Default::default(),
+                        extra_pre_sign_header: Default::default(),
+                        bucket_location: Default::default(),
+                        expires: Default::default(),
+                    },
+                )
+                .await?;
+
+            let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec();
+            process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_bytes)), result_tx.clone());
+        }
+        Ok(())
+    }
+
+    pub async fn remove_incomplete_upload(&self, bucket_name: &str, object_name: &str) -> Result<(), std::io::Error> {
+        let upload_ids = self.find_upload_ids(bucket_name, object_name)?;
+        for upload_id in upload_ids {
+            self.abort_multipart_upload(bucket_name, object_name, &upload_id).await?;
+        }
+
+        Ok(())
+    }
+
+    pub async fn abort_multipart_upload(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        upload_id: &str,
+    ) -> Result<(), std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("uploadId".to_string(), upload_id.to_string());
+
+        let resp = self
+            .execute_method(
+                http::Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: object_name.to_string(),
+                    query_values: url_values,
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: HeaderMap::new(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+        //if resp.is_some() {
+        if resp.status() != StatusCode::NO_CONTENT {
+            let error_response: ErrorResponse;
+            match resp.status() {
+                StatusCode::NOT_FOUND => {
+                    error_response = ErrorResponse {
+                        code: S3ErrorCode::NoSuchUpload,
+                        message: "The specified multipart upload does not exist.".to_string(),
+                        bucket_name: bucket_name.to_string(),
+                        key: object_name.to_string(),
+                        request_id: resp
+                            .headers()
+                            .get("x-amz-request-id")
+                            .expect("err")
+                            .to_str()
+                            .expect("err")
+                            .to_string(),
+                        host_id: resp
+                            .headers()
+                            .get("x-amz-id-2")
+                            .expect("err")
+                            .to_str()
+                            .expect("err")
+                            .to_string(),
+                        region: resp
+                            .headers()
+                            .get("x-amz-bucket-region")
+                            .expect("err")
+                            .to_str()
+                            .expect("err")
+                            .to_string(),
+                        ..Default::default()
+                    };
+                }
+                _ => {
+                    return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
+                }
+            }
+            return Err(std::io::Error::other(error_response));
+        }
+        //}
+        Ok(())
+    }
+}
+
+#[derive(Debug, Default)]
+#[allow(dead_code)]
+pub struct RemoveObjectError {
+    object_name: String,
+    #[allow(dead_code)]
+    version_id: String,
+    err: Option<std::io::Error>,
+}
+
+impl Display for RemoveObjectError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        if self.err.is_none() {
+            return write!(f, "unexpected remove object error result");
+        }
+        write!(f, "{}", self.err.as_ref().expect("err").to_string())
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct RemoveObjectResult {
+    pub object_name: String,
+    pub object_version_id: String,
+    pub delete_marker: bool,
+    pub delete_marker_version_id: String,
+    pub err: Option<std::io::Error>,
+}
+
+impl Clone for RemoveObjectResult {
+    fn clone(&self) -> Self {
+        Self {
+            object_name: self.object_name.clone(),
+            object_version_id: self.object_version_id.clone(),
+            delete_marker: self.delete_marker,
+            delete_marker_version_id: self.delete_marker_version_id.clone(),
+            err: None, //err
+        }
+    }
+}
+
+pub struct RemoveObjectsOptions {
+    pub governance_bypass: bool,
+}
+
+pub fn generate_remove_multi_objects_request(objects: &[ObjectInfo]) -> Vec<u8> {
+    todo!();
+}
+
+pub fn process_remove_multi_objects_response(body: ReaderImpl, result_tx: Sender<RemoveObjectResult>) {
+    todo!();
+}
+
+fn has_invalid_xml_char(str: &str) -> bool {
+    false
+}

crates/ecstore/src/client/api_s3_datatypes.rs

@@ -0,0 +1,351 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use s3s::dto::Owner;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use time::OffsetDateTime;
+
+use crate::checksum::ChecksumMode;
+use crate::client::transition_api::ObjectMultipartInfo;
+use rustfs_utils::crypto::base64_decode;
+
+use super::transition_api;
+
+pub struct ListAllMyBucketsResult {
+    pub owner: Owner,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct CommonPrefix {
+    pub prefix: String,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct ListBucketV2Result {
+    pub common_prefixes: Vec<CommonPrefix>,
+    pub contents: Vec<transition_api::ObjectInfo>,
+    pub delimiter: String,
+    pub encoding_type: String,
+    pub is_truncated: bool,
+    pub max_keys: i64,
+    pub name: String,
+    pub next_continuation_token: String,
+    pub continuation_token: String,
+    pub prefix: String,
+    pub fetch_owner: String,
+    pub start_after: String,
+}
+
+#[allow(dead_code)]
+pub struct Version {
+    etag: String,
+    is_latest: bool,
+    key: String,
+    last_modified: OffsetDateTime,
+    owner: Owner,
+    size: i64,
+    storage_class: String,
+    version_id: String,
+    user_metadata: HashMap<String, String>,
+    user_tags: HashMap<String, String>,
+    is_delete_marker: bool,
+}
+
+pub struct ListVersionsResult {
+    versions: Vec<Version>,
+    common_prefixes: Vec<CommonPrefix>,
+    name: String,
+    prefix: String,
+    delimiter: String,
+    max_keys: i64,
+    encoding_type: String,
+    is_truncated: bool,
+    key_marker: String,
+    version_id_marker: String,
+    next_key_marker: String,
+    next_version_id_marker: String,
+}
+
+pub struct ListBucketResult {
+    common_prefixes: Vec<CommonPrefix>,
+    contents: Vec<transition_api::ObjectInfo>,
+    delimiter: String,
+    encoding_type: String,
+    is_truncated: bool,
+    marker: String,
+    max_keys: i64,
+    name: String,
+    next_marker: String,
+    prefix: String,
+}
+
+pub struct ListMultipartUploadsResult {
+    bucket: String,
+    key_marker: String,
+    upload_id_marker: String,
+    next_key_marker: String,
+    next_upload_id_marker: String,
+    encoding_type: String,
+    max_uploads: i64,
+    is_truncated: bool,
+    uploads: Vec<ObjectMultipartInfo>,
+    prefix: String,
+    delimiter: String,
+    common_prefixes: Vec<CommonPrefix>,
+}
+
+pub struct Initiator {
+    id: String,
+    display_name: String,
+}
+
+pub struct CopyObjectResult {
+    pub etag: String,
+    pub last_modified: OffsetDateTime,
+}
+
+#[derive(Debug, Clone)]
+pub struct ObjectPart {
+    pub etag: String,
+    pub part_num: i64,
+    pub last_modified: OffsetDateTime,
+    pub size: i64,
+    pub checksum_crc32: String,
+    pub checksum_crc32c: String,
+    pub checksum_sha1: String,
+    pub checksum_sha256: String,
+    pub checksum_crc64nvme: String,
+}
+
+impl Default for ObjectPart {
+    fn default() -> Self {
+        ObjectPart {
+            etag: Default::default(),
+            part_num: 0,
+            last_modified: OffsetDateTime::now_utc(),
+            size: 0,
+            checksum_crc32: Default::default(),
+            checksum_crc32c: Default::default(),
+            checksum_sha1: Default::default(),
+            checksum_sha256: Default::default(),
+            checksum_crc64nvme: Default::default(),
+        }
+    }
+}
+
+impl ObjectPart {
+    fn checksum(&self, t: &ChecksumMode) -> String {
+        match t {
+            ChecksumMode::ChecksumCRC32C => {
+                return self.checksum_crc32c.clone();
+            }
+            ChecksumMode::ChecksumCRC32 => {
+                return self.checksum_crc32.clone();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return self.checksum_sha1.clone();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return self.checksum_sha256.clone();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return self.checksum_crc64nvme.clone();
+            }
+            _ => {
+                return "".to_string();
+            }
+        }
+    }
+
+    pub fn checksum_raw(&self, t: &ChecksumMode) -> Result<Vec<u8>, std::io::Error> {
+        let b = self.checksum(t);
+        if b == "" {
+            return Err(std::io::Error::other("no checksum set"));
+        }
+        let decoded = match base64_decode(b.as_bytes()) {
+            Ok(b) => b,
+            Err(e) => return Err(std::io::Error::other(e)),
+        };
+        if decoded.len() != t.raw_byte_len() as usize {
+            return Err(std::io::Error::other("checksum length mismatch"));
+        }
+        Ok(decoded)
+    }
+}
+
+pub struct ListObjectPartsResult {
+    pub bucket: String,
+    pub key: String,
+    pub upload_id: String,
+    pub initiator: Initiator,
+    pub owner: Owner,
+    pub storage_class: String,
+    pub part_number_marker: i32,
+    pub next_part_number_marker: i32,
+    pub max_parts: i32,
+    pub checksum_algorithm: String,
+    pub checksum_type: String,
+    pub is_truncated: bool,
+    pub object_parts: Vec<ObjectPart>,
+    pub encoding_type: String,
+}
+
+#[derive(Debug, Default)]
+pub struct InitiateMultipartUploadResult {
+    pub bucket: String,
+    pub key: String,
+    pub upload_id: String,
+}
+
+#[derive(Debug, Default)]
+pub struct CompleteMultipartUploadResult {
+    pub location: String,
+    pub bucket: String,
+    pub key: String,
+    pub etag: String,
+    pub checksum_crc32: String,
+    pub checksum_crc32c: String,
+    pub checksum_sha1: String,
+    pub checksum_sha256: String,
+    pub checksum_crc64nvme: String,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
+pub struct CompletePart {
+    //api has
+    pub etag: String,
+    pub part_num: i64,
+    pub checksum_crc32: String,
+    pub checksum_crc32c: String,
+    pub checksum_sha1: String,
+    pub checksum_sha256: String,
+    pub checksum_crc64nvme: String,
+}
+
+impl CompletePart {
+    fn checksum(&self, t: &ChecksumMode) -> String {
+        match t {
+            ChecksumMode::ChecksumCRC32C => {
+                return self.checksum_crc32c.clone();
+            }
+            ChecksumMode::ChecksumCRC32 => {
+                return self.checksum_crc32.clone();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return self.checksum_sha1.clone();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return self.checksum_sha256.clone();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return self.checksum_crc64nvme.clone();
+            }
+            _ => {
+                return "".to_string();
+            }
+        }
+    }
+}
+
+pub struct CopyObjectPartResult {
+    pub etag: String,
+    pub last_modified: OffsetDateTime,
+}
+
+#[derive(Debug, Default, serde::Serialize)]
+pub struct CompleteMultipartUpload {
+    pub parts: Vec<CompletePart>,
+}
+
+impl CompleteMultipartUpload {
+    pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
+        //let buf = serde_json::to_string(self)?;
+        let buf = match serde_xml_rs::to_string(self) {
+            Ok(buf) => buf,
+            Err(e) => {
+                return Err(std::io::Error::other(e));
+            }
+        };
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
+        todo!();
+    }
+}
+
+pub struct CreateBucketConfiguration {
+    pub location: String,
+}
+
+#[derive(serde::Serialize)]
+pub struct DeleteObject {
+    //api has
+    pub key: String,
+    pub version_id: String,
+}
+
+pub struct DeletedObject {
+    //s3s has
+    pub key: String,
+    pub version_id: String,
+    pub deletemarker: bool,
+    pub deletemarker_version_id: String,
+}
+
+pub struct NonDeletedObject {
+    pub key: String,
+    pub code: String,
+    pub message: String,
+    pub version_id: String,
+}
+
+#[derive(serde::Serialize)]
+pub struct DeleteMultiObjects {
+    pub quiet: bool,
+    pub objects: Vec<DeleteObject>,
+}
+
+impl DeleteMultiObjects {
+    pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
+        //let buf = serde_json::to_string(self)?;
+        let buf = match serde_xml_rs::to_string(self) {
+            Ok(buf) => buf,
+            Err(e) => {
+                return Err(std::io::Error::other(e));
+            }
+        };
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
+        todo!();
+    }
+}
+
+pub struct DeleteMultiObjectsResult {
+    pub deleted_objects: Vec<DeletedObject>,
+    pub undeleted_objects: Vec<NonDeletedObject>,
+}

crates/ecstore/src/client/bucket_cache.rs

@@ -0,0 +1,240 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use http::Request;
+use hyper::StatusCode;
+use hyper::body::Incoming;
+use std::{collections::HashMap, sync::Arc};
+use tracing::warn;
+use tracing::{debug, error, info};
+
+use crate::client::{
+    api_error_response::{http_resp_to_error_response, to_error_response},
+    transition_api::{Document, TransitionClient},
+};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+use rustfs_utils::hasher::{Hasher, Sha256};
+use s3s::Body;
+use s3s::S3ErrorCode;
+
+use super::constants::UNSIGNED_PAYLOAD;
+use super::credentials::SignatureType;
+
+pub struct BucketLocationCache {
+    items: HashMap<String, String>,
+}
+
+impl BucketLocationCache {
+    pub fn new() -> BucketLocationCache {
+        BucketLocationCache { items: HashMap::new() }
+    }
+
+    pub fn get(&self, bucket_name: &str) -> Option<String> {
+        self.items.get(bucket_name).map(|s| s.clone())
+    }
+
+    pub fn set(&mut self, bucket_name: &str, location: &str) {
+        self.items.insert(bucket_name.to_string(), location.to_string());
+    }
+
+    pub fn delete(&mut self, bucket_name: &str) {
+        self.items.remove(bucket_name);
+    }
+}
+
+impl TransitionClient {
+    pub async fn get_bucket_location(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        Ok(self.get_bucket_location_inner(bucket_name).await?)
+    }
+
+    async fn get_bucket_location_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        if self.region != "" {
+            return Ok(self.region.clone());
+        }
+
+        let mut location;
+        {
+            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
+            let ret = bucket_loc_cache.get(bucket_name);
+            if let Some(location) = ret {
+                return Ok(location);
+            }
+            //location = ret?;
+        }
+
+        let req = self.get_bucket_location_request(bucket_name)?;
+
+        let mut resp = self.doit(req).await?;
+        location = process_bucket_location_response(resp, bucket_name).await?;
+        {
+            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
+            bucket_loc_cache.set(bucket_name, &location);
+        }
+        Ok(location)
+    }
+
+    fn get_bucket_location_request(&self, bucket_name: &str) -> Result<http::Request<Body>, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("location".to_string(), "".to_string());
+
+        let mut target_url = self.endpoint_url.clone();
+        let scheme = self.endpoint_url.scheme();
+        let h = target_url.host().expect("host is none.");
+        let default_port = if scheme == "https" { 443 } else { 80 };
+        let p = target_url.port().unwrap_or(default_port);
+
+        let is_virtual_style = self.is_virtual_host_style_request(&target_url, bucket_name);
+
+        let mut url_str: String = "".to_string();
+
+        if is_virtual_style {
+            url_str = scheme.to_string();
+            url_str.push_str("://");
+            url_str.push_str(bucket_name);
+            url_str.push_str(".");
+            url_str.push_str(target_url.host_str().expect("err"));
+            url_str.push_str("/?location");
+        } else {
+            let mut path = bucket_name.to_string();
+            path.push_str("/");
+            target_url.set_path(&path);
+            {
+                let mut q = target_url.query_pairs_mut();
+                for (k, v) in url_values {
+                    q.append_pair(&k, &urlencoding::encode(&v));
+                }
+            }
+            url_str = target_url.to_string();
+        }
+
+        let mut req_builder = Request::builder().method(http::Method::GET).uri(url_str);
+
+        self.set_user_agent(&mut req_builder);
+
+        let value;
+        {
+            let mut creds_provider = self.creds_provider.lock().unwrap();
+            value = match creds_provider.get_with_context(Some(self.cred_context())) {
+                Ok(v) => v,
+                Err(err) => {
+                    return Err(std::io::Error::other(err));
+                }
+            };
+        }
+
+        let mut signer_type = value.signer_type.clone();
+        let mut access_key_id = value.access_key_id;
+        let mut secret_access_key = value.secret_access_key;
+        let mut session_token = value.session_token;
+
+        if self.override_signer_type != SignatureType::SignatureDefault {
+            signer_type = self.override_signer_type.clone();
+        }
+
+        if value.signer_type == SignatureType::SignatureAnonymous {
+            signer_type = SignatureType::SignatureAnonymous
+        }
+
+        if signer_type == SignatureType::SignatureAnonymous {
+            let req = match req_builder.body(Body::empty()) {
+                Ok(req) => return Ok(req),
+                Err(err) => {
+                    return Err(std::io::Error::other(err));
+                }
+            };
+        }
+
+        if signer_type == SignatureType::SignatureV2 {
+            let req_builder = rustfs_signer::sign_v2(req_builder, 0, &access_key_id, &secret_access_key, is_virtual_style);
+            let req = match req_builder.body(Body::empty()) {
+                Ok(req) => return Ok(req),
+                Err(err) => {
+                    return Err(std::io::Error::other(err));
+                }
+            };
+        }
+
+        let mut content_sha256 = EMPTY_STRING_SHA256_HASH.to_string();
+        if self.secure {
+            content_sha256 = UNSIGNED_PAYLOAD.to_string();
+        }
+
+        req_builder
+            .headers_mut()
+            .expect("err")
+            .insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
+        let req_builder = rustfs_signer::sign_v4(req_builder, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
+        let req = match req_builder.body(Body::empty()) {
+            Ok(req) => return Ok(req),
+            Err(err) => {
+                return Err(std::io::Error::other(err));
+            }
+        };
+    }
+}
+
+async fn process_bucket_location_response(mut resp: http::Response<Body>, bucket_name: &str) -> Result<String, std::io::Error> {
+    //if resp != nil {
+    if resp.status() != StatusCode::OK {
+        let err_resp = http_resp_to_error_response(resp, vec![], bucket_name, "");
+        match err_resp.code {
+                S3ErrorCode::NotImplemented => {
+                    match err_resp.server.as_str() {
+                        "AmazonSnowball" => {
+                            return Ok("snowball".to_string());
+                        }
+                        "cloudflare" => {
+                            return Ok("us-east-1".to_string());
+                        }
+                        _ => {
+                            return Err(std::io::Error::other(err_resp));
+                        }
+                    }
+                }
+                S3ErrorCode::AuthorizationHeaderMalformed |
+                //S3ErrorCode::InvalidRegion |
+                S3ErrorCode::AccessDenied => {
+                    if err_resp.region == "" {
+                      return Ok("us-east-1".to_string());
+                    }
+                    return Ok(err_resp.region);
+                }
+                _ => {
+                    return Err(std::io::Error::other(err_resp));
+                }
+            }
+    }
+    //}
+
+    let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
+    let Document(location_constraint) = serde_xml_rs::from_str::<Document>(&String::from_utf8(b).unwrap()).unwrap();
+
+    let mut location = location_constraint;
+    if location == "" {
+        location = "us-east-1".to_string();
+    }
+
+    if location == "EU" {
+        location = "eu-west-1".to_string();
+    }
+
+    Ok(location)
+}

crates/ecstore/src/client/constants.rs

@@ -0,0 +1,42 @@
+#![allow(unused_imports)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+
+use lazy_static::lazy_static;
+use std::{collections::HashMap, sync::Arc};
+use time::{format_description::FormatItem, macros::format_description};
+
+pub const ABS_MIN_PART_SIZE: i64 = 1024 * 1024 * 5;
+pub const MAX_PARTS_COUNT: i64 = 10000;
+pub const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+pub const MIN_PART_SIZE: i64 = 1024 * 1024 * 16;
+
+pub const MAX_SINGLE_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+
+pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
+pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
+
+pub const TOTAL_WORKERS: i64 = 4;
+
+pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
+pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
+    format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
+
+pub const GET_OBJECT_ATTRIBUTES_TAGS: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts";
+pub const GET_OBJECT_ATTRIBUTES_MAX_PARTS: i64 = 1000;

crates/ecstore/src/client/credentials.rs

@@ -0,0 +1,163 @@
+#![allow(unused_imports)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use std::fmt::{Display, Formatter};
+
+use time::OffsetDateTime;
+
+#[derive(Debug, Default, Clone, Eq, PartialEq)]
+pub enum SignatureType {
+    #[default]
+    SignatureDefault,
+    SignatureV4,
+    SignatureV2,
+    SignatureV4Streaming,
+    SignatureAnonymous,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct Credentials<P: Provider + Default> {
+    creds: Value,
+    force_refresh: bool,
+    provider: P,
+}
+
+impl<P: Provider + Default> Credentials<P> {
+    pub fn new(provider: P) -> Self {
+        Self {
+            provider,
+            force_refresh: true,
+            ..Default::default()
+        }
+    }
+
+    pub fn get(&mut self) -> Result<Value, std::io::Error> {
+        self.get_with_context(None)
+    }
+
+    pub fn get_with_context(&mut self, mut cc: Option<CredContext>) -> Result<Value, std::io::Error> {
+        if self.is_expired() {
+            let creds = self.provider.retrieve_with_cred_context(cc.expect("err"));
+            self.creds = creds;
+            self.force_refresh = false;
+        }
+
+        Ok(self.creds.clone())
+    }
+
+    fn expire(&mut self) {
+        self.force_refresh = true;
+    }
+
+    pub fn is_expired(&self) -> bool {
+        self.force_refresh || self.provider.is_expired()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Value {
+    pub access_key_id: String,
+    pub secret_access_key: String,
+    pub session_token: String,
+    pub expiration: OffsetDateTime,
+    pub signer_type: SignatureType,
+}
+
+impl Default for Value {
+    fn default() -> Self {
+        Self {
+            access_key_id: "".to_string(),
+            secret_access_key: "".to_string(),
+            session_token: "".to_string(),
+            expiration: OffsetDateTime::now_utc(),
+            signer_type: SignatureType::SignatureDefault,
+        }
+    }
+}
+
+pub struct CredContext {
+    //pub client: SendRequest,
+    pub endpoint: String,
+}
+
+pub trait Provider {
+    fn retrieve(&self) -> Value;
+    fn retrieve_with_cred_context(&self, _: CredContext) -> Value;
+    fn is_expired(&self) -> bool;
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct Static(pub Value);
+
+impl Provider for Static {
+    fn retrieve(&self) -> Value {
+        if self.0.access_key_id == "" || self.0.secret_access_key == "" {
+            return Value {
+                signer_type: SignatureType::SignatureAnonymous,
+                ..Default::default()
+            };
+        }
+        self.0.clone()
+    }
+
+    fn retrieve_with_cred_context(&self, _: CredContext) -> Value {
+        self.retrieve()
+    }
+
+    fn is_expired(&self) -> bool {
+        false
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct STSError {
+    pub r#type: String,
+    pub code: String,
+    pub message: String,
+}
+
+#[derive(Debug, Clone, thiserror::Error)]
+pub struct ErrorResponse {
+    pub sts_error: STSError,
+    pub request_id: String,
+}
+
+impl Display for ErrorResponse {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.error())
+    }
+}
+
+impl ErrorResponse {
+    fn error(&self) -> String {
+        if self.sts_error.message == "" {
+            return format!("Error response code {}.", self.sts_error.code);
+        }
+        return self.sts_error.message.clone();
+    }
+}
+
+pub fn xml_decoder<T>(body: &[u8]) -> Result<T, std::io::Error> {
+    todo!();
+}
+
+pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error> {
+    todo!();
+}

crates/ecstore/src/client/hook_reader.rs

@@ -0,0 +1,59 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{collections::HashMap, sync::Arc};
+
+use crate::{
+    disk::{
+        error::{is_unformatted_disk, DiskError},
+        format::{DistributionAlgoVersion, FormatV3},
+        new_disk, DiskAPI, DiskInfo, DiskOption, DiskStore,
+    },
+    store_api::{
+        BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec,
+        ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartInfo, MultipartUploadResult,
+        ObjectIO, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI,
+    },
+    credentials::{Credentials, SignatureType,},
+    api_put_object_multipart::UploadPartParams,
+};
+
+use http::HeaderMap;
+use tokio_util::sync::CancellationToken;
+use tracing::warn;
+use tracing::{error, info};
+use url::Url;
+
+struct HookReader {
+    source: GetObjectReader,
+    hook:   GetObjectReader,
+}
+
+impl HookReader {
+    pub fn new(source: GetObjectReader, hook: GetObjectReader) -> HookReader {
+        HookReader {
+            source,
+            hook,
+        }
+    }
+
+    fn seek(&self, offset: i64, whence: i64) -> Result<i64> {
+        todo!();
+    }
+
+    fn read(&self, b: &[u8]) -> Result<i64> {
+        todo!();
+    }
+}

crates/ecstore/src/client/mod.rs

@@ -0,0 +1,33 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod admin_handler_utils;
+pub mod api_bucket_policy;
+pub mod api_error_response;
+pub mod api_get_object;
+pub mod api_get_options;
+pub mod api_list;
+pub mod api_put_object;
+pub mod api_put_object_common;
+pub mod api_put_object_multipart;
+pub mod api_put_object_streaming;
+pub mod api_remove;
+pub mod api_s3_datatypes;
+pub mod bucket_cache;
+pub mod constants;
+pub mod credentials;
+pub mod object_api_utils;
+pub mod object_handlers_common;
+pub mod transition_api;
+pub mod utils;

crates/ecstore/src/client/object_api_utils.rs

@@ -0,0 +1,157 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use http::HeaderMap;
+use std::io::Cursor;
+use std::{collections::HashMap, sync::Arc};
+use tokio::io::BufReader;
+
+use crate::error::ErrorResponse;
+use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions};
+use rustfs_filemeta::fileinfo::ObjectPartInfo;
+use rustfs_rio::HashReader;
+use s3s::S3ErrorCode;
+
+//#[derive(Clone)]
+pub struct PutObjReader {
+    pub reader: HashReader,
+    pub raw_reader: HashReader,
+    //pub sealMD5Fn: SealMD5CurrFn,
+}
+
+#[allow(dead_code)]
+impl PutObjReader {
+    pub fn new(raw_reader: HashReader) -> Self {
+        todo!();
+    }
+
+    fn md5_current_hex_string(&self) -> String {
+        todo!();
+    }
+
+    fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
+        self.reader = enc_reader;
+
+        Ok(())
+    }
+}
+
+pub type ObjReaderFn = Arc<dyn Fn(BufReader<Cursor<Vec<u8>>>, HeaderMap) -> GetObjectReader + 'static>;
+
+fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRangeSpec> {
+    if oi.size == 0 || oi.parts.len() == 0 {
+        return None;
+    }
+
+    let mut start: i64 = 0;
+    let mut end: i64 = -1;
+    let mut i = 0;
+    while i < oi.parts.len() && i < part_number {
+        start = end + 1;
+        end = start + oi.parts[i].actual_size as i64 - 1;
+        i += 1;
+    }
+
+    Some(HTTPRangeSpec {
+        start,
+        end,
+        is_suffix_length: false,
+    })
+}
+
+fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) {
+    let mut skip_length: i64 = 0;
+    let mut cumulative_actual_size: i64 = 0;
+    let mut first_part_idx: i64 = 0;
+    let mut compressed_offset: i64 = 0;
+    let mut part_skip: i64 = 0;
+    let mut decrypt_skip: i64 = 0;
+    let mut seq_num: u64 = 0;
+    for (i, part) in oi.parts.iter().enumerate() {
+        cumulative_actual_size += part.actual_size as i64;
+        if cumulative_actual_size <= offset {
+            compressed_offset += part.size as i64;
+        } else {
+            first_part_idx = i as i64;
+            skip_length = cumulative_actual_size - part.actual_size as i64;
+            break;
+        }
+    }
+    skip_length = offset - skip_length;
+
+    let parts: &[ObjectPartInfo] = &oi.parts;
+    if skip_length > 0
+        && parts.len() > first_part_idx as usize
+        && parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0
+    {
+        todo!();
+    }
+
+    (compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
+}
+
+pub fn new_getobjectreader(
+    rs: HTTPRangeSpec,
+    oi: &ObjectInfo,
+    opts: &ObjectOptions,
+    h: &HeaderMap,
+) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> {
+    //let (_, mut is_encrypted) = crypto.is_encrypted(oi.user_defined)?;
+    let mut is_encrypted = false;
+    let is_compressed = false; //oi.is_compressed_ok();
+
+    let mut get_fn: ObjReaderFn;
+
+    let (off, length) = match rs.get_offset_length(oi.size) {
+        Ok(x) => x,
+        Err(err) => {
+            return Err(ErrorResponse {
+                code: S3ErrorCode::InvalidRange,
+                message: err.to_string(),
+                key: None,
+                bucket_name: None,
+                region: None,
+                request_id: None,
+                host_id: "".to_string(),
+            });
+        }
+    };
+    get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| {
+        //Box::pin({
+        /*let r = GetObjectReader {
+            object_info: oi.clone(),
+            stream: StreamingBlob::new(HashReader::new(input_reader, 10, None, None, 10)),
+        };
+        r*/
+        todo!();
+        //})
+    });
+
+    Ok((get_fn, off as i64, length as i64))
+}
+
+pub fn extract_etag(metadata: &HashMap<String, String>) -> String {
+    if let Some(etag) = metadata.get("etag") {
+        etag.clone()
+    } else {
+        metadata["md5Sum"].clone()
+    }
+}

crates/ecstore/src/client/object_handlers_common.rs

@@ -0,0 +1,44 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::StorageAPI;
+use crate::bucket::lifecycle::lifecycle;
+use crate::bucket::versioning::VersioningApi;
+use crate::bucket::versioning_sys::BucketVersioningSys;
+use crate::store::ECStore;
+use crate::store_api::{ObjectOptions, ObjectToDelete};
+use rustfs_lock::local_locker::MAX_DELETE_LIST;
+
+pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
+    let mut remaining = to_del;
+    loop {
+        let mut to_del = remaining;
+        if to_del.len() > MAX_DELETE_LIST {
+            remaining = &to_del[MAX_DELETE_LIST..];
+            to_del = &to_del[..MAX_DELETE_LIST];
+        } else {
+            remaining = &[];
+        }
+        let vc = BucketVersioningSys::get(bucket).await.expect("err!");
+        let _deleted_objs = api.delete_objects(
+            bucket,
+            to_del.to_vec(),
+            ObjectOptions {
+                //prefix_enabled_fn:  vc.prefix_enabled(""),
+                version_suspended: vc.suspended(),
+                ..Default::default()
+            },
+        );
+    }
+}

crates/ecstore/src/client/utils.rs

@@ -0,0 +1,92 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use lazy_static::lazy_static;
+use std::collections::HashMap;
+
+use s3s::header::X_AMZ_STORAGE_CLASS;
+
+lazy_static! {
+    static ref SUPPORTED_QUERY_VALUES: HashMap<String, bool> = {
+        let mut m = HashMap::new();
+        m.insert("attributes".to_string(), true);
+        m.insert("partNumber".to_string(), true);
+        m.insert("versionId".to_string(), true);
+        m.insert("response-cache-control".to_string(), true);
+        m.insert("response-content-disposition".to_string(), true);
+        m.insert("response-content-encoding".to_string(), true);
+        m.insert("response-content-language".to_string(), true);
+        m.insert("response-content-type".to_string(), true);
+        m.insert("response-expires".to_string(), true);
+        m
+    };
+    static ref SUPPORTED_HEADERS: HashMap<String, bool> = {
+        let mut m = HashMap::new();
+        m.insert("content-type".to_string(), true);
+        m.insert("cache-control".to_string(), true);
+        m.insert("content-encoding".to_string(), true);
+        m.insert("content-disposition".to_string(), true);
+        m.insert("content-language".to_string(), true);
+        m.insert("x-amz-website-redirect-location".to_string(), true);
+        m.insert("x-amz-object-lock-mode".to_string(), true);
+        m.insert("x-amz-metadata-directive".to_string(), true);
+        m.insert("x-amz-object-lock-retain-until-date".to_string(), true);
+        m.insert("expires".to_string(), true);
+        m.insert("x-amz-replication-status".to_string(), true);
+        m
+    };
+    static ref SSE_HEADERS: HashMap<String, bool> = {
+        let mut m = HashMap::new();
+        m.insert("x-amz-server-side-encryption".to_string(), true);
+        m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true);
+        m.insert("x-amz-server-side-encryption-context".to_string(), true);
+        m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
+        m.insert("x-amz-server-side-encryption-customer-key".to_string(), true);
+        m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true);
+        m
+    };
+}
+
+pub fn is_standard_query_value(qs_key: &str) -> bool {
+    SUPPORTED_QUERY_VALUES[qs_key]
+}
+
+pub fn is_storageclass_header(header_key: &str) -> bool {
+    header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
+}
+
+pub fn is_standard_header(header_key: &str) -> bool {
+    *SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
+}
+
+pub fn is_sse_header(header_key: &str) -> bool {
+    *SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
+}
+
+pub fn is_amz_header(header_key: &str) -> bool {
+    let key = header_key.to_lowercase();
+    key.starts_with("x-amz-meta-")
+        || key.starts_with("x-amz-grant-")
+        || key == "x-amz-acl"
+        || is_sse_header(header_key)
+        || key.starts_with("x-amz-checksum-")
+}
+
+pub fn is_rustfs_header(header_key: &str) -> bool {
+    header_key.to_lowercase().starts_with("x-rustfs-")
+}
+
+pub fn is_minio_header(header_key: &str) -> bool {
+    header_key.to_lowercase().starts_with("x-minio-")
+}

crates/ecstore/src/cmd/bucket_replication_utils.rs

@@ -0,0 +1,69 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use chrono::{DateTime, Utc};
+
+// Representation of the replication status
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum StatusType {
+    Pending,
+    Completed,
+    CompletedLegacy,
+    Failed,
+    Replica,
+}
+
+// Representation of version purge status type (customize as needed)
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum VersionPurgeStatusType {
+    Pending,
+    Completed,
+    Failed,
+}
+
+// ReplicationState struct definition
+#[derive(Debug, Clone)]
+pub struct ReplicationState {
+    // Timestamp when the last replica update was received
+    pub replica_time_stamp: DateTime<Utc>,
+
+    // Replica status
+    pub replica_status: StatusType,
+
+    // Represents DeleteMarker replication state
+    pub delete_marker: bool,
+
+    // Timestamp when the last replication activity happened
+    pub replication_time_stamp: DateTime<Utc>,
+
+    // Stringified representation of all replication activity
+    pub replication_status_internal: String,
+
+    // Stringified representation of all version purge statuses
+    // Example format: "arn1=PENDING;arn2=COMPLETED;"
+    pub version_purge_status_internal: String,
+
+    // Stringified representation of replication decision for each target
+    pub replicate_decision_str: String,
+
+    // Map of ARN -> replication status for ongoing replication activity
+    pub targets: HashMap<String, StatusType>,
+
+    // Map of ARN -> VersionPurgeStatus for all the targets
+    pub purge_targets: HashMap<String, VersionPurgeStatusType>,
+
+    // Map of ARN -> stringified reset id and timestamp for all the targets
+    pub reset_statuses_map: HashMap<String, String>,
+}

crates/ecstore/src/cmd/bucket_targets.rs

@@ -0,0 +1,881 @@
+#![allow(unused_variables)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(dead_code)]
+use crate::{
+    StorageAPI,
+    bucket::{metadata_sys, target::BucketTarget},
+    endpoints::Node,
+    rpc::{PeerS3Client, RemotePeerS3Client},
+};
+use crate::{
+    bucket::{self, target::BucketTargets},
+    new_object_layer_fn, store_api,
+};
+//use tokio::sync::RwLock;
+use aws_sdk_s3::Client as S3Client;
+use chrono::Utc;
+use lazy_static::lazy_static;
+use std::sync::Arc;
+use std::{
+    collections::HashMap,
+    time::{Duration, SystemTime},
+};
+use thiserror::Error;
+use tokio::sync::RwLock;
+
+pub struct TClient {
+    pub s3cli: S3Client,
+    pub remote_peer_client: RemotePeerS3Client,
+    pub arn: String,
+}
+impl TClient {
+    pub fn new(s3cli: S3Client, remote_peer_client: RemotePeerS3Client, arn: String) -> Self {
+        TClient {
+            s3cli,
+            remote_peer_client,
+            arn,
+        }
+    }
+}
+
+pub struct EpHealth {
+    pub endpoint: String,
+    pub scheme: String,
+    pub online: bool,
+    pub last_online: SystemTime,
+    pub last_hc_at: SystemTime,
+    pub offline_duration: Duration,
+    pub latency: LatencyStat, // Assuming LatencyStat is a custom struct
+}
+
+impl EpHealth {
+    pub fn new(
+        endpoint: String,
+        scheme: String,
+        online: bool,
+        last_online: SystemTime,
+        last_hc_at: SystemTime,
+        offline_duration: Duration,
+        latency: LatencyStat,
+    ) -> Self {
+        EpHealth {
+            endpoint,
+            scheme,
+            online,
+            last_online,
+            last_hc_at,
+            offline_duration,
+            latency,
+        }
+    }
+}
+
+pub struct LatencyStat {
+    // Define the fields of LatencyStat as per your requirements
+}
+
+pub struct ArnTarget {
+    client: TargetClient,
+    last_refresh: chrono::DateTime<Utc>,
+}
+impl ArnTarget {
+    pub fn new(bucket: String, endpoint: String, ak: String, sk: String) -> Self {
+        Self {
+            client: TargetClient {
+                bucket,
+                storage_class: "STANDRD".to_string(),
+                disable_proxy: false,
+                health_check_duration: Duration::from_secs(100),
+                endpoint,
+                reset_id: "0".to_string(),
+                replicate_sync: false,
+                secure: false,
+                arn: "".to_string(),
+                client: reqwest::Client::new(),
+                ak,
+                sk,
+            },
+            last_refresh: Utc::now(),
+        }
+    }
+}
+
+// pub fn get_s3client_from_para(
+//     ak: &str,
+//     sk: &str,
+//     url: &str,
+//     _region: &str,
+// ) -> Result<S3Client, Box<dyn Error>> {
+//     let credentials = Credentials::new(ak, sk, None, None, "");
+//     let region = Region::new("us-east-1".to_string());
+
+//     let config = Config::builder()
+//         .region(region)
+//         .endpoint_url(url.to_string())
+//         .credentials_provider(credentials)
+//         .behavior_version(BehaviorVersion::latest()) // Adjust as necessary
+//         .build();
+//     Ok(S3Client::from_conf(config))
+// }
+
+pub struct BucketTargetSys {
+    arn_remote_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
+    targets_map: Arc<RwLock<HashMap<String, Vec<bucket::target::BucketTarget>>>>,
+    hc: HashMap<String, EpHealth>,
+    //store:Option<Arc<ecstore::store::ECStore>>,
+}
+
+lazy_static! {
+    pub static ref GLOBAL_Bucket_Target_Sys: std::sync::OnceLock<BucketTargetSys> = BucketTargetSys::new().into();
+}
+
+//#[derive(Debug)]
+// pub enum SetTargetError {
+//   NotFound,
+// }
+
+pub async fn get_bucket_target_client(bucket: &str, arn: &str) -> Result<TargetClient, SetTargetError> {
+    if let Some(sys) = GLOBAL_Bucket_Target_Sys.get() {
+        sys.get_remote_target_client2(arn).await
+    } else {
+        Err(SetTargetError::TargetNotFound(bucket.to_string()))
+    }
+}
+
+#[derive(Debug)]
+pub struct BucketRemoteTargetNotFound {
+    pub bucket: String,
+}
+
+pub async fn init_bucket_targets(bucket: &str, meta: Arc<bucket::metadata::BucketMetadata>) {
+    println!("140 {bucket}");
+    if let Some(sys) = GLOBAL_Bucket_Target_Sys.get() {
+        if let Some(tgts) = meta.bucket_target_config.clone() {
+            for tgt in tgts.targets {
+                warn!("ak and sk is:{:?}", tgt.credentials);
+                let _ = sys.set_target(bucket, &tgt, false, true).await;
+                //sys.targets_map.
+            }
+        }
+    }
+}
+
+pub async fn remove_bucket_target(bucket: &str, arn_str: &str) {
+    if let Some(sys) = GLOBAL_Bucket_Target_Sys.get() {
+        let _ = sys.remove_target(bucket, arn_str).await;
+    }
+}
+
+impl Default for BucketTargetSys {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl BucketTargetSys {
+    pub fn new() -> Self {
+        BucketTargetSys {
+            arn_remote_map: Arc::new(RwLock::new(HashMap::new())),
+            targets_map: Arc::new(RwLock::new(HashMap::new())),
+            hc: HashMap::new(),
+        }
+    }
+
+    pub async fn list_bucket_targets(&self, bucket: &str) -> Result<BucketTargets, BucketRemoteTargetNotFound> {
+        let targets_map = self.targets_map.read().await;
+        if let Some(targets) = targets_map.get(bucket) {
+            Ok(BucketTargets {
+                targets: targets.clone(),
+            })
+        } else {
+            Err(BucketRemoteTargetNotFound {
+                bucket: bucket.to_string(),
+            })
+        }
+    }
+
+    pub async fn list_targets(&self, bucket: Option<&str>, _arn_type: Option<&str>) -> Vec<BucketTarget> {
+        let _ = _arn_type;
+        //let health_stats = self.health_stats();
+
+        let mut targets = Vec::new();
+
+        if let Some(bucket_name) = bucket {
+            if let Ok(ts) = self.list_bucket_targets(bucket_name).await {
+                for t in ts.targets {
+                    //if arn_type.map_or(true, |arn| t.target_type == arn) {
+                    //if let Some(hs) = health_stats.get(&t.url().host) {
+                    // t.total_downtime = hs.offline_duration;
+                    // t.online = hs.online;
+                    // t.last_online = hs.last_online;
+                    // t.latency = LatencyStat {
+                    //     curr: hs.latency.curr,
+                    //     avg: hs.latency.avg,
+                    //     max: hs.latency.peak,
+                    // };
+                    //}
+                    targets.push(t.clone());
+                    //}
+                }
+            }
+            return targets;
+        }
+
+        // Locking and iterating over all targets in the system
+        let targets_map = self.targets_map.read().await;
+        for tgts in targets_map.values() {
+            for t in tgts {
+                //if arn_type.map_or(true, |arn| t.target_type == arn) {
+                // if let Some(hs) = health_stats.get(&t.url().host) {
+                //     t.total_downtime = hs.offline_duration;
+                //     t.online = hs.online;
+                //     t.last_online = hs.last_online;
+                //     t.latency = LatencyStat {
+                //         curr: hs.latency.curr,
+                //         avg: hs.latency.avg,
+                //         max: hs.latency.peak,
+                //     };
+                // }
+                targets.push(t.clone());
+                //}
+            }
+        }
+
+        targets
+    }
+
+    pub async fn remove_target(&self, bucket: &str, arn_str: &str) -> Result<(), SetTargetError> {
+        //to do need lock;
+        let mut targets_map = self.targets_map.write().await;
+        let tgts = targets_map.get(bucket);
+        let mut arn_remotes_map = self.arn_remote_map.write().await;
+        if tgts.is_none() {
+            //Err(SetTargetError::TargetNotFound(bucket.to_string()));
+            return Ok(());
+        }
+
+        let tgts = tgts.unwrap(); // 安全解引用
+        let mut targets = Vec::with_capacity(tgts.len());
+        let mut found = false;
+
+        // 遍历 targets，找出不匹配的 ARN
+        for tgt in tgts {
+            if tgt.arn != Some(arn_str.to_string()) {
+                targets.push(tgt.clone()); // 克隆符合条件的项
+            } else {
+                found = true; // 找到匹配的 ARN
+            }
+        }
+
+        // 如果没有找到匹配的 ARN，则返回错误
+        if !found {
+            return Ok(());
+        }
+
+        // 更新 targets_map
+        targets_map.insert(bucket.to_string(), targets);
+        arn_remotes_map.remove(arn_str);
+
+        let targets = self.list_targets(Some(bucket), None).await;
+        println!("targets is {}", targets.len());
+        match serde_json::to_vec(&targets) {
+            Ok(json) => {
+                let _ = metadata_sys::update(bucket, "bucket-targets.json", json).await;
+            }
+            Err(e) => {
+                println!("序列化失败{e}");
+            }
+        }
+
+        Ok(())
+    }
+
+    pub async fn get_remote_arn(&self, bucket: &str, target: Option<&BucketTarget>, depl_id: &str) -> (Option<String>, bool) {
+        if target.is_none() {
+            return (None, false);
+        }
+
+        let target = target.unwrap();
+
+        let targets_map = self.targets_map.read().await;
+
+        // 获取锁以访问 arn_remote_map
+        let mut _arn_remotes_map = self.arn_remote_map.read().await;
+        if let Some(tgts) = targets_map.get(bucket) {
+            for tgt in tgts {
+                if tgt.type_ == target.type_
+                    && tgt.target_bucket == target.target_bucket
+                    && tgt.endpoint == target.endpoint
+                    && tgt.credentials.as_ref().unwrap().access_key == target.credentials.as_ref().unwrap().access_key
+                {
+                    return (tgt.arn.clone(), true);
+                }
+            }
+        }
+
+        // if !target.type_.is_valid() {
+        //     return (None, false);
+        // }
+
+        println!("generate_arn");
+
+        (Some(generate_arn(target.clone(), depl_id.to_string())), false)
+    }
+
+    pub async fn get_remote_target_client2(&self, arn: &str) -> Result<TargetClient, SetTargetError> {
+        let map = self.arn_remote_map.read().await;
+        info!("get remote target client and arn is: {}", arn);
+        if let Some(value) = map.get(arn) {
+            let mut x = value.client.clone();
+            x.arn = arn.to_string();
+            Ok(x)
+        } else {
+            error!("not find target");
+            Err(SetTargetError::TargetNotFound(arn.to_string()))
+        }
+    }
+
+    // pub async fn get_remote_target_client(&self, _tgt: &BucketTarget) -> Result<TargetClient, SetTargetError> {
+    //     // Mocked implementation for obtaining a remote client
+    //     let tcli = TargetClient {
+    //         bucket: _tgt.target_bucket.clone(),
+    //         storage_class: "STANDRD".to_string(),
+    //         disable_proxy: false,
+    //         health_check_duration: Duration::from_secs(100),
+    //         endpoint: _tgt.endpoint.clone(),
+    //         reset_id: "0".to_string(),
+    //         replicate_sync: false,
+    //         secure: false,
+    //         arn: "".to_string(),
+    //         client: reqwest::Client::new(),
+    //         ak: _tgt.
+
+    //     };
+    //     Ok(tcli)
+    // }
+    // pub async fn get_remote_target_client_with_bucket(&self, _bucket: String) -> Result<TargetClient, SetTargetError> {
+    //     // Mocked implementation for obtaining a remote client
+    //     let tcli = TargetClient {
+    //         bucket: _tgt.target_bucket.clone(),
+    //         storage_class: "STANDRD".to_string(),
+    //         disable_proxy: false,
+    //         health_check_duration: Duration::from_secs(100),
+    //         endpoint: _tgt.endpoint.clone(),
+    //         reset_id: "0".to_string(),
+    //         replicate_sync: false,
+    //         secure: false,
+    //         arn: "".to_string(),
+    //         client: reqwest::Client::new(),
+    //     };
+    //     Ok(tcli)
+    // }
+
+    async fn local_is_bucket_versioned(&self, _bucket: &str) -> bool {
+        let Some(store) = new_object_layer_fn() else {
+            return false;
+        };
+        //store.get_bucket_info(bucket, opts)
+
+        // let binfo:BucketInfo = store
+        // .get_bucket_info(bucket, &ecstore::store_api::BucketOptions::default()).await;
+        match store.get_bucket_info(_bucket, &store_api::BucketOptions::default()).await {
+            Ok(info) => {
+                println!("Bucket Info: {info:?}");
+                info.versionning
+            }
+            Err(err) => {
+                eprintln!("Error: {err:?}");
+                false
+            }
+        }
+    }
+
+    async fn is_bucket_versioned(&self, _bucket: &str) -> bool {
+        true
+        // let url_str = "http://127.0.0.1:9001";
+
+        // // 转换为 Url 类型
+        // let parsed_url = url::Url::parse(url_str).unwrap();
+
+        // let node = Node {
+        //     url: parsed_url,
+        //     pools: vec![],
+        //     is_local: false,
+        //     grid_host: "".to_string(),
+        // };
+        // let cli = ecstore::peer::RemotePeerS3Client::new(Some(node), None);
+
+        // match cli.get_bucket_info(_bucket, &ecstore::store_api::BucketOptions::default()).await
+        // {
+        //     Ok(info) => {
+        //         println!("Bucket Info: {:?}", info);
+        //         info.versionning
+        //     }
+        //     Err(err) => {
+        //         eprintln!("Error: {:?}", err);
+        //         return false;
+        //     }
+        // }
+    }
+
+    pub async fn set_target(&self, bucket: &str, tgt: &BucketTarget, update: bool, fromdisk: bool) -> Result<(), SetTargetError> {
+        // if !tgt.type_.is_valid() && !update {
+        //     return Err(SetTargetError::InvalidTargetType(bucket.to_string()));
+        // }
+
+        //let client = self.get_remote_target_client(tgt).await?;
+        if tgt.type_ == Some("replication".to_string()) && !fromdisk {
+            let versioning_config = self.local_is_bucket_versioned(bucket).await;
+            if !versioning_config {
+                // println!("111111111");
+                return Err(SetTargetError::TargetNotVersioned(bucket.to_string()));
+            }
+        }
+
+        let url_str = format!("http://{}", tgt.endpoint.clone());
+
+        println!("url str is {url_str}");
+        // 转换为 Url 类型
+        let parsed_url = url::Url::parse(&url_str).unwrap();
+
+        let node = Node {
+            url: parsed_url,
+            pools: vec![],
+            is_local: false,
+            grid_host: "".to_string(),
+        };
+
+        let cli = RemotePeerS3Client::new(Some(node), None);
+
+        match cli
+            .get_bucket_info(&tgt.target_bucket, &store_api::BucketOptions::default())
+            .await
+        {
+            Ok(info) => {
+                println!("Bucket Info: {info:?}");
+                if !info.versionning {
+                    println!("2222222222 {}", info.versionning);
+                    return Err(SetTargetError::TargetNotVersioned(tgt.target_bucket.to_string()));
+                }
+            }
+            Err(err) => {
+                println!("remote bucket 369 is:{}", tgt.target_bucket);
+                eprintln!("Error: {err:?}");
+                return Err(SetTargetError::SourceNotVersioned(tgt.target_bucket.to_string()));
+            }
+        }
+
+        //if tgt.target_type == BucketTargetType::ReplicationService {
+        // Check if target is a rustfs server and alive
+        // let hc_result = tokio::time::timeout(Duration::from_secs(3), client.health_check(&tgt.endpoint)).await;
+        // match hc_result {
+        //     Ok(Ok(true)) => {} // Server is alive
+        //     Ok(Ok(false)) | Ok(Err(_)) | Err(_) => {
+        //         return Err(SetTargetError::HealthCheckFailed(tgt.target_bucket.clone()));
+        //     }
+        // }
+
+        //Lock and update target maps
+        let mut targets_map = self.targets_map.write().await;
+        let mut arn_remotes_map = self.arn_remote_map.write().await;
+
+        let targets = targets_map.entry(bucket.to_string()).or_default();
+        let mut found = false;
+
+        for existing_target in targets.iter_mut() {
+            println!("418 exist:{}", existing_target.source_bucket.clone());
+            if existing_target.type_ == tgt.type_ {
+                if existing_target.arn == tgt.arn {
+                    if !update {
+                        return Err(SetTargetError::TargetAlreadyExists(existing_target.target_bucket.clone()));
+                    }
+                    *existing_target = tgt.clone();
+                    found = true;
+                    break;
+                }
+
+                if existing_target.endpoint == tgt.endpoint {
+                    println!("endpoint is same:{}", tgt.endpoint.clone());
+                    return Err(SetTargetError::TargetAlreadyExists(existing_target.target_bucket.clone()));
+                }
+            }
+        }
+
+        if !found && !update {
+            println!("437 exist:{}", tgt.arn.clone().unwrap());
+            targets.push(tgt.clone());
+        }
+        let arntgt: ArnTarget = ArnTarget::new(
+            tgt.target_bucket.clone(),
+            tgt.endpoint.clone(),
+            tgt.credentials.clone().unwrap().access_key.clone(),
+            tgt.credentials.clone().unwrap().secret_key,
+        );
+
+        arn_remotes_map.insert(tgt.arn.clone().unwrap().clone(), arntgt);
+        //self.update_bandwidth_limit(bucket, &tgt.arn, tgt.bandwidth_limit).await;
+
+        Ok(())
+    }
+}
+
+#[derive(Clone)]
+pub struct TargetClient {
+    pub client: reqwest::Client, // Using reqwest HTTP client
+    pub health_check_duration: Duration,
+    pub bucket: String, // Remote bucket target
+    pub replicate_sync: bool,
+    pub storage_class: String, // Storage class on remote
+    pub disable_proxy: bool,
+    pub arn: String, // ARN to uniquely identify remote target
+    pub reset_id: String,
+    pub endpoint: String,
+    pub secure: bool,
+    pub ak: String,
+    pub sk: String,
+}
+
+#[allow(clippy::too_many_arguments)]
+impl TargetClient {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        client: reqwest::Client,
+        health_check_duration: Duration,
+        bucket: String,
+        replicate_sync: bool,
+        storage_class: String,
+        disable_proxy: bool,
+        arn: String,
+        reset_id: String,
+        endpoint: String,
+        secure: bool,
+        ak: String,
+        sk: String,
+    ) -> Self {
+        TargetClient {
+            client,
+            health_check_duration,
+            bucket,
+            replicate_sync,
+            storage_class,
+            disable_proxy,
+            arn,
+            reset_id,
+            endpoint,
+            secure,
+            ak,
+            sk,
+        }
+    }
+    pub async fn bucket_exists(&self, _bucket: &str) -> Result<bool, SetTargetError> {
+        Ok(true) // Mocked implementation
+    }
+}
+use tracing::{error, info, warn};
+use uuid::Uuid;
+
+#[derive(Debug, Clone)]
+pub struct VersioningConfig {
+    pub enabled: bool,
+}
+
+impl VersioningConfig {
+    pub fn is_enabled(&self) -> bool {
+        self.enabled
+    }
+}
+
+#[derive(Debug)]
+pub struct Client;
+
+impl Client {
+    pub async fn bucket_exists(&self, _bucket: &str) -> Result<bool, SetTargetError> {
+        Ok(true) // Mocked implementation
+    }
+
+    pub async fn get_bucket_versioning(&self, _bucket: &str) -> Result<VersioningConfig, SetTargetError> {
+        Ok(VersioningConfig { enabled: true })
+    }
+
+    pub async fn health_check(&self, _endpoint: &str) -> Result<bool, SetTargetError> {
+        Ok(true) // Mocked health check
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub struct ServiceType(String);
+
+impl ServiceType {
+    pub fn is_valid(&self) -> bool {
+        !self.0.is_empty() // 根据需求添加具体的验证逻辑
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub struct ARN {
+    pub arn_type: String,
+    pub id: String,
+    pub region: String,
+    pub bucket: String,
+}
+
+impl ARN {
+    /// 检查 ARN 是否为空
+    pub fn is_empty(&self) -> bool {
+        //!self.arn_type.is_valid()
+        false
+    }
+
+    // 从字符串解析 ARN
+    pub fn parse(s: &str) -> Result<Self, String> {
+        // ARN 必须是格式 arn:rustfs:<Type>:<REGION>:<ID>:<remote-bucket>
+        if !s.starts_with("arn:rustfs:") {
+            return Err(format!("Invalid ARN {s}"));
+        }
+
+        let tokens: Vec<&str> = s.split(':').collect();
+        if tokens.len() != 6 || tokens[4].is_empty() || tokens[5].is_empty() {
+            return Err(format!("Invalid ARN {s}"));
+        }
+
+        Ok(ARN {
+            arn_type: tokens[2].to_string(),
+            region: tokens[3].to_string(),
+            id: tokens[4].to_string(),
+            bucket: tokens[5].to_string(),
+        })
+    }
+}
+
+// 实现 `Display` trait，使得可以直接使用 `format!` 或 `{}` 输出 ARN
+impl std::fmt::Display for ARN {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "arn:rustfs:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
+    }
+}
+
+fn must_get_uuid() -> String {
+    Uuid::new_v4().to_string()
+    // match Uuid::new_v4() {
+    //     Ok(uuid) => uuid.to_string(),
+    //     Err(err) => {
+    //         error!("Critical error: {}", err);
+    //         panic!("Failed to generate UUID: {}", err); // Ensures similar behavior as Go's logger.CriticalIf
+    //     }
+    // }
+}
+fn generate_arn(target: BucketTarget, depl_id: String) -> String {
+    let mut uuid: String = depl_id;
+    if uuid.is_empty() {
+        uuid = must_get_uuid();
+    }
+
+    let arn: ARN = ARN {
+        arn_type: target.type_.unwrap(),
+        id: (uuid),
+        region: "us-east-1".to_string(),
+        bucket: (target.target_bucket),
+    };
+    arn.to_string()
+}
+
+// use std::collections::HashMap;
+// use std::sync::{Arc, Mutex, RwLock};
+// use std::time::Duration;
+// use tokio::time::timeout;
+// use tokio::sync::RwLock as AsyncRwLock;
+// use serde::Deserialize;
+// use thiserror::Error;
+
+// #[derive(Debug, Clone, PartialEq)]
+// pub enum BucketTargetType {
+//     ReplicationService,
+//     // Add other service types as needed
+// }
+
+// impl BucketTargetType {
+//     pub fn is_valid(&self) -> bool {
+//         matches!(self, BucketTargetType::ReplicationService)
+//     }
+// }
+
+// #[derive(Debug, Clone)]
+// pub struct BucketTarget {
+//     pub arn: String,
+//     pub target_bucket: String,
+//     pub endpoint: String,
+//     pub credentials: Credentials,
+//     pub secure: bool,
+//     pub bandwidth_limit: Option<u64>,
+//     pub target_type: BucketTargetType,
+// }
+
+// #[derive(Debug, Clone)]
+// pub struct Credentials {
+//     pub access_key: String,
+//     pub secret_key: String,
+// }
+
+// #[derive(Debug)]
+// pub struct BucketTargetSys {
+//     targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
+//     arn_remotes_map: Arc<Mutex<HashMap<String, ArnTarget>>>,
+// }
+
+// impl BucketTargetSys {
+//     pub fn new() -> Self {
+//         Self {
+//             targets_map: Arc::new(RwLock::new(HashMap::new())),
+//             arn_remotes_map: Arc::new(Mutex::new(HashMap::new())),
+//         }
+//     }
+
+//     pub async fn set_target(
+//         &self,
+//         bucket: &str,
+//         tgt: &BucketTarget,
+//         update: bool,
+//     ) -> Result<(), SetTargetError> {
+//         if !tgt.target_type.is_valid() && !update {
+//             return Err(SetTargetError::InvalidTargetType(bucket.to_string()));
+//         }
+
+//         let client = self.get_remote_target_client(tgt).await?;
+
+//         // Validate if target credentials are OK
+//         let exists = client.bucket_exists(&tgt.target_bucket).await?;
+//         if !exists {
+//             return Err(SetTargetError::TargetNotFound(tgt.target_bucket.clone()));
+//         }
+
+//         if tgt.target_type == BucketTargetType::ReplicationService {
+//             if !self.is_bucket_versioned(bucket).await {
+//                 return Err(SetTargetError::SourceNotVersioned(bucket.to_string()));
+//             }
+
+//             let versioning_config = client.get_bucket_versioning(&tgt.target_bucket).await?;
+//             if !versioning_config.is_enabled() {
+//                 return Err(SetTargetError::TargetNotVersioned(tgt.target_bucket.clone()));
+//             }
+//         }
+
+//         // Check if target is a rustfs server and alive
+//         let hc_result = timeout(Duration::from_secs(3), client.health_check(&tgt.endpoint)).await;
+//         match hc_result {
+//             Ok(Ok(true)) => {} // Server is alive
+//             Ok(Ok(false)) | Ok(Err(_)) | Err(_) => {
+//                 return Err(SetTargetError::HealthCheckFailed(tgt.target_bucket.clone()));
+//             }
+//         }
+
+//         // Lock and update target maps
+//         let mut targets_map = self.targets_map.write().await;
+//         let mut arn_remotes_map = self.arn_remotes_map.lock().unwrap();
+
+//         let targets = targets_map.entry(bucket.to_string()).or_default();
+//         let mut found = false;
+
+//         for existing_target in targets.iter_mut() {
+//             if existing_target.target_type == tgt.target_type {
+//                 if existing_target.arn == tgt.arn {
+//                     if !update {
+//                         return Err(SetTargetError::TargetAlreadyExists(existing_target.target_bucket.clone()));
+//                     }
+//                     *existing_target = tgt.clone();
+//                     found = true;
+//                     break;
+//                 }
+
+//                 if existing_target.endpoint == tgt.endpoint {
+//                     return Err(SetTargetError::TargetAlreadyExists(existing_target.target_bucket.clone()));
+//                 }
+//             }
+//         }
+
+//         if !found && !update {
+//             targets.push(tgt.clone());
+//         }
+
+//         arn_remotes_map.insert(tgt.arn.clone(), ArnTarget { client });
+//         self.update_bandwidth_limit(bucket, &tgt.arn, tgt.bandwidth_limit).await;
+
+//         Ok(())
+//     }
+
+//     async fn get_remote_target_client(&self, tgt: &BucketTarget) -> Result<Client, SetTargetError> {
+//         // Mocked implementation for obtaining a remote client
+//         Ok(Client {})
+//     }
+
+//     async fn is_bucket_versioned(&self, bucket: &str) -> bool {
+//         // Mocked implementation for checking if a bucket is versioned
+//         true
+//     }
+
+//     async fn update_bandwidth_limit(
+//         &self,
+//         bucket: &str,
+//         arn: &str,
+//         limit: Option<u64>,
+//     ) {
+//         // Mocked implementation for updating bandwidth limits
+//     }
+// }
+
+// #[derive(Debug)]
+// pub struct Client;
+
+// impl Client {
+//     pub async fn bucket_exists(&self, _bucket: &str) -> Result<bool, SetTargetError> {
+//         Ok(true) // Mocked implementation
+//     }
+
+//     pub async fn get_bucket_versioning(
+//         &self,
+//         _bucket: &str,
+//     ) -> Result<VersioningConfig, SetTargetError> {
+//         Ok(VersioningConfig { enabled: true })
+//     }
+
+//     pub async fn health_check(&self, _endpoint: &str) -> Result<bool, SetTargetError> {
+//         Ok(true) // Mocked health check
+//     }
+// }
+
+// #[derive(Debug, Clone)]
+// pub struct ArnTarget {
+//     pub client: Client,
+// }
+
+#[derive(Debug, Error)]
+pub enum SetTargetError {
+    #[error("Invalid target type for bucket {0}")]
+    InvalidTargetType(String),
+
+    #[error("Target bucket {0} not found")]
+    TargetNotFound(String),
+
+    #[error("Source bucket {0} is not versioned")]
+    SourceNotVersioned(String),
+
+    #[error("Target bucket {0} is not versioned")]
+    TargetNotVersioned(String),
+
+    #[error("Health check failed for bucket {0}")]
+    HealthCheckFailed(String),
+
+    #[error("Target bucket {0} already exists")]
+    TargetAlreadyExists(String),
+}

crates/ecstore/src/cmd/bucketreplicationhandler.rs

@@ -0,0 +1,14 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+

crates/ecstore/src/cmd/mod.rs

@@ -0,0 +1,16 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod bucket_replication;
+pub mod bucket_targets;

crates/ecstore/src/compress.rs

@@ -0,0 +1,129 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_utils::string::has_pattern;
+use rustfs_utils::string::has_string_suffix_in_slice;
+use std::env;
+use tracing::error;
+
+pub const MIN_COMPRESSIBLE_SIZE: usize = 4096;
+
+// 环境变量名称，用于控制是否启用压缩
+pub const ENV_COMPRESSION_ENABLED: &str = "RUSTFS_COMPRESSION_ENABLED";
+
+// Some standard object extensions which we strictly dis-allow for compression.
+pub const STANDARD_EXCLUDE_COMPRESS_EXTENSIONS: &[&str] = &[
+    ".gz", ".bz2", ".rar", ".zip", ".7z", ".xz", ".mp4", ".mkv", ".mov", ".jpg", ".png", ".gif",
+];
+
+// Some standard content-types which we strictly dis-allow for compression.
+pub const STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES: &[&str] = &[
+    "video/*",
+    "audio/*",
+    "application/zip",
+    "application/x-gzip",
+    "application/x-zip-compressed",
+    "application/x-compress",
+    "application/x-spoon",
+];
+
+pub fn is_compressible(headers: &http::HeaderMap, object_name: &str) -> bool {
+    // 检查环境变量是否启用压缩，默认关闭
+    if let Ok(compression_enabled) = env::var(ENV_COMPRESSION_ENABLED) {
+        if compression_enabled.to_lowercase() != "true" {
+            error!("Compression is disabled by environment variable");
+            return false;
+        }
+    } else {
+        // 环境变量未设置时默认关闭
+        return false;
+    }
+
+    let content_type = headers.get("content-type").and_then(|s| s.to_str().ok()).unwrap_or("");
+
+    // TODO: crypto request return false
+
+    if has_string_suffix_in_slice(object_name, STANDARD_EXCLUDE_COMPRESS_EXTENSIONS) {
+        error!("object_name: {} is not compressible", object_name);
+        return false;
+    }
+
+    if !content_type.is_empty() && has_pattern(STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES, content_type) {
+        error!("content_type: {} is not compressible", content_type);
+        return false;
+    }
+    true
+
+    // TODO: check from config
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use temp_env;
+
+    #[test]
+    fn test_is_compressible() {
+        use http::HeaderMap;
+
+        let headers = HeaderMap::new();
+
+        // 测试环境变量控制
+        temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("false"), || {
+            assert!(!is_compressible(&headers, "file.txt"));
+        });
+
+        temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || {
+            assert!(is_compressible(&headers, "file.txt"));
+        });
+
+        temp_env::with_var_unset(ENV_COMPRESSION_ENABLED, || {
+            assert!(!is_compressible(&headers, "file.txt"));
+        });
+
+        temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || {
+            let mut headers = HeaderMap::new();
+            // 测试不可压缩的扩展名
+            headers.insert("content-type", "text/plain".parse().unwrap());
+            assert!(!is_compressible(&headers, "file.gz"));
+            assert!(!is_compressible(&headers, "file.zip"));
+            assert!(!is_compressible(&headers, "file.mp4"));
+            assert!(!is_compressible(&headers, "file.jpg"));
+
+            // 测试不可压缩的内容类型
+            headers.insert("content-type", "video/mp4".parse().unwrap());
+            assert!(!is_compressible(&headers, "file.txt"));
+
+            headers.insert("content-type", "audio/mpeg".parse().unwrap());
+            assert!(!is_compressible(&headers, "file.txt"));
+
+            headers.insert("content-type", "application/zip".parse().unwrap());
+            assert!(!is_compressible(&headers, "file.txt"));
+
+            headers.insert("content-type", "application/x-gzip".parse().unwrap());
+            assert!(!is_compressible(&headers, "file.txt"));
+
+            // 测试可压缩的情况
+            headers.insert("content-type", "text/plain".parse().unwrap());
+            assert!(is_compressible(&headers, "file.txt"));
+            assert!(is_compressible(&headers, "file.log"));
+
+            headers.insert("content-type", "text/html".parse().unwrap());
+            assert!(is_compressible(&headers, "file.html"));
+
+            headers.insert("content-type", "application/json".parse().unwrap());
+            assert!(is_compressible(&headers, "file.json"));
+        });
+    }
+}

crates/ecstore/src/config/com.rs

@@ -0,0 +1,230 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{Config, GLOBAL_StorageClass, storageclass};
+use crate::disk::RUSTFS_META_BUCKET;
+use crate::error::{Error, Result};
+use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
+use http::HeaderMap;
+use lazy_static::lazy_static;
+use rustfs_config::DEFAULT_DELIMITER;
+use rustfs_utils::path::SLASH_SEPARATOR;
+use std::collections::HashSet;
+use std::sync::Arc;
+use tracing::{error, warn};
+
+pub const CONFIG_PREFIX: &str = "config";
+const CONFIG_FILE: &str = "config.json";
+
+pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
+
+lazy_static! {
+    static ref CONFIG_BUCKET: String = format!("{}{}{}", RUSTFS_META_BUCKET, SLASH_SEPARATOR, CONFIG_PREFIX);
+    static ref SubSystemsDynamic: HashSet<String> = {
+        let mut h = HashSet::new();
+        h.insert(STORAGE_CLASS_SUB_SYS.to_owned());
+        h
+    };
+}
+pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
+    let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
+    Ok(data)
+}
+
+pub async fn read_config_with_metadata<S: StorageAPI>(
+    api: Arc<S>,
+    file: &str,
+    opts: &ObjectOptions,
+) -> Result<(Vec<u8>, ObjectInfo)> {
+    let h = HeaderMap::new();
+    let mut rd = api
+        .get_object_reader(RUSTFS_META_BUCKET, file, None, h, opts)
+        .await
+        .map_err(|err| {
+            if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) {
+                Error::ConfigNotFound
+            } else {
+                warn!("read_config_with_metadata: err: {:?}, file: {}", err, file);
+                err
+            }
+        })?;
+
+    let data = rd.read_all().await?;
+
+    if data.is_empty() {
+        return Err(Error::ConfigNotFound);
+    }
+
+    Ok((data, rd.object_info))
+}
+
+pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
+    save_config_with_opts(
+        api,
+        file,
+        data,
+        &ObjectOptions {
+            max_parity: true,
+            ..Default::default()
+        },
+    )
+    .await
+}
+
+pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()> {
+    match api
+        .delete_object(
+            RUSTFS_META_BUCKET,
+            file,
+            ObjectOptions {
+                delete_prefix: true,
+                delete_prefix_object: true,
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(_) => Ok(()),
+        Err(err) => {
+            if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) {
+                Err(Error::ConfigNotFound)
+            } else {
+                Err(err)
+            }
+        }
+    }
+}
+
+pub async fn save_config_with_opts<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
+    if let Err(err) = api
+        .put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data), opts)
+        .await
+    {
+        error!("save_config_with_opts: err: {:?}, file: {}", err, file);
+        return Err(err);
+    }
+    Ok(())
+}
+
+fn new_server_config() -> Config {
+    Config::new()
+}
+
+async fn new_and_save_server_config<S: StorageAPI>(api: Arc<S>) -> Result<Config> {
+    let mut cfg = new_server_config();
+    lookup_configs(&mut cfg, api.clone()).await;
+    save_server_config(api, &cfg).await?;
+
+    Ok(cfg)
+}
+
+fn get_config_file() -> String {
+    format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}")
+}
+
+/// Handle the situation where the configuration file does not exist, create and save a new configuration
+async fn handle_missing_config<S: StorageAPI>(api: Arc<S>, context: &str) -> Result<Config> {
+    warn!("Configuration not found ({}): Start initializing new configuration", context);
+    let cfg = new_and_save_server_config(api).await?;
+    warn!("Configuration initialization complete ({})", context);
+    Ok(cfg)
+}
+
+/// Handle configuration file read errors
+fn handle_config_read_error(err: Error, file_path: &str) -> Result<Config> {
+    error!("Read configuration failed (path: '{}'): {:?}", file_path, err);
+    Err(err)
+}
+
+pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<Config> {
+    let config_file = get_config_file();
+
+    // Try to read the configuration file
+    match read_config(api.clone(), &config_file).await {
+        Ok(data) => read_server_config(api, &data).await,
+        Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration").await,
+        Err(err) => handle_config_read_error(err, &config_file),
+    }
+}
+
+async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<Config> {
+    // If the provided data is empty, try to read from the file again
+    if data.is_empty() {
+        let config_file = get_config_file();
+        warn!("Received empty configuration data, try to reread from '{}'", config_file);
+
+        // Try to read the configuration again
+        match read_config(api.clone(), &config_file).await {
+            Ok(cfg_data) => {
+                // TODO: decrypt
+                let cfg = Config::unmarshal(&cfg_data)?;
+                return Ok(cfg.merge());
+            }
+            Err(Error::ConfigNotFound) => return handle_missing_config(api, "Read alternate configuration").await,
+            Err(err) => return handle_config_read_error(err, &config_file),
+        }
+    }
+
+    // Process non-empty configuration data
+    let cfg = Config::unmarshal(data)?;
+    Ok(cfg.merge())
+}
+
+pub async fn save_server_config<S: StorageAPI>(api: Arc<S>, cfg: &Config) -> Result<()> {
+    let data = cfg.marshal()?;
+
+    let config_file = get_config_file();
+
+    save_config(api, &config_file, data).await
+}
+
+pub async fn lookup_configs<S: StorageAPI>(cfg: &mut Config, api: Arc<S>) {
+    // TODO: from etcd
+    if let Err(err) = apply_dynamic_config(cfg, api).await {
+        error!("apply_dynamic_config err {:?}", &err);
+    }
+}
+
+async fn apply_dynamic_config<S: StorageAPI>(cfg: &mut Config, api: Arc<S>) -> Result<()> {
+    for key in SubSystemsDynamic.iter() {
+        apply_dynamic_config_for_sub_sys(cfg, api.clone(), key).await?;
+    }
+
+    Ok(())
+}
+
+async fn apply_dynamic_config_for_sub_sys<S: StorageAPI>(cfg: &mut Config, api: Arc<S>, subsys: &str) -> Result<()> {
+    let set_drive_counts = api.set_drive_counts();
+    if subsys == STORAGE_CLASS_SUB_SYS {
+        let kvs = cfg.get_value(STORAGE_CLASS_SUB_SYS, DEFAULT_DELIMITER).unwrap_or_default();
+
+        for (i, count) in set_drive_counts.iter().enumerate() {
+            match storageclass::lookup_config(&kvs, *count) {
+                Ok(res) => {
+                    if i == 0 && GLOBAL_StorageClass.get().is_none() {
+                        if let Err(r) = GLOBAL_StorageClass.set(res) {
+                            error!("GLOBAL_StorageClass.set failed {:?}", r);
+                        }
+                    }
+                }
+                Err(err) => {
+                    error!("init storage class err:{:?}", &err);
+                    break;
+                }
+            }
+        }
+    }
+
+    Ok(())
+}

crates/ecstore/src/config/heal.rs

@@ -0,0 +1,73 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use rustfs_utils::string::parse_bool;
+use std::time::Duration;
+
+#[derive(Debug, Default)]
+pub struct Config {
+    pub bitrot: String,
+    pub sleep: Duration,
+    pub io_count: usize,
+    pub drive_workers: usize,
+    pub cache: Duration,
+}
+
+impl Config {
+    pub fn bitrot_scan_cycle(&self) -> Duration {
+        self.cache
+    }
+
+    pub fn get_workers(&self) -> usize {
+        self.drive_workers
+    }
+
+    pub fn update(&mut self, nopts: &Config) {
+        self.bitrot = nopts.bitrot.clone();
+        self.io_count = nopts.io_count;
+        self.sleep = nopts.sleep;
+        self.drive_workers = nopts.drive_workers;
+    }
+}
+
+const RUSTFS_BITROT_CYCLE_IN_MONTHS: u64 = 1;
+
+fn parse_bitrot_config(s: &str) -> Result<Duration> {
+    match parse_bool(s) {
+        Ok(enabled) => {
+            if enabled {
+                Ok(Duration::from_secs_f64(0.0))
+            } else {
+                Ok(Duration::from_secs_f64(-1.0))
+            }
+        }
+        Err(_) => {
+            if !s.ends_with("m") {
+                return Err(Error::other("unknown format"));
+            }
+
+            match s.trim_end_matches('m').parse::<u64>() {
+                Ok(months) => {
+                    if months < RUSTFS_BITROT_CYCLE_IN_MONTHS {
+                        return Err(Error::other(format!("minimum bitrot cycle is {RUSTFS_BITROT_CYCLE_IN_MONTHS} month(s)")));
+                    }
+
+                    Ok(Duration::from_secs(months * 30 * 24 * 60))
+                }
+                Err(err) => Err(Error::other(err)),
+            }
+        }
+    }
+}

crates/ecstore/src/config/mod.rs

@@ -0,0 +1,218 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod com;
+#[allow(dead_code)]
+pub mod heal;
+mod notify;
+pub mod storageclass;
+
+use crate::error::Result;
+use crate::store::ECStore;
+use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
+use lazy_static::lazy_static;
+use rustfs_config::DEFAULT_DELIMITER;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::sync::{Arc, OnceLock};
+
+lazy_static! {
+    pub static ref GLOBAL_StorageClass: OnceLock<storageclass::Config> = OnceLock::new();
+    pub static ref DefaultKVS: OnceLock<HashMap<String, KVS>> = OnceLock::new();
+    pub static ref GLOBAL_ServerConfig: OnceLock<Config> = OnceLock::new();
+    pub static ref GLOBAL_ConfigSys: ConfigSys = ConfigSys::new();
+}
+
+/// Standard config keys and values.
+pub const ENABLE_KEY: &str = "enable";
+pub const COMMENT_KEY: &str = "comment";
+
+/// Enable values
+pub const ENABLE_ON: &str = "on";
+pub const ENABLE_OFF: &str = "off";
+
+pub const ENV_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
+pub const ENV_SECRET_KEY: &str = "RUSTFS_SECRET_KEY";
+pub const ENV_ROOT_USER: &str = "RUSTFS_ROOT_USER";
+pub const ENV_ROOT_PASSWORD: &str = "RUSTFS_ROOT_PASSWORD";
+
+pub static RUSTFS_CONFIG_PREFIX: &str = "config";
+
+pub struct ConfigSys {}
+
+impl Default for ConfigSys {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ConfigSys {
+    pub fn new() -> Self {
+        Self {}
+    }
+    pub async fn init(&self, api: Arc<ECStore>) -> Result<()> {
+        let mut cfg = read_config_without_migrate(api.clone().clone()).await?;
+
+        lookup_configs(&mut cfg, api).await;
+
+        let _ = GLOBAL_ServerConfig.set(cfg);
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize, Clone)]
+pub struct KV {
+    pub key: String,
+    pub value: String,
+    pub hidden_if_empty: bool,
+}
+
+#[derive(Debug, Deserialize, Serialize, Clone)]
+pub struct KVS(pub Vec<KV>);
+
+impl Default for KVS {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl KVS {
+    pub fn new() -> Self {
+        KVS(Vec::new())
+    }
+    pub fn get(&self, key: &str) -> String {
+        if let Some(v) = self.lookup(key) { v } else { "".to_owned() }
+    }
+    pub fn lookup(&self, key: &str) -> Option<String> {
+        for kv in self.0.iter() {
+            if kv.key.as_str() == key {
+                return Some(kv.value.clone());
+            }
+        }
+
+        None
+    }
+
+    ///Check if KVS is empty.
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
+    /// Returns a list of all keys for the current KVS.
+    /// If the "comment" key does not exist, it will be added.
+    pub fn keys(&self) -> Vec<String> {
+        let mut found_comment = false;
+        let mut keys: Vec<String> = self
+            .0
+            .iter()
+            .map(|kv| {
+                if kv.key == COMMENT_KEY {
+                    found_comment = true;
+                }
+                kv.key.clone()
+            })
+            .collect();
+
+        if !found_comment {
+            keys.push(COMMENT_KEY.to_owned());
+        }
+
+        keys
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Config(pub HashMap<String, HashMap<String, KVS>>);
+
+impl Default for Config {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Config {
+    pub fn new() -> Self {
+        let mut cfg = Config(HashMap::new());
+        cfg.set_defaults();
+
+        cfg
+    }
+
+    pub fn get_value(&self, sub_sys: &str, key: &str) -> Option<KVS> {
+        if let Some(m) = self.0.get(sub_sys) {
+            m.get(key).cloned()
+        } else {
+            None
+        }
+    }
+
+    pub fn set_defaults(&mut self) {
+        if let Some(defaults) = DefaultKVS.get() {
+            for (k, v) in defaults.iter() {
+                if !self.0.contains_key(k) {
+                    let mut default = HashMap::new();
+                    default.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
+                    self.0.insert(k.clone(), default);
+                } else if !self.0[k].contains_key(DEFAULT_DELIMITER) {
+                    if let Some(m) = self.0.get_mut(k) {
+                        m.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
+                    }
+                }
+            }
+        }
+    }
+
+    pub fn unmarshal(data: &[u8]) -> Result<Config> {
+        let m: HashMap<String, HashMap<String, KVS>> = serde_json::from_slice(data)?;
+        let mut cfg = Config(m);
+        cfg.set_defaults();
+        Ok(cfg)
+    }
+
+    pub fn marshal(&self) -> Result<Vec<u8>> {
+        let data = serde_json::to_vec(&self.0)?;
+        Ok(data)
+    }
+
+    pub fn merge(&self) -> Config {
+        // TODO: merge default
+        self.clone()
+    }
+}
+
+pub fn register_default_kvs(kvs: HashMap<String, KVS>) {
+    let mut p = HashMap::new();
+    for (k, v) in kvs {
+        p.insert(k, v);
+    }
+
+    let _ = DefaultKVS.set(p);
+}
+
+pub fn init() {
+    let mut kvs = HashMap::new();
+    // Load storageclass default configuration
+    kvs.insert(STORAGE_CLASS_SUB_SYS.to_owned(), storageclass::DefaultKVS.clone());
+    // New: Loading default configurations for notify_webhook and notify_mqtt
+    // Referring subsystem names through constants to improve the readability and maintainability of the code
+    kvs.insert(
+        rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS.to_owned(),
+        notify::DefaultWebhookKVS.clone(),
+    );
+    kvs.insert(rustfs_config::notify::NOTIFY_MQTT_SUB_SYS.to_owned(), notify::DefaultMqttKVS.clone());
+
+    // Register all default configurations
+    register_default_kvs(kvs)
+}

crates/ecstore/src/config/notify.rs

@@ -0,0 +1,51 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::config::{ENABLE_KEY, ENABLE_OFF, KV, KVS};
+use lazy_static::lazy_static;
+use rustfs_config::notify::{
+    DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT,
+    MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
+    WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
+};
+
+lazy_static! {
+    /// The default configuration collection of webhooks，
+    /// Use lazy_static! to ensure that these configurations are initialized only once during the program life cycle, enabling high-performance lazy loading.
+    pub static ref DefaultWebhookKVS: KVS = KVS(vec![
+        KV { key: ENABLE_KEY.to_owned(), value: ENABLE_OFF.to_owned(), hidden_if_empty: false },
+        KV { key: WEBHOOK_ENDPOINT.to_owned(), value: "".to_owned(), hidden_if_empty: false },
+        // Sensitive information such as authentication tokens is hidden when the value is empty, enhancing security
+        KV { key: WEBHOOK_AUTH_TOKEN.to_owned(), value: "".to_owned(), hidden_if_empty: true },
+        KV { key: WEBHOOK_QUEUE_LIMIT.to_owned(), value: DEFAULT_LIMIT.to_string().to_owned(), hidden_if_empty: false },
+        KV { key: WEBHOOK_QUEUE_DIR.to_owned(), value: DEFAULT_DIR.to_owned(), hidden_if_empty: false },
+        KV { key: WEBHOOK_CLIENT_CERT.to_owned(), value: "".to_owned(), hidden_if_empty: false },
+        KV { key: WEBHOOK_CLIENT_KEY.to_owned(), value: "".to_owned(), hidden_if_empty: false },
+    ]);
+
+    /// MQTT's default configuration collection
+    pub static ref DefaultMqttKVS: KVS = KVS(vec![
+        KV { key: ENABLE_KEY.to_owned(), value: ENABLE_OFF.to_owned(), hidden_if_empty: false },
+        KV { key: MQTT_BROKER.to_owned(), value: "".to_owned(), hidden_if_empty: false },
+        KV { key: MQTT_TOPIC.to_owned(), value: "".to_owned(), hidden_if_empty: false },
+        // Sensitive information such as passwords are hidden when the value is empty
+        KV { key: MQTT_PASSWORD.to_owned(), value: "".to_owned(), hidden_if_empty: true },
+        KV { key: MQTT_USERNAME.to_owned(), value: "".to_owned(), hidden_if_empty: false },
+        KV { key: MQTT_QOS.to_owned(), value: "0".to_owned(), hidden_if_empty: false },
+        KV { key: MQTT_KEEP_ALIVE_INTERVAL.to_owned(), value: "0s".to_owned(), hidden_if_empty: false },
+        KV { key: MQTT_RECONNECT_INTERVAL.to_owned(), value: "0s".to_owned(), hidden_if_empty: false },
+        KV { key: MQTT_QUEUE_DIR.to_owned(), value: DEFAULT_DIR.to_owned(), hidden_if_empty: false },
+        KV { key: MQTT_QUEUE_LIMIT.to_owned(), value: DEFAULT_LIMIT.to_string().to_owned(), hidden_if_empty: false },
+    ]);
+}

crates/ecstore/src/config/storageclass.rs

@@ -0,0 +1,320 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::KVS;
+use crate::config::KV;
+use crate::error::{Error, Result};
+use lazy_static::lazy_static;
+use serde::{Deserialize, Serialize};
+use std::env;
+use tracing::warn;
+
+/// Default parity count for a given drive count
+/// The default configuration allocates the number of check disks based on the total number of disks
+pub fn default_parity_count(drive: usize) -> usize {
+    match drive {
+        1 => 0,
+        2 | 3 => 1,
+        4 | 5 => 2,
+        6 | 7 => 3,
+        _ => 4,
+    }
+}
+
+// Standard constants for all storage class
+pub const RRS: &str = "REDUCED_REDUNDANCY";
+pub const STANDARD: &str = "STANDARD";
+
+// Standard constants for config info storage class
+pub const CLASS_STANDARD: &str = "standard";
+pub const CLASS_RRS: &str = "rrs";
+pub const OPTIMIZE: &str = "optimize";
+pub const INLINE_BLOCK: &str = "inline_block";
+
+// Reduced redundancy storage class environment variable
+pub const RRS_ENV: &str = "RUSTFS_STORAGE_CLASS_RRS";
+// Standard storage class environment variable
+pub const STANDARD_ENV: &str = "RUSTFS_STORAGE_CLASS_STANDARD";
+// Optimize storage class environment variable
+pub const OPTIMIZE_ENV: &str = "RUSTFS_STORAGE_CLASS_OPTIMIZE";
+// Inline block indicates the size of the shard that is considered for inlining
+pub const INLINE_BLOCK_ENV: &str = "RUSTFS_STORAGE_CLASS_INLINE_BLOCK";
+
+// Supported storage class scheme is EC
+pub const SCHEME_PREFIX: &str = "EC";
+
+// Min parity drives
+pub const MIN_PARITY_DRIVES: usize = 0;
+
+// Default RRS parity is always minimum parity.
+pub const DEFAULT_RRS_PARITY: usize = 1;
+
+pub static DEFAULT_INLINE_BLOCK: usize = 128 * 1024;
+
+lazy_static! {
+    pub static ref DefaultKVS: KVS = {
+        let kvs = vec![
+            KV {
+                key: CLASS_STANDARD.to_owned(),
+                value: "".to_owned(),
+                hidden_if_empty: false,
+            },
+            KV {
+                key: CLASS_RRS.to_owned(),
+                value: "EC:1".to_owned(),
+                hidden_if_empty: false,
+            },
+            KV {
+                key: OPTIMIZE.to_owned(),
+                value: "availability".to_owned(),
+                hidden_if_empty: false,
+            },
+            KV {
+                key: INLINE_BLOCK.to_owned(),
+                value: "".to_owned(),
+                hidden_if_empty: true,
+            },
+        ];
+
+        KVS(kvs)
+    };
+}
+
+// StorageClass - holds storage class information
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct StorageClass {
+    parity: usize,
+}
+
+// Config storage class configuration
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct Config {
+    standard: StorageClass,
+    rrs: StorageClass,
+    optimize: Option<String>,
+    inline_block: usize,
+    initialized: bool,
+}
+
+impl Config {
+    pub fn get_parity_for_sc(&self, sc: &str) -> Option<usize> {
+        match sc.trim() {
+            RRS => {
+                if self.initialized {
+                    Some(self.rrs.parity)
+                } else {
+                    None
+                }
+            }
+            _ => {
+                if self.initialized {
+                    Some(self.standard.parity)
+                } else {
+                    None
+                }
+            }
+        }
+    }
+
+    pub fn should_inline(&self, shard_size: i64, versioned: bool) -> bool {
+        if shard_size < 0 {
+            return false;
+        }
+
+        let shard_size = shard_size as usize;
+
+        let mut inline_block = DEFAULT_INLINE_BLOCK;
+        if self.initialized {
+            inline_block = self.inline_block;
+        }
+
+        if versioned {
+            shard_size <= inline_block / 8
+        } else {
+            shard_size <= inline_block
+        }
+    }
+
+    pub fn inline_block(&self) -> usize {
+        if !self.initialized {
+            DEFAULT_INLINE_BLOCK
+        } else {
+            self.inline_block
+        }
+    }
+
+    pub fn capacity_optimized(&self) -> bool {
+        if !self.initialized {
+            false
+        } else {
+            self.optimize.as_ref().is_some_and(|v| v.as_str() == "capacity")
+        }
+    }
+}
+
+pub fn lookup_config(kvs: &KVS, set_drive_count: usize) -> Result<Config> {
+    let standard = {
+        let ssc_str = {
+            if let Ok(ssc_str) = env::var(STANDARD_ENV) {
+                ssc_str
+            } else {
+                kvs.get(CLASS_STANDARD)
+            }
+        };
+
+        if !ssc_str.is_empty() {
+            parse_storage_class(&ssc_str)?
+        } else {
+            StorageClass {
+                parity: default_parity_count(set_drive_count),
+            }
+        }
+    };
+
+    let rrs = {
+        let ssc_str = {
+            if let Ok(ssc_str) = env::var(RRS_ENV) {
+                ssc_str
+            } else {
+                kvs.get(RRS)
+            }
+        };
+
+        if !ssc_str.is_empty() {
+            parse_storage_class(&ssc_str)?
+        } else {
+            StorageClass {
+                parity: { if set_drive_count == 1 { 0 } else { DEFAULT_RRS_PARITY } },
+            }
+        }
+    };
+
+    validate_parity_inner(standard.parity, rrs.parity, set_drive_count)?;
+
+    let optimize = { env::var(OPTIMIZE_ENV).ok() };
+
+    let inline_block = {
+        if let Ok(ev) = env::var(INLINE_BLOCK_ENV) {
+            if let Ok(block) = ev.parse::<bytesize::ByteSize>() {
+                if block.as_u64() as usize > DEFAULT_INLINE_BLOCK {
+                    warn!(
+                        "inline block value bigger than recommended max of 128KiB -> {}, performance may degrade for PUT please benchmark the changes",
+                        block
+                    );
+                }
+                block.as_u64() as usize
+            } else {
+                return Err(Error::other(format!("parse {INLINE_BLOCK_ENV} format failed")));
+            }
+        } else {
+            DEFAULT_INLINE_BLOCK
+        }
+    };
+
+    Ok(Config {
+        standard,
+        rrs,
+        optimize,
+        inline_block,
+        initialized: true,
+    })
+}
+
+pub fn parse_storage_class(env: &str) -> Result<StorageClass> {
+    let s: Vec<&str> = env.split(':').collect();
+
+    // only two elements allowed in the string - "scheme" and "number of parity drives"
+    if s.len() != 2 {
+        return Err(Error::other(format!(
+            "Invalid storage class format: {env}. Expected 'Scheme:Number of parity drives'."
+        )));
+    }
+
+    // only allowed scheme is "EC"
+    if s[0] != SCHEME_PREFIX {
+        return Err(Error::other(format!("Unsupported scheme {}. Supported scheme is EC.", s[0])));
+    }
+
+    // Number of parity drives should be integer
+    let parity_drives: usize = match s[1].parse() {
+        Ok(num) => num,
+        Err(_) => return Err(Error::other(format!("Failed to parse parity value: {}.", s[1]))),
+    };
+
+    Ok(StorageClass { parity: parity_drives })
+}
+
+// ValidateParity validates standard storage class parity.
+pub fn validate_parity(ss_parity: usize, set_drive_count: usize) -> Result<()> {
+    // if ss_parity > 0 && ss_parity < MIN_PARITY_DRIVES {
+    //     return Err(Error::other(format!(
+    //         "parity {} should be greater than or equal to {}",
+    //         ss_parity, MIN_PARITY_DRIVES
+    //     )));
+    // }
+
+    if ss_parity > set_drive_count / 2 {
+        return Err(Error::other(format!(
+            "parity {} should be less than or equal to {}",
+            ss_parity,
+            set_drive_count / 2
+        )));
+    }
+
+    Ok(())
+}
+
+// Validates the parity drives.
+pub fn validate_parity_inner(ss_parity: usize, rrs_parity: usize, set_drive_count: usize) -> Result<()> {
+    // if ss_parity > 0 && ss_parity < MIN_PARITY_DRIVES {
+    //     return Err(Error::other(format!(
+    //         "Standard storage class parity {} should be greater than or equal to {}",
+    //         ss_parity, MIN_PARITY_DRIVES
+    //     )));
+    // }
+
+    // RRS parity drives should be greater than or equal to minParityDrives.
+    // Parity below minParityDrives is not supported.
+    // if rrs_parity > 0 && rrs_parity < MIN_PARITY_DRIVES {
+    //     return Err(Error::other(format!(
+    //         "Reduced redundancy storage class parity {} should be greater than or equal to {}",
+    //         rrs_parity, MIN_PARITY_DRIVES
+    //     )));
+    // }
+
+    if set_drive_count > 2 {
+        if ss_parity > set_drive_count / 2 {
+            return Err(Error::other(format!(
+                "Standard storage class parity {} should be less than or equal to {}",
+                ss_parity,
+                set_drive_count / 2
+            )));
+        }
+
+        if rrs_parity > set_drive_count / 2 {
+            return Err(Error::other(format!(
+                "Reduced redundancy storage class parity {} should be less than or equal to {}",
+                rrs_parity,
+                set_drive_count / 2
+            )));
+        }
+    }
+
+    if ss_parity > 0 && rrs_parity > 0 && ss_parity < rrs_parity {
+        return Err(Error::other(format!(
+            "Standard storage class parity drives {ss_parity} should be greater than or equal to Reduced redundancy storage class parity drives {rrs_parity}"
+        )));
+    }
+    Ok(())
+}

crates/ecstore/src/disk/endpoint.rs

@@ -0,0 +1,539 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::error::{Error, Result};
+use path_absolutize::Absolutize;
+use rustfs_utils::{is_local_host, is_socket_addr};
+use std::{fmt::Display, path::Path};
+use tracing::debug;
+use url::{ParseError, Url};
+
+/// enum for endpoint type.
+#[derive(PartialEq, Eq, Debug)]
+pub enum EndpointType {
+    /// path style endpoint type enum.
+    Path,
+
+    /// URL style endpoint type enum.
+    Url,
+}
+
+/// any type of endpoint.
+#[derive(Debug, PartialEq, Eq, Clone, Hash)]
+pub struct Endpoint {
+    pub url: Url,
+    pub is_local: bool,
+
+    pub pool_idx: i32,
+    pub set_idx: i32,
+    pub disk_idx: i32,
+}
+
+impl Display for Endpoint {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if self.url.scheme() == "file" {
+            write!(f, "{}", self.get_file_path())
+        } else {
+            write!(f, "{}", self.url)
+        }
+    }
+}
+
+impl TryFrom<&str> for Endpoint {
+    /// The type returned in the event of a conversion error.
+    type Error = Error;
+
+    /// Performs the conversion.
+    fn try_from(value: &str) -> std::result::Result<Self, Self::Error> {
+        // check whether given path is not empty.
+        if ["", "/", "\\"].iter().any(|&v| v.eq(value)) {
+            return Err(Error::other("empty or root endpoint is not supported"));
+        }
+
+        let mut is_local = false;
+        let url = match Url::parse(value) {
+            #[allow(unused_mut)]
+            Ok(mut url) if url.has_host() => {
+                // URL style of endpoint.
+                // Valid URL style endpoint is
+                // - Scheme field must contain "http" or "https"
+                // - All field should be empty except Host and Path.
+                if !((url.scheme() == "http" || url.scheme() == "https")
+                    && url.username().is_empty()
+                    && url.fragment().is_none()
+                    && url.query().is_none())
+                {
+                    return Err(Error::other("invalid URL endpoint format"));
+                }
+
+                let path = url.path().to_string();
+
+                #[cfg(not(windows))]
+                let path = Path::new(&path).absolutize()?;
+
+                // On windows having a preceding SlashSeparator will cause problems, if the
+                // command line already has C:/<export-folder/ in it. Final resulting
+                // path on windows might become C:/C:/ this will cause problems
+                // of starting rustfs server properly in distributed mode on windows.
+                // As a special case make sure to trim the separator.
+                #[cfg(windows)]
+                let path = Path::new(&path[1..]).absolutize()?;
+
+                debug!("endpoint try_from: path={}", path.display());
+
+                if path.parent().is_none() || Path::new("").eq(&path) {
+                    return Err(Error::other("empty or root path is not supported in URL endpoint"));
+                }
+
+                match path.to_str() {
+                    Some(v) => url.set_path(v),
+                    None => return Err(Error::other("invalid path")),
+                }
+
+                url
+            }
+            Ok(_) => {
+                // like d:/foo
+                is_local = true;
+                url_parse_from_file_path(value)?
+            }
+            Err(e) => match e {
+                ParseError::InvalidPort => {
+                    return Err(Error::other("invalid URL endpoint format: port number must be between 1 to 65535"));
+                }
+                ParseError::EmptyHost => return Err(Error::other("invalid URL endpoint format: empty host name")),
+                ParseError::RelativeUrlWithoutBase => {
+                    // like /foo
+                    is_local = true;
+                    url_parse_from_file_path(value)?
+                }
+                _ => return Err(Error::other(format!("invalid URL endpoint format: {e}"))),
+            },
+        };
+
+        Ok(Endpoint {
+            url,
+            is_local,
+            pool_idx: -1,
+            set_idx: -1,
+            disk_idx: -1,
+        })
+    }
+}
+
+impl Endpoint {
+    /// returns type of endpoint.
+    pub fn get_type(&self) -> EndpointType {
+        if self.url.scheme() == "file" {
+            EndpointType::Path
+        } else {
+            EndpointType::Url
+        }
+    }
+
+    /// sets a specific pool number to this node
+    pub fn set_pool_index(&mut self, idx: usize) {
+        self.pool_idx = idx as i32
+    }
+
+    /// sets a specific set number to this node
+    pub fn set_set_index(&mut self, idx: usize) {
+        self.set_idx = idx as i32
+    }
+
+    /// sets a specific disk number to this node
+    pub fn set_disk_index(&mut self, idx: usize) {
+        self.disk_idx = idx as i32
+    }
+
+    /// resolves the host and updates if it is local or not.
+    pub fn update_is_local(&mut self, local_port: u16) -> Result<()> {
+        match (self.url.scheme(), self.url.host()) {
+            (v, Some(host)) if v != "file" => {
+                self.is_local = is_local_host(host, self.url.port().unwrap_or_default(), local_port)?;
+            }
+            _ => {}
+        }
+
+        Ok(())
+    }
+
+    /// returns the host to be used for grid connections.
+    pub fn grid_host(&self) -> String {
+        match (self.url.host(), self.url.port()) {
+            (Some(host), Some(port)) => {
+                debug!("grid_host scheme={}: host={}, port={}", self.url.scheme(), host, port);
+                format!("{}://{}:{}", self.url.scheme(), host, port)
+            }
+            (Some(host), None) => {
+                debug!("grid_host scheme={}: host={}", self.url.scheme(), host);
+                format!("{}://{}", self.url.scheme(), host)
+            }
+            _ => String::new(),
+        }
+    }
+
+    pub fn host_port(&self) -> String {
+        match (self.url.host(), self.url.port()) {
+            (Some(host), Some(port)) => {
+                debug!("host_port host={}, port={}", host, port);
+                format!("{host}:{port}")
+            }
+            (Some(host), None) => {
+                debug!("host_port host={}, port={}", host, self.url.port().unwrap_or(0));
+                format!("{host}")
+            }
+            _ => String::new(),
+        }
+    }
+
+    pub fn get_file_path(&self) -> &str {
+        let path = self.url.path();
+        #[cfg(windows)]
+        if self.url.scheme() == "file" {
+            let stripped = path.strip_prefix('/').unwrap_or(path);
+            debug!("get_file_path windows: path={}", stripped);
+            return stripped;
+        }
+        path
+    }
+}
+
+/// parse a file path into a URL.
+fn url_parse_from_file_path(value: &str) -> Result<Url> {
+    // Only check if the arg is an ip address and ask for scheme since its absent.
+    // localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
+    // /mnt/export1. So we go ahead and start the rustfs server in FS modes in these cases.
+    let addr: Vec<&str> = value.splitn(2, '/').collect();
+    if is_socket_addr(addr[0]) {
+        return Err(Error::other("invalid URL endpoint format: missing scheme http or https"));
+    }
+
+    let file_path = match Path::new(value).absolutize() {
+        Ok(path) => path,
+        Err(err) => return Err(Error::other(format!("absolute path failed: {err}"))),
+    };
+
+    match Url::from_file_path(file_path) {
+        Ok(url) => Ok(url),
+        Err(_) => Err(Error::other("Convert a file path into an URL failed")),
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_new_endpoint() {
+        #[derive(Default)]
+        struct TestCase<'a> {
+            arg: &'a str,
+            expected_endpoint: Option<Endpoint>,
+            expected_type: Option<EndpointType>,
+            expected_err: Option<Error>,
+        }
+
+        let u2 = Url::parse("https://example.org/path").unwrap();
+        let u4 = Url::parse("http://192.168.253.200/path").unwrap();
+        let u6 = Url::parse("http://server:/path").unwrap();
+        let root_slash_foo = Url::from_file_path("/foo").unwrap();
+
+        let test_cases = [
+            TestCase {
+                arg: "/foo",
+                expected_endpoint: Some(Endpoint {
+                    url: root_slash_foo,
+                    is_local: true,
+                    pool_idx: -1,
+                    set_idx: -1,
+                    disk_idx: -1,
+                }),
+                expected_type: Some(EndpointType::Path),
+                expected_err: None,
+            },
+            TestCase {
+                arg: "https://example.org/path",
+                expected_endpoint: Some(Endpoint {
+                    url: u2,
+                    is_local: false,
+                    pool_idx: -1,
+                    set_idx: -1,
+                    disk_idx: -1,
+                }),
+                expected_type: Some(EndpointType::Url),
+                expected_err: None,
+            },
+            TestCase {
+                arg: "http://192.168.253.200/path",
+                expected_endpoint: Some(Endpoint {
+                    url: u4,
+                    is_local: false,
+                    pool_idx: -1,
+                    set_idx: -1,
+                    disk_idx: -1,
+                }),
+                expected_type: Some(EndpointType::Url),
+                expected_err: None,
+            },
+            TestCase {
+                arg: "",
+                expected_endpoint: None,
+                expected_type: None,
+                expected_err: Some(Error::other("empty or root endpoint is not supported")),
+            },
+            TestCase {
+                arg: "/",
+                expected_endpoint: None,
+                expected_type: None,
+                expected_err: Some(Error::other("empty or root endpoint is not supported")),
+            },
+            TestCase {
+                arg: "\\",
+                expected_endpoint: None,
+                expected_type: None,
+                expected_err: Some(Error::other("empty or root endpoint is not supported")),
+            },
+            TestCase {
+                arg: "c://foo",
+                expected_endpoint: None,
+                expected_type: None,
+                expected_err: Some(Error::other("invalid URL endpoint format")),
+            },
+            TestCase {
+                arg: "ftp://foo",
+                expected_endpoint: None,
+                expected_type: None,
+                expected_err: Some(Error::other("invalid URL endpoint format")),
+            },
+            TestCase {
+                arg: "http://server/path?location",
+                expected_endpoint: None,
+                expected_type: None,
+                expected_err: Some(Error::other("invalid URL endpoint format")),
+            },
+            TestCase {
+                arg: "http://:/path",
+                expected_endpoint: None,
+                expected_type: None,
+                expected_err: Some(Error::other("invalid URL endpoint format: empty host name")),
+            },
+            TestCase {
+                arg: "http://:8080/path",
+                expected_endpoint: None,
+                expected_type: None,
+                expected_err: Some(Error::other("invalid URL endpoint format: empty host name")),
+            },
+            TestCase {
+                arg: "http://server:/path",
+                expected_endpoint: Some(Endpoint {
+                    url: u6,
+                    is_local: false,
+                    pool_idx: -1,
+                    set_idx: -1,
+                    disk_idx: -1,
+                }),
+                expected_type: Some(EndpointType::Url),
+                expected_err: None,
+            },
+            TestCase {
+                arg: "https://93.184.216.34:808080/path",
+                expected_endpoint: None,
+                expected_type: None,
+                expected_err: Some(Error::other("invalid URL endpoint format: port number must be between 1 to 65535")),
+            },
+            TestCase {
+                arg: "http://server:8080//",
+                expected_endpoint: None,
+                expected_type: None,
+                expected_err: Some(Error::other("empty or root path is not supported in URL endpoint")),
+            },
+            TestCase {
+                arg: "http://server:8080/",
+                expected_endpoint: None,
+                expected_type: None,
+                expected_err: Some(Error::other("empty or root path is not supported in URL endpoint")),
+            },
+            TestCase {
+                arg: "192.168.1.210:9000",
+                expected_endpoint: None,
+                expected_type: None,
+                expected_err: Some(Error::other("invalid URL endpoint format: missing scheme http or https")),
+            },
+        ];
+
+        for test_case in test_cases {
+            let ret = Endpoint::try_from(test_case.arg);
+            if test_case.expected_err.is_none() && ret.is_err() {
+                panic!("{}: error: expected = <nil>, got = {:?}", test_case.arg, ret);
+            }
+            if test_case.expected_err.is_some() && ret.is_ok() {
+                panic!("{}: error: expected = {:?}, got = <nil>", test_case.arg, test_case.expected_err);
+            }
+            match (test_case.expected_err, ret) {
+                (None, Err(e)) => panic!("{}: error: expected = <nil>, got = {}", test_case.arg, e),
+                (None, Ok(mut ep)) => {
+                    let _ = ep.update_is_local(9000);
+                    if test_case.expected_type != Some(ep.get_type()) {
+                        panic!(
+                            "{}: type: expected = {:?}, got = {:?}",
+                            test_case.arg,
+                            test_case.expected_type,
+                            ep.get_type()
+                        );
+                    }
+
+                    assert_eq!(test_case.expected_endpoint, Some(ep), "{}: endpoint", test_case.arg);
+                }
+                (Some(e), Ok(_)) => panic!("{}: error: expected = {}, got = <nil>", test_case.arg, e),
+                (Some(e), Err(e2)) => {
+                    assert_eq!(e.to_string(), e2.to_string(), "{}: error: expected = {}, got = {}", test_case.arg, e, e2)
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn test_endpoint_display() {
+        // Test file path display
+        let file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
+        let display_str = format!("{file_endpoint}");
+        assert_eq!(display_str, "/tmp/data");
+
+        // Test URL display
+        let url_endpoint = Endpoint::try_from("http://example.com:9000/path").unwrap();
+        let display_str = format!("{url_endpoint}");
+        assert_eq!(display_str, "http://example.com:9000/path");
+    }
+
+    #[test]
+    fn test_endpoint_type() {
+        let file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
+        assert_eq!(file_endpoint.get_type(), EndpointType::Path);
+
+        let url_endpoint = Endpoint::try_from("http://example.com:9000/path").unwrap();
+        assert_eq!(url_endpoint.get_type(), EndpointType::Url);
+    }
+
+    #[test]
+    fn test_endpoint_indexes() {
+        let mut endpoint = Endpoint::try_from("/tmp/data").unwrap();
+
+        // Test initial values
+        assert_eq!(endpoint.pool_idx, -1);
+        assert_eq!(endpoint.set_idx, -1);
+        assert_eq!(endpoint.disk_idx, -1);
+
+        // Test setting indexes
+        endpoint.set_pool_index(2);
+        endpoint.set_set_index(3);
+        endpoint.set_disk_index(4);
+
+        assert_eq!(endpoint.pool_idx, 2);
+        assert_eq!(endpoint.set_idx, 3);
+        assert_eq!(endpoint.disk_idx, 4);
+    }
+
+    #[test]
+    fn test_endpoint_grid_host() {
+        let endpoint = Endpoint::try_from("http://example.com:9000/path").unwrap();
+        assert_eq!(endpoint.grid_host(), "http://example.com:9000");
+
+        let endpoint_no_port = Endpoint::try_from("https://example.com/path").unwrap();
+        assert_eq!(endpoint_no_port.grid_host(), "https://example.com");
+
+        let file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
+        assert_eq!(file_endpoint.grid_host(), "");
+    }
+
+    #[test]
+    fn test_endpoint_host_port() {
+        let endpoint = Endpoint::try_from("http://example.com:9000/path").unwrap();
+        assert_eq!(endpoint.host_port(), "example.com:9000");
+
+        let endpoint_no_port = Endpoint::try_from("https://example.com/path").unwrap();
+        assert_eq!(endpoint_no_port.host_port(), "example.com");
+
+        let file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
+        assert_eq!(file_endpoint.host_port(), "");
+    }
+
+    #[test]
+    fn test_endpoint_get_file_path() {
+        let file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
+        assert_eq!(file_endpoint.get_file_path(), "/tmp/data");
+
+        let url_endpoint = Endpoint::try_from("http://example.com:9000/path/to/data").unwrap();
+        assert_eq!(url_endpoint.get_file_path(), "/path/to/data");
+    }
+
+    #[test]
+    fn test_endpoint_clone_and_equality() {
+        let endpoint1 = Endpoint::try_from("/tmp/data").unwrap();
+        let endpoint2 = endpoint1.clone();
+
+        assert_eq!(endpoint1, endpoint2);
+        assert_eq!(endpoint1.url, endpoint2.url);
+        assert_eq!(endpoint1.is_local, endpoint2.is_local);
+        assert_eq!(endpoint1.pool_idx, endpoint2.pool_idx);
+        assert_eq!(endpoint1.set_idx, endpoint2.set_idx);
+        assert_eq!(endpoint1.disk_idx, endpoint2.disk_idx);
+    }
+
+    #[test]
+    fn test_endpoint_with_special_paths() {
+        // Test with complex paths
+        let complex_path = "/var/lib/rustfs/data/bucket1";
+        let endpoint = Endpoint::try_from(complex_path).unwrap();
+        assert_eq!(endpoint.get_file_path(), complex_path);
+        assert!(endpoint.is_local);
+        assert_eq!(endpoint.get_type(), EndpointType::Path);
+    }
+
+    #[test]
+    fn test_endpoint_update_is_local() {
+        let mut endpoint = Endpoint::try_from("http://localhost:9000/path").unwrap();
+        let result = endpoint.update_is_local(9000);
+        assert!(result.is_ok());
+
+        let mut file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
+        let result = file_endpoint.update_is_local(9000);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_url_parse_from_file_path() {
+        let result = url_parse_from_file_path("/tmp/test");
+        assert!(result.is_ok());
+
+        let url = result.unwrap();
+        assert_eq!(url.scheme(), "file");
+    }
+
+    #[test]
+    fn test_endpoint_hash() {
+        use std::collections::HashSet;
+
+        let endpoint1 = Endpoint::try_from("/tmp/data1").unwrap();
+        let endpoint2 = Endpoint::try_from("/tmp/data2").unwrap();
+        let endpoint3 = endpoint1.clone();
+
+        let mut set = HashSet::new();
+        set.insert(endpoint1);
+        set.insert(endpoint2);
+        set.insert(endpoint3); // Should not be added as it's equal to endpoint1
+
+        assert_eq!(set.len(), 2);
+    }
+}

crates/ecstore/src/disk/error.rs

@@ -0,0 +1,878 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// use crate::quorum::CheckErrorFn;
+use std::hash::{Hash, Hasher};
+use std::io::{self};
+use std::path::PathBuf;
+use tracing::error;
+
+pub type Error = DiskError;
+pub type Result<T> = core::result::Result<T, Error>;
+
+// DiskError == StorageErr
+#[derive(Debug, thiserror::Error)]
+pub enum DiskError {
+    #[error("maximum versions exceeded, please delete few versions to proceed")]
+    MaxVersionsExceeded,
+
+    #[error("unexpected error")]
+    Unexpected,
+
+    #[error("corrupted format")]
+    CorruptedFormat,
+
+    #[error("corrupted backend")]
+    CorruptedBackend,
+
+    #[error("unformatted disk error")]
+    UnformattedDisk,
+
+    #[error("inconsistent drive found")]
+    InconsistentDisk,
+
+    #[error("drive does not support O_DIRECT")]
+    UnsupportedDisk,
+
+    #[error("drive path full")]
+    DiskFull,
+
+    #[error("disk not a dir")]
+    DiskNotDir,
+
+    #[error("disk not found")]
+    DiskNotFound,
+
+    #[error("drive still did not complete the request")]
+    DiskOngoingReq,
+
+    #[error("drive is part of root drive, will not be used")]
+    DriveIsRoot,
+
+    #[error("remote drive is faulty")]
+    FaultyRemoteDisk,
+
+    #[error("drive is faulty")]
+    FaultyDisk,
+
+    #[error("drive access denied")]
+    DiskAccessDenied,
+
+    #[error("file not found")]
+    FileNotFound,
+
+    #[error("file version not found")]
+    FileVersionNotFound,
+
+    #[error("too many open files, please increase 'ulimit -n'")]
+    TooManyOpenFiles,
+
+    #[error("file name too long")]
+    FileNameTooLong,
+
+    #[error("volume already exists")]
+    VolumeExists,
+
+    #[error("not of regular file type")]
+    IsNotRegular,
+
+    #[error("path not found")]
+    PathNotFound,
+
+    #[error("volume not found")]
+    VolumeNotFound,
+
+    #[error("volume is not empty")]
+    VolumeNotEmpty,
+
+    #[error("volume access denied")]
+    VolumeAccessDenied,
+
+    #[error("disk access denied")]
+    FileAccessDenied,
+
+    #[error("file is corrupted")]
+    FileCorrupt,
+
+    #[error("short write")]
+    ShortWrite,
+
+    #[error("bit-rot hash algorithm is invalid")]
+    BitrotHashAlgoInvalid,
+
+    #[error("Rename across devices not allowed, please fix your backend configuration")]
+    CrossDeviceLink,
+
+    #[error("less data available than what was requested")]
+    LessData,
+
+    #[error("more data was sent than what was advertised")]
+    MoreData,
+
+    #[error("outdated XL meta")]
+    OutdatedXLMeta,
+
+    #[error("part missing or corrupt")]
+    PartMissingOrCorrupt,
+
+    #[error("No healing is required")]
+    NoHealRequired,
+
+    #[error("method not allowed")]
+    MethodNotAllowed,
+
+    #[error("erasure write quorum")]
+    ErasureWriteQuorum,
+
+    #[error("erasure read quorum")]
+    ErasureReadQuorum,
+
+    #[error("io error {0}")]
+    Io(io::Error),
+}
+
+impl DiskError {
+    pub fn other<E>(error: E) -> Self
+    where
+        E: Into<Box<dyn std::error::Error + Send + Sync>>,
+    {
+        DiskError::Io(std::io::Error::other(error))
+    }
+
+    pub fn is_all_not_found(errs: &[Option<DiskError>]) -> bool {
+        for err in errs.iter() {
+            if let Some(err) = err {
+                if err == &DiskError::FileNotFound || err == &DiskError::FileVersionNotFound {
+                    continue;
+                }
+
+                return false;
+            }
+
+            return false;
+        }
+
+        !errs.is_empty()
+    }
+
+    pub fn is_err_object_not_found(err: &DiskError) -> bool {
+        matches!(err, &DiskError::FileNotFound) || matches!(err, &DiskError::VolumeNotFound)
+    }
+
+    pub fn is_err_version_not_found(err: &DiskError) -> bool {
+        matches!(err, &DiskError::FileVersionNotFound)
+    }
+
+    // /// If all errors are of the same fatal disk error type, returns the corresponding error.
+    // /// Otherwise, returns Ok.
+    // pub fn check_disk_fatal_errs(errs: &[Option<Error>]) -> Result<()> {
+    //     if DiskError::UnsupportedDisk.count_errs(errs) == errs.len() {
+    //         return Err(DiskError::UnsupportedDisk.into());
+    //     }
+
+    //     if DiskError::FileAccessDenied.count_errs(errs) == errs.len() {
+    //         return Err(DiskError::FileAccessDenied.into());
+    //     }
+
+    //     if DiskError::DiskNotDir.count_errs(errs) == errs.len() {
+    //         return Err(DiskError::DiskNotDir.into());
+    //     }
+
+    //     Ok(())
+    // }
+
+    // pub fn count_errs(&self, errs: &[Option<Error>]) -> usize {
+    //     errs.iter()
+    //         .filter(|&err| match err {
+    //             None => false,
+    //             Some(e) => self.is(e),
+    //         })
+    //         .count()
+    // }
+
+    // pub fn quorum_unformatted_disks(errs: &[Option<Error>]) -> bool {
+    //     DiskError::UnformattedDisk.count_errs(errs) > (errs.len() / 2)
+    // }
+
+    // pub fn should_init_erasure_disks(errs: &[Option<Error>]) -> bool {
+    //     DiskError::UnformattedDisk.count_errs(errs) == errs.len()
+    // }
+
+    // // Check if the error is a disk error
+    // pub fn is(&self, err: &DiskError) -> bool {
+    //     if let Some(e) = err.downcast_ref::<DiskError>() {
+    //         e == self
+    //     } else {
+    //         false
+    //     }
+    // }
+}
+
+impl From<rustfs_filemeta::Error> for DiskError {
+    fn from(e: rustfs_filemeta::Error) -> Self {
+        match e {
+            rustfs_filemeta::Error::Io(e) => DiskError::other(e),
+            rustfs_filemeta::Error::FileNotFound => DiskError::FileNotFound,
+            rustfs_filemeta::Error::FileVersionNotFound => DiskError::FileVersionNotFound,
+            rustfs_filemeta::Error::FileCorrupt => DiskError::FileCorrupt,
+            rustfs_filemeta::Error::MethodNotAllowed => DiskError::MethodNotAllowed,
+            e => DiskError::other(e),
+        }
+    }
+}
+
+impl From<std::io::Error> for DiskError {
+    fn from(e: std::io::Error) -> Self {
+        e.downcast::<DiskError>().unwrap_or_else(DiskError::Io)
+    }
+}
+
+impl From<DiskError> for std::io::Error {
+    fn from(e: DiskError) -> Self {
+        match e {
+            DiskError::Io(io_error) => io_error,
+            e => std::io::Error::other(e),
+        }
+    }
+}
+
+impl From<tonic::Status> for DiskError {
+    fn from(e: tonic::Status) -> Self {
+        DiskError::other(e.message().to_string())
+    }
+}
+
+impl From<rustfs_protos::proto_gen::node_service::Error> for DiskError {
+    fn from(e: rustfs_protos::proto_gen::node_service::Error) -> Self {
+        if let Some(err) = DiskError::from_u32(e.code) {
+            if matches!(err, DiskError::Io(_)) {
+                DiskError::other(e.error_info)
+            } else {
+                err
+            }
+        } else {
+            DiskError::other(e.error_info)
+        }
+    }
+}
+
+impl From<DiskError> for rustfs_protos::proto_gen::node_service::Error {
+    fn from(e: DiskError) -> Self {
+        rustfs_protos::proto_gen::node_service::Error {
+            code: e.to_u32(),
+            error_info: e.to_string(),
+        }
+    }
+}
+
+impl From<serde_json::Error> for DiskError {
+    fn from(e: serde_json::Error) -> Self {
+        DiskError::other(e)
+    }
+}
+
+impl From<rmp_serde::encode::Error> for DiskError {
+    fn from(e: rmp_serde::encode::Error) -> Self {
+        DiskError::other(e)
+    }
+}
+
+impl From<rmp::encode::ValueWriteError> for DiskError {
+    fn from(e: rmp::encode::ValueWriteError) -> Self {
+        DiskError::other(e)
+    }
+}
+
+impl From<rmp::decode::ValueReadError> for DiskError {
+    fn from(e: rmp::decode::ValueReadError) -> Self {
+        DiskError::other(e)
+    }
+}
+
+impl From<std::string::FromUtf8Error> for DiskError {
+    fn from(e: std::string::FromUtf8Error) -> Self {
+        DiskError::other(e)
+    }
+}
+
+impl From<rmp::decode::NumValueReadError> for DiskError {
+    fn from(e: rmp::decode::NumValueReadError) -> Self {
+        DiskError::other(e)
+    }
+}
+
+impl From<tokio::task::JoinError> for DiskError {
+    fn from(e: tokio::task::JoinError) -> Self {
+        DiskError::other(e)
+    }
+}
+
+impl Clone for DiskError {
+    fn clone(&self) -> Self {
+        match self {
+            DiskError::Io(io_error) => DiskError::Io(std::io::Error::new(io_error.kind(), io_error.to_string())),
+            DiskError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
+            DiskError::Unexpected => DiskError::Unexpected,
+            DiskError::CorruptedFormat => DiskError::CorruptedFormat,
+            DiskError::CorruptedBackend => DiskError::CorruptedBackend,
+            DiskError::UnformattedDisk => DiskError::UnformattedDisk,
+            DiskError::InconsistentDisk => DiskError::InconsistentDisk,
+            DiskError::UnsupportedDisk => DiskError::UnsupportedDisk,
+            DiskError::DiskFull => DiskError::DiskFull,
+            DiskError::DiskNotDir => DiskError::DiskNotDir,
+            DiskError::DiskNotFound => DiskError::DiskNotFound,
+            DiskError::DiskOngoingReq => DiskError::DiskOngoingReq,
+            DiskError::DriveIsRoot => DiskError::DriveIsRoot,
+            DiskError::FaultyRemoteDisk => DiskError::FaultyRemoteDisk,
+            DiskError::FaultyDisk => DiskError::FaultyDisk,
+            DiskError::DiskAccessDenied => DiskError::DiskAccessDenied,
+            DiskError::FileNotFound => DiskError::FileNotFound,
+            DiskError::FileVersionNotFound => DiskError::FileVersionNotFound,
+            DiskError::TooManyOpenFiles => DiskError::TooManyOpenFiles,
+            DiskError::FileNameTooLong => DiskError::FileNameTooLong,
+            DiskError::VolumeExists => DiskError::VolumeExists,
+            DiskError::IsNotRegular => DiskError::IsNotRegular,
+            DiskError::PathNotFound => DiskError::PathNotFound,
+            DiskError::VolumeNotFound => DiskError::VolumeNotFound,
+            DiskError::VolumeNotEmpty => DiskError::VolumeNotEmpty,
+            DiskError::VolumeAccessDenied => DiskError::VolumeAccessDenied,
+            DiskError::FileAccessDenied => DiskError::FileAccessDenied,
+            DiskError::FileCorrupt => DiskError::FileCorrupt,
+            DiskError::BitrotHashAlgoInvalid => DiskError::BitrotHashAlgoInvalid,
+            DiskError::CrossDeviceLink => DiskError::CrossDeviceLink,
+            DiskError::LessData => DiskError::LessData,
+            DiskError::MoreData => DiskError::MoreData,
+            DiskError::OutdatedXLMeta => DiskError::OutdatedXLMeta,
+            DiskError::PartMissingOrCorrupt => DiskError::PartMissingOrCorrupt,
+            DiskError::NoHealRequired => DiskError::NoHealRequired,
+            DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
+            DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
+            DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
+            DiskError::ShortWrite => DiskError::ShortWrite,
+        }
+    }
+}
+
+impl DiskError {
+    pub fn to_u32(&self) -> u32 {
+        match self {
+            DiskError::MaxVersionsExceeded => 0x01,
+            DiskError::Unexpected => 0x02,
+            DiskError::CorruptedFormat => 0x03,
+            DiskError::CorruptedBackend => 0x04,
+            DiskError::UnformattedDisk => 0x05,
+            DiskError::InconsistentDisk => 0x06,
+            DiskError::UnsupportedDisk => 0x07,
+            DiskError::DiskFull => 0x08,
+            DiskError::DiskNotDir => 0x09,
+            DiskError::DiskNotFound => 0x0A,
+            DiskError::DiskOngoingReq => 0x0B,
+            DiskError::DriveIsRoot => 0x0C,
+            DiskError::FaultyRemoteDisk => 0x0D,
+            DiskError::FaultyDisk => 0x0E,
+            DiskError::DiskAccessDenied => 0x0F,
+            DiskError::FileNotFound => 0x10,
+            DiskError::FileVersionNotFound => 0x11,
+            DiskError::TooManyOpenFiles => 0x12,
+            DiskError::FileNameTooLong => 0x13,
+            DiskError::VolumeExists => 0x14,
+            DiskError::IsNotRegular => 0x15,
+            DiskError::PathNotFound => 0x16,
+            DiskError::VolumeNotFound => 0x17,
+            DiskError::VolumeNotEmpty => 0x18,
+            DiskError::VolumeAccessDenied => 0x19,
+            DiskError::FileAccessDenied => 0x1A,
+            DiskError::FileCorrupt => 0x1B,
+            DiskError::BitrotHashAlgoInvalid => 0x1C,
+            DiskError::CrossDeviceLink => 0x1D,
+            DiskError::LessData => 0x1E,
+            DiskError::MoreData => 0x1F,
+            DiskError::OutdatedXLMeta => 0x20,
+            DiskError::PartMissingOrCorrupt => 0x21,
+            DiskError::NoHealRequired => 0x22,
+            DiskError::MethodNotAllowed => 0x23,
+            DiskError::Io(_) => 0x24,
+            DiskError::ErasureWriteQuorum => 0x25,
+            DiskError::ErasureReadQuorum => 0x26,
+            DiskError::ShortWrite => 0x27,
+        }
+    }
+
+    pub fn from_u32(error: u32) -> Option<Self> {
+        match error {
+            0x01 => Some(DiskError::MaxVersionsExceeded),
+            0x02 => Some(DiskError::Unexpected),
+            0x03 => Some(DiskError::CorruptedFormat),
+            0x04 => Some(DiskError::CorruptedBackend),
+            0x05 => Some(DiskError::UnformattedDisk),
+            0x06 => Some(DiskError::InconsistentDisk),
+            0x07 => Some(DiskError::UnsupportedDisk),
+            0x08 => Some(DiskError::DiskFull),
+            0x09 => Some(DiskError::DiskNotDir),
+            0x0A => Some(DiskError::DiskNotFound),
+            0x0B => Some(DiskError::DiskOngoingReq),
+            0x0C => Some(DiskError::DriveIsRoot),
+            0x0D => Some(DiskError::FaultyRemoteDisk),
+            0x0E => Some(DiskError::FaultyDisk),
+            0x0F => Some(DiskError::DiskAccessDenied),
+            0x10 => Some(DiskError::FileNotFound),
+            0x11 => Some(DiskError::FileVersionNotFound),
+            0x12 => Some(DiskError::TooManyOpenFiles),
+            0x13 => Some(DiskError::FileNameTooLong),
+            0x14 => Some(DiskError::VolumeExists),
+            0x15 => Some(DiskError::IsNotRegular),
+            0x16 => Some(DiskError::PathNotFound),
+            0x17 => Some(DiskError::VolumeNotFound),
+            0x18 => Some(DiskError::VolumeNotEmpty),
+            0x19 => Some(DiskError::VolumeAccessDenied),
+            0x1A => Some(DiskError::FileAccessDenied),
+            0x1B => Some(DiskError::FileCorrupt),
+            0x1C => Some(DiskError::BitrotHashAlgoInvalid),
+            0x1D => Some(DiskError::CrossDeviceLink),
+            0x1E => Some(DiskError::LessData),
+            0x1F => Some(DiskError::MoreData),
+            0x20 => Some(DiskError::OutdatedXLMeta),
+            0x21 => Some(DiskError::PartMissingOrCorrupt),
+            0x22 => Some(DiskError::NoHealRequired),
+            0x23 => Some(DiskError::MethodNotAllowed),
+            0x24 => Some(DiskError::Io(std::io::Error::other(String::new()))),
+            0x25 => Some(DiskError::ErasureWriteQuorum),
+            0x26 => Some(DiskError::ErasureReadQuorum),
+            0x27 => Some(DiskError::ShortWrite),
+            _ => None,
+        }
+    }
+}
+
+impl PartialEq for DiskError {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (DiskError::Io(e1), DiskError::Io(e2)) => e1.kind() == e2.kind() && e1.to_string() == e2.to_string(),
+            _ => self.to_u32() == other.to_u32(),
+        }
+    }
+}
+
+impl Eq for DiskError {}
+
+impl Hash for DiskError {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.to_u32().hash(state);
+    }
+}
+
+// NOTE: Remove commented out code later if not needed
+// Some error-related helper functions and complex error handling logic
+// is currently commented out to avoid complexity. These can be re-enabled
+// when needed for specific disk quorum checking and error aggregation logic.
+
+/// Bitrot errors
+#[derive(Debug, thiserror::Error)]
+pub enum BitrotErrorType {
+    #[error("bitrot checksum verification failed")]
+    BitrotChecksumMismatch { expected: String, got: String },
+}
+
+impl From<BitrotErrorType> for DiskError {
+    fn from(e: BitrotErrorType) -> Self {
+        DiskError::other(e)
+    }
+}
+
+/// Context wrapper for file access errors
+#[derive(Debug, thiserror::Error)]
+pub struct FileAccessDeniedWithContext {
+    pub path: PathBuf,
+    #[source]
+    pub source: io::Error,
+}
+
+impl std::fmt::Display for FileAccessDeniedWithContext {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "file access denied for path: {}", self.path.display())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::HashMap;
+
+    #[test]
+    fn test_disk_error_variants() {
+        let errors = vec![
+            DiskError::MaxVersionsExceeded,
+            DiskError::Unexpected,
+            DiskError::CorruptedFormat,
+            DiskError::CorruptedBackend,
+            DiskError::UnformattedDisk,
+            DiskError::InconsistentDisk,
+            DiskError::UnsupportedDisk,
+            DiskError::DiskFull,
+            DiskError::DiskNotDir,
+            DiskError::DiskNotFound,
+            DiskError::DiskOngoingReq,
+            DiskError::DriveIsRoot,
+            DiskError::FaultyRemoteDisk,
+            DiskError::FaultyDisk,
+            DiskError::DiskAccessDenied,
+            DiskError::FileNotFound,
+            DiskError::FileVersionNotFound,
+            DiskError::TooManyOpenFiles,
+            DiskError::FileNameTooLong,
+            DiskError::VolumeExists,
+            DiskError::IsNotRegular,
+            DiskError::PathNotFound,
+            DiskError::VolumeNotFound,
+            DiskError::VolumeNotEmpty,
+            DiskError::VolumeAccessDenied,
+            DiskError::FileAccessDenied,
+            DiskError::FileCorrupt,
+            DiskError::ShortWrite,
+            DiskError::BitrotHashAlgoInvalid,
+            DiskError::CrossDeviceLink,
+            DiskError::LessData,
+            DiskError::MoreData,
+            DiskError::OutdatedXLMeta,
+            DiskError::PartMissingOrCorrupt,
+            DiskError::NoHealRequired,
+            DiskError::MethodNotAllowed,
+            DiskError::ErasureWriteQuorum,
+            DiskError::ErasureReadQuorum,
+        ];
+
+        for error in errors {
+            // Test error display
+            assert!(!error.to_string().is_empty());
+
+            // Test error conversion to u32 and back
+            let code = error.to_u32();
+            let converted_back = DiskError::from_u32(code);
+            assert!(converted_back.is_some());
+        }
+    }
+
+    #[test]
+    fn test_disk_error_other() {
+        let custom_error = DiskError::other("custom error message");
+        assert!(matches!(custom_error, DiskError::Io(_)));
+        // The error message format might vary, so just check it's not empty
+        assert!(!custom_error.to_string().is_empty());
+    }
+
+    #[test]
+    fn test_disk_error_from_io_error() {
+        let io_error = std::io::Error::new(std::io::ErrorKind::NotFound, "file not found");
+        let disk_error = DiskError::from(io_error);
+        assert!(matches!(disk_error, DiskError::Io(_)));
+    }
+
+    #[test]
+    fn test_is_all_not_found() {
+        // Empty slice
+        assert!(!DiskError::is_all_not_found(&[]));
+
+        // All file not found
+        let all_not_found = vec![
+            Some(DiskError::FileNotFound),
+            Some(DiskError::FileVersionNotFound),
+            Some(DiskError::FileNotFound),
+        ];
+        assert!(DiskError::is_all_not_found(&all_not_found));
+
+        // Mixed errors
+        let mixed_errors = vec![
+            Some(DiskError::FileNotFound),
+            Some(DiskError::DiskNotFound),
+            Some(DiskError::FileNotFound),
+        ];
+        assert!(!DiskError::is_all_not_found(&mixed_errors));
+
+        // Contains None
+        let with_none = vec![Some(DiskError::FileNotFound), None, Some(DiskError::FileNotFound)];
+        assert!(!DiskError::is_all_not_found(&with_none));
+    }
+
+    #[test]
+    fn test_is_err_object_not_found() {
+        assert!(DiskError::is_err_object_not_found(&DiskError::FileNotFound));
+        assert!(DiskError::is_err_object_not_found(&DiskError::VolumeNotFound));
+        assert!(!DiskError::is_err_object_not_found(&DiskError::DiskNotFound));
+        assert!(!DiskError::is_err_object_not_found(&DiskError::FileCorrupt));
+    }
+
+    #[test]
+    fn test_is_err_version_not_found() {
+        assert!(DiskError::is_err_version_not_found(&DiskError::FileVersionNotFound));
+        assert!(!DiskError::is_err_version_not_found(&DiskError::FileNotFound));
+        assert!(!DiskError::is_err_version_not_found(&DiskError::VolumeNotFound));
+    }
+
+    #[test]
+    fn test_disk_error_to_u32_from_u32() {
+        let test_cases = vec![
+            (DiskError::MaxVersionsExceeded, 1),
+            (DiskError::Unexpected, 2),
+            (DiskError::CorruptedFormat, 3),
+            (DiskError::UnformattedDisk, 5),
+            (DiskError::DiskNotFound, 10),
+            (DiskError::FileNotFound, 16),
+            (DiskError::VolumeNotFound, 23),
+        ];
+
+        for (error, expected_code) in test_cases {
+            assert_eq!(error.to_u32(), expected_code);
+            assert_eq!(DiskError::from_u32(expected_code), Some(error));
+        }
+
+        // Test unknown error code
+        assert_eq!(DiskError::from_u32(999), None);
+    }
+
+    #[test]
+    fn test_disk_error_equality() {
+        assert_eq!(DiskError::FileNotFound, DiskError::FileNotFound);
+        assert_ne!(DiskError::FileNotFound, DiskError::VolumeNotFound);
+
+        let error1 = DiskError::other("test");
+        let error2 = DiskError::other("test");
+        // IO errors with the same message should be equal
+        assert_eq!(error1, error2);
+    }
+
+    #[test]
+    fn test_disk_error_clone() {
+        let original = DiskError::FileNotFound;
+        let cloned = original.clone();
+        assert_eq!(original, cloned);
+
+        let io_error = DiskError::other("test error");
+        let cloned_io = io_error.clone();
+        assert_eq!(io_error, cloned_io);
+    }
+
+    #[test]
+    fn test_disk_error_hash() {
+        let mut map = HashMap::new();
+        map.insert(DiskError::FileNotFound, "file not found");
+        map.insert(DiskError::VolumeNotFound, "volume not found");
+
+        assert_eq!(map.get(&DiskError::FileNotFound), Some(&"file not found"));
+        assert_eq!(map.get(&DiskError::VolumeNotFound), Some(&"volume not found"));
+        assert_eq!(map.get(&DiskError::DiskNotFound), None);
+    }
+
+    #[test]
+    fn test_error_conversions() {
+        // Test From implementations
+        let io_error = std::io::Error::new(std::io::ErrorKind::NotFound, "test");
+        let _disk_error: DiskError = io_error.into();
+
+        let json_str = r#"{"invalid": json}"#; // Invalid JSON
+        let json_error = serde_json::from_str::<serde_json::Value>(json_str).unwrap_err();
+        let _disk_error: DiskError = json_error.into();
+    }
+
+    #[test]
+    fn test_bitrot_error_type() {
+        let bitrot_error = BitrotErrorType::BitrotChecksumMismatch {
+            expected: "abc123".to_string(),
+            got: "def456".to_string(),
+        };
+
+        assert!(bitrot_error.to_string().contains("bitrot checksum verification failed"));
+
+        let disk_error: DiskError = bitrot_error.into();
+        assert!(matches!(disk_error, DiskError::Io(_)));
+    }
+
+    #[test]
+    fn test_file_access_denied_with_context() {
+        let path = PathBuf::from("/test/path");
+        let io_error = std::io::Error::new(std::io::ErrorKind::PermissionDenied, "permission denied");
+
+        let context_error = FileAccessDeniedWithContext {
+            path: path.clone(),
+            source: io_error,
+        };
+
+        let display_str = format!("{context_error}");
+        assert!(display_str.contains("/test/path"));
+        assert!(display_str.contains("file access denied"));
+    }
+
+    #[test]
+    fn test_error_debug_format() {
+        let error = DiskError::FileNotFound;
+        let debug_str = format!("{error:?}");
+        assert_eq!(debug_str, "FileNotFound");
+
+        let io_error = DiskError::other("test error");
+        let debug_str = format!("{io_error:?}");
+        assert!(debug_str.contains("Io"));
+    }
+
+    #[test]
+    fn test_error_source() {
+        use std::error::Error;
+
+        let io_error = std::io::Error::new(std::io::ErrorKind::NotFound, "test");
+        let disk_error = DiskError::Io(io_error);
+
+        // DiskError should have a source
+        if let DiskError::Io(ref inner) = disk_error {
+            assert!(inner.source().is_none()); // std::io::Error typically doesn't have a source
+        }
+    }
+
+    #[test]
+    fn test_io_error_roundtrip_conversion() {
+        // Test DiskError -> std::io::Error -> DiskError roundtrip
+        let original_disk_errors = vec![
+            DiskError::FileNotFound,
+            DiskError::VolumeNotFound,
+            DiskError::DiskFull,
+            DiskError::FileCorrupt,
+            DiskError::MethodNotAllowed,
+        ];
+
+        for original_error in original_disk_errors {
+            // Convert to io::Error and back
+            let io_error: std::io::Error = original_error.clone().into();
+            let recovered_error: DiskError = io_error.into();
+
+            // For non-Io variants, they become Io(ErrorKind::Other) and then back to the original
+            match &original_error {
+                DiskError::Io(_) => {
+                    // Io errors should maintain their kind
+                    assert!(matches!(recovered_error, DiskError::Io(_)));
+                }
+                _ => {
+                    // Other errors become Io(Other) and then are recovered via downcast
+                    // The recovered error should be functionally equivalent
+                    assert_eq!(original_error.to_u32(), recovered_error.to_u32());
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn test_io_error_with_disk_error_inside() {
+        // Test that io::Error containing DiskError can be properly converted back
+        let original_disk_error = DiskError::FileNotFound;
+        let io_with_disk_error = std::io::Error::other(original_disk_error.clone());
+
+        // Convert io::Error back to DiskError
+        let recovered_disk_error: DiskError = io_with_disk_error.into();
+        assert_eq!(original_disk_error, recovered_disk_error);
+    }
+
+    #[test]
+    fn test_io_error_different_kinds() {
+        use std::io::ErrorKind;
+
+        let test_cases = vec![
+            (ErrorKind::NotFound, "file not found"),
+            (ErrorKind::PermissionDenied, "permission denied"),
+            (ErrorKind::ConnectionRefused, "connection refused"),
+            (ErrorKind::TimedOut, "timed out"),
+            (ErrorKind::InvalidInput, "invalid input"),
+        ];
+
+        for (kind, message) in test_cases {
+            let io_error = std::io::Error::new(kind, message);
+            let disk_error: DiskError = io_error.into();
+
+            // Should become DiskError::Io with the same kind and message
+            match disk_error {
+                DiskError::Io(inner_io) => {
+                    assert_eq!(inner_io.kind(), kind);
+                    assert!(inner_io.to_string().contains(message));
+                }
+                _ => panic!("Expected DiskError::Io variant"),
+            }
+        }
+    }
+
+    #[test]
+    fn test_disk_error_to_io_error_preserves_information() {
+        let test_cases = vec![
+            DiskError::FileNotFound,
+            DiskError::VolumeNotFound,
+            DiskError::DiskFull,
+            DiskError::FileCorrupt,
+            DiskError::MethodNotAllowed,
+            DiskError::ErasureReadQuorum,
+            DiskError::ErasureWriteQuorum,
+        ];
+
+        for disk_error in test_cases {
+            let io_error: std::io::Error = disk_error.clone().into();
+
+            // Error message should be preserved
+            assert!(io_error.to_string().contains(&disk_error.to_string()));
+
+            // Should be able to downcast back to DiskError
+            let recovered_error = io_error.downcast::<DiskError>();
+            assert!(recovered_error.is_ok());
+            assert_eq!(recovered_error.unwrap(), disk_error);
+        }
+    }
+
+    #[test]
+    fn test_io_error_downcast_chain() {
+        // Test nested error downcasting chain
+        let original_disk_error = DiskError::FileNotFound;
+
+        // Create a chain: DiskError -> io::Error -> DiskError -> io::Error
+        let io_error1: std::io::Error = original_disk_error.clone().into();
+        let disk_error2: DiskError = io_error1.into();
+        let io_error2: std::io::Error = disk_error2.into();
+
+        // Final io::Error should still contain the original DiskError
+        let final_disk_error = io_error2.downcast::<DiskError>();
+        assert!(final_disk_error.is_ok());
+        assert_eq!(final_disk_error.unwrap(), original_disk_error);
+    }
+
+    #[test]
+    fn test_io_error_with_original_io_content() {
+        // Test DiskError::Io variant preserves original io::Error
+        let original_io = std::io::Error::new(std::io::ErrorKind::BrokenPipe, "broken pipe");
+        let disk_error = DiskError::Io(original_io);
+
+        let converted_io: std::io::Error = disk_error.into();
+        assert_eq!(converted_io.kind(), std::io::ErrorKind::BrokenPipe);
+        assert!(converted_io.to_string().contains("broken pipe"));
+    }
+
+    #[test]
+    fn test_error_display_preservation() {
+        let disk_errors = vec![
+            DiskError::MaxVersionsExceeded,
+            DiskError::CorruptedFormat,
+            DiskError::UnformattedDisk,
+            DiskError::DiskNotFound,
+            DiskError::FileAccessDenied,
+        ];
+
+        for disk_error in disk_errors {
+            let original_message = disk_error.to_string();
+            let io_error: std::io::Error = disk_error.clone().into();
+
+            // The io::Error should contain the original error message
+            assert!(io_error.to_string().contains(&original_message));
+        }
+    }
+}

crates/ecstore/src/disk/error_conv.rs

@@ -0,0 +1,449 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::error::DiskError;
+
+pub fn to_file_error(io_err: std::io::Error) -> std::io::Error {
+    match io_err.kind() {
+        std::io::ErrorKind::NotFound => DiskError::FileNotFound.into(),
+        std::io::ErrorKind::PermissionDenied => DiskError::FileAccessDenied.into(),
+        std::io::ErrorKind::IsADirectory => DiskError::IsNotRegular.into(),
+        std::io::ErrorKind::NotADirectory => DiskError::FileAccessDenied.into(),
+        std::io::ErrorKind::DirectoryNotEmpty => DiskError::FileAccessDenied.into(),
+        std::io::ErrorKind::UnexpectedEof => DiskError::FaultyDisk.into(),
+        std::io::ErrorKind::TooManyLinks => DiskError::TooManyOpenFiles.into(),
+        std::io::ErrorKind::InvalidInput => DiskError::FileNotFound.into(),
+        std::io::ErrorKind::InvalidData => DiskError::FileCorrupt.into(),
+        std::io::ErrorKind::StorageFull => DiskError::DiskFull.into(),
+        _ => io_err,
+    }
+}
+
+pub fn to_volume_error(io_err: std::io::Error) -> std::io::Error {
+    match io_err.kind() {
+        std::io::ErrorKind::NotFound => DiskError::VolumeNotFound.into(),
+        std::io::ErrorKind::PermissionDenied => DiskError::DiskAccessDenied.into(),
+        std::io::ErrorKind::DirectoryNotEmpty => DiskError::VolumeNotEmpty.into(),
+        std::io::ErrorKind::NotADirectory => DiskError::IsNotRegular.into(),
+        std::io::ErrorKind::Other => match io_err.downcast::<DiskError>() {
+            Ok(err) => match err {
+                DiskError::FileNotFound => DiskError::VolumeNotFound.into(),
+                DiskError::FileAccessDenied => DiskError::DiskAccessDenied.into(),
+                err => err.into(),
+            },
+            Err(err) => to_file_error(err),
+        },
+        _ => to_file_error(io_err),
+    }
+}
+
+pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error {
+    match io_err.kind() {
+        std::io::ErrorKind::NotFound => DiskError::DiskNotFound.into(),
+        std::io::ErrorKind::PermissionDenied => DiskError::DiskAccessDenied.into(),
+        std::io::ErrorKind::Other => match io_err.downcast::<DiskError>() {
+            Ok(err) => match err {
+                DiskError::FileNotFound => DiskError::DiskNotFound.into(),
+                DiskError::VolumeNotFound => DiskError::DiskNotFound.into(),
+                DiskError::FileAccessDenied => DiskError::DiskAccessDenied.into(),
+                DiskError::VolumeAccessDenied => DiskError::DiskAccessDenied.into(),
+                err => err.into(),
+            },
+            Err(err) => to_volume_error(err),
+        },
+        _ => to_volume_error(io_err),
+    }
+}
+
+// only errors from FileSystem operations
+pub fn to_access_error(io_err: std::io::Error, per_err: DiskError) -> std::io::Error {
+    match io_err.kind() {
+        std::io::ErrorKind::PermissionDenied => per_err.into(),
+        std::io::ErrorKind::NotADirectory => per_err.into(),
+        std::io::ErrorKind::NotFound => DiskError::VolumeNotFound.into(),
+        std::io::ErrorKind::UnexpectedEof => DiskError::FaultyDisk.into(),
+        std::io::ErrorKind::Other => match io_err.downcast::<DiskError>() {
+            Ok(err) => match err {
+                DiskError::DiskAccessDenied => per_err.into(),
+                DiskError::FileAccessDenied => per_err.into(),
+                DiskError::FileNotFound => DiskError::VolumeNotFound.into(),
+                err => err.into(),
+            },
+            Err(err) => to_volume_error(err),
+        },
+        _ => to_volume_error(io_err),
+    }
+}
+
+pub fn to_unformatted_disk_error(io_err: std::io::Error) -> std::io::Error {
+    match io_err.kind() {
+        std::io::ErrorKind::NotFound => DiskError::UnformattedDisk.into(),
+        std::io::ErrorKind::PermissionDenied => DiskError::DiskAccessDenied.into(),
+        std::io::ErrorKind::Other => match io_err.downcast::<DiskError>() {
+            Ok(err) => match err {
+                DiskError::FileNotFound => DiskError::UnformattedDisk.into(),
+                DiskError::DiskNotFound => DiskError::UnformattedDisk.into(),
+                DiskError::VolumeNotFound => DiskError::UnformattedDisk.into(),
+                DiskError::FileAccessDenied => DiskError::DiskAccessDenied.into(),
+                DiskError::DiskAccessDenied => DiskError::DiskAccessDenied.into(),
+                _ => DiskError::CorruptedBackend.into(),
+            },
+            Err(_err) => DiskError::CorruptedBackend.into(),
+        },
+        _ => DiskError::CorruptedBackend.into(),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::{Error as IoError, ErrorKind};
+
+    // Helper function to create IO errors with specific kinds
+    fn create_io_error(kind: ErrorKind) -> IoError {
+        IoError::new(kind, "test error")
+    }
+
+    // Helper function to create IO errors with DiskError as the source
+    fn create_io_error_with_disk_error(disk_error: DiskError) -> IoError {
+        IoError::other(disk_error)
+    }
+
+    // Helper function to check if an IoError contains a specific DiskError
+    fn contains_disk_error(io_error: IoError, expected: DiskError) -> bool {
+        if let Ok(disk_error) = io_error.downcast::<DiskError>() {
+            std::mem::discriminant(&disk_error) == std::mem::discriminant(&expected)
+        } else {
+            false
+        }
+    }
+
+    #[test]
+    fn test_to_file_error_basic_conversions() {
+        // Test NotFound -> FileNotFound
+        let result = to_file_error(create_io_error(ErrorKind::NotFound));
+        assert!(contains_disk_error(result, DiskError::FileNotFound));
+
+        // Test PermissionDenied -> FileAccessDenied
+        let result = to_file_error(create_io_error(ErrorKind::PermissionDenied));
+        assert!(contains_disk_error(result, DiskError::FileAccessDenied));
+
+        // Test IsADirectory -> IsNotRegular
+        let result = to_file_error(create_io_error(ErrorKind::IsADirectory));
+        assert!(contains_disk_error(result, DiskError::IsNotRegular));
+
+        // Test NotADirectory -> FileAccessDenied
+        let result = to_file_error(create_io_error(ErrorKind::NotADirectory));
+        assert!(contains_disk_error(result, DiskError::FileAccessDenied));
+
+        // Test DirectoryNotEmpty -> FileAccessDenied
+        let result = to_file_error(create_io_error(ErrorKind::DirectoryNotEmpty));
+        assert!(contains_disk_error(result, DiskError::FileAccessDenied));
+
+        // Test UnexpectedEof -> FaultyDisk
+        let result = to_file_error(create_io_error(ErrorKind::UnexpectedEof));
+        assert!(contains_disk_error(result, DiskError::FaultyDisk));
+
+        // Test TooManyLinks -> TooManyOpenFiles
+        #[cfg(unix)]
+        {
+            let result = to_file_error(create_io_error(ErrorKind::TooManyLinks));
+            assert!(contains_disk_error(result, DiskError::TooManyOpenFiles));
+        }
+
+        // Test InvalidInput -> FileNotFound
+        let result = to_file_error(create_io_error(ErrorKind::InvalidInput));
+        assert!(contains_disk_error(result, DiskError::FileNotFound));
+
+        // Test InvalidData -> FileCorrupt
+        let result = to_file_error(create_io_error(ErrorKind::InvalidData));
+        assert!(contains_disk_error(result, DiskError::FileCorrupt));
+
+        // Test StorageFull -> DiskFull
+        #[cfg(unix)]
+        {
+            let result = to_file_error(create_io_error(ErrorKind::StorageFull));
+            assert!(contains_disk_error(result, DiskError::DiskFull));
+        }
+    }
+
+    #[test]
+    fn test_to_file_error_passthrough_unknown() {
+        // Test that unknown error kinds are passed through unchanged
+        let original = create_io_error(ErrorKind::Interrupted);
+        let result = to_file_error(original);
+        assert_eq!(result.kind(), ErrorKind::Interrupted);
+    }
+
+    #[test]
+    fn test_to_volume_error_basic_conversions() {
+        // Test NotFound -> VolumeNotFound
+        let result = to_volume_error(create_io_error(ErrorKind::NotFound));
+        assert!(contains_disk_error(result, DiskError::VolumeNotFound));
+
+        // Test PermissionDenied -> DiskAccessDenied
+        let result = to_volume_error(create_io_error(ErrorKind::PermissionDenied));
+        assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
+
+        // Test DirectoryNotEmpty -> VolumeNotEmpty
+        let result = to_volume_error(create_io_error(ErrorKind::DirectoryNotEmpty));
+        assert!(contains_disk_error(result, DiskError::VolumeNotEmpty));
+
+        // Test NotADirectory -> IsNotRegular
+        let result = to_volume_error(create_io_error(ErrorKind::NotADirectory));
+        assert!(contains_disk_error(result, DiskError::IsNotRegular));
+    }
+
+    #[test]
+    fn test_to_volume_error_other_with_disk_error() {
+        // Test Other error kind with FileNotFound DiskError -> VolumeNotFound
+        let io_error = create_io_error_with_disk_error(DiskError::FileNotFound);
+        let result = to_volume_error(io_error);
+        assert!(contains_disk_error(result, DiskError::VolumeNotFound));
+
+        // Test Other error kind with FileAccessDenied DiskError -> DiskAccessDenied
+        let io_error = create_io_error_with_disk_error(DiskError::FileAccessDenied);
+        let result = to_volume_error(io_error);
+        assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
+
+        // Test Other error kind with other DiskError -> passthrough
+        let io_error = create_io_error_with_disk_error(DiskError::DiskFull);
+        let result = to_volume_error(io_error);
+        assert!(contains_disk_error(result, DiskError::DiskFull));
+    }
+
+    #[test]
+    fn test_to_volume_error_fallback_to_file_error() {
+        // Test fallback to to_file_error for unknown error kinds
+        let result = to_volume_error(create_io_error(ErrorKind::Interrupted));
+        assert_eq!(result.kind(), ErrorKind::Interrupted);
+    }
+
+    #[test]
+    fn test_to_disk_error_basic_conversions() {
+        // Test NotFound -> DiskNotFound
+        let result = to_disk_error(create_io_error(ErrorKind::NotFound));
+        assert!(contains_disk_error(result, DiskError::DiskNotFound));
+
+        // Test PermissionDenied -> DiskAccessDenied
+        let result = to_disk_error(create_io_error(ErrorKind::PermissionDenied));
+        assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
+    }
+
+    #[test]
+    fn test_to_disk_error_other_with_disk_error() {
+        // Test Other error kind with FileNotFound DiskError -> DiskNotFound
+        let io_error = create_io_error_with_disk_error(DiskError::FileNotFound);
+        let result = to_disk_error(io_error);
+        assert!(contains_disk_error(result, DiskError::DiskNotFound));
+
+        // Test Other error kind with VolumeNotFound DiskError -> DiskNotFound
+        let io_error = create_io_error_with_disk_error(DiskError::VolumeNotFound);
+        let result = to_disk_error(io_error);
+        assert!(contains_disk_error(result, DiskError::DiskNotFound));
+
+        // Test Other error kind with FileAccessDenied DiskError -> DiskAccessDenied
+        let io_error = create_io_error_with_disk_error(DiskError::FileAccessDenied);
+        let result = to_disk_error(io_error);
+        assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
+
+        // Test Other error kind with VolumeAccessDenied DiskError -> DiskAccessDenied
+        let io_error = create_io_error_with_disk_error(DiskError::VolumeAccessDenied);
+        let result = to_disk_error(io_error);
+        assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
+
+        // Test Other error kind with other DiskError -> passthrough
+        let io_error = create_io_error_with_disk_error(DiskError::DiskFull);
+        let result = to_disk_error(io_error);
+        assert!(contains_disk_error(result, DiskError::DiskFull));
+    }
+
+    #[test]
+    fn test_to_disk_error_fallback_to_volume_error() {
+        // Test fallback to to_volume_error for unknown error kinds
+        let result = to_disk_error(create_io_error(ErrorKind::Interrupted));
+        assert_eq!(result.kind(), ErrorKind::Interrupted);
+    }
+
+    #[test]
+    fn test_to_access_error_basic_conversions() {
+        let permission_error = DiskError::FileAccessDenied;
+
+        // Test PermissionDenied -> specified permission error
+        let result = to_access_error(create_io_error(ErrorKind::PermissionDenied), permission_error);
+        assert!(contains_disk_error(result, DiskError::FileAccessDenied));
+
+        // Test NotADirectory -> specified permission error
+        let result = to_access_error(create_io_error(ErrorKind::NotADirectory), DiskError::FileAccessDenied);
+        assert!(contains_disk_error(result, DiskError::FileAccessDenied));
+
+        // Test NotFound -> VolumeNotFound
+        let result = to_access_error(create_io_error(ErrorKind::NotFound), DiskError::FileAccessDenied);
+        assert!(contains_disk_error(result, DiskError::VolumeNotFound));
+
+        // Test UnexpectedEof -> FaultyDisk
+        let result = to_access_error(create_io_error(ErrorKind::UnexpectedEof), DiskError::FileAccessDenied);
+        assert!(contains_disk_error(result, DiskError::FaultyDisk));
+    }
+
+    #[test]
+    fn test_to_access_error_other_with_disk_error() {
+        let permission_error = DiskError::VolumeAccessDenied;
+
+        // Test Other error kind with DiskAccessDenied -> specified permission error
+        let io_error = create_io_error_with_disk_error(DiskError::DiskAccessDenied);
+        let result = to_access_error(io_error, permission_error);
+        assert!(contains_disk_error(result, DiskError::VolumeAccessDenied));
+
+        // Test Other error kind with FileAccessDenied -> specified permission error
+        let io_error = create_io_error_with_disk_error(DiskError::FileAccessDenied);
+        let result = to_access_error(io_error, DiskError::VolumeAccessDenied);
+        assert!(contains_disk_error(result, DiskError::VolumeAccessDenied));
+
+        // Test Other error kind with FileNotFound -> VolumeNotFound
+        let io_error = create_io_error_with_disk_error(DiskError::FileNotFound);
+        let result = to_access_error(io_error, DiskError::VolumeAccessDenied);
+        assert!(contains_disk_error(result, DiskError::VolumeNotFound));
+
+        // Test Other error kind with other DiskError -> passthrough
+        let io_error = create_io_error_with_disk_error(DiskError::DiskFull);
+        let result = to_access_error(io_error, DiskError::VolumeAccessDenied);
+        assert!(contains_disk_error(result, DiskError::DiskFull));
+    }
+
+    #[test]
+    fn test_to_access_error_fallback_to_volume_error() {
+        let permission_error = DiskError::FileAccessDenied;
+
+        // Test fallback to to_volume_error for unknown error kinds
+        let result = to_access_error(create_io_error(ErrorKind::Interrupted), permission_error);
+        assert_eq!(result.kind(), ErrorKind::Interrupted);
+    }
+
+    #[test]
+    fn test_to_unformatted_disk_error_basic_conversions() {
+        // Test NotFound -> UnformattedDisk
+        let result = to_unformatted_disk_error(create_io_error(ErrorKind::NotFound));
+        assert!(contains_disk_error(result, DiskError::UnformattedDisk));
+
+        // Test PermissionDenied -> DiskAccessDenied
+        let result = to_unformatted_disk_error(create_io_error(ErrorKind::PermissionDenied));
+        assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
+    }
+
+    #[test]
+    fn test_to_unformatted_disk_error_other_with_disk_error() {
+        // Test Other error kind with FileNotFound -> UnformattedDisk
+        let io_error = create_io_error_with_disk_error(DiskError::FileNotFound);
+        let result = to_unformatted_disk_error(io_error);
+        assert!(contains_disk_error(result, DiskError::UnformattedDisk));
+
+        // Test Other error kind with DiskNotFound -> UnformattedDisk
+        let io_error = create_io_error_with_disk_error(DiskError::DiskNotFound);
+        let result = to_unformatted_disk_error(io_error);
+        assert!(contains_disk_error(result, DiskError::UnformattedDisk));
+
+        // Test Other error kind with VolumeNotFound -> UnformattedDisk
+        let io_error = create_io_error_with_disk_error(DiskError::VolumeNotFound);
+        let result = to_unformatted_disk_error(io_error);
+        assert!(contains_disk_error(result, DiskError::UnformattedDisk));
+
+        // Test Other error kind with FileAccessDenied -> DiskAccessDenied
+        let io_error = create_io_error_with_disk_error(DiskError::FileAccessDenied);
+        let result = to_unformatted_disk_error(io_error);
+        assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
+
+        // Test Other error kind with DiskAccessDenied -> DiskAccessDenied
+        let io_error = create_io_error_with_disk_error(DiskError::DiskAccessDenied);
+        let result = to_unformatted_disk_error(io_error);
+        assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
+
+        // Test Other error kind with other DiskError -> CorruptedBackend
+        let io_error = create_io_error_with_disk_error(DiskError::DiskFull);
+        let result = to_unformatted_disk_error(io_error);
+        assert!(contains_disk_error(result, DiskError::CorruptedBackend));
+    }
+
+    #[test]
+    fn test_to_unformatted_disk_error_recursive_behavior() {
+        // Test with non-Other error kind that should be handled without infinite recursion
+        let result = to_unformatted_disk_error(create_io_error(ErrorKind::Interrupted));
+        // This should not cause infinite recursion and should produce CorruptedBackend
+        assert!(contains_disk_error(result, DiskError::CorruptedBackend));
+    }
+
+    #[test]
+    fn test_error_chain_conversions() {
+        // Test complex error conversion chains
+        let original_error = create_io_error(ErrorKind::NotFound);
+
+        // Chain: NotFound -> FileNotFound (via to_file_error) -> VolumeNotFound (via to_volume_error)
+        let file_error = to_file_error(original_error);
+        let volume_error = to_volume_error(file_error);
+        assert!(contains_disk_error(volume_error, DiskError::VolumeNotFound));
+    }
+
+    #[test]
+    fn test_cross_platform_error_kinds() {
+        // Test error kinds that may not be available on all platforms
+        #[cfg(unix)]
+        {
+            let result = to_file_error(create_io_error(ErrorKind::TooManyLinks));
+            assert!(contains_disk_error(result, DiskError::TooManyOpenFiles));
+        }
+
+        #[cfg(unix)]
+        {
+            let result = to_file_error(create_io_error(ErrorKind::StorageFull));
+            assert!(contains_disk_error(result, DiskError::DiskFull));
+        }
+    }
+
+    #[test]
+    fn test_error_conversion_with_different_kinds() {
+        // Test multiple error kinds to ensure comprehensive coverage
+        let test_cases = vec![
+            (ErrorKind::NotFound, DiskError::FileNotFound),
+            (ErrorKind::PermissionDenied, DiskError::FileAccessDenied),
+            (ErrorKind::IsADirectory, DiskError::IsNotRegular),
+            (ErrorKind::InvalidData, DiskError::FileCorrupt),
+        ];
+
+        for (kind, expected_disk_error) in test_cases {
+            let result = to_file_error(create_io_error(kind));
+            assert!(
+                contains_disk_error(result, expected_disk_error.clone()),
+                "Failed for ErrorKind::{kind:?} -> DiskError::{expected_disk_error:?}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_volume_error_conversion_chain() {
+        // Test volume error conversion with different input types
+        let test_cases = vec![
+            (ErrorKind::NotFound, DiskError::VolumeNotFound),
+            (ErrorKind::PermissionDenied, DiskError::DiskAccessDenied),
+            (ErrorKind::DirectoryNotEmpty, DiskError::VolumeNotEmpty),
+        ];
+
+        for (kind, expected_disk_error) in test_cases {
+            let result = to_volume_error(create_io_error(kind));
+            assert!(
+                contains_disk_error(result, expected_disk_error.clone()),
+                "Failed for ErrorKind::{kind:?} -> DiskError::{expected_disk_error:?}"
+            );
+        }
+    }
+}

crates/ecstore/src/disk/error_reduce.rs

@@ -0,0 +1,176 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::error::Error;
+
+pub static OBJECT_OP_IGNORED_ERRS: &[Error] = &[
+    Error::DiskNotFound,
+    Error::FaultyDisk,
+    Error::FaultyRemoteDisk,
+    Error::DiskAccessDenied,
+    Error::DiskOngoingReq,
+    Error::UnformattedDisk,
+];
+
+pub static BUCKET_OP_IGNORED_ERRS: &[Error] = &[
+    Error::DiskNotFound,
+    Error::FaultyDisk,
+    Error::FaultyRemoteDisk,
+    Error::DiskAccessDenied,
+    Error::UnformattedDisk,
+];
+
+pub static BASE_IGNORED_ERRS: &[Error] = &[Error::DiskNotFound, Error::FaultyDisk, Error::FaultyRemoteDisk];
+
+pub fn reduce_write_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error], quorun: usize) -> Option<Error> {
+    reduce_quorum_errs(errors, ignored_errs, quorun, Error::ErasureWriteQuorum)
+}
+
+pub fn reduce_read_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error], quorun: usize) -> Option<Error> {
+    reduce_quorum_errs(errors, ignored_errs, quorun, Error::ErasureReadQuorum)
+}
+
+pub fn reduce_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error], quorun: usize, quorun_err: Error) -> Option<Error> {
+    let (max_count, err) = reduce_errs(errors, ignored_errs);
+    if max_count >= quorun { err } else { Some(quorun_err) }
+}
+
+pub fn reduce_errs(errors: &[Option<Error>], ignored_errs: &[Error]) -> (usize, Option<Error>) {
+    let nil_error = Error::other("nil".to_string());
+
+    // 首先统计 None 的数量（作为 nil 错误）
+    let nil_count = errors.iter().filter(|e| e.is_none()).count();
+
+    let err_counts = errors
+        .iter()
+        .filter_map(|e| e.as_ref()) // 只处理 Some 的错误
+        .fold(std::collections::HashMap::new(), |mut acc, e| {
+            if is_ignored_err(ignored_errs, e) {
+                return acc;
+            }
+            *acc.entry(e.clone()).or_insert(0) += 1;
+            acc
+        });
+
+    // 找到最高频率的非 nil 错误
+    let (best_err, best_count) = err_counts
+        .into_iter()
+        .max_by(|(_, c1), (_, c2)| c1.cmp(c2))
+        .unwrap_or((nil_error.clone(), 0));
+
+    // 比较 nil 错误和最高频率的非 nil 错误, 优先选择 nil 错误
+    if nil_count > best_count || (nil_count == best_count && nil_count > 0) {
+        (nil_count, None)
+    } else {
+        (best_count, Some(best_err))
+    }
+}
+
+pub fn is_ignored_err(ignored_errs: &[Error], err: &Error) -> bool {
+    ignored_errs.iter().any(|e| e == err)
+}
+
+pub fn count_errs(errors: &[Option<Error>], err: &Error) -> usize {
+    errors.iter().filter(|&e| e.as_ref() == Some(err)).count()
+}
+
+pub fn is_all_buckets_not_found(errs: &[Option<Error>]) -> bool {
+    for err in errs.iter() {
+        if let Some(err) = err {
+            if err == &Error::DiskNotFound || err == &Error::VolumeNotFound {
+                continue;
+            }
+
+            return false;
+        }
+        return false;
+    }
+
+    !errs.is_empty()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn err_io(msg: &str) -> Error {
+        Error::Io(std::io::Error::other(msg))
+    }
+
+    #[test]
+    fn test_reduce_errs_basic() {
+        let e1 = err_io("a");
+        let e2 = err_io("b");
+        let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None];
+        let ignored = vec![];
+        let (count, err) = reduce_errs(&errors, &ignored);
+        assert_eq!(count, 2);
+        assert_eq!(err, Some(e1));
+    }
+
+    #[test]
+    fn test_reduce_errs_ignored() {
+        let e1 = err_io("a");
+        let e2 = err_io("b");
+        let errors = vec![Some(e1.clone()), Some(e2.clone()), Some(e1.clone()), Some(e2.clone()), None];
+        let ignored = vec![e2.clone()];
+        let (count, err) = reduce_errs(&errors, &ignored);
+        assert_eq!(count, 2);
+        assert_eq!(err, Some(e1));
+    }
+
+    #[test]
+    fn test_reduce_quorum_errs() {
+        let e1 = err_io("a");
+        let e2 = err_io("b");
+        let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None];
+        let ignored = vec![];
+        let quorum_err = Error::FaultyDisk;
+        // quorum = 2, should return e1
+        let res = reduce_quorum_errs(&errors, &ignored, 2, quorum_err.clone());
+        assert_eq!(res, Some(e1));
+        // quorum = 3, should return quorum error
+        let res = reduce_quorum_errs(&errors, &ignored, 3, quorum_err.clone());
+        assert_eq!(res, Some(quorum_err));
+    }
+
+    #[test]
+    fn test_count_errs() {
+        let e1 = err_io("a");
+        let e2 = err_io("b");
+        let errors = vec![Some(e1.clone()), Some(e2.clone()), Some(e1.clone()), None];
+        assert_eq!(count_errs(&errors, &e1), 2);
+        assert_eq!(count_errs(&errors, &e2), 1);
+    }
+
+    #[test]
+    fn test_is_ignored_err() {
+        let e1 = err_io("a");
+        let e2 = err_io("b");
+        let ignored = vec![e1.clone()];
+        assert!(is_ignored_err(&ignored, &e1));
+        assert!(!is_ignored_err(&ignored, &e2));
+    }
+
+    #[test]
+    fn test_reduce_errs_nil_tiebreak() {
+        // Error::Nil and another error have the same count, should prefer Nil
+        let e1 = err_io("a");
+        let errors = vec![Some(e1.clone()), None, Some(e1.clone()), None]; // e1:2, Nil:2
+        let ignored = vec![];
+        let (count, err) = reduce_errs(&errors, &ignored);
+        assert_eq!(count, 2);
+        assert_eq!(err, None); // None means Error::Nil is preferred
+    }
+}

crates/ecstore/src/disk/format.rs

@@ -0,0 +1,546 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::error::{Error, Result};
+use super::{DiskInfo, error::DiskError};
+use serde::{Deserialize, Serialize};
+use serde_json::Error as JsonError;
+use uuid::Uuid;
+
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub enum FormatMetaVersion {
+    #[serde(rename = "1")]
+    V1,
+
+    #[serde(other)]
+    Unknown,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub enum FormatBackend {
+    #[serde(rename = "xl")]
+    Erasure,
+    #[serde(rename = "xl-single")]
+    ErasureSingle,
+
+    #[serde(other)]
+    Unknown,
+}
+
+/// Represents the V3 backend disk structure version
+/// under `.rustfs.sys` and actual data namespace.
+///
+/// FormatErasureV3 - structure holds format config version '3'.
+///
+/// The V3 format to support "large bucket" support where a bucket
+/// can span multiple erasure sets.
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub struct FormatErasureV3 {
+    /// Version of 'xl' format.
+    pub version: FormatErasureVersion,
+
+    /// This field carries assigned disk uuid.
+    pub this: Uuid,
+
+    /// Sets field carries the input disk order generated the first
+    /// time when fresh disks were supplied, it is a two-dimensional
+    /// array second dimension represents list of disks used per set.
+    pub sets: Vec<Vec<Uuid>>,
+
+    /// Distribution algorithm represents the hashing algorithm
+    /// to pick the right set index for an object.
+    #[serde(rename = "distributionAlgo")]
+    pub distribution_algo: DistributionAlgoVersion,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub enum FormatErasureVersion {
+    #[serde(rename = "1")]
+    V1,
+    #[serde(rename = "2")]
+    V2,
+    #[serde(rename = "3")]
+    V3,
+
+    #[serde(other)]
+    Unknown,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub enum DistributionAlgoVersion {
+    #[serde(rename = "CRCMOD")]
+    V1,
+    #[serde(rename = "SIPMOD")]
+    V2,
+    #[serde(rename = "SIPMOD+PARITY")]
+    V3,
+}
+
+/// format.json currently has the format:
+///
+/// ```json
+/// {
+///   "version": "1",
+///   "format": "XXXXX",
+///   "id": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
+///   "XXXXX": {
+//
+///   }
+/// }
+/// ```
+///
+/// Ideally we will never have a situation where we will have to change the
+/// fields of this struct and deal with related migration.
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub struct FormatV3 {
+    /// Version of the format config.
+    pub version: FormatMetaVersion,
+
+    /// Format indicates the backend format type, supports two values 'xl' and 'xl-single'.
+    pub format: FormatBackend,
+
+    /// ID is the identifier for the rustfs deployment
+    pub id: Uuid,
+
+    #[serde(rename = "xl")]
+    pub erasure: FormatErasureV3,
+    // /// DiskInfo is an extended type which returns current
+    // /// disk usage per path.
+    #[serde(skip)]
+    pub disk_info: Option<DiskInfo>,
+}
+
+impl TryFrom<&[u8]> for FormatV3 {
+    type Error = JsonError;
+
+    fn try_from(data: &[u8]) -> std::result::Result<Self, Self::Error> {
+        serde_json::from_slice(data)
+    }
+}
+
+impl TryFrom<&str> for FormatV3 {
+    type Error = JsonError;
+
+    fn try_from(data: &str) -> std::result::Result<Self, Self::Error> {
+        serde_json::from_str(data)
+    }
+}
+
+impl FormatV3 {
+    /// Create a new format config with the given number of sets and set length.
+    pub fn new(num_sets: usize, set_len: usize) -> Self {
+        let format = if set_len == 1 {
+            FormatBackend::ErasureSingle
+        } else {
+            FormatBackend::Erasure
+        };
+
+        let erasure = FormatErasureV3 {
+            version: FormatErasureVersion::V3,
+            this: Uuid::nil(),
+            sets: (0..num_sets)
+                .map(|_| (0..set_len).map(|_| Uuid::new_v4()).collect())
+                .collect(),
+            distribution_algo: DistributionAlgoVersion::V3,
+        };
+
+        Self {
+            version: FormatMetaVersion::V1,
+            format,
+            id: Uuid::new_v4(),
+            erasure,
+            disk_info: None,
+        }
+    }
+
+    /// Returns the number of drives in the erasure set.
+    pub fn drives(&self) -> usize {
+        self.erasure.sets.iter().map(|v| v.len()).sum()
+    }
+
+    pub fn to_json(&self) -> std::result::Result<String, JsonError> {
+        serde_json::to_string(self)
+    }
+
+    /// returns the i,j'th position of the input `diskID` against the reference
+    ///
+    /// format, after successful validation.
+    ///   - i'th position is the set index
+    ///   - j'th position is the disk index in the current set
+    pub fn find_disk_index_by_disk_id(&self, disk_id: Uuid) -> Result<(usize, usize)> {
+        if disk_id == Uuid::nil() {
+            return Err(Error::from(DiskError::DiskNotFound));
+        }
+        if disk_id == Uuid::max() {
+            return Err(Error::other("disk offline"));
+        }
+
+        for (i, set) in self.erasure.sets.iter().enumerate() {
+            for (j, d) in set.iter().enumerate() {
+                if disk_id.eq(d) {
+                    return Ok((i, j));
+                }
+            }
+        }
+
+        Err(Error::other(format!("disk id not found {disk_id}")))
+    }
+
+    pub fn check_other(&self, other: &FormatV3) -> Result<()> {
+        let mut tmp = other.clone();
+        let this = tmp.erasure.this;
+        tmp.erasure.this = Uuid::nil();
+
+        if self.erasure.sets.len() != other.erasure.sets.len() {
+            return Err(Error::other(format!(
+                "Expected number of sets {}, got {}",
+                self.erasure.sets.len(),
+                other.erasure.sets.len()
+            )));
+        }
+
+        for i in 0..self.erasure.sets.len() {
+            if self.erasure.sets[i].len() != other.erasure.sets[i].len() {
+                return Err(Error::other(format!(
+                    "Each set should be of same size, expected {}, got {}",
+                    self.erasure.sets[i].len(),
+                    other.erasure.sets[i].len()
+                )));
+            }
+
+            for j in 0..self.erasure.sets[i].len() {
+                if self.erasure.sets[i][j] != other.erasure.sets[i][j] {
+                    return Err(Error::other(format!(
+                        "UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)",
+                        i,
+                        j,
+                        self.erasure.sets[i][j].to_string(),
+                        other.erasure.sets[i][j].to_string(),
+                    )));
+                }
+            }
+        }
+
+        for i in 0..tmp.erasure.sets.len() {
+            for j in 0..tmp.erasure.sets[i].len() {
+                if this == tmp.erasure.sets[i][j] {
+                    return Ok(());
+                }
+            }
+        }
+
+        Err(Error::other(format!(
+            "DriveID {:?} not found in any drive sets {:?}",
+            this, other.erasure.sets
+        )))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_format_v1() {
+        let format = FormatV3::new(1, 4);
+
+        let str = serde_json::to_string(&format);
+        println!("{str:?}");
+
+        let data = r#"
+        {
+            "version": "1",
+            "format": "xl",
+            "id": "321b3874-987d-4c15-8fa5-757c956b1243",
+            "xl": {
+                "version": "1",
+                "this": null,
+                "sets": [
+                    [
+                        "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
+                        "c26315da-05cf-4778-a9ea-b44ea09f58c5",
+                        "fb87a891-18d3-44cf-a46f-bcc15093a038",
+                        "356a925c-57b9-4313-88b3-053edf1104dc"
+                    ]
+                ],
+                "distributionAlgo": "CRCMOD"
+            }
+        }"#;
+
+        let p = FormatV3::try_from(data);
+
+        println!("{p:?}");
+    }
+
+    #[test]
+    fn test_format_v3_new_single_disk() {
+        let format = FormatV3::new(1, 1);
+
+        assert_eq!(format.version, FormatMetaVersion::V1);
+        assert_eq!(format.format, FormatBackend::ErasureSingle);
+        assert_eq!(format.erasure.version, FormatErasureVersion::V3);
+        assert_eq!(format.erasure.sets.len(), 1);
+        assert_eq!(format.erasure.sets[0].len(), 1);
+        assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3);
+        assert_eq!(format.erasure.this, Uuid::nil());
+    }
+
+    #[test]
+    fn test_format_v3_new_multiple_sets() {
+        let format = FormatV3::new(2, 4);
+
+        assert_eq!(format.version, FormatMetaVersion::V1);
+        assert_eq!(format.format, FormatBackend::Erasure);
+        assert_eq!(format.erasure.version, FormatErasureVersion::V3);
+        assert_eq!(format.erasure.sets.len(), 2);
+        assert_eq!(format.erasure.sets[0].len(), 4);
+        assert_eq!(format.erasure.sets[1].len(), 4);
+        assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3);
+    }
+
+    #[test]
+    fn test_format_v3_drives() {
+        let format = FormatV3::new(2, 4);
+        assert_eq!(format.drives(), 8); // 2 sets * 4 drives each
+
+        let format_single = FormatV3::new(1, 1);
+        assert_eq!(format_single.drives(), 1); // 1 set * 1 drive
+    }
+
+    #[test]
+    fn test_format_v3_to_json() {
+        let format = FormatV3::new(1, 2);
+        let json_result = format.to_json();
+
+        assert!(json_result.is_ok());
+        let json_str = json_result.unwrap();
+        assert!(json_str.contains("\"version\":\"1\""));
+        assert!(json_str.contains("\"format\":\"xl\""));
+    }
+
+    #[test]
+    fn test_format_v3_from_json() {
+        let json_data = r#"{
+            "version": "1",
+            "format": "xl-single",
+            "id": "321b3874-987d-4c15-8fa5-757c956b1243",
+            "xl": {
+                "version": "3",
+                "this": "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
+                "sets": [
+                    [
+                        "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5"
+                    ]
+                ],
+                "distributionAlgo": "SIPMOD+PARITY"
+            }
+        }"#;
+
+        let format = FormatV3::try_from(json_data);
+        assert!(format.is_ok());
+
+        let format = format.unwrap();
+        assert_eq!(format.format, FormatBackend::ErasureSingle);
+        assert_eq!(format.erasure.version, FormatErasureVersion::V3);
+        assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3);
+        assert_eq!(format.erasure.sets.len(), 1);
+        assert_eq!(format.erasure.sets[0].len(), 1);
+    }
+
+    #[test]
+    fn test_format_v3_from_bytes() {
+        let json_data = r#"{
+            "version": "1",
+            "format": "xl",
+            "id": "321b3874-987d-4c15-8fa5-757c956b1243",
+            "xl": {
+                "version": "2",
+                "this": "00000000-0000-0000-0000-000000000000",
+                "sets": [
+                    [
+                        "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
+                        "c26315da-05cf-4778-a9ea-b44ea09f58c5"
+                    ]
+                ],
+                "distributionAlgo": "SIPMOD"
+            }
+        }"#;
+
+        let format = FormatV3::try_from(json_data.as_bytes());
+        assert!(format.is_ok());
+
+        let format = format.unwrap();
+        assert_eq!(format.erasure.version, FormatErasureVersion::V2);
+        assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V2);
+        assert_eq!(format.erasure.sets[0].len(), 2);
+    }
+
+    #[test]
+    fn test_format_v3_invalid_json() {
+        let invalid_json = r#"{"invalid": "json"}"#;
+        let format = FormatV3::try_from(invalid_json);
+        assert!(format.is_err());
+    }
+
+    #[test]
+    fn test_find_disk_index_by_disk_id() {
+        let mut format = FormatV3::new(2, 2);
+        let target_disk_id = Uuid::new_v4();
+        format.erasure.sets[1][0] = target_disk_id;
+
+        let result = format.find_disk_index_by_disk_id(target_disk_id);
+        assert!(result.is_ok());
+        assert_eq!(result.unwrap(), (1, 0));
+    }
+
+    #[test]
+    fn test_find_disk_index_nil_uuid() {
+        let format = FormatV3::new(1, 2);
+        let result = format.find_disk_index_by_disk_id(Uuid::nil());
+        assert!(result.is_err());
+        assert!(matches!(result.unwrap_err(), Error::DiskNotFound));
+    }
+
+    #[test]
+    fn test_find_disk_index_max_uuid() {
+        let format = FormatV3::new(1, 2);
+        let result = format.find_disk_index_by_disk_id(Uuid::max());
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_find_disk_index_not_found() {
+        let format = FormatV3::new(1, 2);
+        let non_existent_id = Uuid::new_v4();
+        let result = format.find_disk_index_by_disk_id(non_existent_id);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_check_other_identical() {
+        let format1 = FormatV3::new(2, 4);
+        let mut format2 = format1.clone();
+        format2.erasure.this = format1.erasure.sets[0][0];
+
+        let result = format1.check_other(&format2);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_check_other_different_set_count() {
+        let format1 = FormatV3::new(2, 4);
+        let format2 = FormatV3::new(3, 4);
+
+        let result = format1.check_other(&format2);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_check_other_different_set_size() {
+        let format1 = FormatV3::new(2, 4);
+        let format2 = FormatV3::new(2, 6);
+
+        let result = format1.check_other(&format2);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_check_other_different_disk_id() {
+        let format1 = FormatV3::new(1, 2);
+        let mut format2 = format1.clone();
+        format2.erasure.sets[0][0] = Uuid::new_v4();
+
+        let result = format1.check_other(&format2);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_check_other_disk_not_in_sets() {
+        let format1 = FormatV3::new(1, 2);
+        let mut format2 = format1.clone();
+        format2.erasure.this = Uuid::new_v4(); // Set to a UUID not in any set
+
+        let result = format1.check_other(&format2);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_format_meta_version_serialization() {
+        let v1 = FormatMetaVersion::V1;
+        let json = serde_json::to_string(&v1).unwrap();
+        assert_eq!(json, "\"1\"");
+
+        let unknown = FormatMetaVersion::Unknown;
+        let deserialized: FormatMetaVersion = serde_json::from_str("\"unknown\"").unwrap();
+        assert_eq!(deserialized, unknown);
+    }
+
+    #[test]
+    fn test_format_backend_serialization() {
+        let erasure = FormatBackend::Erasure;
+        let json = serde_json::to_string(&erasure).unwrap();
+        assert_eq!(json, "\"xl\"");
+
+        let single = FormatBackend::ErasureSingle;
+        let json = serde_json::to_string(&single).unwrap();
+        assert_eq!(json, "\"xl-single\"");
+
+        let unknown = FormatBackend::Unknown;
+        let deserialized: FormatBackend = serde_json::from_str("\"unknown\"").unwrap();
+        assert_eq!(deserialized, unknown);
+    }
+
+    #[test]
+    fn test_format_erasure_version_serialization() {
+        let v1 = FormatErasureVersion::V1;
+        let json = serde_json::to_string(&v1).unwrap();
+        assert_eq!(json, "\"1\"");
+
+        let v2 = FormatErasureVersion::V2;
+        let json = serde_json::to_string(&v2).unwrap();
+        assert_eq!(json, "\"2\"");
+
+        let v3 = FormatErasureVersion::V3;
+        let json = serde_json::to_string(&v3).unwrap();
+        assert_eq!(json, "\"3\"");
+    }
+
+    #[test]
+    fn test_distribution_algo_version_serialization() {
+        let v1 = DistributionAlgoVersion::V1;
+        let json = serde_json::to_string(&v1).unwrap();
+        assert_eq!(json, "\"CRCMOD\"");
+
+        let v2 = DistributionAlgoVersion::V2;
+        let json = serde_json::to_string(&v2).unwrap();
+        assert_eq!(json, "\"SIPMOD\"");
+
+        let v3 = DistributionAlgoVersion::V3;
+        let json = serde_json::to_string(&v3).unwrap();
+        assert_eq!(json, "\"SIPMOD+PARITY\"");
+    }
+
+    #[test]
+    fn test_format_v3_round_trip_serialization() {
+        let original = FormatV3::new(2, 3);
+        let json = original.to_json().unwrap();
+        let deserialized = FormatV3::try_from(json.as_str()).unwrap();
+
+        assert_eq!(original.version, deserialized.version);
+        assert_eq!(original.format, deserialized.format);
+        assert_eq!(original.erasure.version, deserialized.erasure.version);
+        assert_eq!(original.erasure.sets.len(), deserialized.erasure.sets.len());
+        assert_eq!(original.erasure.distribution_algo, deserialized.erasure.distribution_algo);
+    }
+}

crates/ecstore/src/disk/fs.rs

@@ -0,0 +1,537 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{fs::Metadata, path::Path};
+
+use tokio::{
+    fs::{self, File},
+    io,
+};
+
+#[cfg(not(windows))]
+pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool {
+    use std::os::unix::fs::MetadataExt;
+
+    if f1.dev() != f2.dev() {
+        return false;
+    }
+
+    if f1.ino() != f2.ino() {
+        return false;
+    }
+
+    if f1.size() != f2.size() {
+        return false;
+    }
+    if f1.permissions() != f2.permissions() {
+        return false;
+    }
+
+    if f1.mtime() != f2.mtime() {
+        return false;
+    }
+
+    true
+}
+
+#[cfg(windows)]
+pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool {
+    if f1.permissions() != f2.permissions() {
+        return false;
+    }
+
+    if f1.file_type() != f2.file_type() {
+        return false;
+    }
+
+    if f1.len() != f2.len() {
+        return false;
+    }
+    true
+}
+
+type FileMode = usize;
+
+pub const O_RDONLY: FileMode = 0x00000;
+pub const O_WRONLY: FileMode = 0x00001;
+pub const O_RDWR: FileMode = 0x00002;
+pub const O_CREATE: FileMode = 0x00040;
+// pub const O_EXCL: FileMode = 0x00080;
+// pub const O_NOCTTY: FileMode = 0x00100;
+pub const O_TRUNC: FileMode = 0x00200;
+// pub const O_NONBLOCK: FileMode = 0x00800;
+pub const O_APPEND: FileMode = 0x00400;
+// pub const O_SYNC: FileMode = 0x01000;
+// pub const O_ASYNC: FileMode = 0x02000;
+// pub const O_CLOEXEC: FileMode = 0x80000;
+
+//      read: bool,
+//     write: bool,
+//     append: bool,
+//     truncate: bool,
+//     create: bool,
+//     create_new: bool,
+
+pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
+    let mut opts = fs::OpenOptions::new();
+
+    match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
+        O_RDONLY => {
+            opts.read(true);
+        }
+        O_WRONLY => {
+            opts.write(true);
+        }
+        O_RDWR => {
+            opts.read(true);
+            opts.write(true);
+        }
+        _ => (),
+    };
+
+    if mode & O_CREATE != 0 {
+        opts.create(true);
+    }
+
+    if mode & O_APPEND != 0 {
+        opts.append(true);
+    }
+
+    if mode & O_TRUNC != 0 {
+        opts.truncate(true);
+    }
+
+    opts.open(path.as_ref()).await
+}
+
+pub async fn access(path: impl AsRef<Path>) -> io::Result<()> {
+    fs::metadata(path).await?;
+    Ok(())
+}
+
+pub fn access_std(path: impl AsRef<Path>) -> io::Result<()> {
+    tokio::task::block_in_place(|| std::fs::metadata(path))?;
+    Ok(())
+}
+
+pub async fn lstat(path: impl AsRef<Path>) -> io::Result<Metadata> {
+    fs::metadata(path).await
+}
+
+pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
+    tokio::task::block_in_place(|| std::fs::metadata(path))
+}
+
+pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
+    fs::create_dir_all(path.as_ref()).await
+}
+
+#[tracing::instrument(level = "debug", skip_all)]
+pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
+    let meta = fs::metadata(path.as_ref()).await?;
+    if meta.is_dir() {
+        fs::remove_dir(path.as_ref()).await
+    } else {
+        fs::remove_file(path.as_ref()).await
+    }
+}
+
+pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
+    let meta = fs::metadata(path.as_ref()).await?;
+    if meta.is_dir() {
+        fs::remove_dir_all(path.as_ref()).await
+    } else {
+        fs::remove_file(path.as_ref()).await
+    }
+}
+
+#[tracing::instrument(level = "debug", skip_all)]
+pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
+    let path = path.as_ref();
+    tokio::task::block_in_place(|| {
+        let meta = std::fs::metadata(path)?;
+        if meta.is_dir() {
+            std::fs::remove_dir(path)
+        } else {
+            std::fs::remove_file(path)
+        }
+    })
+}
+
+pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
+    let path = path.as_ref();
+    tokio::task::block_in_place(|| {
+        let meta = std::fs::metadata(path)?;
+        if meta.is_dir() {
+            std::fs::remove_dir_all(path)
+        } else {
+            std::fs::remove_file(path)
+        }
+    })
+}
+
+pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
+    fs::create_dir(path.as_ref()).await
+}
+
+pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
+    fs::rename(from, to).await
+}
+
+pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
+    tokio::task::block_in_place(|| std::fs::rename(from, to))
+}
+
+#[tracing::instrument(level = "debug", skip_all)]
+pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
+    fs::read(path.as_ref()).await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+    use tokio::io::AsyncWriteExt;
+
+    #[tokio::test]
+    async fn test_file_mode_constants() {
+        assert_eq!(O_RDONLY, 0x00000);
+        assert_eq!(O_WRONLY, 0x00001);
+        assert_eq!(O_RDWR, 0x00002);
+        assert_eq!(O_CREATE, 0x00040);
+        assert_eq!(O_TRUNC, 0x00200);
+        assert_eq!(O_APPEND, 0x00400);
+    }
+
+    #[tokio::test]
+    async fn test_open_file_read_only() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_readonly.txt");
+
+        // Create a test file
+        tokio::fs::write(&file_path, b"test content").await.unwrap();
+
+        // Test opening in read-only mode
+        let file = open_file(&file_path, O_RDONLY).await;
+        assert!(file.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_open_file_write_only() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_writeonly.txt");
+
+        // Test opening in write-only mode with create flag
+        let mut file = open_file(&file_path, O_WRONLY | O_CREATE).await.unwrap();
+
+        // Should be able to write
+        file.write_all(b"write test").await.unwrap();
+        file.flush().await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_open_file_read_write() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_readwrite.txt");
+
+        // Test opening in read-write mode with create flag
+        let mut file = open_file(&file_path, O_RDWR | O_CREATE).await.unwrap();
+
+        // Should be able to write and read
+        file.write_all(b"read-write test").await.unwrap();
+        file.flush().await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_open_file_append() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_append.txt");
+
+        // Create initial content
+        tokio::fs::write(&file_path, b"initial").await.unwrap();
+
+        // Open in append mode
+        let mut file = open_file(&file_path, O_WRONLY | O_APPEND).await.unwrap();
+        file.write_all(b" appended").await.unwrap();
+        file.flush().await.unwrap();
+
+        // Verify content
+        let content = tokio::fs::read_to_string(&file_path).await.unwrap();
+        assert_eq!(content, "initial appended");
+    }
+
+    #[tokio::test]
+    async fn test_open_file_truncate() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_truncate.txt");
+
+        // Create initial content
+        tokio::fs::write(&file_path, b"initial content").await.unwrap();
+
+        // Open with truncate flag
+        let mut file = open_file(&file_path, O_WRONLY | O_TRUNC).await.unwrap();
+        file.write_all(b"new").await.unwrap();
+        file.flush().await.unwrap();
+
+        // Verify content was truncated
+        let content = tokio::fs::read_to_string(&file_path).await.unwrap();
+        assert_eq!(content, "new");
+    }
+
+    #[tokio::test]
+    async fn test_access() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_access.txt");
+
+        // Should fail for non-existent file
+        assert!(access(&file_path).await.is_err());
+
+        // Create file and test again
+        tokio::fs::write(&file_path, b"test").await.unwrap();
+        assert!(access(&file_path).await.is_ok());
+    }
+
+    #[test]
+    fn test_access_std() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_access_std.txt");
+
+        // Should fail for non-existent file
+        assert!(access_std(&file_path).is_err());
+
+        // Create file and test again
+        std::fs::write(&file_path, b"test").unwrap();
+        assert!(access_std(&file_path).is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_lstat() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_lstat.txt");
+
+        // Create test file
+        tokio::fs::write(&file_path, b"test content").await.unwrap();
+
+        // Test lstat
+        let metadata = lstat(&file_path).await.unwrap();
+        assert!(metadata.is_file());
+        assert_eq!(metadata.len(), 12); // "test content" is 12 bytes
+    }
+
+    #[test]
+    fn test_lstat_std() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_lstat_std.txt");
+
+        // Create test file
+        std::fs::write(&file_path, b"test content").unwrap();
+
+        // Test lstat_std
+        let metadata = lstat_std(&file_path).unwrap();
+        assert!(metadata.is_file());
+        assert_eq!(metadata.len(), 12); // "test content" is 12 bytes
+    }
+
+    #[tokio::test]
+    async fn test_make_dir_all() {
+        let temp_dir = TempDir::new().unwrap();
+        let nested_path = temp_dir.path().join("level1").join("level2").join("level3");
+
+        // Should create nested directories
+        assert!(make_dir_all(&nested_path).await.is_ok());
+        assert!(nested_path.exists());
+        assert!(nested_path.is_dir());
+    }
+
+    #[tokio::test]
+    async fn test_remove_file() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_remove.txt");
+
+        // Create test file
+        tokio::fs::write(&file_path, b"test").await.unwrap();
+        assert!(file_path.exists());
+
+        // Remove file
+        assert!(remove(&file_path).await.is_ok());
+        assert!(!file_path.exists());
+    }
+
+    #[tokio::test]
+    async fn test_remove_directory() {
+        let temp_dir = TempDir::new().unwrap();
+        let dir_path = temp_dir.path().join("test_remove_dir");
+
+        // Create test directory
+        tokio::fs::create_dir(&dir_path).await.unwrap();
+        assert!(dir_path.exists());
+
+        // Remove directory
+        assert!(remove(&dir_path).await.is_ok());
+        assert!(!dir_path.exists());
+    }
+
+    #[tokio::test]
+    async fn test_remove_all() {
+        let temp_dir = TempDir::new().unwrap();
+        let dir_path = temp_dir.path().join("test_remove_all");
+        let file_path = dir_path.join("nested_file.txt");
+
+        // Create nested structure
+        tokio::fs::create_dir(&dir_path).await.unwrap();
+        tokio::fs::write(&file_path, b"nested content").await.unwrap();
+
+        // Remove all
+        assert!(remove_all(&dir_path).await.is_ok());
+        assert!(!dir_path.exists());
+    }
+
+    #[test]
+    fn test_remove_std() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_remove_std.txt");
+
+        // Create test file
+        std::fs::write(&file_path, b"test").unwrap();
+        assert!(file_path.exists());
+
+        // Remove file
+        assert!(remove_std(&file_path).is_ok());
+        assert!(!file_path.exists());
+    }
+
+    #[test]
+    fn test_remove_all_std() {
+        let temp_dir = TempDir::new().unwrap();
+        let dir_path = temp_dir.path().join("test_remove_all_std");
+        let file_path = dir_path.join("nested_file.txt");
+
+        // Create nested structure
+        std::fs::create_dir(&dir_path).unwrap();
+        std::fs::write(&file_path, b"nested content").unwrap();
+
+        // Remove all
+        assert!(remove_all_std(&dir_path).is_ok());
+        assert!(!dir_path.exists());
+    }
+
+    #[tokio::test]
+    async fn test_mkdir() {
+        let temp_dir = TempDir::new().unwrap();
+        let dir_path = temp_dir.path().join("test_mkdir");
+
+        // Create directory
+        assert!(mkdir(&dir_path).await.is_ok());
+        assert!(dir_path.exists());
+        assert!(dir_path.is_dir());
+    }
+
+    #[tokio::test]
+    async fn test_rename() {
+        let temp_dir = TempDir::new().unwrap();
+        let old_path = temp_dir.path().join("old_name.txt");
+        let new_path = temp_dir.path().join("new_name.txt");
+
+        // Create test file
+        tokio::fs::write(&old_path, b"test content").await.unwrap();
+        assert!(old_path.exists());
+        assert!(!new_path.exists());
+
+        // Rename file
+        assert!(rename(&old_path, &new_path).await.is_ok());
+        assert!(!old_path.exists());
+        assert!(new_path.exists());
+
+        // Verify content preserved
+        let content = tokio::fs::read_to_string(&new_path).await.unwrap();
+        assert_eq!(content, "test content");
+    }
+
+    #[test]
+    fn test_rename_std() {
+        let temp_dir = TempDir::new().unwrap();
+        let old_path = temp_dir.path().join("old_name_std.txt");
+        let new_path = temp_dir.path().join("new_name_std.txt");
+
+        // Create test file
+        std::fs::write(&old_path, b"test content").unwrap();
+        assert!(old_path.exists());
+        assert!(!new_path.exists());
+
+        // Rename file
+        assert!(rename_std(&old_path, &new_path).is_ok());
+        assert!(!old_path.exists());
+        assert!(new_path.exists());
+
+        // Verify content preserved
+        let content = std::fs::read_to_string(&new_path).unwrap();
+        assert_eq!(content, "test content");
+    }
+
+    #[tokio::test]
+    async fn test_read_file() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_read.txt");
+
+        let test_content = b"This is test content for reading";
+        tokio::fs::write(&file_path, test_content).await.unwrap();
+
+        // Read file
+        let read_content = read_file(&file_path).await.unwrap();
+        assert_eq!(read_content, test_content);
+    }
+
+    #[tokio::test]
+    async fn test_read_file_nonexistent() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("nonexistent.txt");
+
+        // Should fail for non-existent file
+        assert!(read_file(&file_path).await.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_same_file() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_same.txt");
+
+        // Create test file
+        tokio::fs::write(&file_path, b"test content").await.unwrap();
+
+        // Get metadata twice
+        let metadata1 = tokio::fs::metadata(&file_path).await.unwrap();
+        let metadata2 = tokio::fs::metadata(&file_path).await.unwrap();
+
+        // Should be the same file
+        assert!(same_file(&metadata1, &metadata2));
+    }
+
+    #[tokio::test]
+    async fn test_different_files() {
+        let temp_dir = TempDir::new().unwrap();
+        let file1_path = temp_dir.path().join("file1.txt");
+        let file2_path = temp_dir.path().join("file2.txt");
+
+        // Create two different files
+        tokio::fs::write(&file1_path, b"content1").await.unwrap();
+        tokio::fs::write(&file2_path, b"content2").await.unwrap();
+
+        // Get metadata
+        let metadata1 = tokio::fs::metadata(&file1_path).await.unwrap();
+        let metadata2 = tokio::fs::metadata(&file2_path).await.unwrap();
+
+        // Should be different files
+        assert!(!same_file(&metadata1, &metadata2));
+    }
+}

crates/ecstore/src/disk/os.rs

@@ -0,0 +1,229 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{
+    io,
+    path::{Component, Path},
+};
+
+use super::error::Result;
+use crate::disk::error_conv::to_file_error;
+use rustfs_utils::path::SLASH_SEPARATOR;
+use tokio::fs;
+use tracing::warn;
+
+use super::error::DiskError;
+
+pub fn check_path_length(path_name: &str) -> Result<()> {
+    // Apple OS X path length is limited to 1016
+    if cfg!(target_os = "macos") && path_name.len() > 1016 {
+        return Err(DiskError::FileNameTooLong);
+    }
+
+    // Disallow more than 1024 characters on windows, there
+    // are no known name_max limits on Windows.
+    if cfg!(target_os = "windows") && path_name.len() > 1024 {
+        return Err(DiskError::FileNameTooLong);
+    }
+
+    // On Unix we reject paths if they are just '.', '..' or '/'
+    let invalid_paths = [".", "..", "/"];
+    if invalid_paths.contains(&path_name) {
+        return Err(DiskError::FileAccessDenied);
+    }
+
+    // Check each path segment length is > 255 on all Unix
+    // platforms, look for this value as NAME_MAX in
+    // /usr/include/linux/limits.h
+    let mut count = 0usize;
+    for c in path_name.chars() {
+        match c {
+            '/' | '\\' if cfg!(target_os = "windows") => count = 0, // Reset
+            _ => {
+                count += 1;
+                if count > 255 {
+                    return Err(DiskError::FileNameTooLong);
+                }
+            }
+        }
+    }
+
+    // Success.
+    Ok(())
+}
+
+pub fn is_root_disk(disk_path: &str, root_disk: &str) -> Result<bool> {
+    if cfg!(target_os = "windows") {
+        return Ok(false);
+    }
+
+    rustfs_utils::os::same_disk(disk_path, root_disk).map_err(|e| to_file_error(e).into())
+}
+
+pub async fn make_dir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> Result<()> {
+    check_path_length(path.as_ref().to_string_lossy().to_string().as_str())?;
+
+    reliable_mkdir_all(path.as_ref(), base_dir.as_ref())
+        .await
+        .map_err(to_file_error)?;
+
+    Ok(())
+}
+
+pub async fn is_empty_dir(path: impl AsRef<Path>) -> bool {
+    read_dir(path.as_ref(), 1).await.is_ok_and(|v| v.is_empty())
+}
+
+// read_dir  count read limit. when count == 0 unlimit.
+pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> std::io::Result<Vec<String>> {
+    let mut entries = fs::read_dir(path.as_ref()).await?;
+
+    let mut volumes = Vec::new();
+
+    let mut count = count;
+
+    while let Some(entry) = entries.next_entry().await? {
+        let name = entry.file_name().to_string_lossy().to_string();
+
+        if name.is_empty() || name == "." || name == ".." {
+            continue;
+        }
+
+        let file_type = entry.file_type().await?;
+
+        if file_type.is_file() {
+            volumes.push(name);
+        } else if file_type.is_dir() {
+            volumes.push(format!("{name}{SLASH_SEPARATOR}"));
+        }
+        count -= 1;
+        if count == 0 {
+            break;
+        }
+    }
+
+    Ok(volumes)
+}
+
+#[tracing::instrument(level = "debug", skip_all)]
+pub async fn rename_all(
+    src_file_path: impl AsRef<Path>,
+    dst_file_path: impl AsRef<Path>,
+    base_dir: impl AsRef<Path>,
+) -> Result<()> {
+    reliable_rename(src_file_path, dst_file_path.as_ref(), base_dir)
+        .await
+        .map_err(to_file_error)?;
+
+    Ok(())
+}
+
+async fn reliable_rename(
+    src_file_path: impl AsRef<Path>,
+    dst_file_path: impl AsRef<Path>,
+    base_dir: impl AsRef<Path>,
+) -> io::Result<()> {
+    if let Some(parent) = dst_file_path.as_ref().parent() {
+        if !file_exists(parent) {
+            // info!("reliable_rename reliable_mkdir_all parent: {:?}", parent);
+            reliable_mkdir_all(parent, base_dir.as_ref()).await?;
+        }
+    }
+
+    let mut i = 0;
+    loop {
+        if let Err(e) = super::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
+            if e.kind() == io::ErrorKind::NotFound {
+                break;
+            }
+
+            if i == 0 {
+                i += 1;
+                continue;
+            }
+            warn!(
+                "reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
+                src_file_path.as_ref(),
+                dst_file_path.as_ref(),
+                base_dir.as_ref(),
+                e
+            );
+            return Err(e);
+        }
+
+        break;
+    }
+
+    Ok(())
+}
+
+pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
+    let mut i = 0;
+
+    let mut base_dir = base_dir.as_ref();
+    loop {
+        if let Err(e) = os_mkdir_all(path.as_ref(), base_dir).await {
+            if e.kind() == io::ErrorKind::NotFound && i == 0 {
+                i += 1;
+
+                if let Some(base_parent) = base_dir.parent() {
+                    if let Some(c) = base_parent.components().next() {
+                        if c != Component::RootDir {
+                            base_dir = base_parent
+                        }
+                    }
+                }
+                continue;
+            }
+
+            return Err(e);
+        }
+
+        break;
+    }
+
+    Ok(())
+}
+
+pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
+    if !base_dir.as_ref().to_string_lossy().is_empty() && base_dir.as_ref().starts_with(dir_path.as_ref()) {
+        return Ok(());
+    }
+
+    if let Some(parent) = dir_path.as_ref().parent() {
+        // 不支持递归，直接 create_dir_all 了
+        if let Err(e) = super::fs::make_dir_all(&parent).await {
+            if e.kind() == io::ErrorKind::AlreadyExists {
+                return Ok(());
+            }
+
+            return Err(e);
+        }
+        // Box::pin(os_mkdir_all(&parent, &base_dir)).await?;
+    }
+
+    if let Err(e) = super::fs::mkdir(dir_path.as_ref()).await {
+        if e.kind() == io::ErrorKind::AlreadyExists {
+            return Ok(());
+        }
+
+        return Err(e);
+    }
+
+    Ok(())
+}
+
+pub fn file_exists(path: impl AsRef<Path>) -> bool {
+    std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false)
+}

crates/ecstore/src/disks_layout.rs

@@ -0,0 +1,927 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_utils::string::{ArgPattern, find_ellipses_patterns, has_ellipses};
+use serde::Deserialize;
+use std::collections::HashSet;
+use std::env;
+use std::io::{Error, Result};
+use tracing::debug;
+
+/// Supported set sizes this is used to find the optimal
+/// single set size.
+const SET_SIZES: [usize; 15] = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16];
+const ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT: &str = "RUSTFS_ERASURE_SET_DRIVE_COUNT";
+
+#[derive(Deserialize, Debug, Default)]
+pub struct PoolDisksLayout {
+    cmd_line: String,
+    layout: Vec<Vec<String>>,
+}
+
+impl PoolDisksLayout {
+    fn new(args: impl Into<String>, layout: Vec<Vec<String>>) -> Self {
+        PoolDisksLayout {
+            cmd_line: args.into(),
+            layout,
+        }
+    }
+
+    fn count(&self) -> usize {
+        self.layout.len()
+    }
+
+    fn get_cmd_line(&self) -> &str {
+        &self.cmd_line
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = &Vec<String>> {
+        self.layout.iter()
+    }
+}
+
+#[derive(Deserialize, Debug, Default)]
+pub struct DisksLayout {
+    pub legacy: bool,
+    pub pools: Vec<PoolDisksLayout>,
+}
+
+// impl<T: AsRef<str>> TryFrom<&[T]> for DisksLayout {
+//     type Error = Error;
+
+//     fn try_from(args: &[T]) -> Result<Self, Self::Error> {
+//         if args.is_empty() {
+//             return Err(Error::from_string("Invalid argument"));
+//         }
+
+//         let is_ellipses = args.iter().any(|v| has_ellipses(&[v]));
+
+//         // None of the args have ellipses use the old style.
+//         if !is_ellipses {
+//             let set_args = get_all_sets(is_ellipses, args)?;
+
+//             return Ok(DisksLayout {
+//                 legacy: true,
+//                 pools: vec![PoolDisksLayout::new(
+//                     args.iter().map(AsRef::as_ref).collect::<Vec<&str>>().join(" "),
+//                     set_args,
+//                 )],
+//             });
+//         }
+
+//         let mut layout = Vec::with_capacity(args.len());
+//         for arg in args.iter() {
+//             if !has_ellipses(&[arg]) && args.len() > 1 {
+//                 return Err(Error::from_string(
+//                     "all args must have ellipses for pool expansion (Invalid arguments specified)",
+//                 ));
+//             }
+
+//             let set_args = get_all_sets(is_ellipses, &[arg])?;
+
+//             layout.push(PoolDisksLayout::new(arg.as_ref(), set_args));
+//         }
+
+//         Ok(DisksLayout {
+//             legacy: false,
+//             pools: layout,
+//         })
+//     }
+// }
+
+impl DisksLayout {
+    pub fn from_volumes<T: AsRef<str>>(args: &[T]) -> Result<Self> {
+        if args.is_empty() {
+            return Err(Error::other("Invalid argument"));
+        }
+
+        let is_ellipses = args.iter().any(|v| has_ellipses(&[v]));
+
+        let set_drive_count_env = env::var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT).unwrap_or_else(|err| {
+            debug!("{} not set use default:0, {:?}", ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, err);
+            "0".to_string()
+        });
+        let set_drive_count: usize = set_drive_count_env.parse().map_err(Error::other)?;
+
+        // None of the args have ellipses use the old style.
+        if !is_ellipses {
+            let set_args = get_all_sets(set_drive_count, is_ellipses, args)?;
+
+            return Ok(DisksLayout {
+                legacy: true,
+                pools: vec![PoolDisksLayout::new(
+                    args.iter().map(AsRef::as_ref).collect::<Vec<&str>>().join(" "),
+                    set_args,
+                )],
+            });
+        }
+
+        let mut layout = Vec::with_capacity(args.len());
+        for arg in args.iter() {
+            if !has_ellipses(&[arg]) && args.len() > 1 {
+                return Err(Error::other(
+                    "all args must have ellipses for pool expansion (Invalid arguments specified)",
+                ));
+            }
+
+            let set_args = get_all_sets(set_drive_count, is_ellipses, &[arg])?;
+
+            layout.push(PoolDisksLayout::new(arg.as_ref(), set_args));
+        }
+
+        Ok(DisksLayout {
+            legacy: false,
+            pools: layout,
+        })
+    }
+
+    pub fn is_empty_layout(&self) -> bool {
+        self.pools.is_empty()
+            || self.pools[0].layout.is_empty()
+            || self.pools[0].layout[0].is_empty()
+            || self.pools[0].layout[0][0].is_empty()
+    }
+
+    pub fn is_single_drive_layout(&self) -> bool {
+        self.pools.len() == 1 && self.pools[0].layout.len() == 1 && self.pools[0].layout[0].len() == 1
+    }
+
+    pub fn get_single_drive_layout(&self) -> &str {
+        &self.pools[0].layout[0][0]
+    }
+
+    /// returns the total number of sets in the layout.
+    pub fn get_set_count(&self, i: usize) -> usize {
+        self.pools.get(i).map_or(0, |v| v.count())
+    }
+
+    /// returns the total number of drives in the layout.
+    pub fn get_drives_per_set(&self, i: usize) -> usize {
+        self.pools.get(i).map_or(0, |v| v.layout.first().map_or(0, |v| v.len()))
+    }
+
+    /// returns the command line for the given index.
+    pub fn get_cmd_line(&self, i: usize) -> String {
+        self.pools.get(i).map_or(String::new(), |v| v.get_cmd_line().to_owned())
+    }
+}
+
+/// parses all ellipses input arguments, expands them into
+/// corresponding list of endpoints chunked evenly in accordance with a
+/// specific set size.
+///
+/// For example: {1...64} is divided into 4 sets each of size 16.
+/// This applies to even distributed setup syntax as well.
+fn get_all_sets<T: AsRef<str>>(set_drive_count: usize, is_ellipses: bool, args: &[T]) -> Result<Vec<Vec<String>>> {
+    let endpoint_set = if is_ellipses {
+        EndpointSet::from_volumes(args, set_drive_count)?
+    } else {
+        let set_indexes = if args.len() > 1 {
+            get_set_indexes(args, &[args.len()], set_drive_count, &[])?
+        } else {
+            vec![vec![args.len()]]
+        };
+        let endpoints = args.iter().map(|v| v.as_ref().to_string()).collect();
+
+        EndpointSet::new(endpoints, set_indexes)
+    };
+
+    let set_args = endpoint_set.get();
+
+    let mut unique_args = HashSet::with_capacity(set_args.len());
+    for args in set_args.iter() {
+        for arg in args {
+            if unique_args.contains(arg) {
+                return Err(Error::other(format!("Input args {arg} has duplicate ellipses")));
+            }
+            unique_args.insert(arg);
+        }
+    }
+
+    Ok(set_args)
+}
+
+/// represents parsed ellipses values, also provides
+/// methods to get the sets of endpoints.
+#[derive(Debug, Default)]
+struct EndpointSet {
+    _arg_patterns: Vec<ArgPattern>,
+    endpoints: Vec<String>,
+    set_indexes: Vec<Vec<usize>>,
+}
+
+// impl<T: AsRef<str>> TryFrom<&[T]> for EndpointSet {
+//     type Error = Error;
+
+//     fn try_from(args: &[T]) -> Result<Self, Self::Error> {
+//         let mut arg_patterns = Vec::with_capacity(args.len());
+//         for arg in args {
+//             arg_patterns.push(find_ellipses_patterns(arg.as_ref())?);
+//         }
+
+//         let total_sizes = get_total_sizes(&arg_patterns);
+//         let set_indexes = get_set_indexes(args, &total_sizes, &arg_patterns)?;
+
+//         let mut endpoints = Vec::new();
+//         for ap in arg_patterns.iter() {
+//             let aps = ap.expand();
+//             for bs in aps {
+//                 endpoints.push(bs.join(""));
+//             }
+//         }
+
+//         Ok(EndpointSet {
+//             set_indexes,
+//             _arg_patterns: arg_patterns,
+//             endpoints,
+//         })
+//     }
+// }
+
+impl EndpointSet {
+    /// Create a new EndpointSet with the given endpoints and set indexes.
+    pub fn new(endpoints: Vec<String>, set_indexes: Vec<Vec<usize>>) -> Self {
+        Self {
+            endpoints,
+            set_indexes,
+            ..Default::default()
+        }
+    }
+
+    pub fn from_volumes<T: AsRef<str>>(args: &[T], set_drive_count: usize) -> Result<Self> {
+        let mut arg_patterns = Vec::with_capacity(args.len());
+        for arg in args {
+            arg_patterns.push(find_ellipses_patterns(arg.as_ref())?);
+        }
+
+        let total_sizes = get_total_sizes(&arg_patterns);
+        let set_indexes = get_set_indexes(args, &total_sizes, set_drive_count, &arg_patterns)?;
+
+        let mut endpoints = Vec::new();
+        for ap in arg_patterns.iter() {
+            let aps = ap.expand();
+            for bs in aps {
+                endpoints.push(bs.join(""));
+            }
+        }
+
+        Ok(EndpointSet {
+            set_indexes,
+            _arg_patterns: arg_patterns,
+            endpoints,
+        })
+    }
+
+    /// returns the sets representation of the endpoints
+    /// this function also intelligently decides on what will
+    /// be the right set size etc.
+    pub fn get(&self) -> Vec<Vec<String>> {
+        let mut sets: Vec<Vec<String>> = Vec::new();
+
+        let mut start = 0;
+        for set_idx in self.set_indexes.iter() {
+            for idx in set_idx {
+                let end = idx + start;
+                sets.push(self.endpoints[start..end].to_vec());
+                start = end;
+            }
+        }
+        sets
+    }
+}
+
+/// returns the greatest common divisor of all the ellipses sizes.
+fn get_divisible_size(total_sizes: &[usize]) -> usize {
+    fn gcd(mut x: usize, mut y: usize) -> usize {
+        while y != 0 {
+            // be equivalent to: x, y = y, x%y
+            std::mem::swap(&mut x, &mut y);
+            y %= x;
+        }
+        x
+    }
+
+    total_sizes.iter().skip(1).fold(total_sizes[0], |acc, &y| gcd(acc, y))
+}
+
+fn possible_set_counts(set_size: usize) -> Vec<usize> {
+    let mut ss = Vec::new();
+    for s in SET_SIZES {
+        if set_size % s == 0 {
+            ss.push(s);
+        }
+    }
+    ss
+}
+
+/// checks whether given count is a valid set size for erasure coding.
+fn is_valid_set_size(count: usize) -> bool {
+    count >= SET_SIZES[0] && count <= SET_SIZES[SET_SIZES.len() - 1]
+}
+
+/// Final set size with all the symmetry accounted for.
+fn common_set_drive_count(divisible_size: usize, set_counts: &[usize]) -> usize {
+    // prefers set_counts to be sorted for optimal behavior.
+    if divisible_size < set_counts[set_counts.len() - 1] {
+        return divisible_size;
+    }
+
+    let mut prev_d = divisible_size / set_counts[0];
+    let mut set_size = 0;
+    for &cnt in set_counts {
+        if divisible_size % cnt == 0 {
+            let d = divisible_size / cnt;
+            if d <= prev_d {
+                prev_d = d;
+                set_size = cnt;
+            }
+        }
+    }
+    set_size
+}
+
+/// returns symmetrical setCounts based on the input argument patterns,
+/// the symmetry calculation is to ensure that we also use uniform number
+/// of drives common across all ellipses patterns.
+fn possible_set_counts_with_symmetry(set_counts: &[usize], arg_patterns: &[ArgPattern]) -> Vec<usize> {
+    let mut new_set_counts: HashSet<usize> = HashSet::new();
+
+    for &ss in set_counts {
+        let mut symmetry = false;
+        for arg_pattern in arg_patterns {
+            for p in arg_pattern.as_ref().iter() {
+                if p.len() > ss {
+                    symmetry = (p.len() % ss) == 0;
+                } else {
+                    symmetry = (ss % p.len()) == 0;
+                }
+            }
+        }
+
+        if !new_set_counts.contains(&ss) && (symmetry || arg_patterns.is_empty()) {
+            new_set_counts.insert(ss);
+        }
+    }
+
+    let mut set_counts: Vec<usize> = new_set_counts.into_iter().collect();
+    set_counts.sort_unstable();
+
+    set_counts
+}
+
+/// returns list of indexes which provides the set size
+/// on each index, this function also determines the final set size
+/// The final set size has the affinity towards choosing smaller
+/// indexes (total sets)
+fn get_set_indexes<T: AsRef<str>>(
+    args: &[T],
+    total_sizes: &[usize],
+    set_drive_count: usize,
+    arg_patterns: &[ArgPattern],
+) -> Result<Vec<Vec<usize>>> {
+    if args.is_empty() || total_sizes.is_empty() {
+        return Err(Error::other("Invalid argument"));
+    }
+
+    for &size in total_sizes {
+        // Check if total_sizes has minimum range upto set_size
+        if size < SET_SIZES[0] || size < set_drive_count {
+            return Err(Error::other(format!("Incorrect number of endpoints provided, size {size}")));
+        }
+    }
+
+    let common_size = get_divisible_size(total_sizes);
+    let mut set_counts = possible_set_counts(common_size);
+    if set_counts.is_empty() {
+        return Err(Error::other(format!(
+            "Incorrect number of endpoints provided, number of drives {} is not divisible by any supported erasure set sizes {}",
+            common_size, 0
+        )));
+    }
+
+    // Returns possible set counts with symmetry.
+    set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns);
+    if set_counts.is_empty() {
+        return Err(Error::other("No symmetric distribution detected with input endpoints provided"));
+    }
+
+    let set_size = {
+        if set_drive_count > 0 {
+            let has_set_drive_count = set_counts.contains(&set_drive_count);
+
+            if !has_set_drive_count {
+                return Err(Error::other(format!(
+                    "Invalid set drive count {}. Acceptable values for {:?} number drives are {:?}",
+                    set_drive_count, common_size, &set_counts
+                )));
+            }
+            set_drive_count
+        } else {
+            set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns);
+            if set_counts.is_empty() {
+                return Err(Error::other(format!(
+                    "No symmetric distribution detected with input endpoints , drives {} cannot be spread symmetrically by any supported erasure set sizes {:?}",
+                    common_size, &set_counts
+                )));
+            }
+            // Final set size with all the symmetry accounted for.
+            common_set_drive_count(common_size, &set_counts)
+        }
+    };
+
+    if !is_valid_set_size(set_size) {
+        return Err(Error::other("Incorrect number of endpoints provided3"));
+    }
+
+    Ok(total_sizes
+        .iter()
+        .map(|&size| (0..(size / set_size)).map(|_| set_size).collect())
+        .collect())
+}
+
+/// Return the total size for each argument patterns.
+fn get_total_sizes(arg_patterns: &[ArgPattern]) -> Vec<usize> {
+    arg_patterns.iter().map(|v| v.total_sizes()).collect()
+}
+
+#[cfg(test)]
+mod test {
+    use rustfs_utils::string::Pattern;
+
+    use super::*;
+
+    impl PartialEq for EndpointSet {
+        fn eq(&self, other: &Self) -> bool {
+            self._arg_patterns == other._arg_patterns && self.set_indexes == other.set_indexes
+        }
+    }
+
+    #[test]
+    fn test_get_divisible_size() {
+        struct TestCase {
+            total_sizes: Vec<usize>,
+            result: usize,
+        }
+
+        let test_cases = [
+            TestCase {
+                total_sizes: vec![24, 32, 16],
+                result: 8,
+            },
+            TestCase {
+                total_sizes: vec![32, 8, 4],
+                result: 4,
+            },
+            TestCase {
+                total_sizes: vec![8, 8, 8],
+                result: 8,
+            },
+            TestCase {
+                total_sizes: vec![24],
+                result: 24,
+            },
+        ];
+
+        for (i, test_case) in test_cases.iter().enumerate() {
+            let ret = get_divisible_size(&test_case.total_sizes);
+            assert_eq!(ret, test_case.result, "Test{}: Expected {}, got {}", i + 1, test_case.result, ret);
+        }
+    }
+
+    #[test]
+    fn test_get_set_indexes() {
+        #[derive(Default)]
+        struct TestCase<'a> {
+            num: usize,
+            args: Vec<&'a str>,
+            total_sizes: Vec<usize>,
+            indexes: Vec<Vec<usize>>,
+            success: bool,
+        }
+
+        let test_cases = [
+            TestCase {
+                num: 1,
+                args: vec!["data{1...17}/export{1...52}"],
+                total_sizes: vec![14144],
+                ..Default::default()
+            },
+            TestCase {
+                num: 2,
+                args: vec!["data{1...3}"],
+                total_sizes: vec![3],
+                indexes: vec![vec![3]],
+                success: true,
+            },
+            TestCase {
+                num: 3,
+                args: vec!["data/controller1/export{1...2}, data/controller2/export{1...4}, data/controller3/export{1...8}"],
+                total_sizes: vec![2, 4, 8],
+                indexes: vec![vec![2], vec![2, 2], vec![2, 2, 2, 2]],
+                success: true,
+            },
+            TestCase {
+                num: 4,
+                args: vec!["data{1...27}"],
+                total_sizes: vec![27],
+                indexes: vec![vec![9, 9, 9]],
+                success: true,
+            },
+            TestCase {
+                num: 5,
+                args: vec!["http://host{1...3}/data{1...180}"],
+                total_sizes: vec![540],
+                indexes: vec![vec![
+                    15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
+                    15, 15, 15, 15, 15, 15, 15, 15, 15,
+                ]],
+                success: true,
+            },
+            TestCase {
+                num: 6,
+                args: vec!["http://host{1...2}.rack{1...4}/data{1...180}"],
+                total_sizes: vec![1440],
+                indexes: vec![vec![
+                    16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
+                    16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
+                    16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
+                    16, 16, 16, 16, 16, 16, 16, 16, 16,
+                ]],
+                success: true,
+            },
+            TestCase {
+                num: 7,
+                args: vec!["http://host{1...2}/data{1...180}"],
+                total_sizes: vec![360],
+                indexes: vec![vec![
+                    12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
+                    12, 12, 12,
+                ]],
+                success: true,
+            },
+            TestCase {
+                num: 8,
+                args: vec!["data/controller1/export{1...4}, data/controller2/export{1...8}, data/controller3/export{1...12}"],
+                total_sizes: vec![4, 8, 12],
+                indexes: vec![vec![4], vec![4, 4], vec![4, 4, 4]],
+                success: true,
+            },
+            TestCase {
+                num: 9,
+                args: vec!["data{1...64}"],
+                total_sizes: vec![64],
+                indexes: vec![vec![16, 16, 16, 16]],
+                success: true,
+            },
+            TestCase {
+                num: 10,
+                args: vec!["data{1...24}"],
+                total_sizes: vec![24],
+                indexes: vec![vec![12, 12]],
+                success: true,
+            },
+            TestCase {
+                num: 11,
+                args: vec!["data/controller{1...11}/export{1...8}"],
+                total_sizes: vec![88],
+                indexes: vec![vec![11, 11, 11, 11, 11, 11, 11, 11]],
+                success: true,
+            },
+            TestCase {
+                num: 12,
+                args: vec!["data{1...4}"],
+                total_sizes: vec![4],
+                indexes: vec![vec![4]],
+                success: true,
+            },
+            TestCase {
+                num: 13,
+                args: vec!["data/controller1/export{1...10}, data/controller2/export{1...10}, data/controller3/export{1...10}"],
+                total_sizes: vec![10, 10, 10],
+                indexes: vec![vec![10], vec![10], vec![10]],
+                success: true,
+            },
+            TestCase {
+                num: 14,
+                args: vec!["data{1...16}/export{1...52}"],
+                total_sizes: vec![832],
+                indexes: vec![vec![
+                    16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
+                    16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
+                ]],
+                success: true,
+            },
+            TestCase {
+                num: 15,
+                args: vec!["https://node{1...3}.example.net/mnt/drive{1...8}"],
+                total_sizes: vec![24],
+                indexes: vec![vec![12, 12]],
+                success: true,
+            },
+        ];
+
+        for test_case in test_cases {
+            let mut arg_patterns = Vec::new();
+            for v in test_case.args.iter() {
+                match find_ellipses_patterns(v) {
+                    Ok(patterns) => {
+                        arg_patterns.push(patterns);
+                    }
+                    Err(err) => {
+                        panic!("Test{}: Unexpected failure {:?}", test_case.num, err);
+                    }
+                }
+            }
+
+            match get_set_indexes(test_case.args.as_slice(), test_case.total_sizes.as_slice(), 0, arg_patterns.as_slice()) {
+                Ok(got_indexes) => {
+                    if !test_case.success {
+                        panic!("Test{}: Expected failure but passed instead", test_case.num);
+                    }
+
+                    assert_eq!(
+                        test_case.indexes, got_indexes,
+                        "Test{}: Expected {:?}, got {:?}",
+                        test_case.num, test_case.indexes, got_indexes
+                    )
+                }
+                Err(err) => {
+                    if test_case.success {
+                        panic!("Test{}: Expected success but failed instead {:?}", test_case.num, err);
+                    }
+                }
+            }
+        }
+    }
+
+    fn get_sequences(start: usize, number: usize, padding_len: usize) -> Vec<String> {
+        let mut seq = Vec::new();
+        for i in start..=number {
+            if padding_len == 0 {
+                seq.push(format!("{i}"));
+            } else {
+                seq.push(format!("{i:0padding_len$}"));
+            }
+        }
+        seq
+    }
+
+    #[test]
+    fn test_into_endpoint_set() {
+        #[derive(Default)]
+        struct TestCase<'a> {
+            num: usize,
+            arg: &'a str,
+            es: EndpointSet,
+            success: bool,
+        }
+
+        let test_cases = [
+            // Tests invalid inputs.
+            TestCase {
+                num: 1,
+                arg: "...",
+                ..Default::default()
+            },
+            // No range specified.
+            TestCase {
+                num: 2,
+                arg: "{...}",
+                ..Default::default()
+            },
+            // Invalid range.
+            TestCase {
+                num: 3,
+                arg: "http://rustfs{2...3}/export/set{1...0}",
+                ..Default::default()
+            },
+            // Range cannot be smaller than 4 minimum.
+            TestCase {
+                num: 4,
+                arg: "/export{1..2}",
+                ..Default::default()
+            },
+            // Unsupported characters.
+            TestCase {
+                num: 5,
+                arg: "/export/test{1...2O}",
+                ..Default::default()
+            },
+            // Tests valid inputs.
+            TestCase {
+                num: 6,
+                arg: "{1...27}",
+                es: EndpointSet {
+                    _arg_patterns: vec![ArgPattern::new(vec![Pattern {
+                        seq: get_sequences(1, 27, 0),
+                        ..Default::default()
+                    }])],
+                    set_indexes: vec![vec![9, 9, 9]],
+                    ..Default::default()
+                },
+                success: true,
+            },
+            TestCase {
+                num: 7,
+                arg: "/export/set{1...64}",
+                es: EndpointSet {
+                    _arg_patterns: vec![ArgPattern::new(vec![Pattern {
+                        seq: get_sequences(1, 64, 0),
+                        prefix: "/export/set".to_owned(),
+                        ..Default::default()
+                    }])],
+                    set_indexes: vec![vec![16, 16, 16, 16]],
+                    ..Default::default()
+                },
+                success: true,
+            },
+            // Valid input for distributed setup.
+            TestCase {
+                num: 8,
+                arg: "http://rustfs{2...3}/export/set{1...64}",
+                es: EndpointSet {
+                    _arg_patterns: vec![ArgPattern::new(vec![
+                        Pattern {
+                            seq: get_sequences(1, 64, 0),
+                            ..Default::default()
+                        },
+                        Pattern {
+                            seq: get_sequences(2, 3, 0),
+                            prefix: "http://rustfs".to_owned(),
+                            suffix: "/export/set".to_owned(),
+                        },
+                    ])],
+                    set_indexes: vec![vec![16, 16, 16, 16, 16, 16, 16, 16]],
+                    ..Default::default()
+                },
+                success: true,
+            },
+            // Supporting some advanced cases.
+            TestCase {
+                num: 9,
+                arg: "http://rustfs{1...64}.mydomain.net/data",
+                es: EndpointSet {
+                    _arg_patterns: vec![ArgPattern::new(vec![Pattern {
+                        seq: get_sequences(1, 64, 0),
+                        prefix: "http://rustfs".to_owned(),
+                        suffix: ".mydomain.net/data".to_owned(),
+                    }])],
+                    set_indexes: vec![vec![16, 16, 16, 16]],
+                    ..Default::default()
+                },
+                success: true,
+            },
+            TestCase {
+                num: 10,
+                arg: "http://rack{1...4}.mydomain.rustfs{1...16}/data",
+                es: EndpointSet {
+                    _arg_patterns: vec![ArgPattern::new(vec![
+                        Pattern {
+                            seq: get_sequences(1, 16, 0),
+                            suffix: "/data".to_owned(),
+                            ..Default::default()
+                        },
+                        Pattern {
+                            seq: get_sequences(1, 4, 0),
+                            prefix: "http://rack".to_owned(),
+                            suffix: ".mydomain.rustfs".to_owned(),
+                        },
+                    ])],
+                    set_indexes: vec![vec![16, 16, 16, 16]],
+                    ..Default::default()
+                },
+                success: true,
+            },
+            // Supporting kubernetes cases.
+            TestCase {
+                num: 11,
+                arg: "http://rustfs{0...15}.mydomain.net/data{0...1}",
+                es: EndpointSet {
+                    _arg_patterns: vec![ArgPattern::new(vec![
+                        Pattern {
+                            seq: get_sequences(0, 1, 0),
+                            ..Default::default()
+                        },
+                        Pattern {
+                            seq: get_sequences(0, 15, 0),
+                            prefix: "http://rustfs".to_owned(),
+                            suffix: ".mydomain.net/data".to_owned(),
+                        },
+                    ])],
+                    set_indexes: vec![vec![16, 16]],
+                    ..Default::default()
+                },
+                success: true,
+            },
+            // No host regex, just disks.
+            TestCase {
+                num: 12,
+                arg: "http://server1/data{1...32}",
+                es: EndpointSet {
+                    _arg_patterns: vec![ArgPattern::new(vec![Pattern {
+                        seq: get_sequences(1, 32, 0),
+                        prefix: "http://server1/data".to_owned(),
+                        ..Default::default()
+                    }])],
+                    set_indexes: vec![vec![16, 16]],
+                    ..Default::default()
+                },
+                success: true,
+            },
+            // No host regex, just disks with two position numerics.
+            TestCase {
+                num: 13,
+                arg: "http://server1/data{01...32}",
+                es: EndpointSet {
+                    _arg_patterns: vec![ArgPattern::new(vec![Pattern {
+                        seq: get_sequences(1, 32, 2),
+                        prefix: "http://server1/data".to_owned(),
+                        ..Default::default()
+                    }])],
+                    set_indexes: vec![vec![16, 16]],
+                    ..Default::default()
+                },
+                success: true,
+            },
+            // More than 2 ellipses are supported as well.
+            TestCase {
+                num: 14,
+                arg: "http://rustfs{2...3}/export/set{1...64}/test{1...2}",
+                es: EndpointSet {
+                    _arg_patterns: vec![ArgPattern::new(vec![
+                        Pattern {
+                            seq: get_sequences(1, 2, 0),
+                            ..Default::default()
+                        },
+                        Pattern {
+                            seq: get_sequences(1, 64, 0),
+                            suffix: "/test".to_owned(),
+                            ..Default::default()
+                        },
+                        Pattern {
+                            seq: get_sequences(2, 3, 0),
+                            prefix: "http://rustfs".to_owned(),
+                            suffix: "/export/set".to_owned(),
+                        },
+                    ])],
+                    set_indexes: vec![vec![16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16]],
+                    ..Default::default()
+                },
+                success: true,
+            },
+            // More than an ellipse per argument for standalone setup.
+            TestCase {
+                num: 15,
+                arg: "/export{1...10}/disk{1...10}",
+                es: EndpointSet {
+                    _arg_patterns: vec![ArgPattern::new(vec![
+                        Pattern {
+                            seq: get_sequences(1, 10, 0),
+                            ..Default::default()
+                        },
+                        Pattern {
+                            seq: get_sequences(1, 10, 0),
+                            prefix: "/export".to_owned(),
+                            suffix: "/disk".to_owned(),
+                        },
+                    ])],
+                    set_indexes: vec![vec![10, 10, 10, 10, 10, 10, 10, 10, 10, 10]],
+                    ..Default::default()
+                },
+                success: true,
+            },
+        ];
+
+        for test_case in test_cases {
+            match EndpointSet::from_volumes([test_case.arg].as_slice(), 0) {
+                Ok(got_es) => {
+                    if !test_case.success {
+                        panic!("Test{}: Expected failure but passed instead", test_case.num);
+                    }
+
+                    assert_eq!(
+                        test_case.es, got_es,
+                        "Test{}: Expected {:?}, got {:?}",
+                        test_case.num, test_case.es, got_es
+                    )
+                }
+                Err(err) => {
+                    if test_case.success {
+                        panic!("Test{}: Expected success but failed instead {:?}", test_case.num, err);
+                    }
+                }
+            }
+        }
+    }
+}

crates/ecstore/src/erasure.rs

@@ -0,0 +1,586 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::bitrot::{BitrotReader, BitrotWriter};
+use crate::disk::error::{Error, Result};
+use crate::disk::error_reduce::{reduce_write_quorum_errs, OBJECT_OP_IGNORED_ERRS};
+use crate::io::Etag;
+use bytes::{Bytes, BytesMut};
+use futures::future::join_all;
+use reed_solomon_erasure::galois_8::ReedSolomon;
+use smallvec::SmallVec;
+use std::any::Any;
+use std::io::ErrorKind;
+use std::sync::{mpsc, Arc};
+use tokio::io::{AsyncRead, AsyncWrite};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use tokio::sync::mpsc;
+use tracing::warn;
+use tracing::{error, info};
+use uuid::Uuid;
+
+use crate::disk::error::DiskError;
+
+#[derive(Default)]
+pub struct Erasure {
+    data_shards: usize,
+    parity_shards: usize,
+    encoder: Option<ReedSolomon>,
+    pub block_size: usize,
+    _id: Uuid,
+    _buf: Vec<u8>,
+}
+
+impl Erasure {
+    pub fn new(data_shards: usize, parity_shards: usize, block_size: usize) -> Self {
+        // debug!(
+        //     "Erasure new data_shards {},parity_shards {} block_size {} ",
+        //     data_shards, parity_shards, block_size
+        // );
+        let mut encoder = None;
+        if parity_shards > 0 {
+            encoder = Some(ReedSolomon::new(data_shards, parity_shards).unwrap());
+        }
+
+        Erasure {
+            data_shards,
+            parity_shards,
+            block_size,
+            encoder,
+            _id: Uuid::new_v4(),
+            _buf: vec![0u8; block_size],
+        }
+    }
+
+    #[tracing::instrument(level = "info", skip(self, reader, writers))]
+    pub async fn encode<S>(
+        self: Arc<Self>,
+        mut reader: S,
+        writers: &mut [Option<BitrotWriter>],
+        // block_size: usize,
+        total_size: usize,
+        write_quorum: usize,
+    ) -> Result<(usize, String)>
+    where
+        S: AsyncRead + Etag + Unpin + Send + 'static,
+    {
+        let (tx, mut rx) = mpsc::channel(5);
+        let task = tokio::spawn(async move {
+            let mut buf = vec![0u8; self.block_size];
+            let mut total: usize = 0;
+            loop {
+                if total_size > 0 {
+                    let new_len = {
+                        let remain = total_size - total;
+                        if remain > self.block_size { self.block_size } else { remain }
+                    };
+
+                    if new_len == 0 && total > 0 {
+                        break;
+                    }
+
+                    buf.resize(new_len, 0u8);
+                    match reader.read_exact(&mut buf).await {
+                        Ok(res) => res,
+                        Err(e) => {
+                            if let ErrorKind::UnexpectedEof = e.kind() {
+                                break;
+                            } else {
+                                return Err(e.into());
+                            }
+                        }
+                    };
+                    total += buf.len();
+                }
+                let blocks = Arc::new(Box::pin(self.clone().encode_data(&buf)?));
+                let _ = tx.send(blocks).await;
+                if total_size == 0 {
+                    break;
+                }
+            }
+            let etag = reader.etag().await;
+            Ok((total, etag))
+        });
+
+        while let Some(blocks) = rx.recv().await {
+            let write_futures = writers.iter_mut().enumerate().map(|(i, w_op)| {
+                let i_inner = i;
+                let blocks_inner = blocks.clone();
+                async move {
+                    if let Some(w) = w_op {
+                        w.write(blocks_inner[i_inner].clone()).await.err()
+                    } else {
+                        Some(DiskError::DiskNotFound)
+                    }
+                }
+            });
+            let errs = join_all(write_futures).await;
+            let none_count = errs.iter().filter(|&x| x.is_none()).count();
+            if none_count >= write_quorum {
+                if total_size == 0 {
+                    break;
+                }
+                continue;
+            }
+
+            if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
+                warn!("Erasure encode errs {:?}", &errs);
+                return Err(err);
+            }
+        }
+        task.await?
+    }
+
+    pub async fn decode<W>(
+        &self,
+        writer: &mut W,
+        readers: Vec<Option<BitrotReader>>,
+        offset: usize,
+        length: usize,
+        total_length: usize,
+    ) -> (usize, Option<Error>)
+    where
+        W: AsyncWriteExt + Send + Unpin + 'static,
+    {
+        if length == 0 {
+            return (0, None);
+        }
+
+        let mut reader = ShardReader::new(readers, self, offset, total_length);
+
+        // debug!("ShardReader {:?}", &reader);
+
+        let start_block = offset / self.block_size;
+        let end_block = (offset + length) / self.block_size;
+
+        // debug!("decode block from {} to {}", start_block, end_block);
+
+        let mut bytes_written = 0;
+
+        for block_idx in start_block..=end_block {
+            let (block_offset, block_length) = if start_block == end_block {
+                (offset % self.block_size, length)
+            } else if block_idx == start_block {
+                let block_offset = offset % self.block_size;
+                (block_offset, self.block_size - block_offset)
+            } else if block_idx == end_block {
+                (0, (offset + length) % self.block_size)
+            } else {
+                (0, self.block_size)
+            };
+
+            if block_length == 0 {
+                // debug!("block_length == 0 break");
+                break;
+            }
+
+            // debug!("decode {} block_offset {},block_length {} ", block_idx, block_offset, block_length);
+
+            let mut bufs = match reader.read().await {
+                Ok(bufs) => bufs,
+                Err(err) => return (bytes_written, Some(err)),
+            };
+
+            if self.parity_shards > 0 {
+                if let Err(err) = self.decode_data(&mut bufs) {
+                    return (bytes_written, Some(err));
+                }
+            }
+
+            let written_n = match self
+                .write_data_blocks(writer, bufs, self.data_shards, block_offset, block_length)
+                .await
+            {
+                Ok(n) => n,
+                Err(err) => {
+                    error!("write_data_blocks err {:?}", &err);
+                    return (bytes_written, Some(err));
+                }
+            };
+
+            bytes_written += written_n;
+
+            // debug!("decode {} written_n {}, total_written: {} ", block_idx, written_n, bytes_written);
+        }
+
+        if bytes_written != length {
+            // debug!("bytes_written != length: {} != {} ", bytes_written, length);
+            return (bytes_written, Some(Error::other("erasure decode less data")));
+        }
+
+        (bytes_written, None)
+    }
+
+    async fn write_data_blocks<W>(
+        &self,
+        writer: &mut W,
+        bufs: Vec<Option<Vec<u8>>>,
+        data_blocks: usize,
+        offset: usize,
+        length: usize,
+    ) -> Result<usize>
+    where
+        W: AsyncWrite + Send + Unpin + 'static,
+    {
+        if bufs.len() < data_blocks {
+            return Err(Error::other("read bufs not match data_blocks"));
+        }
+
+        let data_len: usize = bufs
+            .iter()
+            .take(data_blocks)
+            .filter(|v| v.is_some())
+            .map(|v| v.as_ref().unwrap().len())
+            .sum();
+        if data_len < length {
+            return Err(Error::other(format!("write_data_blocks data_len < length {} < {}", data_len, length)));
+        }
+
+        let mut offset = offset;
+
+        // debug!("write_data_blocks offset {}, length {}", offset, length);
+
+        let mut write = length;
+        let mut total_written = 0;
+
+        for opt_buf in bufs.iter().take(data_blocks) {
+            let buf = opt_buf.as_ref().unwrap();
+
+            if offset >= buf.len() {
+                offset -= buf.len();
+                continue;
+            }
+
+            let buf = &buf[offset..];
+
+            offset = 0;
+
+            // debug!("write_data_blocks write buf len {}", buf.len());
+
+            if write < buf.len() {
+                let buf = &buf[..write];
+
+                // debug!("write_data_blocks write buf less len {}", buf.len());
+                writer.write_all(buf).await?;
+                // debug!("write_data_blocks write done len {}", buf.len());
+                total_written += buf.len();
+                break;
+            }
+
+            writer.write_all(buf).await?;
+            let n = buf.len();
+
+            // debug!("write_data_blocks write done len {}", n);
+            write -= n;
+            total_written += n;
+        }
+
+        Ok(total_written)
+    }
+
+    pub fn total_shard_count(&self) -> usize {
+        self.data_shards + self.parity_shards
+    }
+
+    #[tracing::instrument(level = "info", skip_all, fields(data_len=data.len()))]
+    pub fn encode_data(self: Arc<Self>, data: &[u8]) -> Result<Vec<Bytes>> {
+        let (shard_size, total_size) = self.need_size(data.len());
+
+        // 生成一个新的 所需的所有分片数据长度
+        let mut data_buffer = BytesMut::with_capacity(total_size);
+
+        // 复制源数据
+        data_buffer.extend_from_slice(data);
+        data_buffer.resize(total_size, 0u8);
+
+        {
+            // ec encode, 结果会写进 data_buffer
+            let data_slices: SmallVec<[&mut [u8]; 16]> = data_buffer.chunks_exact_mut(shard_size).collect();
+
+            // partiy 数量大于 0 才 ec
+            if self.parity_shards > 0 {
+                self.encoder.as_ref().unwrap().encode(data_slices).map_err(Error::other)?;
+            }
+        }
+
+        // 零拷贝分片，所有 shard 引用 data_buffer
+        let mut data_buffer = data_buffer.freeze();
+        let mut shards = Vec::with_capacity(self.total_shard_count());
+        for _ in 0..self.total_shard_count() {
+            let shard = data_buffer.split_to(shard_size);
+            shards.push(shard);
+        }
+
+        Ok(shards)
+    }
+
+    pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> Result<()> {
+        if self.parity_shards > 0 {
+            self.encoder.as_ref().unwrap().reconstruct(shards).map_err(Error::other)?;
+        }
+
+        Ok(())
+    }
+
+    // 每个分片长度，所需要的总长度
+    fn need_size(&self, data_size: usize) -> (usize, usize) {
+        let shard_size = self.shard_size(data_size);
+        (shard_size, shard_size * (self.total_shard_count()))
+    }
+
+    // 算出每个分片大小
+    pub fn shard_size(&self, data_size: usize) -> usize {
+        data_size.div_ceil(self.data_shards)
+    }
+    // returns final erasure size from original size.
+    pub fn shard_file_size(&self, total_size: usize) -> usize {
+        if total_size == 0 {
+            return 0;
+        }
+
+        let num_shards = total_size / self.block_size;
+        let last_block_size = total_size % self.block_size;
+        let last_shard_size = last_block_size.div_ceil(self.data_shards);
+        num_shards * self.shard_size(self.block_size) + last_shard_size
+
+        // // 因为写入的时候 ec 需要补全，所以最后一个长度应该也是一样的
+        // if last_block_size != 0 {
+        //     num_shards += 1
+        // }
+        // num_shards * self.shard_size(self.block_size)
+    }
+
+    // where erasure reading begins.
+    pub fn shard_file_offset(&self, start_offset: usize, length: usize, total_length: usize) -> usize {
+        let shard_size = self.shard_size(self.block_size);
+        let shard_file_size = self.shard_file_size(total_length);
+        let end_shard = (start_offset + length) / self.block_size;
+        let mut till_offset = end_shard * shard_size + shard_size;
+        if till_offset > shard_file_size {
+            till_offset = shard_file_size;
+        }
+
+        till_offset
+    }
+
+    pub async fn heal(
+        &self,
+        writers: &mut [Option<BitrotWriter>],
+        readers: Vec<Option<BitrotReader>>,
+        total_length: usize,
+        _prefer: &[bool],
+    ) -> Result<()> {
+        info!(
+            "Erasure heal, writers len: {}, readers len: {}, total_length: {}",
+            writers.len(),
+            readers.len(),
+            total_length
+        );
+        if writers.len() != self.parity_shards + self.data_shards {
+            return Err(Error::other("invalid argument"));
+        }
+        let mut reader = ShardReader::new(readers, self, 0, total_length);
+
+        let start_block = 0;
+        let mut end_block = total_length / self.block_size;
+        if total_length % self.block_size != 0 {
+            end_block += 1;
+        }
+
+        let mut errs = Vec::new();
+        for _ in start_block..end_block {
+            let mut bufs = reader.read().await?;
+
+            if self.parity_shards > 0 {
+                self.encoder.as_ref().unwrap().reconstruct(&mut bufs).map_err(Error::other)?;
+            }
+
+            let shards = bufs.into_iter().flatten().map(Bytes::from).collect::<Vec<_>>();
+            if shards.len() != self.parity_shards + self.data_shards {
+                return Err(Error::other("can not reconstruct data"));
+            }
+
+            for (i, w) in writers.iter_mut().enumerate() {
+                if w.is_none() {
+                    continue;
+                }
+                match w.as_mut().unwrap().write(shards[i].clone()).await {
+                    Ok(_) => {}
+                    Err(e) => {
+                        info!("write failed, err: {:?}", e);
+                        errs.push(e);
+                    }
+                }
+            }
+        }
+        if !errs.is_empty() {
+            return Err(errs[0].clone().into());
+        }
+
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+pub trait Writer {
+    fn as_any(&self) -> &dyn Any;
+    async fn write(&mut self, buf: Bytes) -> Result<()>;
+    async fn close(&mut self) -> Result<()> {
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+pub trait ReadAt {
+    async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec<u8>, usize)>;
+}
+
+pub struct ShardReader {
+    readers: Vec<Option<BitrotReader>>, // 磁盘
+    data_block_count: usize,            // 总的分片数量
+    parity_block_count: usize,
+    shard_size: usize,      // 每个分片的块大小 一次读取一块
+    shard_file_size: usize, // 分片文件总长度
+    offset: usize,          // 在分片中的 offset
+}
+
+impl ShardReader {
+    pub fn new(readers: Vec<Option<BitrotReader>>, ec: &Erasure, offset: usize, total_length: usize) -> Self {
+        Self {
+            readers,
+            data_block_count: ec.data_shards,
+            parity_block_count: ec.parity_shards,
+            shard_size: ec.shard_size(ec.block_size),
+            shard_file_size: ec.shard_file_size(total_length),
+            offset: (offset / ec.block_size) * ec.shard_size(ec.block_size),
+        }
+    }
+
+    pub async fn read(&mut self) -> Result<Vec<Option<Vec<u8>>>> {
+        // let mut disks = self.readers;
+        let reader_length = self.readers.len();
+        // 需要读取的块长度
+        let mut read_length = self.shard_size;
+        if self.offset + read_length > self.shard_file_size {
+            read_length = self.shard_file_size - self.offset
+        }
+
+        if read_length == 0 {
+            return Ok(vec![None; reader_length]);
+        }
+
+        // debug!("shard reader read offset {}, shard_size {}", self.offset, read_length);
+
+        let mut futures = Vec::with_capacity(reader_length);
+        let mut errors = Vec::with_capacity(reader_length);
+
+        let mut ress = Vec::with_capacity(reader_length);
+
+        for disk in self.readers.iter_mut() {
+            // if disk.is_none() {
+            //     ress.push(None);
+            //     errors.push(Some(Error::new(DiskError::DiskNotFound)));
+            //     continue;
+            // }
+
+            // let disk: &mut BitrotReader = disk.as_mut().unwrap();
+            let offset = self.offset;
+            futures.push(async move {
+                if let Some(disk) = disk {
+                    disk.read_at(offset, read_length).await
+                } else {
+                    Err(DiskError::DiskNotFound)
+                }
+            });
+        }
+
+        let results = join_all(futures).await;
+        for result in results {
+            match result {
+                Ok((res, _)) => {
+                    ress.push(Some(res));
+                    errors.push(None);
+                }
+                Err(e) => {
+                    ress.push(None);
+                    errors.push(Some(e));
+                }
+            }
+        }
+
+        if !self.can_decode(&ress) {
+            warn!("ec decode read ress {:?}", &ress);
+            warn!("ec decode read errors {:?}", &errors);
+
+            return Err(Error::other("shard reader read failed"));
+        }
+
+        self.offset += self.shard_size;
+
+        Ok(ress)
+    }
+
+    fn can_decode(&self, bufs: &[Option<Vec<u8>>]) -> bool {
+        let c = bufs.iter().filter(|v| v.is_some()).count();
+        if self.parity_block_count > 0 {
+            c >= self.data_block_count
+        } else {
+            c == self.data_block_count
+        }
+    }
+}
+
+// fn shards_to_option_shards<T: Clone>(shards: &[Vec<T>]) -> Vec<Option<Vec<T>>> {
+//     let mut result = Vec::with_capacity(shards.len());
+
+//     for v in shards.iter() {
+//         let inner: Vec<T> = v.clone();
+//         result.push(Some(inner));
+//     }
+//     result
+// }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_erasure() {
+        let data_shards = 3;
+        let parity_shards = 2;
+        let data: &[u8] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
+        let ec = Erasure::new(data_shards, parity_shards, 1);
+        let shards = Arc::new(ec).encode_data(data).unwrap();
+        println!("shards:{:?}", shards);
+
+        let mut s: Vec<_> = shards
+            .iter()
+            .map(|d| if d.is_empty() { None } else { Some(d.to_vec()) })
+            .collect();
+
+        // let mut s = shards_to_option_shards(&shards);
+
+        // s[0] = None;
+        s[4] = None;
+        s[3] = None;
+
+        println!("sss:{:?}", &s);
+
+        let ec = Erasure::new(data_shards, parity_shards, 1);
+        ec.decode_data(&mut s).unwrap();
+        // ec.encoder.reconstruct(&mut s).unwrap();
+
+        println!("sss:{:?}", &s);
+    }
+}

crates/ecstore/src/erasure_coding/bitrot.rs

@@ -0,0 +1,482 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use bytes::Bytes;
+use pin_project_lite::pin_project;
+use rustfs_utils::HashAlgorithm;
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
+use tracing::error;
+use uuid::Uuid;
+
+pin_project! {
+    /// BitrotReader reads (hash+data) blocks from an async reader and verifies hash integrity.
+    pub struct BitrotReader<R> {
+        #[pin]
+        inner: R,
+        hash_algo: HashAlgorithm,
+        shard_size: usize,
+        buf: Vec<u8>,
+        hash_buf: Vec<u8>,
+        // hash_read: usize,
+        // data_buf: Vec<u8>,
+        // data_read: usize,
+        // hash_checked: bool,
+        id: Uuid,
+    }
+}
+
+impl<R> BitrotReader<R>
+where
+    R: AsyncRead + Unpin + Send + Sync,
+{
+    /// Create a new BitrotReader.
+    pub fn new(inner: R, shard_size: usize, algo: HashAlgorithm) -> Self {
+        let hash_size = algo.size();
+        Self {
+            inner,
+            hash_algo: algo,
+            shard_size,
+            buf: Vec::new(),
+            hash_buf: vec![0u8; hash_size],
+            // hash_read: 0,
+            // data_buf: Vec::new(),
+            // data_read: 0,
+            // hash_checked: false,
+            id: Uuid::new_v4(),
+        }
+    }
+
+    /// Read a single (hash+data) block, verify hash, and return the number of bytes read into `out`.
+    /// Returns an error if hash verification fails or data exceeds shard_size.
+    pub async fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
+        if out.len() > self.shard_size {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                format!("data size {} exceeds shard size {}", out.len(), self.shard_size),
+            ));
+        }
+
+        let hash_size = self.hash_algo.size();
+        // Read hash
+
+        if hash_size > 0 {
+            self.inner.read_exact(&mut self.hash_buf).await.map_err(|e| {
+                error!("bitrot reader read hash error: {}", e);
+                e
+            })?;
+        }
+
+        // Read data
+        let mut data_len = 0;
+        while data_len < out.len() {
+            let n = self.inner.read(&mut out[data_len..]).await.map_err(|e| {
+                error!("bitrot reader read data error: {}", e);
+                e
+            })?;
+            if n == 0 {
+                break;
+            }
+            data_len += n;
+        }
+
+        if hash_size > 0 {
+            let actual_hash = self.hash_algo.hash_encode(&out[..data_len]);
+            if actual_hash.as_ref() != self.hash_buf.as_slice() {
+                error!("bitrot reader hash mismatch, id={} data_len={}, out_len={}", self.id, data_len, out.len());
+                return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
+            }
+        }
+        Ok(data_len)
+    }
+}
+
+pin_project! {
+    /// BitrotWriter writes (hash+data) blocks to an async writer.
+    pub struct BitrotWriter<W> {
+        #[pin]
+        inner: W,
+        hash_algo: HashAlgorithm,
+        shard_size: usize,
+        buf: Vec<u8>,
+        finished: bool,
+    }
+}
+
+impl<W> BitrotWriter<W>
+where
+    W: AsyncWrite + Unpin + Send + Sync,
+{
+    /// Create a new BitrotWriter.
+    pub fn new(inner: W, shard_size: usize, algo: HashAlgorithm) -> Self {
+        let hash_algo = algo;
+        Self {
+            inner,
+            hash_algo,
+            shard_size,
+            buf: Vec::new(),
+            finished: false,
+        }
+    }
+
+    pub fn into_inner(self) -> W {
+        self.inner
+    }
+
+    /// Write a (hash+data) block. Returns the number of data bytes written.
+    /// Returns an error if called after a short write or if data exceeds shard_size.
+    pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        if buf.is_empty() {
+            return Ok(0);
+        }
+
+        if self.finished {
+            return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "bitrot writer already finished"));
+        }
+
+        if buf.len() > self.shard_size {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                format!("data size {} exceeds shard size {}", buf.len(), self.shard_size),
+            ));
+        }
+
+        if buf.len() < self.shard_size {
+            self.finished = true;
+        }
+
+        let hash_algo = &self.hash_algo;
+
+        if hash_algo.size() > 0 {
+            let hash = hash_algo.hash_encode(buf);
+            self.buf.extend_from_slice(hash.as_ref());
+        }
+
+        self.buf.extend_from_slice(buf);
+
+        self.inner.write_all(&self.buf).await?;
+
+        // self.inner.flush().await?;
+
+        let n = buf.len();
+
+        self.buf.clear();
+
+        Ok(n)
+    }
+
+    pub async fn shutdown(&mut self) -> std::io::Result<()> {
+        self.inner.shutdown().await
+    }
+}
+
+pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorithm) -> usize {
+    if algo != HashAlgorithm::HighwayHash256S {
+        return size;
+    }
+    size.div_ceil(shard_size) * algo.size() + size
+}
+
+pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
+    mut r: R,
+    want_size: usize,
+    part_size: usize,
+    algo: HashAlgorithm,
+    _want: Bytes, // FIXME: useless parameter?
+    mut shard_size: usize,
+) -> std::io::Result<()> {
+    let mut hash_buf = vec![0; algo.size()];
+    let mut left = want_size;
+
+    if left != bitrot_shard_file_size(part_size, shard_size, algo.clone()) {
+        return Err(std::io::Error::other("bitrot shard file size mismatch"));
+    }
+
+    while left > 0 {
+        let n = r.read_exact(&mut hash_buf).await?;
+        left -= n;
+
+        if left < shard_size {
+            shard_size = left;
+        }
+
+        let mut buf = vec![0; shard_size];
+        let read = r.read_exact(&mut buf).await?;
+
+        let actual_hash = algo.hash_encode(&buf);
+        if actual_hash.as_ref() != &hash_buf[0..n] {
+            return Err(std::io::Error::other("bitrot hash mismatch"));
+        }
+
+        left -= read;
+    }
+
+    Ok(())
+}
+
+/// Custom writer enum that supports inline buffer storage
+pub enum CustomWriter {
+    /// Inline buffer writer - stores data in memory
+    InlineBuffer(Vec<u8>),
+    /// Disk-based writer using tokio file
+    Other(Box<dyn AsyncWrite + Unpin + Send + Sync>),
+}
+
+impl CustomWriter {
+    /// Create a new inline buffer writer
+    pub fn new_inline_buffer() -> Self {
+        Self::InlineBuffer(Vec::new())
+    }
+
+    /// Create a new disk writer from any AsyncWrite implementation
+    pub fn new_tokio_writer<W>(writer: W) -> Self
+    where
+        W: AsyncWrite + Unpin + Send + Sync + 'static,
+    {
+        Self::Other(Box::new(writer))
+    }
+
+    /// Get the inline buffer data if this is an inline buffer writer
+    pub fn get_inline_data(&self) -> Option<&[u8]> {
+        match self {
+            Self::InlineBuffer(data) => Some(data),
+            Self::Other(_) => None,
+        }
+    }
+
+    /// Extract the inline buffer data, consuming the writer
+    pub fn into_inline_data(self) -> Option<Vec<u8>> {
+        match self {
+            Self::InlineBuffer(data) => Some(data),
+            Self::Other(_) => None,
+        }
+    }
+}
+
+impl AsyncWrite for CustomWriter {
+    fn poll_write(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &[u8],
+    ) -> std::task::Poll<std::io::Result<usize>> {
+        match self.get_mut() {
+            Self::InlineBuffer(data) => {
+                data.extend_from_slice(buf);
+                std::task::Poll::Ready(Ok(buf.len()))
+            }
+            Self::Other(writer) => {
+                let pinned_writer = std::pin::Pin::new(writer.as_mut());
+                pinned_writer.poll_write(cx, buf)
+            }
+        }
+    }
+
+    fn poll_flush(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
+        match self.get_mut() {
+            Self::InlineBuffer(_) => std::task::Poll::Ready(Ok(())),
+            Self::Other(writer) => {
+                let pinned_writer = std::pin::Pin::new(writer.as_mut());
+                pinned_writer.poll_flush(cx)
+            }
+        }
+    }
+
+    fn poll_shutdown(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
+        match self.get_mut() {
+            Self::InlineBuffer(_) => std::task::Poll::Ready(Ok(())),
+            Self::Other(writer) => {
+                let pinned_writer = std::pin::Pin::new(writer.as_mut());
+                pinned_writer.poll_shutdown(cx)
+            }
+        }
+    }
+}
+
+/// Wrapper around BitrotWriter that uses our custom writer
+pub struct BitrotWriterWrapper {
+    bitrot_writer: BitrotWriter<CustomWriter>,
+    writer_type: WriterType,
+}
+
+/// Enum to track the type of writer we're using
+enum WriterType {
+    InlineBuffer,
+    Other,
+}
+
+impl std::fmt::Debug for BitrotWriterWrapper {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("BitrotWriterWrapper")
+            .field(
+                "writer_type",
+                &match self.writer_type {
+                    WriterType::InlineBuffer => "InlineBuffer",
+                    WriterType::Other => "Other",
+                },
+            )
+            .finish()
+    }
+}
+
+impl BitrotWriterWrapper {
+    /// Create a new BitrotWriterWrapper with custom writer
+    pub fn new(writer: CustomWriter, shard_size: usize, checksum_algo: HashAlgorithm) -> Self {
+        let writer_type = match &writer {
+            CustomWriter::InlineBuffer(_) => WriterType::InlineBuffer,
+            CustomWriter::Other(_) => WriterType::Other,
+        };
+
+        Self {
+            bitrot_writer: BitrotWriter::new(writer, shard_size, checksum_algo),
+            writer_type,
+        }
+    }
+
+    /// Write data to the bitrot writer
+    pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.bitrot_writer.write(buf).await
+    }
+
+    pub async fn shutdown(&mut self) -> std::io::Result<()> {
+        self.bitrot_writer.shutdown().await
+    }
+
+    /// Extract the inline buffer data, consuming the wrapper
+    pub fn into_inline_data(self) -> Option<Vec<u8>> {
+        match self.writer_type {
+            WriterType::InlineBuffer => {
+                let writer = self.bitrot_writer.into_inner();
+                writer.into_inline_data()
+            }
+            WriterType::Other => None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::BitrotReader;
+    use super::BitrotWriter;
+    use rustfs_utils::HashAlgorithm;
+    use std::io::Cursor;
+
+    #[tokio::test]
+    async fn test_bitrot_read_write_ok() {
+        let data = b"hello world! this is a test shard.";
+        let data_size = data.len();
+        let shard_size = 8;
+
+        let buf: Vec<u8> = Vec::new();
+        let writer = Cursor::new(buf);
+        let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::HighwayHash256);
+
+        let mut n = 0;
+        for chunk in data.chunks(shard_size) {
+            n += bitrot_writer.write(chunk).await.unwrap();
+        }
+        assert_eq!(n, data.len());
+
+        // 读
+        let reader = bitrot_writer.into_inner();
+        let reader = Cursor::new(reader.into_inner());
+        let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256);
+        let mut out = Vec::new();
+        let mut n = 0;
+        while n < data_size {
+            let mut buf = vec![0u8; shard_size];
+            let m = bitrot_reader.read(&mut buf).await.unwrap();
+            assert_eq!(&buf[..m], &data[n..n + m]);
+
+            out.extend_from_slice(&buf[..m]);
+            n += m;
+        }
+
+        assert_eq!(n, data_size);
+        assert_eq!(data, &out[..]);
+    }
+
+    #[tokio::test]
+    async fn test_bitrot_read_hash_mismatch() {
+        let data = b"test data for bitrot";
+        let data_size = data.len();
+        let shard_size = 8;
+        let buf: Vec<u8> = Vec::new();
+        let writer = Cursor::new(buf);
+        let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::HighwayHash256);
+        for chunk in data.chunks(shard_size) {
+            let _ = bitrot_writer.write(chunk).await.unwrap();
+        }
+        let mut written = bitrot_writer.into_inner().into_inner();
+        // change the last byte to make hash mismatch
+        let pos = written.len() - 1;
+        written[pos] ^= 0xFF;
+        let reader = Cursor::new(written);
+        let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256);
+
+        let count = data_size.div_ceil(shard_size);
+
+        let mut idx = 0;
+        let mut n = 0;
+        while n < data_size {
+            let mut buf = vec![0u8; shard_size];
+            let res = bitrot_reader.read(&mut buf).await;
+
+            if idx == count - 1 {
+                // 最后一个块，应该返回错误
+                assert!(res.is_err());
+                assert_eq!(res.unwrap_err().kind(), std::io::ErrorKind::InvalidData);
+                break;
+            }
+
+            let m = res.unwrap();
+
+            assert_eq!(&buf[..m], &data[n..n + m]);
+
+            n += m;
+            idx += 1;
+        }
+    }
+
+    #[tokio::test]
+    async fn test_bitrot_read_write_none_hash() {
+        let data = b"bitrot none hash test data!";
+        let data_size = data.len();
+        let shard_size = 8;
+
+        let buf: Vec<u8> = Vec::new();
+        let writer = Cursor::new(buf);
+        let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::None);
+
+        let mut n = 0;
+        for chunk in data.chunks(shard_size) {
+            n += bitrot_writer.write(chunk).await.unwrap();
+        }
+        assert_eq!(n, data.len());
+
+        let reader = bitrot_writer.into_inner();
+        let reader = Cursor::new(reader.into_inner());
+        let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::None);
+        let mut out = Vec::new();
+        let mut n = 0;
+        while n < data_size {
+            let mut buf = vec![0u8; shard_size];
+            let m = bitrot_reader.read(&mut buf).await.unwrap();
+            assert_eq!(&buf[..m], &data[n..n + m]);
+            out.extend_from_slice(&buf[..m]);
+            n += m;
+        }
+        assert_eq!(n, data_size);
+        assert_eq!(data, &out[..]);
+    }
+}

crates/ecstore/src/erasure_coding/decode.rs

@@ -0,0 +1,296 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::BitrotReader;
+use super::Erasure;
+use crate::disk::error::Error;
+use crate::disk::error_reduce::reduce_errs;
+use futures::future::join_all;
+use pin_project_lite::pin_project;
+use std::io;
+use std::io::ErrorKind;
+use tokio::io::AsyncRead;
+use tokio::io::AsyncWrite;
+use tokio::io::AsyncWriteExt;
+use tracing::error;
+
+pin_project! {
+pub(crate) struct ParallelReader<R> {
+    #[pin]
+    readers: Vec<Option<BitrotReader<R>>>,
+    offset: usize,
+    shard_size: usize,
+    shard_file_size: usize,
+    data_shards: usize,
+    total_shards: usize,
+}
+}
+
+impl<R> ParallelReader<R>
+where
+    R: AsyncRead + Unpin + Send + Sync,
+{
+    // readers传入前应处理disk错误，确保每个reader达到可用数量的BitrotReader
+    pub fn new(readers: Vec<Option<BitrotReader<R>>>, e: Erasure, offset: usize, total_length: usize) -> Self {
+        let shard_size = e.shard_size();
+        let shard_file_size = e.shard_file_size(total_length as i64) as usize;
+
+        let offset = (offset / e.block_size) * shard_size;
+
+        // 确保offset不超过shard_file_size
+
+        ParallelReader {
+            readers,
+            offset,
+            shard_size,
+            shard_file_size,
+            data_shards: e.data_shards,
+            total_shards: e.data_shards + e.parity_shards,
+        }
+    }
+}
+
+impl<R> ParallelReader<R>
+where
+    R: AsyncRead + Unpin + Send + Sync,
+{
+    pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
+        // if self.readers.len() != self.total_shards {
+        //     return Err(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers"));
+        // }
+
+        let shard_size = if self.offset + self.shard_size > self.shard_file_size {
+            self.shard_file_size - self.offset
+        } else {
+            self.shard_size
+        };
+
+        if shard_size == 0 {
+            return (vec![None; self.readers.len()], vec![None; self.readers.len()]);
+        }
+
+        // 使用并发读取所有分片
+        let mut read_futs = Vec::with_capacity(self.readers.len());
+
+        for (i, opt_reader) in self.readers.iter_mut().enumerate() {
+            let future = if let Some(reader) = opt_reader.as_mut() {
+                Box::pin(async move {
+                    let mut buf = vec![0u8; shard_size];
+                    match reader.read(&mut buf).await {
+                        Ok(n) => {
+                            buf.truncate(n);
+                            (i, Ok(buf))
+                        }
+                        Err(e) => (i, Err(Error::from(e))),
+                    }
+                }) as std::pin::Pin<Box<dyn std::future::Future<Output = (usize, Result<Vec<u8>, Error>)> + Send>>
+            } else {
+                // reader是None时返回FileNotFound错误
+                Box::pin(async move { (i, Err(Error::FileNotFound)) })
+                    as std::pin::Pin<Box<dyn std::future::Future<Output = (usize, Result<Vec<u8>, Error>)> + Send>>
+            };
+            read_futs.push(future);
+        }
+
+        let results = join_all(read_futs).await;
+
+        let mut shards: Vec<Option<Vec<u8>>> = vec![None; self.readers.len()];
+        let mut errs = vec![None; self.readers.len()];
+
+        for (i, shard) in results.into_iter() {
+            match shard {
+                Ok(data) => {
+                    if !data.is_empty() {
+                        shards[i] = Some(data);
+                    }
+                }
+                Err(e) => {
+                    // error!("Error reading shard {}: {}", i, e);
+                    errs[i] = Some(e);
+                }
+            }
+        }
+
+        self.offset += shard_size;
+
+        (shards, errs)
+    }
+
+    pub fn can_decode(&self, shards: &[Option<Vec<u8>>]) -> bool {
+        shards.iter().filter(|s| s.is_some()).count() >= self.data_shards
+    }
+}
+
+/// 获取数据块总长度
+fn get_data_block_len(shards: &[Option<Vec<u8>>], data_blocks: usize) -> usize {
+    let mut size = 0;
+    for shard in shards.iter().take(data_blocks).flatten() {
+        size += shard.len();
+    }
+
+    size
+}
+
+/// 将编码块中的数据块写入目标，支持 offset 和 length
+async fn write_data_blocks<W>(
+    writer: &mut W,
+    en_blocks: &[Option<Vec<u8>>],
+    data_blocks: usize,
+    mut offset: usize,
+    length: usize,
+) -> std::io::Result<usize>
+where
+    W: tokio::io::AsyncWrite + Send + Sync + Unpin,
+{
+    if get_data_block_len(en_blocks, data_blocks) < length {
+        error!("write_data_blocks get_data_block_len < length");
+        return Err(io::Error::new(ErrorKind::UnexpectedEof, "Not enough data blocks to write"));
+    }
+
+    let mut total_written = 0;
+    let mut write_left = length;
+
+    for block_op in &en_blocks[..data_blocks] {
+        if block_op.is_none() {
+            error!("write_data_blocks block_op.is_none()");
+            return Err(io::Error::new(ErrorKind::UnexpectedEof, "Missing data block"));
+        }
+
+        let block = block_op.as_ref().unwrap();
+
+        if offset >= block.len() {
+            offset -= block.len();
+            continue;
+        }
+
+        let block_slice = &block[offset..];
+        offset = 0;
+
+        if write_left < block.len() {
+            writer.write_all(&block_slice[..write_left]).await.map_err(|e| {
+                error!("write_data_blocks write_all err: {}", e);
+                e
+            })?;
+
+            total_written += write_left;
+            break;
+        }
+
+        let n = block_slice.len();
+
+        writer.write_all(block_slice).await.map_err(|e| {
+            error!("write_data_blocks write_all2 err: {}", e);
+            e
+        })?;
+
+        write_left -= n;
+
+        total_written += n;
+    }
+
+    Ok(total_written)
+}
+
+impl Erasure {
+    pub async fn decode<W, R>(
+        &self,
+        writer: &mut W,
+        readers: Vec<Option<BitrotReader<R>>>,
+        offset: usize,
+        length: usize,
+        total_length: usize,
+    ) -> (usize, Option<std::io::Error>)
+    where
+        W: AsyncWrite + Send + Sync + Unpin,
+        R: AsyncRead + Unpin + Send + Sync,
+    {
+        if readers.len() != self.data_shards + self.parity_shards {
+            return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")));
+        }
+
+        if offset + length > total_length {
+            return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")));
+        }
+
+        let mut ret_err = None;
+
+        if length == 0 {
+            return (0, ret_err);
+        }
+
+        let mut written = 0;
+
+        let mut reader = ParallelReader::new(readers, self.clone(), offset, total_length);
+
+        let start = offset / self.block_size;
+        let end = (offset + length) / self.block_size;
+
+        for i in start..=end {
+            let (block_offset, block_length) = if start == end {
+                (offset % self.block_size, length)
+            } else if i == start {
+                (offset % self.block_size, self.block_size - (offset % self.block_size))
+            } else if i == end {
+                (0, (offset + length) % self.block_size)
+            } else {
+                (0, self.block_size)
+            };
+
+            if block_length == 0 {
+                // error!("erasure decode decode block_length == 0");
+                break;
+            }
+
+            let (mut shards, errs) = reader.read().await;
+
+            if ret_err.is_none() {
+                if let (_, Some(err)) = reduce_errs(&errs, &[]) {
+                    if err == Error::FileNotFound || err == Error::FileCorrupt {
+                        ret_err = Some(err.into());
+                    }
+                }
+            }
+
+            if !reader.can_decode(&shards) {
+                error!("erasure decode can_decode errs: {:?}", &errs);
+                ret_err = Some(Error::ErasureReadQuorum.into());
+                break;
+            }
+
+            // Decode the shards
+            if let Err(e) = self.decode_data(&mut shards) {
+                error!("erasure decode decode_data err: {:?}", e);
+                ret_err = Some(e);
+                break;
+            }
+
+            let n = match write_data_blocks(writer, &shards, self.data_shards, block_offset, block_length).await {
+                Ok(n) => n,
+                Err(e) => {
+                    error!("erasure decode write_data_blocks err: {:?}", e);
+                    ret_err = Some(e);
+                    break;
+                }
+            };
+
+            written += n;
+        }
+
+        if written < length {
+            ret_err = Some(Error::LessData.into());
+        }
+
+        (written, ret_err)
+    }
+}

crates/ecstore/src/erasure_coding/encode.rs

@@ -0,0 +1,174 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::BitrotWriterWrapper;
+use super::Erasure;
+use crate::disk::error::Error;
+use crate::disk::error_reduce::count_errs;
+use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_write_quorum_errs};
+use bytes::Bytes;
+use futures::StreamExt;
+use futures::stream::FuturesUnordered;
+use std::sync::Arc;
+use std::vec;
+use tokio::io::AsyncRead;
+use tokio::sync::mpsc;
+use tracing::error;
+
+pub(crate) struct MultiWriter<'a> {
+    writers: &'a mut [Option<BitrotWriterWrapper>],
+    write_quorum: usize,
+    errs: Vec<Option<Error>>,
+}
+
+impl<'a> MultiWriter<'a> {
+    pub fn new(writers: &'a mut [Option<BitrotWriterWrapper>], write_quorum: usize) -> Self {
+        let length = writers.len();
+        MultiWriter {
+            writers,
+            write_quorum,
+            errs: vec![None; length],
+        }
+    }
+
+    async fn write_shard(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>, shard: &Bytes) {
+        match writer_opt {
+            Some(writer) => {
+                match writer.write(shard).await {
+                    Ok(n) => {
+                        if n < shard.len() {
+                            *err = Some(Error::ShortWrite);
+                            *writer_opt = None; // Mark as failed
+                        } else {
+                            *err = None;
+                        }
+                    }
+                    Err(e) => {
+                        *err = Some(Error::from(e));
+                    }
+                }
+            }
+            None => {
+                *err = Some(Error::DiskNotFound);
+            }
+        }
+    }
+
+    pub async fn write(&mut self, data: Vec<Bytes>) -> std::io::Result<()> {
+        assert_eq!(data.len(), self.writers.len());
+
+        {
+            let mut futures = FuturesUnordered::new();
+            for ((writer_opt, err), shard) in self.writers.iter_mut().zip(self.errs.iter_mut()).zip(data.iter()) {
+                if err.is_some() {
+                    continue; // Skip if we already have an error for this writer
+                }
+                futures.push(Self::write_shard(writer_opt, err, shard));
+            }
+            while let Some(()) = futures.next().await {}
+        }
+
+        let nil_count = self.errs.iter().filter(|&e| e.is_none()).count();
+        if nil_count >= self.write_quorum {
+            return Ok(());
+        }
+
+        if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
+            error!(
+                "reduce_write_quorum_errs: {:?}, offline-disks={}/{}, errs={:?}",
+                write_err,
+                count_errs(&self.errs, &Error::DiskNotFound),
+                self.writers.len(),
+                self.errs
+            );
+            return Err(std::io::Error::other(format!(
+                "Failed to write data: {} (offline-disks={}/{})",
+                write_err,
+                count_errs(&self.errs, &Error::DiskNotFound),
+                self.writers.len()
+            )));
+        }
+
+        Err(std::io::Error::other(format!(
+            "Failed to write data:  (offline-disks={}/{}): {}",
+            count_errs(&self.errs, &Error::DiskNotFound),
+            self.writers.len(),
+            self.errs
+                .iter()
+                .map(|e| e.as_ref().map_or("<nil>".to_string(), |e| e.to_string()))
+                .collect::<Vec<_>>()
+                .join(", ")
+        )))
+    }
+
+    pub async fn _shutdown(&mut self) -> std::io::Result<()> {
+        for writer in self.writers.iter_mut().flatten() {
+            writer.shutdown().await?;
+        }
+        Ok(())
+    }
+}
+
+impl Erasure {
+    pub async fn encode<R>(
+        self: Arc<Self>,
+        mut reader: R,
+        writers: &mut [Option<BitrotWriterWrapper>],
+        quorum: usize,
+    ) -> std::io::Result<(R, usize)>
+    where
+        R: AsyncRead + Send + Sync + Unpin + 'static,
+    {
+        let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(8);
+
+        let task = tokio::spawn(async move {
+            let block_size = self.block_size;
+            let mut total = 0;
+            let mut buf = vec![0u8; block_size];
+            loop {
+                match rustfs_utils::read_full(&mut reader, &mut buf).await {
+                    Ok(n) if n > 0 => {
+                        total += n;
+                        let res = self.encode_data(&buf[..n])?;
+                        if let Err(err) = tx.send(res).await {
+                            return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
+                        }
+                    }
+                    Ok(_) => break,
+                    Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
+                        break;
+                    }
+                    Err(e) => {
+                        return Err(e);
+                    }
+                }
+            }
+
+            Ok((reader, total))
+        });
+
+        let mut writers = MultiWriter::new(writers, quorum);
+
+        while let Some(block) = rx.recv().await {
+            if block.is_empty() {
+                break;
+            }
+            writers.write(block).await?;
+        }
+
+        let (reader, total) = task.await??;
+        // writers.shutdown().await?;
+        Ok((reader, total))
+    }
+}

crates/ecstore/src/erasure_coding/heal.rs

@@ -0,0 +1,74 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::BitrotReader;
+use super::BitrotWriterWrapper;
+use super::decode::ParallelReader;
+use crate::disk::error::{Error, Result};
+use crate::erasure_coding::encode::MultiWriter;
+use bytes::Bytes;
+use tokio::io::AsyncRead;
+use tracing::info;
+
+impl super::Erasure {
+    pub async fn heal<R>(
+        &self,
+        writers: &mut [Option<BitrotWriterWrapper>],
+        readers: Vec<Option<BitrotReader<R>>>,
+        total_length: usize,
+        _prefer: &[bool],
+    ) -> Result<()>
+    where
+        R: AsyncRead + Unpin + Send + Sync,
+    {
+        info!(
+            "Erasure heal, writers len: {}, readers len: {}, total_length: {}",
+            writers.len(),
+            readers.len(),
+            total_length
+        );
+        if writers.len() != self.parity_shards + self.data_shards {
+            return Err(Error::other("invalid argument"));
+        }
+        let mut reader = ParallelReader::new(readers, self.clone(), 0, total_length);
+
+        let start_block = 0;
+        let mut end_block = total_length / self.block_size;
+        if total_length % self.block_size != 0 {
+            end_block += 1;
+        }
+
+        for _ in start_block..end_block {
+            let (mut shards, errs) = reader.read().await;
+
+            if errs.iter().filter(|e| e.is_none()).count() < self.data_shards {
+                return Err(Error::other(format!("can not reconstruct data: not enough data shards {errs:?}")));
+            }
+
+            if self.parity_shards > 0 {
+                self.decode_data(&mut shards)?;
+            }
+
+            let shards = shards
+                .into_iter()
+                .map(|s| Bytes::from(s.unwrap_or_default()))
+                .collect::<Vec<_>>();
+
+            let mut writers = MultiWriter::new(writers, self.data_shards);
+            writers.write(shards).await?;
+        }
+
+        Ok(())
+    }
+}

crates/ecstore/src/erasure_coding/mod.rs

@@ -0,0 +1,23 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod decode;
+pub mod encode;
+pub mod erasure;
+pub mod heal;
+
+mod bitrot;
+pub use bitrot::*;
+
+pub use erasure::{Erasure, ReedSolomonEncoder, calc_shard_size};

crates/ecstore/src/event/mod.rs

@@ -0,0 +1,17 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod name;
+pub mod targetid;
+pub mod targetlist;

crates/ecstore/src/event/name.rs

@@ -0,0 +1,162 @@
+#![allow(unused_variables)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[derive(Default)]
+pub enum EventName {
+    ObjectAccessedGet,
+    ObjectAccessedGetRetention,
+    ObjectAccessedGetLegalHold,
+    ObjectAccessedHead,
+    ObjectAccessedAttributes,
+    ObjectCreatedCompleteMultipartUpload,
+    ObjectCreatedCopy,
+    ObjectCreatedPost,
+    ObjectCreatedPut,
+    ObjectCreatedPutRetention,
+    ObjectCreatedPutLegalHold,
+    ObjectCreatedPutTagging,
+    ObjectCreatedDeleteTagging,
+    ObjectRemovedDelete,
+    ObjectRemovedDeleteMarkerCreated,
+    ObjectRemovedDeleteAllVersions,
+    ObjectRemovedNoOP,
+    BucketCreated,
+    BucketRemoved,
+    ObjectReplicationFailed,
+    ObjectReplicationComplete,
+    ObjectReplicationMissedThreshold,
+    ObjectReplicationReplicatedAfterThreshold,
+    ObjectReplicationNotTracked,
+    ObjectRestorePost,
+    ObjectRestoreCompleted,
+    ObjectTransitionFailed,
+    ObjectTransitionComplete,
+    ObjectManyVersions,
+    ObjectLargeVersions,
+    PrefixManyFolders,
+    ILMDelMarkerExpirationDelete,
+    ObjectSingleTypesEnd,
+    ObjectAccessedAll,
+    ObjectCreatedAll,
+    ObjectRemovedAll,
+    ObjectReplicationAll,
+    ObjectRestoreAll,
+    ObjectTransitionAll,
+    ObjectScannerAll,
+    #[default]
+    Everything,
+}
+
+impl EventName {
+    fn expand(&self) -> Vec<EventName> {
+        todo!();
+    }
+
+    fn mask(&self) -> u64 {
+        todo!();
+    }
+}
+
+impl AsRef<str> for EventName {
+    fn as_ref(&self) -> &str {
+        match self {
+            EventName::BucketCreated => "s3:BucketCreated:*",
+            EventName::BucketRemoved => "s3:BucketRemoved:*",
+            EventName::ObjectAccessedAll => "s3:ObjectAccessed:*",
+            EventName::ObjectAccessedGet => "s3:ObjectAccessed:Get",
+            EventName::ObjectAccessedGetRetention => "s3:ObjectAccessed:GetRetention",
+            EventName::ObjectAccessedGetLegalHold => "s3:ObjectAccessed:GetLegalHold",
+            EventName::ObjectAccessedHead => "s3:ObjectAccessed:Head",
+            EventName::ObjectAccessedAttributes => "s3:ObjectAccessed:Attributes",
+            EventName::ObjectCreatedAll => "s3:ObjectCreated:*",
+            EventName::ObjectCreatedCompleteMultipartUpload => "s3:ObjectCreated:CompleteMultipartUpload",
+            EventName::ObjectCreatedCopy => "s3:ObjectCreated:Copy",
+            EventName::ObjectCreatedPost => "s3:ObjectCreated:Post",
+            EventName::ObjectCreatedPut => "s3:ObjectCreated:Put",
+            EventName::ObjectCreatedPutTagging => "s3:ObjectCreated:PutTagging",
+            EventName::ObjectCreatedDeleteTagging => "s3:ObjectCreated:DeleteTagging",
+            EventName::ObjectCreatedPutRetention => "s3:ObjectCreated:PutRetention",
+            EventName::ObjectCreatedPutLegalHold => "s3:ObjectCreated:PutLegalHold",
+            EventName::ObjectRemovedAll => "s3:ObjectRemoved:*",
+            EventName::ObjectRemovedDelete => "s3:ObjectRemoved:Delete",
+            EventName::ObjectRemovedDeleteMarkerCreated => "s3:ObjectRemoved:DeleteMarkerCreated",
+            EventName::ObjectRemovedNoOP => "s3:ObjectRemoved:NoOP",
+            EventName::ObjectRemovedDeleteAllVersions => "s3:ObjectRemoved:DeleteAllVersions",
+            EventName::ILMDelMarkerExpirationDelete => "s3:LifecycleDelMarkerExpiration:Delete",
+            EventName::ObjectReplicationAll => "s3:Replication:*",
+            EventName::ObjectReplicationFailed => "s3:Replication:OperationFailedReplication",
+            EventName::ObjectReplicationComplete => "s3:Replication:OperationCompletedReplication",
+            EventName::ObjectReplicationNotTracked => "s3:Replication:OperationNotTracked",
+            EventName::ObjectReplicationMissedThreshold => "s3:Replication:OperationMissedThreshold",
+            EventName::ObjectReplicationReplicatedAfterThreshold => "s3:Replication:OperationReplicatedAfterThreshold",
+            EventName::ObjectRestoreAll => "s3:ObjectRestore:*",
+            EventName::ObjectRestorePost => "s3:ObjectRestore:Post",
+            EventName::ObjectRestoreCompleted => "s3:ObjectRestore:Completed",
+            EventName::ObjectTransitionAll => "s3:ObjectTransition:*",
+            EventName::ObjectTransitionFailed => "s3:ObjectTransition:Failed",
+            EventName::ObjectTransitionComplete => "s3:ObjectTransition:Complete",
+            EventName::ObjectManyVersions => "s3:Scanner:ManyVersions",
+            EventName::ObjectLargeVersions => "s3:Scanner:LargeVersions",
+            EventName::PrefixManyFolders => "s3:Scanner:BigPrefix",
+            _ => "",
+        }
+    }
+}
+
+impl From<&str> for EventName {
+    fn from(s: &str) -> Self {
+        match s {
+            "s3:BucketCreated:*" => EventName::BucketCreated,
+            "s3:BucketRemoved:*" => EventName::BucketRemoved,
+            "s3:ObjectAccessed:*" => EventName::ObjectAccessedAll,
+            "s3:ObjectAccessed:Get" => EventName::ObjectAccessedGet,
+            "s3:ObjectAccessed:GetRetention" => EventName::ObjectAccessedGetRetention,
+            "s3:ObjectAccessed:GetLegalHold" => EventName::ObjectAccessedGetLegalHold,
+            "s3:ObjectAccessed:Head" => EventName::ObjectAccessedHead,
+            "s3:ObjectAccessed:Attributes" => EventName::ObjectAccessedAttributes,
+            "s3:ObjectCreated:*" => EventName::ObjectCreatedAll,
+            "s3:ObjectCreated:CompleteMultipartUpload" => EventName::ObjectCreatedCompleteMultipartUpload,
+            "s3:ObjectCreated:Copy" => EventName::ObjectCreatedCopy,
+            "s3:ObjectCreated:Post" => EventName::ObjectCreatedPost,
+            "s3:ObjectCreated:Put" => EventName::ObjectCreatedPut,
+            "s3:ObjectCreated:PutRetention" => EventName::ObjectCreatedPutRetention,
+            "s3:ObjectCreated:PutLegalHold" => EventName::ObjectCreatedPutLegalHold,
+            "s3:ObjectCreated:PutTagging" => EventName::ObjectCreatedPutTagging,
+            "s3:ObjectCreated:DeleteTagging" => EventName::ObjectCreatedDeleteTagging,
+            "s3:ObjectRemoved:*" => EventName::ObjectRemovedAll,
+            "s3:ObjectRemoved:Delete" => EventName::ObjectRemovedDelete,
+            "s3:ObjectRemoved:DeleteMarkerCreated" => EventName::ObjectRemovedDeleteMarkerCreated,
+            "s3:ObjectRemoved:NoOP" => EventName::ObjectRemovedNoOP,
+            "s3:ObjectRemoved:DeleteAllVersions" => EventName::ObjectRemovedDeleteAllVersions,
+            "s3:LifecycleDelMarkerExpiration:Delete" => EventName::ILMDelMarkerExpirationDelete,
+            "s3:Replication:*" => EventName::ObjectReplicationAll,
+            "s3:Replication:OperationFailedReplication" => EventName::ObjectReplicationFailed,
+            "s3:Replication:OperationCompletedReplication" => EventName::ObjectReplicationComplete,
+            "s3:Replication:OperationMissedThreshold" => EventName::ObjectReplicationMissedThreshold,
+            "s3:Replication:OperationReplicatedAfterThreshold" => EventName::ObjectReplicationReplicatedAfterThreshold,
+            "s3:Replication:OperationNotTracked" => EventName::ObjectReplicationNotTracked,
+            "s3:ObjectRestore:*" => EventName::ObjectRestoreAll,
+            "s3:ObjectRestore:Post" => EventName::ObjectRestorePost,
+            "s3:ObjectRestore:Completed" => EventName::ObjectRestoreCompleted,
+            "s3:ObjectTransition:Failed" => EventName::ObjectTransitionFailed,
+            "s3:ObjectTransition:Complete" => EventName::ObjectTransitionComplete,
+            "s3:ObjectTransition:*" => EventName::ObjectTransitionAll,
+            "s3:Scanner:ManyVersions" => EventName::ObjectManyVersions,
+            "s3:Scanner:LargeVersions" => EventName::ObjectLargeVersions,
+            "s3:Scanner:BigPrefix" => EventName::PrefixManyFolders,
+            _ => EventName::Everything,
+        }
+    }
+}

crates/ecstore/src/event/targetid.rs

@@ -0,0 +1,25 @@
+#![allow(clippy::all)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub struct TargetID {
+    id: String,
+    name: String,
+}
+
+impl TargetID {
+    fn to_string(&self) -> String {
+        format!("{}:{}", self.id, self.name)
+    }
+}

crates/ecstore/src/event/targetlist.rs

@@ -0,0 +1,45 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::atomic::AtomicI64;
+
+use super::targetid::TargetID;
+
+#[derive(Default)]
+pub struct TargetList {
+    pub current_send_calls: AtomicI64,
+    pub total_events: AtomicI64,
+    pub events_skipped: AtomicI64,
+    pub events_errors_total: AtomicI64,
+    //pub targets: HashMap<TargetID, Target>,
+    //pub queue:   AsyncEvent,
+    //pub targetStats: HashMap<TargetID, TargetStat>,
+}
+
+impl TargetList {
+    pub fn new() -> TargetList {
+        TargetList::default()
+    }
+}
+
+struct TargetStat {
+    current_send_calls: i64,
+    total_events: i64,
+    failed_events: i64,
+}
+
+struct TargetIDResult {
+    id: TargetID,
+    err: std::io::Error,
+}

crates/ecstore/src/event_notification.rs

@@ -0,0 +1,75 @@
+#![allow(unused_imports)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_variables)]
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+use crate::bucket::metadata::BucketMetadata;
+use crate::event::name::EventName;
+use crate::event::targetlist::TargetList;
+use crate::store::ECStore;
+use crate::store_api::ObjectInfo;
+
+pub struct EventNotifier {
+    target_list: TargetList,
+    //bucket_rules_map: HashMap<String , HashMap<EventName, Rules>>,
+}
+
+impl EventNotifier {
+    pub fn new() -> Arc<RwLock<Self>> {
+        Arc::new(RwLock::new(Self {
+            target_list: TargetList::new(),
+            //bucket_rules_map: HashMap::new(),
+        }))
+    }
+
+    fn get_arn_list(&self) -> Vec<String> {
+        todo!();
+    }
+
+    fn set(&self, bucket: &str, meta: BucketMetadata) {
+        todo!();
+    }
+
+    fn init_bucket_targets(&self, api: ECStore) -> Result<(), std::io::Error> {
+        /*if err := self.target_list.Add(globalNotifyTargetList.Targets()...); err != nil {
+          return err
+        }
+        self.target_list = self.target_list.Init(runtime.GOMAXPROCS(0)) // TODO: make this configurable (y4m4)
+        nil*/
+        todo!();
+    }
+
+    fn send(&self, args: EventArgs) {
+        todo!();
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct EventArgs {
+    pub event_name: String,
+    pub bucket_name: String,
+    pub object: ObjectInfo,
+    pub req_params: HashMap<String, String>,
+    pub resp_elements: HashMap<String, String>,
+    pub host: String,
+    pub user_agent: String,
+}
+
+impl EventArgs {}
+
+pub fn send_event(args: EventArgs) {}

crates/ecstore/src/global.rs

@@ -0,0 +1,184 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::heal::mrf::MRFState;
+use crate::{
+    bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys,
+    disk::DiskStore,
+    endpoints::{EndpointServerPools, PoolEndpoints, SetupType},
+    event_notification::EventNotifier,
+    heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState},
+    store::ECStore,
+    tier::tier::TierConfigMgr,
+};
+use lazy_static::lazy_static;
+use rustfs_policy::auth::Credentials;
+use std::{
+    collections::HashMap,
+    sync::{Arc, OnceLock},
+    time::SystemTime,
+};
+use tokio::sync::{OnceCell, RwLock};
+use uuid::Uuid;
+
+pub const DISK_ASSUME_UNKNOWN_SIZE: u64 = 1 << 30;
+pub const DISK_MIN_INODES: u64 = 1000;
+pub const DISK_FILL_FRACTION: f64 = 0.99;
+pub const DISK_RESERVE_FRACTION: f64 = 0.15;
+
+pub const DEFAULT_PORT: u16 = 9000;
+
+lazy_static! {
+static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new();
+pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new();
+pub static ref GLOBAL_LOCAL_DISK: Arc<RwLock<Vec<Option<DiskStore>>>> = Arc::new(RwLock::new(Vec::new()));
+pub static ref GLOBAL_IsErasure: RwLock<bool> = RwLock::new(false);
+pub static ref GLOBAL_IsDistErasure: RwLock<bool> = RwLock::new(false);
+pub static ref GLOBAL_IsErasureSD: RwLock<bool> = RwLock::new(false);
+pub static ref GLOBAL_LOCAL_DISK_MAP: Arc<RwLock<HashMap<String, Option<DiskStore>>>> = Arc::new(RwLock::new(HashMap::new()));
+pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<TypeLocalDiskSetDrives>> = Arc::new(RwLock::new(Vec::new()));
+pub static ref GLOBAL_Endpoints: OnceLock<EndpointServerPools> = OnceLock::new();
+pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0);
+pub static ref GLOBAL_BackgroundHealRoutine: Arc<HealRoutine> = HealRoutine::new();
+pub static ref GLOBAL_BackgroundHealState: Arc<AllHealState> = AllHealState::new(false);
+pub static ref GLOBAL_TierConfigMgr: Arc<RwLock<TierConfigMgr>> = TierConfigMgr::new();
+pub static ref GLOBAL_LifecycleSys: Arc<LifecycleSys> = LifecycleSys::new();
+pub static ref GLOBAL_EventNotifier: Arc<RwLock<EventNotifier>> = EventNotifier::new();
+//pub static ref GLOBAL_RemoteTargetTransport
+pub static ref GLOBAL_ALlHealState: Arc<AllHealState> = AllHealState::new(false);
+pub static ref GLOBAL_MRFState: Arc<MRFState> = Arc::new(MRFState::new());
+static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
+pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new();
+pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string();
+pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes());
+pub static ref GLOBAL_NodeNamesHex: HashMap<String, ()> = HashMap::new();}
+
+static GLOBAL_ACTIVE_CRED: OnceLock<Credentials> = OnceLock::new();
+
+pub fn init_global_action_cred(ak: Option<String>, sk: Option<String>) {
+    let ak = {
+        if let Some(k) = ak {
+            k
+        } else {
+            rustfs_utils::string::gen_access_key(20).unwrap_or_default()
+        }
+    };
+
+    let sk = {
+        if let Some(k) = sk {
+            k
+        } else {
+            rustfs_utils::string::gen_secret_key(32).unwrap_or_default()
+        }
+    };
+
+    GLOBAL_ACTIVE_CRED
+        .set(Credentials {
+            access_key: ak,
+            secret_key: sk,
+            ..Default::default()
+        })
+        .unwrap();
+}
+
+pub fn get_global_action_cred() -> Option<Credentials> {
+    GLOBAL_ACTIVE_CRED.get().cloned()
+}
+
+/// Get the global rustfs port
+pub fn global_rustfs_port() -> u16 {
+    if let Some(p) = GLOBAL_RUSTFS_PORT.get() {
+        *p
+    } else {
+        rustfs_config::DEFAULT_PORT
+    }
+}
+
+/// Set the global rustfs port
+pub fn set_global_rustfs_port(value: u16) {
+    GLOBAL_RUSTFS_PORT.set(value).expect("set_global_rustfs_port fail");
+}
+
+/// Get the global rustfs port
+pub fn set_global_deployment_id(id: Uuid) {
+    globalDeploymentIDPtr.set(id).unwrap();
+}
+
+/// Get the global deployment id
+pub fn get_global_deployment_id() -> Option<String> {
+    globalDeploymentIDPtr.get().map(|v| v.to_string())
+}
+/// Get the global deployment id
+pub fn set_global_endpoints(eps: Vec<PoolEndpoints>) {
+    GLOBAL_Endpoints
+        .set(EndpointServerPools::from(eps))
+        .expect("GLOBAL_Endpoints set failed")
+}
+
+/// Get the global endpoints
+pub fn get_global_endpoints() -> EndpointServerPools {
+    if let Some(eps) = GLOBAL_Endpoints.get() {
+        eps.clone()
+    } else {
+        EndpointServerPools::default()
+    }
+}
+
+pub fn new_object_layer_fn() -> Option<Arc<ECStore>> {
+    GLOBAL_OBJECT_API.get().cloned()
+}
+
+pub async fn set_object_layer(o: Arc<ECStore>) {
+    GLOBAL_OBJECT_API.set(o).expect("set_object_layer fail ")
+}
+
+pub async fn is_dist_erasure() -> bool {
+    let lock = GLOBAL_IsDistErasure.read().await;
+    *lock
+}
+
+pub async fn is_erasure_sd() -> bool {
+    let lock = GLOBAL_IsErasureSD.read().await;
+    *lock
+}
+
+pub async fn is_erasure() -> bool {
+    let lock = GLOBAL_IsErasure.read().await;
+    *lock
+}
+
+pub async fn update_erasure_type(setup_type: SetupType) {
+    let mut is_erasure = GLOBAL_IsErasure.write().await;
+    *is_erasure = setup_type == SetupType::Erasure;
+
+    let mut is_dist_erasure = GLOBAL_IsDistErasure.write().await;
+    *is_dist_erasure = setup_type == SetupType::DistErasure;
+
+    if *is_dist_erasure {
+        *is_erasure = true
+    }
+
+    let mut is_erasure_sd = GLOBAL_IsErasureSD.write().await;
+    *is_erasure_sd = setup_type == SetupType::ErasureSD;
+}
+
+// pub fn is_legacy() -> bool {
+//     if let Some(endpoints) = GLOBAL_Endpoints.get() {
+//         endpoints.as_ref().len() == 1 && endpoints.as_ref()[0].legacy
+//     } else {
+//         false
+//     }
+// }
+
+type TypeLocalDiskSetDrives = Vec<Vec<Vec<Option<DiskStore>>>>;

crates/ecstore/src/heal/background_heal_ops.rs

@@ -0,0 +1,484 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use futures::future::join_all;
+use rustfs_madmin::heal_commands::HealResultItem;
+use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
+use std::{cmp::Ordering, env, path::PathBuf, sync::Arc, time::Duration};
+use tokio::{
+    spawn,
+    sync::{
+        RwLock,
+        mpsc::{self, Receiver, Sender},
+    },
+    time::interval,
+};
+use tracing::{error, info};
+use uuid::Uuid;
+
+use super::{
+    heal_commands::HealOpts,
+    heal_ops::{HealSequence, new_bg_heal_sequence},
+};
+use crate::error::{Error, Result};
+use crate::global::GLOBAL_MRFState;
+use crate::heal::error::ERR_RETRY_HEALING;
+use crate::heal::heal_commands::{HEAL_ITEM_BUCKET, HealScanMode};
+use crate::heal::heal_ops::{BG_HEALING_UUID, HealSource};
+use crate::{
+    config::RUSTFS_CONFIG_PREFIX,
+    disk::{BUCKET_META_PREFIX, DiskAPI, DiskInfoOptions, RUSTFS_META_BUCKET, endpoint::Endpoint, error::DiskError},
+    global::{GLOBAL_BackgroundHealRoutine, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP},
+    heal::{
+        data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT},
+        data_usage_cache::DataUsageCache,
+        heal_commands::{init_healing_tracker, load_healing_tracker},
+        heal_ops::NOP_HEAL,
+    },
+    new_object_layer_fn,
+    store::get_disk_via_endpoint,
+    store_api::{BucketInfo, BucketOptions, StorageAPI},
+};
+
+pub static DEFAULT_MONITOR_NEW_DISK_INTERVAL: Duration = Duration::from_secs(10);
+
+pub async fn init_auto_heal() {
+    init_background_healing().await;
+    let v = env::var("_RUSTFS_AUTO_DRIVE_HEALING").unwrap_or("on".to_string());
+    if v == "on" {
+        info!("start monitor local disks and heal");
+        GLOBAL_BackgroundHealState
+            .push_heal_local_disks(&get_local_disks_to_heal().await)
+            .await;
+        spawn(async {
+            monitor_local_disks_and_heal().await;
+        });
+    }
+    spawn(async {
+        GLOBAL_MRFState.heal_routine().await;
+    });
+}
+
+async fn init_background_healing() {
+    let bg_seq = Arc::new(new_bg_heal_sequence());
+    for _ in 0..GLOBAL_BackgroundHealRoutine.workers {
+        let bg_seq_clone = bg_seq.clone();
+        spawn(async {
+            GLOBAL_BackgroundHealRoutine.add_worker(bg_seq_clone).await;
+        });
+    }
+    let _ = GLOBAL_BackgroundHealState.launch_new_heal_sequence(bg_seq).await;
+}
+
+pub async fn get_local_disks_to_heal() -> Vec<Endpoint> {
+    let mut disks_to_heal = Vec::new();
+    for (_, disk) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() {
+        if let Some(disk) = disk {
+            if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await {
+                if err == DiskError::UnformattedDisk {
+                    info!("get_local_disks_to_heal, disk is unformatted: {}", err);
+                    disks_to_heal.push(disk.endpoint());
+                }
+            }
+            let h = disk.healing().await;
+            if let Some(h) = h {
+                if !h.finished {
+                    info!("get_local_disks_to_heal, disk healing not finished");
+                    disks_to_heal.push(disk.endpoint());
+                }
+            }
+        }
+    }
+
+    // todo
+    // if disks_to_heal.len() == GLOBAL_Endpoints.read().await.n {
+
+    // }
+    disks_to_heal
+}
+
+async fn monitor_local_disks_and_heal() {
+    let mut interval = interval(DEFAULT_MONITOR_NEW_DISK_INTERVAL);
+
+    loop {
+        interval.tick().await;
+        let heal_disks = GLOBAL_BackgroundHealState.get_heal_local_disk_endpoints().await;
+        if heal_disks.is_empty() {
+            info!("heal local disks is empty");
+            interval.reset();
+            continue;
+        }
+
+        info!("heal local disks: {:?}", heal_disks);
+
+        let store = new_object_layer_fn().expect("errServerNotInitialized");
+        if let (_result, Some(err)) = store.heal_format(false).await.expect("heal format failed") {
+            error!("heal local disk format error: {}", err);
+            if err == Error::NoHealRequired {
+            } else {
+                info!("heal format err: {}", err.to_string());
+                interval.reset();
+                continue;
+            }
+        }
+
+        let mut futures = Vec::new();
+        for disk in heal_disks.into_ref().iter() {
+            let disk_clone = disk.clone();
+            futures.push(async move {
+                GLOBAL_BackgroundHealState
+                    .set_disk_healing_status(disk_clone.clone(), true)
+                    .await;
+                if heal_fresh_disk(&disk_clone).await.is_err() {
+                    info!("heal_fresh_disk is err");
+                    GLOBAL_BackgroundHealState
+                        .set_disk_healing_status(disk_clone.clone(), false)
+                        .await;
+                    return;
+                }
+                GLOBAL_BackgroundHealState.pop_heal_local_disks(&[disk_clone]).await;
+            });
+        }
+        let _ = join_all(futures).await;
+        interval.reset();
+    }
+}
+
+async fn heal_fresh_disk(endpoint: &Endpoint) -> Result<()> {
+    let (pool_idx, set_idx) = (endpoint.pool_idx as usize, endpoint.set_idx as usize);
+    let disk = match get_disk_via_endpoint(endpoint).await {
+        Some(disk) => disk,
+        None => {
+            return Err(Error::other(format!(
+                "Unexpected error disk must be initialized by now after formatting: {endpoint}"
+            )));
+        }
+    };
+
+    if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await {
+        match err {
+            DiskError::DriveIsRoot => {
+                return Ok(());
+            }
+            DiskError::UnformattedDisk => {}
+            _ => {
+                return Err(err.into());
+            }
+        }
+    }
+
+    let mut tracker = match load_healing_tracker(&Some(disk.clone())).await {
+        Ok(tracker) => tracker,
+        Err(err) => {
+            match err {
+                DiskError::FileNotFound => {
+                    return Ok(());
+                }
+                _ => {
+                    info!(
+                        "Unable to load healing tracker on '{}': {}, re-initializing..",
+                        disk.to_string(),
+                        err.to_string()
+                    );
+                }
+            }
+            init_healing_tracker(disk.clone(), &Uuid::new_v4().to_string()).await?
+        }
+    };
+
+    info!(
+        "Healing drive '{}' - 'mc admin heal alias/ --verbose' to check the current status.",
+        endpoint.to_string()
+    );
+
+    let Some(store) = new_object_layer_fn() else {
+        return Err(Error::other("errServerNotInitialized"));
+    };
+
+    let mut buckets = store.list_bucket(&BucketOptions::default()).await?;
+    buckets.push(BucketInfo {
+        name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(RUSTFS_CONFIG_PREFIX)])
+            .to_string_lossy()
+            .to_string(),
+        ..Default::default()
+    });
+    buckets.push(BucketInfo {
+        name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(BUCKET_META_PREFIX)])
+            .to_string_lossy()
+            .to_string(),
+        ..Default::default()
+    });
+
+    buckets.sort_by(|a, b| {
+        let a_has_prefix = a.name.starts_with(RUSTFS_META_BUCKET);
+        let b_has_prefix = b.name.starts_with(RUSTFS_META_BUCKET);
+
+        match (a_has_prefix, b_has_prefix) {
+            (true, false) => Ordering::Less,
+            (false, true) => Ordering::Greater,
+            _ => b.created.cmp(&a.created),
+        }
+    });
+
+    if let Ok(cache) = DataUsageCache::load(&store.pools[pool_idx].disk_set[set_idx], DATA_USAGE_CACHE_NAME).await {
+        let data_usage_info = cache.dui(DATA_USAGE_ROOT, &Vec::new());
+        tracker.objects_total_count = data_usage_info.objects_total_count;
+        tracker.objects_total_size = data_usage_info.objects_total_size;
+    };
+
+    tracker.set_queue_buckets(&buckets).await;
+    tracker.save().await?;
+
+    let tracker = Arc::new(RwLock::new(tracker));
+    let qb = tracker.read().await.queue_buckets.clone();
+    store.pools[pool_idx].disk_set[set_idx]
+        .clone()
+        .heal_erasure_set(&qb, tracker.clone())
+        .await?;
+    let mut tracker_w = tracker.write().await;
+    if tracker_w.items_failed > 0 && tracker_w.retry_attempts < 4 {
+        tracker_w.retry_attempts += 1;
+        tracker_w.reset_healing().await;
+        if let Err(err) = tracker_w.update().await {
+            info!("update tracker failed: {}", err.to_string());
+        }
+        return Err(Error::other(ERR_RETRY_HEALING));
+    }
+
+    if tracker_w.items_failed > 0 {
+        info!(
+            "Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}, failed: {}).",
+            disk.to_string(),
+            tracker_w.retry_attempts,
+            tracker_w.items_healed,
+            tracker_w.item_skipped,
+            tracker_w.items_failed
+        );
+    } else if tracker_w.retry_attempts > 0 {
+        info!(
+            "Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}).",
+            disk.to_string(),
+            tracker_w.retry_attempts,
+            tracker_w.items_healed,
+            tracker_w.item_skipped
+        );
+    } else {
+        info!(
+            "Healing of drive '{}' is finished (healed: {}, skipped: {}).",
+            disk.to_string(),
+            tracker_w.items_healed,
+            tracker_w.item_skipped
+        );
+    }
+
+    if tracker_w.heal_id.is_empty() {
+        if let Err(err) = tracker_w.delete().await {
+            error!("delete tracker failed: {}", err.to_string());
+        }
+    }
+    let Some(store) = new_object_layer_fn() else {
+        return Err(Error::other("errServerNotInitialized"));
+    };
+    let disks = store.get_disks(pool_idx, set_idx).await?;
+    for disk in disks.into_iter() {
+        if disk.is_none() {
+            continue;
+        }
+        let mut tracker = match load_healing_tracker(&disk).await {
+            Ok(tracker) => tracker,
+            Err(err) => {
+                match err {
+                    DiskError::FileNotFound => {}
+                    _ => {
+                        info!("Unable to load healing tracker on '{:?}': {}, re-initializing..", disk, err.to_string());
+                    }
+                }
+                continue;
+            }
+        };
+        if tracker.heal_id == tracker_w.heal_id {
+            tracker.finished = true;
+            tracker.update().await?;
+        }
+    }
+    Ok(())
+}
+
+#[derive(Debug)]
+pub struct HealTask {
+    pub bucket: String,
+    pub object: String,
+    pub version_id: String,
+    pub opts: HealOpts,
+    pub resp_tx: Option<Sender<HealResult>>,
+    pub resp_rx: Option<Receiver<HealResult>>,
+}
+
+impl HealTask {
+    pub fn new(bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Self {
+        Self {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            version_id: version_id.to_string(),
+            opts: *opts,
+            resp_tx: None,
+            resp_rx: None,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct HealResult {
+    pub result: HealResultItem,
+    pub err: Option<Error>,
+}
+
+pub struct HealRoutine {
+    pub tasks_tx: Sender<HealTask>,
+    tasks_rx: RwLock<Receiver<HealTask>>,
+    workers: usize,
+}
+
+impl HealRoutine {
+    pub fn new() -> Arc<Self> {
+        let mut workers = num_cpus::get() / 2;
+        if let Ok(env_heal_workers) = env::var("_RUSTFS_HEAL_WORKERS") {
+            if let Ok(num_healers) = env_heal_workers.parse::<usize>() {
+                workers = num_healers;
+            }
+        }
+
+        if workers == 0 {
+            workers = 4;
+        }
+
+        let (tx, rx) = mpsc::channel(100);
+        Arc::new(Self {
+            tasks_tx: tx,
+            tasks_rx: RwLock::new(rx),
+            workers,
+        })
+    }
+
+    pub async fn add_worker(&self, bgseq: Arc<HealSequence>) {
+        loop {
+            let mut d_res = HealResultItem::default();
+            let d_err: Option<Error>;
+            match self.tasks_rx.write().await.recv().await {
+                Some(task) => {
+                    info!("got task: {:?}", task);
+                    if task.bucket == NOP_HEAL {
+                        d_err = Some(Error::other("skip file"));
+                    } else if task.bucket == SLASH_SEPARATOR {
+                        match heal_disk_format(task.opts).await {
+                            Ok((res, err)) => {
+                                d_res = res;
+                                d_err = err;
+                            }
+                            Err(err) => d_err = Some(err),
+                        }
+                    } else {
+                        let store = new_object_layer_fn().expect("errServerNotInitialized");
+                        if task.object.is_empty() {
+                            match store.heal_bucket(&task.bucket, &task.opts).await {
+                                Ok(res) => {
+                                    d_res = res;
+                                    d_err = None;
+                                }
+                                Err(err) => d_err = Some(err),
+                            }
+                        } else {
+                            match store
+                                .heal_object(&task.bucket, &task.object, &task.version_id, &task.opts)
+                                .await
+                            {
+                                Ok((res, err)) => {
+                                    d_res = res;
+                                    d_err = err;
+                                }
+                                Err(err) => d_err = Some(err),
+                            }
+                        }
+                    }
+                    info!("task finished, task: {:?}", task);
+                    if let Some(resp_tx) = task.resp_tx {
+                        let _ = resp_tx
+                            .send(HealResult {
+                                result: d_res,
+                                err: d_err,
+                            })
+                            .await;
+                    } else {
+                        // when respCh is not set caller is not waiting but we
+                        // update the relevant metrics for them
+                        if d_err.is_none() {
+                            bgseq.count_healed(d_res.heal_item_type).await;
+                        } else {
+                            bgseq.count_failed(d_res.heal_item_type).await;
+                        }
+                    }
+                }
+                None => {
+                    info!("add_worker, tasks_rx was closed, return");
+                    return;
+                }
+            }
+        }
+    }
+}
+
+// pub fn active_listeners() -> Result<usize> {
+
+// }
+
+async fn heal_disk_format(opts: HealOpts) -> Result<(HealResultItem, Option<Error>)> {
+    let Some(store) = new_object_layer_fn() else {
+        return Err(Error::other("errServerNotInitialized"));
+    };
+
+    let (res, err) = store.heal_format(opts.dry_run).await?;
+    // return any error, ignore error returned when disks have
+    // already healed.
+    if err.is_some() {
+        return Ok((HealResultItem::default(), err));
+    }
+    Ok((res, err))
+}
+
+pub(crate) async fn heal_bucket(bucket: &str) -> Result<()> {
+    let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await;
+    if ok {
+        // bg_seq must be Some when ok is true
+        return bg_seq
+            .unwrap()
+            .queue_heal_task(
+                HealSource {
+                    bucket: bucket.to_string(),
+                    ..Default::default()
+                },
+                HEAL_ITEM_BUCKET.to_string(),
+            )
+            .await;
+    }
+    Ok(())
+}
+
+pub(crate) async fn heal_object(bucket: &str, object: &str, version_id: &str, scan_mode: HealScanMode) -> Result<()> {
+    let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await;
+    if ok {
+        // bg_seq must be Some when ok is true
+        return HealSequence::heal_object(bg_seq.unwrap(), bucket, object, version_id, scan_mode).await;
+    }
+    Ok(())
+}

crates/ecstore/src/heal/data_scanner_metric.rs

@@ -0,0 +1,486 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::data_scanner::CurrentScannerCycle;
+use crate::bucket::lifecycle::lifecycle;
+use chrono::Utc;
+use lazy_static::lazy_static;
+use rustfs_common::last_minute::{AccElem, LastMinuteLatency};
+use rustfs_madmin::metrics::ScannerMetrics as M_ScannerMetrics;
+use std::{
+    collections::HashMap,
+    pin::Pin,
+    sync::{
+        Arc,
+        atomic::{AtomicU64, Ordering},
+    },
+    time::{Duration, SystemTime},
+};
+use tokio::sync::{Mutex, RwLock};
+
+lazy_static! {
+    pub static ref globalScannerMetrics: Arc<ScannerMetrics> = Arc::new(ScannerMetrics::new());
+}
+
+#[derive(Clone, Debug, PartialEq, PartialOrd)]
+pub enum ScannerMetric {
+    // START Realtime metrics, that only records
+    // last minute latencies and total operation count.
+    ReadMetadata = 0,
+    CheckMissing,
+    SaveUsage,
+    ApplyAll,
+    ApplyVersion,
+    TierObjSweep,
+    HealCheck,
+    Ilm,
+    CheckReplication,
+    Yield,
+    CleanAbandoned,
+    ApplyNonCurrent,
+    HealAbandonedVersion,
+
+    // START Trace metrics:
+    StartTrace,
+    ScanObject, // Scan object. All operations included.
+    HealAbandonedObject,
+
+    // END realtime metrics:
+    LastRealtime,
+
+    // Trace only metrics:
+    ScanFolder,      // Scan a folder on disk, recursively.
+    ScanCycle,       // Full cycle, cluster global.
+    ScanBucketDrive, // Single bucket on one drive.
+    CompactFolder,   // Folder compacted.
+
+    // Must be last:
+    Last,
+}
+
+impl ScannerMetric {
+    /// Convert to string representation for metrics
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::ReadMetadata => "read_metadata",
+            Self::CheckMissing => "check_missing",
+            Self::SaveUsage => "save_usage",
+            Self::ApplyAll => "apply_all",
+            Self::ApplyVersion => "apply_version",
+            Self::TierObjSweep => "tier_obj_sweep",
+            Self::HealCheck => "heal_check",
+            Self::Ilm => "ilm",
+            Self::CheckReplication => "check_replication",
+            Self::Yield => "yield",
+            Self::CleanAbandoned => "clean_abandoned",
+            Self::ApplyNonCurrent => "apply_non_current",
+            Self::HealAbandonedVersion => "heal_abandoned_version",
+            Self::StartTrace => "start_trace",
+            Self::ScanObject => "scan_object",
+            Self::HealAbandonedObject => "heal_abandoned_object",
+            Self::LastRealtime => "last_realtime",
+            Self::ScanFolder => "scan_folder",
+            Self::ScanCycle => "scan_cycle",
+            Self::ScanBucketDrive => "scan_bucket_drive",
+            Self::CompactFolder => "compact_folder",
+            Self::Last => "last",
+        }
+    }
+
+    /// Convert from index back to enum (safe version)
+    pub fn from_index(index: usize) -> Option<Self> {
+        if index >= Self::Last as usize {
+            return None;
+        }
+        // Safe conversion using match instead of unsafe transmute
+        match index {
+            0 => Some(Self::ReadMetadata),
+            1 => Some(Self::CheckMissing),
+            2 => Some(Self::SaveUsage),
+            3 => Some(Self::ApplyAll),
+            4 => Some(Self::ApplyVersion),
+            5 => Some(Self::TierObjSweep),
+            6 => Some(Self::HealCheck),
+            7 => Some(Self::Ilm),
+            8 => Some(Self::CheckReplication),
+            9 => Some(Self::Yield),
+            10 => Some(Self::CleanAbandoned),
+            11 => Some(Self::ApplyNonCurrent),
+            12 => Some(Self::HealAbandonedVersion),
+            13 => Some(Self::StartTrace),
+            14 => Some(Self::ScanObject),
+            15 => Some(Self::HealAbandonedObject),
+            16 => Some(Self::LastRealtime),
+            17 => Some(Self::ScanFolder),
+            18 => Some(Self::ScanCycle),
+            19 => Some(Self::ScanBucketDrive),
+            20 => Some(Self::CompactFolder),
+            21 => Some(Self::Last),
+            _ => None,
+        }
+    }
+}
+
+/// Thread-safe wrapper for LastMinuteLatency with atomic operations
+#[derive(Default)]
+pub struct LockedLastMinuteLatency {
+    latency: Arc<Mutex<LastMinuteLatency>>,
+}
+
+impl Clone for LockedLastMinuteLatency {
+    fn clone(&self) -> Self {
+        Self {
+            latency: Arc::clone(&self.latency),
+        }
+    }
+}
+
+impl LockedLastMinuteLatency {
+    pub fn new() -> Self {
+        Self {
+            latency: Arc::new(Mutex::new(LastMinuteLatency::default())),
+        }
+    }
+
+    /// Add a duration measurement
+    pub async fn add(&self, duration: Duration) {
+        self.add_size(duration, 0).await;
+    }
+
+    /// Add a duration measurement with size
+    pub async fn add_size(&self, duration: Duration, size: u64) {
+        let mut latency = self.latency.lock().await;
+        let now = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_secs();
+
+        let elem = AccElem {
+            n: 1,
+            total: duration.as_secs(),
+            size,
+        };
+        latency.add_all(now, &elem);
+    }
+
+    /// Get total accumulated metrics for the last minute
+    pub async fn total(&self) -> AccElem {
+        let mut latency = self.latency.lock().await;
+        latency.get_total()
+    }
+}
+
+/// Current path tracker for monitoring active scan paths
+struct CurrentPathTracker {
+    current_path: Arc<RwLock<String>>,
+}
+
+impl CurrentPathTracker {
+    fn new(initial_path: String) -> Self {
+        Self {
+            current_path: Arc::new(RwLock::new(initial_path)),
+        }
+    }
+
+    async fn update_path(&self, path: String) {
+        *self.current_path.write().await = path;
+    }
+
+    async fn get_path(&self) -> String {
+        self.current_path.read().await.clone()
+    }
+}
+
+/// Main scanner metrics structure
+pub struct ScannerMetrics {
+    // All fields must be accessed atomically and aligned.
+    operations: Vec<AtomicU64>,
+    latency: Vec<LockedLastMinuteLatency>,
+    actions: Vec<AtomicU64>,
+    actions_latency: Vec<LockedLastMinuteLatency>,
+    // Current paths contains disk -> tracker mappings
+    current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
+
+    // Cycle information
+    cycle_info: Arc<RwLock<Option<CurrentScannerCycle>>>,
+}
+
+impl ScannerMetrics {
+    pub fn new() -> Self {
+        let operations = (0..ScannerMetric::Last as usize).map(|_| AtomicU64::new(0)).collect();
+
+        let latency = (0..ScannerMetric::LastRealtime as usize)
+            .map(|_| LockedLastMinuteLatency::new())
+            .collect();
+
+        Self {
+            operations,
+            latency,
+            actions: (0..ScannerMetric::Last as usize).map(|_| AtomicU64::new(0)).collect(),
+            actions_latency: vec![LockedLastMinuteLatency::default(); ScannerMetric::LastRealtime as usize],
+            current_paths: Arc::new(RwLock::new(HashMap::new())),
+            cycle_info: Arc::new(RwLock::new(None)),
+        }
+    }
+
+    /// Log scanner action with custom metadata - compatible with existing usage
+    pub fn log(metric: ScannerMetric) -> impl Fn(&HashMap<String, String>) {
+        let metric = metric as usize;
+        let start_time = SystemTime::now();
+        move |_custom: &HashMap<String, String>| {
+            let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
+
+            // Update operation count
+            globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
+
+            // Update latency for realtime metrics (spawn async task for this)
+            if (metric) < ScannerMetric::LastRealtime as usize {
+                let metric_index = metric;
+                tokio::spawn(async move {
+                    globalScannerMetrics.latency[metric_index].add(duration).await;
+                });
+            }
+
+            // Log trace metrics
+            if metric as u8 > ScannerMetric::StartTrace as u8 {
+                //debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric");
+            }
+        }
+    }
+
+    /// Time scanner action with size - returns function that takes size
+    pub fn time_size(metric: ScannerMetric) -> impl Fn(u64) {
+        let metric = metric as usize;
+        let start_time = SystemTime::now();
+        move |size: u64| {
+            let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
+
+            // Update operation count
+            globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
+
+            // Update latency for realtime metrics with size (spawn async task)
+            if (metric) < ScannerMetric::LastRealtime as usize {
+                let metric_index = metric;
+                tokio::spawn(async move {
+                    globalScannerMetrics.latency[metric_index].add_size(duration, size).await;
+                });
+            }
+        }
+    }
+
+    /// Time a scanner action - returns a closure to call when done
+    pub fn time(metric: ScannerMetric) -> impl Fn() {
+        let metric = metric as usize;
+        let start_time = SystemTime::now();
+        move || {
+            let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
+
+            // Update operation count
+            globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
+
+            // Update latency for realtime metrics (spawn async task)
+            if (metric) < ScannerMetric::LastRealtime as usize {
+                let metric_index = metric;
+                tokio::spawn(async move {
+                    globalScannerMetrics.latency[metric_index].add(duration).await;
+                });
+            }
+        }
+    }
+
+    /// Time N scanner actions - returns function that takes count, then returns completion function
+    pub fn time_n(metric: ScannerMetric) -> Box<dyn Fn(usize) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
+        let metric = metric as usize;
+        let start_time = SystemTime::now();
+        Box::new(move |count: usize| {
+            Box::new(move || {
+                let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
+
+                // Update operation count
+                globalScannerMetrics.operations[metric].fetch_add(count as u64, Ordering::Relaxed);
+
+                // Update latency for realtime metrics (spawn async task)
+                if (metric) < ScannerMetric::LastRealtime as usize {
+                    let metric_index = metric;
+                    tokio::spawn(async move {
+                        globalScannerMetrics.latency[metric_index].add(duration).await;
+                    });
+                }
+            })
+        })
+    }
+
+    pub fn time_ilm(a: lifecycle::IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
+        let a_clone = a as usize;
+        if a_clone == lifecycle::IlmAction::NoneAction as usize || a_clone >= lifecycle::IlmAction::ActionCount as usize {
+            return Box::new(move |_: u64| Box::new(move || {}));
+        }
+        let start = SystemTime::now();
+        Box::new(move |versions: u64| {
+            Box::new(move || {
+                let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
+                tokio::spawn(async move {
+                    globalScannerMetrics.actions[a_clone].fetch_add(versions, Ordering::Relaxed);
+                    globalScannerMetrics.actions_latency[a_clone].add(duration).await;
+                });
+            })
+        })
+    }
+
+    /// Increment time with specific duration
+    pub async fn inc_time(metric: ScannerMetric, duration: Duration) {
+        let metric = metric as usize;
+        // Update operation count
+        globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
+
+        // Update latency for realtime metrics
+        if (metric) < ScannerMetric::LastRealtime as usize {
+            globalScannerMetrics.latency[metric].add(duration).await;
+        }
+    }
+
+    /// Get lifetime operation count for a metric
+    pub fn lifetime(&self, metric: ScannerMetric) -> u64 {
+        let metric = metric as usize;
+        if (metric) >= ScannerMetric::Last as usize {
+            return 0;
+        }
+        self.operations[metric].load(Ordering::Relaxed)
+    }
+
+    /// Get last minute statistics for a metric
+    pub async fn last_minute(&self, metric: ScannerMetric) -> AccElem {
+        let metric = metric as usize;
+        if (metric) >= ScannerMetric::LastRealtime as usize {
+            return AccElem::default();
+        }
+        self.latency[metric].total().await
+    }
+
+    /// Set current cycle information
+    pub async fn set_cycle(&self, cycle: Option<CurrentScannerCycle>) {
+        *self.cycle_info.write().await = cycle;
+    }
+
+    /// Get current cycle information
+    pub async fn get_cycle(&self) -> Option<CurrentScannerCycle> {
+        self.cycle_info.read().await.clone()
+    }
+
+    /// Get current active paths
+    pub async fn get_current_paths(&self) -> Vec<String> {
+        let mut result = Vec::new();
+        let paths = self.current_paths.read().await;
+
+        for (disk, tracker) in paths.iter() {
+            let path = tracker.get_path().await;
+            result.push(format!("{disk}/{path}"));
+        }
+
+        result
+    }
+
+    /// Get number of active drives
+    pub async fn active_drives(&self) -> usize {
+        self.current_paths.read().await.len()
+    }
+
+    /// Generate metrics report
+    pub async fn report(&self) -> M_ScannerMetrics {
+        let mut metrics = M_ScannerMetrics::default();
+
+        // Set cycle information
+        if let Some(cycle) = self.get_cycle().await {
+            metrics.current_cycle = cycle.current;
+            metrics.cycles_completed_at = cycle.cycle_completed;
+            metrics.current_started = cycle.started;
+        }
+
+        metrics.collected_at = Utc::now();
+        metrics.active_paths = self.get_current_paths().await;
+
+        // Lifetime operations
+        for i in 0..ScannerMetric::Last as usize {
+            let count = self.operations[i].load(Ordering::Relaxed);
+            if count > 0 {
+                if let Some(metric) = ScannerMetric::from_index(i) {
+                    metrics.life_time_ops.insert(metric.as_str().to_string(), count);
+                }
+            }
+        }
+
+        // Last minute statistics for realtime metrics
+        for i in 0..ScannerMetric::LastRealtime as usize {
+            let last_min = self.latency[i].total().await;
+            if last_min.n > 0 {
+                if let Some(_metric) = ScannerMetric::from_index(i) {
+                    // Convert to madmin TimedAction format if needed
+                    // This would require implementing the conversion
+                }
+            }
+        }
+
+        metrics
+    }
+}
+
+// Type aliases for compatibility with existing code
+pub type UpdateCurrentPathFn = Arc<dyn Fn(&str) -> Pin<Box<dyn std::future::Future<Output = ()> + Send>> + Send + Sync>;
+pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn std::future::Future<Output = ()> + Send>> + Send + Sync>;
+
+/// Create a current path updater for tracking scan progress
+pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
+    let tracker = Arc::new(CurrentPathTracker::new(initial.to_string()));
+    let disk_name = disk.to_string();
+
+    // Store the tracker in global metrics
+    let tracker_clone = Arc::clone(&tracker);
+    let disk_clone = disk_name.clone();
+    tokio::spawn(async move {
+        globalScannerMetrics
+            .current_paths
+            .write()
+            .await
+            .insert(disk_clone, tracker_clone);
+    });
+
+    let update_fn = {
+        let tracker = Arc::clone(&tracker);
+        Arc::new(move |path: &str| -> Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
+            let tracker = Arc::clone(&tracker);
+            let path = path.to_string();
+            Box::pin(async move {
+                tracker.update_path(path).await;
+            })
+        })
+    };
+
+    let done_fn = {
+        let disk_name = disk_name.clone();
+        Arc::new(move || -> Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
+            let disk_name = disk_name.clone();
+            Box::pin(async move {
+                globalScannerMetrics.current_paths.write().await.remove(&disk_name);
+            })
+        })
+    };
+
+    (update_fn, done_fn)
+}
+
+impl Default for ScannerMetrics {
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/ecstore/src/heal/data_usage.rs

@@ -0,0 +1,221 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use crate::{
+    bucket::metadata_sys::get_replication_config,
+    config::com::{read_config, save_config},
+    disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
+    error::to_object_err,
+    new_object_layer_fn,
+    store::ECStore,
+};
+use lazy_static::lazy_static;
+use rustfs_utils::path::SLASH_SEPARATOR;
+use serde::{Deserialize, Serialize};
+use std::{collections::HashMap, sync::Arc, time::SystemTime};
+use tokio::sync::mpsc::Receiver;
+use tracing::{error, warn};
+
+pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
+const DATA_USAGE_OBJ_NAME: &str = ".usage.json";
+const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
+pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
+lazy_static! {
+    pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}", RUSTFS_META_BUCKET, SLASH_SEPARATOR, BUCKET_META_PREFIX);
+    pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_OBJ_NAME);
+    pub static ref DATA_USAGE_BLOOM_NAME_PATH: String =
+        format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_BLOOM_NAME);
+    pub static ref BACKGROUND_HEAL_INFO_PATH: String =
+        format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, ".background-heal.json");
+}
+
+// BucketTargetUsageInfo - bucket target usage info provides
+// - replicated size for all objects sent to this target
+// - replica size for all objects received from this target
+// - replication pending size for all objects pending replication to this target
+// - replication failed size for all objects failed replication to this target
+// - replica pending count
+// - replica failed count
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct BucketTargetUsageInfo {
+    pub replication_pending_size: u64,
+    pub replication_failed_size: u64,
+    pub replicated_size: u64,
+    pub replica_size: u64,
+    pub replication_pending_count: u64,
+    pub replication_failed_count: u64,
+    pub replicated_count: u64,
+}
+
+// BucketUsageInfo - bucket usage info provides
+// - total size of the bucket
+// - total objects in a bucket
+// - object size histogram per bucket
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct BucketUsageInfo {
+    pub size: u64,
+    // Following five fields suffixed with V1 are here for backward compatibility
+    // Total Size for objects that have not yet been replicated
+    pub replication_pending_size_v1: u64,
+    // Total size for objects that have witness one or more failures and will be retried
+    pub replication_failed_size_v1: u64,
+    // Total size for objects that have been replicated to destination
+    pub replicated_size_v1: u64,
+    // Total number of objects pending replication
+    pub replication_pending_count_v1: u64,
+    // Total number of objects that failed replication
+    pub replication_failed_count_v1: u64,
+
+    pub objects_count: u64,
+    pub object_size_histogram: HashMap<String, u64>,
+    pub object_versions_histogram: HashMap<String, u64>,
+    pub versions_count: u64,
+    pub delete_markers_count: u64,
+    pub replica_size: u64,
+    pub replica_count: u64,
+    pub replication_info: HashMap<String, BucketTargetUsageInfo>,
+}
+
+// DataUsageInfo represents data usage stats of the underlying Object API
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct DataUsageInfo {
+    pub total_capacity: u64,
+    pub total_used_capacity: u64,
+    pub total_free_capacity: u64,
+
+    // LastUpdate is the timestamp of when the data usage info was last updated.
+    // This does not indicate a full scan.
+    pub last_update: Option<SystemTime>,
+
+    // Objects total count across all buckets
+    pub objects_total_count: u64,
+    // Versions total count across all buckets
+    pub versions_total_count: u64,
+    // Delete markers total count across all buckets
+    pub delete_markers_total_count: u64,
+    // Objects total size across all buckets
+    pub objects_total_size: u64,
+    pub replication_info: HashMap<String, BucketTargetUsageInfo>,
+
+    // Total number of buckets in this cluster
+    pub buckets_count: u64,
+    // Buckets usage info provides following information across all buckets
+    // - total size of the bucket
+    // - total objects in a bucket
+    // - object size histogram per bucket
+    pub buckets_usage: HashMap<String, BucketUsageInfo>,
+    // Deprecated kept here for backward compatibility reasons.
+    pub bucket_sizes: HashMap<String, u64>,
+    // Todo: TierStats
+    // TierStats contains per-tier stats of all configured remote tiers
+}
+
+pub async fn store_data_usage_in_backend(mut rx: Receiver<DataUsageInfo>) {
+    let Some(store) = new_object_layer_fn() else {
+        error!("errServerNotInitialized");
+        return;
+    };
+
+    let mut attempts = 1;
+    loop {
+        match rx.recv().await {
+            Some(data_usage_info) => {
+                if let Ok(data) = serde_json::to_vec(&data_usage_info) {
+                    if attempts > 10 {
+                        let _ =
+                            save_config(store.clone(), &format!("{}{}", *DATA_USAGE_OBJ_NAME_PATH, ".bkp"), data.clone()).await;
+                        attempts += 1;
+                    }
+                    let _ = save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data).await;
+                    attempts += 1;
+                } else {
+                    continue;
+                }
+            }
+            None => {
+                return;
+            }
+        }
+    }
+}
+
+// TODO: cancel ctx
+pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo> {
+    let buf = match read_config(store, &DATA_USAGE_OBJ_NAME_PATH).await {
+        Ok(data) => data,
+        Err(e) => {
+            error!("Failed to read data usage info from backend: {}", e);
+            if e == Error::ConfigNotFound {
+                return Ok(DataUsageInfo::default());
+            }
+
+            return Err(to_object_err(e, vec![RUSTFS_META_BUCKET, &DATA_USAGE_OBJ_NAME_PATH]));
+        }
+    };
+
+    let mut data_usage_info: DataUsageInfo = serde_json::from_slice(&buf)?;
+
+    warn!("Loaded data usage info from backend {:?}", &data_usage_info);
+
+    if data_usage_info.buckets_usage.is_empty() {
+        data_usage_info.buckets_usage = data_usage_info
+            .bucket_sizes
+            .iter()
+            .map(|(bucket, &size)| {
+                (
+                    bucket.clone(),
+                    BucketUsageInfo {
+                        size,
+                        ..Default::default()
+                    },
+                )
+            })
+            .collect();
+    }
+
+    if data_usage_info.bucket_sizes.is_empty() {
+        data_usage_info.bucket_sizes = data_usage_info
+            .buckets_usage
+            .iter()
+            .map(|(bucket, bui)| (bucket.clone(), bui.size))
+            .collect();
+    }
+
+    for (bucket, bui) in &data_usage_info.buckets_usage {
+        if bui.replicated_size_v1 > 0
+            || bui.replication_failed_count_v1 > 0
+            || bui.replication_failed_size_v1 > 0
+            || bui.replication_pending_count_v1 > 0
+        {
+            if let Ok((cfg, _)) = get_replication_config(bucket).await {
+                if !cfg.role.is_empty() {
+                    data_usage_info.replication_info.insert(
+                        cfg.role.clone(),
+                        BucketTargetUsageInfo {
+                            replication_failed_size: bui.replication_failed_size_v1,
+                            replication_failed_count: bui.replication_failed_count_v1,
+                            replicated_size: bui.replicated_size_v1,
+                            replication_pending_count: bui.replication_pending_count_v1,
+                            replication_pending_size: bui.replication_pending_size_v1,
+                            ..Default::default()
+                        },
+                    );
+                }
+            }
+        }
+    }
+
+    Ok(data_usage_info)
+}

crates/ecstore/src/heal/data_usage_cache.rs

@@ -0,0 +1,928 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::config::com::save_config;
+use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
+use crate::error::{Error, Result};
+use crate::new_object_layer_fn;
+use crate::set_disk::SetDisks;
+use crate::store_api::{BucketInfo, ObjectIO, ObjectOptions};
+use bytesize::ByteSize;
+use http::HeaderMap;
+use path_clean::PathClean;
+use rand::Rng;
+use rmp_serde::Serializer;
+use s3s::dto::{BucketLifecycleConfiguration, ReplicationConfiguration};
+use serde::{Deserialize, Serialize};
+use std::collections::{HashMap, HashSet};
+use std::hash::{DefaultHasher, Hash, Hasher};
+use std::path::Path;
+use std::time::{Duration, SystemTime};
+use tokio::sync::mpsc::Sender;
+use tokio::time::sleep;
+
+use super::data_scanner::{DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS, SizeSummary};
+use super::data_usage::{BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo};
+
+// DATA_USAGE_BUCKET_LEN must be length of ObjectsHistogramIntervals
+pub const DATA_USAGE_BUCKET_LEN: usize = 11;
+pub const DATA_USAGE_VERSION_LEN: usize = 7;
+
+pub type DataUsageHashMap = HashSet<String>;
+
+struct ObjectHistogramInterval {
+    name: &'static str,
+    start: u64,
+    end: u64,
+}
+
+const OBJECTS_HISTOGRAM_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_BUCKET_LEN] = [
+    ObjectHistogramInterval {
+        name: "LESS_THAN_1024_B",
+        start: 0,
+        end: ByteSize::kib(1).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_1024_B_AND_64_KB",
+        start: ByteSize::kib(1).as_u64(),
+        end: ByteSize::kib(64).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_64_KB_AND_256_KB",
+        start: ByteSize::kib(64).as_u64(),
+        end: ByteSize::kib(256).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_256_KB_AND_512_KB",
+        start: ByteSize::kib(256).as_u64(),
+        end: ByteSize::kib(512).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_512_KB_AND_1_MB",
+        start: ByteSize::kib(512).as_u64(),
+        end: ByteSize::mib(1).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_1024B_AND_1_MB",
+        start: ByteSize::kib(1).as_u64(),
+        end: ByteSize::mib(1).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_1_MB_AND_10_MB",
+        start: ByteSize::mib(1).as_u64(),
+        end: ByteSize::mib(10).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_10_MB_AND_64_MB",
+        start: ByteSize::mib(10).as_u64(),
+        end: ByteSize::mib(64).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_64_MB_AND_128_MB",
+        start: ByteSize::mib(64).as_u64(),
+        end: ByteSize::mib(128).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_128_MB_AND_512_MB",
+        start: ByteSize::mib(128).as_u64(),
+        end: ByteSize::mib(512).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "GREATER_THAN_512_MB",
+        start: ByteSize::mib(512).as_u64(),
+        end: u64::MAX,
+    },
+];
+
+const OBJECTS_VERSION_COUNT_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_VERSION_LEN] = [
+    ObjectHistogramInterval {
+        name: "UNVERSIONED",
+        start: 0,
+        end: 0,
+    },
+    ObjectHistogramInterval {
+        name: "SINGLE_VERSION",
+        start: 1,
+        end: 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_2_AND_10",
+        start: 2,
+        end: 9,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_10_AND_100",
+        start: 10,
+        end: 99,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_100_AND_1000",
+        start: 100,
+        end: 999,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_1000_AND_10000",
+        start: 1000,
+        end: 9999,
+    },
+    ObjectHistogramInterval {
+        name: "GREATER_THAN_10000",
+        start: 10000,
+        end: u64::MAX,
+    },
+];
+
+#[derive(Clone, Copy, Default)]
+pub struct TierStats {
+    pub total_size: u64,
+    pub num_versions: i32,
+    pub num_objects: i32,
+}
+
+impl TierStats {
+    pub fn add(&self, u: &TierStats) -> TierStats {
+        TierStats {
+            total_size: self.total_size + u.total_size,
+            num_versions: self.num_versions + u.num_versions,
+            num_objects: self.num_objects + u.num_objects,
+        }
+    }
+}
+
+struct AllTierStats {
+    tiers: HashMap<String, TierStats>,
+}
+
+impl AllTierStats {
+    pub fn new() -> Self {
+        Self { tiers: HashMap::new() }
+    }
+
+    fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
+        for (tier, st) in tiers {
+            self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st));
+        }
+    }
+
+    fn merge(&mut self, other: AllTierStats) {
+        for (tier, st) in other.tiers {
+            self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st));
+        }
+    }
+
+    fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) {
+        for (tier, st) in &self.tiers {
+            stats.insert(
+                tier.clone(),
+                TierStats {
+                    total_size: st.total_size,
+                    num_versions: st.num_versions,
+                    num_objects: st.num_objects,
+                },
+            );
+        }
+    }
+}
+
+// sizeHistogram is a size histogram.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct SizeHistogram(Vec<u64>);
+
+impl Default for SizeHistogram {
+    fn default() -> Self {
+        Self(vec![0; DATA_USAGE_BUCKET_LEN])
+    }
+}
+
+impl SizeHistogram {
+    fn add(&mut self, size: u64) {
+        for (idx, interval) in OBJECTS_HISTOGRAM_INTERVALS.iter().enumerate() {
+            if size >= interval.start && size <= interval.end {
+                self.0[idx] += 1;
+                break;
+            }
+        }
+    }
+
+    pub fn to_map(&self) -> HashMap<String, u64> {
+        let mut res = HashMap::new();
+        let mut spl_count = 0;
+        for (count, oh) in self.0.iter().zip(OBJECTS_HISTOGRAM_INTERVALS.iter()) {
+            if ByteSize::kib(1).as_u64() == oh.start && oh.end == ByteSize::mib(1).as_u64() - 1 {
+                res.insert(oh.name.to_string(), spl_count);
+            } else if ByteSize::kib(1).as_u64() <= oh.start && oh.end < ByteSize::mib(1).as_u64() {
+                spl_count += count;
+                res.insert(oh.name.to_string(), *count);
+            } else {
+                res.insert(oh.name.to_string(), *count);
+            }
+        }
+        res
+    }
+}
+
+// versionsHistogram is a histogram of number of versions in an object.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct VersionsHistogram(Vec<u64>);
+
+impl Default for VersionsHistogram {
+    fn default() -> Self {
+        Self(vec![0; DATA_USAGE_VERSION_LEN])
+    }
+}
+
+impl VersionsHistogram {
+    fn add(&mut self, size: u64) {
+        for (idx, interval) in OBJECTS_VERSION_COUNT_INTERVALS.iter().enumerate() {
+            if size >= interval.start && size <= interval.end {
+                self.0[idx] += 1;
+                break;
+            }
+        }
+    }
+
+    pub fn to_map(&self) -> HashMap<String, u64> {
+        let mut res = HashMap::new();
+        for (count, ov) in self.0.iter().zip(OBJECTS_VERSION_COUNT_INTERVALS.iter()) {
+            res.insert(ov.name.to_string(), *count);
+        }
+        res
+    }
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct ReplicationStats {
+    pub pending_size: u64,
+    pub replicated_size: u64,
+    pub failed_size: u64,
+    pub failed_count: u64,
+    pub pending_count: u64,
+    pub missed_threshold_size: u64,
+    pub after_threshold_size: u64,
+    pub missed_threshold_count: u64,
+    pub after_threshold_count: u64,
+    pub replicated_count: u64,
+}
+
+impl ReplicationStats {
+    pub fn empty(&self) -> bool {
+        self.replicated_size == 0 && self.failed_size == 0 && self.failed_count == 0
+    }
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct ReplicationAllStats {
+    pub targets: HashMap<String, ReplicationStats>,
+    pub replica_size: u64,
+    pub replica_count: u64,
+}
+
+impl ReplicationAllStats {
+    pub fn empty(&self) -> bool {
+        if self.replica_size != 0 && self.replica_count != 0 {
+            return false;
+        }
+        for (_, v) in self.targets.iter() {
+            if !v.empty() {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
+pub struct DataUsageEntry {
+    pub children: DataUsageHashMap,
+    // These fields do not include any children.
+    pub size: usize,
+    pub objects: usize,
+    pub versions: usize,
+    pub delete_markers: usize,
+    pub obj_sizes: SizeHistogram,
+    pub obj_versions: VersionsHistogram,
+    pub replication_stats: Option<ReplicationAllStats>,
+    // Todo: tier
+    // pub all_tier_stats: ,
+    pub compacted: bool,
+}
+
+impl DataUsageEntry {
+    pub fn add_child(&mut self, hash: &DataUsageHash) {
+        if self.children.contains(&hash.key()) {
+            return;
+        }
+
+        self.children.insert(hash.key());
+    }
+
+    pub fn add_sizes(&mut self, summary: &SizeSummary) {
+        self.size += summary.total_size;
+        self.versions += summary.versions;
+        self.delete_markers += summary.delete_markers;
+        self.obj_sizes.add(summary.total_size as u64);
+        self.obj_versions.add(summary.versions as u64);
+
+        let replication_stats = if self.replication_stats.is_none() {
+            self.replication_stats = Some(ReplicationAllStats::default());
+            self.replication_stats.as_mut().unwrap()
+        } else {
+            self.replication_stats.as_mut().unwrap()
+        };
+        replication_stats.replica_size += summary.replica_size as u64;
+        replication_stats.replica_count += summary.replica_count as u64;
+
+        for (arn, st) in &summary.repl_target_stats {
+            let tgt_stat = replication_stats
+                .targets
+                .entry(arn.to_string())
+                .or_insert(ReplicationStats::default());
+            tgt_stat.pending_size += st.pending_size as u64;
+            tgt_stat.failed_size += st.failed_size as u64;
+            tgt_stat.replicated_size += st.replicated_size as u64;
+            tgt_stat.replicated_count += st.replicated_count as u64;
+            tgt_stat.failed_count += st.failed_count as u64;
+            tgt_stat.pending_count += st.pending_count as u64;
+        }
+        // Todo:: tiers
+    }
+
+    pub fn merge(&mut self, other: &DataUsageEntry) {
+        self.objects += other.objects;
+        self.versions += other.versions;
+        self.delete_markers += other.delete_markers;
+        self.size += other.size;
+        if let Some(o_rep) = &other.replication_stats {
+            if self.replication_stats.is_none() {
+                self.replication_stats = Some(ReplicationAllStats::default());
+            }
+            let s_rep = self.replication_stats.as_mut().unwrap();
+            s_rep.targets.clear();
+            s_rep.replica_size += o_rep.replica_size;
+            s_rep.replica_count += o_rep.replica_count;
+            for (arn, stat) in o_rep.targets.iter() {
+                let st = s_rep.targets.entry(arn.clone()).or_default();
+                *st = ReplicationStats {
+                    pending_size: stat.pending_size + st.pending_size,
+                    failed_size: stat.failed_size + st.failed_size,
+                    replicated_size: stat.replicated_size + st.replicated_size,
+                    pending_count: stat.pending_count + st.pending_count,
+                    failed_count: stat.failed_count + st.failed_count,
+                    replicated_count: stat.replicated_count + st.replicated_count,
+                    ..Default::default()
+                };
+            }
+        }
+
+        for (i, v) in other.obj_sizes.0.iter().enumerate() {
+            self.obj_sizes.0[i] += v;
+        }
+
+        for (i, v) in other.obj_versions.0.iter().enumerate() {
+            self.obj_versions.0[i] += v;
+        }
+
+        // todo: tiers
+    }
+}
+
+#[derive(Clone)]
+pub struct DataUsageEntryInfo {
+    pub name: String,
+    pub parent: String,
+    pub entry: DataUsageEntry,
+}
+
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
+pub struct DataUsageCacheInfo {
+    pub name: String,
+    pub next_cycle: u32,
+    pub last_update: Option<SystemTime>,
+    pub skip_healing: bool,
+    #[serde(skip)]
+    pub lifecycle: Option<BucketLifecycleConfiguration>,
+    #[serde(skip)]
+    pub updates: Option<Sender<DataUsageEntry>>,
+    #[serde(skip)]
+    pub replication: Option<ReplicationConfiguration>,
+}
+
+// impl Default for DataUsageCacheInfo {
+//     fn default() -> Self {
+//         Self {
+//             name: Default::default(),
+//             next_cycle: Default::default(),
+//             last_update: SystemTime::now(),
+//             skip_healing: Default::default(),
+//             updates: Default::default(),
+//             replication: Default::default(),
+//         }
+//     }
+// }
+
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
+pub struct DataUsageCache {
+    pub info: DataUsageCacheInfo,
+    pub cache: HashMap<String, DataUsageEntry>,
+}
+
+impl DataUsageCache {
+    pub async fn load(store: &SetDisks, name: &str) -> Result<Self> {
+        let mut d = DataUsageCache::default();
+        let mut retries = 0;
+        while retries < 5 {
+            let path = Path::new(BUCKET_META_PREFIX).join(name);
+            // warn!("Loading data usage cache from backend: {}", path.display());
+            match store
+                .get_object_reader(
+                    RUSTFS_META_BUCKET,
+                    path.to_str().unwrap(),
+                    None,
+                    HeaderMap::new(),
+                    &ObjectOptions {
+                        no_lock: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+            {
+                Ok(mut reader) => {
+                    if let Ok(info) = Self::unmarshal(&reader.read_all().await?) {
+                        d = info
+                    }
+                    break;
+                }
+                Err(err) => {
+                    // warn!("Failed to load data usage cache from backend: {}", &err);
+                    match err {
+                        Error::FileNotFound | Error::VolumeNotFound => {
+                            match store
+                                .get_object_reader(
+                                    RUSTFS_META_BUCKET,
+                                    name,
+                                    None,
+                                    HeaderMap::new(),
+                                    &ObjectOptions {
+                                        no_lock: true,
+                                        ..Default::default()
+                                    },
+                                )
+                                .await
+                            {
+                                Ok(mut reader) => {
+                                    if let Ok(info) = Self::unmarshal(&reader.read_all().await?) {
+                                        d = info
+                                    }
+                                    break;
+                                }
+                                Err(_) => match err {
+                                    Error::FileNotFound | Error::VolumeNotFound => {
+                                        break;
+                                    }
+                                    _ => {}
+                                },
+                            }
+                        }
+                        _ => {
+                            break;
+                        }
+                    }
+                }
+            }
+            retries += 1;
+            let dur = {
+                let mut rng = rand::rng();
+                rng.random_range(0..1_000)
+            };
+            sleep(Duration::from_millis(dur)).await;
+        }
+        Ok(d)
+    }
+
+    pub async fn save(&self, name: &str) -> Result<()> {
+        let Some(store) = new_object_layer_fn() else {
+            return Err(Error::other("errServerNotInitialized"));
+        };
+        let buf = self.marshal_msg()?;
+        let buf_clone = buf.clone();
+
+        let store_clone = store.clone();
+
+        let name = Path::new(BUCKET_META_PREFIX).join(name).to_string_lossy().to_string();
+
+        let name_clone = name.clone();
+        tokio::spawn(async move {
+            let _ = save_config(store_clone, &format!("{}{}", &name_clone, ".bkp"), buf_clone).await;
+        });
+        save_config(store, &name, buf).await?;
+        Ok(())
+    }
+
+    pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) {
+        let hash = hash_path(path);
+        self.cache.insert(hash.key(), e);
+        if !parent.is_empty() {
+            let phash = hash_path(parent);
+            let p = {
+                let p = self.cache.entry(phash.key()).or_default();
+                p.add_child(&hash);
+                p.clone()
+            };
+            self.cache.insert(phash.key(), p);
+        }
+    }
+
+    pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, e: &DataUsageEntry) {
+        self.cache.insert(hash.key(), e.clone());
+        if let Some(parent) = parent {
+            self.cache.entry(parent.key()).or_default().add_child(hash);
+        }
+    }
+
+    pub fn find(&self, path: &str) -> Option<DataUsageEntry> {
+        self.cache.get(&hash_path(path).key()).cloned()
+    }
+
+    pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap {
+        self.cache.entry(h.string()).or_default().children.clone()
+    }
+
+    pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry {
+        let mut root = root.clone();
+        for id in root.children.clone().iter() {
+            if let Some(e) = self.cache.get(id) {
+                let mut e = e.clone();
+                if !e.children.is_empty() {
+                    e = self.flatten(&e);
+                }
+                root.merge(&e);
+            }
+        }
+        root.children.clear();
+        root
+    }
+
+    pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
+        if let Some(e) = src.cache.get(&hash.string()) {
+            self.cache.insert(hash.key(), e.clone());
+            for ch in e.children.iter() {
+                if *ch == hash.key() {
+                    return;
+                }
+                self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone()));
+            }
+            if let Some(parent) = parent {
+                let p = self.cache.entry(parent.key()).or_default();
+                p.add_child(hash);
+            }
+        }
+    }
+
+    pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
+        let mut need_remove = Vec::new();
+        if let Some(v) = self.cache.get(&hash.string()) {
+            for child in v.children.iter() {
+                need_remove.push(child.clone());
+            }
+        }
+        self.cache.remove(&hash.string());
+        need_remove.iter().for_each(|child| {
+            self.delete_recursive(&DataUsageHash(child.to_string()));
+        });
+    }
+
+    pub fn size_recursive(&self, path: &str) -> Option<DataUsageEntry> {
+        match self.find(path) {
+            Some(root) => {
+                if root.children.is_empty() {
+                    return Some(root);
+                }
+                let mut flat = self.flatten(&root);
+                if flat.replication_stats.is_some() && flat.replication_stats.as_ref().unwrap().empty() {
+                    flat.replication_stats = None;
+                }
+                Some(flat)
+            }
+            None => None,
+        }
+    }
+
+    pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
+        let want = hash.key();
+        if let Some(last_index) = want.rfind('/') {
+            if let Some(v) = self.find(&want[0..last_index]) {
+                if v.children.contains(&want) {
+                    let found = hash_path(&want[0..last_index]);
+                    return Some(found);
+                }
+            }
+        }
+
+        for (k, v) in self.cache.iter() {
+            if v.children.contains(&want) {
+                let found = DataUsageHash(k.clone());
+                return Some(found);
+            }
+        }
+        None
+    }
+
+    pub fn is_compacted(&self, hash: &DataUsageHash) -> bool {
+        match self.cache.get(&hash.key()) {
+            Some(due) => due.compacted,
+            None => false,
+        }
+    }
+
+    pub fn force_compact(&mut self, limit: usize) {
+        if self.cache.len() < limit {
+            return;
+        }
+        let top = hash_path(&self.info.name).key();
+        let top_e = match self.find(&top) {
+            Some(e) => e,
+            None => return,
+        };
+        if top_e.children.len() > <u64 as TryInto<usize>>::try_into(DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS).unwrap() {
+            self.reduce_children_of(&hash_path(&self.info.name), limit, true);
+        }
+        if self.cache.len() <= limit {
+            return;
+        }
+
+        let mut found = HashSet::new();
+        found.insert(top);
+        mark(self, &top_e, &mut found);
+        self.cache.retain(|k, _| {
+            if !found.contains(k) {
+                return false;
+            }
+            true
+        });
+    }
+
+    pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
+        let e = match self.cache.get(&path.key()) {
+            Some(e) => e,
+            None => return,
+        };
+
+        if e.compacted {
+            return;
+        }
+
+        if e.children.len() > limit && compact_self {
+            let mut flat = self.size_recursive(&path.key()).unwrap_or_default();
+            flat.compacted = true;
+            self.delete_recursive(path);
+            self.replace_hashed(path, &None, &flat);
+            return;
+        }
+        let total = self.total_children_rec(&path.key());
+        if total < limit {
+            return;
+        }
+
+        let mut leaves = Vec::new();
+        let mut remove = total - limit;
+        add(self, path, &mut leaves);
+        leaves.sort_by(|a, b| a.objects.cmp(&b.objects));
+
+        while remove > 0 && !leaves.is_empty() {
+            let e = leaves.first().unwrap();
+            let candidate = e.path.clone();
+            if candidate == *path && !compact_self {
+                break;
+            }
+            let removing = self.total_children_rec(&candidate.key());
+            let mut flat = match self.size_recursive(&candidate.key()) {
+                Some(flat) => flat,
+                None => {
+                    leaves.remove(0);
+                    continue;
+                }
+            };
+
+            flat.compacted = true;
+            self.delete_recursive(&candidate);
+            self.replace_hashed(&candidate, &None, &flat);
+
+            remove -= removing;
+            leaves.remove(0);
+        }
+    }
+
+    pub fn total_children_rec(&self, path: &str) -> usize {
+        let root = self.find(path);
+
+        if root.is_none() {
+            return 0;
+        }
+        let root = root.unwrap();
+        if root.children.is_empty() {
+            return 0;
+        }
+
+        let mut n = root.children.len();
+        for ch in root.children.iter() {
+            n += self.total_children_rec(ch);
+        }
+        n
+    }
+
+    pub fn merge(&mut self, o: &DataUsageCache) {
+        let mut existing_root = self.root();
+        let other_root = o.root();
+        if existing_root.is_none() && other_root.is_none() {
+            return;
+        }
+        if other_root.is_none() {
+            return;
+        }
+        if existing_root.is_none() {
+            *self = o.clone();
+            return;
+        }
+        if o.info.last_update.gt(&self.info.last_update) {
+            self.info.last_update = o.info.last_update;
+        }
+
+        existing_root.as_mut().unwrap().merge(other_root.as_ref().unwrap());
+        self.cache.insert(hash_path(&self.info.name).key(), existing_root.unwrap());
+        let e_hash = self.root_hash();
+        for key in other_root.as_ref().unwrap().children.iter() {
+            let entry = &o.cache[key];
+            let flat = o.flatten(entry);
+            let mut existing = self.cache[key].clone();
+            existing.merge(&flat);
+            self.replace_hashed(&DataUsageHash(key.clone()), &Some(e_hash.clone()), &existing);
+        }
+    }
+
+    pub fn root_hash(&self) -> DataUsageHash {
+        hash_path(&self.info.name)
+    }
+
+    pub fn root(&self) -> Option<DataUsageEntry> {
+        self.find(&self.info.name)
+    }
+
+    pub fn dui(&self, path: &str, buckets: &[BucketInfo]) -> DataUsageInfo {
+        let e = match self.find(path) {
+            Some(e) => e,
+            None => return DataUsageInfo::default(),
+        };
+        let flat = self.flatten(&e);
+        DataUsageInfo {
+            last_update: self.info.last_update,
+            objects_total_count: flat.objects as u64,
+            versions_total_count: flat.versions as u64,
+            delete_markers_total_count: flat.delete_markers as u64,
+            objects_total_size: flat.size as u64,
+            buckets_count: e.children.len() as u64,
+            buckets_usage: self.buckets_usage_info(buckets),
+            ..Default::default()
+        }
+    }
+
+    pub fn buckets_usage_info(&self, buckets: &[BucketInfo]) -> HashMap<String, BucketUsageInfo> {
+        let mut dst = HashMap::new();
+        for bucket in buckets.iter() {
+            let e = match self.find(&bucket.name) {
+                Some(e) => e,
+                None => continue,
+            };
+            let flat = self.flatten(&e);
+            let mut bui = BucketUsageInfo {
+                size: flat.size as u64,
+                versions_count: flat.versions as u64,
+                objects_count: flat.objects as u64,
+                delete_markers_count: flat.delete_markers as u64,
+                object_size_histogram: flat.obj_sizes.to_map(),
+                object_versions_histogram: flat.obj_versions.to_map(),
+                ..Default::default()
+            };
+            if let Some(rs) = &flat.replication_stats {
+                bui.replica_size = rs.replica_size;
+                bui.replica_count = rs.replica_count;
+
+                for (arn, stat) in rs.targets.iter() {
+                    bui.replication_info.insert(
+                        arn.clone(),
+                        BucketTargetUsageInfo {
+                            replication_pending_size: stat.pending_size,
+                            replicated_size: stat.replicated_size,
+                            replication_failed_size: stat.failed_size,
+                            replication_pending_count: stat.pending_count,
+                            replication_failed_count: stat.failed_count,
+                            replicated_count: stat.replicated_count,
+                            ..Default::default()
+                        },
+                    );
+                }
+            }
+            dst.insert(bucket.name.clone(), bui);
+        }
+        dst
+    }
+
+    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
+        let mut buf = Vec::new();
+
+        self.serialize(&mut Serializer::new(&mut buf))?;
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self> {
+        let t: Self = rmp_serde::from_slice(buf)?;
+        Ok(t)
+    }
+}
+
+#[derive(Default, Clone)]
+struct Inner {
+    objects: usize,
+    path: DataUsageHash,
+}
+
+fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, leaves: &mut Vec<Inner>) {
+    let e = match data_usage_cache.cache.get(&path.key()) {
+        Some(e) => e,
+        None => return,
+    };
+    if !e.children.is_empty() {
+        return;
+    }
+
+    let sz = data_usage_cache.size_recursive(&path.key()).unwrap_or_default();
+    leaves.push(Inner {
+        objects: sz.objects,
+        path: path.clone(),
+    });
+    for ch in e.children.iter() {
+        add(data_usage_cache, &DataUsageHash(ch.clone()), leaves);
+    }
+}
+
+fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String>) {
+    for k in entry.children.iter() {
+        found.insert(k.to_string());
+        if let Some(ch) = duc.cache.get(k) {
+            mark(duc, ch, found);
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+pub struct DataUsageHash(pub String);
+
+impl DataUsageHash {
+    pub fn string(&self) -> String {
+        self.0.clone()
+    }
+
+    pub fn key(&self) -> String {
+        self.0.clone()
+    }
+
+    pub fn mod_(&self, cycle: u32, cycles: u32) -> bool {
+        if cycles <= 1 {
+            return cycles == 1;
+        }
+
+        let hash = self.calculate_hash();
+        hash as u32 % cycles == cycle % cycles
+    }
+
+    pub fn mod_alt(&self, cycle: u32, cycles: u32) -> bool {
+        if cycles <= 1 {
+            return cycles == 1;
+        }
+
+        let hash = self.calculate_hash();
+        (hash >> 32) as u32 % cycles == cycle % cycles
+    }
+
+    fn calculate_hash(&self) -> u64 {
+        let mut hasher = DefaultHasher::new();
+        self.0.hash(&mut hasher);
+        hasher.finish()
+    }
+}
+
+pub fn hash_path(data: &str) -> DataUsageHash {
+    DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string())
+}

crates/ecstore/src/heal/error.rs

@@ -0,0 +1,19 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const ERR_IGNORE_FILE_CONTRIB: &str = "ignore this file's contribution toward data-usage";
+pub const ERR_SKIP_FILE: &str = "skip this file";
+pub const ERR_HEAL_STOP_SIGNALLED: &str = "heal stop signaled";
+pub const ERR_HEAL_IDLE_TIMEOUT: &str = "healing results were not consumed for too long";
+pub const ERR_RETRY_HEALING: &str = "some items failed to heal, we will retry healing this drive again";

crates/ecstore/src/heal/heal_commands.rs

@@ -0,0 +1,544 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{
+    collections::{HashMap, HashSet},
+    path::Path,
+    time::SystemTime,
+};
+
+use crate::{
+    config::storageclass::{RRS, STANDARD},
+    disk::{BUCKET_META_PREFIX, DeleteOptions, DiskAPI, DiskStore, RUSTFS_META_BUCKET, error::DiskError, fs::read_file},
+    global::GLOBAL_BackgroundHealState,
+    heal::heal_ops::HEALING_TRACKER_FILENAME,
+    new_object_layer_fn,
+    store_api::{BucketInfo, StorageAPI},
+};
+use crate::{disk, error::Result};
+use chrono::{DateTime, Utc};
+use lazy_static::lazy_static;
+use serde::{Deserialize, Serialize};
+use time::OffsetDateTime;
+use tokio::sync::RwLock;
+
+use super::{background_heal_ops::get_local_disks_to_heal, heal_ops::BG_HEALING_UUID};
+
+pub type HealScanMode = usize;
+
+pub const HEAL_UNKNOWN_SCAN: HealScanMode = 0;
+pub const HEAL_NORMAL_SCAN: HealScanMode = 1;
+pub const HEAL_DEEP_SCAN: HealScanMode = 2;
+
+pub const HEAL_ITEM_METADATA: &str = "metadata";
+pub const HEAL_ITEM_BUCKET: &str = "bucket";
+pub const HEAL_ITEM_BUCKET_METADATA: &str = "bucket-metadata";
+pub const HEAL_ITEM_OBJECT: &str = "object";
+
+pub const DRIVE_STATE_OK: &str = "ok";
+pub const DRIVE_STATE_OFFLINE: &str = "offline";
+pub const DRIVE_STATE_CORRUPT: &str = "corrupt";
+pub const DRIVE_STATE_MISSING: &str = "missing";
+pub const DRIVE_STATE_PERMISSION: &str = "permission-denied";
+pub const DRIVE_STATE_FAULTY: &str = "faulty";
+pub const DRIVE_STATE_ROOT_MOUNT: &str = "root-mount";
+pub const DRIVE_STATE_UNKNOWN: &str = "unknown";
+pub const DRIVE_STATE_UNFORMATTED: &str = "unformatted"; // only returned by disk
+
+lazy_static! {
+    pub static ref TIME_SENTINEL: OffsetDateTime = OffsetDateTime::from_unix_timestamp(0).unwrap();
+}
+
+#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)]
+pub struct HealOpts {
+    pub recursive: bool,
+    #[serde(rename = "dryRun")]
+    pub dry_run: bool,
+    pub remove: bool,
+    pub recreate: bool,
+    #[serde(rename = "scanMode")]
+    pub scan_mode: HealScanMode,
+    #[serde(rename = "updateParity")]
+    pub update_parity: bool,
+    #[serde(rename = "nolock")]
+    pub no_lock: bool,
+    pub pool: Option<usize>,
+    pub set: Option<usize>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct HealStartSuccess {
+    #[serde(rename = "clientToken")]
+    pub client_token: String,
+    #[serde(rename = "clientAddress")]
+    pub client_address: String,
+    #[serde(rename = "startTime")]
+    pub start_time: DateTime<Utc>,
+}
+
+impl Default for HealStartSuccess {
+    fn default() -> Self {
+        Self {
+            client_token: Default::default(),
+            client_address: Default::default(),
+            start_time: Utc::now(),
+        }
+    }
+}
+
+pub type HealStopSuccess = HealStartSuccess;
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct HealingTracker {
+    #[serde(skip_serializing, skip_deserializing)]
+    pub disk: Option<DiskStore>,
+    pub id: String,
+    pub pool_index: Option<usize>,
+    pub set_index: Option<usize>,
+    pub disk_index: Option<usize>,
+    pub path: String,
+    pub endpoint: String,
+    pub started: Option<OffsetDateTime>,
+    pub last_update: Option<SystemTime>,
+    pub objects_total_count: u64,
+    pub objects_total_size: u64,
+    pub items_healed: u64,
+    pub items_failed: u64,
+    pub item_skipped: u64,
+    pub bytes_done: u64,
+    pub bytes_failed: u64,
+    pub bytes_skipped: u64,
+    pub bucket: String,
+    pub object: String,
+    pub resume_items_healed: u64,
+    pub resume_items_failed: u64,
+    pub resume_items_skipped: u64,
+    pub resume_bytes_done: u64,
+    pub resume_bytes_failed: u64,
+    pub resume_bytes_skipped: u64,
+    pub queue_buckets: Vec<String>,
+    pub healed_buckets: Vec<String>,
+    pub heal_id: String,
+    pub retry_attempts: u64,
+    pub finished: bool,
+    #[serde(skip_serializing, skip_deserializing)]
+    pub mu: RwLock<bool>,
+}
+
+impl HealingTracker {
+    pub fn marshal_msg(&self) -> disk::error::Result<Vec<u8>> {
+        Ok(serde_json::to_vec(self)?)
+    }
+
+    pub fn unmarshal_msg(data: &[u8]) -> disk::error::Result<Self> {
+        Ok(serde_json::from_slice::<HealingTracker>(data)?)
+    }
+
+    pub async fn reset_healing(&mut self) {
+        let _ = self.mu.write().await;
+        self.items_healed = 0;
+        self.items_failed = 0;
+        self.bytes_done = 0;
+        self.bytes_failed = 0;
+        self.resume_items_healed = 0;
+        self.resume_items_failed = 0;
+        self.resume_bytes_done = 0;
+        self.resume_bytes_failed = 0;
+        self.item_skipped = 0;
+        self.bytes_skipped = 0;
+
+        self.healed_buckets = Vec::new();
+        self.bucket = String::new();
+        self.object = String::new();
+    }
+
+    pub async fn get_last_update(&self) -> Option<SystemTime> {
+        let _ = self.mu.read().await;
+
+        self.last_update
+    }
+
+    pub async fn get_bucket(&self) -> String {
+        let _ = self.mu.read().await;
+
+        self.bucket.clone()
+    }
+
+    pub async fn set_bucket(&mut self, bucket: &str) {
+        let _ = self.mu.write().await;
+
+        self.bucket = bucket.to_string();
+    }
+
+    pub async fn get_object(&self) -> String {
+        let _ = self.mu.read().await;
+
+        self.object.clone()
+    }
+
+    pub async fn set_object(&mut self, object: &str) {
+        let _ = self.mu.write().await;
+
+        self.object = object.to_string();
+    }
+
+    pub async fn update_progress(&mut self, success: bool, skipped: bool, by: u64) {
+        let _ = self.mu.write().await;
+
+        if success {
+            self.items_healed += 1;
+            self.bytes_done += by;
+        } else if skipped {
+            self.item_skipped += 1;
+            self.bytes_skipped += by;
+        } else {
+            self.items_failed += 1;
+            self.bytes_failed += by;
+        }
+    }
+
+    pub async fn update(&mut self) -> disk::error::Result<()> {
+        if let Some(disk) = &self.disk {
+            if healing(disk.path().to_string_lossy().as_ref()).await?.is_none() {
+                return Err(DiskError::other(format!("healingTracker: drive {} is not marked as healing", self.id)));
+            }
+            let _ = self.mu.write().await;
+            if self.id.is_empty() || self.pool_index.is_none() || self.set_index.is_none() || self.disk_index.is_none() {
+                self.id = disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string());
+                let disk_location = disk.get_disk_location();
+                self.pool_index = disk_location.pool_idx;
+                self.set_index = disk_location.set_idx;
+                self.disk_index = disk_location.disk_idx;
+            }
+        }
+
+        self.save().await
+    }
+
+    pub async fn save(&mut self) -> disk::error::Result<()> {
+        let _ = self.mu.write().await;
+        if self.pool_index.is_none() || self.set_index.is_none() || self.disk_index.is_none() {
+            let Some(store) = new_object_layer_fn() else {
+                return Err(DiskError::other("errServerNotInitialized"));
+            };
+
+            // TODO: check error type
+            (self.pool_index, self.set_index, self.disk_index) =
+                store.get_pool_and_set(&self.id).await.map_err(|_| DiskError::DiskNotFound)?;
+        }
+
+        self.last_update = Some(SystemTime::now());
+
+        let htracker_bytes = self.marshal_msg()?;
+
+        GLOBAL_BackgroundHealState.update_heal_status(self).await;
+
+        if let Some(disk) = &self.disk {
+            let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME);
+            disk.write_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap(), htracker_bytes.into())
+                .await?;
+        }
+        Ok(())
+    }
+
+    pub async fn delete(&self) -> Result<()> {
+        if let Some(disk) = &self.disk {
+            let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME);
+            disk.delete(
+                RUSTFS_META_BUCKET,
+                file_path.to_str().unwrap(),
+                DeleteOptions {
+                    recursive: false,
+                    immediate: false,
+                    ..Default::default()
+                },
+            )
+            .await?;
+        }
+
+        Ok(())
+    }
+
+    pub async fn is_healed(&self, bucket: &str) -> bool {
+        let _ = self.mu.read().await;
+        for v in self.healed_buckets.iter() {
+            if v == bucket {
+                return true;
+            }
+        }
+
+        false
+    }
+
+    pub async fn resume(&mut self) {
+        let _ = self.mu.write().await;
+
+        self.items_healed = self.resume_items_healed;
+        self.items_failed = self.resume_items_failed;
+        self.item_skipped = self.resume_items_skipped;
+        self.bytes_done = self.resume_bytes_done;
+        self.bytes_failed = self.resume_bytes_failed;
+        self.bytes_skipped = self.resume_bytes_skipped;
+    }
+
+    pub async fn bucket_done(&mut self, bucket: &str) {
+        let _ = self.mu.write().await;
+
+        self.resume_items_healed = self.items_healed;
+        self.resume_items_failed = self.items_failed;
+        self.resume_items_skipped = self.item_skipped;
+        self.resume_bytes_done = self.bytes_done;
+        self.resume_bytes_failed = self.bytes_failed;
+        self.resume_bytes_skipped = self.bytes_skipped;
+        self.healed_buckets.push(bucket.to_string());
+
+        self.queue_buckets.retain(|x| x != bucket);
+    }
+
+    pub async fn set_queue_buckets(&mut self, buckets: &[BucketInfo]) {
+        let _ = self.mu.write().await;
+
+        buckets.iter().for_each(|bucket| {
+            if !self.healed_buckets.contains(&bucket.name) {
+                self.queue_buckets.push(bucket.name.clone());
+            }
+        });
+    }
+
+    pub async fn to_healing_disk(&self) -> rustfs_madmin::HealingDisk {
+        let _ = self.mu.read().await;
+
+        rustfs_madmin::HealingDisk {
+            id: self.id.clone(),
+            heal_id: self.heal_id.clone(),
+            pool_index: self.pool_index,
+            set_index: self.set_index,
+            disk_index: self.disk_index,
+            endpoint: self.endpoint.clone(),
+            path: self.path.clone(),
+            started: self.started,
+            last_update: self.last_update,
+            retry_attempts: self.retry_attempts,
+            objects_total_count: self.objects_total_count,
+            objects_total_size: self.objects_total_size,
+            items_healed: self.items_healed,
+            items_failed: self.items_failed,
+            item_skipped: self.item_skipped,
+            bytes_done: self.bytes_done,
+            bytes_failed: self.bytes_failed,
+            bytes_skipped: self.bytes_skipped,
+            objects_healed: self.items_healed,
+            objects_failed: self.items_failed,
+            bucket: self.bucket.clone(),
+            object: self.object.clone(),
+            queue_buckets: self.queue_buckets.clone(),
+            healed_buckets: self.healed_buckets.clone(),
+            finished: self.finished,
+        }
+    }
+}
+
+impl Clone for HealingTracker {
+    fn clone(&self) -> Self {
+        Self {
+            disk: self.disk.clone(),
+            id: self.id.clone(),
+            pool_index: self.pool_index,
+            set_index: self.set_index,
+            disk_index: self.disk_index,
+            path: self.path.clone(),
+            endpoint: self.endpoint.clone(),
+            started: self.started,
+            last_update: self.last_update,
+            objects_total_count: self.objects_total_count,
+            objects_total_size: self.objects_total_size,
+            items_healed: self.items_healed,
+            items_failed: self.items_failed,
+            item_skipped: self.item_skipped,
+            bytes_done: self.bytes_done,
+            bytes_failed: self.bytes_failed,
+            bytes_skipped: self.bytes_skipped,
+            bucket: self.bucket.clone(),
+            object: self.object.clone(),
+            resume_items_healed: self.resume_items_healed,
+            resume_items_failed: self.resume_items_failed,
+            resume_items_skipped: self.resume_items_skipped,
+            resume_bytes_done: self.resume_bytes_done,
+            resume_bytes_failed: self.resume_bytes_failed,
+            resume_bytes_skipped: self.resume_bytes_skipped,
+            queue_buckets: self.queue_buckets.clone(),
+            healed_buckets: self.healed_buckets.clone(),
+            heal_id: self.heal_id.clone(),
+            retry_attempts: self.retry_attempts,
+            finished: self.finished,
+            mu: RwLock::new(false),
+        }
+    }
+}
+
+pub async fn load_healing_tracker(disk: &Option<DiskStore>) -> disk::error::Result<HealingTracker> {
+    if let Some(disk) = disk {
+        let disk_id = disk.get_disk_id().await?;
+        if let Some(disk_id) = disk_id {
+            let disk_id = disk_id.to_string();
+            let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME);
+            let data = disk.read_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap()).await?;
+            let mut healing_tracker = HealingTracker::unmarshal_msg(&data)?;
+            if healing_tracker.id != disk_id && !healing_tracker.id.is_empty() {
+                return Err(DiskError::other(format!(
+                    "loadHealingTracker: drive id mismatch expected {}, got {}",
+                    healing_tracker.id, disk_id
+                )));
+            }
+            healing_tracker.id = disk_id;
+            healing_tracker.disk = Some(disk.clone());
+            Ok(healing_tracker)
+        } else {
+            Err(DiskError::other("loadHealingTracker: disk not have id"))
+        }
+    } else {
+        Err(DiskError::other("loadHealingTracker: nil drive given"))
+    }
+}
+
+pub async fn init_healing_tracker(disk: DiskStore, heal_id: &str) -> disk::error::Result<HealingTracker> {
+    let disk_location = disk.get_disk_location();
+    Ok(HealingTracker {
+        id: disk
+            .get_disk_id()
+            .await
+            .map_or("".to_string(), |id| id.map_or("".to_string(), |id| id.to_string())),
+        heal_id: heal_id.to_string(),
+        path: disk.to_string(),
+        endpoint: disk.endpoint().to_string(),
+        started: Some(OffsetDateTime::now_utc()),
+        pool_index: disk_location.pool_idx,
+        set_index: disk_location.set_idx,
+        disk_index: disk_location.disk_idx,
+        disk: Some(disk),
+        ..Default::default()
+    })
+}
+
+pub async fn healing(derive_path: &str) -> disk::error::Result<Option<HealingTracker>> {
+    let healing_file = Path::new(derive_path)
+        .join(RUSTFS_META_BUCKET)
+        .join(BUCKET_META_PREFIX)
+        .join(HEALING_TRACKER_FILENAME);
+
+    let b = read_file(healing_file).await?;
+    if b.is_empty() {
+        return Ok(None);
+    }
+
+    let healing_tracker = HealingTracker::unmarshal_msg(&b)?;
+
+    Ok(Some(healing_tracker))
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct MRFStatus {
+    bytes_healed: u64,
+    items_healed: u64,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct SetStatus {
+    pub id: String,
+    pub pool_index: i32,
+    pub set_index: i32,
+    pub heal_status: String,
+    pub heal_priority: String,
+    pub total_objects: usize,
+    pub disks: Vec<rustfs_madmin::Disk>,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct BgHealState {
+    offline_endpoints: Vec<String>,
+    scanned_items_count: u64,
+    heal_disks: Vec<String>,
+    sets: Vec<SetStatus>,
+    mrf: HashMap<String, MRFStatus>,
+    scparity: HashMap<String, usize>,
+}
+
+pub async fn get_local_background_heal_status() -> (BgHealState, bool) {
+    let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await;
+    if !ok {
+        return (BgHealState::default(), false);
+    }
+    let bg_seq = bg_seq.unwrap();
+    let mut status = BgHealState {
+        scanned_items_count: bg_seq.get_scanned_items_count().await as u64,
+        ..Default::default()
+    };
+    let mut heal_disks_map = HashSet::new();
+    for ep in get_local_disks_to_heal().await.iter() {
+        heal_disks_map.insert(ep.to_string());
+    }
+
+    let Some(store) = new_object_layer_fn() else {
+        let healing = GLOBAL_BackgroundHealState.get_local_healing_disks().await;
+        for disk in healing.values() {
+            status.heal_disks.push(disk.endpoint.clone());
+        }
+        return (status, true);
+    };
+
+    let si = store.local_storage_info().await;
+    let mut indexed = HashMap::new();
+    for disk in si.disks.iter() {
+        let set_idx = format!("{}-{}", disk.pool_index, disk.set_index);
+        // indexed.insert(set_idx, disk);
+        indexed.entry(set_idx).or_insert(Vec::new()).push(disk);
+    }
+
+    for (id, disks) in indexed {
+        let mut ss = SetStatus {
+            id,
+            set_index: disks[0].set_index,
+            pool_index: disks[0].pool_index,
+            ..Default::default()
+        };
+        for disk in disks {
+            ss.disks.push(disk.clone());
+            if disk.healing {
+                ss.heal_status = "healing".to_string();
+                ss.heal_priority = "high".to_string();
+                status.heal_disks.push(disk.endpoint.clone());
+            }
+        }
+        ss.disks.sort_by(|a, b| {
+            if a.pool_index != b.pool_index {
+                return a.pool_index.cmp(&b.pool_index);
+            }
+            if a.set_index != b.set_index {
+                return a.set_index.cmp(&b.set_index);
+            }
+            a.disk_index.cmp(&b.disk_index)
+        });
+        status.sets.push(ss);
+    }
+    status.sets.sort_by(|a, b| a.id.cmp(&b.id));
+    let backend_info = store.backend_info().await;
+    status
+        .scparity
+        .insert(STANDARD.to_string(), backend_info.standard_sc_parity.unwrap_or_default());
+    status
+        .scparity
+        .insert(RRS.to_string(), backend_info.rr_sc_parity.unwrap_or_default());
+
+    (status, true)
+}

crates/ecstore/src/heal/heal_ops.rs

@@ -0,0 +1,842 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{
+    background_heal_ops::HealTask,
+    data_scanner::HEAL_DELETE_DANGLING,
+    error::ERR_SKIP_FILE,
+    heal_commands::{HEAL_ITEM_BUCKET_METADATA, HealOpts, HealScanMode, HealStopSuccess, HealingTracker},
+};
+use crate::error::{Error, Result};
+use crate::heal::heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT};
+use crate::store_api::StorageAPI;
+use crate::{
+    config::com::CONFIG_PREFIX,
+    disk::RUSTFS_META_BUCKET,
+    global::GLOBAL_BackgroundHealRoutine,
+    heal::{error::ERR_HEAL_STOP_SIGNALLED, heal_commands::DRIVE_STATE_OK},
+};
+use crate::{
+    disk::endpoint::Endpoint,
+    endpoints::Endpoints,
+    global::GLOBAL_IsDistErasure,
+    heal::heal_commands::{HEAL_UNKNOWN_SCAN, HealStartSuccess},
+    new_object_layer_fn,
+};
+use chrono::Utc;
+use futures::join;
+use lazy_static::lazy_static;
+use rustfs_filemeta::MetaCacheEntry;
+use rustfs_madmin::heal_commands::{HealDriveInfo, HealItemType, HealResultItem};
+use rustfs_utils::path::has_prefix;
+use rustfs_utils::path::path_join;
+use serde::{Deserialize, Serialize};
+use std::{
+    collections::HashMap,
+    future::Future,
+    path::PathBuf,
+    pin::Pin,
+    sync::Arc,
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+use tokio::{
+    select, spawn,
+    sync::{
+        RwLock, broadcast,
+        mpsc::{self, Receiver as M_Receiver, Sender as M_Sender},
+        watch::{self, Receiver as W_Receiver, Sender as W_Sender},
+    },
+    time::{interval, sleep},
+};
+use tracing::{error, info};
+use uuid::Uuid;
+
+type HealStatusSummary = String;
+type ItemsMap = HashMap<HealItemType, usize>;
+pub type HealEntryFn =
+    Arc<dyn Fn(String, MetaCacheEntry, HealScanMode) -> Pin<Box<dyn Future<Output = Result<()>> + Send>> + Send + Sync + 'static>;
+
+pub const BG_HEALING_UUID: &str = "0000-0000-0000-0000";
+pub const HEALING_TRACKER_FILENAME: &str = ".healing.bin";
+const KEEP_HEAL_SEQ_STATE_DURATION: Duration = Duration::from_secs(10 * 60);
+const HEAL_NOT_STARTED_STATUS: &str = "not started";
+const HEAL_RUNNING_STATUS: &str = "running";
+const HEAL_STOPPED_STATUS: &str = "stopped";
+const HEAL_FINISHED_STATUS: &str = "finished";
+
+pub const RUSTFS_RESERVED_BUCKET: &str = "rustfs";
+pub const RUSTFS_RESERVED_BUCKET_PATH: &str = "/rustfs";
+pub const LOGIN_PATH_PREFIX: &str = "/login";
+
+const MAX_UNCONSUMED_HEAL_RESULT_ITEMS: usize = 1000;
+const HEAL_UNCONSUMED_TIMEOUT: Duration = Duration::from_secs(24 * 60 * 60);
+pub const NOP_HEAL: &str = "";
+
+lazy_static! {}
+
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
+pub struct HealSequenceStatus {
+    pub summary: HealStatusSummary,
+    pub failure_detail: String,
+    pub start_time: u64,
+    pub heal_setting: HealOpts,
+    pub items: Vec<HealResultItem>,
+}
+
+#[derive(Debug, Default)]
+pub struct HealSource {
+    pub bucket: String,
+    pub object: String,
+    pub version_id: String,
+    pub no_wait: bool,
+    pub opts: Option<HealOpts>,
+}
+
+#[derive(Debug)]
+pub struct HealSequence {
+    pub bucket: String,
+    pub object: String,
+    pub report_progress: bool,
+    pub start_time: SystemTime,
+    pub end_time: Arc<RwLock<SystemTime>>,
+    pub client_token: String,
+    pub client_address: String,
+    pub force_started: bool,
+    pub setting: HealOpts,
+    pub current_status: Arc<RwLock<HealSequenceStatus>>,
+    pub last_sent_result_index: RwLock<usize>,
+    pub scanned_items_map: RwLock<ItemsMap>,
+    pub healed_items_map: RwLock<ItemsMap>,
+    pub heal_failed_items_map: RwLock<ItemsMap>,
+    pub last_heal_activity: RwLock<SystemTime>,
+
+    traverse_and_heal_done_tx: Arc<RwLock<M_Sender<Option<Error>>>>,
+    traverse_and_heal_done_rx: Arc<RwLock<M_Receiver<Option<Error>>>>,
+
+    tx: W_Sender<bool>,
+    rx: W_Receiver<bool>,
+}
+
+pub fn new_bg_heal_sequence() -> HealSequence {
+    let hs = HealOpts {
+        remove: HEAL_DELETE_DANGLING,
+        ..Default::default()
+    };
+
+    HealSequence {
+        start_time: SystemTime::now(),
+        client_token: BG_HEALING_UUID.to_string(),
+        bucket: RUSTFS_RESERVED_BUCKET.to_string(),
+        setting: hs,
+        current_status: Arc::new(RwLock::new(HealSequenceStatus {
+            summary: HEAL_NOT_STARTED_STATUS.to_string(),
+            heal_setting: hs,
+            ..Default::default()
+        })),
+        report_progress: false,
+        scanned_items_map: HashMap::new().into(),
+        healed_items_map: HashMap::new().into(),
+        heal_failed_items_map: HashMap::new().into(),
+        ..Default::default()
+    }
+}
+
+pub fn new_heal_sequence(bucket: &str, obj_prefix: &str, client_addr: &str, hs: HealOpts, force_start: bool) -> HealSequence {
+    let client_token = Uuid::new_v4().to_string();
+    let (tx, rx) = mpsc::channel(10);
+    HealSequence {
+        bucket: bucket.to_string(),
+        object: obj_prefix.to_string(),
+        report_progress: true,
+        start_time: SystemTime::now(),
+        client_token,
+        client_address: client_addr.to_string(),
+        force_started: force_start,
+        setting: hs,
+        current_status: Arc::new(RwLock::new(HealSequenceStatus {
+            summary: HEAL_NOT_STARTED_STATUS.to_string(),
+            heal_setting: hs,
+            ..Default::default()
+        })),
+        traverse_and_heal_done_tx: Arc::new(RwLock::new(tx)),
+        traverse_and_heal_done_rx: Arc::new(RwLock::new(rx)),
+        scanned_items_map: HashMap::new().into(),
+        healed_items_map: HashMap::new().into(),
+        heal_failed_items_map: HashMap::new().into(),
+        ..Default::default()
+    }
+}
+
+impl Default for HealSequence {
+    fn default() -> Self {
+        let (h_tx, h_rx) = mpsc::channel(1);
+        let (tx, rx) = watch::channel(false);
+        Self {
+            bucket: Default::default(),
+            object: Default::default(),
+            report_progress: Default::default(),
+            start_time: SystemTime::now(),
+            end_time: Arc::new(RwLock::new(SystemTime::now())),
+            client_token: Default::default(),
+            client_address: Default::default(),
+            force_started: Default::default(),
+            setting: Default::default(),
+            current_status: Default::default(),
+            last_sent_result_index: Default::default(),
+            scanned_items_map: Default::default(),
+            healed_items_map: Default::default(),
+            heal_failed_items_map: Default::default(),
+            last_heal_activity: RwLock::new(SystemTime::now()),
+            traverse_and_heal_done_tx: Arc::new(RwLock::new(h_tx)),
+            traverse_and_heal_done_rx: Arc::new(RwLock::new(h_rx)),
+            tx,
+            rx,
+        }
+    }
+}
+
+impl HealSequence {
+    pub fn new(bucket: &str, obj_prefix: &str, client_addr: &str, hs: HealOpts, force_start: bool) -> Self {
+        let client_token = Uuid::new_v4().to_string();
+
+        Self {
+            bucket: bucket.to_string(),
+            object: obj_prefix.to_string(),
+            report_progress: true,
+            client_token,
+            client_address: client_addr.to_string(),
+            force_started: force_start,
+            setting: hs,
+            current_status: Arc::new(RwLock::new(HealSequenceStatus {
+                summary: HEAL_NOT_STARTED_STATUS.to_string(),
+                heal_setting: hs,
+                ..Default::default()
+            })),
+            ..Default::default()
+        }
+    }
+}
+
+impl HealSequence {
+    pub async fn get_scanned_items_count(&self) -> usize {
+        self.scanned_items_map.read().await.values().sum()
+    }
+
+    async fn _get_scanned_items_map(&self) -> ItemsMap {
+        self.scanned_items_map.read().await.clone()
+    }
+
+    async fn _get_healed_items_map(&self) -> ItemsMap {
+        self.healed_items_map.read().await.clone()
+    }
+
+    async fn _get_heal_failed_items_map(&self) -> ItemsMap {
+        self.heal_failed_items_map.read().await.clone()
+    }
+
+    pub async fn count_failed(&self, heal_type: HealItemType) {
+        *self.heal_failed_items_map.write().await.entry(heal_type).or_insert(0) += 1;
+        *self.last_heal_activity.write().await = SystemTime::now();
+    }
+
+    pub async fn count_scanned(&self, heal_type: HealItemType) {
+        *self.scanned_items_map.write().await.entry(heal_type).or_insert(0) += 1;
+        *self.last_heal_activity.write().await = SystemTime::now();
+    }
+
+    pub async fn count_healed(&self, heal_type: HealItemType) {
+        *self.healed_items_map.write().await.entry(heal_type).or_insert(0) += 1;
+        *self.last_heal_activity.write().await = SystemTime::now();
+    }
+
+    async fn is_quitting(&self) -> bool {
+        if let Ok(true) = self.rx.has_changed() {
+            info!("quited");
+            return true;
+        }
+        false
+    }
+
+    async fn has_ended(&self) -> bool {
+        if self.client_token == *BG_HEALING_UUID {
+            return false;
+        }
+
+        *(self.end_time.read().await) != self.start_time
+    }
+
+    async fn stop(&self) {
+        let _ = self.tx.send(true);
+    }
+
+    async fn push_heal_result_item(&self, r: &HealResultItem) -> Result<()> {
+        let mut r = r.clone();
+        let mut interval_timer = interval(HEAL_UNCONSUMED_TIMEOUT);
+        #[allow(unused_assignments)]
+        let mut items_len = 0;
+        loop {
+            {
+                let current_status_r = self.current_status.read().await;
+                items_len = current_status_r.items.len();
+            }
+
+            if items_len == MAX_UNCONSUMED_HEAL_RESULT_ITEMS {
+                select! {
+                    _ = sleep(Duration::from_secs(1)) => {
+
+                    }
+                    _ = self.is_done() => {
+                        return Err(Error::other("stopped"));
+                    }
+                    _ = interval_timer.tick() => {
+                        return Err(Error::other("timeout"));
+                    }
+                }
+            } else {
+                break;
+            }
+        }
+
+        let mut current_status_w = self.current_status.write().await;
+        if items_len > 0 {
+            r.result_index = 1 + current_status_w.items[items_len - 1].result_index;
+        } else {
+            r.result_index = 1 + *self.last_sent_result_index.read().await;
+        }
+
+        current_status_w.items.push(r);
+
+        Ok(())
+    }
+
+    pub async fn queue_heal_task(&self, source: HealSource, heal_type: HealItemType) -> Result<()> {
+        let mut task = HealTask::new(&source.bucket, &source.object, &source.version_id, &self.setting);
+        info!("queue_heal_task, {:?}", task);
+        if let Some(opts) = source.opts {
+            task.opts = opts;
+        } else {
+            task.opts.scan_mode = HEAL_UNKNOWN_SCAN;
+        }
+
+        self.count_scanned(heal_type.clone()).await;
+
+        if source.no_wait {
+            let task_str = format!("{task:?}");
+            if GLOBAL_BackgroundHealRoutine.tasks_tx.try_send(task).is_ok() {
+                info!("Task in the queue: {:?}", task_str);
+            }
+            return Ok(());
+        }
+
+        let (resp_tx, mut resp_rx) = mpsc::channel(1);
+        task.resp_tx = Some(resp_tx);
+
+        let task_str = format!("{task:?}");
+        if GLOBAL_BackgroundHealRoutine.tasks_tx.try_send(task).is_ok() {
+            info!("Task in the queue: {:?}", task_str);
+        } else {
+            error!("push task to queue failed");
+        }
+        let count_ok_drives = |drivers: &[HealDriveInfo]| {
+            let mut count = 0;
+            for drive in drivers.iter() {
+                if drive.state == DRIVE_STATE_OK {
+                    count += 1;
+                }
+            }
+            count
+        };
+
+        match resp_rx.recv().await {
+            Some(mut res) => {
+                if res.err.is_none() {
+                    self.count_healed(heal_type.clone()).await;
+                } else {
+                    self.count_failed(heal_type.clone()).await;
+                }
+                if !self.report_progress {
+                    return if let Some(err) = res.err {
+                        if err.to_string() == ERR_SKIP_FILE {
+                            return Ok(());
+                        }
+                        Err(err)
+                    } else {
+                        Ok(())
+                    };
+                }
+                res.result.heal_item_type = heal_type.clone();
+                if let Some(err) = res.err.as_ref() {
+                    res.result.detail = err.to_string();
+                }
+                if res.result.parity_blocks > 0 && res.result.data_blocks > 0 && res.result.data_blocks > res.result.parity_blocks
+                {
+                    let got = count_ok_drives(&res.result.after.drives);
+                    if got < res.result.parity_blocks {
+                        res.result.detail = format!(
+                            "quorum loss - expected {} minimum, got drive states in OK {}",
+                            res.result.parity_blocks, got
+                        );
+                    }
+                }
+
+                info!("queue_heal_task, HealResult: {:?}", res);
+                self.push_heal_result_item(&res.result).await
+            }
+            None => Ok(()),
+        }
+    }
+
+    async fn heal_disk_meta(h: Arc<HealSequence>) -> Result<()> {
+        HealSequence::heal_rustfs_sys_meta(h, CONFIG_PREFIX).await
+    }
+
+    async fn heal_items(h: Arc<HealSequence>, buckets_only: bool) -> Result<()> {
+        if h.client_token == *BG_HEALING_UUID {
+            return Ok(());
+        }
+
+        let bucket = h.bucket.clone();
+        let task1 = Self::heal_disk_meta(h.clone());
+        let task2 = Self::heal_bucket(h.clone(), &bucket, buckets_only);
+        let results = join!(task1, task2);
+        results.0?;
+        results.1?;
+
+        Ok(())
+    }
+
+    async fn traverse_and_heal(h: Arc<HealSequence>) {
+        let buckets_only = false;
+        let result = Self::heal_items(h.clone(), buckets_only).await.err();
+        let _ = h.traverse_and_heal_done_tx.read().await.send(result).await;
+    }
+
+    async fn heal_rustfs_sys_meta(h: Arc<HealSequence>, meta_prefix: &str) -> Result<()> {
+        info!("heal_rustfs_sys_meta, h: {:?}", h);
+        let Some(store) = new_object_layer_fn() else {
+            return Err(Error::other("errServerNotInitialized"));
+        };
+        let setting = h.setting;
+        store
+            .heal_objects(RUSTFS_META_BUCKET, meta_prefix, &setting, h.clone(), true)
+            .await
+    }
+
+    async fn is_done(&self) -> bool {
+        if let Ok(true) = self.rx.has_changed() {
+            return true;
+        }
+        false
+    }
+
+    pub async fn heal_bucket(hs: Arc<HealSequence>, bucket: &str, bucket_only: bool) -> Result<()> {
+        info!("heal_bucket, hs: {:?}", hs);
+        let (object, setting) = {
+            hs.queue_heal_task(
+                HealSource {
+                    bucket: bucket.to_string(),
+                    ..Default::default()
+                },
+                HEAL_ITEM_BUCKET.to_string(),
+            )
+            .await?;
+
+            if bucket_only {
+                return Ok(());
+            }
+
+            if !hs.setting.recursive {
+                if !hs.object.is_empty() {
+                    HealSequence::heal_object(hs.clone(), bucket, &hs.object, "", hs.setting.scan_mode).await?;
+                }
+                return Ok(());
+            }
+            (hs.object.clone(), hs.setting)
+        };
+        let Some(store) = new_object_layer_fn() else {
+            return Err(Error::other("errServerNotInitialized"));
+        };
+        store.heal_objects(bucket, &object, &setting, hs.clone(), false).await
+    }
+
+    pub async fn heal_object(
+        hs: Arc<HealSequence>,
+        bucket: &str,
+        object: &str,
+        version_id: &str,
+        _scan_mode: HealScanMode,
+    ) -> Result<()> {
+        info!("heal_object");
+        if hs.is_quitting().await {
+            info!("heal_object hs is quitting");
+            return Err(Error::other(ERR_HEAL_STOP_SIGNALLED));
+        }
+
+        info!("will queue task");
+        hs.queue_heal_task(
+            HealSource {
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+                version_id: version_id.to_string(),
+                opts: Some(hs.setting),
+                ..Default::default()
+            },
+            HEAL_ITEM_OBJECT.to_string(),
+        )
+        .await?;
+
+        Ok(())
+    }
+
+    pub async fn heal_meta_object(
+        hs: Arc<HealSequence>,
+        bucket: &str,
+        object: &str,
+        version_id: &str,
+        _scan_mode: HealScanMode,
+    ) -> Result<()> {
+        if hs.is_quitting().await {
+            return Err(Error::other(ERR_HEAL_STOP_SIGNALLED));
+        }
+
+        hs.queue_heal_task(
+            HealSource {
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+                version_id: version_id.to_string(),
+                ..Default::default()
+            },
+            HEAL_ITEM_BUCKET_METADATA.to_string(),
+        )
+        .await?;
+
+        Ok(())
+    }
+}
+
+pub async fn heal_sequence_start(h: Arc<HealSequence>) {
+    {
+        let mut current_status_w = h.current_status.write().await;
+        current_status_w.summary = HEAL_RUNNING_STATUS.to_string();
+        current_status_w.start_time = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("Time went backwards")
+            .as_secs();
+    }
+
+    let h_clone = h.clone();
+    spawn(async move {
+        HealSequence::traverse_and_heal(h_clone).await;
+    });
+
+    let h_clone_1 = h.clone();
+    let mut x = h.traverse_and_heal_done_rx.write().await;
+    select! {
+        _ = h.is_done() => {
+            *(h.end_time.write().await) = SystemTime::now();
+            let mut current_status_w = h.current_status.write().await;
+            current_status_w.summary = HEAL_FINISHED_STATUS.to_string();
+
+            spawn(async move {
+                let mut rx_w = h_clone_1.traverse_and_heal_done_rx.write().await;
+                rx_w.recv().await;
+            });
+        }
+        result = x.recv() => {
+            if let Some(err) = result {
+                match err {
+                    Some(err) => {
+                        let mut current_status_w = h.current_status.write().await;
+                        current_status_w.summary = HEAL_STOPPED_STATUS.to_string();
+                        current_status_w.failure_detail = err.to_string();
+                    },
+                    None => {
+                        let mut current_status_w = h.current_status.write().await;
+                        current_status_w.summary = HEAL_FINISHED_STATUS.to_string();
+                    }
+                }
+            }
+
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct AllHealState {
+    mu: RwLock<bool>,
+
+    heal_seq_map: RwLock<HashMap<String, Arc<HealSequence>>>,
+    heal_local_disks: RwLock<HashMap<Endpoint, bool>>,
+    heal_status: RwLock<HashMap<String, HealingTracker>>,
+}
+
+impl AllHealState {
+    pub fn new(cleanup: bool) -> Arc<Self> {
+        let state = Arc::new(AllHealState::default());
+        let (_, mut rx) = broadcast::channel(1);
+        if cleanup {
+            let state_clone = state.clone();
+            spawn(async move {
+                loop {
+                    select! {
+                        result = rx.recv() =>{
+                            if let Ok(true) = result {
+                                return;
+                            }
+                        }
+                        _ = sleep(Duration::from_secs(5 * 60)) => {
+                            state_clone.periodic_heal_seqs_clean().await;
+                        }
+                    }
+                }
+            });
+        }
+
+        state
+    }
+
+    pub async fn pop_heal_local_disks(&self, heal_local_disks: &[Endpoint]) {
+        let _ = self.mu.write().await;
+
+        self.heal_local_disks.write().await.retain(|k, _| {
+            if heal_local_disks.contains(k) {
+                return false;
+            }
+            true
+        });
+
+        let heal_local_disks = heal_local_disks.iter().map(|s| s.to_string()).collect::<Vec<_>>();
+        self.heal_status.write().await.retain(|_, v| {
+            if heal_local_disks.contains(&v.endpoint) {
+                return false;
+            }
+
+            true
+        });
+    }
+
+    pub async fn pop_heal_status_json(&self, heal_path: &str, client_token: &str) -> Result<Vec<u8>> {
+        match self.get_heal_sequence(heal_path).await {
+            Some(h) => {
+                if client_token != h.client_token {
+                    info!("err heal invalid client token");
+                    return Err(Error::other("err heal invalid client token"));
+                }
+                let num_items = h.current_status.read().await.items.len();
+                let mut last_result_index = *h.last_sent_result_index.read().await;
+                if num_items > 0 {
+                    if let Some(item) = h.current_status.read().await.items.last() {
+                        last_result_index = item.result_index;
+                    }
+                }
+                *h.last_sent_result_index.write().await = last_result_index;
+                let data = h.current_status.read().await.clone();
+                match serde_json::to_vec(&data) {
+                    Ok(b) => {
+                        h.current_status.write().await.items.clear();
+                        Ok(b)
+                    }
+                    Err(e) => {
+                        h.current_status.write().await.items.clear();
+                        info!("json encode err, e: {}", e);
+                        Err(Error::other(e.to_string()))
+                    }
+                }
+            }
+            None => serde_json::to_vec(&HealSequenceStatus {
+                summary: HEAL_FINISHED_STATUS.to_string(),
+                ..Default::default()
+            })
+            .map_err(|e| {
+                info!("json encode err, e: {}", e);
+                Error::other(e.to_string())
+            }),
+        }
+    }
+
+    pub async fn update_heal_status(&self, tracker: &HealingTracker) {
+        let _ = self.mu.write().await;
+        let _ = tracker.mu.read().await;
+
+        self.heal_status.write().await.insert(tracker.id.clone(), tracker.clone());
+    }
+
+    pub async fn get_local_healing_disks(&self) -> HashMap<String, rustfs_madmin::HealingDisk> {
+        let _ = self.mu.read().await;
+
+        let mut dst = HashMap::new();
+        for v in self.heal_status.read().await.values() {
+            dst.insert(v.endpoint.clone(), v.to_healing_disk().await);
+        }
+
+        dst
+    }
+
+    pub async fn get_heal_local_disk_endpoints(&self) -> Endpoints {
+        let _ = self.mu.read().await;
+
+        let mut endpoints = Vec::new();
+        self.heal_local_disks.read().await.iter().for_each(|(k, v)| {
+            if !v {
+                endpoints.push(k.clone());
+            }
+        });
+
+        Endpoints::from(endpoints)
+    }
+
+    pub async fn set_disk_healing_status(&self, ep: Endpoint, healing: bool) {
+        let _ = self.mu.write().await;
+
+        self.heal_local_disks.write().await.insert(ep, healing);
+    }
+
+    pub async fn push_heal_local_disks(&self, heal_local_disks: &[Endpoint]) {
+        let _ = self.mu.write().await;
+
+        for heal_local_disk in heal_local_disks.iter() {
+            self.heal_local_disks.write().await.insert(heal_local_disk.clone(), false);
+        }
+    }
+
+    pub async fn periodic_heal_seqs_clean(&self) {
+        let _ = self.mu.write().await;
+        let now = SystemTime::now();
+
+        let mut keys_to_remove = Vec::new();
+        for (k, v) in self.heal_seq_map.read().await.iter() {
+            if v.has_ended().await && now.duration_since(*(v.end_time.read().await)).unwrap() > KEEP_HEAL_SEQ_STATE_DURATION {
+                keys_to_remove.push(k.clone())
+            }
+        }
+        for key in keys_to_remove.iter() {
+            self.heal_seq_map.write().await.remove(key);
+        }
+    }
+
+    pub async fn get_heal_sequence_by_token(&self, token: &str) -> (Option<Arc<HealSequence>>, bool) {
+        let _ = self.mu.read().await;
+
+        for v in self.heal_seq_map.read().await.values() {
+            if v.client_token == token {
+                return (Some(v.clone()), true);
+            }
+        }
+
+        (None, false)
+    }
+
+    pub async fn get_heal_sequence(&self, path: &str) -> Option<Arc<HealSequence>> {
+        let _ = self.mu.read().await;
+
+        self.heal_seq_map.read().await.get(path).cloned()
+    }
+
+    pub async fn stop_heal_sequence(&self, path: &str) -> Result<Vec<u8>> {
+        let mut hsp = HealStopSuccess::default();
+        if let Some(he) = self.get_heal_sequence(path).await {
+            let client_token = he.client_token.clone();
+            if *GLOBAL_IsDistErasure.read().await {
+                // TODO: proxy
+            }
+
+            hsp.client_token = client_token;
+            hsp.client_address = he.client_address.clone();
+            hsp.start_time = Utc::now();
+
+            he.stop().await;
+
+            loop {
+                if he.has_ended().await {
+                    break;
+                }
+
+                sleep(Duration::from_secs(1)).await;
+            }
+
+            let _ = self.mu.write().await;
+            self.heal_seq_map.write().await.remove(path);
+        } else {
+            hsp.client_token = "unknown".to_string();
+        }
+
+        let b = serde_json::to_string(&hsp)?;
+        Ok(b.as_bytes().to_vec())
+    }
+
+    // LaunchNewHealSequence - launches a background routine that performs
+    // healing according to the healSequence argument. For each heal
+    // sequence, state is stored in the `globalAllHealState`, which is a
+    // map of the heal path to `healSequence` which holds state about the
+    // heal sequence.
+    //
+    // Heal results are persisted in server memory for
+    // `keepHealSeqStateDuration`. This function also launches a
+    // background routine to clean up heal results after the
+    // aforementioned duration.
+    pub async fn launch_new_heal_sequence(&self, heal_sequence: Arc<HealSequence>) -> Result<Vec<u8>> {
+        let path = path_join(&[
+            PathBuf::from(heal_sequence.bucket.clone()),
+            PathBuf::from(heal_sequence.object.clone()),
+        ]);
+        let path_s = path.to_str().unwrap();
+        if heal_sequence.force_started {
+            self.stop_heal_sequence(path_s).await?;
+        } else if let Some(hs) = self.get_heal_sequence(path_s).await {
+            if !hs.has_ended().await {
+                return Err(Error::other(format!(
+                    "Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {:?}, token is {}",
+                    heal_sequence.client_address, heal_sequence.start_time, heal_sequence.client_token
+                )));
+            }
+        }
+
+        let _ = self.mu.write().await;
+
+        for (k, v) in self.heal_seq_map.read().await.iter() {
+            if (has_prefix(k, path_s) || has_prefix(path_s, k)) && !v.has_ended().await {
+                return Err(Error::other(format!(
+                    "The provided heal sequence path overlaps with an existing heal path: {k}"
+                )));
+            }
+        }
+
+        self.heal_seq_map
+            .write()
+            .await
+            .insert(path_s.to_string(), heal_sequence.clone());
+
+        let client_token = heal_sequence.client_token.clone();
+        if *GLOBAL_IsDistErasure.read().await {
+            // TODO: proxy
+        }
+
+        if heal_sequence.client_token == BG_HEALING_UUID {
+            // For background heal do nothing, do not spawn an unnecessary goroutine.
+        } else {
+            let heal_sequence_clone = heal_sequence.clone();
+            spawn(async {
+                heal_sequence_start(heal_sequence_clone).await;
+            });
+        }
+
+        let b = serde_json::to_vec(&HealStartSuccess {
+            client_token,
+            client_address: heal_sequence.client_address.clone(),
+            // start_time: Utc::now(),
+            start_time: heal_sequence.start_time.into(),
+        })?;
+        Ok(b)
+    }
+}

crates/ecstore/src/heal/mod.rs

@@ -0,0 +1,23 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod background_heal_ops;
+pub mod data_scanner;
+pub mod data_scanner_metric;
+pub mod data_usage;
+pub mod data_usage_cache;
+pub mod error;
+pub mod heal_commands;
+pub mod heal_ops;
+pub mod mrf;

crates/ecstore/src/heal/mrf.rs

@@ -0,0 +1,142 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
+use crate::heal::background_heal_ops::{heal_bucket, heal_object};
+use crate::heal::heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN};
+use chrono::{DateTime, Utc};
+use lazy_static::lazy_static;
+use regex::Regex;
+use rustfs_utils::path::SLASH_SEPARATOR;
+use std::ops::Sub;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
+use tokio::sync::RwLock;
+use tokio::sync::mpsc::{Receiver, Sender};
+use tokio::time::sleep;
+use tracing::error;
+use uuid::Uuid;
+
+pub const MRF_OPS_QUEUE_SIZE: u64 = 100000;
+pub const HEAL_DIR: &str = ".heal";
+pub const HEAL_MRFMETA_FORMAT: u64 = 1;
+pub const HEAL_MRFMETA_VERSION_V1: u64 = 1;
+
+lazy_static! {
+    pub static ref HEAL_MRF_DIR: String =
+        format!("{}{}{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, HEAL_DIR, SLASH_SEPARATOR, "mrf");
+    static ref PATTERNS: Vec<Regex> = vec![
+        Regex::new(r"^buckets/.*/.metacache/.*").unwrap(),
+        Regex::new(r"^tmp/.*").unwrap(),
+        Regex::new(r"^multipart/.*").unwrap(),
+        Regex::new(r"^tmp-old/.*").unwrap(),
+    ];
+}
+
+#[derive(Default)]
+pub struct PartialOperation {
+    pub bucket: String,
+    pub object: String,
+    pub version_id: Option<String>,
+    pub versions: Vec<u8>,
+    pub set_index: usize,
+    pub pool_index: usize,
+    pub queued: DateTime<Utc>,
+    pub bitrot_scan: bool,
+}
+
+pub struct MRFState {
+    tx: Sender<PartialOperation>,
+    rx: RwLock<Receiver<PartialOperation>>,
+    closed: AtomicBool,
+    closing: AtomicBool,
+}
+
+impl Default for MRFState {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl MRFState {
+    pub fn new() -> MRFState {
+        let (tx, rx) = tokio::sync::mpsc::channel(MRF_OPS_QUEUE_SIZE as usize);
+        MRFState {
+            tx,
+            rx: RwLock::new(rx),
+            closed: Default::default(),
+            closing: Default::default(),
+        }
+    }
+
+    pub async fn add_partial(&self, op: PartialOperation) {
+        if self.closed.load(Ordering::SeqCst) || self.closing.load(Ordering::SeqCst) {
+            return;
+        }
+        let _ = self.tx.send(op).await;
+    }
+
+    pub async fn heal_routine(&self) {
+        loop {
+            // rx used only there,
+            if let Some(op) = self.rx.write().await.recv().await {
+                if op.bucket == RUSTFS_META_BUCKET {
+                    for pattern in &*PATTERNS {
+                        if pattern.is_match(&op.object) {
+                            return;
+                        }
+                    }
+                }
+
+                let now = Utc::now();
+                if now.sub(op.queued).num_seconds() < 1 {
+                    sleep(Duration::from_secs(1)).await;
+                }
+
+                let scan_mode = if op.bitrot_scan { HEAL_DEEP_SCAN } else { HEAL_NORMAL_SCAN };
+                if op.object.is_empty() {
+                    if let Err(err) = heal_bucket(&op.bucket).await {
+                        error!("heal bucket failed, bucket: {}, err: {:?}", op.bucket, err);
+                    }
+                } else if op.versions.is_empty() {
+                    if let Err(err) =
+                        heal_object(&op.bucket, &op.object, &op.version_id.clone().unwrap_or_default(), scan_mode).await
+                    {
+                        error!("heal object failed, bucket: {}, object: {}, err: {:?}", op.bucket, op.object, err);
+                    }
+                } else {
+                    let vers = op.versions.len() / 16;
+                    if vers > 0 {
+                        for i in 0..vers {
+                            let start = i * 16;
+                            let end = start + 16;
+                            if let Err(err) = heal_object(
+                                &op.bucket,
+                                &op.object,
+                                &Uuid::from_slice(&op.versions[start..end]).expect("").to_string(),
+                                scan_mode,
+                            )
+                            .await
+                            {
+                                error!("heal object failed, bucket: {}, object: {}, err: {:?}", op.bucket, op.object, err);
+                            }
+                        }
+                    }
+                }
+            } else {
+                return;
+            }
+        }
+    }
+}

crates/ecstore/src/lib.rs

@@ -0,0 +1,57 @@
+#![allow(dead_code)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+extern crate core;
+
+pub mod admin_server_info;
+pub mod bitrot;
+pub mod bucket;
+pub mod cache_value;
+mod chunk_stream;
+pub mod cmd;
+pub mod compress;
+pub mod config;
+pub mod disk;
+pub mod disks_layout;
+pub mod endpoints;
+pub mod erasure_coding;
+pub mod error;
+pub mod global;
+pub mod heal;
+pub mod metrics_realtime;
+pub mod notification_sys;
+pub mod pools;
+pub mod rebalance;
+pub mod rpc;
+pub mod set_disk;
+mod sets;
+pub mod store;
+pub mod store_api;
+mod store_init;
+pub mod store_list_objects;
+pub mod store_utils;
+
+pub mod checksum;
+pub mod client;
+pub mod event;
+pub mod event_notification;
+pub mod tier;
+
+pub use global::new_object_layer_fn;
+pub use global::set_global_endpoints;
+pub use global::update_erasure_type;
+
+pub use global::GLOBAL_Endpoints;
+pub use store_api::StorageAPI;

crates/ecstore/src/metrics_realtime.rs

@@ -0,0 +1,230 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{HashMap, HashSet};
+
+use chrono::Utc;
+use rustfs_common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Addr};
+use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, RealtimeMetrics};
+use rustfs_utils::os::get_drive_stats;
+use serde::{Deserialize, Serialize};
+use tracing::info;
+
+use crate::{
+    admin_server_info::get_local_server_property,
+    heal::{
+        data_scanner_metric::globalScannerMetrics,
+        heal_commands::{DRIVE_STATE_OK, DRIVE_STATE_UNFORMATTED},
+    },
+    new_object_layer_fn,
+    store_api::StorageAPI,
+    // utils::os::get_drive_stats,
+};
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct CollectMetricsOpts {
+    pub hosts: HashSet<String>,
+    pub disks: HashSet<String>,
+    pub job_id: String,
+    pub dep_id: String,
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct MetricType(u32);
+
+impl MetricType {
+    // 定义一些常量
+    pub const NONE: MetricType = MetricType(0);
+    pub const SCANNER: MetricType = MetricType(1 << 0);
+    pub const DISK: MetricType = MetricType(1 << 1);
+    pub const OS: MetricType = MetricType(1 << 2);
+    pub const BATCH_JOBS: MetricType = MetricType(1 << 3);
+    pub const SITE_RESYNC: MetricType = MetricType(1 << 4);
+    pub const NET: MetricType = MetricType(1 << 5);
+    pub const MEM: MetricType = MetricType(1 << 6);
+    pub const CPU: MetricType = MetricType(1 << 7);
+    pub const RPC: MetricType = MetricType(1 << 8);
+
+    // MetricsAll must be last.
+    pub const ALL: MetricType = MetricType((1 << 9) - 1);
+
+    pub fn new(t: u32) -> Self {
+        Self(t)
+    }
+}
+
+impl MetricType {
+    fn contains(&self, x: &MetricType) -> bool {
+        (self.0 & x.0) == x.0
+    }
+}
+
+pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts) -> RealtimeMetrics {
+    info!("collect_local_metrics");
+    let mut real_time_metrics = RealtimeMetrics::default();
+    if types.0 == MetricType::NONE.0 {
+        info!("types is None, return");
+        return real_time_metrics;
+    }
+
+    let mut by_host_name = GLOBAL_Rustfs_Addr.read().await.clone();
+    if !opts.hosts.is_empty() {
+        let server = get_local_server_property().await;
+        if opts.hosts.contains(&server.endpoint) {
+            by_host_name = server.endpoint;
+        } else {
+            return real_time_metrics;
+        }
+    }
+    let local_node_name = GLOBAL_Local_Node_Name.read().await.clone();
+    if by_host_name.starts_with(":") && !local_node_name.starts_with(":") {
+        by_host_name = local_node_name;
+    }
+
+    if types.contains(&MetricType::DISK) {
+        info!("start get disk metrics");
+        let mut aggr = DiskMetric {
+            collected_at: Utc::now(),
+            ..Default::default()
+        };
+        for (name, disk) in collect_local_disks_metrics(&opts.disks).await.into_iter() {
+            info!("got disk metric, name: {name}, metric: {disk:?}");
+            real_time_metrics.by_disk.insert(name, disk.clone());
+            aggr.merge(&disk);
+        }
+        real_time_metrics.aggregated.disk = Some(aggr);
+    }
+
+    if types.contains(&MetricType::SCANNER) {
+        info!("start get scanner metrics");
+        let metrics = globalScannerMetrics.report().await;
+        real_time_metrics.aggregated.scanner = Some(metrics);
+    }
+
+    // if types.contains(&MetricType::OS) {}
+
+    // if types.contains(&MetricType::BATCH_JOBS) {}
+
+    // if types.contains(&MetricType::SITE_RESYNC) {}
+
+    // if types.contains(&MetricType::NET) {}
+
+    // if types.contains(&MetricType::MEM) {}
+
+    // if types.contains(&MetricType::CPU) {}
+
+    // if types.contains(&MetricType::RPC) {}
+
+    real_time_metrics
+        .by_host
+        .insert(by_host_name.clone(), real_time_metrics.aggregated.clone());
+    real_time_metrics.hosts.push(by_host_name);
+
+    real_time_metrics
+}
+
+async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String, DiskMetric> {
+    let store = match new_object_layer_fn() {
+        Some(store) => store,
+        None => return HashMap::new(),
+    };
+
+    let mut metrics = HashMap::new();
+    let storage_info = store.local_storage_info().await;
+    for d in storage_info.disks.iter() {
+        if !disks.is_empty() && !disks.contains(&d.endpoint) {
+            continue;
+        }
+
+        if d.state != *DRIVE_STATE_OK && d.state != *DRIVE_STATE_UNFORMATTED {
+            metrics.insert(
+                d.endpoint.clone(),
+                DiskMetric {
+                    n_disks: 1,
+                    offline: 1,
+                    ..Default::default()
+                },
+            );
+            continue;
+        }
+
+        let mut dm = DiskMetric {
+            n_disks: 1,
+            ..Default::default()
+        };
+        if d.healing {
+            dm.healing += 1;
+        }
+
+        if let Some(m) = &d.metrics {
+            for (k, v) in m.api_calls.iter() {
+                if *v != 0 {
+                    dm.life_time_ops.insert(k.clone(), *v);
+                }
+            }
+            for (k, v) in m.last_minute.iter() {
+                if v.count != 0 {
+                    dm.last_minute.operations.insert(k.clone(), v.clone());
+                }
+            }
+        }
+
+        if let Ok(st) = get_drive_stats(d.major, d.minor) {
+            dm.io_stats = DiskIOStats {
+                read_ios: st.read_ios,
+                read_merges: st.read_merges,
+                read_sectors: st.read_sectors,
+                read_ticks: st.read_ticks,
+                write_ios: st.write_ios,
+                write_merges: st.write_merges,
+                write_sectors: st.write_sectors,
+                write_ticks: st.write_ticks,
+                current_ios: st.current_ios,
+                total_ticks: st.total_ticks,
+                req_ticks: st.req_ticks,
+                discard_ios: st.discard_ios,
+                discard_merges: st.discard_merges,
+                discard_sectors: st.discard_sectors,
+                discard_ticks: st.discard_ticks,
+                flush_ios: st.flush_ios,
+                flush_ticks: st.flush_ticks,
+            };
+        }
+        metrics.insert(d.endpoint.clone(), dm);
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod test {
+    use super::MetricType;
+
+    #[test]
+    fn tes_types() {
+        let t = MetricType::ALL;
+        assert!(t.contains(&MetricType::NONE));
+        assert!(t.contains(&MetricType::DISK));
+        assert!(t.contains(&MetricType::OS));
+        assert!(t.contains(&MetricType::BATCH_JOBS));
+        assert!(t.contains(&MetricType::SITE_RESYNC));
+        assert!(t.contains(&MetricType::NET));
+        assert!(t.contains(&MetricType::MEM));
+        assert!(t.contains(&MetricType::CPU));
+        assert!(t.contains(&MetricType::RPC));
+
+        let disk = MetricType::new(1 << 1);
+        assert!(disk.contains(&MetricType::DISK));
+    }
+}