refactor: consolidate ecstore owner module layout (#3934)

* refactor: shrink ecstore root owner facades

* refactor: remove ecstore core store root shims

* refactor: move ecstore erasure owner modules

* refactor: remove ecstore root rpc facade

* refactor: move ecstore services domain modules

crates/ecstore/src/services/metrics_realtime.rs

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::admin_server_info::get_local_server_property;
-use crate::runtime_sources;
+use crate::diagnostics::admin_server_info::get_local_server_property;
+use crate::runtime::sources as runtime_sources;
 use crate::storage_api_contracts::admin::StorageAdminApi;
 use chrono::Utc;
 use rustfs_common::{heal_channel::DriveState, metrics::global_metrics};

crates/ecstore/src/services/mod.rs

@@ -0,0 +1,20 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) mod batch_processor;
+pub(crate) mod event_notification;
+pub(crate) mod metrics_realtime;
+pub(crate) mod notification_sys;
+pub(crate) mod rebalance;
+pub(crate) mod tier;

crates/ecstore/src/services/notification_sys.rs

@@ -12,13 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::admin_server_info::get_commit_id;
-use crate::endpoints::EndpointServerPools;
+use crate::cluster::rpc::PeerRestClient;
+use crate::diagnostics::admin_server_info::get_commit_id;
 use crate::error::{Error, Result};
-use crate::metrics_realtime::{CollectMetricsOpts, MetricType};
-use crate::rebalance::RebalSaveOpt;
-use crate::rpc::PeerRestClient;
-use crate::runtime_sources;
+use crate::layout::endpoints::EndpointServerPools;
+use crate::runtime::sources as runtime_sources;
+use crate::services::metrics_realtime::{CollectMetricsOpts, MetricType};
+use crate::services::rebalance::RebalSaveOpt;
 use crate::storage_api_contracts::admin::StorageAdminApi;
 use futures::future::join_all;
 use lazy_static::lazy_static;
@@ -232,7 +232,12 @@ impl NotificationSys {
             futures.push(async move {
                 if let Some(client) = client {
                     match client
-                        .signal_service(crate::rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC, &sub_sys, false, SystemTime::UNIX_EPOCH)
+                        .signal_service(
+                            crate::cluster::rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC,
+                            &sub_sys,
+                            false,
+                            SystemTime::UNIX_EPOCH,
+                        )
                         .await
                     {
                         Ok(_) => NotificationPeerErr {
@@ -261,7 +266,7 @@ impl NotificationSys {
             futures.push(async move {
                 if let Some(client) = client {
                     match client
-                        .signal_service(crate::rpc::SERVICE_SIGNAL_REFRESH_CONFIG, "", false, SystemTime::UNIX_EPOCH)
+                        .signal_service(crate::cluster::rpc::SERVICE_SIGNAL_REFRESH_CONFIG, "", false, SystemTime::UNIX_EPOCH)
                         .await
                     {
                         Ok(_) => NotificationPeerErr {

crates/ecstore/src/services/rebalance/control.rs

@@ -0,0 +1,780 @@
+use super::meta::{
+    RebalanceMetaMergeOutcome, clone_first_arc, clone_rebalance_pool_stats, defer_bucket_in_rebalance_queue,
+    ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error, is_rebalance_actively_running,
+    is_rebalance_conflicting_with_decommission, mark_rebalance_bucket_done, merge_rebalance_meta, percent_free_ratio,
+    rebalance_metadata_not_initialized_error, record_rebalance_cleanup_warning_in_meta,
+    record_rebalance_stop_propagation_snapshot, resolve_next_rebalance_bucket, rollback_rebalance_start_meta_snapshot_for_id,
+    should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot_for_id,
+    validate_init_rebalance_state,
+};
+use super::worker::{
+    rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result, resolve_rebalance_meta_load_result,
+    resolve_rebalance_meta_save_result,
+};
+use super::{
+    DiskStat, EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
+    RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
+    encode_rebalance_stop_propagation_record,
+};
+use crate::error::{Error, Result};
+use crate::object_api::ObjectOptions;
+use crate::set_disk::get_lock_acquire_timeout;
+use crate::storage_api_contracts::{
+    admin::StorageAdminApi, namespace::NamespaceLocking as StorageNamespaceLocking, object::EcstoreObjectIO,
+};
+use crate::store::ECStore;
+use rustfs_filemeta::FileInfo;
+use std::sync::Arc;
+use time::OffsetDateTime;
+use tracing::{debug, info};
+use uuid::Uuid;
+
+pub(super) fn validate_rebalance_disk_stats_coverage(disk_stats: &[DiskStat]) -> Result<()> {
+    for (idx, disk_stat) in disk_stats.iter().enumerate() {
+        if disk_stat.total_space == 0 {
+            return Err(Error::other(format!(
+                "rebalance storage info is incomplete: pool {idx} has no reported capacity"
+            )));
+        }
+    }
+
+    Ok(())
+}
+
+fn pool_rebalance_status_from_meta(meta: Option<&RebalanceMeta>, pool_index: usize) -> (RebalStatus, bool) {
+    meta.and_then(|meta| meta.pool_stats.get(pool_index))
+        .filter(|pool_stat| pool_stat.participating)
+        .map(|pool_stat| (pool_stat.info.status, pool_stat.info.stopping))
+        .unwrap_or_default()
+}
+
+fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) {
+    if persisted.id.is_empty() && persisted.pool_stats.is_empty() {
+        clear_rebalance_status_refresh(current);
+        return;
+    }
+
+    match current.as_mut() {
+        Some(current_meta) => {
+            if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict
+                && !is_rebalance_actively_running(current_meta)
+            {
+                *current = Some(persisted);
+            }
+        }
+        None => {
+            *current = Some(persisted);
+        }
+    }
+}
+
+fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) {
+    if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) {
+        *current = None;
+    }
+}
+
+impl ECStore {
+    pub(super) async fn save_rebalance_meta_with_merge<S>(
+        &self,
+        pool: Arc<S>,
+        local_snapshot: &RebalanceMeta,
+        stage: &str,
+    ) -> Result<()>
+    where
+        S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
+    {
+        let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
+        let _guard = ns_lock
+            .get_write_lock(get_lock_acquire_timeout())
+            .await
+            .map_err(rebalance_meta_lock_error)?;
+
+        let opts = ObjectOptions {
+            no_lock: true,
+            ..Default::default()
+        };
+        let mut merged = RebalanceMeta::new();
+        match merged.load_with_opts(pool.clone(), opts.clone()).await {
+            Ok(()) => {
+                if merge_rebalance_meta(&mut merged, local_snapshot) == RebalanceMetaMergeOutcome::RejectedActiveConflict {
+                    return Err(Error::RebalanceAlreadyRunning);
+                }
+            }
+            Err(Error::ConfigNotFound) => {
+                merged = local_snapshot.clone();
+            }
+            Err(err) => return Err(Error::other(format!("rebalance meta load before save failed during {stage}: {err}"))),
+        }
+
+        merged.save_with_opts(pool, opts).await
+    }
+
+    #[tracing::instrument(skip_all)]
+    pub async fn load_rebalance_meta(&self) -> Result<()> {
+        let mut meta = RebalanceMeta::new();
+        debug!(
+            event = EVENT_REBALANCE_STATE,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            state = "metadata_loading",
+            "Loading rebalance metadata"
+        );
+        let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?;
+        if resolve_rebalance_meta_load_result(meta.load(pool).await)? {
+            {
+                let mut rebalance_meta = self.rebalance_meta.write().await;
+
+                *rebalance_meta = Some(meta);
+
+                drop(rebalance_meta);
+            }
+
+            resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?;
+            debug!(
+                event = EVENT_REBALANCE_STATE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                state = "metadata_loaded",
+                "Loaded rebalance metadata"
+            );
+        } else {
+            {
+                let mut rebalance_meta = self.rebalance_meta.write().await;
+                *rebalance_meta = None;
+            }
+            debug!(
+                event = EVENT_REBALANCE_STATE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                state = "metadata_missing",
+                reason = "rebalance_not_started",
+                "Rebalance metadata not found"
+            );
+        }
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip_all)]
+    pub async fn refresh_rebalance_status_meta(&self) -> Result<()> {
+        let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?;
+        let mut persisted = RebalanceMeta::new();
+        match persisted.load(pool).await {
+            Ok(()) => {
+                let mut rebalance_meta = self.rebalance_meta.write().await;
+                merge_rebalance_status_refresh(&mut rebalance_meta, persisted);
+            }
+            Err(Error::ConfigNotFound) => {
+                let mut rebalance_meta = self.rebalance_meta.write().await;
+                clear_rebalance_status_refresh(&mut rebalance_meta);
+            }
+            Err(err) => {
+                return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}")));
+            }
+        }
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip_all)]
+    pub async fn update_rebalance_stats(&self) -> Result<()> {
+        let mut ok = false;
+
+        let pool_stats = {
+            let rebalance_meta = self.rebalance_meta.read().await;
+            clone_rebalance_pool_stats(rebalance_meta.as_ref())?
+        };
+
+        debug!(
+            event = EVENT_REBALANCE_STATE,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            pool_count = pool_stats.len(),
+            "Refreshing rebalance stats snapshot"
+        );
+
+        for i in 0..self.pools.len() {
+            if pool_stats.get(i).is_none() {
+                let mut rebalance_meta = self.rebalance_meta.write().await;
+                debug!(
+                    event = EVENT_REBALANCE_STATE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index = i,
+                    state = "pool_stat_missing",
+                    "Adding missing rebalance pool stats entry"
+                );
+                if let Some(meta) = rebalance_meta.as_mut() {
+                    meta.pool_stats.push(RebalanceStats::default());
+                }
+                ok = true;
+                drop(rebalance_meta);
+            }
+        }
+
+        if ok {
+            debug!(
+                event = EVENT_REBALANCE_STATE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                state = "metadata_saving",
+                "Saving rebalance metadata after stats refresh"
+            );
+
+            let rebalance_meta = self.rebalance_meta.read().await;
+            if let Some(meta) = rebalance_meta.as_ref() {
+                let pool = clone_first_arc(&self.pools, "update_rebalance_stats: no pools available")?;
+                resolve_rebalance_meta_save_result(
+                    self.save_rebalance_meta_with_merge(pool, meta, "update_rebalance_stats")
+                        .await,
+                    "update_rebalance_stats",
+                )?;
+            }
+        }
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self))]
+    pub async fn init_rebalance_meta(&self, bucktes: Vec<String>) -> Result<String> {
+        info!(
+            event = EVENT_REBALANCE_STATE,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            state = "initializing",
+            bucket_count = bucktes.len(),
+            "Initializing rebalance metadata"
+        );
+        let si = StorageAdminApi::storage_info(self).await;
+
+        let mut disk_stats = vec![DiskStat::default(); self.pools.len()];
+
+        let mut total_cap = 0;
+        let mut total_free = 0;
+        for disk in si.disks.iter() {
+            if disk.pool_index < 0 || disk_stats.len() <= disk.pool_index as usize {
+                continue;
+            }
+
+            total_cap += disk.total_space;
+            total_free += disk.available_space;
+
+            disk_stats[disk.pool_index as usize].total_space += disk.total_space;
+            disk_stats[disk.pool_index as usize].available_space += disk.available_space;
+        }
+
+        let percent_free_goal = percent_free_ratio(total_free, total_cap);
+        validate_rebalance_disk_stats_coverage(&disk_stats)?;
+
+        let mut pool_stats = Vec::with_capacity(self.pools.len());
+
+        let now = OffsetDateTime::now_utc();
+
+        for disk_stat in disk_stats.iter() {
+            let mut pool_stat = RebalanceStats {
+                init_free_space: disk_stat.available_space,
+                init_capacity: disk_stat.total_space,
+                buckets: bucktes.clone(),
+                rebalanced_buckets: Vec::with_capacity(bucktes.len()),
+                ..Default::default()
+            };
+
+            if should_pool_participate(disk_stat.available_space, disk_stat.total_space, percent_free_goal) {
+                pool_stat.participating = true;
+                pool_stat.info = RebalanceInfo {
+                    start_time: Some(now),
+                    status: RebalStatus::Started,
+                    ..Default::default()
+                };
+            }
+
+            pool_stats.push(pool_stat);
+        }
+
+        let meta = RebalanceMeta {
+            id: Uuid::new_v4().to_string(),
+            percent_free_goal,
+            pool_stats,
+            ..Default::default()
+        };
+
+        let pool = clone_first_arc(&self.pools, "init_rebalance_meta: no pools available")?;
+        resolve_rebalance_meta_save_result(
+            self.save_rebalance_meta_with_merge(pool, &meta, "init_rebalance_meta").await,
+            "init_rebalance_meta",
+        )?;
+
+        info!(
+            event = EVENT_REBALANCE_STATE,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            state = "metadata_initialized",
+            bucket_count = bucktes.len(),
+            "Rebalance metadata initialized"
+        );
+
+        let id = meta.id.clone();
+
+        {
+            let mut rebalance_meta = self.rebalance_meta.write().await;
+            *rebalance_meta = Some(meta);
+            drop(rebalance_meta);
+        }
+
+        Ok(id)
+    }
+
+    #[tracing::instrument(skip(self, bucktes))]
+    pub async fn init_and_start_rebalance(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
+        let _start_guard = self.start_gate.lock().await;
+
+        let decommission_running = self.is_decommission_running().await;
+        {
+            let rebalance_meta = self.rebalance_meta.read().await;
+            validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
+        }
+
+        let id = self.init_rebalance_meta(bucktes).await?;
+        if let Err(start_err) = self.start_rebalance().await {
+            if let Err(rollback_err) = self
+                .rollback_rebalance_start_without_worker_for_id(Some(&id), start_err.to_string())
+                .await
+            {
+                return Err(Error::other(format!(
+                    "failed to start rebalance after metadata initialized for {id}: {start_err}; rollback failed: {rollback_err}"
+                )));
+            }
+
+            return Err(Error::other(format!(
+                "failed to start rebalance after metadata initialized for {id}; local metadata was finalized as failed: {start_err}"
+            )));
+        }
+
+        Ok(id)
+    }
+
+    #[tracing::instrument(skip(self, fi))]
+    pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> {
+        self.update_pool_stats_batch(pool_index, bucket, &[fi]).await
+    }
+
+    #[tracing::instrument(skip(self, versions))]
+    pub async fn update_pool_stats_batch(&self, pool_index: usize, bucket: String, versions: &[&FileInfo]) -> Result<()> {
+        if versions.is_empty() {
+            return Ok(());
+        }
+
+        let mut rebalance_meta = self.rebalance_meta.write().await;
+        if let Some(meta) = rebalance_meta.as_mut() {
+            if !should_accept_rebalance_stats_update(meta, pool_index) {
+                return Ok(());
+            }
+
+            if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
+                pool_stat.update_batch(bucket, versions);
+            }
+        }
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self))]
+    pub async fn next_rebal_bucket(&self, pool_index: usize) -> Result<Option<String>> {
+        let rebalance_meta = self.rebalance_meta.read().await;
+        debug!(
+            event = EVENT_REBALANCE_BUCKET,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            pool_index,
+            has_meta = rebalance_meta.is_some(),
+            state = "next_bucket_lookup",
+            "Rebalance next bucket lookup"
+        );
+        resolve_next_rebalance_bucket(rebalance_meta.as_ref(), pool_index)
+    }
+
+    #[tracing::instrument(skip(self))]
+    pub async fn bucket_rebalance_done(&self, pool_index: usize, bucket: String) -> Result<()> {
+        let mut rebalance_meta = self.rebalance_meta.write().await;
+        mark_rebalance_bucket_done(rebalance_meta.as_mut(), pool_index, &bucket)
+    }
+
+    pub(super) async fn record_rebalance_cleanup_warning(
+        &self,
+        pool_index: usize,
+        bucket: &str,
+        object: &str,
+        message: String,
+    ) -> Result<()> {
+        let mut rebalance_meta = self.rebalance_meta.write().await;
+        record_rebalance_cleanup_warning_in_meta(
+            rebalance_meta.as_mut(),
+            pool_index,
+            bucket,
+            object,
+            message,
+            OffsetDateTime::now_utc(),
+        )
+    }
+
+    pub(super) async fn defer_rebalance_bucket(&self, pool_index: usize, bucket: String, last_error: String) -> Result<()> {
+        let mut rebalance_meta = self.rebalance_meta.write().await;
+        let Some(meta) = rebalance_meta.as_mut() else {
+            return Err(rebalance_metadata_not_initialized_error("defer rebalance bucket"));
+        };
+        let pool_count = meta.pool_stats.len();
+        ensure_valid_rebalance_pool_index(pool_count, pool_index)?;
+        let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) else {
+            return Err(invalid_rebalance_pool_index_error(pool_index, pool_count));
+        };
+
+        defer_bucket_in_rebalance_queue(pool_stat, &bucket)?;
+        pool_stat.info.last_error = Some(last_error);
+        meta.last_refreshed_at = Some(OffsetDateTime::now_utc());
+        Ok(())
+    }
+
+    pub async fn is_rebalance_started(&self) -> bool {
+        let rebalance_meta = self.rebalance_meta.read().await;
+        if let Some(meta) = rebalance_meta.as_ref() {
+            meta.pool_stats.iter().enumerate().for_each(|(i, v)| {
+                debug!(
+                    event = EVENT_REBALANCE_STATE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index = i,
+                    participating = v.participating,
+                    status = ?v.info.status,
+                    state = "status_inspected",
+                    "Rebalance status inspected"
+                );
+            });
+
+            let started = is_rebalance_conflicting_with_decommission(meta);
+            if started {
+                debug!(
+                    event = EVENT_REBALANCE_STATE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    state = "running",
+                    "Rebalance is running"
+                );
+                return true;
+            }
+        }
+
+        debug!(
+            event = EVENT_REBALANCE_STATE,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            state = "not_running",
+            "Rebalance is not running"
+        );
+        false
+    }
+
+    pub async fn is_rebalance_conflicting_with_decommission(&self) -> bool {
+        let rebalance_meta = self.rebalance_meta.read().await;
+        rebalance_meta
+            .as_ref()
+            .is_some_and(is_rebalance_conflicting_with_decommission)
+    }
+
+    pub async fn is_pool_rebalancing(&self, pool_index: usize) -> bool {
+        let rebalance_meta = self.rebalance_meta.read().await;
+        if let Some(ref meta) = *rebalance_meta {
+            if meta.stopped_at.is_some() {
+                return false;
+            }
+
+            if let Some(pool_stat) = meta.pool_stats.get(pool_index) {
+                return pool_stat.participating && pool_stat.info.status == RebalStatus::Started;
+            }
+        }
+
+        false
+    }
+
+    pub async fn pool_rebalance_status(&self, pool_index: usize) -> (RebalStatus, bool) {
+        let rebalance_meta = self.rebalance_meta.read().await;
+        pool_rebalance_status_from_meta(rebalance_meta.as_ref(), pool_index)
+    }
+
+    pub async fn current_rebalance_id(&self) -> Option<String> {
+        let rebalance_meta = self.rebalance_meta.read().await;
+        rebalance_meta
+            .as_ref()
+            .and_then(|meta| (!meta.id.is_empty()).then(|| meta.id.clone()))
+    }
+
+    #[tracing::instrument(skip(self))]
+    pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> {
+        self.stop_rebalance_for_id(None).await
+    }
+
+    #[tracing::instrument(skip(self))]
+    pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> {
+        let meta_to_save = {
+            let mut rebalance_meta = self.rebalance_meta.write().await;
+            stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id)
+        }?;
+
+        if let Some(meta_to_save) = meta_to_save {
+            let pool = clone_first_arc(self.pools.as_slice(), "stop_rebalance: no pools available")?;
+            resolve_rebalance_meta_save_result(
+                self.save_rebalance_meta_with_merge(pool, &meta_to_save, "stop_rebalance")
+                    .await,
+                "stop_rebalance",
+            )?;
+        }
+
+        Ok(())
+    }
+
+    async fn rollback_rebalance_start_without_worker_for_id(
+        self: &Arc<Self>,
+        expected_id: Option<&str>,
+        start_error: String,
+    ) -> Result<()> {
+        let meta_to_save = {
+            let mut rebalance_meta = self.rebalance_meta.write().await;
+            rollback_rebalance_start_meta_snapshot_for_id(
+                rebalance_meta.as_mut(),
+                OffsetDateTime::now_utc(),
+                expected_id,
+                start_error,
+            )
+        };
+
+        if let Some(meta_to_save) = meta_to_save {
+            let pool = clone_first_arc(self.pools.as_slice(), "rollback_rebalance_start: no pools available")?;
+            resolve_rebalance_meta_save_result(
+                self.save_rebalance_meta_with_merge(pool, &meta_to_save, "rollback_rebalance_start")
+                    .await,
+                "rollback_rebalance_start",
+            )?;
+        }
+
+        Ok(())
+    }
+
+    pub async fn record_rebalance_stop_propagation(self: &Arc<Self>, record: RebalanceStopPropagationRecord) -> Result<()> {
+        if !record.has_failures() {
+            return Ok(());
+        }
+
+        let encoded_error = encode_rebalance_stop_propagation_record(&record);
+        let meta_to_save = {
+            let mut rebalance_meta = self.rebalance_meta.write().await;
+            record_rebalance_stop_propagation_snapshot(rebalance_meta.as_mut(), encoded_error, OffsetDateTime::now_utc())
+        };
+
+        if let Some(meta_to_save) = meta_to_save {
+            let pool = clone_first_arc(self.pools.as_slice(), "record_rebalance_stop_propagation: no pools available")?;
+            resolve_rebalance_meta_save_result(
+                self.save_rebalance_meta_with_merge(pool, &meta_to_save, "record_rebalance_stop_propagation")
+                    .await,
+                "record_rebalance_stop_propagation",
+            )?;
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn pool_rebalance_status_ignores_non_participating_pool_state() {
+        let meta = RebalanceMeta {
+            pool_stats: vec![
+                RebalanceStats {
+                    participating: false,
+                    info: RebalanceInfo {
+                        status: RebalStatus::Started,
+                        stopping: true,
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                },
+                RebalanceStats {
+                    participating: true,
+                    info: RebalanceInfo {
+                        status: RebalStatus::Started,
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                },
+            ],
+            ..Default::default()
+        };
+
+        assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 0), (RebalStatus::None, false));
+        assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 1), (RebalStatus::Started, false));
+        assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 2), (RebalStatus::None, false));
+    }
+
+    #[test]
+    fn rebalance_status_refresh_applies_persisted_terminal_state() {
+        let rebalance_id = "rebalance-id".to_string();
+        let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
+        let mut current = Some(RebalanceMeta {
+            id: rebalance_id.clone(),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    start_time: Some(now),
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        });
+        let persisted = RebalanceMeta {
+            id: rebalance_id,
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Completed,
+                    start_time: Some(now),
+                    end_time: Some(now),
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+
+        merge_rebalance_status_refresh(&mut current, persisted);
+
+        let refreshed = current.as_ref().expect("refresh should keep rebalance metadata");
+        assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
+        assert_eq!(refreshed.pool_stats[0].info.end_time, Some(now));
+    }
+
+    #[test]
+    fn rebalance_status_refresh_preserves_runtime_cancel_token() {
+        let rebalance_id = "rebalance-id".to_string();
+        let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
+        let mut current = Some(RebalanceMeta {
+            id: rebalance_id.clone(),
+            cancel: Some(tokio_util::sync::CancellationToken::new()),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    start_time: Some(now),
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        });
+        let persisted = RebalanceMeta {
+            id: rebalance_id,
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    start_time: Some(now),
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+
+        merge_rebalance_status_refresh(&mut current, persisted);
+
+        assert!(
+            current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(),
+            "status refresh must not drop the runtime cancellation token"
+        );
+    }
+
+    #[test]
+    fn rebalance_status_refresh_preserves_local_active_different_id_conflict() {
+        let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
+        let mut current = Some(RebalanceMeta {
+            id: "old-active-id".to_string(),
+            cancel: Some(tokio_util::sync::CancellationToken::new()),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    start_time: Some(now),
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        });
+        let persisted = RebalanceMeta {
+            id: "new-terminal-id".to_string(),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Completed,
+                    start_time: Some(now),
+                    end_time: Some(now),
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+
+        merge_rebalance_status_refresh(&mut current, persisted);
+
+        let refreshed = current.as_ref().expect("local active metadata should remain visible");
+        assert_eq!(refreshed.id, "old-active-id");
+        assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Started);
+        assert!(
+            refreshed.cancel.is_some(),
+            "status refresh must not drop a live runtime cancellation token"
+        );
+    }
+
+    #[test]
+    fn rebalance_status_refresh_replaces_stale_memory_without_runtime_token() {
+        let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
+        let mut current = Some(RebalanceMeta {
+            id: "old-stale-id".to_string(),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    start_time: Some(now),
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        });
+        let persisted = RebalanceMeta {
+            id: "new-terminal-id".to_string(),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Completed,
+                    start_time: Some(now),
+                    end_time: Some(now),
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+
+        merge_rebalance_status_refresh(&mut current, persisted);
+
+        let refreshed = current.as_ref().expect("persisted metadata should replace stale memory");
+        assert_eq!(refreshed.id, "new-terminal-id");
+        assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
+        assert!(refreshed.cancel.is_none());
+    }
+}

crates/ecstore/src/services/rebalance/entry.rs

@@ -0,0 +1,533 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::meta::{
+    clone_arc_by_index, ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error,
+    rebalance_metadata_not_initialized_error, should_ignore_rebalance_data_usage_cache,
+};
+use super::migration::migrate_entry_version;
+use super::worker::{
+    RebalanceEntryTask, load_rebalance_bucket_configs, rebalance_max_attempts, resolve_rebalance_bucket_error,
+    resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
+    resolve_rebalance_migrate_result_error, resolve_rebalance_stats_update_result, resolve_rebalance_worker_result,
+    run_rebalance_listing_with_retry, should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete,
+    should_defer_rebalance_entry_failure, should_skip_rebalance_delete_marker, wait_rebalance_entry_tasks,
+    with_rebalance_entry_context,
+};
+use super::{
+    EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_ENTRY, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
+    REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome,
+};
+use crate::core::pools::ListCallback;
+use crate::data_movement;
+use crate::data_movement::backpressure::{self, DataMovementOperation};
+use crate::error::{Error, Result};
+use crate::object_api::{GetObjectReader, ObjectOptions};
+use crate::set_disk::SetDisks;
+use crate::storage_api_contracts::object::ObjectOperations as _;
+use crate::store::ECStore;
+use rustfs_filemeta::MetaCacheEntry;
+use rustfs_utils::path::encode_dir_object;
+use std::sync::Arc;
+use time::OffsetDateTime;
+use tokio_util::sync::CancellationToken;
+use tracing::{debug, error, warn};
+
+impl ECStore {
+    #[allow(unused_assignments)]
+    #[tracing::instrument(skip(self, set))]
+    async fn rebalance_entry(
+        self: Arc<Self>,
+        bucket: String,
+        pool_index: usize,
+        entry: MetaCacheEntry,
+        set: Arc<SetDisks>,
+        bucket_configs: Arc<RebalanceBucketConfigs>,
+        // wk: Arc<Workers>,
+    ) -> Result<RebalanceEntryOutcome> {
+        debug!(
+            event = EVENT_REBALANCE_ENTRY,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            bucket = %bucket,
+            object = %entry.name,
+            pool_index,
+            state = "started",
+            "Starting rebalance entry"
+        );
+
+        // defer!(|| async {
+        //     warn!("rebalance_entry: defer give worker start");
+        //     wk.give().await;
+        //     warn!("rebalance_entry: defer give worker done");
+        // });
+
+        if entry.is_dir() {
+            debug!(
+                event = EVENT_REBALANCE_ENTRY,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                bucket = %bucket,
+                object = %entry.name,
+                pool_index,
+                state = "skipped",
+                reason = "directory_entry",
+                "Skipped rebalance entry"
+            );
+            return Ok(RebalanceEntryOutcome::Completed);
+        }
+
+        if self.check_if_rebalance_done(pool_index).await {
+            debug!(
+                event = EVENT_REBALANCE_ENTRY,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                pool_index,
+                bucket = %bucket,
+                object = %entry.name,
+                state = "skipped",
+                reason = "pool_completed",
+                "Skipped rebalance entry"
+            );
+            return Ok(RebalanceEntryOutcome::Completed);
+        }
+
+        let mut fivs =
+            resolve_rebalance_file_info_versions_result(entry.file_info_versions(&bucket), bucket.as_str(), entry.name.as_str())?;
+
+        fivs.versions
+            .sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
+
+        let mut rebalanced: usize = 0;
+        let mut expired: usize = 0;
+        let mut stats_updates = Vec::with_capacity(fivs.versions.len());
+        for version in fivs.versions.iter() {
+            if crate::core::pools::should_skip_lifecycle_for_data_movement(
+                self.clone(),
+                &bucket,
+                version,
+                bucket_configs.lifecycle_config.as_ref(),
+                bucket_configs.lock_retention.clone(),
+                bucket_configs.replication_config.clone(),
+                true,
+                &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Rebal,
+            )
+            .await?
+            {
+                expired += 1;
+                debug!(
+                    event = EVENT_REBALANCE_ENTRY,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index,
+                    bucket = %bucket,
+                    object = %version.name,
+                    state = "skipped",
+                    reason = "expired_by_lifecycle",
+                    "Skipped rebalance version"
+                );
+                continue;
+            }
+
+            let remaining_versions = fivs.versions.len() - expired;
+            if should_skip_rebalance_delete_marker(version, remaining_versions, bucket_configs.replication_config.is_some()) {
+                rebalanced += 1;
+                debug!(
+                    event = EVENT_REBALANCE_ENTRY,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index,
+                    bucket = %bucket,
+                    object = %version.name,
+                    state = "skipped",
+                    reason = "last_delete_marker_without_replication",
+                    "Skipped rebalance version"
+                );
+                continue;
+            }
+
+            let version_id = version.version_id.map(|v| v.to_string());
+            let mut transfer = |src_pool_idx: usize, bucket: String, rd: GetObjectReader| {
+                let store = self.clone();
+                async move { store.rebalance_object(src_pool_idx, bucket, rd).await }
+            };
+            let result = migrate_entry_version(
+                set.as_ref(),
+                bucket.clone(),
+                pool_index,
+                version,
+                version_id.clone(),
+                rebalance_max_attempts(),
+                should_ignore_rebalance_data_usage_cache(bucket.as_str()),
+                &mut transfer,
+            )
+            .await;
+
+            if result.ignored {
+                if should_count_rebalance_version_complete(&result) {
+                    rebalanced += 1;
+                }
+                debug!(
+                    event = EVENT_REBALANCE_ENTRY,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index,
+                    bucket = %bucket,
+                    object = %version.name,
+                    state = "skipped",
+                    reason = "already_deleted",
+                    "Skipped rebalance version"
+                );
+                continue;
+            }
+
+            if result.failed {
+                let err = resolve_rebalance_migrate_result_error(
+                    result.error,
+                    pool_index,
+                    bucket.as_str(),
+                    version.name.as_str(),
+                    version_id.as_deref(),
+                );
+                error!(
+                    "rebalance_entry {} Error rebalancing entry {}/{:?}: {:?}",
+                    &bucket, &version.name, &version.version_id, err
+                );
+                if should_defer_rebalance_entry_failure(&err) {
+                    let deferred_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} {err}");
+                    warn!(
+                        event = EVENT_REBALANCE_ENTRY,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        pool_index,
+                        bucket = %bucket,
+                        object = %version.name,
+                        state = "deferred",
+                        error = %err,
+                        "Deferred rebalance entry after transient migration failure"
+                    );
+                    if let Err(stats_err) = self.update_rebalance_last_error(pool_index, deferred_error.clone()).await {
+                        error!(
+                            "rebalance_entry {} failed to record deferred transient failure for {}: {}",
+                            &bucket, &entry.name, stats_err
+                        );
+                    }
+                    return Ok(RebalanceEntryOutcome::Deferred {
+                        last_error: deferred_error,
+                    });
+                }
+                let entry_err =
+                    with_rebalance_entry_context(result.stage.unwrap_or("migrate"), bucket.as_str(), version.name.as_str(), err);
+
+                if !stats_updates.is_empty()
+                    && let Err(stats_err) = self
+                        .update_pool_stats_batch(pool_index, bucket.clone(), stats_updates.as_slice())
+                        .await
+                {
+                    error!(
+                        "rebalance_entry {} failed to update stats before returning migration error for {}: {}",
+                        &bucket, &entry.name, stats_err
+                    );
+                }
+
+                return Err(entry_err);
+            }
+
+            stats_updates.push(version);
+            if should_count_rebalance_version_complete(&result) {
+                rebalanced += 1;
+            }
+        }
+
+        resolve_rebalance_stats_update_result(
+            self.update_pool_stats_batch(pool_index, bucket.clone(), stats_updates.as_slice())
+                .await,
+            pool_index,
+            bucket.as_str(),
+            entry.name.as_str(),
+        )?;
+
+        if should_cleanup_rebalance_source_entry(rebalanced, fivs.versions.len()) {
+            let cleanup_warning = resolve_rebalance_entry_cleanup_delete_result(
+                set.delete_object(
+                    bucket.as_str(),
+                    &encode_dir_object(&entry.name),
+                    ObjectOptions {
+                        delete_prefix: true,
+                        delete_prefix_object: true,
+
+                        ..Default::default()
+                    },
+                )
+                .await,
+                bucket.as_str(),
+                entry.name.as_str(),
+            )?;
+            if let Some(message) = cleanup_warning {
+                warn!(
+                    event = EVENT_REBALANCE_ENTRY,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index,
+                    bucket = %bucket,
+                    object = %entry.name,
+                    stage = "cleanup_source",
+                    cleanup_status = "failed_ignored",
+                    error = %message,
+                    "Ignored rebalance source cleanup failure"
+                );
+                if let Err(err) = self
+                    .record_rebalance_cleanup_warning(pool_index, bucket.as_str(), entry.name.as_str(), message)
+                    .await
+                {
+                    error!(
+                        event = EVENT_REBALANCE_ENTRY,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        pool_index,
+                        bucket = %bucket,
+                        object = %entry.name,
+                        stage = "cleanup_source",
+                        error = ?err,
+                        "Failed to record rebalance source cleanup warning"
+                    );
+                }
+            } else {
+                debug!(
+                    event = EVENT_REBALANCE_ENTRY,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index,
+                    bucket = %bucket,
+                    object = %entry.name,
+                    state = "source_deleted",
+                    "Deleted rebalance source entry"
+                );
+            }
+        }
+
+        Ok(RebalanceEntryOutcome::Completed)
+    }
+
+    #[tracing::instrument(skip(self, rd))]
+    async fn rebalance_object(self: Arc<Self>, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> {
+        data_movement::migrate_object(self, pool_idx, bucket, rd, "rebalance_object").await
+    }
+
+    async fn update_rebalance_last_error(&self, pool_idx: usize, message: String) -> Result<()> {
+        let mut rebalance_meta = self.rebalance_meta.write().await;
+        let Some(meta) = rebalance_meta.as_mut() else {
+            return Err(rebalance_metadata_not_initialized_error("record rebalance last error"));
+        };
+        let pool_count = meta.pool_stats.len();
+        ensure_valid_rebalance_pool_index(pool_count, pool_idx)?;
+        let Some(pool_stat) = meta.pool_stats.get_mut(pool_idx) else {
+            return Err(invalid_rebalance_pool_index_error(pool_idx, pool_count));
+        };
+
+        pool_stat.info.last_error = Some(message);
+        meta.last_refreshed_at = Some(OffsetDateTime::now_utc());
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, rx))]
+    pub(super) async fn rebalance_bucket(
+        self: &Arc<Self>,
+        rx: CancellationToken,
+        bucket: String,
+        pool_index: usize,
+    ) -> Result<RebalanceBucketOutcome> {
+        ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?;
+
+        // Placeholder for actual bucket rebalance logic
+        debug!(
+            event = EVENT_REBALANCE_BUCKET,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            pool_index,
+            bucket = %bucket,
+            state = "entry_scan_started",
+            "Rebalance bucket entry scan started"
+        );
+
+        // TODO: other config
+        // if bucket != RUSTFS_META_BUCKET{
+
+        // }
+
+        let pool = clone_arc_by_index(self.pools.as_slice(), pool_index, "invalid rebalance pool index")?;
+        let bucket_configs = Arc::new(load_rebalance_bucket_configs(&bucket).await?);
+
+        let mut jobs = Vec::new();
+        let entry_error = Arc::new(tokio::sync::Mutex::new(None::<Error>));
+        let entry_workers = Arc::new(tokio::sync::Semaphore::new(pool.disk_set.len().max(1)));
+
+        for (set_idx, set) in pool.disk_set.iter().enumerate() {
+            let entry_tasks = Arc::new(tokio::sync::Mutex::new(Vec::<RebalanceEntryTask>::new()));
+            let rebalance_entry: ListCallback = Arc::new({
+                let this = Arc::clone(self);
+                let bucket = bucket.clone();
+                let entry_error = entry_error.clone();
+                let callback_rx = rx.clone();
+                let set = set.clone();
+                let bucket_configs = bucket_configs.clone();
+                let entry_tasks = entry_tasks.clone();
+                let entry_workers = entry_workers.clone();
+                move |entry: MetaCacheEntry| {
+                    let this = this.clone();
+                    let bucket = bucket.clone();
+                    let entry_error = entry_error.clone();
+                    let callback_rx = callback_rx.clone();
+                    let set = set.clone();
+                    let bucket_configs = bucket_configs.clone();
+                    let entry_tasks = entry_tasks.clone();
+                    let entry_workers = entry_workers.clone();
+                    Box::pin(async move {
+                        if callback_rx.is_cancelled() {
+                            return;
+                        }
+                        if entry_error.lock().await.is_some() {
+                            return;
+                        }
+
+                        if let Err(err) = backpressure::wait_for_data_movement_admission(
+                            DataMovementOperation::Rebalance,
+                            pool_index,
+                            &callback_rx,
+                        )
+                        .await
+                        {
+                            if matches!(err, Error::OperationCanceled) {
+                                return;
+                            }
+                            error!("rebalance_entry: data movement admission failed: {err}");
+                            let mut first_err = entry_error.lock().await;
+                            if first_err.is_none() {
+                                *first_err = Some(err);
+                                callback_rx.cancel();
+                            }
+                            return;
+                        }
+
+                        let permit = tokio::select! {
+                            _ = callback_rx.cancelled() => return,
+                            permit = entry_workers.clone().acquire_owned() => match permit {
+                                Ok(permit) => permit,
+                                Err(err) => {
+                                    error!("rebalance_entry: worker semaphore closed: {err}");
+                                    return;
+                                }
+                            },
+                        };
+
+                        if entry_error.lock().await.is_some() {
+                            return;
+                        }
+
+                        let task = tokio::spawn(async move {
+                            let _permit = permit;
+                            debug!(
+                                event = EVENT_REBALANCE_ENTRY,
+                                component = LOG_COMPONENT_ECSTORE,
+                                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                                set_index = set_idx,
+                                state = "task_started",
+                                "Started rebalance entry task"
+                            );
+                            let result = this.rebalance_entry(bucket, pool_index, entry, set, bucket_configs).await;
+                            if let Err(err) = &result {
+                                error!("rebalance_entry: rebalance entry failed: {err}");
+                                let mut first_err = entry_error.lock().await;
+                                if first_err.is_none() {
+                                    *first_err = Some(err.clone());
+                                    callback_rx.cancel();
+                                }
+                            }
+                            debug!(
+                                event = EVENT_REBALANCE_ENTRY,
+                                component = LOG_COMPONENT_ECSTORE,
+                                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                                set_index = set_idx,
+                                state = "task_completed",
+                                "Completed rebalance entry task"
+                            );
+                            result
+                        });
+
+                        entry_tasks.lock().await.push(task);
+                    })
+                }
+            });
+
+            let set = set.clone();
+            let rx = rx.clone();
+            let bucket = bucket.clone();
+            let entry_tasks = entry_tasks.clone();
+
+            let job = tokio::spawn(async move {
+                let list_result =
+                    run_rebalance_listing_with_retry(set, rx, bucket.clone(), rebalance_entry, set_idx, rebalance_max_attempts())
+                        .await;
+                let entry_result = wait_rebalance_entry_tasks(set_idx, entry_tasks).await;
+                let result = list_result.and(entry_result);
+                if let Err(err) = &result {
+                    error!("Rebalance worker {} error: {}", set_idx, err);
+                } else {
+                    debug!(
+                        event = EVENT_REBALANCE_STATE,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        set_index = set_idx,
+                        state = "worker_completed",
+                        "Completed rebalance worker"
+                    );
+                }
+                result
+            });
+
+            jobs.push((set_idx, job));
+        }
+
+        let mut worker_error: Option<Error> = None;
+        let mut deferred_error: Option<String> = None;
+        for (set_idx, job) in jobs {
+            match resolve_rebalance_worker_result(set_idx, job.await) {
+                Ok(Some(last_error)) if deferred_error.is_none() => {
+                    deferred_error = Some(last_error);
+                }
+                Ok(_) => {}
+                Err(err) if worker_error.is_none() => {
+                    worker_error = Some(err);
+                }
+                Err(_) => {}
+            }
+        }
+        let entry_error = entry_error.lock().await.clone();
+        resolve_rebalance_bucket_error(entry_error, worker_error)?;
+        if let Some(last_error) = deferred_error {
+            return Ok(RebalanceBucketOutcome::Deferred { last_error });
+        }
+
+        debug!(
+            event = EVENT_REBALANCE_BUCKET,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            pool_index,
+            bucket = %bucket,
+            state = "completed",
+            "Finished rebalance bucket"
+        );
+        Ok(RebalanceBucketOutcome::Completed)
+    }
+}

crates/ecstore/src/services/rebalance/meta.rs

@@ -0,0 +1,988 @@
+use super::{
+    EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, Error, GetObjectReader, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
+    ObjectInfo, ObjectOptions, PutObjReader, REBAL_META_FMT, REBAL_META_NAME, REBAL_META_VER,
+    REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT, REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_STOP_PROPAGATION_ERROR_PREFIX,
+    RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats,
+    RebalanceStopPropagationRecord, Result,
+};
+use crate::config::com::{read_config_with_metadata, save_config_with_opts};
+use crate::error::is_err_operation_canceled;
+use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec};
+use http::HeaderMap;
+use rustfs_filemeta::FileInfo;
+use std::{collections::HashSet, fmt, io::Cursor, sync::Arc};
+use time::OffsetDateTime;
+use tracing::{debug, info};
+
+impl RebalanceStats {
+    pub fn update(&mut self, bucket: String, fi: &FileInfo) {
+        if fi.is_latest {
+            self.num_objects += 1;
+        }
+
+        self.num_versions += 1;
+        let on_disk_size = if fi.deleted || fi.erasure.data_blocks == 0 || fi.size <= 0 {
+            0
+        } else {
+            let data_blocks = fi.erasure.data_blocks as i64;
+            let total_blocks = fi.erasure.data_blocks.saturating_add(fi.erasure.parity_blocks) as i64;
+            fi.size
+                .saturating_mul(total_blocks)
+                .checked_div(data_blocks)
+                .unwrap_or(0)
+                .max(0) as u64
+        };
+        self.bytes = self.bytes.saturating_add(on_disk_size);
+        self.bucket = bucket;
+        self.object = fi.name.clone();
+    }
+
+    pub fn update_batch(&mut self, bucket: String, versions: &[&FileInfo]) {
+        for version in versions {
+            self.update(bucket.clone(), version);
+        }
+    }
+}
+
+impl fmt::Display for RebalStatus {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let status = match self {
+            RebalStatus::None => "None",
+            RebalStatus::Started => "Started",
+            RebalStatus::Completed => "Completed",
+            RebalStatus::Stopped => "Stopped",
+            RebalStatus::Failed => "Failed",
+        };
+        write!(f, "{status}")
+    }
+}
+
+impl From<u8> for RebalStatus {
+    fn from(value: u8) -> Self {
+        match value {
+            1 => RebalStatus::Started,
+            2 => RebalStatus::Completed,
+            3 => RebalStatus::Stopped,
+            4 => RebalStatus::Failed,
+            _ => RebalStatus::None,
+        }
+    }
+}
+
+impl RebalanceMeta {
+    pub fn new() -> Self {
+        Self::default()
+    }
+    pub async fn load<S>(&mut self, store: Arc<S>) -> Result<()>
+    where
+        S: ObjectIO<
+                Error = Error,
+                RangeSpec = HTTPRangeSpec,
+                HeaderMap = HeaderMap,
+                ObjectOptions = ObjectOptions,
+                ObjectInfo = ObjectInfo,
+                GetObjectReader = GetObjectReader,
+                PutObjectReader = PutObjReader,
+            >,
+    {
+        self.load_with_opts(store, ObjectOptions::default()).await
+    }
+
+    pub async fn load_with_opts<S>(&mut self, store: Arc<S>, opts: ObjectOptions) -> Result<()>
+    where
+        S: ObjectIO<
+                Error = Error,
+                RangeSpec = HTTPRangeSpec,
+                HeaderMap = HeaderMap,
+                ObjectOptions = ObjectOptions,
+                ObjectInfo = ObjectInfo,
+                GetObjectReader = GetObjectReader,
+                PutObjectReader = PutObjReader,
+            >,
+    {
+        let (data, _) = read_config_with_metadata(store, REBAL_META_NAME, &opts).await?;
+        if data.is_empty() {
+            debug!(
+                event = EVENT_REBALANCE_STATE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                state = "metadata_empty",
+                "Rebalance metadata is empty"
+            );
+            return Ok(());
+        }
+        if data.len() <= 4 {
+            return Err(rebalance_meta_load_no_data_error());
+        }
+
+        // Read header
+        match u16::from_le_bytes([data[0], data[1]]) {
+            REBAL_META_FMT => {}
+            fmt => return Err(rebalance_meta_load_unknown_format_error(fmt)),
+        }
+        match u16::from_le_bytes([data[2], data[3]]) {
+            REBAL_META_VER => {}
+            ver => return Err(rebalance_meta_load_unknown_version_error(ver)),
+        }
+
+        let meta: Self = rmp_serde::from_read(Cursor::new(&data[4..]))?;
+        *self = meta;
+
+        self.last_refreshed_at = Some(OffsetDateTime::now_utc());
+
+        debug!(
+            event = EVENT_REBALANCE_STATE,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            state = "metadata_loaded",
+            "Loaded rebalance metadata"
+        );
+        Ok(())
+    }
+
+    pub async fn save<S>(&self, store: Arc<S>) -> Result<()>
+    where
+        S: ObjectIO<
+                Error = Error,
+                RangeSpec = HTTPRangeSpec,
+                HeaderMap = HeaderMap,
+                ObjectOptions = ObjectOptions,
+                ObjectInfo = ObjectInfo,
+                GetObjectReader = GetObjectReader,
+                PutObjectReader = PutObjReader,
+            >,
+    {
+        self.save_with_opts(store, ObjectOptions::default()).await
+    }
+
+    pub async fn save_with_opts<S>(&self, store: Arc<S>, opts: ObjectOptions) -> Result<()>
+    where
+        S: ObjectIO<
+                Error = Error,
+                RangeSpec = HTTPRangeSpec,
+                HeaderMap = HeaderMap,
+                ObjectOptions = ObjectOptions,
+                ObjectInfo = ObjectInfo,
+                GetObjectReader = GetObjectReader,
+                PutObjectReader = PutObjReader,
+            >,
+    {
+        if self.pool_stats.is_empty() {
+            debug!(
+                event = EVENT_REBALANCE_STATE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                state = "metadata_save_skipped",
+                reason = "no_pool_stats",
+                "Skipped rebalance metadata save"
+            );
+            return Ok(());
+        }
+
+        let mut data = Vec::new();
+
+        // Initialize the header
+        data.extend(&REBAL_META_FMT.to_le_bytes());
+        data.extend(&REBAL_META_VER.to_le_bytes());
+
+        let msg = rmp_serde::to_vec(self)?;
+        data.extend(msg);
+
+        save_config_with_opts(store, REBAL_META_NAME, data, &opts).await?;
+
+        Ok(())
+    }
+}
+
+pub(super) fn is_rebalance_pool_started(pool_stat: &RebalanceStats) -> bool {
+    pool_stat.participating && pool_stat.info.status == RebalStatus::Started
+}
+
+pub(super) fn is_rebalance_pool_active(pool_stat: &RebalanceStats) -> bool {
+    is_rebalance_pool_started(pool_stat) || pool_stat.info.stopping
+}
+
+pub(super) fn is_rebalance_in_progress(meta: &RebalanceMeta) -> bool {
+    meta.pool_stats.iter().any(is_rebalance_pool_active)
+}
+
+pub(crate) fn is_rebalance_conflicting_with_decommission(meta: &RebalanceMeta) -> bool {
+    is_rebalance_in_progress(meta)
+}
+
+pub(super) fn first_rebalance_bucket(pool_stat: &RebalanceStats) -> Option<String> {
+    pool_stat.buckets.first().cloned()
+}
+
+pub(super) fn rebalance_meta_load_no_data_error() -> Error {
+    Error::other("rebalance metadata load failed: metadata payload is too short")
+}
+
+pub(super) fn rebalance_meta_load_unknown_format_error(fmt: u16) -> Error {
+    Error::other(format!("rebalance metadata load failed: unknown format {fmt}"))
+}
+
+pub(super) fn rebalance_meta_load_unknown_version_error(ver: u16) -> Error {
+    Error::other(format!("rebalance metadata load failed: unknown version {ver}"))
+}
+
+pub fn encode_rebalance_stop_propagation_record(record: &RebalanceStopPropagationRecord) -> String {
+    match serde_json::to_string(record) {
+        Ok(payload) => format!("{REBALANCE_STOP_PROPAGATION_ERROR_PREFIX}{payload}"),
+        Err(err) => {
+            let payload = serde_json::json!({
+                "encodeError": err.to_string(),
+                "stopFailures": [],
+                "terminalReloadFailures": [],
+            });
+            format!("{REBALANCE_STOP_PROPAGATION_ERROR_PREFIX}{payload}")
+        }
+    }
+}
+
+pub fn decode_rebalance_stop_propagation_record(message: &str) -> Option<RebalanceStopPropagationRecord> {
+    let payload = message.strip_prefix(REBALANCE_STOP_PROPAGATION_ERROR_PREFIX)?;
+    serde_json::from_str(payload).ok()
+}
+
+fn is_rebalance_stop_propagation_error(message: Option<&str>) -> bool {
+    message.is_some_and(|message| message.starts_with(REBALANCE_STOP_PROPAGATION_ERROR_PREFIX))
+}
+
+pub(super) fn rebalance_goal_reached(init_free_space: u64, init_capacity: u64, bytes: u64, percent_free_goal: f64) -> bool {
+    if init_capacity == 0 {
+        return false;
+    }
+
+    let pfi = (init_free_space + bytes) as f64 / init_capacity as f64;
+    pfi + f64::EPSILON >= percent_free_goal
+}
+
+pub(super) fn percent_free_ratio(total_free: u64, total_cap: u64) -> f64 {
+    if total_cap == 0 {
+        return 0.0;
+    }
+    total_free as f64 / total_cap as f64
+}
+
+pub(super) fn next_rebal_bucket_from_stat(pool_stat: &RebalanceStats) -> Option<String> {
+    if pool_stat.buckets.is_empty() {
+        return None;
+    }
+
+    first_rebalance_bucket(pool_stat)
+}
+
+pub(super) fn rebalance_metadata_not_initialized_error(operation: &str) -> Error {
+    Error::other(format!("failed to {operation}: rebalance metadata not initialized"))
+}
+
+pub(super) fn invalid_rebalance_pool_index_error(pool_index: usize, pool_count: usize) -> Error {
+    Error::other(format!("invalid rebalance pool index {pool_index} for {pool_count} pools"))
+}
+
+pub(super) fn clone_rebalance_pool_stats(meta: Option<&RebalanceMeta>) -> Result<Vec<RebalanceStats>> {
+    let Some(meta) = meta else {
+        return Err(rebalance_metadata_not_initialized_error("clone rebalance pool stats"));
+    };
+    Ok(meta.pool_stats.clone())
+}
+
+pub(super) fn should_accept_rebalance_stats_update(meta: &RebalanceMeta, pool_index: usize) -> bool {
+    if meta.stopped_at.is_some() {
+        return false;
+    }
+
+    meta.pool_stats
+        .get(pool_index)
+        .is_some_and(|pool_stat| pool_stat.info.status == RebalStatus::Started)
+}
+
+pub(super) fn resolve_next_rebalance_bucket(meta: Option<&RebalanceMeta>, pool_index: usize) -> Result<Option<String>> {
+    let Some(meta) = meta else {
+        return Err(rebalance_metadata_not_initialized_error("resolve next rebalance bucket"));
+    };
+
+    ensure_valid_rebalance_pool_index(meta.pool_stats.len(), pool_index)?;
+    let Some(pool_stat) = meta.pool_stats.get(pool_index) else {
+        return Err(invalid_rebalance_pool_index_error(pool_index, meta.pool_stats.len()));
+    };
+
+    if pool_stat.info.status == RebalStatus::Completed || !pool_stat.participating {
+        debug!(
+            event = EVENT_REBALANCE_BUCKET,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            pool_index,
+            state = "unavailable",
+            reason = "completed_or_not_participating",
+            "No rebalance bucket available"
+        );
+        return Ok(None);
+    }
+
+    if pool_stat.buckets.is_empty() {
+        debug!(
+            event = EVENT_REBALANCE_BUCKET,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            pool_index,
+            state = "unavailable",
+            reason = "bucket_queue_empty",
+            "No rebalance bucket available"
+        );
+        return Ok(None);
+    }
+
+    if let Some(bucket) = next_rebal_bucket_from_stat(pool_stat) {
+        debug!(
+            event = EVENT_REBALANCE_BUCKET,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            pool_index,
+            bucket = %bucket,
+            state = "selected",
+            "Selected rebalance bucket"
+        );
+        return Ok(Some(bucket));
+    }
+
+    debug!(
+        event = EVENT_REBALANCE_BUCKET,
+        component = LOG_COMPONENT_ECSTORE,
+        subsystem = LOG_SUBSYSTEM_REBALANCE,
+        pool_index,
+        state = "unavailable",
+        reason = "selection_returned_none",
+        "No rebalance bucket available"
+    );
+    Ok(None)
+}
+
+pub(super) fn mark_rebalance_bucket_done(meta: Option<&mut RebalanceMeta>, pool_index: usize, bucket: &str) -> Result<()> {
+    let Some(meta) = meta else {
+        return Err(rebalance_metadata_not_initialized_error("mark rebalance bucket done"));
+    };
+
+    ensure_valid_rebalance_pool_index(meta.pool_stats.len(), pool_index)?;
+    let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) else {
+        return Err(invalid_rebalance_pool_index_error(pool_index, meta.pool_stats.len()));
+    };
+
+    if take_bucket_from_rebalance_queue(pool_stat, bucket) {
+        debug!(
+            event = EVENT_REBALANCE_BUCKET,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            pool_index,
+            bucket = %bucket,
+            state = "queue_removed",
+            remaining_bucket_count = pool_stat.buckets.len(),
+            "Removed bucket from rebalance queue"
+        );
+        if has_deferred_rebalance_error(pool_stat) {
+            pool_stat.info.last_error = None;
+        }
+        Ok(())
+    } else {
+        Err(Error::other(format!(
+            "failed to mark rebalance bucket done: bucket {bucket} was not queued for pool {pool_index}"
+        )))
+    }
+}
+
+pub(super) fn record_rebalance_cleanup_warning_in_meta(
+    meta: Option<&mut RebalanceMeta>,
+    pool_index: usize,
+    bucket: &str,
+    object: &str,
+    message: String,
+    now: OffsetDateTime,
+) -> Result<()> {
+    let Some(meta) = meta else {
+        return Err(rebalance_metadata_not_initialized_error("record rebalance cleanup warning"));
+    };
+
+    ensure_valid_rebalance_pool_index(meta.pool_stats.len(), pool_index)?;
+    let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) else {
+        return Err(invalid_rebalance_pool_index_error(pool_index, meta.pool_stats.len()));
+    };
+
+    pool_stat.cleanup_warnings.count = pool_stat.cleanup_warnings.count.saturating_add(1);
+    pool_stat.cleanup_warnings.last_message = Some(message.clone());
+    pool_stat.cleanup_warnings.last_bucket = Some(bucket.to_string());
+    pool_stat.cleanup_warnings.last_object = Some(object.to_string());
+    pool_stat.cleanup_warnings.last_at = Some(now);
+    pool_stat.cleanup_warnings.entries.push(RebalanceCleanupWarningEntry {
+        bucket: bucket.to_string(),
+        object: object.to_string(),
+        message,
+        timestamp: Some(now),
+    });
+    truncate_rebalance_cleanup_warning_entries(&mut pool_stat.cleanup_warnings.entries);
+    meta.last_refreshed_at = Some(now);
+    Ok(())
+}
+
+pub(super) fn take_bucket_from_rebalance_queue(pool_stat: &mut RebalanceStats, bucket: &str) -> bool {
+    let mut found = false;
+    pool_stat.buckets.retain(|name| {
+        if name == bucket {
+            found = true;
+            pool_stat.rebalanced_buckets.push(name.clone());
+            false
+        } else {
+            true
+        }
+    });
+
+    found
+}
+
+pub(super) fn defer_bucket_in_rebalance_queue(pool_stat: &mut RebalanceStats, bucket: &str) -> Result<()> {
+    let Some(pos) = pool_stat.buckets.iter().position(|name| name == bucket) else {
+        return Err(Error::other(format!("failed to defer rebalance bucket {bucket}: bucket was not queued")));
+    };
+
+    let bucket = pool_stat.buckets.remove(pos);
+    pool_stat.buckets.push(bucket);
+    Ok(())
+}
+
+pub(super) fn should_pool_participate(init_free_space: u64, init_capacity: u64, percent_free_goal: f64) -> bool {
+    init_capacity > 0 && percent_free_ratio(init_free_space, init_capacity) < percent_free_goal
+}
+
+pub(super) fn complete_rebalance_pools_at_goal(meta: &mut RebalanceMeta, now: OffsetDateTime) -> bool {
+    let mut changed = false;
+
+    for pool_stat in meta.pool_stats.iter_mut() {
+        if !is_rebalance_pool_started(pool_stat)
+            || has_deferred_rebalance_error(pool_stat)
+            || has_rebalance_cleanup_warnings(pool_stat)
+        {
+            continue;
+        }
+
+        if rebalance_goal_reached(
+            pool_stat.init_free_space,
+            pool_stat.init_capacity,
+            pool_stat.bytes,
+            meta.percent_free_goal,
+        ) {
+            pool_stat.info.status = RebalStatus::Completed;
+            pool_stat.info.end_time = Some(now);
+            pool_stat.info.last_error = None;
+            changed = true;
+        }
+    }
+
+    changed
+}
+
+pub(super) fn complete_rebalance_pools_with_empty_queue(meta: &mut RebalanceMeta, now: OffsetDateTime) -> bool {
+    let mut changed = false;
+
+    for pool_stat in meta.pool_stats.iter_mut() {
+        if !is_rebalance_pool_started(pool_stat) || !pool_stat.buckets.is_empty() || has_rebalance_cleanup_warnings(pool_stat) {
+            continue;
+        }
+
+        pool_stat.info.status = RebalStatus::Completed;
+        pool_stat.info.end_time = Some(now);
+        pool_stat.info.last_error = None;
+        changed = true;
+    }
+
+    changed
+}
+
+pub(super) fn has_deferred_rebalance_error(pool_stat: &RebalanceStats) -> bool {
+    pool_stat
+        .info
+        .last_error
+        .as_deref()
+        .is_some_and(|last_error| last_error.starts_with(REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX))
+}
+
+pub(super) fn has_rebalance_cleanup_warnings(pool_stat: &RebalanceStats) -> bool {
+    pool_stat.cleanup_warnings.count > 0
+}
+
+pub(super) fn clone_first_arc<T>(values: &[Arc<T>], err_msg: &str) -> Result<Arc<T>> {
+    values.first().cloned().ok_or_else(|| Error::other(err_msg))
+}
+
+pub(super) fn clone_arc_by_index<T>(values: &[Arc<T>], idx: usize, err_prefix: &str) -> Result<Arc<T>> {
+    values
+        .get(idx)
+        .cloned()
+        .ok_or_else(|| Error::other(format!("{err_prefix}: {idx}")))
+}
+
+pub(super) fn ensure_valid_rebalance_pool_index(pool_count: usize, idx: usize) -> Result<()> {
+    if idx >= pool_count {
+        return Err(invalid_rebalance_pool_index_error(idx, pool_count));
+    }
+
+    Ok(())
+}
+
+pub(super) enum RebalanceTerminalEvent {
+    Completed { msg: String },
+    Stopped { msg: String },
+    Failed { msg: String, last_error: String },
+    ChannelClosed { msg: String, last_error: String },
+}
+
+impl RebalanceTerminalEvent {
+    pub(super) fn message(&self) -> &str {
+        match self {
+            RebalanceTerminalEvent::Completed { msg }
+            | RebalanceTerminalEvent::Stopped { msg }
+            | RebalanceTerminalEvent::Failed { msg, .. }
+            | RebalanceTerminalEvent::ChannelClosed { msg, .. } => msg,
+        }
+    }
+}
+
+pub(super) fn apply_rebalance_terminal_event(
+    status: &mut RebalStatus,
+    end_time: &mut Option<OffsetDateTime>,
+    last_error: &mut Option<String>,
+    terminal_event: RebalanceTerminalEvent,
+    now: OffsetDateTime,
+) {
+    match terminal_event {
+        RebalanceTerminalEvent::Completed { .. } => {
+            *status = RebalStatus::Completed;
+            *end_time = Some(now);
+            *last_error = None;
+        }
+        RebalanceTerminalEvent::Stopped { .. } => {
+            *status = RebalStatus::Stopped;
+            *end_time = Some(now);
+            *last_error = None;
+        }
+        RebalanceTerminalEvent::Failed { last_error: err, .. }
+        | RebalanceTerminalEvent::ChannelClosed { last_error: err, .. } => {
+            *status = RebalStatus::Failed;
+            *end_time = Some(now);
+            *last_error = Some(err);
+        }
+    }
+}
+
+pub(super) fn classify_rebalance_terminal_event(signal: Option<Result<()>>, now: OffsetDateTime) -> RebalanceTerminalEvent {
+    match signal {
+        Some(Ok(())) => RebalanceTerminalEvent::Completed {
+            msg: format!("Rebalance completed at {now:?}"),
+        },
+        Some(Err(err)) => {
+            if is_err_operation_canceled(&err) {
+                RebalanceTerminalEvent::Stopped {
+                    msg: format!("Rebalance stopped at {now:?}"),
+                }
+            } else {
+                RebalanceTerminalEvent::Failed {
+                    msg: format!("Rebalance failed at {now:?} with err {err:?}"),
+                    last_error: err.to_string(),
+                }
+            }
+        }
+        None => RebalanceTerminalEvent::ChannelClosed {
+            msg: format!("Rebalance save task channel closed unexpectedly at {now:?}"),
+            last_error: format!("rebalance save channel closed before terminal event at {now:?}"),
+        },
+    }
+}
+
+pub(super) fn ensure_rebalance_not_decommissioning(decommission_running: bool) -> bool {
+    !decommission_running
+}
+
+pub(super) fn validate_start_rebalance_state(decommission_running: bool, meta_loaded: bool) -> Result<()> {
+    if !ensure_rebalance_not_decommissioning(decommission_running) {
+        return Err(Error::DecommissionAlreadyRunning);
+    }
+    if !meta_loaded {
+        return Err(Error::ConfigNotFound);
+    }
+
+    Ok(())
+}
+
+pub(super) fn validate_init_rebalance_state(decommission_running: bool, current_meta: Option<&RebalanceMeta>) -> Result<()> {
+    if !ensure_rebalance_not_decommissioning(decommission_running) {
+        return Err(Error::DecommissionAlreadyRunning);
+    }
+    if current_meta.is_some_and(|meta| !is_rebalance_meta_replaceable_for_new_id(meta)) {
+        return Err(Error::RebalanceAlreadyRunning);
+    }
+
+    Ok(())
+}
+
+pub(super) fn should_skip_start_rebalance(cancel_attached: bool, in_progress: bool) -> bool {
+    cancel_attached && in_progress
+}
+
+pub(super) fn is_rebalance_stopped_terminal_event(terminal_event: &RebalanceTerminalEvent) -> bool {
+    matches!(terminal_event, RebalanceTerminalEvent::Stopped { .. })
+}
+
+pub(super) fn should_preserve_rebalance_stopped_state(
+    meta_stopped: bool,
+    status: RebalStatus,
+    terminal_event: &RebalanceTerminalEvent,
+) -> bool {
+    (meta_stopped || status == RebalStatus::Stopped) && !is_rebalance_stopped_terminal_event(terminal_event)
+}
+
+pub(super) fn resolve_rebalance_participants(pool_stats: &[RebalanceStats], pool_count: usize) -> Vec<bool> {
+    let mut participants = vec![false; pool_count];
+
+    for (idx, pool_stat) in pool_stats.iter().enumerate() {
+        if idx >= participants.len() {
+            break;
+        }
+
+        if pool_stat.info.status == RebalStatus::Started {
+            participants[idx] = pool_stat.participating;
+        }
+    }
+
+    participants
+}
+
+pub(super) fn is_rebalance_actively_running(meta: &RebalanceMeta) -> bool {
+    meta.cancel.is_some() && is_rebalance_in_progress(meta)
+}
+
+pub(super) fn should_ignore_rebalance_data_usage_cache(bucket: &str) -> bool {
+    bucket == crate::disk::RUSTFS_META_BUCKET
+}
+
+pub(super) fn apply_rebalance_save_option(meta: &mut RebalanceMeta, pool_idx: usize, opt: RebalSaveOpt, now: OffsetDateTime) {
+    match opt {
+        RebalSaveOpt::Stats => {
+            if pool_idx >= meta.pool_stats.len() {
+                info!("save_rebalance_stats: pool_idx {pool_idx} out of range for pool_stats");
+            }
+        }
+        RebalSaveOpt::StoppedAt => {
+            apply_stopped_at(meta, now);
+        }
+    }
+
+    meta.last_refreshed_at = Some(now);
+}
+
+pub(super) fn is_rebalance_terminal_status(status: RebalStatus) -> bool {
+    matches!(status, RebalStatus::Completed | RebalStatus::Stopped | RebalStatus::Failed)
+}
+
+pub(super) fn merge_rebalance_bucket_lists(remote: &mut Vec<String>, local: &[String]) {
+    let mut existing: HashSet<String> = remote.iter().cloned().collect();
+    for bucket in local {
+        if existing.insert(bucket.clone()) {
+            remote.push(bucket.clone());
+        }
+    }
+}
+
+pub(super) fn remove_rebalanced_buckets_from_queue(pool_stat: &mut RebalanceStats) {
+    let rebalanced_buckets: HashSet<String> = pool_stat.rebalanced_buckets.iter().cloned().collect();
+    pool_stat.buckets.retain(|bucket| !rebalanced_buckets.contains(bucket));
+}
+
+pub(super) fn merge_rebalance_cleanup_warnings(remote: &mut RebalanceCleanupWarnings, local: &RebalanceCleanupWarnings) {
+    remote.count = remote.count.max(local.count);
+
+    if should_replace_rebalance_cleanup_warning(remote.last_at, local.last_at) {
+        remote.last_message = local.last_message.clone();
+        remote.last_bucket = local.last_bucket.clone();
+        remote.last_object = local.last_object.clone();
+        remote.last_at = local.last_at;
+    }
+
+    merge_rebalance_cleanup_warning_entries(&mut remote.entries, &local.entries);
+    let retained_entries = u64::try_from(remote.entries.len()).unwrap_or(u64::MAX);
+    remote.count = remote.count.max(retained_entries);
+}
+
+pub(super) fn merge_rebalance_cleanup_warning_entries(
+    remote: &mut Vec<RebalanceCleanupWarningEntry>,
+    local: &[RebalanceCleanupWarningEntry],
+) {
+    for entry in local {
+        if !remote.iter().any(|existing| existing == entry) {
+            remote.push(entry.clone());
+        }
+    }
+
+    remote.sort_by_key(|entry| entry.timestamp);
+    truncate_rebalance_cleanup_warning_entries(remote);
+}
+
+fn truncate_rebalance_cleanup_warning_entries(entries: &mut Vec<RebalanceCleanupWarningEntry>) {
+    if entries.len() > REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT {
+        let remove_count = entries.len() - REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT;
+        entries.drain(0..remove_count);
+    }
+}
+
+pub(super) fn should_replace_rebalance_cleanup_warning(
+    remote_at: Option<OffsetDateTime>,
+    local_at: Option<OffsetDateTime>,
+) -> bool {
+    match (remote_at, local_at) {
+        (_, None) => false,
+        (None, Some(_)) => true,
+        (Some(remote), Some(local)) => local >= remote,
+    }
+}
+
+pub(super) fn merge_rebalance_stop_propagation_error(remote: Option<String>, local: Option<String>) -> Option<String> {
+    if is_rebalance_stop_propagation_error(local.as_deref()) {
+        local
+    } else if is_rebalance_stop_propagation_error(remote.as_deref()) {
+        remote
+    } else {
+        None
+    }
+}
+
+pub(super) fn merge_rebalance_pool_stats(remote: &mut RebalanceStats, local: &RebalanceStats) {
+    remote.init_free_space = remote.init_free_space.max(local.init_free_space);
+    remote.init_capacity = remote.init_capacity.max(local.init_capacity);
+    remote.participating |= local.participating;
+
+    merge_rebalance_bucket_lists(&mut remote.buckets, &local.buckets);
+    merge_rebalance_bucket_lists(&mut remote.rebalanced_buckets, &local.rebalanced_buckets);
+    remove_rebalanced_buckets_from_queue(remote);
+
+    let local_is_newer = local.num_versions >= remote.num_versions;
+    remote.num_objects = remote.num_objects.max(local.num_objects);
+    remote.num_versions = remote.num_versions.max(local.num_versions);
+    remote.bytes = remote.bytes.max(local.bytes);
+    merge_rebalance_cleanup_warnings(&mut remote.cleanup_warnings, &local.cleanup_warnings);
+
+    if local_is_newer {
+        remote.bucket = local.bucket.clone();
+        remote.object = local.object.clone();
+    }
+
+    if remote.info.start_time.is_none() {
+        remote.info.start_time = local.info.start_time;
+    }
+
+    if is_rebalance_terminal_status(remote.info.status) && local.info.status == RebalStatus::Started {
+        return;
+    }
+
+    if remote.info.status == RebalStatus::Stopped && matches!(local.info.status, RebalStatus::Started | RebalStatus::Completed) {
+        return;
+    }
+
+    match local.info.status {
+        RebalStatus::Failed => {
+            remote.info.status = RebalStatus::Failed;
+            remote.info.stopping = false;
+            remote.info.end_time = local.info.end_time.or(remote.info.end_time);
+            remote.info.last_error = local.info.last_error.clone().or_else(|| remote.info.last_error.clone());
+        }
+        RebalStatus::Stopped => {
+            if remote.info.status != RebalStatus::Failed {
+                remote.info.status = RebalStatus::Stopped;
+                remote.info.stopping = false;
+                remote.info.end_time = local.info.end_time.or(remote.info.end_time);
+                remote.info.last_error =
+                    merge_rebalance_stop_propagation_error(remote.info.last_error.clone(), local.info.last_error.clone());
+            }
+        }
+        RebalStatus::Completed => {
+            if !matches!(remote.info.status, RebalStatus::Failed | RebalStatus::Stopped) {
+                remote.info.status = RebalStatus::Completed;
+                remote.info.stopping = false;
+                remote.info.end_time = local.info.end_time.or(remote.info.end_time);
+                remote.info.last_error = None;
+            }
+        }
+        RebalStatus::Started => {
+            if !is_rebalance_terminal_status(remote.info.status) {
+                remote.info.status = RebalStatus::Started;
+                remote.info.stopping |= local.info.stopping;
+                remote.info.last_error = local.info.last_error.clone();
+            }
+        }
+        RebalStatus::None => {}
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum RebalanceMetaMergeOutcome {
+    Merged,
+    Replaced,
+    RejectedActiveConflict,
+}
+
+pub(super) fn is_rebalance_meta_replaceable_for_new_id(meta: &RebalanceMeta) -> bool {
+    meta.stopped_at.is_some() || !is_rebalance_in_progress(meta)
+}
+
+pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &RebalanceMeta) -> RebalanceMetaMergeOutcome {
+    if remote.id.is_empty() {
+        *remote = local.clone();
+        return RebalanceMetaMergeOutcome::Replaced;
+    }
+
+    if !local.id.is_empty() && remote.id != local.id {
+        if is_rebalance_meta_replaceable_for_new_id(remote) {
+            *remote = local.clone();
+            return RebalanceMetaMergeOutcome::Replaced;
+        }
+        return RebalanceMetaMergeOutcome::RejectedActiveConflict;
+    }
+
+    remote.percent_free_goal = local.percent_free_goal;
+    remote.last_refreshed_at = Some(OffsetDateTime::now_utc());
+    if remote.stopped_at.is_none() {
+        remote.stopped_at = local.stopped_at;
+    }
+
+    if remote.pool_stats.len() < local.pool_stats.len() {
+        remote.pool_stats.resize_with(local.pool_stats.len(), RebalanceStats::default);
+    }
+
+    for (idx, local_pool_stat) in local.pool_stats.iter().enumerate() {
+        if let Some(remote_pool_stat) = remote.pool_stats.get_mut(idx) {
+            merge_rebalance_pool_stats(remote_pool_stat, local_pool_stat);
+        }
+    }
+
+    RebalanceMetaMergeOutcome::Merged
+}
+
+pub(super) fn mark_started_rebalance_pools_stopped(meta: &mut RebalanceMeta, stop_time: OffsetDateTime) {
+    for pool_stat in meta.pool_stats.iter_mut() {
+        if pool_stat.info.status == RebalStatus::Started {
+            pool_stat.info.status = RebalStatus::Stopped;
+            pool_stat.info.stopping = false;
+            pool_stat.info.end_time.get_or_insert(stop_time);
+            if !is_rebalance_stop_propagation_error(pool_stat.info.last_error.as_deref()) {
+                pool_stat.info.last_error = None;
+            }
+        }
+    }
+}
+
+pub(super) fn mark_started_rebalance_pools_stopping(meta: &mut RebalanceMeta) {
+    for pool_stat in meta.pool_stats.iter_mut() {
+        if pool_stat.info.status == RebalStatus::Started {
+            pool_stat.info.stopping = true;
+            pool_stat.info.last_error = None;
+        }
+    }
+}
+
+pub(super) fn apply_stopped_at(meta: &mut RebalanceMeta, now: OffsetDateTime) {
+    meta.stopped_at.get_or_insert(now);
+    mark_started_rebalance_pools_stopping(meta);
+}
+
+pub(super) fn clear_rebalance_cancel_token(meta: Option<&mut RebalanceMeta>) -> bool {
+    let Some(meta) = meta else {
+        return false;
+    };
+    if let Some(cancel_tx) = meta.cancel.take() {
+        cancel_tx.cancel();
+        return true;
+    }
+    false
+}
+
+pub(super) fn stop_rebalance_state(meta: &mut RebalanceMeta, now: OffsetDateTime) {
+    clear_rebalance_cancel_token(Some(meta));
+    if meta.stopped_at.is_none() && is_rebalance_in_progress(meta) {
+        apply_stopped_at(meta, now);
+    } else if meta.stopped_at.is_some() {
+        mark_started_rebalance_pools_stopping(meta);
+    }
+}
+
+pub(super) fn stop_rebalance_meta_snapshot_for_id(
+    meta: Option<&mut RebalanceMeta>,
+    now: OffsetDateTime,
+    expected_id: Option<&str>,
+) -> Result<Option<RebalanceMeta>> {
+    let Some(meta) = meta else {
+        return Ok(None);
+    };
+
+    if let Some(expected_id) = expected_id
+        && !expected_id.is_empty()
+        && meta.id != expected_id
+    {
+        return Err(Error::other(format!(
+            "rebalance stop id mismatch: expected {expected_id}, found {}",
+            meta.id
+        )));
+    }
+
+    stop_rebalance_state(meta, now);
+    meta.last_refreshed_at = Some(now);
+    Ok(Some(meta.clone()))
+}
+
+pub(super) fn rollback_rebalance_start_meta_snapshot_for_id(
+    meta: Option<&mut RebalanceMeta>,
+    now: OffsetDateTime,
+    expected_id: Option<&str>,
+    start_error: String,
+) -> Option<RebalanceMeta> {
+    meta.and_then(|meta| {
+        if let Some(expected_id) = expected_id
+            && !expected_id.is_empty()
+            && meta.id != expected_id
+        {
+            return None;
+        }
+
+        clear_rebalance_cancel_token(Some(meta));
+        meta.stopped_at.get_or_insert(now);
+        meta.last_refreshed_at = Some(now);
+        for pool_stat in meta.pool_stats.iter_mut() {
+            if pool_stat.info.status == RebalStatus::Started {
+                pool_stat.info.status = RebalStatus::Failed;
+                pool_stat.info.stopping = false;
+                pool_stat.info.end_time.get_or_insert(now);
+                pool_stat.info.last_error = Some(start_error.clone());
+            }
+        }
+        Some(meta.clone())
+    })
+}
+
+pub(super) fn stop_rebalance_meta_snapshot(meta: Option<&mut RebalanceMeta>, now: OffsetDateTime) -> Option<RebalanceMeta> {
+    let meta = meta?;
+    stop_rebalance_state(meta, now);
+    meta.last_refreshed_at = Some(now);
+    Some(meta.clone())
+}
+
+pub(super) fn record_rebalance_stop_propagation_snapshot(
+    meta: Option<&mut RebalanceMeta>,
+    encoded_error: String,
+    now: OffsetDateTime,
+) -> Option<RebalanceMeta> {
+    meta.map(|meta| {
+        for pool_stat in meta.pool_stats.iter_mut() {
+            if pool_stat.participating || pool_stat.info.stopping || pool_stat.info.status == RebalStatus::Stopped {
+                pool_stat.info.last_error = Some(encoded_error.clone());
+            }
+        }
+        meta.last_refreshed_at = Some(now);
+        meta.clone()
+    })
+}

crates/ecstore/src/services/rebalance/migration.rs

@@ -0,0 +1,339 @@
+use super::worker::{is_transient_rebalance_error, rebalance_migration_retry_delay, sleep_rebalance_migration_retry};
+use crate::data_usage::DATA_USAGE_CACHE_NAME;
+use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_found};
+use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
+use crate::set_disk::SetDisks;
+use crate::storage_api_contracts::{
+    object::{ObjectIO, ObjectOperations as _},
+    range::HTTPRangeSpec,
+};
+use http::HeaderMap;
+use rustfs_filemeta::FileInfo;
+use rustfs_utils::path::encode_dir_object;
+use std::future::Future;
+use tokio::time::Duration;
+
+#[derive(Debug, Default, Clone)]
+pub(crate) struct MigrationVersionResult {
+    pub moved: bool,
+    pub ignored: bool,
+    pub cleanup_ignored: bool,
+    pub failed: bool,
+    pub stage: Option<&'static str>,
+    pub error: Option<Error>,
+}
+
+pub(super) fn rebalance_delete_marker_opts(version: &FileInfo, version_id: Option<String>, src_pool_idx: usize) -> ObjectOptions {
+    ObjectOptions {
+        versioned: true,
+        version_id,
+        mod_time: version.mod_time,
+        src_pool_idx,
+        data_movement: true,
+        delete_marker: true,
+        skip_decommissioned: true,
+        delete_replication: version.replication_state_internal.clone(),
+        ..Default::default()
+    }
+}
+
+fn rebalance_remote_tiered_opts(version: &FileInfo, version_id: Option<String>, src_pool_idx: usize) -> ObjectOptions {
+    ObjectOptions {
+        versioned: version_id.is_some(),
+        version_id,
+        mod_time: version.mod_time,
+        user_defined: version.metadata.clone(),
+        src_pool_idx,
+        data_movement: true,
+        ..Default::default()
+    }
+}
+
+#[async_trait::async_trait]
+pub(crate) trait MigrationBackend: Send + Sync {
+    async fn get_object_reader_for_migration(
+        &self,
+        bucket: &str,
+        object: &str,
+        range: Option<HTTPRangeSpec>,
+        h: HeaderMap,
+        opts: &ObjectOptions,
+    ) -> Result<GetObjectReader>;
+
+    async fn delete_object_for_migration(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo>;
+
+    async fn move_remote_version_for_migration(
+        &self,
+        bucket: &str,
+        object: &str,
+        fi: &FileInfo,
+        opts: &ObjectOptions,
+    ) -> Result<()>;
+}
+
+#[async_trait::async_trait]
+impl MigrationBackend for SetDisks {
+    async fn get_object_reader_for_migration(
+        &self,
+        bucket: &str,
+        object: &str,
+        range: Option<HTTPRangeSpec>,
+        h: HeaderMap,
+        opts: &ObjectOptions,
+    ) -> Result<GetObjectReader> {
+        self.get_object_reader(bucket, object, range, h, opts).await
+    }
+
+    async fn delete_object_for_migration(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
+        self.delete_object(bucket, object, opts).await
+    }
+
+    async fn move_remote_version_for_migration(
+        &self,
+        bucket: &str,
+        object: &str,
+        fi: &FileInfo,
+        opts: &ObjectOptions,
+    ) -> Result<()> {
+        self.decommission_tiered_object(bucket, object, fi, opts).await
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+pub(crate) async fn migrate_entry_version<Backend, F, Fut>(
+    set: &Backend,
+    bucket: String,
+    pool_index: usize,
+    version: &FileInfo,
+    version_id: Option<String>,
+    max_attempts: usize,
+    ignore_data_usage_cache: bool,
+    transfer: F,
+) -> MigrationVersionResult
+where
+    Backend: MigrationBackend + ?Sized,
+    F: FnMut(usize, String, GetObjectReader) -> Fut + Send,
+    Fut: Future<Output = Result<()>> + Send,
+{
+    migrate_entry_version_with_retry_wait(
+        set,
+        bucket,
+        pool_index,
+        version,
+        version_id,
+        max_attempts,
+        ignore_data_usage_cache,
+        transfer,
+        sleep_rebalance_migration_retry,
+    )
+    .await
+}
+
+#[allow(clippy::too_many_arguments)]
+pub(super) async fn migrate_entry_version_with_retry_wait<Backend, F, Fut, W, WFut>(
+    set: &Backend,
+    bucket: String,
+    pool_index: usize,
+    version: &FileInfo,
+    version_id: Option<String>,
+    max_attempts: usize,
+    ignore_data_usage_cache: bool,
+    mut transfer: F,
+    mut wait_retry: W,
+) -> MigrationVersionResult
+where
+    Backend: MigrationBackend + ?Sized,
+    F: FnMut(usize, String, GetObjectReader) -> Fut + Send,
+    Fut: Future<Output = Result<()>> + Send,
+    W: FnMut(Duration) -> WFut + Send,
+    WFut: Future<Output = ()> + Send,
+{
+    let max_attempts = max_attempts.max(1);
+
+    if ignore_data_usage_cache && bucket == crate::disk::RUSTFS_META_BUCKET && version.name.contains(DATA_USAGE_CACHE_NAME) {
+        return MigrationVersionResult {
+            moved: false,
+            ignored: true,
+            cleanup_ignored: false,
+            failed: false,
+            stage: None,
+            error: None,
+        };
+    }
+
+    if version.is_remote() {
+        if let Err(err) = set
+            .move_remote_version_for_migration(
+                &bucket,
+                &version.name,
+                version,
+                &rebalance_remote_tiered_opts(version, version_id, pool_index),
+            )
+            .await
+        {
+            if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
+                return MigrationVersionResult {
+                    moved: false,
+                    ignored: true,
+                    cleanup_ignored: true,
+                    failed: false,
+                    stage: Some("move_remote_version"),
+                    error: None,
+                };
+            }
+
+            return MigrationVersionResult {
+                moved: false,
+                ignored: false,
+                cleanup_ignored: false,
+                failed: true,
+                stage: Some("move_remote_version"),
+                error: Some(err),
+            };
+        }
+
+        return MigrationVersionResult {
+            moved: true,
+            ignored: false,
+            cleanup_ignored: false,
+            failed: false,
+            stage: None,
+            error: None,
+        };
+    }
+
+    if version.deleted {
+        if let Err(err) = set
+            .delete_object_for_migration(&bucket, &version.name, rebalance_delete_marker_opts(version, version_id, pool_index))
+            .await
+        {
+            if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
+                return MigrationVersionResult {
+                    moved: false,
+                    ignored: true,
+                    cleanup_ignored: true,
+                    failed: false,
+                    stage: Some("delete_marker"),
+                    error: None,
+                };
+            }
+
+            return MigrationVersionResult {
+                moved: false,
+                ignored: false,
+                cleanup_ignored: false,
+                failed: true,
+                stage: Some("delete_marker"),
+                error: Some(err),
+            };
+        }
+
+        return MigrationVersionResult {
+            moved: true,
+            ignored: false,
+            cleanup_ignored: false,
+            failed: false,
+            stage: None,
+            error: None,
+        };
+    }
+
+    let mut last_error: Option<Error> = None;
+    for attempt in 0..max_attempts {
+        let rd = match set
+            .get_object_reader_for_migration(
+                &bucket,
+                &encode_dir_object(&version.name),
+                None,
+                HeaderMap::new(),
+                &ObjectOptions {
+                    version_id: version_id.clone(),
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+        {
+            Ok(rd) => rd,
+            Err(err) => {
+                if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
+                    return MigrationVersionResult {
+                        moved: false,
+                        ignored: true,
+                        cleanup_ignored: true,
+                        failed: false,
+                        stage: Some("read_source"),
+                        error: None,
+                    };
+                }
+
+                last_error = Some(err);
+                let Some(err) = last_error.as_ref() else {
+                    continue;
+                };
+                if attempt + 1 >= max_attempts || !is_transient_rebalance_error(err) {
+                    return MigrationVersionResult {
+                        moved: false,
+                        ignored: false,
+                        cleanup_ignored: false,
+                        failed: true,
+                        stage: Some("read_source"),
+                        error: last_error,
+                    };
+                }
+
+                wait_retry(rebalance_migration_retry_delay(attempt, err)).await;
+                continue;
+            }
+        };
+
+        if let Err(err) = transfer(pool_index, bucket.clone(), rd).await {
+            if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
+                return MigrationVersionResult {
+                    moved: false,
+                    ignored: true,
+                    cleanup_ignored: true,
+                    failed: false,
+                    stage: Some("write_target"),
+                    error: None,
+                };
+            }
+
+            last_error = Some(err);
+            let Some(err) = last_error.as_ref() else {
+                continue;
+            };
+            if attempt + 1 >= max_attempts || !is_transient_rebalance_error(err) {
+                return MigrationVersionResult {
+                    moved: false,
+                    ignored: false,
+                    cleanup_ignored: false,
+                    failed: true,
+                    stage: Some("write_target"),
+                    error: last_error,
+                };
+            }
+
+            wait_retry(rebalance_migration_retry_delay(attempt, err)).await;
+            continue;
+        }
+
+        return MigrationVersionResult {
+            moved: true,
+            ignored: false,
+            cleanup_ignored: false,
+            failed: false,
+            stage: None,
+            error: None,
+        };
+    }
+
+    MigrationVersionResult {
+        moved: false,
+        ignored: false,
+        cleanup_ignored: false,
+        failed: true,
+        stage: Some("migrate"),
+        error: last_error,
+    }
+}

crates/ecstore/src/services/rebalance/mod.rs

@@ -0,0 +1,55 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
+use tokio::time::Duration;
+
+const LOG_COMPONENT_ECSTORE: &str = "ecstore";
+const LOG_SUBSYSTEM_REBALANCE: &str = "rebalance";
+const EVENT_REBALANCE_STATE: &str = "rebalance_state";
+const EVENT_REBALANCE_BUCKET: &str = "rebalance_bucket";
+const EVENT_REBALANCE_ENTRY: &str = "rebalance_entry";
+const EVENT_REBALANCE_LISTING: &str = "rebalance_listing";
+
+const REBAL_META_FMT: u16 = 1; // Replace with actual format value
+const REBAL_META_VER: u16 = 1; // Replace with actual version value
+pub(crate) const REBAL_META_NAME: &str = "rebalance.bin";
+const DEFAULT_REBALANCE_MAX_ATTEMPTS: usize = 3;
+const REBALANCE_MAX_ATTEMPTS_ENV: &str = "RUSTFS_REBALANCE_MAX_ATTEMPTS";
+const REBALANCE_STOP_PROPAGATION_ERROR_PREFIX: &str = "rebalance stop propagation incomplete: ";
+const REBALANCE_LISTING_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
+const REBALANCE_MIGRATION_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
+const REBALANCE_MIGRATION_LOCK_RETRY_CAP: Duration = Duration::from_secs(10);
+const REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX: &str = "deferred transient rebalance entry failure:";
+const REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT: usize = 10;
+
+mod control;
+mod entry;
+mod meta;
+mod migration;
+mod runtime;
+mod types;
+mod worker;
+
+pub(crate) use meta::is_rebalance_conflicting_with_decommission;
+pub use meta::{decode_rebalance_stop_propagation_record, encode_rebalance_stop_propagation_record};
+pub use types::{
+    DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta,
+    RebalanceStats, RebalanceStopPropagationRecord,
+};
+use types::{RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome};
+
+#[cfg(test)]
+mod rebalance_unit_tests;

crates/ecstore/src/services/rebalance/runtime.rs

@@ -0,0 +1,586 @@
+use super::meta::{
+    apply_rebalance_save_option, apply_rebalance_terminal_event, classify_rebalance_terminal_event, clone_first_arc,
+    complete_rebalance_pools_at_goal, complete_rebalance_pools_with_empty_queue, ensure_valid_rebalance_pool_index,
+    has_deferred_rebalance_error, has_rebalance_cleanup_warnings, is_rebalance_in_progress, rebalance_goal_reached,
+    resolve_rebalance_participants, should_preserve_rebalance_stopped_state, should_skip_start_rebalance,
+    validate_start_rebalance_state,
+};
+use super::worker::{
+    resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result, resolve_rebalance_save_task_result,
+    resolve_rebalance_terminal_error, send_rebalance_done_signal,
+};
+use super::{
+    EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, RebalSaveOpt, RebalStatus,
+    RebalanceBucketOutcome,
+};
+use crate::error::{Error, Result};
+use crate::runtime::sources as runtime_sources;
+use crate::store::ECStore;
+use std::collections::HashSet;
+use std::sync::Arc;
+use time::OffsetDateTime;
+use tokio::time::{Duration, Instant};
+use tokio_util::sync::CancellationToken;
+use tracing::{debug, error, info, warn};
+
+impl ECStore {
+    #[tracing::instrument(skip_all)]
+    pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
+        info!(
+            event = EVENT_REBALANCE_STATE,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            state = "starting",
+            "Starting rebalance"
+        );
+        let decommission_running = self.is_decommission_running().await;
+        // let rebalance_meta = self.rebalance_meta.read().await;
+
+        let cancel_tx = CancellationToken::new();
+        let rx = cancel_tx.clone();
+        let mut meta_to_save = None;
+
+        {
+            let mut rebalance_meta = self.rebalance_meta.write().await;
+            validate_start_rebalance_state(decommission_running, rebalance_meta.is_some())?;
+
+            let Some(meta) = rebalance_meta.as_mut() else {
+                return Err(Error::ConfigNotFound);
+            };
+            if should_skip_start_rebalance(meta.cancel.is_some(), is_rebalance_in_progress(meta)) {
+                debug!(
+                    event = EVENT_REBALANCE_STATE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    state = "start_skipped",
+                    reason = "already_in_progress",
+                    "Skipped duplicate rebalance start"
+                );
+                return Ok(());
+            }
+            let now = OffsetDateTime::now_utc();
+            if complete_rebalance_pools_at_goal(meta, now) {
+                meta_to_save = Some(meta.clone());
+            }
+            if complete_rebalance_pools_with_empty_queue(meta, now) {
+                meta_to_save = Some(meta.clone());
+            }
+            meta.cancel = Some(cancel_tx);
+
+            drop(rebalance_meta);
+        }
+
+        if let Some(meta) = meta_to_save {
+            let pool = clone_first_arc(self.pools.as_slice(), "start_rebalance: no pools available")?;
+            resolve_rebalance_meta_save_result(
+                self.save_rebalance_meta_with_merge(pool, &meta, "start_rebalance complete pools at goal")
+                    .await,
+                "start_rebalance complete pools at goal",
+            )?;
+        }
+
+        let participants = if let Some(ref meta) = *self.rebalance_meta.read().await {
+            resolve_rebalance_participants(meta.pool_stats.as_slice(), self.pools.len())
+        } else {
+            debug!(
+                event = EVENT_REBALANCE_STATE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                state = "start_skipped",
+                reason = "metadata_missing",
+                "Skipped rebalance start because metadata is unavailable"
+            );
+            Vec::new()
+        };
+
+        if !participants.iter().any(|participating| *participating) {
+            debug!(
+                event = EVENT_REBALANCE_STATE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                state = "start_skipped",
+                reason = "no_participants",
+                "Skipped rebalance start because no pools are participating"
+            );
+            return Ok(());
+        }
+
+        let mut workers_started = 0usize;
+        for (idx, participating) in participants.iter().enumerate() {
+            if !*participating {
+                debug!(
+                    event = EVENT_REBALANCE_STATE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index = idx,
+                    state = "pool_skipped",
+                    reason = "not_participating",
+                    "Skipped rebalance pool"
+                );
+                continue;
+            }
+
+            if !runtime_sources::endpoint_pool_is_local(idx) {
+                debug!(
+                    event = EVENT_REBALANCE_STATE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index = idx,
+                    state = "pool_skipped",
+                    reason = "not_local",
+                    "Skipped non-local rebalance pool"
+                );
+                continue;
+            }
+
+            let pool_idx = idx;
+            let store = self.clone();
+            let rx_clone = rx.clone();
+            workers_started += 1;
+            tokio::spawn(async move {
+                if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx).await {
+                    error!(
+                        event = EVENT_REBALANCE_STATE,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        pool_index = pool_idx,
+                        state = "pool_failed",
+                        error = %err,
+                        "Rebalance pool failed"
+                    );
+                } else {
+                    debug!(
+                        event = EVENT_REBALANCE_STATE,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        pool_index = pool_idx,
+                        state = "completed",
+                        "Rebalance pool completed"
+                    );
+                }
+            });
+        }
+
+        if workers_started == 0 {
+            debug!(
+                event = EVENT_REBALANCE_STATE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                state = "start_skipped",
+                reason = "no_local_participants",
+                "Skipped rebalance start because no local pools are participating"
+            );
+            return Ok(());
+        }
+
+        info!(
+            event = EVENT_REBALANCE_STATE,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            state = "started",
+            worker_count = workers_started,
+            "Rebalance started"
+        );
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, rx))]
+    async fn rebalance_buckets(self: &Arc<Self>, rx: CancellationToken, pool_index: usize) -> Result<()> {
+        ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?;
+
+        let (done_tx, mut done_rx) = tokio::sync::mpsc::channel::<Result<()>>(1);
+
+        // Save rebalance metadata periodically
+        let store = self.clone();
+        let save_task = tokio::spawn(async move {
+            let mut timer = tokio::time::interval_at(Instant::now() + Duration::from_secs(30), Duration::from_secs(10));
+            let mut msg: String;
+            let mut quit = false;
+
+            loop {
+                tokio::select! {
+                    result = done_rx.recv() => {
+                        quit = true;
+                        let now = OffsetDateTime::now_utc();
+                        let terminal_event = classify_rebalance_terminal_event(result, now);
+                        msg = terminal_event.message().to_string();
+                        let mut rebalance_meta = store.rebalance_meta.write().await;
+                        if let Some(meta) = rebalance_meta.as_mut() {
+                            let meta_stopped = meta.stopped_at.is_some();
+                            if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
+                                if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
+                                    && has_rebalance_cleanup_warnings(pool_stat)
+                                {
+                                    pool_stat.info.stopping = false;
+                                    pool_stat.info.status = RebalStatus::Failed;
+                                    pool_stat.info.end_time = Some(now);
+                                    pool_stat.info.last_error = Some(
+                                        pool_stat
+                                            .cleanup_warnings
+                                            .last_message
+                                            .clone()
+                                            .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
+                                    );
+                                } else if should_preserve_rebalance_stopped_state(
+                                    meta_stopped,
+                                    pool_stat.info.status,
+                                    &terminal_event,
+                                ) {
+                                    debug!(
+                                        event = EVENT_REBALANCE_STATE,
+                                        component = LOG_COMPONENT_ECSTORE,
+                                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                                        pool_index,
+                                        state = "stopped_preserved",
+                                        "Preserved stopped rebalance status"
+                                    );
+                                } else {
+                                    pool_stat.info.stopping = false;
+                                    apply_rebalance_terminal_event(
+                                        &mut pool_stat.info.status,
+                                        &mut pool_stat.info.end_time,
+                                        &mut pool_stat.info.last_error,
+                                        terminal_event,
+                                        now,
+                                    );
+                                }
+                            }
+                        }
+                    }
+                    _ = timer.tick() => {
+                        let now = OffsetDateTime::now_utc();
+                        msg = format!("Saving rebalance metadata at {now:?}");
+                    }
+                }
+
+                if let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await {
+                    let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
+                    error!("{} err: {:?}", msg, wrapped);
+                    if quit {
+                        return Err(wrapped);
+                    }
+                } else {
+                    debug!(
+                        event = EVENT_REBALANCE_STATE,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        pool_index,
+                        state = "metadata_saved",
+                        message = %msg,
+                        "Saved rebalance metadata"
+                    );
+                }
+
+                if quit {
+                    debug!(
+                        event = EVENT_REBALANCE_STATE,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        pool_index,
+                        state = "save_task_exiting",
+                        message = %msg,
+                        "Exiting rebalance save task"
+                    );
+                    return Ok(());
+                }
+
+                timer.reset();
+            }
+        });
+
+        info!(
+            event = EVENT_REBALANCE_STATE,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            pool_index,
+            state = "pool_started",
+            "Rebalance worker started"
+        );
+        let mut final_result: Result<()> = Ok(());
+        let mut deferred_buckets = HashSet::new();
+
+        loop {
+            if rx.is_cancelled() {
+                info!(
+                    event = EVENT_REBALANCE_STATE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index,
+                    state = "pool_stopped",
+                    reason = "cancelled",
+                    "Stopped rebalance worker"
+                );
+                let err = Error::OperationCanceled;
+                final_result = Err(resolve_rebalance_terminal_error(
+                    err.clone(),
+                    send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
+                ));
+                break;
+            }
+
+            let next_bucket = match self.next_rebal_bucket(pool_index).await {
+                Ok(bucket) => bucket,
+                Err(err) => {
+                    error!(
+                        event = EVENT_REBALANCE_BUCKET,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        pool_index,
+                        state = "next_bucket_failed",
+                        error = ?err,
+                        "Rebalance next bucket lookup failed"
+                    );
+                    final_result = Err(resolve_rebalance_terminal_error(
+                        err.clone(),
+                        send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
+                    ));
+                    break;
+                }
+            };
+
+            if let Some(bucket) = next_bucket {
+                debug!(
+                    event = EVENT_REBALANCE_BUCKET,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index,
+                    bucket = %bucket,
+                    state = "started",
+                    "Starting rebalance bucket"
+                );
+
+                let outcome = match resolve_rebalance_bucket_result(
+                    self.rebalance_bucket(rx.clone(), bucket.clone(), pool_index).await,
+                    pool_index,
+                    &bucket,
+                ) {
+                    Ok(outcome) => outcome,
+                    Err(err) => {
+                        error!(
+                            event = EVENT_REBALANCE_BUCKET,
+                            component = LOG_COMPONENT_ECSTORE,
+                            subsystem = LOG_SUBSYSTEM_REBALANCE,
+                            pool_index,
+                            bucket = %bucket,
+                            state = "bucket_failed",
+                            error = ?err,
+                            "Rebalance bucket failed"
+                        );
+                        final_result = Err(resolve_rebalance_terminal_error(
+                            err.clone(),
+                            send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
+                        ));
+                        break;
+                    }
+                };
+
+                if let RebalanceBucketOutcome::Deferred { last_error } = outcome {
+                    if !deferred_buckets.insert(bucket.clone()) {
+                        let err = Error::other(format!(
+                            "rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}"
+                        ));
+                        error!(
+                            event = EVENT_REBALANCE_BUCKET,
+                            component = LOG_COMPONENT_ECSTORE,
+                            subsystem = LOG_SUBSYSTEM_REBALANCE,
+                            pool_index,
+                            bucket = %bucket,
+                            state = "bucket_deferred_repeatedly",
+                            error = ?err,
+                            "Rebalance bucket failed after repeated deferral"
+                        );
+                        final_result = Err(resolve_rebalance_terminal_error(
+                            err.clone(),
+                            send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
+                        ));
+                        break;
+                    }
+
+                    warn!(
+                        event = EVENT_REBALANCE_BUCKET,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        pool_index,
+                        bucket = %bucket,
+                        state = "deferred",
+                        error = %last_error,
+                        "Deferred rebalance bucket after transient object failures"
+                    );
+                    if let Err(err) = self.defer_rebalance_bucket(pool_index, bucket.clone(), last_error).await {
+                        error!(
+                            event = EVENT_REBALANCE_BUCKET,
+                            component = LOG_COMPONENT_ECSTORE,
+                            subsystem = LOG_SUBSYSTEM_REBALANCE,
+                            pool_index,
+                            bucket = %bucket,
+                            state = "defer_failed",
+                            error = ?err,
+                            "Rebalance bucket defer failed"
+                        );
+                        final_result = Err(resolve_rebalance_terminal_error(
+                            err.clone(),
+                            send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
+                        ));
+                        break;
+                    }
+                    continue;
+                }
+
+                debug!(
+                    event = EVENT_REBALANCE_BUCKET,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index,
+                    bucket = %bucket,
+                    state = "completed",
+                    "Completed rebalance bucket"
+                );
+                if let Err(err) = self.bucket_rebalance_done(pool_index, bucket).await {
+                    error!(
+                        event = EVENT_REBALANCE_BUCKET,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        pool_index,
+                        state = "bucket_done_mark_failed",
+                        error = ?err,
+                        "Rebalance bucket completion mark failed"
+                    );
+                    final_result = Err(resolve_rebalance_terminal_error(
+                        err.clone(),
+                        send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
+                    ));
+                    break;
+                }
+            } else {
+                debug!(
+                    event = EVENT_REBALANCE_BUCKET,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index,
+                    state = "idle",
+                    reason = "no_bucket_to_rebalance",
+                    "No rebalance bucket available"
+                );
+                break;
+            }
+        }
+
+        info!(
+            event = EVENT_REBALANCE_STATE,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            pool_index,
+            state = "pool_done",
+            "Rebalance worker finished"
+        );
+
+        if final_result.is_ok()
+            && let Err(err) = send_rebalance_done_signal(&done_tx, Ok(()), pool_index).await
+        {
+            final_result = Err(err);
+        }
+        drop(done_tx);
+        if let Err(err) = resolve_rebalance_save_task_result(pool_index, save_task.await)
+            && final_result.is_ok()
+        {
+            final_result = Err(err);
+        }
+        debug!(
+            event = EVENT_REBALANCE_STATE,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            pool_index,
+            state = "pool_result_returned",
+            "Rebalance worker result returned"
+        );
+        final_result
+    }
+
+    pub(super) async fn check_if_rebalance_done(&self, pool_index: usize) -> bool {
+        let mut rebalance_meta = self.rebalance_meta.write().await;
+
+        if let Some(meta) = rebalance_meta.as_mut()
+            && let Some(pool_stat) = meta.pool_stats.get_mut(pool_index)
+        {
+            // Check if the pool's rebalance status is already completed
+            if pool_stat.info.status == RebalStatus::Completed {
+                debug!(
+                    event = EVENT_REBALANCE_STATE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index,
+                    state = "already_completed",
+                    "Rebalance pool is already completed"
+                );
+                return true;
+            }
+
+            // Mark pool rebalance as done only after it reaches the PercentFreeGoal.
+            let pfi = if pool_stat.init_capacity == 0 {
+                0.0
+            } else {
+                (pool_stat.init_free_space + pool_stat.bytes) as f64 / pool_stat.init_capacity as f64
+            };
+
+            if !has_deferred_rebalance_error(pool_stat)
+                && !has_rebalance_cleanup_warnings(pool_stat)
+                && rebalance_goal_reached(
+                    pool_stat.init_free_space,
+                    pool_stat.init_capacity,
+                    pool_stat.bytes,
+                    meta.percent_free_goal,
+                )
+            {
+                pool_stat.info.status = RebalStatus::Completed;
+                pool_stat.info.end_time = Some(OffsetDateTime::now_utc());
+                info!(
+                    event = EVENT_REBALANCE_STATE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    pool_index,
+                    state = "completed",
+                    percent_free = pfi,
+                    "Marked rebalance pool completed"
+                );
+                return true;
+            }
+        }
+
+        false
+    }
+}
+
+impl ECStore {
+    #[tracing::instrument(skip(self))]
+    pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
+        let meta_to_save = {
+            let mut rebalance_meta = self.rebalance_meta.write().await;
+            let Some(meta) = rebalance_meta.as_mut() else {
+                return Ok(());
+            };
+
+            let now = OffsetDateTime::now_utc();
+            apply_rebalance_save_option(meta, pool_idx, opt, now);
+            meta.clone()
+        };
+
+        let pool = clone_first_arc(&self.pools, "save_rebalance_stats: no pools available")?;
+
+        debug!(
+            event = EVENT_REBALANCE_STATE,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            pool_index = pool_idx,
+            save_opt = ?opt,
+            state = "metadata_save_requested",
+            "Rebalance metadata save requested"
+        );
+        let stage = format!("save_rebalance_stats for pool {pool_idx} opt {opt:?}");
+        resolve_rebalance_meta_save_result(
+            self.save_rebalance_meta_with_merge(pool, &meta_to_save, stage.as_str()).await,
+            stage.as_str(),
+        )?;
+
+        Ok(())
+    }
+}

crates/ecstore/src/services/rebalance/types.rs

@@ -0,0 +1,159 @@
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+use time::OffsetDateTime;
+use tokio_util::sync::CancellationToken;
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct RebalanceStats {
+    #[serde(rename = "ifs")]
+    pub init_free_space: u64, // Pool free space at the start of rebalance
+    #[serde(rename = "ic")]
+    pub init_capacity: u64, // Pool capacity at the start of rebalance
+    #[serde(rename = "bus")]
+    pub buckets: Vec<String>, // Buckets being rebalanced or to be rebalanced
+    #[serde(rename = "rbs")]
+    pub rebalanced_buckets: Vec<String>, // Buckets rebalanced
+    #[serde(rename = "bu")]
+    pub bucket: String, // Last rebalanced bucket
+    #[serde(rename = "ob")]
+    pub object: String, // Last rebalanced object
+    #[serde(rename = "no")]
+    pub num_objects: u64, // Number of objects rebalanced
+    #[serde(rename = "nv")]
+    pub num_versions: u64, // Number of versions rebalanced
+    #[serde(rename = "bs")]
+    pub bytes: u64, // Number of bytes rebalanced
+    #[serde(rename = "par")]
+    pub participating: bool, // Whether the pool is participating in rebalance
+    #[serde(rename = "inf")]
+    pub info: RebalanceInfo, // Rebalance operation info
+    #[serde(rename = "cw", default)]
+    pub cleanup_warnings: RebalanceCleanupWarnings,
+}
+
+pub type RStats = Vec<Arc<RebalanceStats>>;
+
+#[derive(Debug, Default)]
+pub(super) struct RebalanceBucketConfigs {
+    pub(super) lifecycle_config: Option<s3s::dto::BucketLifecycleConfiguration>,
+    pub(super) lock_retention: Option<s3s::dto::DefaultRetention>,
+    pub(super) replication_config: Option<(s3s::dto::ReplicationConfiguration, OffsetDateTime)>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(super) enum RebalanceBucketOutcome {
+    Completed,
+    Deferred { last_error: String },
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(super) enum RebalanceEntryOutcome {
+    Completed,
+    Deferred { last_error: String },
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+pub enum RebalStatus {
+    #[default]
+    None,
+    Started,
+    Completed,
+    Stopped,
+    Failed,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+pub enum RebalSaveOpt {
+    #[default]
+    Stats,
+    StoppedAt,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct RebalanceInfo {
+    #[serde(rename = "startTs")]
+    pub start_time: Option<OffsetDateTime>, // Time at which rebalance-start was issued
+    #[serde(rename = "stopTs")]
+    pub end_time: Option<OffsetDateTime>, // Time at which rebalance operation completed or rebalance-stop was called
+    #[serde(rename = "err")]
+    pub last_error: Option<String>, // Last rebalance error message
+    #[serde(rename = "status")]
+    pub status: RebalStatus, // Current state of rebalance operation
+    #[serde(rename = "stopping", default)]
+    pub stopping: bool, // True after stop is requested and before worker terminal acknowledgement
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct RebalanceCleanupWarningEntry {
+    #[serde(rename = "bucket", default)]
+    pub bucket: String,
+    #[serde(rename = "object", default)]
+    pub object: String,
+    #[serde(rename = "message", default)]
+    pub message: String,
+    #[serde(rename = "timestamp", default)]
+    pub timestamp: Option<OffsetDateTime>,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct RebalanceCleanupWarnings {
+    #[serde(rename = "count", default)]
+    pub count: u64,
+    #[serde(rename = "lastMsg", default)]
+    pub last_message: Option<String>,
+    #[serde(rename = "lastBucket", default)]
+    pub last_bucket: Option<String>,
+    #[serde(rename = "lastObject", default)]
+    pub last_object: Option<String>,
+    #[serde(rename = "lastAt", default)]
+    pub last_at: Option<OffsetDateTime>,
+    #[serde(rename = "entries", default)]
+    pub entries: Vec<RebalanceCleanupWarningEntry>,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct RebalanceStopPropagationRecord {
+    #[serde(rename = "stopAttemptAt", default)]
+    pub stop_attempt_at: Option<OffsetDateTime>,
+    #[serde(rename = "stopFailures", default)]
+    pub stop_failures: Vec<String>,
+    #[serde(rename = "terminalReloadAttemptAt", default)]
+    pub terminal_reload_attempt_at: Option<OffsetDateTime>,
+    #[serde(rename = "terminalReloadFailures", default)]
+    pub terminal_reload_failures: Vec<String>,
+}
+
+impl RebalanceStopPropagationRecord {
+    pub fn has_failures(&self) -> bool {
+        !self.stop_failures.is_empty() || !self.terminal_reload_failures.is_empty()
+    }
+}
+
+#[allow(dead_code)]
+#[derive(Debug, Clone, Default)]
+pub struct DiskStat {
+    pub total_space: u64,
+    pub available_space: u64,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize, Clone)]
+#[serde(deny_unknown_fields)]
+pub struct RebalanceMeta {
+    #[serde(skip)]
+    pub cancel: Option<CancellationToken>, // To be invoked on rebalance-stop
+    #[serde(skip)]
+    pub last_refreshed_at: Option<OffsetDateTime>,
+    #[serde(rename = "stopTs")]
+    pub stopped_at: Option<OffsetDateTime>, // Time when rebalance-stop was issued
+    #[serde(rename = "id")]
+    pub id: String, // ID of the ongoing rebalance operation
+    #[serde(rename = "pf")]
+    pub percent_free_goal: f64, // Computed from total free space and capacity at the start of rebalance
+    #[serde(rename = "rss")]
+    pub pool_stats: Vec<RebalanceStats>, // Per-pool rebalance stats keyed by pool index
+}

crates/ecstore/src/services/rebalance/worker.rs

@@ -0,0 +1,547 @@
+use super::migration::MigrationVersionResult;
+use super::{
+    DEFAULT_REBALANCE_MAX_ATTEMPTS, EVENT_REBALANCE_LISTING, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
+    REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_MAX_ATTEMPTS_ENV, REBALANCE_MIGRATION_LOCK_RETRY_CAP,
+    REBALANCE_MIGRATION_RETRY_BASE_DELAY, RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome, Result,
+};
+use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
+use crate::core::pools::ListCallback;
+use crate::disk::error::DiskError;
+use crate::error::{
+    Error, is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found, is_network_or_host_down,
+};
+use crate::runtime::sources as runtime_sources;
+use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
+use rand::RngExt as _;
+use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
+use std::sync::Arc;
+use tokio::time::Duration;
+use tokio_util::sync::CancellationToken;
+use tracing::{debug, error, info};
+
+pub(super) fn resolve_rebalance_worker_result<T>(
+    set_idx: usize,
+    worker_result: std::result::Result<Result<T>, tokio::task::JoinError>,
+) -> Result<T> {
+    match worker_result {
+        Ok(result) => result,
+        Err(err) => Err(Error::other(format!("rebalance worker {set_idx} task join error: {err}"))),
+    }
+}
+
+pub(super) type RebalanceEntryTask = tokio::task::JoinHandle<Result<RebalanceEntryOutcome>>;
+
+pub(super) async fn wait_rebalance_entry_tasks(
+    set_idx: usize,
+    tasks: Arc<tokio::sync::Mutex<Vec<RebalanceEntryTask>>>,
+) -> Result<Option<String>> {
+    let tasks = {
+        let mut tasks = tasks.lock().await;
+        std::mem::take(&mut *tasks)
+    };
+
+    let mut first_error = None;
+    let mut first_deferred = None;
+    for task in tasks {
+        match task.await {
+            Ok(Ok(RebalanceEntryOutcome::Completed)) => {}
+            Ok(Ok(RebalanceEntryOutcome::Deferred { last_error })) => {
+                if first_deferred.is_none() {
+                    first_deferred = Some(last_error);
+                }
+            }
+            Ok(Err(err)) => {
+                error!("rebalance entry task failed for set {}: {}", set_idx, err);
+                if first_error.is_none() {
+                    first_error = Some(err);
+                }
+            }
+            Err(err) => {
+                let err = Error::other(format!("rebalance entry task join error for set {set_idx}: {err}"));
+                error!("{}", err);
+                if first_error.is_none() {
+                    first_error = Some(err);
+                }
+            }
+        }
+    }
+
+    if let Some(err) = first_error {
+        Err(err)
+    } else {
+        Ok(first_deferred)
+    }
+}
+
+pub(super) fn resolve_rebalance_save_task_result(
+    pool_idx: usize,
+    save_task_result: std::result::Result<Result<()>, tokio::task::JoinError>,
+) -> Result<()> {
+    match save_task_result {
+        Ok(result) => result.map_err(|err| Error::other(format!("rebalance save_task failed for pool {pool_idx}: {err}"))),
+        Err(err) => Err(Error::other(format!("rebalance save_task for pool {pool_idx} join error: {err}"))),
+    }
+}
+
+pub(super) fn resolve_rebalance_meta_save_result(result: Result<()>, stage: &str) -> Result<()> {
+    result.map_err(|err| Error::other(format!("rebalance meta save failed during {stage}: {err}")))
+}
+
+pub(super) fn rebalance_meta_lock_error(err: rustfs_lock::LockError) -> Error {
+    match err {
+        rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
+            mode: "write",
+            bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
+            object: REBAL_META_NAME.to_string(),
+            required,
+            achieved,
+        },
+        other => Error::other(format!(
+            "failed to acquire rebalance metadata write lock on {}/{}: {other}",
+            crate::disk::RUSTFS_META_BUCKET,
+            REBAL_META_NAME
+        )),
+    }
+}
+
+pub(super) fn resolve_rebalance_meta_load_result(result: Result<()>) -> Result<bool> {
+    match result {
+        Ok(()) => Ok(true),
+        Err(Error::ConfigNotFound) => Ok(false),
+        Err(err) => {
+            error!("rebalanceMeta: load rebalance meta err {:?}", &err);
+            Err(Error::other(format!("rebalance metadata load failed during load_rebalance_meta: {err}")))
+        }
+    }
+}
+
+pub(super) fn resolve_rebalance_stats_update_result(
+    result: Result<()>,
+    pool_idx: usize,
+    bucket: &str,
+    object_name: &str,
+) -> Result<()> {
+    result.map_err(|err| {
+        Error::other(format!(
+            "rebalance stats update failed for pool {pool_idx} bucket {bucket} object {object_name}: {err}"
+        ))
+    })
+}
+
+pub(super) fn resolve_rebalance_file_info_versions_result<T, E>(
+    result: std::result::Result<T, E>,
+    bucket: &str,
+    object_name: &str,
+) -> Result<T>
+where
+    E: std::fmt::Display,
+{
+    result.map_err(|err| Error::other(format!("rebalance file_info_versions failed for {bucket}/{object_name}: {err}")))
+}
+
+pub(super) fn resolve_rebalance_entry_cleanup_delete_result(
+    result: Result<crate::object_api::ObjectInfo>,
+    bucket: &str,
+    object_name: &str,
+) -> Result<Option<String>> {
+    match result {
+        Ok(_) => Ok(None),
+        Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
+        Err(err) => Ok(Some(format!("rebalance cleanup delete failed for {bucket}/{object_name}: {err}"))),
+    }
+}
+
+pub(super) fn resolve_rebalance_migrate_result_error(
+    err: Option<Error>,
+    pool_idx: usize,
+    bucket: &str,
+    object_name: &str,
+    version_id: Option<&str>,
+) -> Error {
+    err.unwrap_or_else(|| {
+        Error::other(format!(
+            "rebalance migration reported failure without error for pool {pool_idx} entry {bucket}/{object_name} version {}",
+            version_id.unwrap_or("none")
+        ))
+    })
+}
+
+pub(super) fn should_defer_rebalance_entry_failure(err: &Error) -> bool {
+    is_transient_rebalance_error(err)
+}
+
+pub(super) fn resolve_load_rebalance_stats_update_result(result: Result<()>) -> Result<()> {
+    result.map_err(|err| Error::other(format!("rebalance metadata stats refresh failed after load: {err}")))
+}
+
+pub(super) async fn send_rebalance_done_signal(
+    done_tx: &tokio::sync::mpsc::Sender<Result<()>>,
+    signal: Result<()>,
+    pool_idx: usize,
+) -> Result<()> {
+    done_tx
+        .send(signal)
+        .await
+        .map_err(|err| Error::other(format!("rebalance done signal send failed for pool {pool_idx}: {err}")))
+}
+
+pub(super) fn resolve_rebalance_terminal_error(primary_err: Error, signal_result: Result<()>) -> Error {
+    match signal_result {
+        Ok(()) => primary_err,
+        Err(signal_err) => Error::other(format!("rebalance terminal signal failed after error {primary_err}: {signal_err}")),
+    }
+}
+
+pub(super) fn resolve_rebalance_bucket_error(entry_error: Option<Error>, worker_error: Option<Error>) -> Result<()> {
+    if let Some(err) = entry_error {
+        return Err(err);
+    }
+
+    if let Some(err) = worker_error {
+        return Err(err);
+    }
+
+    Ok(())
+}
+
+pub(super) fn resolve_rebalance_bucket_result(
+    result: Result<RebalanceBucketOutcome>,
+    pool_idx: usize,
+    bucket: &str,
+) -> Result<RebalanceBucketOutcome> {
+    match result {
+        Ok(outcome) => Ok(outcome),
+        Err(err) if is_err_operation_canceled(&err) => Err(err),
+        Err(err) => Err(Error::other(format!("rebalance bucket {bucket} failed for pool {pool_idx}: {err}"))),
+    }
+}
+
+pub(super) fn is_transient_rebalance_error(err: &Error) -> bool {
+    match err {
+        Error::SlowDown
+        | Error::ErasureReadQuorum
+        | Error::ErasureWriteQuorum
+        | Error::InsufficientReadQuorum(_, _)
+        | Error::InsufficientWriteQuorum(_, _) => true,
+        Error::Lock(lock_err) => is_rebalance_transient_lock_error(lock_err),
+        Error::Io(io_err) => is_rebalance_transient_io_error(io_err) || is_rebalance_transient_message(&io_err.to_string()),
+        _ => is_rebalance_transient_message(&err.to_string()) || is_network_or_host_down(&err.to_string(), true),
+    }
+}
+
+fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
+    match err {
+        rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::Network { .. } => true,
+        rustfs_lock::LockError::Internal { message } => is_rebalance_transient_message(message),
+        _ => false,
+    }
+}
+
+fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
+    if err.kind() == std::io::ErrorKind::TimedOut {
+        return true;
+    }
+
+    if let Some(disk_err) = err.get_ref().and_then(|err| err.downcast_ref::<DiskError>())
+        && *disk_err == DiskError::Timeout
+    {
+        return true;
+    }
+
+    let message = err.to_string();
+    message.eq_ignore_ascii_case("timeout") || is_rebalance_transient_message(&message)
+}
+
+fn is_rebalance_transient_message(message: &str) -> bool {
+    let message = message.to_ascii_lowercase();
+    message.contains("lock acquisition timed out")
+        || message.contains("remote lock rpc timed out")
+        || message.contains("keepalivetimedout")
+        || message.contains("i/o timeout")
+        || message.contains("operation timed out")
+}
+
+pub(super) fn should_retry_rebalance_listing(err: &Error, attempt: usize, max_attempts: usize) -> bool {
+    attempt + 1 < max_attempts && is_transient_rebalance_error(err)
+}
+
+pub(super) fn parse_rebalance_max_attempts(value: Option<&str>) -> usize {
+    value
+        .and_then(|value| value.trim().parse::<usize>().ok())
+        .filter(|attempts| *attempts > 0)
+        .unwrap_or(DEFAULT_REBALANCE_MAX_ATTEMPTS)
+}
+
+pub(super) fn rebalance_max_attempts() -> usize {
+    parse_rebalance_max_attempts(std::env::var(REBALANCE_MAX_ATTEMPTS_ENV).ok().as_deref())
+}
+
+pub(super) fn rebalance_listing_retry_delay(attempt: usize) -> Duration {
+    let multiplier = u32::try_from(attempt.saturating_add(1)).unwrap_or(u32::MAX);
+    REBALANCE_LISTING_RETRY_BASE_DELAY.saturating_mul(multiplier)
+}
+
+fn is_rebalance_lock_or_rpc_timeout(err: &Error) -> bool {
+    match err {
+        Error::Lock(rustfs_lock::LockError::Timeout { .. }) | Error::Lock(rustfs_lock::LockError::Network { .. }) => true,
+        Error::Io(io_err) => is_rebalance_lock_or_rpc_timeout_message(&io_err.to_string()),
+        _ => is_rebalance_lock_or_rpc_timeout_message(&err.to_string()),
+    }
+}
+
+fn is_rebalance_lock_or_rpc_timeout_message(message: &str) -> bool {
+    let message = message.to_ascii_lowercase();
+    message.contains("lock acquisition timed out")
+        || message.contains("remote lock rpc timed out")
+        || message.contains("keepalivetimedout")
+}
+
+pub(super) fn rebalance_migration_retry_delay(attempt: usize, err: &Error) -> Duration {
+    if is_rebalance_lock_or_rpc_timeout(err) {
+        return rebalance_lock_retry_delay(attempt);
+    }
+
+    let multiplier = u32::try_from(attempt.saturating_add(1)).unwrap_or(u32::MAX);
+    REBALANCE_MIGRATION_RETRY_BASE_DELAY.saturating_mul(multiplier)
+}
+
+fn rebalance_lock_retry_delay(attempt: usize) -> Duration {
+    let lock_timeout = get_lock_acquire_timeout();
+    let attempt_shift = u32::try_from(attempt.min(4)).unwrap_or(4);
+    let multiplier = 1_u32.checked_shl(attempt_shift).unwrap_or(u32::MAX);
+    let cap = lock_timeout
+        .saturating_mul(multiplier)
+        .min(REBALANCE_MIGRATION_LOCK_RETRY_CAP)
+        .max(REBALANCE_MIGRATION_RETRY_BASE_DELAY);
+    let max_millis = u64::try_from(cap.as_millis()).unwrap_or(u64::MAX).max(1);
+    let jitter_millis = rand::rng().random_range(1..=max_millis);
+    Duration::from_millis(jitter_millis)
+}
+
+pub(super) async fn sleep_rebalance_migration_retry(delay: Duration) {
+    tokio::time::sleep(delay).await;
+}
+
+pub(super) async fn wait_rebalance_listing_retry(rx: &CancellationToken, delay: Duration) -> Result<()> {
+    tokio::select! {
+        _ = rx.cancelled() => Err(Error::OperationCanceled),
+        _ = tokio::time::sleep(delay) => Ok(()),
+    }
+}
+
+pub(super) fn ensure_rebalance_listing_disks_available(has_disks: bool, bucket: &str) -> Result<()> {
+    if !has_disks {
+        return Err(Error::other(format!(
+            "failed to list objects to rebalance for bucket {bucket}: no disks available"
+        )));
+    }
+
+    Ok(())
+}
+
+pub(super) fn with_rebalance_entry_context(stage: &str, bucket: &str, object_name: &str, err: Error) -> Error {
+    Error::other(format!("rebalance entry {stage} failed for {bucket}/{object_name}: {err}"))
+}
+
+pub(super) fn should_count_rebalance_version_complete(result: &MigrationVersionResult) -> bool {
+    result.cleanup_ignored || (result.moved && !result.failed)
+}
+
+pub(super) fn should_cleanup_rebalance_source_entry(rebalanced: usize, total_versions: usize) -> bool {
+    rebalanced == total_versions
+}
+
+pub(super) fn should_skip_rebalance_delete_marker(
+    version: &rustfs_filemeta::FileInfo,
+    remaining_versions: usize,
+    replication_configured: bool,
+) -> bool {
+    version.deleted && remaining_versions == 1 && !replication_configured
+}
+
+pub(super) fn resolve_rebalance_optional_bucket_config_result<T>(
+    bucket: &str,
+    stage: &str,
+    result: Result<T>,
+) -> Result<Option<T>> {
+    match result {
+        Ok(config) => Ok(Some(config)),
+        Err(Error::ConfigNotFound) => Ok(None),
+        Err(err) => Err(Error::other(format!("rebalance {stage} config load failed for bucket {bucket}: {err}"))),
+    }
+}
+
+pub(super) async fn load_rebalance_bucket_configs(bucket: &str) -> Result<RebalanceBucketConfigs> {
+    if bucket == crate::disk::RUSTFS_META_BUCKET {
+        return Ok(RebalanceBucketConfigs::default());
+    }
+
+    let _ = resolve_rebalance_optional_bucket_config_result(
+        bucket,
+        "versioning",
+        crate::bucket::versioning_sys::BucketVersioningSys::get(bucket).await,
+    )?;
+
+    Ok(RebalanceBucketConfigs {
+        lifecycle_config: runtime_sources::bucket_lifecycle_config(bucket).await,
+        lock_retention: crate::bucket::object_lock::objectlock_sys::BucketObjectLockSys::get(bucket).await,
+        replication_config: resolve_rebalance_optional_bucket_config_result(
+            bucket,
+            "replication",
+            crate::bucket::metadata_sys::get_replication_config(bucket).await,
+        )?,
+    })
+}
+
+pub(super) async fn run_rebalance_listing_with_retry(
+    set: Arc<SetDisks>,
+    rx: CancellationToken,
+    bucket: String,
+    cb: ListCallback,
+    set_idx: usize,
+    max_attempts: usize,
+) -> Result<()> {
+    let max_attempts = max_attempts.max(1);
+    let mut last_error = None;
+
+    for attempt in 0..max_attempts {
+        match set.list_objects_to_rebalance(rx.clone(), bucket.clone(), cb.clone()).await {
+            Ok(()) => return Ok(()),
+            Err(err) if should_retry_rebalance_listing(&err, attempt, max_attempts) => {
+                let next_attempt = attempt + 2;
+                let delay = rebalance_listing_retry_delay(attempt);
+                error!(
+                    "rebalance listing failed for bucket {} set {} attempt {}/{}: {}; retrying in {:?}",
+                    bucket,
+                    set_idx,
+                    attempt + 1,
+                    max_attempts,
+                    err,
+                    delay
+                );
+                last_error = Some(err);
+                wait_rebalance_listing_retry(&rx, delay).await?;
+                info!(
+                    "rebalance listing retrying bucket {} set {} attempt {}/{}",
+                    bucket, set_idx, next_attempt, max_attempts
+                );
+            }
+            Err(err) => {
+                return Err(Error::other(format!(
+                    "rebalance listing failed for bucket {bucket} set {set_idx} attempt {}/{}: {err}",
+                    attempt + 1,
+                    max_attempts
+                )));
+            }
+        }
+    }
+
+    Err(Error::other(format!(
+        "rebalance listing failed for bucket {bucket} set {set_idx} after {max_attempts} attempts: {}",
+        last_error
+            .map(|err| err.to_string())
+            .unwrap_or_else(|| "unknown listing failure".to_string())
+    )))
+}
+
+impl SetDisks {
+    #[tracing::instrument(skip(self, rx, cb))]
+    pub async fn list_objects_to_rebalance(
+        self: &Arc<Self>,
+        rx: CancellationToken,
+        bucket: String,
+        cb: ListCallback,
+    ) -> Result<()> {
+        debug!(
+            event = EVENT_REBALANCE_LISTING,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            bucket = %bucket,
+            state = "started",
+            "Rebalance listing started"
+        );
+        let (disks, _) = self.get_online_disks_with_healing(false).await;
+        ensure_rebalance_listing_disks_available(!disks.is_empty(), &bucket)?;
+
+        debug!(
+            event = EVENT_REBALANCE_LISTING,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            bucket = %bucket,
+            disk_count = disks.len(),
+            state = "disks_resolved",
+            "Rebalance listing disks resolved"
+        );
+        let listing_quorum = self.set_drive_count.div_ceil(2);
+
+        let resolver = MetadataResolutionParams {
+            dir_quorum: listing_quorum,
+            obj_quorum: listing_quorum,
+            bucket: bucket.clone(),
+            ..Default::default()
+        };
+
+        let cb1 = cb.clone();
+        list_path_raw(
+            rx,
+            ListPathRawOptions {
+                disks: disks.iter().cloned().map(Some).collect(),
+                bucket: bucket.clone(),
+                recursive: true,
+                min_disks: listing_quorum,
+                skip_walkdir_total_timeout: true,
+                agreed: Some(Box::new(move |entry: MetaCacheEntry| {
+                    debug!(
+                        event = EVENT_REBALANCE_LISTING,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        entry = %entry.name,
+                        state = "agreed_entry",
+                        "Rebalance listing agreed entry"
+                    );
+                    Box::pin(cb1(entry))
+                })),
+                partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
+                    let resolver = resolver.clone();
+                    let cb = cb.clone();
+
+                    match entries.resolve(resolver) {
+                        Some(entry) => {
+                            debug!(
+                                event = EVENT_REBALANCE_LISTING,
+                                component = LOG_COMPONENT_ECSTORE,
+                                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                                entry = %entry.name,
+                                state = "resolved_partial_entry",
+                                "Rebalance listing resolved partial entry"
+                            );
+                            Box::pin(async move { cb(entry).await })
+                        }
+                        None => {
+                            debug!(
+                                event = EVENT_REBALANCE_LISTING,
+                                component = LOG_COMPONENT_ECSTORE,
+                                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                                state = "partial_entry_missing",
+                                "Rebalance listing partial entry missing"
+                            );
+                            Box::pin(async {})
+                        }
+                    }
+                })),
+                ..Default::default()
+            },
+        )
+        .await?;
+
+        debug!(
+            event = EVENT_REBALANCE_LISTING,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            bucket = %bucket,
+            state = "completed",
+            "Rebalance listing completed"
+        );
+        Ok(())
+    }
+}

crates/ecstore/src/services/tier/mod.rs

@@ -0,0 +1,30 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod tier;
+pub mod tier_admin;
+pub mod tier_config;
+pub mod tier_gen;
+pub mod tier_handlers;
+pub mod warm_backend;
+pub mod warm_backend_aliyun;
+pub mod warm_backend_azure;
+pub mod warm_backend_gcs;
+pub mod warm_backend_huaweicloud;
+pub mod warm_backend_minio;
+pub mod warm_backend_r2;
+pub mod warm_backend_rustfs;
+pub mod warm_backend_s3;
+pub mod warm_backend_s3sdk;
+pub mod warm_backend_tencent;

crates/ecstore/src/services/tier/tier_admin.rs

@@ -0,0 +1,42 @@
+#![allow(unused_imports)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierCreds {
+    #[serde(rename = "accessKey")]
+    pub access_key: String,
+    #[serde(rename = "secretKey")]
+    pub secret_key: String,
+
+    #[serde(rename = "awsRole")]
+    pub aws_role: bool,
+    #[serde(rename = "awsRoleWebIdentityTokenFile")]
+    pub aws_role_web_identity_token_file: String,
+    #[serde(rename = "awsRoleArn")]
+    pub aws_role_arn: String,
+
+    //azsp: ServicePrincipalAuth,
+
+    //#[serde(rename = "credsJson")]
+    pub creds_json: Vec<u8>,
+}

crates/ecstore/src/services/tier/tier_config.rs

@@ -0,0 +1,600 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use serde::{Deserialize, Serialize};
+use std::fmt::Display;
+use tracing::info;
+
+const C_TIER_CONFIG_VER: &str = "v1";
+
+const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty";
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+pub enum TierType {
+    #[default]
+    Unsupported,
+    #[serde(rename = "s3")]
+    S3,
+    #[serde(rename = "rustfs")]
+    RustFS,
+    #[serde(rename = "minio")]
+    MinIO,
+    #[serde(rename = "aliyun")]
+    Aliyun,
+    #[serde(rename = "tencent")]
+    Tencent,
+    #[serde(rename = "huaweicloud")]
+    Huaweicloud,
+    #[serde(rename = "azure")]
+    Azure,
+    #[serde(rename = "gcs")]
+    GCS,
+    #[serde(rename = "r2")]
+    R2,
+}
+
+impl Display for TierType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            TierType::S3 => {
+                write!(f, "S3")
+            }
+            TierType::RustFS => {
+                write!(f, "RustFS")
+            }
+            TierType::MinIO => {
+                write!(f, "MinIO")
+            }
+            TierType::Aliyun => {
+                write!(f, "Aliyun")
+            }
+            TierType::Tencent => {
+                write!(f, "Tencent")
+            }
+            TierType::Huaweicloud => {
+                write!(f, "Huaweicloud")
+            }
+            TierType::Azure => {
+                write!(f, "Azure")
+            }
+            TierType::GCS => {
+                write!(f, "GCS")
+            }
+            TierType::R2 => {
+                write!(f, "R2")
+            }
+            _ => {
+                write!(f, "Unsupported")
+            }
+        }
+    }
+}
+
+impl TierType {
+    pub fn new(sc_type: &str) -> Self {
+        match sc_type {
+            "S3" => TierType::S3,
+            "RustFS" => TierType::RustFS,
+            "MinIO" => TierType::MinIO,
+            "Aliyun" => TierType::Aliyun,
+            "Tencent" => TierType::Tencent,
+            "Huaweicloud" => TierType::Huaweicloud,
+            "Azure" => TierType::Azure,
+            "GCS" => TierType::GCS,
+            "R2" => TierType::R2,
+            _ => TierType::Unsupported,
+        }
+    }
+
+    pub fn as_lowercase(&self) -> String {
+        match self {
+            TierType::S3 => "s3".to_string(),
+            TierType::RustFS => "rustfs".to_string(),
+            TierType::MinIO => "minio".to_string(),
+            TierType::Aliyun => "aliyun".to_string(),
+            TierType::Tencent => "tencent".to_string(),
+            TierType::Huaweicloud => "huaweicloud".to_string(),
+            TierType::Azure => "azure".to_string(),
+            TierType::GCS => "gcs".to_string(),
+            TierType::R2 => "r2".to_string(),
+            _ => "unsupported".to_string(),
+        }
+    }
+}
+
+#[derive(Default, Debug, Serialize, Deserialize)]
+#[serde(default)]
+pub struct TierConfig {
+    #[serde(skip)]
+    pub version: String,
+    #[serde(rename = "type")]
+    pub tier_type: TierType,
+    #[serde(skip)]
+    pub name: String,
+    #[serde(rename = "s3", skip_serializing_if = "Option::is_none")]
+    pub s3: Option<TierS3>,
+    #[serde(rename = "aliyun", skip_serializing_if = "Option::is_none")]
+    pub aliyun: Option<TierAliyun>,
+    #[serde(rename = "tencent", skip_serializing_if = "Option::is_none")]
+    pub tencent: Option<TierTencent>,
+    #[serde(rename = "huaweicloud", skip_serializing_if = "Option::is_none")]
+    pub huaweicloud: Option<TierHuaweicloud>,
+    #[serde(rename = "azure", skip_serializing_if = "Option::is_none")]
+    pub azure: Option<TierAzure>,
+    #[serde(rename = "gcs", skip_serializing_if = "Option::is_none")]
+    pub gcs: Option<TierGCS>,
+    #[serde(rename = "r2", skip_serializing_if = "Option::is_none")]
+    pub r2: Option<TierR2>,
+    #[serde(rename = "rustfs", skip_serializing_if = "Option::is_none")]
+    pub rustfs: Option<TierRustFS>,
+    #[serde(rename = "minio", skip_serializing_if = "Option::is_none")]
+    pub minio: Option<TierMinIO>,
+}
+
+impl Clone for TierConfig {
+    fn clone(&self) -> TierConfig {
+        let mut s3 = None;
+        let mut r = None;
+        let mut compatible_backend = None;
+        let mut aliyun = None;
+        let mut tencent = None;
+        let mut huaweicloud = None;
+        let mut azure = None;
+        let mut gcs = None;
+        let mut r2 = None;
+        match self.tier_type {
+            TierType::S3 => {
+                if let Some(s3_) = self.s3.as_ref() {
+                    let mut s3_clone = s3_.clone();
+                    s3_clone.secret_key = "REDACTED".to_string();
+                    s3 = Some(s3_clone);
+                }
+            }
+            TierType::RustFS => {
+                if let Some(r_) = self.rustfs.as_ref() {
+                    let mut r_clone = r_.clone();
+                    r_clone.secret_key = "REDACTED".to_string();
+                    r = Some(r_clone);
+                }
+            }
+            TierType::MinIO => {
+                if let Some(compatible_backend_) = self.minio.as_ref() {
+                    let mut compatible_backend_clone = compatible_backend_.clone();
+                    compatible_backend_clone.secret_key = "REDACTED".to_string();
+                    compatible_backend = Some(compatible_backend_clone);
+                }
+            }
+            TierType::Aliyun => {
+                if let Some(aliyun_) = self.aliyun.as_ref() {
+                    let mut aliyun_clone = aliyun_.clone();
+                    aliyun_clone.secret_key = "REDACTED".to_string();
+                    aliyun = Some(aliyun_clone);
+                }
+            }
+            TierType::Tencent => {
+                if let Some(tencent_) = self.tencent.as_ref() {
+                    let mut tencent_clone = tencent_.clone();
+                    tencent_clone.secret_key = "REDACTED".to_string();
+                    tencent = Some(tencent_clone);
+                }
+            }
+            TierType::Huaweicloud => {
+                if let Some(huaweicloud_) = self.huaweicloud.as_ref() {
+                    let mut huaweicloud_clone = huaweicloud_.clone();
+                    huaweicloud_clone.secret_key = "REDACTED".to_string();
+                    huaweicloud = Some(huaweicloud_clone);
+                }
+            }
+            TierType::Azure => {
+                if let Some(azure_) = self.azure.as_ref() {
+                    let mut azure_clone = azure_.clone();
+                    azure_clone.secret_key = "REDACTED".to_string();
+                    azure = Some(azure_clone);
+                }
+            }
+            TierType::GCS => {
+                if let Some(gcs_) = self.gcs.as_ref() {
+                    let mut gcs_clone = gcs_.clone();
+                    gcs_clone.creds = "REDACTED".to_string();
+                    gcs = Some(gcs_clone);
+                }
+            }
+            TierType::R2 => {
+                if let Some(r2_) = self.r2.as_ref() {
+                    let mut r2_clone = r2_.clone();
+                    r2_clone.secret_key = "REDACTED".to_string();
+                    r2 = Some(r2_clone);
+                }
+            }
+            _ => (),
+        }
+        TierConfig {
+            version: self.version.clone(),
+            tier_type: self.tier_type.clone(),
+            name: self.name.clone(),
+            s3,
+            rustfs: r,
+            minio: compatible_backend,
+            aliyun,
+            tencent,
+            huaweicloud,
+            azure,
+            gcs,
+            r2,
+        }
+    }
+}
+
+#[allow(dead_code)]
+impl TierConfig {
+    fn endpoint(&self) -> String {
+        match self.tier_type {
+            TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(),
+            TierType::RustFS => self.rustfs.as_ref().map(|r| r.endpoint.clone()).unwrap_or_default(),
+            TierType::MinIO => self.minio.as_ref().map(|m| m.endpoint.clone()).unwrap_or_default(),
+            TierType::Aliyun => self.aliyun.as_ref().map(|a| a.endpoint.clone()).unwrap_or_default(),
+            TierType::Tencent => self.tencent.as_ref().map(|t| t.endpoint.clone()).unwrap_or_default(),
+            TierType::Huaweicloud => self.huaweicloud.as_ref().map(|h| h.endpoint.clone()).unwrap_or_default(),
+            TierType::Azure => self.azure.as_ref().map(|a| a.endpoint.clone()).unwrap_or_default(),
+            TierType::GCS => self.gcs.as_ref().map(|g| g.endpoint.clone()).unwrap_or_default(),
+            TierType::R2 => self.r2.as_ref().map(|r| r.endpoint.clone()).unwrap_or_default(),
+            _ => {
+                info!("unexpected tier type {}", self.tier_type);
+                "".to_string()
+            }
+        }
+    }
+
+    fn bucket(&self) -> String {
+        match self.tier_type {
+            TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(),
+            TierType::RustFS => self.rustfs.as_ref().map(|r| r.bucket.clone()).unwrap_or_default(),
+            TierType::MinIO => self.minio.as_ref().map(|m| m.bucket.clone()).unwrap_or_default(),
+            TierType::Aliyun => self.aliyun.as_ref().map(|a| a.bucket.clone()).unwrap_or_default(),
+            TierType::Tencent => self.tencent.as_ref().map(|t| t.bucket.clone()).unwrap_or_default(),
+            TierType::Huaweicloud => self.huaweicloud.as_ref().map(|h| h.bucket.clone()).unwrap_or_default(),
+            TierType::Azure => self.azure.as_ref().map(|a| a.bucket.clone()).unwrap_or_default(),
+            TierType::GCS => self.gcs.as_ref().map(|g| g.bucket.clone()).unwrap_or_default(),
+            TierType::R2 => self.r2.as_ref().map(|r| r.bucket.clone()).unwrap_or_default(),
+            _ => {
+                info!("unexpected tier type {}", self.tier_type);
+                "".to_string()
+            }
+        }
+    }
+
+    fn prefix(&self) -> String {
+        match self.tier_type {
+            TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(),
+            TierType::RustFS => self.rustfs.as_ref().map(|r| r.prefix.clone()).unwrap_or_default(),
+            TierType::MinIO => self.minio.as_ref().map(|m| m.prefix.clone()).unwrap_or_default(),
+            TierType::Aliyun => self.aliyun.as_ref().map(|a| a.prefix.clone()).unwrap_or_default(),
+            TierType::Tencent => self.tencent.as_ref().map(|t| t.prefix.clone()).unwrap_or_default(),
+            TierType::Huaweicloud => self.huaweicloud.as_ref().map(|h| h.prefix.clone()).unwrap_or_default(),
+            TierType::Azure => self.azure.as_ref().map(|a| a.prefix.clone()).unwrap_or_default(),
+            TierType::GCS => self.gcs.as_ref().map(|g| g.prefix.clone()).unwrap_or_default(),
+            TierType::R2 => self.r2.as_ref().map(|r| r.prefix.clone()).unwrap_or_default(),
+            _ => {
+                info!("unexpected tier type {}", self.tier_type);
+                "".to_string()
+            }
+        }
+    }
+
+    fn region(&self) -> String {
+        match self.tier_type {
+            TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(),
+            TierType::RustFS => self.rustfs.as_ref().map(|r| r.region.clone()).unwrap_or_default(),
+            TierType::MinIO => self.minio.as_ref().map(|m| m.region.clone()).unwrap_or_default(),
+            TierType::Aliyun => self.aliyun.as_ref().map(|a| a.region.clone()).unwrap_or_default(),
+            TierType::Tencent => self.tencent.as_ref().map(|t| t.region.clone()).unwrap_or_default(),
+            TierType::Huaweicloud => self.huaweicloud.as_ref().map(|h| h.region.clone()).unwrap_or_default(),
+            TierType::Azure => self.azure.as_ref().map(|a| a.region.clone()).unwrap_or_default(),
+            TierType::GCS => self.gcs.as_ref().map(|g| g.region.clone()).unwrap_or_default(),
+            TierType::R2 => self.r2.as_ref().map(|r| r.region.clone()).unwrap_or_default(),
+            _ => {
+                info!("unexpected tier type {}", self.tier_type);
+                "".to_string()
+            }
+        }
+    }
+}
+
+//type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierS3 {
+    pub name: String,
+    pub endpoint: String,
+    #[serde(rename = "accessKey")]
+    pub access_key: String,
+    #[serde(rename = "secretKey")]
+    pub secret_key: String,
+    pub bucket: String,
+    pub prefix: String,
+    pub region: String,
+    #[serde(rename = "storageClass")]
+    pub storage_class: String,
+    #[serde(skip)]
+    pub aws_role: bool,
+    #[serde(skip)]
+    pub aws_role_web_identity_token_file: String,
+    #[serde(skip)]
+    pub aws_role_arn: String,
+    #[serde(skip)]
+    pub aws_role_session_name: String,
+    #[serde(skip)]
+    pub aws_role_duration_seconds: i32,
+}
+
+impl TierS3 {
+    #[allow(dead_code)]
+    fn create<F>(
+        name: &str,
+        access_key: &str,
+        secret_key: &str,
+        bucket: &str,
+        options: Vec<F>,
+    ) -> Result<TierConfig, std::io::Error>
+    where
+        F: Fn(TierS3) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
+    {
+        if name.is_empty() {
+            return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
+        }
+        let sc = TierS3 {
+            access_key: access_key.to_string(),
+            secret_key: secret_key.to_string(),
+            bucket: bucket.to_string(),
+            endpoint: "https://s3.amazonaws.com".to_string(),
+            region: "".to_string(),
+            storage_class: "".to_string(),
+            ..Default::default()
+        };
+
+        for option in options {
+            let option = option(sc.clone());
+            let option = *option;
+            option?;
+        }
+
+        Ok(TierConfig {
+            version: C_TIER_CONFIG_VER.to_string(),
+            tier_type: TierType::S3,
+            name: name.to_string(),
+            s3: Some(sc),
+            ..Default::default()
+        })
+    }
+}
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierRustFS {
+    pub name: String,
+    pub endpoint: String,
+    #[serde(rename = "accessKey")]
+    pub access_key: String,
+    #[serde(rename = "secretKey")]
+    pub secret_key: String,
+    pub bucket: String,
+    pub prefix: String,
+    pub region: String,
+    #[serde(rename = "storageClass")]
+    pub storage_class: String,
+}
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierMinIO {
+    pub name: String,
+    pub endpoint: String,
+    #[serde(rename = "accessKey")]
+    pub access_key: String,
+    #[serde(rename = "secretKey")]
+    pub secret_key: String,
+    pub bucket: String,
+    pub prefix: String,
+    pub region: String,
+}
+
+impl TierMinIO {
+    #[allow(dead_code)]
+    fn create<F>(
+        name: &str,
+        endpoint: &str,
+        access_key: &str,
+        secret_key: &str,
+        bucket: &str,
+        options: Vec<F>,
+    ) -> Result<TierConfig, std::io::Error>
+    where
+        F: Fn(TierMinIO) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
+    {
+        if name.is_empty() {
+            return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
+        }
+        let backend = TierMinIO {
+            access_key: access_key.to_string(),
+            secret_key: secret_key.to_string(),
+            bucket: bucket.to_string(),
+            endpoint: endpoint.to_string(),
+            ..Default::default()
+        };
+
+        for option in options {
+            let option = option(backend.clone());
+            let option = *option;
+            option?;
+        }
+
+        Ok(TierConfig {
+            version: C_TIER_CONFIG_VER.to_string(),
+            tier_type: TierType::MinIO,
+            name: name.to_string(),
+            minio: Some(backend),
+            ..Default::default()
+        })
+    }
+}
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierAliyun {
+    pub name: String,
+    pub endpoint: String,
+    #[serde(rename = "accessKey")]
+    pub access_key: String,
+    #[serde(rename = "secretKey")]
+    pub secret_key: String,
+    pub bucket: String,
+    pub prefix: String,
+    pub region: String,
+}
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierTencent {
+    pub name: String,
+    pub endpoint: String,
+    #[serde(rename = "accessKey")]
+    pub access_key: String,
+    #[serde(rename = "secretKey")]
+    pub secret_key: String,
+    pub bucket: String,
+    pub prefix: String,
+    pub region: String,
+}
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierHuaweicloud {
+    pub name: String,
+    pub endpoint: String,
+    #[serde(rename = "accessKey")]
+    pub access_key: String,
+    #[serde(rename = "secretKey")]
+    pub secret_key: String,
+    pub bucket: String,
+    pub prefix: String,
+    pub region: String,
+}
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct ServicePrincipalAuth {
+    pub tenant_id: String,
+    pub client_id: String,
+    pub client_secret: String,
+}
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierAzure {
+    pub name: String,
+    pub endpoint: String,
+    #[serde(rename = "accessKey")]
+    pub access_key: String,
+    #[serde(rename = "secretKey")]
+    pub secret_key: String,
+    pub bucket: String,
+    pub prefix: String,
+    pub region: String,
+    #[serde(rename = "storageClass")]
+    pub storage_class: String,
+    #[serde(rename = "spAuth")]
+    pub sp_auth: ServicePrincipalAuth,
+}
+
+impl TierAzure {
+    pub fn is_sp_enabled(&self) -> bool {
+        !self.sp_auth.tenant_id.is_empty() && !self.sp_auth.client_id.is_empty() && !self.sp_auth.client_secret.is_empty()
+    }
+}
+
+/*
+fn AzureServicePrincipal(tenantID, clientID, clientSecret string) func(az *TierAzure) error {
+  return func(az *TierAzure) error {
+    if tenantID == "" {
+      return errors.New("empty tenant ID unsupported")
+    }
+    if clientID == "" {
+      return errors.New("empty client ID unsupported")
+    }
+    if clientSecret == "" {
+      return errors.New("empty client secret unsupported")
+    }
+    az.SPAuth.TenantID = tenantID
+    az.SPAuth.ClientID = clientID
+    az.SPAuth.ClientSecret = clientSecret
+    return nil
+  }
+}
+
+fn AzurePrefix(prefix string) func(az *TierAzure) error {
+  return func(az *TierAzure) error {
+    az.Prefix = prefix
+    return nil
+  }
+}
+
+fn AzureEndpoint(endpoint string) func(az *TierAzure) error {
+  return func(az *TierAzure) error {
+    az.Endpoint = endpoint
+    return nil
+  }
+}
+
+fn AzureRegion(region string) func(az *TierAzure) error {
+  return func(az *TierAzure) error {
+    az.Region = region
+    return nil
+  }
+}
+
+fn AzureStorageClass(sc string) func(az *TierAzure) error {
+  return func(az *TierAzure) error {
+    az.StorageClass = sc
+    return nil
+  }
+}*/
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierGCS {
+    pub name: String,
+    pub endpoint: String,
+    #[serde(rename = "creds")]
+    pub creds: String,
+    pub bucket: String,
+    pub prefix: String,
+    pub region: String,
+    #[serde(rename = "storageClass")]
+    pub storage_class: String,
+}
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierR2 {
+    pub name: String,
+    pub endpoint: String,
+    #[serde(rename = "accessKey")]
+    pub access_key: String,
+    #[serde(rename = "secretKey")]
+    pub secret_key: String,
+    pub bucket: String,
+    pub prefix: String,
+    pub region: String,
+}

crates/ecstore/src/services/tier/tier_gen.rs

@@ -0,0 +1,22 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::services::tier::tier::TierConfigMgr;
+
+#[allow(dead_code)]
+impl TierConfigMgr {
+    pub fn msg_size(&self) -> usize {
+        100
+    }
+}

crates/ecstore/src/services/tier/tier_handlers.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::client::admin_handler_utils::AdminError;
+use http::status::StatusCode;
+use lazy_static::lazy_static;
+
+lazy_static! {
+    pub static ref ERR_TIER_ALREADY_EXISTS: AdminError = AdminError {
+        code: "XRustFSAdminTierAlreadyExists".to_string(),
+        message: "Specified remote tier already exists".to_string(),
+        status_code: StatusCode::CONFLICT,
+    };
+    pub static ref ERR_TIER_NOT_FOUND: AdminError = AdminError {
+        code: "XRustFSAdminTierNotFound".to_string(),
+        message: "Specified remote tier was not found".to_string(),
+        status_code: StatusCode::NOT_FOUND,
+    };
+    pub static ref ERR_TIER_NAME_NOT_UPPERCASE: AdminError = AdminError {
+        code: "XRustFSAdminTierNameNotUpperCase".to_string(),
+        message: "Tier name must be in uppercase".to_string(),
+        status_code: StatusCode::BAD_REQUEST,
+    };
+    pub static ref ERR_TIER_BUCKET_NOT_FOUND: AdminError = AdminError {
+        code: "XRustFSAdminTierBucketNotFound".to_string(),
+        message: "Remote tier bucket not found".to_string(),
+        status_code: StatusCode::BAD_REQUEST,
+    };
+    pub static ref ERR_TIER_INVALID_CREDENTIALS: AdminError = AdminError {
+        code: "XRustFSAdminTierInvalidCredentials".to_string(),
+        message: "Invalid remote tier credentials".to_string(),
+        status_code: StatusCode::BAD_REQUEST,
+    };
+    pub static ref ERR_TIER_RESERVED_NAME: AdminError = AdminError {
+        code: "XRustFSAdminTierReserved".to_string(),
+        message: "Cannot use reserved tier name".to_string(),
+        status_code: StatusCode::BAD_REQUEST,
+    };
+    pub static ref ERR_TIER_PERM_ERR: AdminError = AdminError {
+        code: "TierPermErr".to_string(),
+        message: "Tier Perm Err".to_string(),
+        status_code: StatusCode::OK,
+    };
+    pub static ref ERR_TIER_CONNECT_ERR: AdminError = AdminError {
+        code: "TierConnectErr".to_string(),
+        message: "Tier Connect Err".to_string(),
+        status_code: StatusCode::OK,
+    };
+}

crates/ecstore/src/services/tier/warm_backend.rs

@@ -0,0 +1,432 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use crate::client::{
+    admin_handler_utils::AdminError,
+    api_put_object::{AdvancedPutOptions, PutObjectOptions},
+    transition_api::{ReadCloser, ReaderImpl},
+};
+use crate::error::is_err_bucket_not_found;
+use crate::services::tier::{
+    tier::ERR_TIER_TYPE_UNSUPPORTED,
+    tier_config::{TierConfig, TierType},
+    tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
+    warm_backend_aliyun::WarmBackendAliyun,
+    warm_backend_azure::WarmBackendAzure,
+    warm_backend_gcs::WarmBackendGCS,
+    warm_backend_huaweicloud::WarmBackendHuaweicloud,
+    warm_backend_minio::WarmBackendMinIO,
+    warm_backend_r2::WarmBackendR2,
+    warm_backend_rustfs::WarmBackendRustFS,
+    warm_backend_s3::WarmBackendS3,
+    warm_backend_tencent::WarmBackendTencent,
+};
+use bytes::Bytes;
+use http::StatusCode;
+use rustfs_utils::http::headers::{
+    CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
+};
+use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus};
+use s3s::header::{
+    X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
+    X_AMZ_STORAGE_CLASS,
+};
+use std::collections::HashMap;
+use time::OffsetDateTime;
+use time::format_description::well_known::{Rfc2822, Rfc3339};
+use tracing::{info, warn};
+
+pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
+
+const PROBE_OBJECT: &str = "probeobject";
+
+#[derive(Default)]
+pub struct WarmBackendGetOpts {
+    pub start_offset: i64,
+    pub length: i64,
+}
+
+#[async_trait::async_trait]
+pub trait WarmBackend {
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error>;
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error>;
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error>;
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>;
+    async fn in_use(&self) -> Result<bool, std::io::Error>;
+}
+
+fn parse_http_timestamp(value: &str) -> Option<OffsetDateTime> {
+    OffsetDateTime::parse(value, &Rfc3339)
+        .or_else(|_| OffsetDateTime::parse(value, &Rfc2822))
+        .ok()
+}
+
+pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap<String, String>) -> PutObjectOptions {
+    let mut opts = PutObjectOptions {
+        storage_class,
+        send_content_md5: true,
+        legalhold: ObjectLockLegalHoldStatus::from_static(""),
+        internal: AdvancedPutOptions {
+            replication_status: ReplicationStatus::from_static(""),
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+
+    if let Some(content_type) = metadata.lookup(CONTENT_TYPE) {
+        opts.content_type = content_type.to_string();
+    }
+
+    if let Some(content_encoding) = metadata.lookup(CONTENT_ENCODING) {
+        opts.content_encoding = content_encoding.to_string();
+    }
+
+    if let Some(content_language) = metadata.lookup(CONTENT_LANGUAGE) {
+        opts.content_language = content_language.to_string();
+    }
+
+    if let Some(content_disposition) = metadata.lookup(CONTENT_DISPOSITION) {
+        opts.content_disposition = content_disposition.to_string();
+    }
+
+    if let Some(cache_control) = metadata.lookup(CACHE_CONTROL) {
+        opts.cache_control = cache_control.to_string();
+    }
+
+    if let Some(expires) = metadata.lookup(EXPIRES).and_then(parse_http_timestamp) {
+        opts.expires = expires;
+    }
+
+    if let Some(mode) = metadata.lookup(X_AMZ_OBJECT_LOCK_MODE.as_str()) {
+        opts.mode = ObjectLockRetentionMode::from(mode.to_ascii_uppercase());
+    }
+
+    if let Some(retain_until_date) = metadata
+        .lookup(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str())
+        .and_then(parse_http_timestamp)
+    {
+        opts.retain_until_date = retain_until_date;
+    }
+
+    if let Some(legalhold) = metadata.lookup(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) {
+        opts.legalhold = ObjectLockLegalHoldStatus::from(legalhold.to_ascii_uppercase());
+    }
+
+    for key in [
+        CONTENT_TYPE,
+        CONTENT_ENCODING,
+        CONTENT_LANGUAGE,
+        CONTENT_DISPOSITION,
+        CACHE_CONTROL,
+        EXPIRES,
+        X_AMZ_OBJECT_LOCK_MODE.as_str(),
+        X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str(),
+        X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str(),
+        X_AMZ_REPLICATION_STATUS.as_str(),
+        X_AMZ_STORAGE_CLASS.as_str(),
+    ] {
+        metadata.remove(key);
+    }
+
+    opts.user_metadata = metadata;
+    opts
+}
+
+pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
+    let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
+    let remote_version_id = w
+        .put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
+        .await;
+    if let Err(err) = remote_version_id {
+        return Err(ERR_TIER_PERM_ERR.clone());
+    }
+
+    let r = w.get(PROBE_OBJECT, "", WarmBackendGetOpts::default()).await;
+    //xhttp.DrainBody(r);
+    if let Err(err) = r {
+        //if is_err_bucket_not_found(&err) {
+        //    return Err(ERR_TIER_BUCKET_NOT_FOUND);
+        //}
+        /*else if is_err_signature_does_not_match(err) {
+            return Err(ERR_TIER_MISSING_CREDENTIALS);
+        }*/
+        //else {
+        return Err(ERR_TIER_PERM_ERR.clone());
+        //}
+    }
+    if let Ok(version_id) = remote_version_id {
+        if let Err(err) = w.remove(PROBE_OBJECT, &version_id).await {
+            return Err(ERR_TIER_PERM_ERR.clone());
+        };
+    }
+    Ok(())
+}
+
+pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBackendImpl, AdminError> {
+    let mut d: Option<WarmBackendImpl> = None;
+    match tier.tier_type {
+        TierType::S3 => {
+            if let Some(s3_config) = tier.s3.as_ref() {
+                let dd = WarmBackendS3::new(s3_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create S3 backend")));
+            } else {
+                return Err(AdminError {
+                    code: "XRustFSAdminTierInvalidConfig".to_string(),
+                    message: "S3 tier configuration not found".to_string(),
+                    status_code: StatusCode::BAD_REQUEST,
+                });
+            }
+        }
+        TierType::RustFS => {
+            if let Some(rustfs_config) = tier.rustfs.as_ref() {
+                let dd = WarmBackendRustFS::new(rustfs_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create RustFS backend")));
+            } else {
+                return Err(AdminError {
+                    code: "XRustFSAdminTierInvalidConfig".to_string(),
+                    message: "RustFS tier configuration not found".to_string(),
+                    status_code: StatusCode::BAD_REQUEST,
+                });
+            }
+        }
+        TierType::MinIO => {
+            if let Some(minio_config) = tier.minio.as_ref() {
+                let dd = WarmBackendMinIO::new(minio_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create MinIO backend")));
+            } else {
+                return Err(AdminError {
+                    code: "XRustFSAdminTierInvalidConfig".to_string(),
+                    message: "MinIO tier configuration not found".to_string(),
+                    status_code: StatusCode::BAD_REQUEST,
+                });
+            }
+        }
+        TierType::Aliyun => {
+            if let Some(aliyun_config) = tier.aliyun.as_ref() {
+                let dd = WarmBackendAliyun::new(aliyun_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create Aliyun backend")));
+            } else {
+                return Err(AdminError {
+                    code: "XRustFSAdminTierInvalidConfig".to_string(),
+                    message: "Aliyun tier configuration not found".to_string(),
+                    status_code: StatusCode::BAD_REQUEST,
+                });
+            }
+        }
+        TierType::Tencent => {
+            if let Some(tencent_config) = tier.tencent.as_ref() {
+                let dd = WarmBackendTencent::new(tencent_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create Tencent backend")));
+            } else {
+                return Err(AdminError {
+                    code: "XRustFSAdminTierInvalidConfig".to_string(),
+                    message: "Tencent tier configuration not found".to_string(),
+                    status_code: StatusCode::BAD_REQUEST,
+                });
+            }
+        }
+        TierType::Huaweicloud => {
+            if let Some(huaweicloud_config) = tier.huaweicloud.as_ref() {
+                let dd = WarmBackendHuaweicloud::new(huaweicloud_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create Huaweicloud backend")));
+            } else {
+                return Err(AdminError {
+                    code: "XRustFSAdminTierInvalidConfig".to_string(),
+                    message: "Huaweicloud tier configuration not found".to_string(),
+                    status_code: StatusCode::BAD_REQUEST,
+                });
+            }
+        }
+        TierType::Azure => {
+            if let Some(azure_config) = tier.azure.as_ref() {
+                let dd = WarmBackendAzure::new(azure_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create Azure backend")));
+            } else {
+                return Err(AdminError {
+                    code: "XRustFSAdminTierInvalidConfig".to_string(),
+                    message: "Azure tier configuration not found".to_string(),
+                    status_code: StatusCode::BAD_REQUEST,
+                });
+            }
+        }
+        TierType::GCS => {
+            if let Some(gcs_config) = tier.gcs.as_ref() {
+                let dd = WarmBackendGCS::new(gcs_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create GCS backend")));
+            } else {
+                return Err(AdminError {
+                    code: "XRustFSAdminTierInvalidConfig".to_string(),
+                    message: "GCS tier configuration not found".to_string(),
+                    status_code: StatusCode::BAD_REQUEST,
+                });
+            }
+        }
+        TierType::R2 => {
+            if let Some(r2_config) = tier.r2.as_ref() {
+                let dd = WarmBackendR2::new(r2_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create R2 backend")));
+            } else {
+                return Err(AdminError {
+                    code: "XRustFSAdminTierInvalidConfig".to_string(),
+                    message: "R2 tier configuration not found".to_string(),
+                    status_code: StatusCode::BAD_REQUEST,
+                });
+            }
+        }
+        _ => {
+            return Err(ERR_TIER_TYPE_UNSUPPORTED.clone());
+        }
+    }
+
+    d.ok_or_else(|| AdminError {
+        code: "XRustFSAdminTierInvalidConfig".to_string(),
+        message: "Tier backend not initialized".to_string(),
+        status_code: StatusCode::BAD_REQUEST,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn build_transition_put_options_preserves_content_headers() {
+        let mut metadata = HashMap::new();
+        metadata.insert("content-type".to_string(), "text/plain".to_string());
+        metadata.insert("content-encoding".to_string(), "gzip".to_string());
+        metadata.insert("cache-control".to_string(), "max-age=60".to_string());
+
+        let opts = build_transition_put_options("COLD".to_string(), metadata);
+
+        assert_eq!(opts.content_type, "text/plain");
+        assert_eq!(opts.content_encoding, "gzip");
+        assert_eq!(opts.cache_control, "max-age=60");
+        assert_eq!(opts.internal.replication_status.as_str(), "");
+        assert_eq!(opts.legalhold.as_str(), "");
+    }
+
+    #[test]
+    fn build_transition_put_options_preserves_object_lock_headers_when_present() {
+        let mut metadata = HashMap::new();
+        metadata.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), "2026-03-23T00:00:00Z".to_string());
+        metadata.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.to_string(), ObjectLockLegalHoldStatus::ON.to_string());
+        metadata.insert(X_AMZ_OBJECT_LOCK_MODE.to_string(), ObjectLockRetentionMode::GOVERNANCE.to_string());
+
+        let opts = build_transition_put_options("COLD".to_string(), metadata);
+
+        assert_eq!(opts.mode.as_str(), ObjectLockRetentionMode::GOVERNANCE);
+        assert_eq!(opts.legalhold.as_str(), ObjectLockLegalHoldStatus::ON);
+        assert_ne!(opts.retain_until_date, OffsetDateTime::UNIX_EPOCH);
+    }
+
+    #[test]
+    fn build_transition_put_options_filters_promoted_headers_from_user_metadata() {
+        let mut metadata = HashMap::new();
+        metadata.insert("name".to_string(), "object".to_string());
+        metadata.insert(CONTENT_TYPE.to_string(), "text/plain".to_string());
+        metadata.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.to_string(), ObjectLockLegalHoldStatus::ON.to_string());
+        metadata.insert(X_AMZ_REPLICATION_STATUS.to_string(), "PENDING".to_string());
+
+        let opts = build_transition_put_options("COLD".to_string(), metadata);
+
+        assert_eq!(opts.user_metadata.get("name"), Some(&"object".to_string()));
+        assert!(!opts.user_metadata.contains_key(CONTENT_TYPE));
+        assert!(!opts.user_metadata.contains_key(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()));
+        assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
+    }
+}

crates/ecstore/src/services/tier/warm_backend_aliyun.rs

@@ -0,0 +1,155 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use crate::client::{
+    admin_handler_utils::AdminError,
+    api_put_object::PutObjectOptions,
+    credentials::{Credentials, SignatureType, Static, Value},
+    transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
+};
+use crate::services::tier::{
+    tier_config::TierAliyun,
+    warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend_s3::WarmBackendS3,
+};
+use tracing::warn;
+
+const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+const MAX_PARTS_COUNT: i64 = 10000;
+const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
+
+pub struct WarmBackendAliyun(WarmBackendS3);
+
+impl WarmBackendAliyun {
+    pub async fn new(conf: &TierAliyun, tier: &str) -> Result<Self, std::io::Error> {
+        if conf.access_key == "" || conf.secret_key == "" {
+            return Err(std::io::Error::other("both access and secret keys are required"));
+        }
+
+        if conf.bucket == "" {
+            return Err(std::io::Error::other("no bucket name was provided"));
+        }
+
+        let u = match url::Url::parse(&conf.endpoint) {
+            Ok(u) => u,
+            Err(e) => {
+                return Err(std::io::Error::other(e.to_string()));
+            }
+        };
+
+        let creds = Credentials::new(Static(Value {
+            access_key_id: conf.access_key.clone(),
+            secret_access_key: conf.secret_key.clone(),
+            session_token: "".to_string(),
+            signer_type: SignatureType::SignatureV4,
+            ..Default::default()
+        }));
+        let opts = Options {
+            creds,
+            secure: u.scheme() == "https",
+            //transport: GLOBAL_RemoteTargetTransport,
+            trailing_headers: true,
+            region: conf.region.clone(),
+            bucket_lookup: BucketLookupType::BucketLookupDNS,
+            ..Default::default()
+        };
+        let scheme = u.scheme();
+        let default_port = if scheme == "https" { 443 } else { 80 };
+        let host = u
+            .host_str()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "aliyun").await?;
+
+        let client = Arc::new(client);
+        let core = TransitionCore(Arc::clone(&client));
+        Ok(Self(WarmBackendS3 {
+            client,
+            core,
+            bucket: conf.bucket.clone(),
+            prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
+            storage_class: "".to_string(),
+        }))
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendAliyun {
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        let part_size = optimal_part_size(length)?;
+        let client = self.0.client.clone();
+        let res = client
+            .put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
+                let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
+                opts.part_size = part_size as u64;
+                opts.disable_content_sha256 = true;
+                opts
+            })
+            .await?;
+        //self.ToObjectError(err, object)
+        Ok(res.version_id)
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        self.put_with_meta(object, r, length, HashMap::new()).await
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        self.0.get(object, rv, opts).await
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        self.0.remove(object, rv).await
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        self.0.in_use().await
+    }
+}
+
+fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
+    let mut object_size = object_size;
+    if object_size == -1 {
+        object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
+    }
+
+    if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+        return Err(std::io::Error::other("entity too large"));
+    }
+
+    let configured_part_size = MIN_PART_SIZE;
+    let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
+    part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
+
+    let part_size = part_size_flt as i64;
+    if part_size == 0 {
+        return Ok(MIN_PART_SIZE);
+    }
+    Ok(part_size)
+}

crates/ecstore/src/services/tier/warm_backend_azure.rs

@@ -0,0 +1,155 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use crate::client::{
+    admin_handler_utils::AdminError,
+    api_put_object::PutObjectOptions,
+    credentials::{Credentials, SignatureType, Static, Value},
+    transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
+};
+use crate::services::tier::{
+    tier_config::TierAzure,
+    warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend_s3::WarmBackendS3,
+};
+use tracing::warn;
+
+const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+const MAX_PARTS_COUNT: i64 = 10000;
+const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
+
+pub struct WarmBackendAzure(WarmBackendS3);
+
+impl WarmBackendAzure {
+    pub async fn new(conf: &TierAzure, tier: &str) -> Result<Self, std::io::Error> {
+        if conf.access_key == "" || conf.secret_key == "" {
+            return Err(std::io::Error::other("both access and secret keys are required"));
+        }
+
+        if conf.bucket == "" {
+            return Err(std::io::Error::other("no bucket name was provided"));
+        }
+
+        let u = match url::Url::parse(&conf.endpoint) {
+            Ok(u) => u,
+            Err(e) => {
+                return Err(std::io::Error::other(e.to_string()));
+            }
+        };
+
+        let creds = Credentials::new(Static(Value {
+            access_key_id: conf.access_key.clone(),
+            secret_access_key: conf.secret_key.clone(),
+            session_token: "".to_string(),
+            signer_type: SignatureType::SignatureV4,
+            ..Default::default()
+        }));
+        let opts = Options {
+            creds,
+            secure: u.scheme() == "https",
+            //transport: GLOBAL_RemoteTargetTransport,
+            trailing_headers: true,
+            region: conf.region.clone(),
+            bucket_lookup: BucketLookupType::BucketLookupDNS,
+            ..Default::default()
+        };
+        let scheme = u.scheme();
+        let default_port = if scheme == "https" { 443 } else { 80 };
+        let host = u
+            .host_str()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "azure").await?;
+
+        let client = Arc::new(client);
+        let core = TransitionCore(Arc::clone(&client));
+        Ok(Self(WarmBackendS3 {
+            client,
+            core,
+            bucket: conf.bucket.clone(),
+            prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
+            storage_class: "".to_string(),
+        }))
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendAzure {
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        let part_size = optimal_part_size(length)?;
+        let client = self.0.client.clone();
+        let res = client
+            .put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
+                let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
+                opts.part_size = part_size as u64;
+                opts.disable_content_sha256 = true;
+                opts
+            })
+            .await?;
+        //self.ToObjectError(err, object)
+        Ok(res.version_id)
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        self.put_with_meta(object, r, length, HashMap::new()).await
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        self.0.get(object, rv, opts).await
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        self.0.remove(object, rv).await
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        self.0.in_use().await
+    }
+}
+
+fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
+    let mut object_size = object_size;
+    if object_size == -1 {
+        object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
+    }
+
+    if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+        return Err(std::io::Error::other("entity too large"));
+    }
+
+    let configured_part_size = MIN_PART_SIZE;
+    let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
+    part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
+
+    let part_size = part_size_flt as i64;
+    if part_size == 0 {
+        return Ok(MIN_PART_SIZE);
+    }
+    Ok(part_size)
+}

crates/ecstore/src/services/tier/warm_backend_gcs.rs

@@ -0,0 +1,249 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use bytes::Bytes;
+use google_cloud_auth::credentials::Credentials;
+use google_cloud_auth::credentials::user_account::Builder;
+use google_cloud_storage as gcs;
+use google_cloud_storage::client::Storage;
+use std::convert::TryFrom;
+
+use crate::client::{
+    admin_handler_utils::AdminError,
+    api_put_object::PutObjectOptions,
+    transition_api::{Options, ReadCloser, ReaderImpl},
+};
+use crate::services::tier::{
+    tier_config::TierGCS,
+    warm_backend::{WarmBackend, WarmBackendGetOpts},
+};
+use tracing::warn;
+
+const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+const MAX_PARTS_COUNT: i64 = 10000;
+const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
+
+pub struct WarmBackendGCS {
+    pub client: Arc<Storage>,
+    pub bucket: String,
+    pub prefix: String,
+    pub storage_class: String,
+}
+
+impl WarmBackendGCS {
+    pub async fn new(conf: &TierGCS, tier: &str) -> Result<Self, std::io::Error> {
+        if conf.creds == "" {
+            return Err(std::io::Error::other("both access and secret keys are required"));
+        }
+
+        if conf.bucket == "" {
+            return Err(std::io::Error::other("no bucket name was provided"));
+        }
+
+        let authorized_user = serde_json::from_str(&conf.creds)?;
+        let credentials = Builder::new(authorized_user)
+            //.with_retry_policy(AlwaysRetry.with_attempt_limit(3))
+            //.with_backoff_policy(backoff)
+            .build()
+            .map_err(|e| std::io::Error::other(format!("Invalid credentials JSON: {}", e)))?;
+
+        let Ok(client) = Storage::builder()
+            .with_endpoint(conf.endpoint.clone())
+            .with_credentials(credentials)
+            .build()
+            .await
+        else {
+            return Err(std::io::Error::other("Storage::builder error"));
+        };
+        let client = Arc::new(client);
+        Ok(Self {
+            client,
+            bucket: conf.bucket.clone(),
+            prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
+            storage_class: "".to_string(),
+        })
+    }
+
+    pub fn get_dest(&self, object: &str) -> String {
+        let mut dest_obj = object.to_string();
+        if self.prefix != "" {
+            dest_obj = format!("{}/{}", &self.prefix, object);
+        }
+        return dest_obj;
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendGCS {
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        let d = match r {
+            ReaderImpl::Body(content_body) => content_body.to_vec(),
+            ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?,
+        };
+        let Ok(res) = Box::pin(
+            self.client
+                .write_object(&self.bucket, &self.get_dest(object), Bytes::from(d))
+                .send_buffered(),
+        )
+        .await
+        else {
+            return Err(std::io::Error::other("write_object error"));
+        };
+        //self.ToObjectError(err, object)
+        Ok(res.generation.to_string())
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        self.put_with_meta(object, r, length, HashMap::new()).await
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        let Ok(mut reader) = self.client.read_object(&self.bucket, &self.get_dest(object)).send().await else {
+            return Err(std::io::Error::other("read_object error"));
+        };
+        let mut contents = Vec::new();
+        while let Ok(Some(chunk)) = reader.next().await.transpose() {
+            contents.extend_from_slice(&chunk);
+        }
+        Ok(ReadCloser::new(std::io::Cursor::new(contents)))
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        /*self.client
+        .delete_object()
+        .set_bucket(&self.bucket)
+        .set_object(&self.get_dest(object))
+        //.set_generation(object.generation)
+        .send()
+        .await?;*/
+        Ok(())
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        /*let result = self.client
+            .list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
+            .await?;
+
+        Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)*/
+        Ok(false)
+    }
+}
+
+/*fn gcs_to_object_error(err: Error, params: Vec<String>) -> Option<Error> {
+  if err == nil {
+    return nil
+  }
+
+  bucket := ""
+  object := ""
+  uploadID := ""
+  if len(params) >= 1 {
+    bucket = params[0]
+  }
+  if len(params) == 2 {
+    object = params[1]
+  }
+  if len(params) == 3 {
+    uploadID = params[2]
+  }
+
+  // in some cases just a plain error is being returned
+  switch err.Error() {
+  case "storage: bucket doesn't exist":
+    err = BucketNotFound{
+      Bucket: bucket,
+    }
+    return err
+  case "storage: object doesn't exist":
+    if uploadID != "" {
+      err = InvalidUploadID{
+        UploadID: uploadID,
+      }
+    } else {
+      err = ObjectNotFound{
+        Bucket: bucket,
+        Object: object,
+      }
+    }
+    return err
+  }
+
+  googleAPIErr, ok := err.(*googleapi.Error)
+  if !ok {
+    // We don't interpret non MinIO errors. As minio errors will
+    // have StatusCode to help to convert to object errors.
+    return err
+  }
+
+  if len(googleAPIErr.Errors) == 0 {
+    return err
+  }
+
+  reason := googleAPIErr.Errors[0].Reason
+  message := googleAPIErr.Errors[0].Message
+
+  switch reason {
+  case "required":
+    // Anonymous users does not have storage.xyz access to project 123.
+    fallthrough
+  case "keyInvalid":
+    fallthrough
+  case "forbidden":
+    err = PrefixAccessDenied{
+      Bucket: bucket,
+      Object: object,
+    }
+  case "invalid":
+    err = BucketNameInvalid{
+      Bucket: bucket,
+    }
+  case "notFound":
+    if object != "" {
+      err = ObjectNotFound{
+        Bucket: bucket,
+        Object: object,
+      }
+      break
+    }
+    err = BucketNotFound{Bucket: bucket}
+  case "conflict":
+    if message == "You already own this bucket. Please select another name." {
+      err = BucketAlreadyOwnedByYou{Bucket: bucket}
+      break
+    }
+    if message == "Sorry, that name is not available. Please try a different one." {
+      err = BucketAlreadyExists{Bucket: bucket}
+      break
+    }
+    err = BucketNotEmpty{Bucket: bucket}
+  }
+
+  return err
+}*/

crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs

@@ -0,0 +1,156 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use crate::client::{
+    admin_handler_utils::AdminError,
+    api_put_object::PutObjectOptions,
+    credentials::{Credentials, SignatureType, Static, Value},
+    transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
+};
+use crate::services::tier::{
+    tier_config::TierHuaweicloud,
+    warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend_s3::WarmBackendS3,
+};
+use tracing::warn;
+
+const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+const MAX_PARTS_COUNT: i64 = 10000;
+const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
+
+pub struct WarmBackendHuaweicloud(WarmBackendS3);
+
+impl WarmBackendHuaweicloud {
+    pub async fn new(conf: &TierHuaweicloud, tier: &str) -> Result<Self, std::io::Error> {
+        if conf.access_key == "" || conf.secret_key == "" {
+            return Err(std::io::Error::other("both access and secret keys are required"));
+        }
+
+        if conf.bucket == "" {
+            return Err(std::io::Error::other("no bucket name was provided"));
+        }
+
+        let u = match url::Url::parse(&conf.endpoint) {
+            Ok(u) => u,
+            Err(e) => {
+                return Err(std::io::Error::other(e.to_string()));
+            }
+        };
+
+        let creds = Credentials::new(Static(Value {
+            access_key_id: conf.access_key.clone(),
+            secret_access_key: conf.secret_key.clone(),
+            session_token: "".to_string(),
+            signer_type: SignatureType::SignatureV4,
+            ..Default::default()
+        }));
+        let opts = Options {
+            creds,
+            secure: u.scheme() == "https",
+            //transport: GLOBAL_RemoteTargetTransport,
+            trailing_headers: true,
+            region: conf.region.clone(),
+            bucket_lookup: BucketLookupType::BucketLookupDNS,
+            ..Default::default()
+        };
+        let scheme = u.scheme();
+        let default_port = if scheme == "https" { 443 } else { 80 };
+        let host = u
+            .host_str()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client =
+            TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "huaweicloud").await?;
+
+        let client = Arc::new(client);
+        let core = TransitionCore(Arc::clone(&client));
+        Ok(Self(WarmBackendS3 {
+            client,
+            core,
+            bucket: conf.bucket.clone(),
+            prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
+            storage_class: "".to_string(),
+        }))
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendHuaweicloud {
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        let part_size = optimal_part_size(length)?;
+        let client = self.0.client.clone();
+        let res = client
+            .put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
+                let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
+                opts.part_size = part_size as u64;
+                opts.disable_content_sha256 = true;
+                opts
+            })
+            .await?;
+        //self.ToObjectError(err, object)
+        Ok(res.version_id)
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        self.put_with_meta(object, r, length, HashMap::new()).await
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        self.0.get(object, rv, opts).await
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        self.0.remove(object, rv).await
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        self.0.in_use().await
+    }
+}
+
+fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
+    let mut object_size = object_size;
+    if object_size == -1 {
+        object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
+    }
+
+    if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+        return Err(std::io::Error::other("entity too large"));
+    }
+
+    let configured_part_size = MIN_PART_SIZE;
+    let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
+    part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
+
+    let part_size = part_size_flt as i64;
+    if part_size == 0 {
+        return Ok(MIN_PART_SIZE);
+    }
+    Ok(part_size)
+}

crates/ecstore/src/services/tier/warm_backend_minio.rs

@@ -0,0 +1,154 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use crate::client::{
+    admin_handler_utils::AdminError,
+    api_put_object::PutObjectOptions,
+    credentials::{Credentials, SignatureType, Static, Value},
+    transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
+};
+use crate::services::tier::{
+    tier_config::TierMinIO,
+    warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend_s3::WarmBackendS3,
+};
+use tracing::warn;
+
+const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+const MAX_PARTS_COUNT: i64 = 10000;
+const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
+
+pub struct WarmBackendMinIO(WarmBackendS3);
+
+impl WarmBackendMinIO {
+    pub async fn new(conf: &TierMinIO, tier: &str) -> Result<Self, std::io::Error> {
+        if conf.access_key == "" || conf.secret_key == "" {
+            return Err(std::io::Error::other("both access and secret keys are required"));
+        }
+
+        if conf.bucket == "" {
+            return Err(std::io::Error::other("no bucket name was provided"));
+        }
+
+        let u = match url::Url::parse(&conf.endpoint) {
+            Ok(u) => u,
+            Err(e) => {
+                return Err(std::io::Error::other(e.to_string()));
+            }
+        };
+
+        let creds = Credentials::new(Static(Value {
+            access_key_id: conf.access_key.clone(),
+            secret_access_key: conf.secret_key.clone(),
+            session_token: "".to_string(),
+            signer_type: SignatureType::SignatureV4,
+            ..Default::default()
+        }));
+        let opts = Options {
+            creds,
+            secure: u.scheme() == "https",
+            //transport: GLOBAL_RemoteTargetTransport,
+            trailing_headers: true,
+            region: conf.region.clone(),
+            ..Default::default()
+        };
+        let scheme = u.scheme();
+        let default_port = if scheme == "https" { 443 } else { 80 };
+        let host = u
+            .host_str()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "minio").await?;
+
+        let client = Arc::new(client);
+        let core = TransitionCore(Arc::clone(&client));
+        Ok(Self(WarmBackendS3 {
+            client,
+            core,
+            bucket: conf.bucket.clone(),
+            prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
+            storage_class: "".to_string(),
+        }))
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendMinIO {
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        let part_size = optimal_part_size(length)?;
+        let client = self.0.client.clone();
+        let res = client
+            .put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
+                let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
+                opts.part_size = part_size as u64;
+                opts.disable_content_sha256 = true;
+                opts
+            })
+            .await?;
+        //self.ToObjectError(err, object)
+        Ok(res.version_id)
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        self.put_with_meta(object, r, length, HashMap::new()).await
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        self.0.get(object, rv, opts).await
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        self.0.remove(object, rv).await
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        self.0.in_use().await
+    }
+}
+
+fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
+    let mut object_size = object_size;
+    if object_size == -1 {
+        object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
+    }
+
+    if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+        return Err(std::io::Error::other("entity too large"));
+    }
+
+    let configured_part_size = MIN_PART_SIZE;
+    let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
+    part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
+
+    let part_size = part_size_flt as i64;
+    if part_size == 0 {
+        return Ok(MIN_PART_SIZE);
+    }
+    Ok(part_size)
+}

crates/ecstore/src/services/tier/warm_backend_r2.rs

@@ -0,0 +1,154 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use crate::client::{
+    admin_handler_utils::AdminError,
+    api_put_object::PutObjectOptions,
+    credentials::{Credentials, SignatureType, Static, Value},
+    transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
+};
+use crate::services::tier::{
+    tier_config::TierR2,
+    warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend_s3::WarmBackendS3,
+};
+use tracing::warn;
+
+const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+const MAX_PARTS_COUNT: i64 = 10000;
+const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
+
+pub struct WarmBackendR2(WarmBackendS3);
+
+impl WarmBackendR2 {
+    pub async fn new(conf: &TierR2, tier: &str) -> Result<Self, std::io::Error> {
+        if conf.access_key == "" || conf.secret_key == "" {
+            return Err(std::io::Error::other("both access and secret keys are required"));
+        }
+
+        if conf.bucket == "" {
+            return Err(std::io::Error::other("no bucket name was provided"));
+        }
+
+        let u = match url::Url::parse(&conf.endpoint) {
+            Ok(u) => u,
+            Err(e) => {
+                return Err(std::io::Error::other(e.to_string()));
+            }
+        };
+
+        let creds = Credentials::new(Static(Value {
+            access_key_id: conf.access_key.clone(),
+            secret_access_key: conf.secret_key.clone(),
+            session_token: "".to_string(),
+            signer_type: SignatureType::SignatureV4,
+            ..Default::default()
+        }));
+        let opts = Options {
+            creds,
+            secure: u.scheme() == "https",
+            //transport: GLOBAL_RemoteTargetTransport,
+            trailing_headers: true,
+            region: conf.region.clone(),
+            ..Default::default()
+        };
+        let scheme = u.scheme();
+        let default_port = if scheme == "https" { 443 } else { 80 };
+        let host = u
+            .host_str()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "r2").await?;
+
+        let client = Arc::new(client);
+        let core = TransitionCore(Arc::clone(&client));
+        Ok(Self(WarmBackendS3 {
+            client,
+            core,
+            bucket: conf.bucket.clone(),
+            prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
+            storage_class: "".to_string(),
+        }))
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendR2 {
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        let part_size = optimal_part_size(length)?;
+        let client = self.0.client.clone();
+        let res = client
+            .put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
+                let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
+                opts.part_size = part_size as u64;
+                opts.disable_content_sha256 = true;
+                opts
+            })
+            .await?;
+        //self.ToObjectError(err, object)
+        Ok(res.version_id)
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        self.put_with_meta(object, r, length, HashMap::new()).await
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        self.0.get(object, rv, opts).await
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        self.0.remove(object, rv).await
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        self.0.in_use().await
+    }
+}
+
+fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
+    let mut object_size = object_size;
+    if object_size == -1 {
+        object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
+    }
+
+    if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+        return Err(std::io::Error::other("entity too large"));
+    }
+
+    let configured_part_size = MIN_PART_SIZE;
+    let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
+    part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
+
+    let part_size = part_size_flt as i64;
+    if part_size == 0 {
+        return Ok(MIN_PART_SIZE);
+    }
+    Ok(part_size)
+}

crates/ecstore/src/services/tier/warm_backend_rustfs.rs

@@ -0,0 +1,183 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use crate::client::{
+    admin_handler_utils::AdminError,
+    api_put_object::PutObjectOptions,
+    credentials::{Credentials, SignatureType, Static, Value},
+    transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
+};
+use crate::services::tier::{
+    tier_config::TierRustFS,
+    warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend_s3::WarmBackendS3,
+};
+
+const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+const MAX_PARTS_COUNT: i64 = 10000;
+const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
+
+pub struct WarmBackendRustFS(WarmBackendS3);
+
+impl WarmBackendRustFS {
+    pub async fn new(conf: &TierRustFS, tier: &str) -> Result<Self, std::io::Error> {
+        if conf.access_key == "" || conf.secret_key == "" {
+            return Err(std::io::Error::other("both access and secret keys are required"));
+        }
+
+        if conf.bucket == "" {
+            return Err(std::io::Error::other("no bucket name was provided"));
+        }
+
+        let u = match url::Url::parse(&conf.endpoint) {
+            Ok(u) => u,
+            Err(e) => return Err(std::io::Error::other(e)),
+        };
+
+        let creds = Credentials::new(Static(Value {
+            access_key_id: conf.access_key.clone(),
+            secret_access_key: conf.secret_key.clone(),
+            session_token: "".to_string(),
+            signer_type: SignatureType::SignatureV4,
+            ..Default::default()
+        }));
+        let opts = Options {
+            creds,
+            secure: u.scheme() == "https",
+            //transport: GLOBAL_RemoteTargetTransport,
+            trailing_headers: true,
+            region: conf.region.clone(),
+            ..Default::default()
+        };
+        let scheme = u.scheme();
+        let default_port = if scheme == "https" { 443 } else { 80 };
+        let host = u
+            .host_str()
+            .ok_or_else(|| std::io::Error::other("endpoint URL must include a host"))?;
+        let client = TransitionClient::new(&format!("{host}:{}", u.port().unwrap_or(default_port)), opts, "rustfs").await?;
+
+        let client = Arc::new(client);
+        let core = TransitionCore(Arc::clone(&client));
+        Ok(Self(WarmBackendS3 {
+            client,
+            core,
+            bucket: conf.bucket.clone(),
+            prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
+            storage_class: "".to_string(),
+        }))
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendRustFS {
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        let part_size = optimal_part_size(length)?;
+        let client = self.0.client.clone();
+        let res = client
+            .put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
+                let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
+                opts.part_size = part_size as u64;
+                opts.disable_content_sha256 = true;
+                opts
+            })
+            .await?;
+        //self.ToObjectError(err, object)
+        Ok(res.version_id)
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        self.put_with_meta(object, r, length, HashMap::new()).await
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        self.0.get(object, rv, opts).await
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        self.0.remove(object, rv).await
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        self.0.in_use().await
+    }
+}
+
+fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
+    let mut object_size = object_size;
+    if object_size == -1 {
+        object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
+    }
+
+    if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+        return Err(std::io::Error::other("entity too large"));
+    }
+
+    let configured_part_size = MIN_PART_SIZE;
+    let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
+    part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
+
+    let part_size = part_size_flt as i64;
+    if part_size == 0 {
+        return Ok(MIN_PART_SIZE);
+    }
+    Ok(part_size)
+}
+
+#[cfg(test)]
+mod tests {
+    use futures::FutureExt;
+    use std::panic::AssertUnwindSafe;
+
+    use super::*;
+
+    fn rustfs_tier(endpoint: &str) -> TierRustFS {
+        TierRustFS {
+            endpoint: endpoint.to_string(),
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            bucket: "bucket".to_string(),
+            ..Default::default()
+        }
+    }
+
+    #[tokio::test]
+    async fn new_returns_error_when_endpoint_has_no_host() {
+        let conf = rustfs_tier("rustfs://");
+
+        let outcome = AssertUnwindSafe(WarmBackendRustFS::new(&conf, "tier")).catch_unwind().await;
+
+        let result = outcome.expect("initialization should return an error instead of panicking");
+        let err = match result {
+            Ok(_) => panic!("endpoint without host must be rejected"),
+            Err(err) => err,
+        };
+        assert!(err.to_string().contains("host"), "expected host validation error, got: {err}");
+    }
+}

crates/ecstore/src/services/tier/warm_backend_s3.rs

@@ -0,0 +1,208 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use url::Url;
+
+use crate::client::{
+    api_get_options::GetObjectOptions,
+    api_put_object::PutObjectOptions,
+    api_remove::RemoveObjectOptions,
+    credentials::{Credentials, SignatureType, Static, Value},
+    transition_api::{Options, TransitionClient, TransitionCore},
+    transition_api::{ReadCloser, ReaderImpl},
+};
+use crate::error::ErrorResponse;
+use crate::error::error_resp_to_object_err;
+use crate::services::tier::{
+    tier_config::TierS3,
+    warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+};
+use rustfs_utils::egress::validate_outbound_url;
+use rustfs_utils::path::SLASH_SEPARATOR;
+
+pub struct WarmBackendS3 {
+    pub client: Arc<TransitionClient>,
+    pub core: TransitionCore,
+    pub bucket: String,
+    pub prefix: String,
+    pub storage_class: String,
+}
+
+impl WarmBackendS3 {
+    pub async fn new(conf: &TierS3, tier: &str) -> Result<Self, std::io::Error> {
+        let u = match Url::parse(&conf.endpoint) {
+            Ok(u) => u,
+            Err(err) => {
+                return Err(std::io::Error::other(err.to_string()));
+            }
+        };
+        validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
+
+        if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
+            || conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
+        {
+            return Err(std::io::Error::other("both the token file and the role ARN are required"));
+        } else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
+            return Err(std::io::Error::other("both the access and secret keys are required"));
+        } else if conf.aws_role
+            && (conf.aws_role_web_identity_token_file != ""
+                || conf.aws_role_arn != ""
+                || conf.access_key != ""
+                || conf.secret_key != "")
+        {
+            return Err(std::io::Error::other(
+                "AWS Role cannot be activated with static credentials or the web identity token file",
+            ));
+        } else if conf.bucket == "" {
+            return Err(std::io::Error::other("no bucket name was provided"));
+        }
+
+        let creds: Credentials<Static>;
+
+        if conf.access_key != "" && conf.secret_key != "" {
+            //creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, "");
+            creds = Credentials::new(Static(Value {
+                access_key_id: conf.access_key.clone(),
+                secret_access_key: conf.secret_key.clone(),
+                session_token: "".to_string(),
+                signer_type: SignatureType::SignatureV4,
+                ..Default::default()
+            }));
+        } else {
+            return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
+        }
+        let opts = Options {
+            creds,
+            secure: u.scheme() == "https",
+            //transport: GLOBAL_RemoteTargetTransport,
+            region: conf.region.clone(),
+            ..Default::default()
+        };
+        let host = u
+            .host()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client = TransitionClient::new(&host.to_string(), opts, "s3").await?;
+
+        let client = Arc::new(client);
+        let core = TransitionCore(Arc::clone(&client));
+        Ok(Self {
+            client,
+            core,
+            bucket: conf.bucket.clone(),
+            prefix: conf.prefix.clone().trim_matches('/').to_string(),
+            storage_class: conf.storage_class.clone(),
+        })
+    }
+
+    pub fn get_dest(&self, object: &str) -> String {
+        let mut dest_obj = object.to_string();
+        if self.prefix != "" {
+            dest_obj = format!("{}/{}", &self.prefix, object);
+        }
+        return dest_obj;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn new_rejects_loopback_endpoint_before_network_setup() {
+        let conf = TierS3 {
+            endpoint: "https://127.0.0.1:9000".to_string(),
+            bucket: "tier-bucket".to_string(),
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            region: "us-east-1".to_string(),
+            ..Default::default()
+        };
+
+        match WarmBackendS3::new(&conf, "tier").await {
+            Ok(_) => panic!("loopback endpoint should be rejected"),
+            Err(err) => assert!(err.to_string().contains("not allowed")),
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendS3 {
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        let client = self.client.clone();
+        let res = client
+            .put_object(&self.bucket, &self.get_dest(object), r, length, &{
+                let mut opts = build_transition_put_options(self.storage_class.clone(), meta);
+                opts.send_content_md5 = true;
+                opts
+            })
+            .await?;
+        Ok(res.version_id)
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        self.put_with_meta(object, r, length, HashMap::new()).await
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        let mut gopts = GetObjectOptions::default();
+
+        if rv != "" {
+            gopts.version_id = rv.to_string();
+        }
+        if opts.start_offset >= 0 && opts.length > 0 {
+            if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset + opts.length - 1) {
+                return Err(std::io::Error::other(err));
+            }
+        }
+        let c = TransitionCore(Arc::clone(&self.client));
+        let (_, _, r) = c.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
+
+        Ok(r)
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        let mut ropts = RemoveObjectOptions::default();
+        if rv != "" {
+            ropts.version_id = rv.to_string();
+        }
+        let client = self.client.clone();
+        match client.remove_object(&self.bucket, &self.get_dest(object), ropts).await {
+            None => Ok(()),
+            Some(err) => Err(std::io::Error::other(err)),
+        }
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        let result = self
+            .core
+            .list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
+            .await?;
+
+        Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
+    }
+}

crates/ecstore/src/services/tier/warm_backend_s3sdk.rs

@@ -0,0 +1,200 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use url::Url;
+
+use aws_config::meta::region::RegionProviderChain;
+use aws_sdk_s3::Client;
+use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::primitives::ByteStream;
+
+use crate::client::{
+    api_get_options::GetObjectOptions,
+    api_put_object::PutObjectOptions,
+    api_remove::RemoveObjectOptions,
+    transition_api::{ReadCloser, ReaderImpl},
+};
+use crate::error::ErrorResponse;
+use crate::error::error_resp_to_object_err;
+use crate::services::tier::{
+    tier_config::TierS3,
+    warm_backend::{WarmBackend, WarmBackendGetOpts},
+};
+
+pub struct WarmBackendS3 {
+    pub client: Arc<Client>,
+    pub bucket: String,
+    pub prefix: String,
+    pub storage_class: String,
+}
+
+impl WarmBackendS3 {
+    pub async fn new(conf: &TierS3, tier: &str) -> Result<Self, std::io::Error> {
+        let u = match Url::parse(&conf.endpoint) {
+            Ok(u) => u,
+            Err(err) => {
+                return Err(std::io::Error::other(err.to_string()));
+            }
+        };
+
+        if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
+            || conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
+        {
+            return Err(std::io::Error::other("both the token file and the role ARN are required"));
+        } else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
+            return Err(std::io::Error::other("both the access and secret keys are required"));
+        } else if conf.aws_role
+            && (conf.aws_role_web_identity_token_file != ""
+                || conf.aws_role_arn != ""
+                || conf.access_key != ""
+                || conf.secret_key != "")
+        {
+            return Err(std::io::Error::other(
+                "AWS Role cannot be activated with static credentials or the web identity token file",
+            ));
+        } else if conf.bucket == "" {
+            return Err(std::io::Error::other("no bucket name was provided"));
+        }
+
+        let creds;
+        if conf.access_key != "" && conf.secret_key != "" {
+            creds = Credentials::new(
+                conf.access_key.clone(), // access_key_id
+                conf.secret_key.clone(), // secret_access_key
+                None,                    // session_token (optional)
+                None,
+                "Static",
+            );
+        } else {
+            return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
+        }
+        let region_provider = RegionProviderChain::default_provider().or_else(Region::new(conf.region.clone()));
+        #[allow(deprecated)]
+        let config = aws_config::from_env()
+            .endpoint_url(conf.endpoint.clone())
+            .region(region_provider)
+            .credentials_provider(creds)
+            .load()
+            .await;
+        let client = Client::new(&config);
+        let client = Arc::new(client);
+        Ok(Self {
+            client,
+            bucket: conf.bucket.clone(),
+            prefix: conf.prefix.clone().trim_matches('/').to_string(),
+            storage_class: conf.storage_class.clone(),
+        })
+    }
+
+    pub fn get_dest(&self, object: &str) -> String {
+        let mut dest_obj = object.to_string();
+        if self.prefix != "" {
+            dest_obj = format!("{}/{}", &self.prefix, object);
+        }
+        return dest_obj;
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendS3 {
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        let client = self.client.clone();
+        let Ok(res) = client
+            .put_object()
+            .bucket(&self.bucket)
+            .key(&self.get_dest(object))
+            .body(match r {
+                ReaderImpl::Body(content_body) => ByteStream::from(content_body.to_vec()),
+                ReaderImpl::ObjectBody(mut content_body) => ByteStream::from(content_body.read_all().await?),
+            })
+            .send()
+            .await
+        else {
+            return Err(std::io::Error::other("put_object error"));
+        };
+
+        Ok(res.version_id().unwrap_or("").to_string())
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        self.put_with_meta(object, r, length, HashMap::new()).await
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        let client = self.client.clone();
+        let mut req = client.get_object().bucket(&self.bucket).key(&self.get_dest(object));
+
+        if !rv.is_empty() {
+            req = req.version_id(rv);
+        }
+
+        if opts.start_offset >= 0 && opts.length > 0 {
+            let end = opts
+                .start_offset
+                .checked_add(opts.length)
+                .and_then(|v| v.checked_sub(1))
+                .ok_or_else(|| std::io::Error::other("invalid range: overflow"))?;
+            req = req.range(format!("bytes={}-{}", opts.start_offset, end));
+        }
+
+        let res = req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
+
+        Ok(ReadCloser::new(std::io::Cursor::new(
+            res.body.collect().await.map(|data| data.into_bytes().to_vec())?,
+        )))
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        let client = self.client.clone();
+        let mut req = client.delete_object().bucket(&self.bucket).key(&self.get_dest(object));
+
+        if !rv.is_empty() {
+            req = req.version_id(rv);
+        }
+
+        req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
+
+        Ok(())
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        let client = self.client.clone();
+        let Ok(res) = client
+            .list_objects_v2()
+            .bucket(&self.bucket)
+            //.max_keys(10)
+            //.into_paginator()
+            .send()
+            .await
+        else {
+            return Err(std::io::Error::other("list_objects_v2 error"));
+        };
+
+        Ok(res.common_prefixes.unwrap_or_default().len() > 0 || res.contents.unwrap_or_default().len() > 0)
+    }
+}

crates/ecstore/src/services/tier/warm_backend_tencent.rs

@@ -0,0 +1,155 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use crate::client::{
+    admin_handler_utils::AdminError,
+    api_put_object::PutObjectOptions,
+    credentials::{Credentials, SignatureType, Static, Value},
+    transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
+};
+use crate::services::tier::{
+    tier_config::TierTencent,
+    warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend_s3::WarmBackendS3,
+};
+use tracing::warn;
+
+const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+const MAX_PARTS_COUNT: i64 = 10000;
+const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
+
+pub struct WarmBackendTencent(WarmBackendS3);
+
+impl WarmBackendTencent {
+    pub async fn new(conf: &TierTencent, tier: &str) -> Result<Self, std::io::Error> {
+        if conf.access_key == "" || conf.secret_key == "" {
+            return Err(std::io::Error::other("both access and secret keys are required"));
+        }
+
+        if conf.bucket == "" {
+            return Err(std::io::Error::other("no bucket name was provided"));
+        }
+
+        let u = match url::Url::parse(&conf.endpoint) {
+            Ok(u) => u,
+            Err(e) => {
+                return Err(std::io::Error::other(e.to_string()));
+            }
+        };
+
+        let creds = Credentials::new(Static(Value {
+            access_key_id: conf.access_key.clone(),
+            secret_access_key: conf.secret_key.clone(),
+            session_token: "".to_string(),
+            signer_type: SignatureType::SignatureV4,
+            ..Default::default()
+        }));
+        let opts = Options {
+            creds,
+            secure: u.scheme() == "https",
+            //transport: GLOBAL_RemoteTargetTransport,
+            trailing_headers: true,
+            region: conf.region.clone(),
+            bucket_lookup: BucketLookupType::BucketLookupDNS,
+            ..Default::default()
+        };
+        let scheme = u.scheme();
+        let default_port = if scheme == "https" { 443 } else { 80 };
+        let host = u
+            .host_str()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "tencent").await?;
+
+        let client = Arc::new(client);
+        let core = TransitionCore(Arc::clone(&client));
+        Ok(Self(WarmBackendS3 {
+            client,
+            core,
+            bucket: conf.bucket.clone(),
+            prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
+            storage_class: "".to_string(),
+        }))
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendTencent {
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        let part_size = optimal_part_size(length)?;
+        let client = self.0.client.clone();
+        let res = client
+            .put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
+                let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
+                opts.part_size = part_size as u64;
+                opts.disable_content_sha256 = true;
+                opts
+            })
+            .await?;
+        //self.ToObjectError(err, object)
+        Ok(res.version_id)
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        self.put_with_meta(object, r, length, HashMap::new()).await
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        self.0.get(object, rv, opts).await
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        self.0.remove(object, rv).await
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        self.0.in_use().await
+    }
+}
+
+fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
+    let mut object_size = object_size;
+    if object_size == -1 {
+        object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
+    }
+
+    if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+        return Err(std::io::Error::other("entity too large"));
+    }
+
+    let configured_part_size = MIN_PART_SIZE;
+    let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
+    part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
+
+    let part_size = part_size_flt as i64;
+    if part_size == 0 {
+        return Ok(MIN_PART_SIZE);
+    }
+    Ok(part_size)
+}