fix(storage): harden rebalance and decommission state (#3730)

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -664,6 +664,27 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn start_decommission(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::StartDecommissionRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::StartDecommissionResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
+    async fn cancel_decommission(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::CancelDecommissionRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::CancelDecommissionResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
+    async fn clear_decommission(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::ClearDecommissionRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::ClearDecommissionResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn get_metrics(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::GetMetricsRequest>,

crates/ecstore/src/notification_sys.rs

@@ -109,6 +109,14 @@ impl NotificationSys {
         self.all_peer_clients[idx].clone()
     }
 
+    pub fn peer_client_for_grid_host(&self, grid_host: &str) -> Option<PeerRestClient> {
+        self.all_peer_clients
+            .iter()
+            .flatten()
+            .find(|client| client.grid_host == grid_host)
+            .cloned()
+    }
+
     pub async fn delete_policy(&self, policy_name: &str) -> Vec<NotificationPeerErr> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for client in self.peer_clients.iter() {
@@ -1215,6 +1223,24 @@ mod tests {
         assert!(msg.contains("local save failed"));
     }
 
+    #[test]
+    fn peer_client_for_grid_host_matches_exact_grid_host() {
+        let sys = NotificationSys {
+            peer_clients: Vec::new(),
+            all_peer_clients: vec![Some(PeerRestClient::new(
+                "127.0.0.1:9000".to_string().try_into().expect("peer host should parse"),
+                "http://127.0.0.1:9000".to_string(),
+            ))],
+            peer_admin_caches: Vec::new(),
+        };
+
+        let client = sys
+            .peer_client_for_grid_host("http://127.0.0.1:9000")
+            .expect("matching grid host should return peer client");
+        assert_eq!(client.grid_host, "http://127.0.0.1:9000");
+        assert!(sys.peer_client_for_grid_host("http://node-b:9000").is_none());
+    }
+
     #[test]
     fn load_rebalance_meta_aggregate_failures_return_error() {
         let err = aggregate_notification_failures(

crates/ecstore/src/rebalance/control.rs

@@ -1,10 +1,10 @@
 use super::meta::{
-    clone_first_arc, clone_rebalance_pool_stats, defer_bucket_in_rebalance_queue, ensure_valid_rebalance_pool_index,
-    invalid_rebalance_pool_index_error, is_rebalance_conflicting_with_decommission, mark_rebalance_bucket_done,
-    merge_rebalance_meta, percent_free_ratio, rebalance_metadata_not_initialized_error, record_rebalance_cleanup_warning_in_meta,
-    record_rebalance_stop_propagation_snapshot, resolve_next_rebalance_bucket, rollback_rebalance_start_meta_snapshot_for_id,
-    should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot_for_id,
-    validate_init_rebalance_state,
+    RebalanceMetaMergeOutcome, clone_first_arc, clone_rebalance_pool_stats, defer_bucket_in_rebalance_queue,
+    ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error, is_rebalance_conflicting_with_decommission,
+    mark_rebalance_bucket_done, merge_rebalance_meta, percent_free_ratio, rebalance_metadata_not_initialized_error,
+    record_rebalance_cleanup_warning_in_meta, record_rebalance_stop_propagation_snapshot, resolve_next_rebalance_bucket,
+    rollback_rebalance_start_meta_snapshot_for_id, should_accept_rebalance_stats_update, should_pool_participate,
+    stop_rebalance_meta_snapshot_for_id, validate_init_rebalance_state,
 };
 use super::worker::{
     rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result, resolve_rebalance_meta_load_result,
@@ -27,6 +27,18 @@ use time::OffsetDateTime;
 use tracing::{debug, info};
 use uuid::Uuid;
 
+pub(super) fn validate_rebalance_disk_stats_coverage(disk_stats: &[DiskStat]) -> Result<()> {
+    for (idx, disk_stat) in disk_stats.iter().enumerate() {
+        if disk_stat.total_space == 0 {
+            return Err(Error::other(format!(
+                "rebalance storage info is incomplete: pool {idx} has no reported capacity"
+            )));
+        }
+    }
+
+    Ok(())
+}
+
 impl ECStore {
     pub(super) async fn save_rebalance_meta_with_merge<S>(
         &self,
@@ -50,7 +62,9 @@ impl ECStore {
         let mut merged = RebalanceMeta::new();
         match merged.load_with_opts(pool.clone(), opts.clone()).await {
             Ok(()) => {
-                merge_rebalance_meta(&mut merged, local_snapshot);
+                if merge_rebalance_meta(&mut merged, local_snapshot) == RebalanceMetaMergeOutcome::RejectedActiveConflict {
+                    return Err(Error::RebalanceAlreadyRunning);
+                }
             }
             Err(Error::ConfigNotFound) => {
                 merged = local_snapshot.clone();
@@ -90,6 +104,10 @@ impl ECStore {
                 "Loaded rebalance metadata"
             );
         } else {
+            {
+                let mut rebalance_meta = self.rebalance_meta.write().await;
+                *rebalance_meta = None;
+            }
             debug!(
                 event = EVENT_REBALANCE_STATE,
                 component = LOG_COMPONENT_ECSTORE,
@@ -191,6 +209,7 @@ impl ECStore {
         }
 
         let percent_free_goal = percent_free_ratio(total_free, total_cap);
+        validate_rebalance_disk_stats_coverage(&disk_stats)?;
 
         let mut pool_stats = Vec::with_capacity(self.pools.len());
 
@@ -421,6 +440,15 @@ impl ECStore {
         false
     }
 
+    pub async fn pool_rebalance_status(&self, pool_index: usize) -> (RebalStatus, bool) {
+        let rebalance_meta = self.rebalance_meta.read().await;
+        rebalance_meta
+            .as_ref()
+            .and_then(|meta| meta.pool_stats.get(pool_index))
+            .map(|pool_stat| (pool_stat.info.status, pool_stat.info.stopping))
+            .unwrap_or_default()
+    }
+
     pub async fn current_rebalance_id(&self) -> Option<String> {
         let rebalance_meta = self.rebalance_meta.read().await;
         rebalance_meta

crates/ecstore/src/rebalance/meta.rs

@@ -457,7 +457,10 @@ pub(super) fn complete_rebalance_pools_at_goal(meta: &mut RebalanceMeta, now: Of
     let mut changed = false;
 
     for pool_stat in meta.pool_stats.iter_mut() {
-        if !is_rebalance_pool_started(pool_stat) || has_deferred_rebalance_error(pool_stat) {
+        if !is_rebalance_pool_started(pool_stat)
+            || has_deferred_rebalance_error(pool_stat)
+            || has_rebalance_cleanup_warnings(pool_stat)
+        {
             continue;
         }
 
@@ -481,7 +484,7 @@ pub(super) fn complete_rebalance_pools_with_empty_queue(meta: &mut RebalanceMeta
     let mut changed = false;
 
     for pool_stat in meta.pool_stats.iter_mut() {
-        if !is_rebalance_pool_started(pool_stat) || !pool_stat.buckets.is_empty() {
+        if !is_rebalance_pool_started(pool_stat) || !pool_stat.buckets.is_empty() || has_rebalance_cleanup_warnings(pool_stat) {
             continue;
         }
 
@@ -501,6 +504,11 @@ pub(super) fn has_deferred_rebalance_error(pool_stat: &RebalanceStats) -> bool {
         .as_deref()
         .is_some_and(|last_error| last_error.starts_with(REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX))
 }
+
+pub(super) fn has_rebalance_cleanup_warnings(pool_stat: &RebalanceStats) -> bool {
+    pool_stat.cleanup_warnings.count > 0
+}
+
 pub(super) fn clone_first_arc<T>(values: &[Arc<T>], err_msg: &str) -> Result<Arc<T>> {
     values.first().cloned().ok_or_else(|| Error::other(err_msg))
 }
@@ -608,7 +616,7 @@ pub(super) fn validate_init_rebalance_state(decommission_running: bool, current_
     if !ensure_rebalance_not_decommissioning(decommission_running) {
         return Err(Error::DecommissionAlreadyRunning);
     }
-    if current_meta.is_some_and(is_rebalance_in_progress) {
+    if current_meta.is_some_and(|meta| !is_rebalance_meta_replaceable_for_new_id(meta)) {
         return Err(Error::RebalanceAlreadyRunning);
     }
 
@@ -812,14 +820,29 @@ pub(super) fn merge_rebalance_pool_stats(remote: &mut RebalanceStats, local: &Re
     }
 }
 
-pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &RebalanceMeta) {
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum RebalanceMetaMergeOutcome {
+    Merged,
+    Replaced,
+    RejectedActiveConflict,
+}
+
+pub(super) fn is_rebalance_meta_replaceable_for_new_id(meta: &RebalanceMeta) -> bool {
+    meta.stopped_at.is_some() || !is_rebalance_in_progress(meta)
+}
+
+pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &RebalanceMeta) -> RebalanceMetaMergeOutcome {
     if remote.id.is_empty() {
         *remote = local.clone();
-        return;
+        return RebalanceMetaMergeOutcome::Replaced;
     }
 
     if !local.id.is_empty() && remote.id != local.id {
-        return;
+        if is_rebalance_meta_replaceable_for_new_id(remote) {
+            *remote = local.clone();
+            return RebalanceMetaMergeOutcome::Replaced;
+        }
+        return RebalanceMetaMergeOutcome::RejectedActiveConflict;
     }
 
     remote.percent_free_goal = local.percent_free_goal;
@@ -837,6 +860,8 @@ pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &Rebalance
             merge_rebalance_pool_stats(remote_pool_stat, local_pool_stat);
         }
     }
+
+    RebalanceMetaMergeOutcome::Merged
 }
 
 pub(super) fn mark_started_rebalance_pools_stopped(meta: &mut RebalanceMeta, stop_time: OffsetDateTime) {

crates/ecstore/src/rebalance/rebalance_unit_tests.rs

@@ -13,19 +13,21 @@
 // limitations under the License.
 
 use super::REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX;
+use super::control::validate_rebalance_disk_stats_coverage;
 use super::meta::{
-    RebalanceTerminalEvent, apply_rebalance_save_option, apply_rebalance_terminal_event, apply_stopped_at,
-    classify_rebalance_terminal_event, clone_arc_by_index, clone_first_arc, clone_rebalance_pool_stats,
+    RebalanceMetaMergeOutcome, RebalanceTerminalEvent, apply_rebalance_save_option, apply_rebalance_terminal_event,
+    apply_stopped_at, classify_rebalance_terminal_event, clone_arc_by_index, clone_first_arc, clone_rebalance_pool_stats,
     complete_rebalance_pools_at_goal, complete_rebalance_pools_with_empty_queue, defer_bucket_in_rebalance_queue,
     ensure_rebalance_not_decommissioning, ensure_valid_rebalance_pool_index, first_rebalance_bucket,
     has_deferred_rebalance_error, is_rebalance_actively_running, is_rebalance_conflicting_with_decommission,
-    is_rebalance_in_progress, is_rebalance_stopped_terminal_event, mark_rebalance_bucket_done, merge_rebalance_bucket_lists,
-    merge_rebalance_meta, next_rebal_bucket_from_stat, percent_free_ratio, rebalance_goal_reached,
-    rebalance_meta_load_no_data_error, rebalance_meta_load_unknown_format_error, rebalance_meta_load_unknown_version_error,
-    record_rebalance_cleanup_warning_in_meta, remove_rebalanced_buckets_from_queue, resolve_next_rebalance_bucket,
-    resolve_rebalance_participants, should_accept_rebalance_stats_update, should_ignore_rebalance_data_usage_cache,
-    should_pool_participate, should_preserve_rebalance_stopped_state, should_skip_start_rebalance, stop_rebalance_meta_snapshot,
-    stop_rebalance_state, take_bucket_from_rebalance_queue, validate_init_rebalance_state, validate_start_rebalance_state,
+    is_rebalance_in_progress, is_rebalance_meta_replaceable_for_new_id, is_rebalance_stopped_terminal_event,
+    mark_rebalance_bucket_done, merge_rebalance_bucket_lists, merge_rebalance_meta, next_rebal_bucket_from_stat,
+    percent_free_ratio, rebalance_goal_reached, rebalance_meta_load_no_data_error, rebalance_meta_load_unknown_format_error,
+    rebalance_meta_load_unknown_version_error, record_rebalance_cleanup_warning_in_meta, remove_rebalanced_buckets_from_queue,
+    resolve_next_rebalance_bucket, resolve_rebalance_participants, should_accept_rebalance_stats_update,
+    should_ignore_rebalance_data_usage_cache, should_pool_participate, should_preserve_rebalance_stopped_state,
+    should_skip_start_rebalance, stop_rebalance_meta_snapshot, stop_rebalance_state, take_bucket_from_rebalance_queue,
+    validate_init_rebalance_state, validate_start_rebalance_state,
 };
 use super::migration::{
     MigrationBackend, MigrationVersionResult, migrate_entry_version, migrate_entry_version_with_retry_wait,
@@ -44,8 +46,8 @@ use super::worker::{
     wait_rebalance_listing_retry, with_rebalance_entry_context,
 };
 use super::{
-    GetObjectReader, ObjectInfo, ObjectOptions, RebalSaveOpt, RebalStatus, RebalanceBucketOutcome, RebalanceCleanupWarnings,
-    RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta, RebalanceStats,
+    DiskStat, GetObjectReader, ObjectInfo, ObjectOptions, RebalSaveOpt, RebalStatus, RebalanceBucketOutcome,
+    RebalanceCleanupWarnings, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta, RebalanceStats,
 };
 use crate::data_movement;
 use crate::data_usage::DATA_USAGE_CACHE_NAME;
@@ -1207,7 +1209,7 @@ fn test_merge_rebalance_meta_preserves_updates_from_multiple_pools() {
         ..Default::default()
     };
 
-    merge_rebalance_meta(&mut remote, &local);
+    assert_eq!(merge_rebalance_meta(&mut remote, &local), RebalanceMetaMergeOutcome::Merged);
 
     assert_eq!(remote.pool_stats[0].num_versions, 4);
     assert_eq!(remote.pool_stats[0].object, "remote-object");
@@ -1220,6 +1222,163 @@ fn test_merge_rebalance_meta_preserves_updates_from_multiple_pools() {
     assert_eq!(remote.pool_stats[1].cleanup_warnings.last_at, Some(warning_at));
 }
 
+#[test]
+fn test_merge_rebalance_meta_replaces_terminal_metadata_for_new_rebalance() {
+    let old_completed_at = OffsetDateTime::from_unix_timestamp(1_000).expect("valid old completion timestamp");
+    let new_started_at = OffsetDateTime::from_unix_timestamp(2_000).expect("valid new start timestamp");
+    let mut remote = RebalanceMeta {
+        id: "old-rebalance".to_string(),
+        percent_free_goal: 0.25,
+        pool_stats: vec![
+            RebalanceStats {
+                buckets: Vec::new(),
+                rebalanced_buckets: vec!["bucket-a".to_string()],
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Completed,
+                    end_time: Some(old_completed_at),
+                    ..Default::default()
+                },
+                num_versions: 7,
+                bytes: 700,
+                ..Default::default()
+            },
+            RebalanceStats::default(),
+        ],
+        ..Default::default()
+    };
+    let local = RebalanceMeta {
+        id: "new-rebalance".to_string(),
+        percent_free_goal: 0.5,
+        pool_stats: vec![
+            RebalanceStats {
+                buckets: vec!["bucket-a".to_string()],
+                participating: true,
+                info: RebalanceInfo {
+                    start_time: Some(new_started_at),
+                    status: RebalStatus::Started,
+                    ..Default::default()
+                },
+                ..Default::default()
+            },
+            RebalanceStats {
+                buckets: vec!["bucket-a".to_string()],
+                participating: true,
+                info: RebalanceInfo {
+                    start_time: Some(new_started_at),
+                    status: RebalStatus::Started,
+                    ..Default::default()
+                },
+                ..Default::default()
+            },
+            RebalanceStats::default(),
+        ],
+        ..Default::default()
+    };
+
+    assert_eq!(merge_rebalance_meta(&mut remote, &local), RebalanceMetaMergeOutcome::Replaced);
+
+    assert_eq!(remote.id, "new-rebalance");
+    assert_eq!(remote.percent_free_goal, 0.5);
+    assert_eq!(remote.pool_stats.len(), 3);
+    assert_eq!(remote.pool_stats[0].info.status, RebalStatus::Started);
+    assert_eq!(remote.pool_stats[0].buckets, vec!["bucket-a"]);
+    assert_eq!(remote.pool_stats[0].rebalanced_buckets, Vec::<String>::new());
+    assert_eq!(remote.pool_stats[0].num_versions, 0);
+    assert_eq!(remote.pool_stats[1].info.status, RebalStatus::Started);
+    assert!(remote.pool_stats[1].participating);
+}
+
+#[test]
+fn test_merge_rebalance_meta_preserves_active_metadata_for_different_rebalance_id() {
+    let old_started_at = OffsetDateTime::from_unix_timestamp(1_000).expect("valid old start timestamp");
+    let new_started_at = OffsetDateTime::from_unix_timestamp(2_000).expect("valid new start timestamp");
+    let mut remote = RebalanceMeta {
+        id: "active-rebalance".to_string(),
+        percent_free_goal: 0.25,
+        pool_stats: vec![RebalanceStats {
+            buckets: vec!["bucket-a".to_string()],
+            participating: true,
+            info: RebalanceInfo {
+                start_time: Some(old_started_at),
+                status: RebalStatus::Started,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
+    let local = RebalanceMeta {
+        id: "new-rebalance".to_string(),
+        percent_free_goal: 0.5,
+        pool_stats: vec![RebalanceStats {
+            buckets: vec!["bucket-b".to_string()],
+            participating: true,
+            info: RebalanceInfo {
+                start_time: Some(new_started_at),
+                status: RebalStatus::Started,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
+
+    assert_eq!(
+        merge_rebalance_meta(&mut remote, &local),
+        RebalanceMetaMergeOutcome::RejectedActiveConflict
+    );
+
+    assert_eq!(remote.id, "active-rebalance");
+    assert_eq!(remote.percent_free_goal, 0.25);
+    assert_eq!(remote.pool_stats.len(), 1);
+    assert_eq!(remote.pool_stats[0].buckets, vec!["bucket-a"]);
+    assert_eq!(remote.pool_stats[0].info.start_time, Some(old_started_at));
+}
+
+#[test]
+fn test_merge_rebalance_meta_replaces_stopped_started_metadata_for_new_rebalance() {
+    let stopped_at = OffsetDateTime::from_unix_timestamp(1_500).expect("valid stop timestamp");
+    let new_started_at = OffsetDateTime::from_unix_timestamp(2_000).expect("valid new start timestamp");
+    let mut remote = RebalanceMeta {
+        id: "stopped-rebalance".to_string(),
+        stopped_at: Some(stopped_at),
+        pool_stats: vec![RebalanceStats {
+            participating: true,
+            info: RebalanceInfo {
+                status: RebalStatus::Started,
+                stopping: true,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
+    let local = RebalanceMeta {
+        id: "new-rebalance".to_string(),
+        percent_free_goal: 0.5,
+        pool_stats: vec![RebalanceStats {
+            buckets: vec!["bucket-a".to_string()],
+            participating: true,
+            info: RebalanceInfo {
+                start_time: Some(new_started_at),
+                status: RebalStatus::Started,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
+
+    assert!(is_rebalance_meta_replaceable_for_new_id(&remote));
+    assert_eq!(merge_rebalance_meta(&mut remote, &local), RebalanceMetaMergeOutcome::Replaced);
+
+    assert_eq!(remote.id, "new-rebalance");
+    assert_eq!(remote.stopped_at, None);
+    assert_eq!(remote.pool_stats[0].info.status, RebalStatus::Started);
+    assert!(!remote.pool_stats[0].info.stopping);
+}
+
 #[test]
 fn test_merge_rebalance_meta_does_not_overwrite_failed_with_started_stats() {
     let now = OffsetDateTime::from_unix_timestamp(2_000).unwrap();
@@ -2202,6 +2361,7 @@ fn test_validate_init_rebalance_state_rejects_active_rebalance() {
 
 #[test]
 fn test_validate_init_rebalance_state_allows_terminal_or_missing_rebalance() {
+    let stopped_at = OffsetDateTime::now_utc();
     let completed = RebalanceMeta {
         pool_stats: vec![RebalanceStats {
             participating: true,
@@ -2213,9 +2373,23 @@ fn test_validate_init_rebalance_state_allows_terminal_or_missing_rebalance() {
         }],
         ..Default::default()
     };
+    let stopped_started = RebalanceMeta {
+        stopped_at: Some(stopped_at),
+        pool_stats: vec![RebalanceStats {
+            participating: true,
+            info: RebalanceInfo {
+                status: RebalStatus::Started,
+                stopping: true,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
 
     validate_init_rebalance_state(false, None).expect("missing rebalance meta should allow init");
     validate_init_rebalance_state(false, Some(&completed)).expect("terminal rebalance meta should allow init");
+    validate_init_rebalance_state(false, Some(&stopped_started)).expect("stopped rebalance meta should allow new init");
 }
 
 #[tokio::test]
@@ -2298,6 +2472,38 @@ fn test_rebalance_goal_not_reached_for_issue_3137_initial_imbalance() {
     assert!(!rebalance_goal_reached(pool0_free, pool0_capacity, 0, goal));
 }
 
+#[test]
+fn test_validate_rebalance_disk_stats_coverage_rejects_missing_pool_capacity() {
+    let disk_stats = vec![
+        DiskStat {
+            total_space: 1_000,
+            available_space: 100,
+        },
+        DiskStat::default(),
+    ];
+
+    let err = validate_rebalance_disk_stats_coverage(&disk_stats)
+        .expect_err("missing pool capacity should reject rebalance initialization");
+
+    assert!(err.to_string().contains("pool 1 has no reported capacity"));
+}
+
+#[test]
+fn test_validate_rebalance_disk_stats_coverage_accepts_all_pools() {
+    let disk_stats = vec![
+        DiskStat {
+            total_space: 1_000,
+            available_space: 100,
+        },
+        DiskStat {
+            total_space: 2_000,
+            available_space: 1_500,
+        },
+    ];
+
+    assert!(validate_rebalance_disk_stats_coverage(&disk_stats).is_ok());
+}
+
 #[test]
 fn test_complete_rebalance_pools_at_goal_marks_started_participants_completed() {
     let now = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
@@ -2943,7 +3149,7 @@ fn test_record_rebalance_cleanup_warning_in_meta_preserves_last_error() {
 }
 
 #[test]
-fn test_complete_rebalance_pools_with_empty_queue_preserves_cleanup_warnings() {
+fn test_complete_rebalance_pools_with_empty_queue_skips_cleanup_warnings() {
     let warning_at = OffsetDateTime::from_unix_timestamp(9_000).unwrap();
     let completed_at = OffsetDateTime::from_unix_timestamp(10_000).unwrap();
     let mut meta = RebalanceMeta {
@@ -2966,9 +3172,10 @@ fn test_complete_rebalance_pools_with_empty_queue_preserves_cleanup_warnings() {
         ..Default::default()
     };
 
-    assert!(complete_rebalance_pools_with_empty_queue(&mut meta, completed_at));
+    assert!(!complete_rebalance_pools_with_empty_queue(&mut meta, completed_at));
 
-    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Completed);
+    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started);
+    assert_eq!(meta.pool_stats[0].info.end_time, None);
     assert!(meta.pool_stats[0].info.last_error.is_none());
     assert_eq!(meta.pool_stats[0].cleanup_warnings.count, 1);
     assert_eq!(meta.pool_stats[0].cleanup_warnings.last_message.as_deref(), Some("cleanup failed"));

crates/ecstore/src/rebalance/runtime.rs

@@ -1,8 +1,9 @@
 use super::meta::{
     apply_rebalance_save_option, apply_rebalance_terminal_event, classify_rebalance_terminal_event, clone_first_arc,
     complete_rebalance_pools_at_goal, complete_rebalance_pools_with_empty_queue, ensure_valid_rebalance_pool_index,
-    has_deferred_rebalance_error, is_rebalance_in_progress, rebalance_goal_reached, resolve_rebalance_participants,
-    should_preserve_rebalance_stopped_state, should_skip_start_rebalance, validate_start_rebalance_state,
+    has_deferred_rebalance_error, has_rebalance_cleanup_warnings, is_rebalance_in_progress, rebalance_goal_reached,
+    resolve_rebalance_participants, should_preserve_rebalance_stopped_state, should_skip_start_rebalance,
+    validate_start_rebalance_state,
 };
 use super::worker::{
     resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result, resolve_rebalance_save_task_result,
@@ -211,7 +212,20 @@ impl ECStore {
                         if let Some(meta) = rebalance_meta.as_mut() {
                             let meta_stopped = meta.stopped_at.is_some();
                             if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
-                                if should_preserve_rebalance_stopped_state(
+                                if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
+                                    && has_rebalance_cleanup_warnings(pool_stat)
+                                {
+                                    pool_stat.info.stopping = false;
+                                    pool_stat.info.status = RebalStatus::Failed;
+                                    pool_stat.info.end_time = Some(now);
+                                    pool_stat.info.last_error = Some(
+                                        pool_stat
+                                            .cleanup_warnings
+                                            .last_message
+                                            .clone()
+                                            .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
+                                    );
+                                } else if should_preserve_rebalance_stopped_state(
                                     meta_stopped,
                                     pool_stat.info.status,
                                     &terminal_event,
@@ -513,6 +527,7 @@ impl ECStore {
             };
 
             if !has_deferred_rebalance_error(pool_stat)
+                && !has_rebalance_cleanup_warnings(pool_stat)
                 && rebalance_goal_reached(
                     pool_stat.init_free_space,
                     pool_stat.init_capacity,

crates/ecstore/src/rpc/peer_rest_client.rs

@@ -29,12 +29,13 @@ use rustfs_madmin::{
 };
 use rustfs_protos::evict_failed_connection;
 use rustfs_protos::proto_gen::node_service::{
-    DeleteBucketMetadataRequest, DeletePolicyRequest, DeleteServiceAccountRequest, DeleteUserRequest, GetCpusRequest,
-    GetLiveEventsRequest, GetMemInfoRequest, GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest,
-    GetProcInfoRequest, GetSeLinuxInfoRequest, GetSysConfigRequest, GetSysErrorsRequest, LoadBucketMetadataRequest,
-    LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
-    LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest,
-    ReloadSiteReplicationConfigRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest, StopRebalanceRequest,
+    CancelDecommissionRequest, ClearDecommissionRequest, DeleteBucketMetadataRequest, DeletePolicyRequest,
+    DeleteServiceAccountRequest, DeleteUserRequest, GetCpusRequest, GetLiveEventsRequest, GetMemInfoRequest, GetMetricsRequest,
+    GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest, GetSysConfigRequest,
+    GetSysErrorsRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
+    LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
+    LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ServerInfoRequest,
+    SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest,
     node_service_client::NodeServiceClient,
 };
 use rustfs_utils::XHost;
@@ -972,6 +973,79 @@ impl PeerRestClient {
         .await
     }
 
+    pub async fn start_decommission(&self, pool_indices: Vec<usize>) -> Result<()> {
+        self.finalize_result(
+            async {
+                let pool_indices = pool_indices
+                    .into_iter()
+                    .map(|idx| {
+                        u32::try_from(idx).map_err(|_| Error::other(format!("decommission pool index {idx} exceeds RPC range")))
+                    })
+                    .collect::<Result<Vec<_>>>()?;
+                let mut client = self.get_client().await?;
+                let request = Request::new(StartDecommissionRequest { pool_indices });
+
+                let response = client.start_decommission(request).await?.into_inner();
+                if !response.success {
+                    if let Some(msg) = response.error_info {
+                        return Err(Error::other(msg));
+                    }
+                    return Err(Error::other(""));
+                }
+
+                Ok(())
+            }
+            .await,
+        )
+        .await
+    }
+
+    pub async fn decommission_cancel(&self, pool_index: usize) -> Result<()> {
+        self.finalize_result(
+            async {
+                let pool_index = u32::try_from(pool_index)
+                    .map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
+                let mut client = self.get_client().await?;
+                let request = Request::new(CancelDecommissionRequest { pool_index });
+
+                let response = client.cancel_decommission(request).await?.into_inner();
+                if !response.success {
+                    if let Some(msg) = response.error_info {
+                        return Err(Error::other(msg));
+                    }
+                    return Err(Error::other(""));
+                }
+
+                Ok(())
+            }
+            .await,
+        )
+        .await
+    }
+
+    pub async fn clear_decommission(&self, pool_index: usize) -> Result<()> {
+        self.finalize_result(
+            async {
+                let pool_index = u32::try_from(pool_index)
+                    .map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
+                let mut client = self.get_client().await?;
+                let request = Request::new(ClearDecommissionRequest { pool_index });
+
+                let response = client.clear_decommission(request).await?.into_inner();
+                if !response.success {
+                    if let Some(msg) = response.error_info {
+                        return Err(Error::other(msg));
+                    }
+                    return Err(Error::other(""));
+                }
+
+                Ok(())
+            }
+            .await,
+        )
+        .await
+    }
+
     pub async fn load_transition_tier_config(&self) -> Result<()> {
         self.finalize_result(
             async {

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1104,6 +1104,42 @@ pub struct LoadRebalanceMetaResponse {
     #[prost(string, optional, tag = "2")]
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
 }
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct StartDecommissionRequest {
+    #[prost(uint32, repeated, tag = "1")]
+    pub pool_indices: ::prost::alloc::vec::Vec<u32>,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct StartDecommissionResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(string, optional, tag = "2")]
+    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+}
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct CancelDecommissionRequest {
+    #[prost(uint32, tag = "1")]
+    pub pool_index: u32,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct CancelDecommissionResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(string, optional, tag = "2")]
+    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+}
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct ClearDecommissionRequest {
+    #[prost(uint32, tag = "1")]
+    pub pool_index: u32,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct ClearDecommissionResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(string, optional, tag = "2")]
+    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+}
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct LoadTransitionTierConfigRequest {}
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
@@ -2356,6 +2392,51 @@ pub mod node_service_client {
                 .insert(GrpcMethod::new("node_service.NodeService", "LoadRebalanceMeta"));
             self.inner.unary(req, path, codec).await
         }
+        pub async fn start_decommission(
+            &mut self,
+            request: impl tonic::IntoRequest<super::StartDecommissionRequest>,
+        ) -> std::result::Result<tonic::Response<super::StartDecommissionResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/StartDecommission");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "StartDecommission"));
+            self.inner.unary(req, path, codec).await
+        }
+        pub async fn cancel_decommission(
+            &mut self,
+            request: impl tonic::IntoRequest<super::CancelDecommissionRequest>,
+        ) -> std::result::Result<tonic::Response<super::CancelDecommissionResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/CancelDecommission");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "CancelDecommission"));
+            self.inner.unary(req, path, codec).await
+        }
+        pub async fn clear_decommission(
+            &mut self,
+            request: impl tonic::IntoRequest<super::ClearDecommissionRequest>,
+        ) -> std::result::Result<tonic::Response<super::ClearDecommissionResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ClearDecommission");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "ClearDecommission"));
+            self.inner.unary(req, path, codec).await
+        }
         pub async fn load_transition_tier_config(
             &mut self,
             request: impl tonic::IntoRequest<super::LoadTransitionTierConfigRequest>,
@@ -2715,6 +2796,18 @@ pub mod node_service_server {
             &self,
             request: tonic::Request<super::LoadRebalanceMetaRequest>,
         ) -> std::result::Result<tonic::Response<super::LoadRebalanceMetaResponse>, tonic::Status>;
+        async fn start_decommission(
+            &self,
+            request: tonic::Request<super::StartDecommissionRequest>,
+        ) -> std::result::Result<tonic::Response<super::StartDecommissionResponse>, tonic::Status>;
+        async fn cancel_decommission(
+            &self,
+            request: tonic::Request<super::CancelDecommissionRequest>,
+        ) -> std::result::Result<tonic::Response<super::CancelDecommissionResponse>, tonic::Status>;
+        async fn clear_decommission(
+            &self,
+            request: tonic::Request<super::ClearDecommissionRequest>,
+        ) -> std::result::Result<tonic::Response<super::ClearDecommissionResponse>, tonic::Status>;
         async fn load_transition_tier_config(
             &self,
             request: tonic::Request<super::LoadTransitionTierConfigRequest>,
@@ -4927,6 +5020,90 @@ pub mod node_service_server {
                     };
                     Box::pin(fut)
                 }
+                "/node_service.NodeService/StartDecommission" => {
+                    #[allow(non_camel_case_types)]
+                    struct StartDecommissionSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::StartDecommissionRequest> for StartDecommissionSvc<T> {
+                        type Response = super::StartDecommissionResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::StartDecommissionRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::start_decommission(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = StartDecommissionSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/node_service.NodeService/CancelDecommission" => {
+                    #[allow(non_camel_case_types)]
+                    struct CancelDecommissionSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::CancelDecommissionRequest> for CancelDecommissionSvc<T> {
+                        type Response = super::CancelDecommissionResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::CancelDecommissionRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::cancel_decommission(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = CancelDecommissionSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/node_service.NodeService/ClearDecommission" => {
+                    #[allow(non_camel_case_types)]
+                    struct ClearDecommissionSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::ClearDecommissionRequest> for ClearDecommissionSvc<T> {
+                        type Response = super::ClearDecommissionResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::ClearDecommissionRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::clear_decommission(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = ClearDecommissionSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 "/node_service.NodeService/LoadTransitionTierConfig" => {
                     #[allow(non_camel_case_types)]
                     struct LoadTransitionTierConfigSvc<T: NodeService>(pub Arc<T>);

crates/protos/src/node.proto

@@ -782,6 +782,33 @@ message LoadRebalanceMetaResponse {
   optional string error_info = 2;
 }
 
+message StartDecommissionRequest {
+  repeated uint32 pool_indices = 1;
+}
+
+message StartDecommissionResponse {
+  bool success = 1;
+  optional string error_info = 2;
+}
+
+message CancelDecommissionRequest {
+  uint32 pool_index = 1;
+}
+
+message CancelDecommissionResponse {
+  bool success = 1;
+  optional string error_info = 2;
+}
+
+message ClearDecommissionRequest {
+  uint32 pool_index = 1;
+}
+
+message ClearDecommissionResponse {
+  bool success = 1;
+  optional string error_info = 2;
+}
+
 message LoadTransitionTierConfigRequest {}
 
 message LoadTransitionTierConfigResponse {
@@ -895,6 +922,9 @@ service NodeService {
   rpc ReloadPoolMeta(ReloadPoolMetaRequest) returns (ReloadPoolMetaResponse) {};
   rpc StopRebalance(StopRebalanceRequest) returns (StopRebalanceResponse) {};
   rpc LoadRebalanceMeta(LoadRebalanceMetaRequest) returns (LoadRebalanceMetaResponse) {};
+  rpc StartDecommission(StartDecommissionRequest) returns (StartDecommissionResponse) {};
+  rpc CancelDecommission(CancelDecommissionRequest) returns (CancelDecommissionResponse) {};
+  rpc ClearDecommission(ClearDecommissionRequest) returns (ClearDecommissionResponse) {};
   rpc LoadTransitionTierConfig(LoadTransitionTierConfigRequest) returns (LoadTransitionTierConfigResponse) {};
   rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {};
 }