fix(storage): harden rebalance and decommission state (#3730)

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -664,6 +664,27 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn start_decommission(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::StartDecommissionRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::StartDecommissionResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
+    async fn cancel_decommission(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::CancelDecommissionRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::CancelDecommissionResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
+    async fn clear_decommission(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::ClearDecommissionRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::ClearDecommissionResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn get_metrics(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::GetMetricsRequest>,

crates/ecstore/src/notification_sys.rs

@@ -109,6 +109,14 @@ impl NotificationSys {
         self.all_peer_clients[idx].clone()
     }
 
+    pub fn peer_client_for_grid_host(&self, grid_host: &str) -> Option<PeerRestClient> {
+        self.all_peer_clients
+            .iter()
+            .flatten()
+            .find(|client| client.grid_host == grid_host)
+            .cloned()
+    }
+
     pub async fn delete_policy(&self, policy_name: &str) -> Vec<NotificationPeerErr> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for client in self.peer_clients.iter() {
@@ -1215,6 +1223,24 @@ mod tests {
         assert!(msg.contains("local save failed"));
     }
 
+    #[test]
+    fn peer_client_for_grid_host_matches_exact_grid_host() {
+        let sys = NotificationSys {
+            peer_clients: Vec::new(),
+            all_peer_clients: vec![Some(PeerRestClient::new(
+                "127.0.0.1:9000".to_string().try_into().expect("peer host should parse"),
+                "http://127.0.0.1:9000".to_string(),
+            ))],
+            peer_admin_caches: Vec::new(),
+        };
+
+        let client = sys
+            .peer_client_for_grid_host("http://127.0.0.1:9000")
+            .expect("matching grid host should return peer client");
+        assert_eq!(client.grid_host, "http://127.0.0.1:9000");
+        assert!(sys.peer_client_for_grid_host("http://node-b:9000").is_none());
+    }
+
     #[test]
     fn load_rebalance_meta_aggregate_failures_return_error() {
         let err = aggregate_notification_failures(

crates/ecstore/src/rebalance/control.rs

@@ -1,10 +1,10 @@
 use super::meta::{
-    clone_first_arc, clone_rebalance_pool_stats, defer_bucket_in_rebalance_queue, ensure_valid_rebalance_pool_index,
-    invalid_rebalance_pool_index_error, is_rebalance_conflicting_with_decommission, mark_rebalance_bucket_done,
-    merge_rebalance_meta, percent_free_ratio, rebalance_metadata_not_initialized_error, record_rebalance_cleanup_warning_in_meta,
-    record_rebalance_stop_propagation_snapshot, resolve_next_rebalance_bucket, rollback_rebalance_start_meta_snapshot_for_id,
-    should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot_for_id,
-    validate_init_rebalance_state,
+    RebalanceMetaMergeOutcome, clone_first_arc, clone_rebalance_pool_stats, defer_bucket_in_rebalance_queue,
+    ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error, is_rebalance_conflicting_with_decommission,
+    mark_rebalance_bucket_done, merge_rebalance_meta, percent_free_ratio, rebalance_metadata_not_initialized_error,
+    record_rebalance_cleanup_warning_in_meta, record_rebalance_stop_propagation_snapshot, resolve_next_rebalance_bucket,
+    rollback_rebalance_start_meta_snapshot_for_id, should_accept_rebalance_stats_update, should_pool_participate,
+    stop_rebalance_meta_snapshot_for_id, validate_init_rebalance_state,
 };
 use super::worker::{
     rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result, resolve_rebalance_meta_load_result,
@@ -27,6 +27,18 @@ use time::OffsetDateTime;
 use tracing::{debug, info};
 use uuid::Uuid;
 
+pub(super) fn validate_rebalance_disk_stats_coverage(disk_stats: &[DiskStat]) -> Result<()> {
+    for (idx, disk_stat) in disk_stats.iter().enumerate() {
+        if disk_stat.total_space == 0 {
+            return Err(Error::other(format!(
+                "rebalance storage info is incomplete: pool {idx} has no reported capacity"
+            )));
+        }
+    }
+
+    Ok(())
+}
+
 impl ECStore {
     pub(super) async fn save_rebalance_meta_with_merge<S>(
         &self,
@@ -50,7 +62,9 @@ impl ECStore {
         let mut merged = RebalanceMeta::new();
         match merged.load_with_opts(pool.clone(), opts.clone()).await {
             Ok(()) => {
-                merge_rebalance_meta(&mut merged, local_snapshot);
+                if merge_rebalance_meta(&mut merged, local_snapshot) == RebalanceMetaMergeOutcome::RejectedActiveConflict {
+                    return Err(Error::RebalanceAlreadyRunning);
+                }
             }
             Err(Error::ConfigNotFound) => {
                 merged = local_snapshot.clone();
@@ -90,6 +104,10 @@ impl ECStore {
                 "Loaded rebalance metadata"
             );
         } else {
+            {
+                let mut rebalance_meta = self.rebalance_meta.write().await;
+                *rebalance_meta = None;
+            }
             debug!(
                 event = EVENT_REBALANCE_STATE,
                 component = LOG_COMPONENT_ECSTORE,
@@ -191,6 +209,7 @@ impl ECStore {
         }
 
         let percent_free_goal = percent_free_ratio(total_free, total_cap);
+        validate_rebalance_disk_stats_coverage(&disk_stats)?;
 
         let mut pool_stats = Vec::with_capacity(self.pools.len());
 
@@ -421,6 +440,15 @@ impl ECStore {
         false
     }
 
+    pub async fn pool_rebalance_status(&self, pool_index: usize) -> (RebalStatus, bool) {
+        let rebalance_meta = self.rebalance_meta.read().await;
+        rebalance_meta
+            .as_ref()
+            .and_then(|meta| meta.pool_stats.get(pool_index))
+            .map(|pool_stat| (pool_stat.info.status, pool_stat.info.stopping))
+            .unwrap_or_default()
+    }
+
     pub async fn current_rebalance_id(&self) -> Option<String> {
         let rebalance_meta = self.rebalance_meta.read().await;
         rebalance_meta

crates/ecstore/src/rebalance/meta.rs

@@ -457,7 +457,10 @@ pub(super) fn complete_rebalance_pools_at_goal(meta: &mut RebalanceMeta, now: Of
     let mut changed = false;
 
     for pool_stat in meta.pool_stats.iter_mut() {
-        if !is_rebalance_pool_started(pool_stat) || has_deferred_rebalance_error(pool_stat) {
+        if !is_rebalance_pool_started(pool_stat)
+            || has_deferred_rebalance_error(pool_stat)
+            || has_rebalance_cleanup_warnings(pool_stat)
+        {
             continue;
         }
 
@@ -481,7 +484,7 @@ pub(super) fn complete_rebalance_pools_with_empty_queue(meta: &mut RebalanceMeta
     let mut changed = false;
 
     for pool_stat in meta.pool_stats.iter_mut() {
-        if !is_rebalance_pool_started(pool_stat) || !pool_stat.buckets.is_empty() {
+        if !is_rebalance_pool_started(pool_stat) || !pool_stat.buckets.is_empty() || has_rebalance_cleanup_warnings(pool_stat) {
             continue;
         }
 
@@ -501,6 +504,11 @@ pub(super) fn has_deferred_rebalance_error(pool_stat: &RebalanceStats) -> bool {
         .as_deref()
         .is_some_and(|last_error| last_error.starts_with(REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX))
 }
+
+pub(super) fn has_rebalance_cleanup_warnings(pool_stat: &RebalanceStats) -> bool {
+    pool_stat.cleanup_warnings.count > 0
+}
+
 pub(super) fn clone_first_arc<T>(values: &[Arc<T>], err_msg: &str) -> Result<Arc<T>> {
     values.first().cloned().ok_or_else(|| Error::other(err_msg))
 }
@@ -608,7 +616,7 @@ pub(super) fn validate_init_rebalance_state(decommission_running: bool, current_
     if !ensure_rebalance_not_decommissioning(decommission_running) {
         return Err(Error::DecommissionAlreadyRunning);
     }
-    if current_meta.is_some_and(is_rebalance_in_progress) {
+    if current_meta.is_some_and(|meta| !is_rebalance_meta_replaceable_for_new_id(meta)) {
         return Err(Error::RebalanceAlreadyRunning);
     }
 
@@ -812,14 +820,29 @@ pub(super) fn merge_rebalance_pool_stats(remote: &mut RebalanceStats, local: &Re
     }
 }
 
-pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &RebalanceMeta) {
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum RebalanceMetaMergeOutcome {
+    Merged,
+    Replaced,
+    RejectedActiveConflict,
+}
+
+pub(super) fn is_rebalance_meta_replaceable_for_new_id(meta: &RebalanceMeta) -> bool {
+    meta.stopped_at.is_some() || !is_rebalance_in_progress(meta)
+}
+
+pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &RebalanceMeta) -> RebalanceMetaMergeOutcome {
     if remote.id.is_empty() {
         *remote = local.clone();
-        return;
+        return RebalanceMetaMergeOutcome::Replaced;
     }
 
     if !local.id.is_empty() && remote.id != local.id {
-        return;
+        if is_rebalance_meta_replaceable_for_new_id(remote) {
+            *remote = local.clone();
+            return RebalanceMetaMergeOutcome::Replaced;
+        }
+        return RebalanceMetaMergeOutcome::RejectedActiveConflict;
     }
 
     remote.percent_free_goal = local.percent_free_goal;
@@ -837,6 +860,8 @@ pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &Rebalance
             merge_rebalance_pool_stats(remote_pool_stat, local_pool_stat);
         }
     }
+
+    RebalanceMetaMergeOutcome::Merged
 }
 
 pub(super) fn mark_started_rebalance_pools_stopped(meta: &mut RebalanceMeta, stop_time: OffsetDateTime) {

crates/ecstore/src/rebalance/rebalance_unit_tests.rs

@@ -13,19 +13,21 @@
 // limitations under the License.
 
 use super::REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX;
+use super::control::validate_rebalance_disk_stats_coverage;
 use super::meta::{
-    RebalanceTerminalEvent, apply_rebalance_save_option, apply_rebalance_terminal_event, apply_stopped_at,
-    classify_rebalance_terminal_event, clone_arc_by_index, clone_first_arc, clone_rebalance_pool_stats,
+    RebalanceMetaMergeOutcome, RebalanceTerminalEvent, apply_rebalance_save_option, apply_rebalance_terminal_event,
+    apply_stopped_at, classify_rebalance_terminal_event, clone_arc_by_index, clone_first_arc, clone_rebalance_pool_stats,
     complete_rebalance_pools_at_goal, complete_rebalance_pools_with_empty_queue, defer_bucket_in_rebalance_queue,
     ensure_rebalance_not_decommissioning, ensure_valid_rebalance_pool_index, first_rebalance_bucket,
     has_deferred_rebalance_error, is_rebalance_actively_running, is_rebalance_conflicting_with_decommission,
-    is_rebalance_in_progress, is_rebalance_stopped_terminal_event, mark_rebalance_bucket_done, merge_rebalance_bucket_lists,
-    merge_rebalance_meta, next_rebal_bucket_from_stat, percent_free_ratio, rebalance_goal_reached,
-    rebalance_meta_load_no_data_error, rebalance_meta_load_unknown_format_error, rebalance_meta_load_unknown_version_error,
-    record_rebalance_cleanup_warning_in_meta, remove_rebalanced_buckets_from_queue, resolve_next_rebalance_bucket,
-    resolve_rebalance_participants, should_accept_rebalance_stats_update, should_ignore_rebalance_data_usage_cache,
-    should_pool_participate, should_preserve_rebalance_stopped_state, should_skip_start_rebalance, stop_rebalance_meta_snapshot,
-    stop_rebalance_state, take_bucket_from_rebalance_queue, validate_init_rebalance_state, validate_start_rebalance_state,
+    is_rebalance_in_progress, is_rebalance_meta_replaceable_for_new_id, is_rebalance_stopped_terminal_event,
+    mark_rebalance_bucket_done, merge_rebalance_bucket_lists, merge_rebalance_meta, next_rebal_bucket_from_stat,
+    percent_free_ratio, rebalance_goal_reached, rebalance_meta_load_no_data_error, rebalance_meta_load_unknown_format_error,
+    rebalance_meta_load_unknown_version_error, record_rebalance_cleanup_warning_in_meta, remove_rebalanced_buckets_from_queue,
+    resolve_next_rebalance_bucket, resolve_rebalance_participants, should_accept_rebalance_stats_update,
+    should_ignore_rebalance_data_usage_cache, should_pool_participate, should_preserve_rebalance_stopped_state,
+    should_skip_start_rebalance, stop_rebalance_meta_snapshot, stop_rebalance_state, take_bucket_from_rebalance_queue,
+    validate_init_rebalance_state, validate_start_rebalance_state,
 };
 use super::migration::{
     MigrationBackend, MigrationVersionResult, migrate_entry_version, migrate_entry_version_with_retry_wait,
@@ -44,8 +46,8 @@ use super::worker::{
     wait_rebalance_listing_retry, with_rebalance_entry_context,
 };
 use super::{
-    GetObjectReader, ObjectInfo, ObjectOptions, RebalSaveOpt, RebalStatus, RebalanceBucketOutcome, RebalanceCleanupWarnings,
-    RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta, RebalanceStats,
+    DiskStat, GetObjectReader, ObjectInfo, ObjectOptions, RebalSaveOpt, RebalStatus, RebalanceBucketOutcome,
+    RebalanceCleanupWarnings, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta, RebalanceStats,
 };
 use crate::data_movement;
 use crate::data_usage::DATA_USAGE_CACHE_NAME;
@@ -1207,7 +1209,7 @@ fn test_merge_rebalance_meta_preserves_updates_from_multiple_pools() {
         ..Default::default()
     };
 
-    merge_rebalance_meta(&mut remote, &local);
+    assert_eq!(merge_rebalance_meta(&mut remote, &local), RebalanceMetaMergeOutcome::Merged);
 
     assert_eq!(remote.pool_stats[0].num_versions, 4);
     assert_eq!(remote.pool_stats[0].object, "remote-object");
@@ -1220,6 +1222,163 @@ fn test_merge_rebalance_meta_preserves_updates_from_multiple_pools() {
     assert_eq!(remote.pool_stats[1].cleanup_warnings.last_at, Some(warning_at));
 }
 
+#[test]
+fn test_merge_rebalance_meta_replaces_terminal_metadata_for_new_rebalance() {
+    let old_completed_at = OffsetDateTime::from_unix_timestamp(1_000).expect("valid old completion timestamp");
+    let new_started_at = OffsetDateTime::from_unix_timestamp(2_000).expect("valid new start timestamp");
+    let mut remote = RebalanceMeta {
+        id: "old-rebalance".to_string(),
+        percent_free_goal: 0.25,
+        pool_stats: vec![
+            RebalanceStats {
+                buckets: Vec::new(),
+                rebalanced_buckets: vec!["bucket-a".to_string()],
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Completed,
+                    end_time: Some(old_completed_at),
+                    ..Default::default()
+                },
+                num_versions: 7,
+                bytes: 700,
+                ..Default::default()
+            },
+            RebalanceStats::default(),
+        ],
+        ..Default::default()
+    };
+    let local = RebalanceMeta {
+        id: "new-rebalance".to_string(),
+        percent_free_goal: 0.5,
+        pool_stats: vec![
+            RebalanceStats {
+                buckets: vec!["bucket-a".to_string()],
+                participating: true,
+                info: RebalanceInfo {
+                    start_time: Some(new_started_at),
+                    status: RebalStatus::Started,
+                    ..Default::default()
+                },
+                ..Default::default()
+            },
+            RebalanceStats {
+                buckets: vec!["bucket-a".to_string()],
+                participating: true,
+                info: RebalanceInfo {
+                    start_time: Some(new_started_at),
+                    status: RebalStatus::Started,
+                    ..Default::default()
+                },
+                ..Default::default()
+            },
+            RebalanceStats::default(),
+        ],
+        ..Default::default()
+    };
+
+    assert_eq!(merge_rebalance_meta(&mut remote, &local), RebalanceMetaMergeOutcome::Replaced);
+
+    assert_eq!(remote.id, "new-rebalance");
+    assert_eq!(remote.percent_free_goal, 0.5);
+    assert_eq!(remote.pool_stats.len(), 3);
+    assert_eq!(remote.pool_stats[0].info.status, RebalStatus::Started);
+    assert_eq!(remote.pool_stats[0].buckets, vec!["bucket-a"]);
+    assert_eq!(remote.pool_stats[0].rebalanced_buckets, Vec::<String>::new());
+    assert_eq!(remote.pool_stats[0].num_versions, 0);
+    assert_eq!(remote.pool_stats[1].info.status, RebalStatus::Started);
+    assert!(remote.pool_stats[1].participating);
+}
+
+#[test]
+fn test_merge_rebalance_meta_preserves_active_metadata_for_different_rebalance_id() {
+    let old_started_at = OffsetDateTime::from_unix_timestamp(1_000).expect("valid old start timestamp");
+    let new_started_at = OffsetDateTime::from_unix_timestamp(2_000).expect("valid new start timestamp");
+    let mut remote = RebalanceMeta {
+        id: "active-rebalance".to_string(),
+        percent_free_goal: 0.25,
+        pool_stats: vec![RebalanceStats {
+            buckets: vec!["bucket-a".to_string()],
+            participating: true,
+            info: RebalanceInfo {
+                start_time: Some(old_started_at),
+                status: RebalStatus::Started,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
+    let local = RebalanceMeta {
+        id: "new-rebalance".to_string(),
+        percent_free_goal: 0.5,
+        pool_stats: vec![RebalanceStats {
+            buckets: vec!["bucket-b".to_string()],
+            participating: true,
+            info: RebalanceInfo {
+                start_time: Some(new_started_at),
+                status: RebalStatus::Started,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
+
+    assert_eq!(
+        merge_rebalance_meta(&mut remote, &local),
+        RebalanceMetaMergeOutcome::RejectedActiveConflict
+    );
+
+    assert_eq!(remote.id, "active-rebalance");
+    assert_eq!(remote.percent_free_goal, 0.25);
+    assert_eq!(remote.pool_stats.len(), 1);
+    assert_eq!(remote.pool_stats[0].buckets, vec!["bucket-a"]);
+    assert_eq!(remote.pool_stats[0].info.start_time, Some(old_started_at));
+}
+
+#[test]
+fn test_merge_rebalance_meta_replaces_stopped_started_metadata_for_new_rebalance() {
+    let stopped_at = OffsetDateTime::from_unix_timestamp(1_500).expect("valid stop timestamp");
+    let new_started_at = OffsetDateTime::from_unix_timestamp(2_000).expect("valid new start timestamp");
+    let mut remote = RebalanceMeta {
+        id: "stopped-rebalance".to_string(),
+        stopped_at: Some(stopped_at),
+        pool_stats: vec![RebalanceStats {
+            participating: true,
+            info: RebalanceInfo {
+                status: RebalStatus::Started,
+                stopping: true,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
+    let local = RebalanceMeta {
+        id: "new-rebalance".to_string(),
+        percent_free_goal: 0.5,
+        pool_stats: vec![RebalanceStats {
+            buckets: vec!["bucket-a".to_string()],
+            participating: true,
+            info: RebalanceInfo {
+                start_time: Some(new_started_at),
+                status: RebalStatus::Started,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
+
+    assert!(is_rebalance_meta_replaceable_for_new_id(&remote));
+    assert_eq!(merge_rebalance_meta(&mut remote, &local), RebalanceMetaMergeOutcome::Replaced);
+
+    assert_eq!(remote.id, "new-rebalance");
+    assert_eq!(remote.stopped_at, None);
+    assert_eq!(remote.pool_stats[0].info.status, RebalStatus::Started);
+    assert!(!remote.pool_stats[0].info.stopping);
+}
+
 #[test]
 fn test_merge_rebalance_meta_does_not_overwrite_failed_with_started_stats() {
     let now = OffsetDateTime::from_unix_timestamp(2_000).unwrap();
@@ -2202,6 +2361,7 @@ fn test_validate_init_rebalance_state_rejects_active_rebalance() {
 
 #[test]
 fn test_validate_init_rebalance_state_allows_terminal_or_missing_rebalance() {
+    let stopped_at = OffsetDateTime::now_utc();
     let completed = RebalanceMeta {
         pool_stats: vec![RebalanceStats {
             participating: true,
@@ -2213,9 +2373,23 @@ fn test_validate_init_rebalance_state_allows_terminal_or_missing_rebalance() {
         }],
         ..Default::default()
     };
+    let stopped_started = RebalanceMeta {
+        stopped_at: Some(stopped_at),
+        pool_stats: vec![RebalanceStats {
+            participating: true,
+            info: RebalanceInfo {
+                status: RebalStatus::Started,
+                stopping: true,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
 
     validate_init_rebalance_state(false, None).expect("missing rebalance meta should allow init");
     validate_init_rebalance_state(false, Some(&completed)).expect("terminal rebalance meta should allow init");
+    validate_init_rebalance_state(false, Some(&stopped_started)).expect("stopped rebalance meta should allow new init");
 }
 
 #[tokio::test]
@@ -2298,6 +2472,38 @@ fn test_rebalance_goal_not_reached_for_issue_3137_initial_imbalance() {
     assert!(!rebalance_goal_reached(pool0_free, pool0_capacity, 0, goal));
 }
 
+#[test]
+fn test_validate_rebalance_disk_stats_coverage_rejects_missing_pool_capacity() {
+    let disk_stats = vec![
+        DiskStat {
+            total_space: 1_000,
+            available_space: 100,
+        },
+        DiskStat::default(),
+    ];
+
+    let err = validate_rebalance_disk_stats_coverage(&disk_stats)
+        .expect_err("missing pool capacity should reject rebalance initialization");
+
+    assert!(err.to_string().contains("pool 1 has no reported capacity"));
+}
+
+#[test]
+fn test_validate_rebalance_disk_stats_coverage_accepts_all_pools() {
+    let disk_stats = vec![
+        DiskStat {
+            total_space: 1_000,
+            available_space: 100,
+        },
+        DiskStat {
+            total_space: 2_000,
+            available_space: 1_500,
+        },
+    ];
+
+    assert!(validate_rebalance_disk_stats_coverage(&disk_stats).is_ok());
+}
+
 #[test]
 fn test_complete_rebalance_pools_at_goal_marks_started_participants_completed() {
     let now = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
@@ -2943,7 +3149,7 @@ fn test_record_rebalance_cleanup_warning_in_meta_preserves_last_error() {
 }
 
 #[test]
-fn test_complete_rebalance_pools_with_empty_queue_preserves_cleanup_warnings() {
+fn test_complete_rebalance_pools_with_empty_queue_skips_cleanup_warnings() {
     let warning_at = OffsetDateTime::from_unix_timestamp(9_000).unwrap();
     let completed_at = OffsetDateTime::from_unix_timestamp(10_000).unwrap();
     let mut meta = RebalanceMeta {
@@ -2966,9 +3172,10 @@ fn test_complete_rebalance_pools_with_empty_queue_preserves_cleanup_warnings() {
         ..Default::default()
     };
 
-    assert!(complete_rebalance_pools_with_empty_queue(&mut meta, completed_at));
+    assert!(!complete_rebalance_pools_with_empty_queue(&mut meta, completed_at));
 
-    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Completed);
+    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started);
+    assert_eq!(meta.pool_stats[0].info.end_time, None);
     assert!(meta.pool_stats[0].info.last_error.is_none());
     assert_eq!(meta.pool_stats[0].cleanup_warnings.count, 1);
     assert_eq!(meta.pool_stats[0].cleanup_warnings.last_message.as_deref(), Some("cleanup failed"));

crates/ecstore/src/rebalance/runtime.rs

@@ -1,8 +1,9 @@
 use super::meta::{
     apply_rebalance_save_option, apply_rebalance_terminal_event, classify_rebalance_terminal_event, clone_first_arc,
     complete_rebalance_pools_at_goal, complete_rebalance_pools_with_empty_queue, ensure_valid_rebalance_pool_index,
-    has_deferred_rebalance_error, is_rebalance_in_progress, rebalance_goal_reached, resolve_rebalance_participants,
-    should_preserve_rebalance_stopped_state, should_skip_start_rebalance, validate_start_rebalance_state,
+    has_deferred_rebalance_error, has_rebalance_cleanup_warnings, is_rebalance_in_progress, rebalance_goal_reached,
+    resolve_rebalance_participants, should_preserve_rebalance_stopped_state, should_skip_start_rebalance,
+    validate_start_rebalance_state,
 };
 use super::worker::{
     resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result, resolve_rebalance_save_task_result,
@@ -211,7 +212,20 @@ impl ECStore {
                         if let Some(meta) = rebalance_meta.as_mut() {
                             let meta_stopped = meta.stopped_at.is_some();
                             if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
-                                if should_preserve_rebalance_stopped_state(
+                                if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
+                                    && has_rebalance_cleanup_warnings(pool_stat)
+                                {
+                                    pool_stat.info.stopping = false;
+                                    pool_stat.info.status = RebalStatus::Failed;
+                                    pool_stat.info.end_time = Some(now);
+                                    pool_stat.info.last_error = Some(
+                                        pool_stat
+                                            .cleanup_warnings
+                                            .last_message
+                                            .clone()
+                                            .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
+                                    );
+                                } else if should_preserve_rebalance_stopped_state(
                                     meta_stopped,
                                     pool_stat.info.status,
                                     &terminal_event,
@@ -513,6 +527,7 @@ impl ECStore {
             };
 
             if !has_deferred_rebalance_error(pool_stat)
+                && !has_rebalance_cleanup_warnings(pool_stat)
                 && rebalance_goal_reached(
                     pool_stat.init_free_space,
                     pool_stat.init_capacity,

crates/ecstore/src/rpc/peer_rest_client.rs

@@ -29,12 +29,13 @@ use rustfs_madmin::{
 };
 use rustfs_protos::evict_failed_connection;
 use rustfs_protos::proto_gen::node_service::{
-    DeleteBucketMetadataRequest, DeletePolicyRequest, DeleteServiceAccountRequest, DeleteUserRequest, GetCpusRequest,
-    GetLiveEventsRequest, GetMemInfoRequest, GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest,
-    GetProcInfoRequest, GetSeLinuxInfoRequest, GetSysConfigRequest, GetSysErrorsRequest, LoadBucketMetadataRequest,
-    LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
-    LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest,
-    ReloadSiteReplicationConfigRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest, StopRebalanceRequest,
+    CancelDecommissionRequest, ClearDecommissionRequest, DeleteBucketMetadataRequest, DeletePolicyRequest,
+    DeleteServiceAccountRequest, DeleteUserRequest, GetCpusRequest, GetLiveEventsRequest, GetMemInfoRequest, GetMetricsRequest,
+    GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest, GetSysConfigRequest,
+    GetSysErrorsRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
+    LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
+    LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ServerInfoRequest,
+    SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest,
     node_service_client::NodeServiceClient,
 };
 use rustfs_utils::XHost;
@@ -972,6 +973,79 @@ impl PeerRestClient {
         .await
     }
 
+    pub async fn start_decommission(&self, pool_indices: Vec<usize>) -> Result<()> {
+        self.finalize_result(
+            async {
+                let pool_indices = pool_indices
+                    .into_iter()
+                    .map(|idx| {
+                        u32::try_from(idx).map_err(|_| Error::other(format!("decommission pool index {idx} exceeds RPC range")))
+                    })
+                    .collect::<Result<Vec<_>>>()?;
+                let mut client = self.get_client().await?;
+                let request = Request::new(StartDecommissionRequest { pool_indices });
+
+                let response = client.start_decommission(request).await?.into_inner();
+                if !response.success {
+                    if let Some(msg) = response.error_info {
+                        return Err(Error::other(msg));
+                    }
+                    return Err(Error::other(""));
+                }
+
+                Ok(())
+            }
+            .await,
+        )
+        .await
+    }
+
+    pub async fn decommission_cancel(&self, pool_index: usize) -> Result<()> {
+        self.finalize_result(
+            async {
+                let pool_index = u32::try_from(pool_index)
+                    .map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
+                let mut client = self.get_client().await?;
+                let request = Request::new(CancelDecommissionRequest { pool_index });
+
+                let response = client.cancel_decommission(request).await?.into_inner();
+                if !response.success {
+                    if let Some(msg) = response.error_info {
+                        return Err(Error::other(msg));
+                    }
+                    return Err(Error::other(""));
+                }
+
+                Ok(())
+            }
+            .await,
+        )
+        .await
+    }
+
+    pub async fn clear_decommission(&self, pool_index: usize) -> Result<()> {
+        self.finalize_result(
+            async {
+                let pool_index = u32::try_from(pool_index)
+                    .map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
+                let mut client = self.get_client().await?;
+                let request = Request::new(ClearDecommissionRequest { pool_index });
+
+                let response = client.clear_decommission(request).await?.into_inner();
+                if !response.success {
+                    if let Some(msg) = response.error_info {
+                        return Err(Error::other(msg));
+                    }
+                    return Err(Error::other(""));
+                }
+
+                Ok(())
+            }
+            .await,
+        )
+        .await
+    }
+
     pub async fn load_transition_tier_config(&self) -> Result<()> {
         self.finalize_result(
             async {

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1104,6 +1104,42 @@ pub struct LoadRebalanceMetaResponse {
     #[prost(string, optional, tag = "2")]
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
 }
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct StartDecommissionRequest {
+    #[prost(uint32, repeated, tag = "1")]
+    pub pool_indices: ::prost::alloc::vec::Vec<u32>,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct StartDecommissionResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(string, optional, tag = "2")]
+    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+}
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct CancelDecommissionRequest {
+    #[prost(uint32, tag = "1")]
+    pub pool_index: u32,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct CancelDecommissionResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(string, optional, tag = "2")]
+    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+}
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct ClearDecommissionRequest {
+    #[prost(uint32, tag = "1")]
+    pub pool_index: u32,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct ClearDecommissionResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(string, optional, tag = "2")]
+    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+}
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct LoadTransitionTierConfigRequest {}
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
@@ -2356,6 +2392,51 @@ pub mod node_service_client {
                 .insert(GrpcMethod::new("node_service.NodeService", "LoadRebalanceMeta"));
             self.inner.unary(req, path, codec).await
         }
+        pub async fn start_decommission(
+            &mut self,
+            request: impl tonic::IntoRequest<super::StartDecommissionRequest>,
+        ) -> std::result::Result<tonic::Response<super::StartDecommissionResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/StartDecommission");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "StartDecommission"));
+            self.inner.unary(req, path, codec).await
+        }
+        pub async fn cancel_decommission(
+            &mut self,
+            request: impl tonic::IntoRequest<super::CancelDecommissionRequest>,
+        ) -> std::result::Result<tonic::Response<super::CancelDecommissionResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/CancelDecommission");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "CancelDecommission"));
+            self.inner.unary(req, path, codec).await
+        }
+        pub async fn clear_decommission(
+            &mut self,
+            request: impl tonic::IntoRequest<super::ClearDecommissionRequest>,
+        ) -> std::result::Result<tonic::Response<super::ClearDecommissionResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ClearDecommission");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "ClearDecommission"));
+            self.inner.unary(req, path, codec).await
+        }
         pub async fn load_transition_tier_config(
             &mut self,
             request: impl tonic::IntoRequest<super::LoadTransitionTierConfigRequest>,
@@ -2715,6 +2796,18 @@ pub mod node_service_server {
             &self,
             request: tonic::Request<super::LoadRebalanceMetaRequest>,
         ) -> std::result::Result<tonic::Response<super::LoadRebalanceMetaResponse>, tonic::Status>;
+        async fn start_decommission(
+            &self,
+            request: tonic::Request<super::StartDecommissionRequest>,
+        ) -> std::result::Result<tonic::Response<super::StartDecommissionResponse>, tonic::Status>;
+        async fn cancel_decommission(
+            &self,
+            request: tonic::Request<super::CancelDecommissionRequest>,
+        ) -> std::result::Result<tonic::Response<super::CancelDecommissionResponse>, tonic::Status>;
+        async fn clear_decommission(
+            &self,
+            request: tonic::Request<super::ClearDecommissionRequest>,
+        ) -> std::result::Result<tonic::Response<super::ClearDecommissionResponse>, tonic::Status>;
         async fn load_transition_tier_config(
             &self,
             request: tonic::Request<super::LoadTransitionTierConfigRequest>,
@@ -4927,6 +5020,90 @@ pub mod node_service_server {
                     };
                     Box::pin(fut)
                 }
+                "/node_service.NodeService/StartDecommission" => {
+                    #[allow(non_camel_case_types)]
+                    struct StartDecommissionSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::StartDecommissionRequest> for StartDecommissionSvc<T> {
+                        type Response = super::StartDecommissionResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::StartDecommissionRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::start_decommission(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = StartDecommissionSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/node_service.NodeService/CancelDecommission" => {
+                    #[allow(non_camel_case_types)]
+                    struct CancelDecommissionSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::CancelDecommissionRequest> for CancelDecommissionSvc<T> {
+                        type Response = super::CancelDecommissionResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::CancelDecommissionRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::cancel_decommission(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = CancelDecommissionSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/node_service.NodeService/ClearDecommission" => {
+                    #[allow(non_camel_case_types)]
+                    struct ClearDecommissionSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::ClearDecommissionRequest> for ClearDecommissionSvc<T> {
+                        type Response = super::ClearDecommissionResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::ClearDecommissionRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::clear_decommission(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = ClearDecommissionSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 "/node_service.NodeService/LoadTransitionTierConfig" => {
                     #[allow(non_camel_case_types)]
                     struct LoadTransitionTierConfigSvc<T: NodeService>(pub Arc<T>);

crates/protos/src/node.proto

@@ -782,6 +782,33 @@ message LoadRebalanceMetaResponse {
   optional string error_info = 2;
 }
 
+message StartDecommissionRequest {
+  repeated uint32 pool_indices = 1;
+}
+
+message StartDecommissionResponse {
+  bool success = 1;
+  optional string error_info = 2;
+}
+
+message CancelDecommissionRequest {
+  uint32 pool_index = 1;
+}
+
+message CancelDecommissionResponse {
+  bool success = 1;
+  optional string error_info = 2;
+}
+
+message ClearDecommissionRequest {
+  uint32 pool_index = 1;
+}
+
+message ClearDecommissionResponse {
+  bool success = 1;
+  optional string error_info = 2;
+}
+
 message LoadTransitionTierConfigRequest {}
 
 message LoadTransitionTierConfigResponse {
@@ -895,6 +922,9 @@ service NodeService {
   rpc ReloadPoolMeta(ReloadPoolMetaRequest) returns (ReloadPoolMetaResponse) {};
   rpc StopRebalance(StopRebalanceRequest) returns (StopRebalanceResponse) {};
   rpc LoadRebalanceMeta(LoadRebalanceMetaRequest) returns (LoadRebalanceMetaResponse) {};
+  rpc StartDecommission(StartDecommissionRequest) returns (StartDecommissionResponse) {};
+  rpc CancelDecommission(CancelDecommissionRequest) returns (CancelDecommissionResponse) {};
+  rpc ClearDecommission(ClearDecommissionRequest) returns (ClearDecommissionResponse) {};
   rpc LoadTransitionTierConfig(LoadTransitionTierConfigRequest) returns (LoadTransitionTierConfigResponse) {};
   rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {};
 }

docs/architecture/decommission-compatibility.md

@@ -33,6 +33,11 @@ being moved while still allowing a request to contain later targets whose leader
 are different nodes. Later queued targets are recovered or promoted by the
 leader that owns that target.
 
+Admin start, cancel, and clear requests may arrive on any cluster node. When the
+target pool first endpoint is remote, RustFS forwards the operation over the
+authenticated internode RPC channel to that first endpoint. The receiving node
+still enforces the local-leader rule before mutating decommission state.
+
 ### Persisted Metadata Shape
 
 The queue is persisted in pool metadata and decoded with the rest of
@@ -155,6 +160,7 @@ The queued multi-pool contract is guarded by:
 - `test_contextualized_decommission_start_request_allows_multiple_target_pools`
 - `test_decommission_start_local_leader_allows_remote_queued_pool`
 - `test_local_decommission_queue_prefix_stops_at_remote_leader`
+- `test_decommission_peer_target_returns_none_for_local_first_endpoint`
 - `test_pool_meta_queued_decommission_is_not_suspended_until_promoted`
 - `test_pool_meta_promoted_queued_decommission_can_be_canceled`
 - `test_first_resumable_decommission_queue_indices_stops_at_failed_or_canceled_state`

rustfs/src/admin/handlers/pools.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use http::{HeaderMap, StatusCode, Uri};
+use http::{HeaderMap, HeaderValue, StatusCode, Uri};
 use matchit::Params;
 use rustfs_policy::policy::action::{Action, AdminAction};
 use rustfs_utils::{
@@ -26,11 +26,12 @@ use tracing::{error, info, warn};
 
 use crate::{
     admin::{
+        EndpointServerPools, PeerRestClient,
         auth::validate_admin_request,
         router::{AdminOperation, Operation, S3Router},
     },
     app::admin_usecase::{DefaultAdminUsecase, QueryPoolStatusRequest},
-    app::context::{resolve_endpoints_handle, resolve_object_store_handle},
+    app::context::{resolve_endpoints_handle, resolve_notification_system, resolve_object_store_handle},
     auth::{check_key_valid, get_session_token},
     error::ApiError,
     server::{ADMIN_PREFIX, RemoteAddr},
@@ -213,6 +214,84 @@ fn validate_start_decommission_guards(decommission_running: bool, rebalance_runn
     Ok(())
 }
 
+#[cfg(test)]
+fn validate_pool_mutation_leader(
+    endpoints: &EndpointServerPools,
+    idx: usize,
+    operation: &str,
+    audit: PoolAuditContext<'_>,
+) -> s3s::S3Result<()> {
+    let endpoint = endpoints
+        .as_ref()
+        .get(idx)
+        .and_then(|pool| pool.endpoints.as_ref().first())
+        .ok_or_else(|| pool_admin_pool_index_error_with_audit(operation, idx, endpoints.as_ref().len(), audit))?;
+
+    if !endpoint.is_local {
+        log_pool_request_rejected_with_index_audit(
+            operation_to_event(operation),
+            "not_pool_leader",
+            idx,
+            endpoints.as_ref().len(),
+            audit,
+        );
+        return Err(S3Error::with_message(
+            S3ErrorCode::OperationAborted,
+            format!("Failed to {operation}: pool {idx} must be handled by its first endpoint {endpoint}"),
+        ));
+    }
+
+    Ok(())
+}
+
+fn decommission_peer_target(
+    endpoints: &EndpointServerPools,
+    idx: usize,
+    operation: &str,
+    audit: PoolAuditContext<'_>,
+) -> s3s::S3Result<Option<PeerRestClient>> {
+    let endpoint = endpoints
+        .as_ref()
+        .get(idx)
+        .and_then(|pool| pool.endpoints.as_ref().first())
+        .ok_or_else(|| pool_admin_pool_index_error_with_audit(operation, idx, endpoints.as_ref().len(), audit))?;
+
+    if endpoint.is_local {
+        return Ok(None);
+    }
+
+    let grid_host = endpoint.grid_host();
+    let Some(notification_sys) = resolve_notification_system() else {
+        log_pool_request_rejected_with_index_audit(
+            operation_to_event(operation),
+            "notification_sys_not_initialized",
+            idx,
+            endpoints.as_ref().len(),
+            audit,
+        );
+        return Err(S3Error::with_message(
+            S3ErrorCode::OperationAborted,
+            format!("Failed to {operation}: target pool first endpoint is not reachable"),
+        ));
+    };
+
+    let Some(client) = notification_sys.peer_client_for_grid_host(&grid_host) else {
+        log_pool_request_rejected_with_index_audit(
+            operation_to_event(operation),
+            "target_peer_not_found",
+            idx,
+            endpoints.as_ref().len(),
+            audit,
+        );
+        return Err(S3Error::with_message(
+            S3ErrorCode::OperationAborted,
+            format!("Failed to {operation}: target pool first endpoint is not reachable"),
+        ));
+    };
+
+    Ok(Some(client))
+}
+
 fn contextualize_admin_pool_api_error(
     err: crate::error::ApiError,
     operation: &str,
@@ -304,6 +383,7 @@ fn operation_to_event(operation: &str) -> &'static str {
     match operation {
         "list pools" => "list_pools",
         "load pool status" => "query_pool_status",
+        "load decommission status" => "query_decommission_status",
         "start decommission" => "start_decommission",
         "cancel decommission" => "cancel_decommission",
         "clear decommission" => "clear_decommission",
@@ -339,6 +419,12 @@ pub fn register_pool_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<
         AdminOperation(&StatusPool {}),
     )?;
 
+    r.insert(
+        Method::GET,
+        format!("{}{}", ADMIN_PREFIX, "/v3/decommission/status").as_str(),
+        AdminOperation(&StatusDecommission {}),
+    )?;
+
     r.insert(
         Method::POST,
         format!("{}{}", ADMIN_PREFIX, "/v3/pools/decommission").as_str(),
@@ -508,6 +594,62 @@ impl Operation for StatusPool {
     }
 }
 
+pub struct StatusDecommission {}
+
+#[async_trait::async_trait]
+impl Operation for StatusDecommission {
+    // GET <endpoint>/<admin-API>/decommission/status[?pool=http://server{1...4}/disk{1...4}]
+    #[tracing::instrument(skip_all)]
+    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let Some(input_cred) = req.credentials else {
+            return Err(pool_admin_missing_credentials_error("load decommission status"));
+        };
+
+        let (cred, owner) =
+            check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+
+        validate_admin_request(
+            &req.headers,
+            &cred,
+            owner,
+            false,
+            vec![
+                Action::AdminAction(AdminAction::ServerInfoAdminAction),
+                Action::AdminAction(AdminAction::DecommissionAdminAction),
+            ],
+            req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
+        )
+        .await?;
+
+        let query = parse_status_pool_query(&req.uri).map_err(|_| pool_admin_query_parse_error("load decommission status"))?;
+
+        let usecase = DefaultAdminUsecase::from_global();
+        let data = if query.pool.is_empty() {
+            let status = usecase.execute_list_decommission_status().await.map_err(S3Error::from)?;
+            serde_json::to_vec(&status)
+        } else {
+            let status = usecase
+                .execute_query_decommission_status(QueryPoolStatusRequest {
+                    pool: query.pool,
+                    by_id: query.by_id.as_str() == "true",
+                })
+                .await
+                .map_err(S3Error::from)?;
+            serde_json::to_vec(&status)
+        }
+        .map_err(|e| {
+            log_pool_request_failed!("query_decommission_status", "serialize_decommission_status_failed", e);
+            S3Error::with_message(S3ErrorCode::InternalError, "serialize decommission status failed")
+        })?;
+
+        let mut header = HeaderMap::new();
+        header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
+        log_pool_response_emitted!("query_decommission_status");
+
+        Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
+    }
+}
+
 pub struct StartDecommission {}
 
 #[async_trait::async_trait]
@@ -572,27 +714,6 @@ impl Operation for StartDecommission {
             return Err(decommission_admin_not_initialized_error_with_audit("start decommission", audit));
         };
 
-        let decommission_running = store.is_decommission_running().await;
-        let rebalance_running = store.is_rebalance_started().await;
-        if decommission_running {
-            log_pool_request_rejected_with_context(
-                "start_decommission",
-                "decommission_already_running",
-                &request_id,
-                &actor,
-                &remote_addr,
-            );
-        } else if rebalance_running {
-            log_pool_request_rejected_with_context(
-                "start_decommission",
-                "rebalance_in_progress",
-                &request_id,
-                &actor,
-                &remote_addr,
-            );
-        }
-        validate_start_decommission_guards(decommission_running, rebalance_running)?;
-
         let query = parse_mutation_pool_query(&req.uri)
             .map_err(|_| pool_admin_query_parse_error_with_audit("start decommission", audit))?;
         let is_byid = query.by_id.as_str() == "true";
@@ -631,13 +752,49 @@ impl Operation for StartDecommission {
         }
         let pools_indices = parsed_indices;
 
-        if !pools_indices.is_empty() {
+        if let Some(first_idx) = pools_indices.first().copied()
+            && let Some(client) = decommission_peer_target(&endpoints, first_idx, "start decommission", audit)?
+        {
             let pool_context = format!("pools {:?}", &pools_indices);
-            store
-                .decommission(ctx.clone(), pools_indices.clone())
+            client
+                .start_decommission(pools_indices.clone())
                 .await
                 .map_err(ApiError::from)
                 .map_err(|err| contextualize_admin_pool_api_error(err, "start decommission", &pool_context))?;
+        } else {
+            store
+                .load_rebalance_meta()
+                .await
+                .map_err(|e| s3_error!(InternalError, "failed to refresh rebalance metadata before decommission start: {}", e))?;
+            let decommission_running = store.is_decommission_running().await;
+            let rebalance_running = store.is_rebalance_started().await;
+            if decommission_running {
+                log_pool_request_rejected_with_context(
+                    "start_decommission",
+                    "decommission_already_running",
+                    &request_id,
+                    &actor,
+                    &remote_addr,
+                );
+            } else if rebalance_running {
+                log_pool_request_rejected_with_context(
+                    "start_decommission",
+                    "rebalance_in_progress",
+                    &request_id,
+                    &actor,
+                    &remote_addr,
+                );
+            }
+            validate_start_decommission_guards(decommission_running, rebalance_running)?;
+
+            if !pools_indices.is_empty() {
+                let pool_context = format!("pools {:?}", &pools_indices);
+                store
+                    .decommission(ctx.clone(), pools_indices.clone())
+                    .await
+                    .map_err(ApiError::from)
+                    .map_err(|err| contextualize_admin_pool_api_error(err, "start decommission", &pool_context))?;
+            }
         }
 
         info!(
@@ -734,15 +891,23 @@ impl Operation for CancelDecommission {
             return Err(pool_admin_pool_not_found_error_with_audit("cancel decommission", &query.pool, audit));
         };
 
-        let Some(store) = resolve_object_store_handle() else {
-            return Err(decommission_admin_not_initialized_error_with_audit("cancel decommission", audit));
-        };
+        if let Some(client) = decommission_peer_target(&endpoints, idx, "cancel decommission", audit)? {
+            client
+                .decommission_cancel(idx)
+                .await
+                .map_err(ApiError::from)
+                .map_err(|err| contextualize_admin_pool_api_error(err, "cancel decommission", format!("pool {idx}")))?;
+        } else {
+            let Some(store) = resolve_object_store_handle() else {
+                return Err(decommission_admin_not_initialized_error_with_audit("cancel decommission", audit));
+            };
 
-        store
-            .decommission_cancel(idx)
-            .await
-            .map_err(ApiError::from)
-            .map_err(|err| contextualize_admin_pool_api_error(err, "cancel decommission", format!("pool {idx}")))?;
+            store
+                .decommission_cancel(idx)
+                .await
+                .map_err(ApiError::from)
+                .map_err(|err| contextualize_admin_pool_api_error(err, "cancel decommission", format!("pool {idx}")))?;
+        }
 
         info!(
             event = EVENT_ADMIN_RESPONSE_EMITTED,
@@ -839,15 +1004,23 @@ impl Operation for ClearDecommission {
             return Err(pool_admin_pool_not_found_error_with_audit("clear decommission", &query.pool, audit));
         };
 
-        let Some(store) = resolve_object_store_handle() else {
-            return Err(decommission_admin_not_initialized_error_with_audit("clear decommission", audit));
-        };
+        if let Some(client) = decommission_peer_target(&endpoints, idx, "clear decommission", audit)? {
+            client
+                .clear_decommission(idx)
+                .await
+                .map_err(ApiError::from)
+                .map_err(|err| contextualize_admin_pool_api_error(err, "clear decommission", format!("pool {idx}")))?;
+        } else {
+            let Some(store) = resolve_object_store_handle() else {
+                return Err(decommission_admin_not_initialized_error_with_audit("clear decommission", audit));
+            };
 
-        store
-            .clear_decommission(idx)
-            .await
-            .map_err(ApiError::from)
-            .map_err(|err| contextualize_admin_pool_api_error(err, "clear decommission", format!("pool {idx}")))?;
+            store
+                .clear_decommission(idx)
+                .await
+                .map_err(ApiError::from)
+                .map_err(|err| contextualize_admin_pool_api_error(err, "clear decommission", format!("pool {idx}")))?;
+        }
 
         info!(
             event = EVENT_ADMIN_RESPONSE_EMITTED,
@@ -870,12 +1043,26 @@ impl Operation for ClearDecommission {
 mod pools_handler_tests {
     use super::{
         PoolAuditContext, contextualize_admin_pool_api_error, decommission_admin_not_initialized_error_with_audit,
-        has_duplicate_indices, parse_mutation_pool_query, parse_pool_idx_by_id, parse_status_pool_query,
-        pool_admin_missing_credentials_error, pool_admin_missing_credentials_error_with_request,
+        decommission_peer_target, has_duplicate_indices, parse_mutation_pool_query, parse_pool_idx_by_id,
+        parse_status_pool_query, pool_admin_missing_credentials_error, pool_admin_missing_credentials_error_with_request,
         pool_admin_pool_index_error_with_audit, pool_admin_pool_not_found_error_with_audit,
         pool_admin_pool_parse_error_with_audit, pool_admin_query_parse_error, pool_admin_query_parse_error_with_audit,
-        validate_start_decommission_guards,
+        validate_pool_mutation_leader, validate_start_decommission_guards,
     };
+    use crate::admin::{Endpoint, EndpointServerPools, Endpoints, PoolEndpoints};
+
+    fn test_pool_endpoints(is_local: bool) -> EndpointServerPools {
+        let mut endpoint = Endpoint::try_from("http://127.0.0.1:9000/disk").expect("test endpoint should parse");
+        endpoint.is_local = is_local;
+        EndpointServerPools::from(vec![PoolEndpoints {
+            legacy: false,
+            set_count: 1,
+            drives_per_set: 1,
+            cmd_line: "http://127.0.0.1:9000/disk".to_string(),
+            endpoints: Endpoints::from(vec![endpoint]),
+            platform: String::new(),
+        }])
+    }
 
     #[test]
     fn test_parse_pool_idx_by_id_rejects_non_numeric() {
@@ -969,6 +1156,41 @@ mod pools_handler_tests {
         assert!(validate_start_decommission_guards(false, false).is_ok());
     }
 
+    #[test]
+    fn test_validate_pool_mutation_leader_allows_local_first_endpoint() {
+        let endpoints = test_pool_endpoints(true);
+        let audit = PoolAuditContext::new("request", "actor", "remote");
+
+        assert!(validate_pool_mutation_leader(&endpoints, 0, "cancel decommission", audit).is_ok());
+    }
+
+    #[test]
+    fn test_decommission_peer_target_returns_none_for_local_first_endpoint() {
+        let endpoints = test_pool_endpoints(true);
+        let audit = PoolAuditContext::new("request", "actor", "remote");
+
+        let target = decommission_peer_target(&endpoints, 0, "start decommission", audit)
+            .expect("local first endpoint should resolve without peer lookup");
+
+        assert!(target.is_none());
+    }
+
+    #[test]
+    fn test_validate_pool_mutation_leader_rejects_remote_first_endpoint() {
+        let endpoints = test_pool_endpoints(false);
+        let audit = PoolAuditContext::new("request", "actor", "remote");
+
+        let err = validate_pool_mutation_leader(&endpoints, 0, "cancel decommission", audit)
+            .expect_err("remote first endpoint should reject mutation");
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::OperationAborted);
+        assert!(
+            err.message()
+                .expect("rejection should include message")
+                .contains("must be handled by its first endpoint")
+        );
+    }
+
     #[test]
     fn test_contextualize_admin_pool_api_error_preserves_code_and_adds_pool_context() {
         let err = crate::error::ApiError {

rustfs/src/admin/handlers/rebalance.rs

@@ -323,8 +323,8 @@ fn rebalance_used_pct(total: u64, available: u64) -> f64 {
     (total - bounded_available) as f64 / total as f64
 }
 
-fn rebalance_remaining_buckets(buckets: usize, rebalanced_buckets: usize) -> usize {
-    buckets.saturating_sub(rebalanced_buckets)
+fn rebalance_remaining_buckets(buckets: usize, _rebalanced_buckets: usize) -> usize {
+    buckets
 }
 
 fn rebalance_pool_used(disk_stats: &[DiskStat], idx: usize) -> f64 {
@@ -1090,7 +1090,7 @@ mod rebalance_handler_tests {
         assert_eq!(progress.num_objects, 3);
         assert_eq!(progress.num_versions, 5);
         assert_eq!(progress.bytes, 100);
-        assert_eq!(progress.remaining_buckets, 2);
+        assert_eq!(progress.remaining_buckets, 3);
         assert_eq!(progress.bucket, "bucket-b");
         assert_eq!(progress.object, "obj-1");
         assert_eq!(progress.elapsed, 50);
@@ -1157,9 +1157,9 @@ mod rebalance_handler_tests {
     }
 
     #[test]
-    fn test_rebalance_remaining_buckets_is_saturating_sub() {
-        assert_eq!(rebalance_remaining_buckets(10, 7), 3);
-        assert_eq!(rebalance_remaining_buckets(3, 10), 0);
+    fn test_rebalance_remaining_buckets_uses_pending_queue_len() {
+        assert_eq!(rebalance_remaining_buckets(10, 7), 10);
+        assert_eq!(rebalance_remaining_buckets(3, 10), 3);
     }
 
     #[test]
@@ -1227,7 +1227,7 @@ mod rebalance_handler_tests {
         assert_eq!(active.used, 0.5);
         assert_eq!(active.progress.as_ref().unwrap().bucket, "bucket-b");
         assert_eq!(active.progress.as_ref().unwrap().object, "obj-2");
-        assert_eq!(active.progress.as_ref().unwrap().remaining_buckets, 1);
+        assert_eq!(active.progress.as_ref().unwrap().remaining_buckets, 2);
 
         let inactive = &statuses[1];
         assert_eq!(inactive.id, 1);

rustfs/src/admin/route_policy.rs

@@ -235,6 +235,12 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
         DECOMMISSION,
         RouteRiskLevel::High,
     ),
+    admin(
+        HttpMethod::Get,
+        "/rustfs/admin/v3/decommission/status",
+        DECOMMISSION,
+        RouteRiskLevel::Sensitive,
+    ),
     admin(HttpMethod::Post, "/rustfs/admin/v3/pools/cancel", DECOMMISSION, RouteRiskLevel::High),
     admin(HttpMethod::Post, "/rustfs/admin/v3/pools/clear", DECOMMISSION, RouteRiskLevel::High),
     admin(HttpMethod::Post, "/rustfs/admin/v3/rebalance/start", REBALANCE, RouteRiskLevel::High),

rustfs/src/admin/route_registration_test.rs

@@ -173,6 +173,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
         admin_route(Method::GET, "/v3/metrics"),
         admin_route(Method::GET, "/v3/pools/list"),
         admin_route(Method::GET, "/v3/pools/status"),
+        admin_route(Method::GET, "/v3/decommission/status"),
         admin_route(Method::POST, "/v3/pools/decommission"),
         admin_route(Method::POST, "/v3/pools/cancel"),
         admin_route(Method::POST, "/v3/pools/clear"),
@@ -1036,6 +1037,7 @@ fn test_register_routes_cover_representative_admin_paths() {
     assert_route(&router, Method::GET, &admin_path("/v3/metrics"));
 
     assert_route(&router, Method::GET, &admin_path("/v3/pools/list"));
+    assert_route(&router, Method::GET, &admin_path("/v3/decommission/status"));
     assert_route(&router, Method::POST, &admin_path("/v3/rebalance/start"));
     assert_route(&router, Method::GET, &admin_path("/v3/rebalance/status"));
     assert_route(&router, Method::POST, &admin_path("/v3/heal/"));

rustfs/src/app/admin_usecase.rs

@@ -17,7 +17,7 @@
 use super::ECStore;
 use super::EndpointServerPools;
 use super::get_server_info;
-use super::{PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free};
+use super::{PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free};
 use super::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend};
 use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
 use crate::capacity::resolve_admin_used_capacity;
@@ -81,6 +81,8 @@ pub struct AdminPoolDecommissionInfo {
     pub prefix: String,
     #[serde(rename = "object")]
     pub object: String,
+    #[serde(rename = "stage")]
+    pub stage: String,
     #[serde(rename = "objectsDecommissioned")]
     pub items_decommissioned: usize,
     #[serde(rename = "objectsDecommissionedFailed")]
@@ -111,24 +113,58 @@ pub struct AdminPoolStatus {
     pub used: f64,
     #[serde(rename = "status")]
     pub status: String,
+    #[serde(rename = "decommissionStatus")]
+    pub decommission_status: String,
+    #[serde(rename = "rebalanceStatus")]
+    pub rebalance_status: String,
     #[serde(rename = "decommissionInfo")]
     pub decommission: Option<AdminPoolDecommissionInfo>,
 }
 
 pub type AdminPoolListItem = AdminPoolStatus;
 
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct AdminDecommissionPoolStatus {
+    #[serde(rename = "id")]
+    pub id: usize,
+    #[serde(rename = "cmdline")]
+    pub cmd_line: String,
+    #[serde(rename = "status")]
+    pub status: String,
+    #[serde(rename = "poolStatus")]
+    pub pool_status: String,
+    #[serde(rename = "decommissionInfo")]
+    pub decommission: Option<AdminPoolDecommissionInfo>,
+}
+
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct AdminDecommissionStatus {
+    #[serde(rename = "pools")]
+    pub pools: Vec<AdminDecommissionPoolStatus>,
+}
+
 #[derive(Clone, Default)]
 pub struct DefaultAdminUsecase {
     context: Option<Arc<AppContext>>,
 }
 
 impl DefaultAdminUsecase {
-    const POOL_STATUS_ACTIVE: &'static str = "active";
     const POOL_STATUS_CANCELED: &'static str = "canceled";
     const POOL_STATUS_COMPLETE: &'static str = "complete";
     const POOL_STATUS_FAILED: &'static str = "failed";
     const POOL_STATUS_QUEUED: &'static str = "queued";
     const POOL_STATUS_RUNNING: &'static str = "running";
+    const POOL_STATUS_UNKNOWN: &'static str = "unknown";
+    const POOL_STATE_ACTIVE: &'static str = "active";
+    const POOL_STATE_BLOCKED: &'static str = "blocked";
+    const POOL_STATE_DECOMMISSIONED: &'static str = "decommissioned";
+    const POOL_STATE_DECOMMISSIONING: &'static str = "decommissioning";
+    const REBALANCE_STATUS_COMPLETED: &'static str = "completed";
+    const REBALANCE_STATUS_FAILED: &'static str = "failed";
+    const REBALANCE_STATUS_NONE: &'static str = "none";
+    const REBALANCE_STATUS_STARTED: &'static str = "started";
+    const REBALANCE_STATUS_STOPPING: &'static str = "stopping";
+    const REBALANCE_STATUS_STOPPED: &'static str = "stopped";
 
     #[cfg(test)]
     pub fn without_context() -> Self {
@@ -277,19 +313,19 @@ impl DefaultAdminUsecase {
     }
 
     pub async fn execute_list_pools(&self) -> AdminUsecaseResult<Vec<AdminPoolListItem>> {
+        let Some(store) = self.object_store() else {
+            return Err(Self::app_error(S3ErrorCode::InternalError, "Not init"));
+        };
         let pool_statuses = self.execute_list_pool_statuses().await?;
-        Ok(pool_statuses.into_iter().map(Self::pool_list_item_from_status).collect())
+        let mut items = Vec::with_capacity(pool_statuses.len());
+        for status in pool_statuses {
+            let rebalance_status = store.pool_rebalance_status(status.id).await;
+            items.push(Self::pool_list_item_from_status(status, rebalance_status));
+        }
+        Ok(items)
     }
 
-    pub async fn execute_query_pool_status(&self, req: QueryPoolStatusRequest) -> AdminUsecaseResult<AdminPoolStatus> {
-        let Some(endpoints) = self.endpoints() else {
-            return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
-        };
-
-        if endpoints.legacy() {
-            return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
-        }
-
+    fn resolve_pool_index(&self, req: &QueryPoolStatusRequest, endpoints: &EndpointServerPools) -> AdminUsecaseResult<usize> {
         let has_idx = if req.by_id {
             Self::parse_pool_idx_by_id(&req.pool, endpoints.as_ref().len())
         } else {
@@ -301,18 +337,62 @@ impl DefaultAdminUsecase {
             return Err(Self::app_error_default(S3ErrorCode::InvalidArgument));
         };
 
+        Ok(idx)
+    }
+
+    pub async fn execute_query_pool_status(&self, req: QueryPoolStatusRequest) -> AdminUsecaseResult<AdminPoolStatus> {
+        let Some(endpoints) = self.endpoints() else {
+            return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
+        };
+
+        if endpoints.legacy() {
+            return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
+        }
+
+        let idx = self.resolve_pool_index(&req, &endpoints)?;
+
         let Some(store) = self.object_store() else {
             return Err(Self::app_error(S3ErrorCode::InternalError, "Not init"));
         };
 
-        store
-            .status(idx)
-            .await
-            .map(Self::pool_list_item_from_status)
-            .map_err(ApiError::from)
+        let status = store.status(idx).await.map_err(ApiError::from)?;
+        let rebalance_status = store.pool_rebalance_status(idx).await;
+        Ok(Self::pool_list_item_from_status(status, rebalance_status))
     }
 
-    fn pool_list_item_from_status(status: PoolStatus) -> AdminPoolListItem {
+    pub async fn execute_list_decommission_status(&self) -> AdminUsecaseResult<AdminDecommissionStatus> {
+        let pool_statuses = self.execute_list_pool_statuses().await?;
+        Ok(AdminDecommissionStatus {
+            pools: pool_statuses
+                .into_iter()
+                .map(Self::decommission_pool_status_from_status)
+                .collect(),
+        })
+    }
+
+    pub async fn execute_query_decommission_status(
+        &self,
+        req: QueryPoolStatusRequest,
+    ) -> AdminUsecaseResult<AdminDecommissionPoolStatus> {
+        let Some(endpoints) = self.endpoints() else {
+            return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
+        };
+
+        if endpoints.legacy() {
+            return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
+        }
+
+        let idx = self.resolve_pool_index(&req, &endpoints)?;
+
+        let Some(store) = self.object_store() else {
+            return Err(Self::app_error(S3ErrorCode::InternalError, "Not init"));
+        };
+
+        let status = store.status(idx).await.map_err(ApiError::from)?;
+        Ok(Self::decommission_pool_status_from_status(status))
+    }
+
+    fn pool_list_item_from_status(status: PoolStatus, rebalance_status: (RebalStatus, bool)) -> AdminPoolListItem {
         let PoolStatus {
             id,
             cmd_line,
@@ -322,6 +402,7 @@ impl DefaultAdminUsecase {
         let total_size = decommission.as_ref().map(|info| info.total_size).unwrap_or_default();
         let current_size = decommission.as_ref().map(|info| info.current_size).unwrap_or_default();
         let used_size = total_size.saturating_sub(current_size);
+        let pool_state = Self::pool_lifecycle_state(decommission.as_ref());
 
         AdminPoolStatus {
             id,
@@ -331,19 +412,64 @@ impl DefaultAdminUsecase {
             current_size,
             used_size,
             used: Self::used_ratio(total_size, used_size),
-            status: Self::pool_list_status(decommission.as_ref()).to_string(),
+            status: pool_state.to_string(),
+            decommission_status: Self::pool_decommission_status(decommission.as_ref()).to_string(),
+            rebalance_status: Self::pool_rebalance_status(rebalance_status).to_string(),
             decommission: decommission.map(Self::admin_decommission_info_from_pool),
         }
     }
 
-    fn pool_list_status(decommission: Option<&PoolDecommissionInfo>) -> &'static str {
+    fn pool_lifecycle_state(decommission: Option<&PoolDecommissionInfo>) -> &'static str {
+        match decommission {
+            Some(info) if info.complete => Self::POOL_STATE_DECOMMISSIONED,
+            Some(info) if info.failed || info.canceled => Self::POOL_STATE_BLOCKED,
+            Some(_) => Self::POOL_STATE_DECOMMISSIONING,
+            None => Self::POOL_STATE_ACTIVE,
+        }
+    }
+
+    fn pool_decommission_status(decommission: Option<&PoolDecommissionInfo>) -> &'static str {
         match decommission {
             Some(info) if info.complete => Self::POOL_STATUS_COMPLETE,
             Some(info) if info.failed => Self::POOL_STATUS_FAILED,
             Some(info) if info.canceled => Self::POOL_STATUS_CANCELED,
             Some(info) if info.queued => Self::POOL_STATUS_QUEUED,
             Some(info) if info.start_time.is_some() => Self::POOL_STATUS_RUNNING,
-            _ => Self::POOL_STATUS_ACTIVE,
+            Some(_) => Self::POOL_STATUS_UNKNOWN,
+            None => Self::REBALANCE_STATUS_NONE,
+        }
+    }
+
+    fn pool_rebalance_status((status, stopping): (RebalStatus, bool)) -> &'static str {
+        if stopping {
+            return Self::REBALANCE_STATUS_STOPPING;
+        }
+
+        match status {
+            RebalStatus::None => Self::REBALANCE_STATUS_NONE,
+            RebalStatus::Started => Self::REBALANCE_STATUS_STARTED,
+            RebalStatus::Completed => Self::REBALANCE_STATUS_COMPLETED,
+            RebalStatus::Stopped => Self::REBALANCE_STATUS_STOPPED,
+            RebalStatus::Failed => Self::REBALANCE_STATUS_FAILED,
+        }
+    }
+
+    fn decommission_pool_status_from_status(status: PoolStatus) -> AdminDecommissionPoolStatus {
+        let PoolStatus {
+            id,
+            cmd_line,
+            decommission,
+            ..
+        } = status;
+        let pool_status = Self::pool_lifecycle_state(decommission.as_ref()).to_string();
+        let status = Self::pool_decommission_status(decommission.as_ref()).to_string();
+
+        AdminDecommissionPoolStatus {
+            id,
+            cmd_line,
+            status,
+            pool_status,
+            decommission: decommission.map(Self::admin_decommission_info_from_pool),
         }
     }
 
@@ -363,6 +489,7 @@ impl DefaultAdminUsecase {
             bucket: info.bucket,
             prefix: info.prefix,
             object: info.object,
+            stage: info.stage,
             items_decommissioned: info.items_decommissioned,
             items_decommission_failed: info.items_decommission_failed,
             bytes_done: info.bytes_done,
@@ -446,7 +573,7 @@ mod tests {
     }
 
     #[test]
-    fn admin_pool_list_item_maps_capacity_and_active_status() {
+    fn admin_pool_list_item_maps_capacity_and_unknown_decommission_status() {
         let now = OffsetDateTime::UNIX_EPOCH;
         let pool = PoolStatus {
             id: 2,
@@ -459,24 +586,29 @@ mod tests {
             }),
         };
 
-        let item = DefaultAdminUsecase::pool_list_item_from_status(pool);
+        let item = DefaultAdminUsecase::pool_list_item_from_status(pool, (RebalStatus::None, false));
 
         assert_eq!(item.id, 2);
         assert_eq!(item.total_size, 1_000);
         assert_eq!(item.current_size, 250);
         assert_eq!(item.used_size, 750);
         assert!((item.used - 0.75).abs() < f64::EPSILON);
-        assert_eq!(item.status, "active");
+        assert_eq!(item.status, "decommissioning");
+        assert_eq!(item.decommission_status, "unknown");
+        assert_eq!(item.rebalance_status, "none");
     }
 
     #[test]
     fn admin_pool_list_item_serializes_admin_api_fields() {
-        let item = DefaultAdminUsecase::pool_list_item_from_status(PoolStatus {
-            id: 1,
-            cmd_line: "pool-1".to_string(),
-            last_update: OffsetDateTime::UNIX_EPOCH,
-            decommission: None,
-        });
+        let item = DefaultAdminUsecase::pool_list_item_from_status(
+            PoolStatus {
+                id: 1,
+                cmd_line: "pool-1".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: None,
+            },
+            (RebalStatus::Completed, false),
+        );
 
         let value = serde_json::to_value(item).unwrap();
 
@@ -491,6 +623,8 @@ mod tests {
                 "usedSize": 0,
                 "used": 0.0,
                 "status": "active",
+                "decommissionStatus": "none",
+                "rebalanceStatus": "completed",
                 "decommissionInfo": null
             })
         );
@@ -509,7 +643,7 @@ mod tests {
             }),
         };
 
-        let item = DefaultAdminUsecase::pool_list_item_from_status(pool);
+        let item = DefaultAdminUsecase::pool_list_item_from_status(pool, (RebalStatus::None, false));
 
         assert_eq!(item.total_size, 100);
         assert_eq!(item.current_size, 150);
@@ -531,33 +665,40 @@ mod tests {
             }),
         };
 
-        let item = DefaultAdminUsecase::pool_list_item_from_status(pool);
+        let item = DefaultAdminUsecase::pool_list_item_from_status(pool, (RebalStatus::Started, false));
 
-        assert_eq!(item.status, "running");
+        assert_eq!(item.status, "decommissioning");
+        assert_eq!(item.decommission_status, "running");
+        assert_eq!(item.rebalance_status, "started");
     }
 
     #[test]
     fn admin_pool_list_item_exposes_queued_decommission_state() {
-        let item = DefaultAdminUsecase::pool_list_item_from_status(PoolStatus {
-            id: 3,
-            cmd_line: "pool-3".to_string(),
-            last_update: OffsetDateTime::UNIX_EPOCH,
-            decommission: Some(PoolDecommissionInfo {
-                queued: true,
-                queued_buckets: vec!["bucket-a".to_string(), ".rustfs.sys/config".to_string()],
-                decommissioned_buckets: vec!["bucket-done".to_string()],
-                bucket: "bucket-a".to_string(),
-                prefix: "prefix/".to_string(),
-                object: "object.txt".to_string(),
-                items_decommissioned: 7,
-                items_decommission_failed: 1,
-                bytes_done: 1024,
-                bytes_failed: 64,
-                ..Default::default()
-            }),
-        });
+        let item = DefaultAdminUsecase::pool_list_item_from_status(
+            PoolStatus {
+                id: 3,
+                cmd_line: "pool-3".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: Some(PoolDecommissionInfo {
+                    queued: true,
+                    queued_buckets: vec!["bucket-a".to_string(), ".rustfs.sys/config".to_string()],
+                    decommissioned_buckets: vec!["bucket-done".to_string()],
+                    bucket: "bucket-a".to_string(),
+                    prefix: "prefix/".to_string(),
+                    object: "object.txt".to_string(),
+                    stage: "migrate_object".to_string(),
+                    items_decommissioned: 7,
+                    items_decommission_failed: 1,
+                    bytes_done: 1024,
+                    bytes_failed: 64,
+                    ..Default::default()
+                }),
+            },
+            (RebalStatus::None, false),
+        );
 
-        assert_eq!(item.status, "queued");
+        assert_eq!(item.status, "decommissioning");
+        assert_eq!(item.decommission_status, "queued");
         let value = serde_json::to_value(item).expect("admin pool status should serialize");
         assert_eq!(value["decommissionInfo"]["queued"], true);
         assert_eq!(
@@ -568,6 +709,7 @@ mod tests {
         assert_eq!(value["decommissionInfo"]["bucket"], "bucket-a");
         assert_eq!(value["decommissionInfo"]["prefix"], "prefix/");
         assert_eq!(value["decommissionInfo"]["object"], "object.txt");
+        assert_eq!(value["decommissionInfo"]["stage"], "migrate_object");
         assert_eq!(value["decommissionInfo"]["objectsDecommissioned"], 7);
         assert_eq!(value["decommissionInfo"]["objectsDecommissionedFailed"], 1);
         assert_eq!(value["decommissionInfo"]["bytesDecommissioned"], 1024);
@@ -577,28 +719,128 @@ mod tests {
 
     #[test]
     fn admin_pool_list_item_maps_terminal_decommission_statuses() {
-        let complete = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo {
+        let complete = DefaultAdminUsecase::pool_decommission_status(Some(&PoolDecommissionInfo {
             complete: true,
             ..Default::default()
         }));
-        let failed = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo {
+        let failed = DefaultAdminUsecase::pool_decommission_status(Some(&PoolDecommissionInfo {
             failed: true,
             ..Default::default()
         }));
-        let canceled = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo {
+        let canceled = DefaultAdminUsecase::pool_decommission_status(Some(&PoolDecommissionInfo {
             canceled: true,
             ..Default::default()
         }));
-        let queued = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo {
+        let queued = DefaultAdminUsecase::pool_decommission_status(Some(&PoolDecommissionInfo {
             queued: true,
             ..Default::default()
         }));
-        let idle = DefaultAdminUsecase::pool_list_status(None);
+        let idle = DefaultAdminUsecase::pool_decommission_status(None);
 
         assert_eq!(complete, "complete");
         assert_eq!(failed, "failed");
         assert_eq!(canceled, "canceled");
         assert_eq!(queued, "queued");
-        assert_eq!(idle, "active");
+        assert_eq!(idle, "none");
+    }
+
+    #[test]
+    fn admin_pool_list_item_keeps_rebalance_failure_separate_from_pool_state() {
+        let item = DefaultAdminUsecase::pool_list_item_from_status(
+            PoolStatus {
+                id: 1,
+                cmd_line: "pool-1".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: None,
+            },
+            (RebalStatus::Failed, false),
+        );
+
+        assert_eq!(item.status, "active");
+        assert_eq!(item.decommission_status, "none");
+        assert_eq!(item.rebalance_status, "failed");
+    }
+
+    #[test]
+    fn admin_pool_list_item_maps_stopping_rebalance_status() {
+        let item = DefaultAdminUsecase::pool_list_item_from_status(
+            PoolStatus {
+                id: 1,
+                cmd_line: "pool-1".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: None,
+            },
+            (RebalStatus::Started, true),
+        );
+
+        assert_eq!(item.status, "active");
+        assert_eq!(item.decommission_status, "none");
+        assert_eq!(item.rebalance_status, "stopping");
+    }
+
+    #[test]
+    fn admin_pool_lifecycle_state_distinguishes_decommission_terminal_states() {
+        let complete = DefaultAdminUsecase::pool_lifecycle_state(Some(&PoolDecommissionInfo {
+            complete: true,
+            ..Default::default()
+        }));
+        let failed = DefaultAdminUsecase::pool_lifecycle_state(Some(&PoolDecommissionInfo {
+            failed: true,
+            ..Default::default()
+        }));
+        let canceled = DefaultAdminUsecase::pool_lifecycle_state(Some(&PoolDecommissionInfo {
+            canceled: true,
+            ..Default::default()
+        }));
+
+        assert_eq!(complete, "decommissioned");
+        assert_eq!(failed, "blocked");
+        assert_eq!(canceled, "blocked");
+    }
+
+    #[test]
+    fn admin_decommission_status_serializes_task_status_and_pool_status() {
+        let item = DefaultAdminUsecase::decommission_pool_status_from_status(PoolStatus {
+            id: 3,
+            cmd_line: "pool-3".to_string(),
+            last_update: OffsetDateTime::UNIX_EPOCH,
+            decommission: Some(PoolDecommissionInfo {
+                failed: true,
+                ..Default::default()
+            }),
+        });
+
+        let value = serde_json::to_value(item).expect("decommission status should serialize");
+
+        assert_eq!(
+            value,
+            serde_json::json!({
+                "id": 3,
+                "cmdline": "pool-3",
+                "status": "failed",
+                "poolStatus": "blocked",
+                "decommissionInfo": {
+                    "startTime": null,
+                    "startSize": 0,
+                    "totalSize": 0,
+                    "currentSize": 0,
+                    "complete": false,
+                    "failed": true,
+                    "canceled": false,
+                    "queued": false,
+                    "queuedBuckets": [],
+                    "decommissionedBuckets": [],
+                    "bucket": "",
+                    "prefix": "",
+                    "object": "",
+                    "stage": "",
+                    "objectsDecommissioned": 0,
+                    "objectsDecommissionedFailed": 0,
+                    "bytesDecommissioned": 0,
+                    "bytesDecommissionedFailed": 0,
+                    "waitingReason": null
+                }
+            })
+        );
     }
 }

rustfs/src/app/mod.rs

@@ -105,6 +105,7 @@ pub(crate) type ObjectInfo = <ECStore as rustfs_storage_api::ObjectOperations>::
 pub(crate) type ObjectOptions = <ECStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
 pub(crate) type PoolDecommissionInfo = ecstore_capacity::PoolDecommissionInfo;
 pub(crate) type PoolStatus = ecstore_capacity::PoolStatus;
+pub(crate) type RebalStatus = crate::storage::ecstore_rebalance::RebalStatus;
 pub(crate) type StorageError = crate::storage::StorageError;
 pub(crate) type Error = StorageError;
 pub(crate) type TierConfigMgr = crate::storage::TierConfigMgr;

rustfs/src/storage/rpc/node_service.rs

@@ -13,11 +13,11 @@
 // limitations under the License.
 
 use super::super::{
-    CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, FileInfoVersions, LocalPeerS3Client, MetricType,
-    PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG,
-    SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StoragePeerS3ClientExt as _, UpdateMetadataOpts, all_local_disk_path,
-    collect_local_metrics, find_local_disk_by_ref, get_local_server_property, load_bucket_metadata,
-    reload_transition_tier_config, resolve_object_store_handle, set_bucket_metadata,
+    CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, ECStore, Error, FileInfoVersions,
+    LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions,
+    SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StoragePeerS3ClientExt as _,
+    UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, get_local_server_property,
+    load_bucket_metadata, reload_transition_tier_config, resolve_object_store_handle, set_bucket_metadata,
 };
 use crate::admin::service::{
     config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot},
@@ -46,6 +46,7 @@ use std::{collections::HashMap, io::Cursor, pin::Pin, sync::Arc};
 use tokio::spawn;
 use tokio::sync::mpsc;
 use tokio_stream::wrappers::ReceiverStream;
+use tokio_util::sync::CancellationToken;
 use tonic::{Request, Response, Status, Streaming};
 use tracing::{debug, error, info, warn};
 
@@ -140,6 +141,23 @@ fn stop_rebalance_response(result: super::super::Result<()>) -> StopRebalanceRes
     }
 }
 
+fn ensure_rpc_decommission_local_leader(store: &ECStore, idx: usize) -> super::super::Result<()> {
+    let endpoints = store.endpoints();
+    let endpoint = endpoints
+        .as_ref()
+        .get(idx)
+        .and_then(|pool| pool.endpoints.as_ref().first())
+        .ok_or_else(|| Error::other(format!("invalid decommission pool index {idx} for {} pools", endpoints.as_ref().len())))?;
+
+    if !endpoint.is_local {
+        return Err(Error::other(format!(
+            "decommission for pool {idx} must run on the pool first endpoint {endpoint}"
+        )));
+    }
+
+    Ok(())
+}
+
 #[path = "bucket.rs"]
 mod bucket;
 #[path = "disk.rs"]
@@ -1057,6 +1075,101 @@ impl Node for NodeService {
         }))
     }
 
+    async fn start_decommission(
+        &self,
+        request: Request<StartDecommissionRequest>,
+    ) -> Result<Response<StartDecommissionResponse>, Status> {
+        let Some(store) = resolve_object_store_handle() else {
+            return Ok(Response::new(StartDecommissionResponse {
+                success: false,
+                error_info: Some("errServerNotInitialized".to_string()),
+            }));
+        };
+
+        let mut indices = Vec::with_capacity(request.get_ref().pool_indices.len());
+        for idx in request.into_inner().pool_indices {
+            indices.push(
+                usize::try_from(idx)
+                    .map_err(|_| Status::invalid_argument(format!("decommission pool index {idx} exceeds local range")))?,
+            );
+        }
+
+        match store.decommission(CancellationToken::new(), indices).await {
+            Ok(()) => Ok(Response::new(StartDecommissionResponse {
+                success: true,
+                error_info: None,
+            })),
+            Err(err) => Ok(Response::new(StartDecommissionResponse {
+                success: false,
+                error_info: Some(err.to_string()),
+            })),
+        }
+    }
+
+    async fn cancel_decommission(
+        &self,
+        request: Request<CancelDecommissionRequest>,
+    ) -> Result<Response<CancelDecommissionResponse>, Status> {
+        let Some(store) = resolve_object_store_handle() else {
+            return Ok(Response::new(CancelDecommissionResponse {
+                success: false,
+                error_info: Some("errServerNotInitialized".to_string()),
+            }));
+        };
+
+        let idx = usize::try_from(request.into_inner().pool_index)
+            .map_err(|_| Status::invalid_argument("decommission pool index exceeds local range"))?;
+        if let Err(err) = ensure_rpc_decommission_local_leader(&store, idx) {
+            return Ok(Response::new(CancelDecommissionResponse {
+                success: false,
+                error_info: Some(err.to_string()),
+            }));
+        }
+
+        match store.decommission_cancel(idx).await {
+            Ok(()) => Ok(Response::new(CancelDecommissionResponse {
+                success: true,
+                error_info: None,
+            })),
+            Err(err) => Ok(Response::new(CancelDecommissionResponse {
+                success: false,
+                error_info: Some(err.to_string()),
+            })),
+        }
+    }
+
+    async fn clear_decommission(
+        &self,
+        request: Request<ClearDecommissionRequest>,
+    ) -> Result<Response<ClearDecommissionResponse>, Status> {
+        let Some(store) = resolve_object_store_handle() else {
+            return Ok(Response::new(ClearDecommissionResponse {
+                success: false,
+                error_info: Some("errServerNotInitialized".to_string()),
+            }));
+        };
+
+        let idx = usize::try_from(request.into_inner().pool_index)
+            .map_err(|_| Status::invalid_argument("decommission pool index exceeds local range"))?;
+        if let Err(err) = ensure_rpc_decommission_local_leader(&store, idx) {
+            return Ok(Response::new(ClearDecommissionResponse {
+                success: false,
+                error_info: Some(err.to_string()),
+            }));
+        }
+
+        match store.clear_decommission(idx).await {
+            Ok(()) => Ok(Response::new(ClearDecommissionResponse {
+                success: true,
+                error_info: None,
+            })),
+            Err(err) => Ok(Response::new(ClearDecommissionResponse {
+                success: false,
+                error_info: Some(err.to_string()),
+            })),
+        }
+    }
+
     async fn load_transition_tier_config(
         &self,
         _request: Request<LoadTransitionTierConfigRequest>,