fix(storage): harden rebalance decommission state (#3515)

.github/workflows/ci.yml

@@ -127,6 +127,14 @@ jobs:
           cargo nextest run --all --exclude e2e_test
           cargo test --all --doc
 
+      - name: Run rebalance/decommission migration proofs
+        run: |
+          cargo test -p rustfs-ecstore data_movement --lib
+          cargo test -p rustfs-ecstore rebalance --lib
+          cargo test -p rustfs-ecstore decommission --lib
+          cargo test -p rustfs-ecstore source_cleanup --lib
+          cargo test -p rustfs-ecstore delete_marker --lib
+
       - name: Check code formatting
         run: cargo fmt --all --check
 
@@ -258,6 +266,9 @@ jobs:
       - name: Setup Rust toolchain for s3s-e2e installation
         uses: dtolnay/rust-toolchain@stable
 
+      - name: Run delete-marker migration proof
+        run: cargo test -p e2e_test delete_marker_migration_semantics -- --nocapture --test-threads=1
+
       - name: Install s3s-e2e test tool
         uses: taiki-e/cache-cargo-install-action@v2
         with:

crates/e2e_test/src/delete_marker_migration_semantics_test.rs

@@ -0,0 +1,165 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::primitives::ByteStream;
+    use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+    use serial_test::serial;
+
+    async fn create_versioned_bucket(client: &Client, bucket: &str) {
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("create versioning proof bucket");
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("enable versioning for proof bucket");
+    }
+
+    async fn assert_current_get_is_delete_marker_not_found(client: &Client, bucket: &str, key: &str) {
+        let err = client
+            .get_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect_err("current get should fail when latest version is a delete marker");
+        let service_err = err.into_service_error();
+        let code = service_err.meta().code();
+        assert!(
+            matches!(code, Some("NoSuchKey") | Some("NotFound")),
+            "current get should expose delete-marker not-found semantics, got {service_err:?}"
+        );
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_versioning_only_delete_marker_has_minio_compatible_visibility_for_migration_proof() {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
+        env.start_rustfs_server(vec![]).await.expect("start RustFS");
+        let client = env.create_s3_client();
+        let bucket = "delete-marker-migration-proof-a";
+        let key = "only-delete-marker.txt";
+
+        create_versioned_bucket(&client, bucket).await;
+
+        let delete_marker = client
+            .delete_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("create delete marker for absent object");
+        let delete_marker_version_id = delete_marker
+            .version_id()
+            .expect("MinIO-compatible delete marker should have a version id");
+
+        let listed = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("list delete marker only object");
+        let versions = listed.versions();
+        let markers = listed.delete_markers();
+
+        assert!(versions.is_empty(), "only-delete-marker case must not report data versions");
+        assert_eq!(markers.len(), 1, "only-delete-marker case must report the delete marker");
+        assert_eq!(markers[0].version_id(), Some(delete_marker_version_id));
+        assert_eq!(markers[0].is_latest(), Some(true));
+        assert_current_get_is_delete_marker_not_found(&client, bucket, key).await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_versioning_delete_marker_plus_history_remains_visible_for_migration_proof() {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
+        env.start_rustfs_server(vec![]).await.expect("start RustFS");
+        let client = env.create_s3_client();
+        let bucket = "delete-marker-migration-proof-b";
+        let key = "delete-marker-with-history.txt";
+        let body = b"historical version body";
+
+        create_versioned_bucket(&client, bucket).await;
+
+        let put = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(body))
+            .send()
+            .await
+            .expect("put historical version");
+        let data_version_id = put.version_id().expect("put should return data version id");
+
+        let delete_marker = client
+            .delete_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("create delete marker over historical version");
+        let delete_marker_version_id = delete_marker.version_id().expect("delete marker should have a version id");
+
+        let listed = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("list delete marker plus historical version");
+        let versions = listed.versions();
+        let markers = listed.delete_markers();
+
+        assert_eq!(versions.len(), 1, "history case must report the historical data version");
+        assert_eq!(markers.len(), 1, "history case must report the latest delete marker");
+        assert_eq!(versions[0].version_id(), Some(data_version_id));
+        assert_eq!(versions[0].is_latest(), Some(false));
+        assert_eq!(markers[0].version_id(), Some(delete_marker_version_id));
+        assert_eq!(markers[0].is_latest(), Some(true));
+        assert_current_get_is_delete_marker_not_found(&client, bucket, key).await;
+
+        let historical = client
+            .get_object()
+            .bucket(bucket)
+            .key(key)
+            .version_id(data_version_id)
+            .send()
+            .await
+            .expect("historical version get should succeed");
+        let bytes = historical
+            .body
+            .collect()
+            .await
+            .expect("collect historical version body")
+            .into_bytes();
+        assert_eq!(bytes.as_ref(), body);
+    }
+}

crates/e2e_test/src/lib.rs

@@ -83,6 +83,10 @@ mod delete_objects_versioning_test;
 #[cfg(test)]
 mod delete_object_no_content_length_test;
 
+// Delete-marker visibility baseline for data-movement migration proof.
+#[cfg(test)]
+mod delete_marker_migration_semantics_test;
+
 // Regression test for Issue #2252: ListObjectVersions misses newest version after put -> delete -> put
 #[cfg(test)]
 mod list_object_versions_regression_test;

crates/ecstore/src/api/mod.rs

@@ -140,7 +140,9 @@ pub mod object {
 
 pub mod rebalance {
     pub use crate::rebalance::{
-        DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta, RebalanceStats,
+        DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo,
+        RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, decode_rebalance_stop_propagation_record,
+        encode_rebalance_stop_propagation_record,
     };
 }
 

crates/ecstore/src/notification_sys.rs

@@ -32,10 +32,13 @@ use std::hash::{Hash, Hasher};
 use std::sync::{Mutex, OnceLock};
 use std::time::{Duration, SystemTime};
 use tokio::time::timeout;
-use tracing::{error, warn};
+use tracing::{debug, error, info, warn};
 
 /// After this many consecutive admin-call failures, mark the peer as offline.
 const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3;
+const LOG_COMPONENT_ECSTORE: &str = "ecstore";
+const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification";
+const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation";
 
 /// Cached result from the last successful admin call to a peer.
 struct PeerAdminCache {
@@ -472,6 +475,15 @@ impl NotificationSys {
                 let host = client.grid_host.clone();
                 futures.push(async move { client.reload_pool_meta().await.map_err(|err| (host, err)) });
             } else {
+                warn!(
+                    event = EVENT_NOTIFICATION_PEER_PROPAGATION,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    action = "reload_pool_meta",
+                    result = "peer_unreachable",
+                    peer_index = idx,
+                    "notification peer propagation"
+                );
                 failures.push(format!("peer[{idx}] reload_pool_meta failed: peer is not reachable"));
             }
         }
@@ -479,7 +491,16 @@ impl NotificationSys {
         for result in join_all(futures).await {
             if let Err((host, err)) = result {
                 let failure = format!("peer {host} reload_pool_meta failed: {err}");
-                error!("notification reload_pool_meta err {}", failure);
+                error!(
+                    event = EVENT_NOTIFICATION_PEER_PROPAGATION,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    action = "reload_pool_meta",
+                    result = "peer_failed",
+                    peer = %host,
+                    error = %err,
+                    "notification peer propagation"
+                );
                 failures.push(failure);
             }
         }
@@ -489,87 +510,187 @@ impl NotificationSys {
 
     #[tracing::instrument(skip(self))]
     pub async fn load_rebalance_meta(&self, start: bool) -> Result<()> {
+        let failures = self.load_rebalance_meta_failures(start).await?;
+        aggregate_notification_failures("load_rebalance_meta", failures)
+    }
+
+    #[tracing::instrument(skip(self))]
+    pub async fn load_rebalance_meta_failures(&self, start: bool) -> Result<Vec<String>> {
         let operation = format!("load_rebalance_meta(start={start})");
         let mut failures = Vec::new();
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for (idx, client) in self.peer_clients.iter().enumerate() {
             if let Some(client) = client {
-                warn!(
-                    "notification load_rebalance_meta start: {}, index: {}, client: {:?}",
-                    start, idx, client.host
-                );
                 let host = client.grid_host.clone();
-                futures.push(async move { client.load_rebalance_meta(start).await.map_err(|err| (host, err)) });
+                futures.push(async move {
+                    let result = client.load_rebalance_meta(start).await;
+                    (host, result)
+                });
             } else {
+                warn!(
+                    event = EVENT_NOTIFICATION_PEER_PROPAGATION,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    action = "load_rebalance_meta",
+                    result = "peer_unreachable",
+                    peer_index = idx,
+                    start_rebalance = start,
+                    "notification peer propagation"
+                );
                 failures.push(format!("peer[{idx}] {operation} failed: peer is not reachable"));
             }
         }
 
-        for result in join_all(futures).await {
-            if let Err((host, err)) = result {
+        for (host, result) in join_all(futures).await {
+            if let Err(err) = result {
                 let failure = format!("peer {host} {operation} failed: {err}");
-                error!("notification load_rebalance_meta err {}", failure);
+                error!(
+                    event = EVENT_NOTIFICATION_PEER_PROPAGATION,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    action = "load_rebalance_meta",
+                    result = "peer_failed",
+                    peer = %host,
+                    start_rebalance = start,
+                    error = %err,
+                    "notification peer propagation"
+                );
                 failures.push(failure);
             } else {
-                warn!("notification load_rebalance_meta success");
+                debug!(
+                    event = EVENT_NOTIFICATION_PEER_PROPAGATION,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    action = "load_rebalance_meta",
+                    result = "peer_success",
+                    peer = %host,
+                    start_rebalance = start,
+                    "notification peer propagation"
+                );
             }
         }
 
-        aggregate_notification_failures("load_rebalance_meta", failures)
+        Ok(failures)
     }
 
-    pub async fn stop_rebalance(&self) -> Result<()> {
-        warn!("notification stop_rebalance start");
+    pub async fn stop_rebalance(&self, expected_rebalance_id: Option<&str>) -> Result<()> {
+        let failures = self.stop_rebalance_failures(expected_rebalance_id).await?;
+        aggregate_notification_failures("stop_rebalance", failures)
+    }
+
+    pub async fn stop_rebalance_failures(&self, expected_rebalance_id: Option<&str>) -> Result<Vec<String>> {
+        info!(
+            event = EVENT_NOTIFICATION_PEER_PROPAGATION,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+            action = "stop_rebalance",
+            state = "started",
+            "notification peer propagation"
+        );
         let Some(store) = resolve_object_store_handle() else {
-            error!("stop_rebalance: not init");
+            error!(
+                event = EVENT_NOTIFICATION_PEER_PROPAGATION,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                action = "stop_rebalance",
+                result = "failed",
+                reason = "object_layer_not_initialized",
+                "notification peer propagation"
+            );
             return Err(Error::other("stop_rebalance: object layer not initialized"));
         };
 
-        // warn!("notification stop_rebalance load_rebalance_meta");
-        // self.load_rebalance_meta(false).await;
-        // warn!("notification stop_rebalance load_rebalance_meta done");
-
         let mut failures = Vec::new();
 
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for (idx, client) in self.peer_clients.iter().enumerate() {
             if let Some(client) = client {
                 let host = client.grid_host.clone();
-                futures.push(async move { client.stop_rebalance().await.map_err(|err| (host, err)) });
+                futures.push(async move {
+                    let result = client.stop_rebalance(expected_rebalance_id).await;
+                    (host, result)
+                });
             } else {
+                warn!(
+                    event = EVENT_NOTIFICATION_PEER_PROPAGATION,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    action = "stop_rebalance",
+                    result = "peer_unreachable",
+                    peer_index = idx,
+                    "notification peer propagation"
+                );
                 failures.push(format!("peer[{idx}] stop_rebalance failed: peer is not reachable"));
             }
         }
 
-        for result in join_all(futures).await {
-            if let Err((host, err)) = result {
+        for (host, result) in join_all(futures).await {
+            if let Err(err) = result {
                 let failure = format!("peer {host} stop_rebalance failed: {err}");
-                error!("notification stop_rebalance err {}", failure);
+                error!(
+                    event = EVENT_NOTIFICATION_PEER_PROPAGATION,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    action = "stop_rebalance",
+                    result = "peer_failed",
+                    peer = %host,
+                    error = %err,
+                    "notification peer propagation"
+                );
                 failures.push(failure);
+            } else {
+                debug!(
+                    event = EVENT_NOTIFICATION_PEER_PROPAGATION,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    action = "stop_rebalance",
+                    result = "peer_success",
+                    peer = %host,
+                    "notification peer propagation"
+                );
             }
         }
 
-        warn!("notification stop_rebalance stop_rebalance start");
-        match store.stop_rebalance().await {
+        match store.stop_rebalance_for_id(expected_rebalance_id).await {
             Ok(_) => {
                 if let Err(err) = store.save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt).await {
-                    error!("notification stop_rebalance local save err {:?}", err);
+                    error!(
+                        event = EVENT_NOTIFICATION_PEER_PROPAGATION,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                        action = "stop_rebalance",
+                        result = "local_save_failed",
+                        error = %err,
+                        "notification peer propagation"
+                    );
                     return Err(Error::other(format!(
                         "local stop_rebalance save_rebalance_stats(stopped_at) failed: {err}"
                     )));
                 }
             }
             Err(err) => {
-                error!("notification stop_rebalance local stop err {:?}", err);
+                error!(
+                    event = EVENT_NOTIFICATION_PEER_PROPAGATION,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    action = "stop_rebalance",
+                    result = "local_stop_failed",
+                    error = %err,
+                    "notification peer propagation"
+                );
                 return Err(Error::other(format!("local stop_rebalance stop failed: {err}")));
             }
         }
 
-        if let Err(err) = aggregate_notification_failures("stop_rebalance", failures) {
-            warn!("{err}");
-        }
-        warn!("notification stop_rebalance stop_rebalance done");
-        Ok(())
+        info!(
+            event = EVENT_NOTIFICATION_PEER_PROPAGATION,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+            action = "stop_rebalance",
+            result = if failures.is_empty() { "success" } else { "partial_failure" },
+            "notification peer propagation"
+        );
+        Ok(failures)
     }
 
     pub async fn load_bucket_metadata(&self, bucket: &str) -> Result<()> {
@@ -1094,6 +1215,53 @@ mod tests {
         assert!(msg.contains("local save failed"));
     }
 
+    #[test]
+    fn load_rebalance_meta_aggregate_failures_return_error() {
+        let err = aggregate_notification_failures(
+            "load_rebalance_meta(start=true)",
+            vec!["peer[0] load_rebalance_meta failed: peer is not reachable".to_string()],
+        )
+        .expect_err("load_rebalance_meta peer failures must be returned");
+
+        let msg = err.to_string();
+        assert!(msg.contains("load_rebalance_meta(start=true)"));
+        assert!(msg.contains("1 failure(s)"));
+        assert!(msg.contains("peer[0]"));
+    }
+
+    #[test]
+    fn stop_rebalance_aggregate_failures_return_error() {
+        let err = aggregate_notification_failures(
+            "stop_rebalance",
+            vec!["peer[0] stop_rebalance failed: peer is not reachable".to_string()],
+        )
+        .expect_err("stop_rebalance peer failures must be returned");
+
+        let msg = err.to_string();
+        assert!(msg.contains("stop_rebalance"));
+        assert!(msg.contains("1 failure(s)"));
+        assert!(msg.contains("peer[0]"));
+    }
+
+    #[tokio::test]
+    async fn reload_pool_meta_reports_unreachable_peers() {
+        let sys = NotificationSys {
+            peer_clients: vec![None],
+            all_peer_clients: Vec::new(),
+            peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
+        };
+
+        let err = sys
+            .reload_pool_meta()
+            .await
+            .expect_err("unreachable peers should fail pool metadata reload");
+
+        let msg = err.to_string();
+        assert!(msg.contains("reload_pool_meta"));
+        assert!(msg.contains("1 failure(s)"));
+        assert!(msg.contains("peer[0]"));
+    }
+
     #[tokio::test]
     async fn load_bucket_metadata_reports_unreachable_peers() {
         let sys = NotificationSys {

crates/ecstore/src/object_api/readers.rs

@@ -314,6 +314,24 @@ impl ReadPlan {
             rs = http_range_spec_from_object_info(oi, part_number);
         }
 
+        if opts.raw_data_movement_read {
+            let (visible_offset, visible_length) = if let Some(rs) = rs {
+                rs.get_offset_length(oi.size)?
+            } else {
+                (0, oi.size)
+            };
+
+            return Ok(Self {
+                storage_offset: visible_offset,
+                storage_length: visible_length,
+                object_size: oi.size,
+                transform: ReadTransform::Plain {
+                    visible_offset,
+                    visible_length,
+                },
+            });
+        }
+
         let mut is_encrypted = oi.is_encrypted();
         let (algo, compression_backend, mut is_compressed) = oi.compression_read_plan()?;
 
@@ -2162,6 +2180,101 @@ mod tests {
         ));
     }
 
+    #[tokio::test]
+    async fn test_raw_data_movement_read_plan_bypasses_compression_transform() {
+        let object_info = ObjectInfo {
+            size: 3_000_000,
+            user_defined: Arc::new(HashMap::from([
+                ("x-minio-internal-compression".to_string(), "klauspost/compress/s2".to_string()),
+                ("x-minio-internal-actual-size".to_string(), "4194304".to_string()),
+            ])),
+            ..Default::default()
+        };
+        let opts = ObjectOptions {
+            raw_data_movement_read: true,
+            ..Default::default()
+        };
+
+        let plan = ReadPlan::build(None, &object_info, &opts, &HeaderMap::new())
+            .await
+            .expect("raw data movement read should bypass compression planning");
+
+        assert_eq!(plan.storage_offset, 0);
+        assert_eq!(plan.storage_length, object_info.size);
+        assert_eq!(plan.object_size, object_info.size);
+        assert!(matches!(
+            plan.transform,
+            ReadTransform::Plain {
+                visible_offset: 0,
+                visible_length: 3_000_000
+            }
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_raw_data_movement_read_plan_bypasses_encryption_transform() {
+        let object_info = ObjectInfo {
+            size: 128,
+            user_defined: Arc::new(HashMap::from([
+                ("X-Amz-Server-Side-Encryption".to_string(), "aws:kms".to_string()),
+                ("X-Amz-Server-Side-Encryption-Iv".to_string(), "AAAAAAAAAAAAAAAA".to_string()),
+                ("X-Amz-Server-Side-Encryption-Key".to_string(), BASE64_STANDARD.encode([7_u8; 32])),
+                ("x-rustfs-encryption-original-size".to_string(), "64".to_string()),
+            ])),
+            ..Default::default()
+        };
+        let opts = ObjectOptions {
+            raw_data_movement_read: true,
+            ..Default::default()
+        };
+
+        let plan = ReadPlan::build(None, &object_info, &opts, &HeaderMap::new())
+            .await
+            .expect("raw data movement read should not require decryption material");
+
+        assert_eq!(plan.storage_offset, 0);
+        assert_eq!(plan.storage_length, object_info.size);
+        assert_eq!(plan.object_size, object_info.size);
+        assert!(matches!(
+            plan.transform,
+            ReadTransform::Plain {
+                visible_offset: 0,
+                visible_length: 128
+            }
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_raw_data_movement_read_plan_bypasses_ssec_header_resolution() {
+        let object_info = ObjectInfo {
+            size: 256,
+            user_defined: Arc::new(HashMap::from([
+                (SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string()),
+                (SSEC_KEY_MD5_HEADER.to_string(), "stored-key-md5".to_string()),
+            ])),
+            ..Default::default()
+        };
+        let opts = ObjectOptions {
+            raw_data_movement_read: true,
+            ..Default::default()
+        };
+
+        let plan = ReadPlan::build(None, &object_info, &opts, &HeaderMap::new())
+            .await
+            .expect("raw data movement read should not require SSE-C request headers");
+
+        assert_eq!(plan.storage_offset, 0);
+        assert_eq!(plan.storage_length, object_info.size);
+        assert_eq!(plan.object_size, object_info.size);
+        assert!(matches!(
+            plan.transform,
+            ReadTransform::Plain {
+                visible_offset: 0,
+                visible_length: 256
+            }
+        ));
+    }
+
     #[tokio::test]
     async fn test_get_object_reader_allows_encrypted_full_object_passthrough() {
         async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async {

crates/ecstore/src/object_api/types.rs

@@ -25,6 +25,7 @@ pub struct ObjectOptions {
     pub skip_free_version: bool,
 
     pub data_movement: bool,
+    pub raw_data_movement_read: bool,
     pub src_pool_idx: usize,
     pub user_defined: HashMap<String, String>,
     pub preserve_etag: Option<String>,

crates/ecstore/src/rebalance.rs

@@ -25,13 +25,15 @@ const EVENT_REBALANCE_LISTING: &str = "rebalance_listing";
 
 const REBAL_META_FMT: u16 = 1; // Replace with actual format value
 const REBAL_META_VER: u16 = 1; // Replace with actual version value
-const REBAL_META_NAME: &str = "rebalance.bin";
+pub(crate) const REBAL_META_NAME: &str = "rebalance.bin";
 const DEFAULT_REBALANCE_MAX_ATTEMPTS: usize = 3;
 const REBALANCE_MAX_ATTEMPTS_ENV: &str = "RUSTFS_REBALANCE_MAX_ATTEMPTS";
+const REBALANCE_STOP_PROPAGATION_ERROR_PREFIX: &str = "rebalance stop propagation incomplete: ";
 const REBALANCE_LISTING_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
 const REBALANCE_MIGRATION_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
 const REBALANCE_MIGRATION_LOCK_RETRY_CAP: Duration = Duration::from_secs(10);
 const REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX: &str = "deferred transient rebalance entry failure:";
+const REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT: usize = 10;
 
 mod control;
 mod entry;
@@ -41,7 +43,12 @@ mod runtime;
 mod types;
 mod worker;
 
-pub use types::{DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta, RebalanceStats};
+pub(crate) use meta::is_rebalance_conflicting_with_decommission;
+pub use meta::{decode_rebalance_stop_propagation_record, encode_rebalance_stop_propagation_record};
+pub use types::{
+    DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta,
+    RebalanceStats, RebalanceStopPropagationRecord,
+};
 use types::{RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome};
 
 #[cfg(test)]

crates/ecstore/src/rebalance/control.rs

@@ -2,7 +2,9 @@ use super::meta::{
     clone_first_arc, clone_rebalance_pool_stats, defer_bucket_in_rebalance_queue, ensure_valid_rebalance_pool_index,
     invalid_rebalance_pool_index_error, is_rebalance_conflicting_with_decommission, mark_rebalance_bucket_done,
     merge_rebalance_meta, percent_free_ratio, rebalance_metadata_not_initialized_error, record_rebalance_cleanup_warning_in_meta,
-    resolve_next_rebalance_bucket, should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot,
+    record_rebalance_stop_propagation_snapshot, resolve_next_rebalance_bucket, rollback_rebalance_start_meta_snapshot_for_id,
+    should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot_for_id,
+    validate_init_rebalance_state,
 };
 use super::worker::{
     rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result, resolve_rebalance_meta_load_result,
@@ -10,7 +12,8 @@ use super::worker::{
 };
 use super::{
     DiskStat, EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
-    RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats,
+    RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
+    encode_rebalance_stop_propagation_record,
 };
 use crate::error::{Error, Result};
 use crate::object_api::ObjectOptions;
@@ -247,6 +250,35 @@ impl ECStore {
         Ok(id)
     }
 
+    #[tracing::instrument(skip(self, bucktes))]
+    pub async fn init_and_start_rebalance(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
+        let _start_guard = self.start_gate.lock().await;
+
+        let decommission_running = self.is_decommission_running().await;
+        {
+            let rebalance_meta = self.rebalance_meta.read().await;
+            validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
+        }
+
+        let id = self.init_rebalance_meta(bucktes).await?;
+        if let Err(start_err) = self.start_rebalance().await {
+            if let Err(rollback_err) = self
+                .rollback_rebalance_start_without_worker_for_id(Some(&id), start_err.to_string())
+                .await
+            {
+                return Err(Error::other(format!(
+                    "failed to start rebalance after metadata initialized for {id}: {start_err}; rollback failed: {rollback_err}"
+                )));
+            }
+
+            return Err(Error::other(format!(
+                "failed to start rebalance after metadata initialized for {id}; local metadata was finalized as failed: {start_err}"
+            )));
+        }
+
+        Ok(id)
+    }
+
     #[tracing::instrument(skip(self, fi))]
     pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> {
         self.update_pool_stats_batch(pool_index, bucket, &[fi]).await
@@ -389,12 +421,24 @@ impl ECStore {
         false
     }
 
+    pub async fn current_rebalance_id(&self) -> Option<String> {
+        let rebalance_meta = self.rebalance_meta.read().await;
+        rebalance_meta
+            .as_ref()
+            .and_then(|meta| (!meta.id.is_empty()).then(|| meta.id.clone()))
+    }
+
     #[tracing::instrument(skip(self))]
     pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> {
+        self.stop_rebalance_for_id(None).await
+    }
+
+    #[tracing::instrument(skip(self))]
+    pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> {
         let meta_to_save = {
             let mut rebalance_meta = self.rebalance_meta.write().await;
-            stop_rebalance_meta_snapshot(rebalance_meta.as_mut(), OffsetDateTime::now_utc())
-        };
+            stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id)
+        }?;
 
         if let Some(meta_to_save) = meta_to_save {
             let pool = clone_first_arc(self.pools.as_slice(), "stop_rebalance: no pools available")?;
@@ -407,4 +451,54 @@ impl ECStore {
 
         Ok(())
     }
+
+    async fn rollback_rebalance_start_without_worker_for_id(
+        self: &Arc<Self>,
+        expected_id: Option<&str>,
+        start_error: String,
+    ) -> Result<()> {
+        let meta_to_save = {
+            let mut rebalance_meta = self.rebalance_meta.write().await;
+            rollback_rebalance_start_meta_snapshot_for_id(
+                rebalance_meta.as_mut(),
+                OffsetDateTime::now_utc(),
+                expected_id,
+                start_error,
+            )
+        };
+
+        if let Some(meta_to_save) = meta_to_save {
+            let pool = clone_first_arc(self.pools.as_slice(), "rollback_rebalance_start: no pools available")?;
+            resolve_rebalance_meta_save_result(
+                self.save_rebalance_meta_with_merge(pool, &meta_to_save, "rollback_rebalance_start")
+                    .await,
+                "rollback_rebalance_start",
+            )?;
+        }
+
+        Ok(())
+    }
+
+    pub async fn record_rebalance_stop_propagation(self: &Arc<Self>, record: RebalanceStopPropagationRecord) -> Result<()> {
+        if !record.has_failures() {
+            return Ok(());
+        }
+
+        let encoded_error = encode_rebalance_stop_propagation_record(&record);
+        let meta_to_save = {
+            let mut rebalance_meta = self.rebalance_meta.write().await;
+            record_rebalance_stop_propagation_snapshot(rebalance_meta.as_mut(), encoded_error, OffsetDateTime::now_utc())
+        };
+
+        if let Some(meta_to_save) = meta_to_save {
+            let pool = clone_first_arc(self.pools.as_slice(), "record_rebalance_stop_propagation: no pools available")?;
+            resolve_rebalance_meta_save_result(
+                self.save_rebalance_meta_with_merge(pool, &meta_to_save, "record_rebalance_stop_propagation")
+                    .await,
+                "record_rebalance_stop_propagation",
+            )?;
+        }
+
+        Ok(())
+    }
 }

crates/ecstore/src/rebalance/entry.rs

@@ -122,7 +122,7 @@ impl ECStore {
                 true,
                 &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Rebal,
             )
-            .await
+            .await?
             {
                 expired += 1;
                 debug!(

crates/ecstore/src/rebalance/meta.rs

@@ -1,8 +1,9 @@
 use super::{
     EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, Error, GetObjectReader, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
     ObjectInfo, ObjectOptions, PutObjReader, REBAL_META_FMT, REBAL_META_NAME, REBAL_META_VER,
-    REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, RebalSaveOpt, RebalStatus, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats,
-    Result,
+    REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT, REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_STOP_PROPAGATION_ERROR_PREFIX,
+    RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats,
+    RebalanceStopPropagationRecord, Result,
 };
 use crate::config::com::{read_config_with_metadata, save_config_with_opts};
 use crate::error::is_err_operation_canceled;
@@ -197,15 +198,15 @@ pub(super) fn is_rebalance_pool_started(pool_stat: &RebalanceStats) -> bool {
     pool_stat.participating && pool_stat.info.status == RebalStatus::Started
 }
 
-pub(super) fn is_rebalance_in_progress(meta: &RebalanceMeta) -> bool {
-    if meta.stopped_at.is_some() {
-        return false;
-    }
-
-    meta.pool_stats.iter().any(is_rebalance_pool_started)
+pub(super) fn is_rebalance_pool_active(pool_stat: &RebalanceStats) -> bool {
+    is_rebalance_pool_started(pool_stat) || pool_stat.info.stopping
 }
 
-pub(super) fn is_rebalance_conflicting_with_decommission(meta: &RebalanceMeta) -> bool {
+pub(super) fn is_rebalance_in_progress(meta: &RebalanceMeta) -> bool {
+    meta.pool_stats.iter().any(is_rebalance_pool_active)
+}
+
+pub(crate) fn is_rebalance_conflicting_with_decommission(meta: &RebalanceMeta) -> bool {
     is_rebalance_in_progress(meta)
 }
 
@@ -224,6 +225,30 @@ pub(super) fn rebalance_meta_load_unknown_format_error(fmt: u16) -> Error {
 pub(super) fn rebalance_meta_load_unknown_version_error(ver: u16) -> Error {
     Error::other(format!("rebalance metadata load failed: unknown version {ver}"))
 }
+
+pub fn encode_rebalance_stop_propagation_record(record: &RebalanceStopPropagationRecord) -> String {
+    match serde_json::to_string(record) {
+        Ok(payload) => format!("{REBALANCE_STOP_PROPAGATION_ERROR_PREFIX}{payload}"),
+        Err(err) => {
+            let payload = serde_json::json!({
+                "encodeError": err.to_string(),
+                "stopFailures": [],
+                "terminalReloadFailures": [],
+            });
+            format!("{REBALANCE_STOP_PROPAGATION_ERROR_PREFIX}{payload}")
+        }
+    }
+}
+
+pub fn decode_rebalance_stop_propagation_record(message: &str) -> Option<RebalanceStopPropagationRecord> {
+    let payload = message.strip_prefix(REBALANCE_STOP_PROPAGATION_ERROR_PREFIX)?;
+    serde_json::from_str(payload).ok()
+}
+
+fn is_rebalance_stop_propagation_error(message: Option<&str>) -> bool {
+    message.is_some_and(|message| message.starts_with(REBALANCE_STOP_PROPAGATION_ERROR_PREFIX))
+}
+
 pub(super) fn rebalance_goal_reached(init_free_space: u64, init_capacity: u64, bytes: u64, percent_free_goal: f64) -> bool {
     if init_capacity == 0 {
         return false;
@@ -384,10 +409,17 @@ pub(super) fn record_rebalance_cleanup_warning_in_meta(
     };
 
     pool_stat.cleanup_warnings.count = pool_stat.cleanup_warnings.count.saturating_add(1);
-    pool_stat.cleanup_warnings.last_message = Some(message);
+    pool_stat.cleanup_warnings.last_message = Some(message.clone());
     pool_stat.cleanup_warnings.last_bucket = Some(bucket.to_string());
     pool_stat.cleanup_warnings.last_object = Some(object.to_string());
     pool_stat.cleanup_warnings.last_at = Some(now);
+    pool_stat.cleanup_warnings.entries.push(RebalanceCleanupWarningEntry {
+        bucket: bucket.to_string(),
+        object: object.to_string(),
+        message,
+        timestamp: Some(now),
+    });
+    truncate_rebalance_cleanup_warning_entries(&mut pool_stat.cleanup_warnings.entries);
     meta.last_refreshed_at = Some(now);
     Ok(())
 }
@@ -572,6 +604,17 @@ pub(super) fn validate_start_rebalance_state(decommission_running: bool, meta_lo
     Ok(())
 }
 
+pub(super) fn validate_init_rebalance_state(decommission_running: bool, current_meta: Option<&RebalanceMeta>) -> Result<()> {
+    if !ensure_rebalance_not_decommissioning(decommission_running) {
+        return Err(Error::DecommissionAlreadyRunning);
+    }
+    if current_meta.is_some_and(is_rebalance_in_progress) {
+        return Err(Error::RebalanceAlreadyRunning);
+    }
+
+    Ok(())
+}
+
 pub(super) fn should_skip_start_rebalance(cancel_attached: bool, in_progress: bool) -> bool {
     cancel_attached && in_progress
 }
@@ -654,6 +697,31 @@ pub(super) fn merge_rebalance_cleanup_warnings(remote: &mut RebalanceCleanupWarn
         remote.last_object = local.last_object.clone();
         remote.last_at = local.last_at;
     }
+
+    merge_rebalance_cleanup_warning_entries(&mut remote.entries, &local.entries);
+    let retained_entries = u64::try_from(remote.entries.len()).unwrap_or(u64::MAX);
+    remote.count = remote.count.max(retained_entries);
+}
+
+pub(super) fn merge_rebalance_cleanup_warning_entries(
+    remote: &mut Vec<RebalanceCleanupWarningEntry>,
+    local: &[RebalanceCleanupWarningEntry],
+) {
+    for entry in local {
+        if !remote.iter().any(|existing| existing == entry) {
+            remote.push(entry.clone());
+        }
+    }
+
+    remote.sort_by_key(|entry| entry.timestamp);
+    truncate_rebalance_cleanup_warning_entries(remote);
+}
+
+fn truncate_rebalance_cleanup_warning_entries(entries: &mut Vec<RebalanceCleanupWarningEntry>) {
+    if entries.len() > REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT {
+        let remove_count = entries.len() - REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT;
+        entries.drain(0..remove_count);
+    }
 }
 
 pub(super) fn should_replace_rebalance_cleanup_warning(
@@ -667,6 +735,16 @@ pub(super) fn should_replace_rebalance_cleanup_warning(
     }
 }
 
+pub(super) fn merge_rebalance_stop_propagation_error(remote: Option<String>, local: Option<String>) -> Option<String> {
+    if is_rebalance_stop_propagation_error(local.as_deref()) {
+        local
+    } else if is_rebalance_stop_propagation_error(remote.as_deref()) {
+        remote
+    } else {
+        None
+    }
+}
+
 pub(super) fn merge_rebalance_pool_stats(remote: &mut RebalanceStats, local: &RebalanceStats) {
     remote.init_free_space = remote.init_free_space.max(local.init_free_space);
     remote.init_capacity = remote.init_capacity.max(local.init_capacity);
@@ -702,19 +780,23 @@ pub(super) fn merge_rebalance_pool_stats(remote: &mut RebalanceStats, local: &Re
     match local.info.status {
         RebalStatus::Failed => {
             remote.info.status = RebalStatus::Failed;
+            remote.info.stopping = false;
             remote.info.end_time = local.info.end_time.or(remote.info.end_time);
             remote.info.last_error = local.info.last_error.clone().or_else(|| remote.info.last_error.clone());
         }
         RebalStatus::Stopped => {
             if remote.info.status != RebalStatus::Failed {
                 remote.info.status = RebalStatus::Stopped;
+                remote.info.stopping = false;
                 remote.info.end_time = local.info.end_time.or(remote.info.end_time);
-                remote.info.last_error = None;
+                remote.info.last_error =
+                    merge_rebalance_stop_propagation_error(remote.info.last_error.clone(), local.info.last_error.clone());
             }
         }
         RebalStatus::Completed => {
             if !matches!(remote.info.status, RebalStatus::Failed | RebalStatus::Stopped) {
                 remote.info.status = RebalStatus::Completed;
+                remote.info.stopping = false;
                 remote.info.end_time = local.info.end_time.or(remote.info.end_time);
                 remote.info.last_error = None;
             }
@@ -722,6 +804,7 @@ pub(super) fn merge_rebalance_pool_stats(remote: &mut RebalanceStats, local: &Re
         RebalStatus::Started => {
             if !is_rebalance_terminal_status(remote.info.status) {
                 remote.info.status = RebalStatus::Started;
+                remote.info.stopping |= local.info.stopping;
                 remote.info.last_error = local.info.last_error.clone();
             }
         }
@@ -736,7 +819,6 @@ pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &Rebalance
     }
 
     if !local.id.is_empty() && remote.id != local.id {
-        *remote = local.clone();
         return;
     }
 
@@ -761,35 +843,120 @@ pub(super) fn mark_started_rebalance_pools_stopped(meta: &mut RebalanceMeta, sto
     for pool_stat in meta.pool_stats.iter_mut() {
         if pool_stat.info.status == RebalStatus::Started {
             pool_stat.info.status = RebalStatus::Stopped;
+            pool_stat.info.stopping = false;
             pool_stat.info.end_time.get_or_insert(stop_time);
+            if !is_rebalance_stop_propagation_error(pool_stat.info.last_error.as_deref()) {
+                pool_stat.info.last_error = None;
+            }
+        }
+    }
+}
+
+pub(super) fn mark_started_rebalance_pools_stopping(meta: &mut RebalanceMeta) {
+    for pool_stat in meta.pool_stats.iter_mut() {
+        if pool_stat.info.status == RebalStatus::Started {
+            pool_stat.info.stopping = true;
             pool_stat.info.last_error = None;
         }
     }
 }
 
 pub(super) fn apply_stopped_at(meta: &mut RebalanceMeta, now: OffsetDateTime) {
-    meta.stopped_at = Some(now);
-    mark_started_rebalance_pools_stopped(meta, now);
+    meta.stopped_at.get_or_insert(now);
+    mark_started_rebalance_pools_stopping(meta);
+}
+
+pub(super) fn clear_rebalance_cancel_token(meta: Option<&mut RebalanceMeta>) -> bool {
+    let Some(meta) = meta else {
+        return false;
+    };
+    if let Some(cancel_tx) = meta.cancel.take() {
+        cancel_tx.cancel();
+        return true;
+    }
+    false
 }
 
 pub(super) fn stop_rebalance_state(meta: &mut RebalanceMeta, now: OffsetDateTime) {
-    if let Some(cancel_tx) = meta.cancel.take() {
-        cancel_tx.cancel();
-    }
-
-    let stop_time = meta.stopped_at.unwrap_or(now);
+    clear_rebalance_cancel_token(Some(meta));
     if meta.stopped_at.is_none() && is_rebalance_in_progress(meta) {
-        meta.stopped_at = Some(stop_time);
-    }
-
-    if meta.stopped_at.is_some() {
-        mark_started_rebalance_pools_stopped(meta, stop_time);
+        apply_stopped_at(meta, now);
+    } else if meta.stopped_at.is_some() {
+        mark_started_rebalance_pools_stopping(meta);
     }
 }
 
+pub(super) fn stop_rebalance_meta_snapshot_for_id(
+    meta: Option<&mut RebalanceMeta>,
+    now: OffsetDateTime,
+    expected_id: Option<&str>,
+) -> Result<Option<RebalanceMeta>> {
+    let Some(meta) = meta else {
+        return Ok(None);
+    };
+
+    if let Some(expected_id) = expected_id
+        && !expected_id.is_empty()
+        && meta.id != expected_id
+    {
+        return Err(Error::other(format!(
+            "rebalance stop id mismatch: expected {expected_id}, found {}",
+            meta.id
+        )));
+    }
+
+    stop_rebalance_state(meta, now);
+    meta.last_refreshed_at = Some(now);
+    Ok(Some(meta.clone()))
+}
+
+pub(super) fn rollback_rebalance_start_meta_snapshot_for_id(
+    meta: Option<&mut RebalanceMeta>,
+    now: OffsetDateTime,
+    expected_id: Option<&str>,
+    start_error: String,
+) -> Option<RebalanceMeta> {
+    meta.and_then(|meta| {
+        if let Some(expected_id) = expected_id
+            && !expected_id.is_empty()
+            && meta.id != expected_id
+        {
+            return None;
+        }
+
+        clear_rebalance_cancel_token(Some(meta));
+        meta.stopped_at.get_or_insert(now);
+        meta.last_refreshed_at = Some(now);
+        for pool_stat in meta.pool_stats.iter_mut() {
+            if pool_stat.info.status == RebalStatus::Started {
+                pool_stat.info.status = RebalStatus::Failed;
+                pool_stat.info.stopping = false;
+                pool_stat.info.end_time.get_or_insert(now);
+                pool_stat.info.last_error = Some(start_error.clone());
+            }
+        }
+        Some(meta.clone())
+    })
+}
+
 pub(super) fn stop_rebalance_meta_snapshot(meta: Option<&mut RebalanceMeta>, now: OffsetDateTime) -> Option<RebalanceMeta> {
+    let meta = meta?;
+    stop_rebalance_state(meta, now);
+    meta.last_refreshed_at = Some(now);
+    Some(meta.clone())
+}
+
+pub(super) fn record_rebalance_stop_propagation_snapshot(
+    meta: Option<&mut RebalanceMeta>,
+    encoded_error: String,
+    now: OffsetDateTime,
+) -> Option<RebalanceMeta> {
     meta.map(|meta| {
-        stop_rebalance_state(meta, now);
+        for pool_stat in meta.pool_stats.iter_mut() {
+            if pool_stat.participating || pool_stat.info.stopping || pool_stat.info.status == RebalStatus::Stopped {
+                pool_stat.info.last_error = Some(encoded_error.clone());
+            }
+        }
         meta.last_refreshed_at = Some(now);
         meta.clone()
     })

crates/ecstore/src/rebalance/rebalance_unit_tests.rs

@@ -25,7 +25,7 @@ use super::meta::{
     record_rebalance_cleanup_warning_in_meta, remove_rebalanced_buckets_from_queue, resolve_next_rebalance_bucket,
     resolve_rebalance_participants, should_accept_rebalance_stats_update, should_ignore_rebalance_data_usage_cache,
     should_pool_participate, should_preserve_rebalance_stopped_state, should_skip_start_rebalance, stop_rebalance_meta_snapshot,
-    stop_rebalance_state, take_bucket_from_rebalance_queue, validate_start_rebalance_state,
+    stop_rebalance_state, take_bucket_from_rebalance_queue, validate_init_rebalance_state, validate_start_rebalance_state,
 };
 use super::migration::{
     MigrationBackend, MigrationVersionResult, migrate_entry_version, migrate_entry_version_with_retry_wait,
@@ -1199,6 +1199,7 @@ fn test_merge_rebalance_meta_preserves_updates_from_multiple_pools() {
                     last_bucket: Some("bucket-a".to_string()),
                     last_object: Some("local-object".to_string()),
                     last_at: Some(warning_at),
+                    entries: Vec::new(),
                 },
                 ..Default::default()
             },
@@ -1242,7 +1243,7 @@ fn test_merge_rebalance_meta_does_not_overwrite_failed_with_started_stats() {
         pool_stats: vec![RebalanceStats {
             participating: true,
             info: RebalanceInfo {
-                status: RebalStatus::Started,
+                status: RebalStatus::Stopped,
                 ..Default::default()
             },
             num_versions: 8,
@@ -1307,7 +1308,7 @@ fn test_merge_rebalance_meta_preserves_failed_status_over_stopped() {
         stopped_at: Some(stopped_at),
         pool_stats: vec![RebalanceStats {
             info: RebalanceInfo {
-                status: RebalStatus::Stopped,
+                status: RebalStatus::Started,
                 end_time: Some(stopped_at),
                 ..Default::default()
             },
@@ -1812,7 +1813,7 @@ fn test_should_accept_rebalance_stats_update_rejects_stopped_meta() {
         stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
         pool_stats: vec![RebalanceStats {
             info: RebalanceInfo {
-                status: RebalStatus::Started,
+                status: RebalStatus::Stopped,
                 ..Default::default()
             },
             ..Default::default()
@@ -2173,6 +2174,93 @@ fn test_validate_start_rebalance_state_allows_loaded_meta() {
     validate_start_rebalance_state(false, true).expect("loaded rebalance meta should allow start");
 }
 
+#[test]
+fn test_validate_init_rebalance_state_rejects_running_decommission() {
+    let err = validate_init_rebalance_state(true, None).expect_err("running decommission should block rebalance init");
+    assert!(matches!(err, Error::DecommissionAlreadyRunning));
+}
+
+#[test]
+fn test_validate_init_rebalance_state_rejects_active_rebalance() {
+    let now = OffsetDateTime::now_utc();
+    let meta = RebalanceMeta {
+        pool_stats: vec![RebalanceStats {
+            participating: true,
+            info: RebalanceInfo {
+                start_time: Some(now),
+                status: RebalStatus::Started,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
+
+    let err = validate_init_rebalance_state(false, Some(&meta)).expect_err("active rebalance should block rebalance init");
+    assert!(matches!(err, Error::RebalanceAlreadyRunning));
+}
+
+#[test]
+fn test_validate_init_rebalance_state_allows_terminal_or_missing_rebalance() {
+    let completed = RebalanceMeta {
+        pool_stats: vec![RebalanceStats {
+            participating: true,
+            info: RebalanceInfo {
+                status: RebalStatus::Completed,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
+
+    validate_init_rebalance_state(false, None).expect("missing rebalance meta should allow init");
+    validate_init_rebalance_state(false, Some(&completed)).expect("terminal rebalance meta should allow init");
+}
+
+#[tokio::test]
+async fn test_init_and_start_rebalance_rejects_second_start_after_gate() {
+    let now = OffsetDateTime::now_utc();
+    let active_meta = RebalanceMeta {
+        pool_stats: vec![RebalanceStats {
+            participating: true,
+            info: RebalanceInfo {
+                start_time: Some(now),
+                status: RebalStatus::Started,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
+
+    let endpoint_pools: crate::endpoints::EndpointServerPools = Vec::new().into();
+    let store = Arc::new(crate::store::ECStore {
+        id: uuid::Uuid::new_v4(),
+        disk_map: std::collections::HashMap::new(),
+        pools: Vec::new(),
+        peer_sys: crate::rpc::S3PeerSys::new(&endpoint_pools),
+        pool_meta: tokio::sync::RwLock::new(crate::pools::PoolMeta::default()),
+        rebalance_meta: tokio::sync::RwLock::new(Some(active_meta)),
+        decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
+        start_gate: tokio::sync::Mutex::new(()),
+        pool_meta_save_gate: tokio::sync::Mutex::new(()),
+        local_disk_map: crate::global::GLOBAL_LOCAL_DISK_MAP.clone(),
+        local_disk_id_map: crate::global::GLOBAL_LOCAL_DISK_ID_MAP.clone(),
+        local_disk_set_drives: crate::global::GLOBAL_LOCAL_DISK_SET_DRIVES.clone(),
+        tier_config_mgr: crate::tier::tier::TierConfigMgr::new(),
+        event_notifier: crate::event_notification::EventNotifier::new(),
+        bucket_monitor: std::sync::OnceLock::new(),
+    });
+
+    let err = store
+        .init_and_start_rebalance(vec!["bucket".to_string()])
+        .await
+        .expect_err("second rebalance start should be rejected before metadata init");
+
+    assert!(matches!(err, Error::RebalanceAlreadyRunning));
+}
+
 #[test]
 fn test_percent_free_ratio_zero_capacity_is_zero() {
     assert_eq!(percent_free_ratio(100, 0), 0.0);
@@ -2412,7 +2500,7 @@ fn test_resolve_rebalance_participants_respects_runtime_pool_count() {
         RebalanceStats {
             participating: false,
             info: RebalanceInfo {
-                status: RebalStatus::Started,
+                status: RebalStatus::Stopped,
                 start_time: Some(now),
                 ..Default::default()
             },
@@ -2421,7 +2509,7 @@ fn test_resolve_rebalance_participants_respects_runtime_pool_count() {
         RebalanceStats {
             participating: true,
             info: RebalanceInfo {
-                status: RebalStatus::Started,
+                status: RebalStatus::Stopped,
                 start_time: Some(now),
                 ..Default::default()
             },
@@ -2542,7 +2630,7 @@ fn test_is_rebalance_conflicting_with_decommission_false_when_stopped() {
             participating: true,
             info: RebalanceInfo {
                 start_time: Some(now),
-                status: RebalStatus::Started,
+                status: RebalStatus::Stopped,
                 ..Default::default()
             },
             ..Default::default()
@@ -2562,7 +2650,7 @@ fn test_is_rebalance_in_progress_stopped_takes_precedence() {
             participating: true,
             info: RebalanceInfo {
                 start_time: Some(now),
-                status: RebalStatus::Started,
+                status: RebalStatus::Stopped,
                 ..Default::default()
             },
             ..Default::default()
@@ -2871,6 +2959,7 @@ fn test_complete_rebalance_pools_with_empty_queue_preserves_cleanup_warnings() {
                 last_bucket: Some("bucket-a".to_string()),
                 last_object: Some("obj.txt".to_string()),
                 last_at: Some(warning_at),
+                entries: Vec::new(),
             },
             ..Default::default()
         }],
@@ -2916,8 +3005,9 @@ fn test_apply_stopped_at_transitions_started_pools_only() {
     apply_stopped_at(&mut meta, now);
 
     assert_eq!(meta.stopped_at, Some(now));
-    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
-    assert_eq!(meta.pool_stats[0].info.end_time, Some(now));
+    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started);
+    assert!(meta.pool_stats[0].info.stopping);
+    assert_eq!(meta.pool_stats[0].info.end_time, None);
     assert_eq!(meta.pool_stats[0].info.last_error, None);
 
     assert_eq!(meta.pool_stats[1].info.status, RebalStatus::Failed);
@@ -2948,8 +3038,9 @@ fn test_stop_rebalance_state_cancels_token_and_marks_stopped_when_in_progress()
     assert!(cancel_clone.is_cancelled());
     assert!(meta.cancel.is_none());
     assert_eq!(meta.stopped_at, Some(now));
-    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
-    assert_eq!(meta.pool_stats[0].info.end_time, Some(now));
+    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started);
+    assert!(meta.pool_stats[0].info.stopping);
+    assert_eq!(meta.pool_stats[0].info.end_time, None);
 }
 
 #[test]
@@ -3005,8 +3096,9 @@ fn test_stop_rebalance_state_normalizes_started_pool_when_stopped_at_already_set
     assert!(cancel_clone.is_cancelled());
     assert!(meta.cancel.is_none());
     assert_eq!(meta.stopped_at, Some(stopped_at));
-    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
-    assert_eq!(meta.pool_stats[0].info.end_time, Some(stopped_at));
+    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started);
+    assert!(meta.pool_stats[0].info.stopping);
+    assert_eq!(meta.pool_stats[0].info.end_time, None);
     assert_eq!(meta.pool_stats[0].info.last_error, None);
 }
 
@@ -3040,13 +3132,15 @@ fn test_stop_rebalance_meta_snapshot_stops_meta_and_returns_snapshot() {
     assert!(meta.cancel.is_none());
     assert_eq!(meta.stopped_at, Some(now));
     assert_eq!(meta.last_refreshed_at, Some(now));
-    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
+    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started);
+    assert!(meta.pool_stats[0].info.stopping);
 
     assert!(snapshot.cancel.is_none());
     assert_eq!(snapshot.stopped_at, Some(now));
     assert_eq!(snapshot.last_refreshed_at, Some(now));
-    assert_eq!(snapshot.pool_stats[0].info.status, RebalStatus::Stopped);
-    assert_eq!(snapshot.pool_stats[0].info.end_time, Some(now));
+    assert_eq!(snapshot.pool_stats[0].info.status, RebalStatus::Started);
+    assert!(snapshot.pool_stats[0].info.stopping);
+    assert_eq!(snapshot.pool_stats[0].info.end_time, None);
 }
 
 #[test]
@@ -3108,8 +3202,9 @@ fn test_apply_rebalance_save_option_stopped_at_updates_refresh_and_statuses() {
 
     assert_eq!(meta.stopped_at, Some(now));
     assert_eq!(meta.last_refreshed_at, Some(now));
-    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
-    assert_eq!(meta.pool_stats[0].info.end_time, Some(now));
+    assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started);
+    assert!(meta.pool_stats[0].info.stopping);
+    assert_eq!(meta.pool_stats[0].info.end_time, None);
     assert!(meta.pool_stats[0].info.last_error.is_none());
     assert_eq!(meta.pool_stats[1].info.status, RebalStatus::Failed);
     assert_eq!(meta.pool_stats[1].info.last_error.as_deref(), Some("previous failure"));

crates/ecstore/src/rebalance/runtime.rs

@@ -225,6 +225,7 @@ impl ECStore {
                                         "Preserved stopped rebalance status"
                                     );
                                 } else {
+                                    pool_stat.info.stopping = false;
                                     apply_rebalance_terminal_event(
                                         &mut pool_stat.info.status,
                                         &mut pool_stat.info.end_time,

crates/ecstore/src/rebalance/types.rs

@@ -4,6 +4,7 @@ use time::OffsetDateTime;
 use tokio_util::sync::CancellationToken;
 
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
 pub struct RebalanceStats {
     #[serde(rename = "ifs")]
     pub init_free_space: u64, // Pool free space at the start of rebalance
@@ -70,6 +71,7 @@ pub enum RebalSaveOpt {
 }
 
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
 pub struct RebalanceInfo {
     #[serde(rename = "startTs")]
     pub start_time: Option<OffsetDateTime>, // Time at which rebalance-start was issued
@@ -79,9 +81,25 @@ pub struct RebalanceInfo {
     pub last_error: Option<String>, // Last rebalance error message
     #[serde(rename = "status")]
     pub status: RebalStatus, // Current state of rebalance operation
+    #[serde(rename = "stopping", default)]
+    pub stopping: bool, // True after stop is requested and before worker terminal acknowledgement
 }
 
 #[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct RebalanceCleanupWarningEntry {
+    #[serde(rename = "bucket", default)]
+    pub bucket: String,
+    #[serde(rename = "object", default)]
+    pub object: String,
+    #[serde(rename = "message", default)]
+    pub message: String,
+    #[serde(rename = "timestamp", default)]
+    pub timestamp: Option<OffsetDateTime>,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
 pub struct RebalanceCleanupWarnings {
     #[serde(rename = "count", default)]
     pub count: u64,
@@ -93,6 +111,27 @@ pub struct RebalanceCleanupWarnings {
     pub last_object: Option<String>,
     #[serde(rename = "lastAt", default)]
     pub last_at: Option<OffsetDateTime>,
+    #[serde(rename = "entries", default)]
+    pub entries: Vec<RebalanceCleanupWarningEntry>,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct RebalanceStopPropagationRecord {
+    #[serde(rename = "stopAttemptAt", default)]
+    pub stop_attempt_at: Option<OffsetDateTime>,
+    #[serde(rename = "stopFailures", default)]
+    pub stop_failures: Vec<String>,
+    #[serde(rename = "terminalReloadAttemptAt", default)]
+    pub terminal_reload_attempt_at: Option<OffsetDateTime>,
+    #[serde(rename = "terminalReloadFailures", default)]
+    pub terminal_reload_failures: Vec<String>,
+}
+
+impl RebalanceStopPropagationRecord {
+    pub fn has_failures(&self) -> bool {
+        !self.stop_failures.is_empty() || !self.terminal_reload_failures.is_empty()
+    }
 }
 
 #[allow(dead_code)]
@@ -103,6 +142,7 @@ pub struct DiskStat {
 }
 
 #[derive(Debug, Default, Serialize, Deserialize, Clone)]
+#[serde(deny_unknown_fields)]
 pub struct RebalanceMeta {
     #[serde(skip)]
     pub cancel: Option<CancellationToken>, // To be invoked on rebalance-stop

crates/ecstore/src/rpc/peer_rest_client.rs

@@ -52,7 +52,7 @@ use tokio::{net::TcpStream, time::Duration};
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
 use tonic::transport::Channel;
-use tracing::{Instrument, warn};
+use tracing::{Instrument, debug, warn};
 
 pub const PEER_RESTSIGNAL: &str = "signal";
 pub const PEER_RESTSUB_SYS: &str = "sub-sys";
@@ -916,11 +916,13 @@ impl PeerRestClient {
         .await
     }
 
-    pub async fn stop_rebalance(&self) -> Result<()> {
+    pub async fn stop_rebalance(&self, expected_rebalance_id: Option<&str>) -> Result<()> {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(StopRebalanceRequest {});
+                let request = Request::new(StopRebalanceRequest {
+                    expected_rebalance_id: expected_rebalance_id.unwrap_or_default().to_string(),
+                });
 
                 let response = client.stop_rebalance(request).await?.into_inner();
                 if !response.success {
@@ -945,7 +947,17 @@ impl PeerRestClient {
 
                 let response = client.load_rebalance_meta(request).await?.into_inner();
 
-                warn!("load_rebalance_meta response {:?}, grid_host: {:?}", response, &self.grid_host);
+                debug!(
+                    event = "peer_rebalance_meta",
+                    component = "ecstore",
+                    subsystem = "peer_rest_client",
+                    action = "load_rebalance_meta",
+                    result = "response_received",
+                    peer = %self.grid_host,
+                    success = response.success,
+                    start_rebalance = start_rebalance,
+                    "peer rebalance metadata response"
+                );
                 if !response.success {
                     if let Some(msg) = response.error_info {
                         return Err(Error::other(msg));

crates/ecstore/src/rpc/peer_s3_client.rs

@@ -346,7 +346,8 @@ impl S3PeerSys {
 
 #[derive(Debug)]
 pub struct LocalPeerS3Client {
-    // pub local_disks: Vec<DiskStore>,
+    #[cfg(test)]
+    local_disks: Option<Vec<DiskStore>>,
     // pub node: Node,
     pub pools: Option<Vec<usize>>,
 }
@@ -354,13 +355,29 @@ pub struct LocalPeerS3Client {
 impl LocalPeerS3Client {
     pub fn new(_node: Option<Node>, pools: Option<Vec<usize>>) -> Self {
         Self {
-            // local_disks,
+            #[cfg(test)]
+            local_disks: None,
             // node,
             pools,
         }
     }
 
+    #[cfg(test)]
+    fn new_with_local_disks(_node: Option<Node>, pools: Option<Vec<usize>>, local_disks: Vec<DiskStore>) -> Self {
+        Self {
+            local_disks: Some(local_disks),
+            pools,
+        }
+    }
+
     async fn local_disks_for_pools(&self) -> Vec<DiskStore> {
+        #[cfg(test)]
+        let local_disks = if let Some(local_disks) = self.local_disks.as_ref() {
+            local_disks.clone()
+        } else {
+            all_local_disk().await
+        };
+        #[cfg(not(test))]
         let local_disks = all_local_disk().await;
         let Some(pools) = self.pools.as_ref() else {
             return local_disks;
@@ -1323,7 +1340,7 @@ mod tests {
 
             assert_eq!(walk_err, DiskError::Timeout);
 
-            let info = LocalPeerS3Client::new(None, Some(vec![0]))
+            let info = LocalPeerS3Client::new_with_local_disks(None, Some(vec![0]), disks.clone())
                 .get_bucket_info(bucket, &BucketOptions::default())
                 .await
                 .expect("bucket info should still succeed after prior walk timeout");
@@ -1347,7 +1364,7 @@ mod tests {
             .await
             .expect("bucket should be created on one disk");
 
-        let err = LocalPeerS3Client::new(None, Some(vec![0]))
+        let err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![0]), disks.clone())
             .get_bucket_info("partial-bucket", &BucketOptions::default())
             .await
             .expect_err("partial bucket should not satisfy local write quorum");
@@ -1375,19 +1392,19 @@ mod tests {
             .await
             .expect("bucket should be created on pool 0 disk 1");
 
-        let pool0_info = LocalPeerS3Client::new(None, Some(vec![0]))
+        let pool0_info = LocalPeerS3Client::new_with_local_disks(None, Some(vec![0]), disks.clone())
             .get_bucket_info(bucket, &BucketOptions::default())
             .await
             .expect("pool 0 peer should see bucket on pool 0 disks");
         assert_eq!(pool0_info.name, bucket);
 
-        let pool1_err = LocalPeerS3Client::new(None, Some(vec![1]))
+        let pool1_err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone())
             .get_bucket_info(bucket, &BucketOptions::default())
             .await
             .expect_err("pool 1 peer should not count pool 0 disks");
         assert_eq!(pool1_err, Error::VolumeNotFound);
 
-        let pool1_buckets = LocalPeerS3Client::new(None, Some(vec![1]))
+        let pool1_buckets = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone())
             .list_bucket(&BucketOptions::default())
             .await
             .expect("pool 1 local listing should succeed against its own disks");

crates/ecstore/src/set_disk.rs

@@ -1234,7 +1234,10 @@ impl rustfs_storage_api::ObjectIO for SetDisks {
 
             //TODO: userDefined
 
-            let etag = data.stream.try_resolve_etag().unwrap_or_default();
+            let mut etag = data.stream.try_resolve_etag().unwrap_or_default();
+            if let Some(ref tag) = opts.preserve_etag {
+                etag = tag.clone();
+            }
 
             user_defined.insert("etag".to_owned(), etag.clone());
 
@@ -2275,7 +2278,7 @@ impl rustfs_storage_api::ObjectOperations for SetDisks {
         if let Some(err) = &gerr
             && goi.name.is_empty()
         {
-            if opts.delete_marker {
+            if should_force_delete_marker_for_missing_version(&opts) {
                 version_found = false;
             } else {
                 return Err(err.clone());
@@ -2336,7 +2339,7 @@ impl rustfs_storage_api::ObjectOperations for SetDisks {
                 fi.set_skip_tier_free_version();
             }
 
-            fi.version_id = if let Some(vid) = opts.version_id {
+            fi.version_id = if let Some(vid) = opts.version_id.as_ref() {
                 Some(Uuid::parse_str(vid.as_str())?)
             } else if opts.versioned {
                 Some(Uuid::new_v4())
@@ -2344,7 +2347,7 @@ impl rustfs_storage_api::ObjectOperations for SetDisks {
                 None
             };
 
-            self.delete_object_version(bucket, object, &fi, opts.delete_marker)
+            self.delete_object_version(bucket, object, &fi, should_force_delete_marker_for_missing_version(&opts))
                 .await
                 .map_err(|e| to_object_err(e, vec![bucket, object]))?;
 
@@ -2905,8 +2908,12 @@ fn should_preserve_delete_replication_state(opts: &ObjectOptions) -> bool {
     }) || opts.version_purge_status() == VersionPurgeStatusType::Complete
 }
 
+fn should_force_delete_marker_for_missing_version(opts: &ObjectOptions) -> bool {
+    opts.delete_marker || (opts.versioned && opts.version_id.is_none() && !opts.data_movement)
+}
+
 fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) {
-    let mut mark_delete = goi.version_id.is_some();
+    let mut mark_delete = goi.version_id.is_some() || (opts.versioned && opts.version_id.is_none());
     let mut delete_marker = opts.versioned;
 
     if opts.version_id.is_some() {
@@ -3956,15 +3963,7 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
             object_size += ext_part.size;
             object_actual_size += ext_part.actual_size;
 
-            fi.parts.push(ObjectPartInfo {
-                etag: ext_part.etag.clone(),
-                number: p.part_num,
-                size: ext_part.size,
-                mod_time: ext_part.mod_time,
-                actual_size: ext_part.actual_size,
-                index: ext_part.index.clone(),
-                ..Default::default()
-            });
+            fi.parts.push(completed_multipart_object_part(p.part_num, ext_part));
         }
 
         if let Some(wtcs) = opts.want_checksum.as_ref() {
@@ -4913,6 +4912,19 @@ fn get_complete_multipart_md5(parts: &[CompletePart]) -> String {
     format!("{}-{}", etag_hex, parts.len())
 }
 
+fn completed_multipart_object_part(part_num: usize, ext_part: &ObjectPartInfo) -> ObjectPartInfo {
+    ObjectPartInfo {
+        etag: ext_part.etag.clone(),
+        number: part_num,
+        size: ext_part.size,
+        mod_time: ext_part.mod_time,
+        actual_size: ext_part.actual_size,
+        index: ext_part.index.clone(),
+        checksums: ext_part.checksums.clone(),
+        ..Default::default()
+    }
+}
+
 fn complete_part_checksum(part: &CompletePart, checksum_type: rustfs_rio::ChecksumType) -> Option<Option<String>> {
     match checksum_type.base() {
         rustfs_rio::ChecksumType::SHA256 => Some(part.checksum_sha256.clone()),
@@ -5300,6 +5312,42 @@ mod tests {
         assert!(delete_marker);
     }
 
+    #[test]
+    fn resolve_delete_version_state_creates_marker_for_missing_latest_versioned_delete() {
+        let opts = ObjectOptions {
+            versioned: true,
+            ..Default::default()
+        };
+
+        let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &ObjectInfo::default(), false);
+
+        assert!(mark_delete);
+        assert!(delete_marker);
+    }
+
+    #[test]
+    fn should_force_delete_marker_for_missing_version_rejects_data_movement_latest_delete() {
+        let opts = ObjectOptions {
+            versioned: true,
+            data_movement: true,
+            ..Default::default()
+        };
+
+        assert!(!should_force_delete_marker_for_missing_version(&opts));
+    }
+
+    #[test]
+    fn should_force_delete_marker_for_missing_version_allows_explicit_marker_creation() {
+        let opts = ObjectOptions {
+            versioned: true,
+            data_movement: true,
+            delete_marker: true,
+            ..Default::default()
+        };
+
+        assert!(should_force_delete_marker_for_missing_version(&opts));
+    }
+
     #[test]
     fn resolve_delete_version_state_skips_marker_creation_for_replica_purge_when_version_missing() {
         let opts = ObjectOptions {
@@ -5433,6 +5481,33 @@ mod tests {
         assert!(single_result.ends_with("-1"));
     }
 
+    #[test]
+    fn test_completed_multipart_object_part_preserves_checksums() {
+        let checksums = HashMap::from([
+            (rustfs_rio::ChecksumType::CRC32.to_string(), "crc32-value".to_string()),
+            (rustfs_rio::ChecksumType::CRC32C.to_string(), "crc32c-value".to_string()),
+        ]);
+        let ext_part = ObjectPartInfo {
+            number: 7,
+            etag: "etag-7".to_string(),
+            size: 123,
+            actual_size: 456,
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+            index: Some(Bytes::from_static(&[1, 2, 3])),
+            checksums: Some(checksums.clone()),
+            ..Default::default()
+        };
+
+        let completed = completed_multipart_object_part(7, &ext_part);
+
+        assert_eq!(completed.number, 7);
+        assert_eq!(completed.etag, ext_part.etag);
+        assert_eq!(completed.size, ext_part.size);
+        assert_eq!(completed.actual_size, ext_part.actual_size);
+        assert_eq!(completed.index, ext_part.index);
+        assert_eq!(completed.checksums, Some(checksums));
+    }
+
     #[test]
     fn test_get_upload_id_dir() {
         // Test upload ID directory path generation
@@ -6305,6 +6380,71 @@ mod tests {
         drop(temp_dirs);
     }
 
+    #[tokio::test]
+    async fn load_file_info_versions_exact_returns_none_for_explicit_not_found() {
+        let format = FormatV3::new(1, 1);
+        let (temp_dir, endpoint, disk) = make_formatted_local_disk_for_info_test(0, &format).await;
+        let bucket = "bucket";
+
+        disk.make_volume(bucket).await.expect("bucket should be created");
+
+        let set_disks = SetDisks::new(
+            "test-owner".to_string(),
+            Arc::new(RwLock::new(vec![Some(disk)])),
+            1,
+            0,
+            0,
+            0,
+            vec![endpoint],
+            format,
+            Vec::new(),
+        )
+        .await;
+
+        let versions = set_disks
+            .load_file_info_versions_exact(bucket, "missing-object")
+            .await
+            .expect("explicit object not found should be accepted");
+
+        assert!(versions.is_none());
+        drop(temp_dir);
+    }
+
+    #[tokio::test]
+    async fn load_file_info_versions_exact_rejects_corrupt_metadata() {
+        let format = FormatV3::new(1, 1);
+        let (temp_dir, endpoint, disk) = make_formatted_local_disk_for_info_test(0, &format).await;
+        let bucket = "bucket";
+        let object = "object.txt";
+
+        disk.make_volume(bucket).await.expect("bucket should be created");
+        let metadata_path = format!("{object}/{STORAGE_FORMAT_FILE}");
+        disk.write_all(bucket, &metadata_path, bytes::Bytes::from_static(b"not-xl-meta"))
+            .await
+            .expect("corrupt metadata file should be written");
+
+        let set_disks = SetDisks::new(
+            "test-owner".to_string(),
+            Arc::new(RwLock::new(vec![Some(disk)])),
+            1,
+            0,
+            0,
+            0,
+            vec![endpoint],
+            format,
+            Vec::new(),
+        )
+        .await;
+
+        let err = set_disks
+            .load_file_info_versions_exact(bucket, object)
+            .await
+            .expect_err("corrupt exact metadata must fail closed");
+
+        assert!(!is_err_object_not_found(&err), "corrupt metadata must not be treated as not found: {err}");
+        drop(temp_dir);
+    }
+
     #[tokio::test]
     async fn list_path_still_uses_disk_after_prior_walk_timeout() {
         use std::pin::Pin;

crates/ecstore/src/set_disk/read.rs

@@ -342,6 +342,54 @@ impl SetDisks {
         Self::pick_latest_quorum_files_info(fileinfos, errs, bucket, object, read_data, incl_free_vers).await
     }
 
+    pub(crate) async fn load_file_info_versions_exact(
+        &self,
+        bucket: &str,
+        object: &str,
+    ) -> Result<Option<rustfs_filemeta::FileInfoVersions>> {
+        let disks = self.get_disks_internal().await;
+        if disks.is_empty() {
+            return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object]));
+        }
+
+        let read_quorum = disks.len().div_ceil(2).max(1);
+        let (raw_fileinfos, errs) = Self::read_all_raw_file_info(&disks, bucket, object, false).await;
+
+        if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
+            let object_err = to_object_err(err.into(), vec![bucket, object]);
+            if is_err_object_not_found(&object_err) || is_err_version_not_found(&object_err) {
+                return Ok(None);
+            }
+            return Err(object_err);
+        }
+
+        let mut shallow_versions = Vec::with_capacity(raw_fileinfos.len());
+        for raw_fileinfo in raw_fileinfos.into_iter().flatten() {
+            let meta = FileMeta::load(&raw_fileinfo.buf)
+                .map_err(|err| Error::other(format!("exact object metadata decode failed for {bucket}/{object}: {err}")))?;
+            shallow_versions.push(meta.versions);
+        }
+
+        if shallow_versions.len() < read_quorum {
+            return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object]));
+        }
+
+        let versions = merge_file_meta_versions(read_quorum, true, 0, &shallow_versions);
+        if versions.is_empty() {
+            return Err(Error::other(format!(
+                "exact object metadata read returned no quorum versions for {bucket}/{object}"
+            )));
+        }
+
+        FileMeta {
+            versions,
+            ..Default::default()
+        }
+        .get_all_file_info_versions(bucket, object, true)
+        .map(Some)
+        .map_err(|err| Error::other(format!("exact object versions decode failed for {bucket}/{object}: {err}")))
+    }
+
     pub(super) async fn read_all_raw_file_info(
         disks: &[Option<DiskStore>],
         bucket: &str,

crates/ecstore/src/store.rs

@@ -93,7 +93,7 @@ use std::{
 };
 use time::OffsetDateTime;
 use tokio::select;
-use tokio::sync::RwLock;
+use tokio::sync::{Mutex, RwLock};
 use tokio::time::sleep;
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, instrument, warn};
@@ -188,6 +188,18 @@ pub struct ECStore {
     pub pool_meta: RwLock<PoolMeta>,
     pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
     pub decommission_cancelers: RwLock<Vec<Option<CancellationToken>>>,
+    /// Serializes rebalance/decommission start transitions.
+    ///
+    /// Lock order: acquire `start_gate` before `pool_meta`, `rebalance_meta`,
+    /// or `decommission_cancelers`. The guarded sections may perform bounded
+    /// async metadata work so check/init/start cannot race across operations.
+    pub(crate) start_gate: Mutex<()>,
+    /// Serializes full-document pool metadata saves.
+    ///
+    /// Lock order: acquire `pool_meta_save_gate` without holding `pool_meta`.
+    /// The saver then clones the latest `pool_meta` under a short read lock and
+    /// releases it before awaiting disk writes.
+    pub(crate) pool_meta_save_gate: Mutex<()>,
 
     // Phase 2 migration pending - do not use directly.
     /// Local disk maps (migrated from GLOBAL_LOCAL_DISK_MAP/ID_MAP/SET_DRIVES)

crates/ecstore/src/store/init.rs

@@ -18,6 +18,7 @@ use crate::global::{
     GLOBAL_EventNotifier, GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_TierConfigMgr,
     get_global_bucket_monitor, is_dist_erasure, is_first_cluster_node_local,
 };
+use crate::pools::local_decommission_queue_prefix;
 use tracing::{debug, error, info, warn};
 
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
@@ -54,6 +55,22 @@ fn should_retry_local_decommission_resume(err: &Error, attempt: usize) -> bool {
     matches!(err, Error::ConfigNotFound) && attempt < LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES
 }
 
+fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_meta_loaded: bool) -> bool {
+    rebalance_meta_loaded && !decommission_running
+}
+
+fn pool_meta_has_active_decommission(meta: &PoolMeta) -> bool {
+    meta.pools.iter().any(|pool| {
+        pool.decommission
+            .as_ref()
+            .is_some_and(|info| !info.complete && !info.failed && !info.canceled)
+    })
+}
+
+fn should_auto_start_rebalance_after_recovered_meta(pool_meta: &PoolMeta, rebalance_meta_loaded: bool) -> bool {
+    should_auto_start_rebalance_after_init(pool_meta_has_active_decommission(pool_meta), rebalance_meta_loaded)
+}
+
 async fn wait_for_local_decommission_resume_delay(rx: &CancellationToken, delay: Duration) -> bool {
     tokio::select! {
         _ = rx.cancelled() => false,
@@ -305,6 +322,8 @@ impl ECStore {
             pool_meta: RwLock::new(pool_meta),
             rebalance_meta: RwLock::new(None),
             decommission_cancelers,
+            start_gate: tokio::sync::Mutex::new(()),
+            pool_meta_save_gate: tokio::sync::Mutex::new(()),
 
             local_disk_map: GLOBAL_LOCAL_DISK_MAP.clone(),
             local_disk_id_map: GLOBAL_LOCAL_DISK_ID_MAP.clone(),
@@ -354,11 +373,6 @@ impl ECStore {
     pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
         GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await;
 
-        resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?;
-        if self.rebalance_meta.read().await.is_some() {
-            resolve_store_init_stage_result(self.start_rebalance().await, "start_rebalance")?;
-        }
-
         let mut meta = PoolMeta::default();
         resolve_store_init_stage_result(
             meta.load(
@@ -375,11 +389,8 @@ impl ECStore {
         let endpoints = get_global_endpoints();
         let should_persist_pool_meta = is_first_cluster_node_local().await;
 
-        if !update {
-            {
-                let mut pool_meta = self.pool_meta.write().await;
-                *pool_meta = meta.clone();
-            }
+        let installed_pool_meta = if !update {
+            meta.clone()
         } else {
             let new_meta = PoolMeta::new(&self.pools, &meta);
             // Only one local node should persist validated pool metadata here; otherwise
@@ -387,13 +398,32 @@ impl ECStore {
             if should_persist_pool_meta {
                 resolve_store_init_stage_result(new_meta.save(self.pools.clone()).await, "save_validated_pool_meta")?;
             }
-            {
-                let mut pool_meta = self.pool_meta.write().await;
-                *pool_meta = new_meta;
-            }
+            new_meta
+        };
+
+        {
+            let mut pool_meta = self.pool_meta.write().await;
+            *pool_meta = installed_pool_meta.clone();
         }
 
-        let pools = meta.return_resumable_pools();
+        resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?;
+        let rebalance_meta_loaded = self.rebalance_meta.read().await.is_some();
+        let decommission_running =
+            pool_meta_has_active_decommission(&installed_pool_meta) || self.is_decommission_running().await;
+        if should_auto_start_rebalance_after_init(decommission_running, rebalance_meta_loaded) {
+            resolve_store_init_stage_result(self.start_rebalance().await, "start_rebalance")?;
+        } else if decommission_running && rebalance_meta_loaded {
+            warn!(
+                event = EVENT_ECSTORE_INIT_STATUS,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_STORE_INIT,
+                stage = "start_rebalance",
+                reason = "active_decommission",
+                "Deferred rebalance auto-start during store init because decommission is active"
+            );
+        }
+
+        let pools = installed_pool_meta.return_resumable_pools();
         let mut pool_indices = Vec::with_capacity(pools.len());
 
         for p in pools.iter() {
@@ -407,18 +437,16 @@ impl ECStore {
             }
         }
 
-        if !pool_indices.is_empty() {
-            let idx = pool_indices[0];
-            if should_resume_local_decommission(&endpoints, idx)? {
-                let store = self.clone();
+        let local_pool_indices = local_decommission_queue_prefix(&endpoints, &pool_indices)?;
+        if !local_pool_indices.is_empty() {
+            let store = self.clone();
 
-                tokio::spawn(async move {
-                    if !wait_for_local_decommission_resume_delay(&rx, LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY).await {
-                        return;
-                    }
-                    resume_local_decommission_after_init(store, rx, pool_indices).await;
-                });
-            }
+            tokio::spawn(async move {
+                if !wait_for_local_decommission_resume_delay(&rx, LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY).await {
+                    return;
+                }
+                resume_local_decommission_after_init(store, rx, local_pool_indices).await;
+            });
         }
 
         let num_nodes = get_global_endpoints().get_nodes().len() as u64;
@@ -448,16 +476,33 @@ impl ECStore {
 mod tests {
     use super::{
         LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES, pool_first_endpoint_is_local, resolve_store_init_stage_result,
+        should_auto_start_rebalance_after_init, should_auto_start_rebalance_after_recovered_meta,
         should_resume_local_decommission, should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay,
     };
     use crate::{
         disk::endpoint::Endpoint,
         endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
         error::StorageError,
+        pools::{POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolStatus},
+        rebalance::RebalanceMeta,
     };
     use std::time::Duration;
+    use time::OffsetDateTime;
     use tokio_util::sync::CancellationToken;
 
+    fn init_test_pool_meta(decommission: Option<PoolDecommissionInfo>) -> PoolMeta {
+        PoolMeta {
+            version: POOL_META_VERSION,
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission,
+            }],
+            dont_save: false,
+        }
+    }
+
     #[test]
     fn test_should_resume_local_decommission_respects_local_flag() {
         let mut local_endpoint = Endpoint::try_from("http://127.0.0.1:9000/data").expect("endpoint should parse");
@@ -519,6 +564,43 @@ mod tests {
         assert!(!should_retry_local_decommission_resume(&StorageError::SlowDown, 0));
     }
 
+    #[test]
+    fn test_should_auto_start_rebalance_after_init_allows_loaded_rebalance_without_decommission() {
+        assert!(should_auto_start_rebalance_after_init(false, true));
+    }
+
+    #[test]
+    fn test_should_auto_start_rebalance_after_init_rejects_active_decommission() {
+        assert!(!should_auto_start_rebalance_after_init(true, true));
+    }
+
+    #[test]
+    fn test_should_auto_start_rebalance_after_init_rejects_missing_rebalance_meta() {
+        assert!(!should_auto_start_rebalance_after_init(false, false));
+    }
+
+    #[test]
+    fn test_store_init_recovery_skips_rebalance_when_decommission_metadata_is_active() {
+        let pool_meta = init_test_pool_meta(Some(PoolDecommissionInfo {
+            start_time: Some(OffsetDateTime::UNIX_EPOCH),
+            complete: false,
+            failed: false,
+            canceled: false,
+            ..Default::default()
+        }));
+        let rebalance_meta = Some(RebalanceMeta::default());
+
+        assert!(!should_auto_start_rebalance_after_recovered_meta(&pool_meta, rebalance_meta.is_some()));
+    }
+
+    #[test]
+    fn test_store_init_recovery_allows_rebalance_when_only_rebalance_metadata_exists() {
+        let pool_meta = init_test_pool_meta(None);
+        let rebalance_meta = Some(RebalanceMeta::default());
+
+        assert!(should_auto_start_rebalance_after_recovered_meta(&pool_meta, rebalance_meta.is_some()));
+    }
+
     #[test]
     fn test_resolve_store_init_stage_result_passthrough_ok() {
         resolve_store_init_stage_result(Ok(()), "load_rebalance_meta").expect("successful stage should pass through");

crates/ecstore/src/store/object.rs

@@ -216,6 +216,10 @@ fn resolve_latest_object_access(
     Ok((info, idx))
 }
 
+fn should_create_delete_marker_for_missing_object(opts: &ObjectOptions) -> bool {
+    opts.versioned && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement
+}
+
 fn version_aware_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
     let mut lookup_opts = opts.clone();
     lookup_opts.no_lock = no_lock;
@@ -234,6 +238,12 @@ fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> Object
     lookup_opts
 }
 
+fn transition_restore_pool_opts(opts: &ObjectOptions) -> ObjectOptions {
+    let mut lookup_opts = opts.clone();
+    lookup_opts.skip_decommissioned = true;
+    lookup_opts
+}
+
 fn effective_object_actual_size(info: &ObjectInfo) -> Option<i64> {
     info.get_actual_size().ok()
 }
@@ -243,27 +253,44 @@ fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &Objec
         && is_data_movement_delete_marker(target)
         && source.version_id == target.version_id
         && source.mod_time == target.mod_time
+        && source.user_defined == target.user_defined
+        && source.user_tags == target.user_tags
+        && source.replication_status_internal == target.replication_status_internal
+        && source.replication_status == target.replication_status
+        && source.version_purge_status_internal == target.version_purge_status_internal
+        && source.version_purge_status == target.version_purge_status
 }
 
 fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool {
     info.delete_marker
 }
 
+fn expected_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo) -> ObjectInfo {
+    ObjectInfo::from_file_info(source, "", &source.name, source.version_id.is_some())
+}
+
 fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool {
+    let expected = expected_data_movement_tiered_object(source);
+
     source.version_id == target.version_id
         && !target.delete_marker
         && source.size == target.size
         && source.get_etag() == target.etag
         && source.checksum == target.checksum
         && source.mod_time == target.mod_time
-        && source.transition_status == target.transitioned_object.status
-        && source.transitioned_objname == target.transitioned_object.name
-        && source.transition_tier == target.transitioned_object.tier
-        && source
-            .transition_version_id
-            .map(|version_id| version_id.to_string())
-            .unwrap_or_default()
-            == target.transitioned_object.version_id
+        && expected.user_defined == target.user_defined
+        && expected.user_tags == target.user_tags
+        && expected.expires == target.expires
+        && expected.storage_class == target.storage_class
+        && expected.replication_status_internal == target.replication_status_internal
+        && expected.replication_status == target.replication_status
+        && expected.version_purge_status_internal == target.version_purge_status_internal
+        && expected.version_purge_status == target.version_purge_status
+        && expected.transitioned_object.status == target.transitioned_object.status
+        && expected.transitioned_object.name == target.transitioned_object.name
+        && expected.transitioned_object.tier == target.transitioned_object.tier
+        && expected.transitioned_object.version_id == target.transitioned_object.version_id
+        && expected.transitioned_object.free_version == target.transitioned_object.free_version
         && effective_object_actual_size(target) == Some(source.size)
 }
 
@@ -836,7 +863,7 @@ impl ECStore {
             let target_pool_idx =
                 resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx);
 
-            if opts.src_pool_idx == selected_target_pool_idx {
+            if !should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
                 if let Ok((source_pool_info, _)) = existing_pool_info
                     && opts.delete_marker
                     && is_data_movement_delete_marker(&source_pool_info.object_info)
@@ -862,24 +889,23 @@ impl ECStore {
                 ));
             }
 
-            let mut obj = self.pools[selected_target_pool_idx]
-                .delete_object(bucket, object, opts)
-                .await?;
+            let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
             obj.name = decode_dir_object(obj.name.as_str());
             return Ok(obj);
         }
 
         // Determine which pool contains it
-        let (mut pinfo, errs) = self
-            .get_pool_info_existing_with_opts(bucket, object, &gopts)
-            .await
-            .map_err(|e| {
-                if is_err_read_quorum(&e) {
-                    StorageError::ErasureWriteQuorum
-                } else {
-                    e
-                }
-            })?;
+        let (mut pinfo, errs) = match self.get_pool_info_existing_with_opts(bucket, object, &gopts).await {
+            Ok(res) => res,
+            Err(err) if is_err_read_quorum(&err) => return Err(StorageError::ErasureWriteQuorum),
+            Err(err) if is_err_object_not_found(&err) && should_create_delete_marker_for_missing_object(&opts) => {
+                let target_pool_idx = self.get_pool_idx_no_lock(bucket, object, 0).await?;
+                let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
+                obj.name = decode_dir_object(object);
+                return Ok(obj);
+            }
+            Err(err) => return Err(err),
+        };
 
         if pinfo.object_info.delete_marker && opts.version_id.is_none() {
             pinfo.object_info.name = decode_dir_object(object);
@@ -1133,11 +1159,12 @@ impl ECStore {
             return self.pools[0].transition_object(bucket, &object, opts).await;
         }
 
-        //opts.skip_decommissioned = true;
-        //opts.no_lock = true;
-        let (_, idx) = self.get_latest_accessible_object_info_with_idx(bucket, &object, opts).await?;
+        let opts = transition_restore_pool_opts(opts);
+        let (_, idx) = self
+            .get_latest_accessible_object_info_with_idx(bucket, &object, &opts)
+            .await?;
 
-        self.pools[idx].transition_object(bucket, &object, opts).await
+        self.pools[idx].transition_object(bucket, &object, &opts).await
     }
 
     #[instrument(skip(self))]
@@ -1152,15 +1179,14 @@ impl ECStore {
             return self.pools[0].clone().restore_transitioned_object(bucket, &object, opts).await;
         }
 
-        //opts.skip_decommissioned = true;
-        //opts.nolock = true;
+        let opts = transition_restore_pool_opts(opts);
         let (_, idx) = self
-            .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), opts)
+            .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
             .await?;
 
         self.pools[idx]
             .clone()
-            .restore_transitioned_object(bucket, &object, opts)
+            .restore_transitioned_object(bucket, &object, &opts)
             .await
     }
 
@@ -1269,8 +1295,8 @@ mod tests {
     use super::*;
     use crate::bucket::lifecycle::core::TRANSITION_COMPLETE;
     use bytes::Bytes;
-    use rustfs_storage_api::TransitionedObject;
     use std::io::Cursor;
+    use std::sync::Arc;
     use tokio::io::AsyncReadExt;
 
     #[test]
@@ -1313,6 +1339,33 @@ mod tests {
         assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched));
     }
 
+    #[test]
+    fn equivalent_data_movement_delete_marker_rejects_metadata_and_replication_mismatch() {
+        let version_id = Uuid::nil();
+        let mod_time = OffsetDateTime::UNIX_EPOCH;
+        let source = ObjectInfo {
+            version_id: Some(version_id),
+            delete_marker: true,
+            mod_time: Some(mod_time),
+            user_defined: Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "true".to_string())])),
+            replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()),
+            version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()),
+            ..Default::default()
+        };
+
+        let mut target = source.clone();
+        target.user_defined = Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "false".to_string())]));
+        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
+
+        let mut target = source.clone();
+        target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string());
+        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
+
+        let mut target = source.clone();
+        target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string());
+        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
+    }
+
     #[test]
     fn equivalent_data_movement_delete_marker_rejects_live_object() {
         let source = ObjectInfo {
@@ -1367,6 +1420,7 @@ mod tests {
     fn data_movement_resume_target_uses_resolved_non_source_pool_when_selected_is_source() {
         let target_pool_idx = resolve_data_movement_resume_target_pool(1, Some(3), 1);
         assert_eq!(target_pool_idx, 3);
+        assert!(should_check_data_movement_resume_target(1, target_pool_idx));
     }
 
     #[test]
@@ -1386,12 +1440,12 @@ mod tests {
         assert!(matches!(result, Err(Error::SlowDown)));
     }
 
-    #[test]
-    fn equivalent_data_movement_tiered_object_accepts_matching_transition_metadata() {
+    fn tiered_equivalence_source() -> FileInfo {
         let version_id = Uuid::nil();
         let transition_version_id = Uuid::new_v4();
         let mod_time = OffsetDateTime::UNIX_EPOCH;
-        let source = FileInfo {
+
+        FileInfo {
             version_id: Some(version_id),
             size: 1024,
             mod_time: Some(mod_time),
@@ -1400,80 +1454,86 @@ mod tests {
             transitioned_objname: "remote/object".to_string(),
             transition_tier: "WARM".to_string(),
             transition_version_id: Some(transition_version_id),
-            metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
-            ..Default::default()
-        };
-        let target = ObjectInfo {
-            version_id: Some(version_id),
-            size: 1024,
-            mod_time: Some(mod_time),
-            checksum: Some(Bytes::from_static(b"checksum")),
-            etag: Some("etag-value".to_string()),
-            transitioned_object: TransitionedObject {
-                name: "remote/object".to_string(),
-                version_id: transition_version_id.to_string(),
-                tier: "WARM".to_string(),
-                status: TRANSITION_COMPLETE.to_string(),
+            replication_state_internal: Some(rustfs_filemeta::ReplicationState {
+                replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()),
+                targets: rustfs_filemeta::replication_statuses_map("arn:minio:replication:target=COMPLETED;"),
+                version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()),
+                purge_targets: rustfs_filemeta::version_purge_statuses_map("arn:minio:replication:target=PENDING;"),
                 ..Default::default()
-            },
+            }),
+            metadata: HashMap::from([
+                ("etag".to_string(), "etag-value".to_string()),
+                ("x-amz-meta-key".to_string(), "metadata-value".to_string()),
+                (rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), "tag=value".to_string()),
+                ("expires".to_string(), "1970-01-01T00:33:20Z".to_string()),
+            ]),
             ..Default::default()
-        };
+        }
+    }
+
+    fn tiered_equivalence_target(source: &FileInfo) -> ObjectInfo {
+        ObjectInfo::from_file_info(source, "bucket", "object", source.version_id.is_some())
+    }
+
+    #[test]
+    fn equivalent_data_movement_tiered_object_accepts_matching_persisted_metadata() {
+        let source = tiered_equivalence_source();
+        let target = tiered_equivalence_target(&source);
 
         assert!(is_equivalent_data_movement_tiered_object(&source, &target));
     }
 
     #[test]
     fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() {
-        let source = FileInfo {
-            version_id: Some(Uuid::nil()),
-            size: 1024,
-            transition_status: TRANSITION_COMPLETE.to_string(),
-            transitioned_objname: "remote/source".to_string(),
-            transition_tier: "WARM".to_string(),
-            ..Default::default()
-        };
-        let target = ObjectInfo {
-            version_id: source.version_id,
-            size: 1024,
-            transitioned_object: TransitionedObject {
-                name: "remote/target".to_string(),
-                tier: "WARM".to_string(),
-                status: TRANSITION_COMPLETE.to_string(),
-                ..Default::default()
-            },
-            ..Default::default()
-        };
+        let source = tiered_equivalence_source();
+        let mut target = tiered_equivalence_target(&source);
+        target.transitioned_object.name = "remote/target".to_string();
+
+        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
+    }
+
+    #[test]
+    fn equivalent_data_movement_tiered_object_rejects_user_metadata_mismatch() {
+        let source = tiered_equivalence_source();
+        let mut target = tiered_equivalence_target(&source);
+        target.user_defined = Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "target-value".to_string())]));
+
+        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
+    }
+
+    #[test]
+    fn equivalent_data_movement_tiered_object_rejects_tag_mismatch() {
+        let source = tiered_equivalence_source();
+        let mut target = tiered_equivalence_target(&source);
+        target.user_tags = Arc::new("tag=target".to_string());
+
+        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
+    }
+
+    #[test]
+    fn equivalent_data_movement_tiered_object_rejects_replication_mismatch() {
+        let source = tiered_equivalence_source();
+        let mut target = tiered_equivalence_target(&source);
+        target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string());
+        target.replication_status = rustfs_filemeta::ReplicationStatusType::Failed;
+
+        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
+    }
+
+    #[test]
+    fn equivalent_data_movement_tiered_object_rejects_version_purge_mismatch() {
+        let source = tiered_equivalence_source();
+        let mut target = tiered_equivalence_target(&source);
+        target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string());
+        target.version_purge_status = rustfs_filemeta::VersionPurgeStatusType::Complete;
 
         assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
     }
 
     #[test]
     fn data_movement_tiered_resume_accepts_equivalent_target() {
-        let version_id = Uuid::nil();
-        let transition_version_id = Uuid::new_v4();
-        let source = FileInfo {
-            version_id: Some(version_id),
-            size: 1024,
-            transition_status: TRANSITION_COMPLETE.to_string(),
-            transitioned_objname: "remote/object".to_string(),
-            transition_tier: "WARM".to_string(),
-            transition_version_id: Some(transition_version_id),
-            metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
-            ..Default::default()
-        };
-        let target = ObjectInfo {
-            version_id: Some(version_id),
-            size: 1024,
-            etag: Some("etag-value".to_string()),
-            transitioned_object: TransitionedObject {
-                name: "remote/object".to_string(),
-                version_id: transition_version_id.to_string(),
-                tier: "WARM".to_string(),
-                status: TRANSITION_COMPLETE.to_string(),
-                ..Default::default()
-            },
-            ..Default::default()
-        };
+        let source = tiered_equivalence_source();
+        let target = tiered_equivalence_target(&source);
 
         let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 1)
             .expect("equivalent tiered target should be evaluated");
@@ -1483,28 +1543,8 @@ mod tests {
 
     #[test]
     fn data_movement_tiered_resume_rejects_source_pool_target() {
-        let version_id = Uuid::nil();
-        let source = FileInfo {
-            version_id: Some(version_id),
-            size: 1024,
-            transition_status: TRANSITION_COMPLETE.to_string(),
-            transitioned_objname: "remote/object".to_string(),
-            transition_tier: "WARM".to_string(),
-            metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
-            ..Default::default()
-        };
-        let target = ObjectInfo {
-            version_id: Some(version_id),
-            size: 1024,
-            etag: Some("etag-value".to_string()),
-            transitioned_object: TransitionedObject {
-                name: "remote/object".to_string(),
-                tier: "WARM".to_string(),
-                status: TRANSITION_COMPLETE.to_string(),
-                ..Default::default()
-            },
-            ..Default::default()
-        };
+        let source = tiered_equivalence_source();
+        let target = tiered_equivalence_target(&source);
 
         let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 0)
             .expect("source-pool target should be rejected before target lookup");
@@ -1623,6 +1663,39 @@ mod tests {
         assert!(matches!(err, Error::MethodNotAllowed));
     }
 
+    #[test]
+    fn should_create_delete_marker_for_missing_object_allows_latest_versioned_delete() {
+        let opts = ObjectOptions {
+            versioned: true,
+            ..Default::default()
+        };
+
+        assert!(should_create_delete_marker_for_missing_object(&opts));
+    }
+
+    #[test]
+    fn should_create_delete_marker_for_missing_object_rejects_specialized_deletes() {
+        let version_delete = ObjectOptions {
+            versioned: true,
+            version_id: Some("vid-1".to_string()),
+            ..Default::default()
+        };
+        let delete_marker_replication = ObjectOptions {
+            versioned: true,
+            delete_marker: true,
+            ..Default::default()
+        };
+        let data_movement = ObjectOptions {
+            versioned: true,
+            data_movement: true,
+            ..Default::default()
+        };
+
+        assert!(!should_create_delete_marker_for_missing_object(&version_delete));
+        assert!(!should_create_delete_marker_for_missing_object(&delete_marker_replication));
+        assert!(!should_create_delete_marker_for_missing_object(&data_movement));
+    }
+
     #[test]
     fn resolve_decommission_target_pool_idx_result_passthrough_ok() {
         let idx = ECStore::resolve_decommission_target_pool_idx_result(Ok(3), "bucket", "object").unwrap();
@@ -1715,6 +1788,29 @@ mod tests {
         assert!(lookup_opts.skip_rebalancing);
     }
 
+    #[test]
+    fn transition_restore_pool_opts_skips_decommissioned_and_preserves_locking() {
+        let lookup_opts = transition_restore_pool_opts(&ObjectOptions {
+            no_lock: false,
+            skip_decommissioned: false,
+            ..Default::default()
+        });
+
+        assert!(lookup_opts.skip_decommissioned);
+        assert!(!lookup_opts.no_lock);
+    }
+
+    #[test]
+    fn transition_restore_pool_opts_preserves_existing_no_lock() {
+        let lookup_opts = transition_restore_pool_opts(&ObjectOptions {
+            no_lock: true,
+            ..Default::default()
+        });
+
+        assert!(lookup_opts.skip_decommissioned);
+        assert!(lookup_opts.no_lock);
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn reader_lock_is_held_when_optimization_is_disabled() {

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1080,8 +1080,11 @@ pub struct ReloadPoolMetaResponse {
     #[prost(string, optional, tag = "2")]
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
 }
-#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
-pub struct StopRebalanceRequest {}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct StopRebalanceRequest {
+    #[prost(string, tag = "1")]
+    pub expected_rebalance_id: ::prost::alloc::string::String,
+}
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct StopRebalanceResponse {
     #[prost(bool, tag = "1")]

crates/protos/src/node.proto

@@ -764,7 +764,9 @@ message ReloadPoolMetaResponse {
   optional string error_info = 2;
 }
 
-message StopRebalanceRequest {}
+message StopRebalanceRequest {
+  string expected_rebalance_id = 1;
+}
 
 message StopRebalanceResponse {
   bool success = 1;

docs/architecture/decommission-compatibility.md

@@ -0,0 +1,165 @@
+# Decommission Compatibility Scope
+
+This note records the current RustFS decommission contract for admin/API
+compatibility reviews.
+
+## Current Contract
+
+RustFS supports queued multi-pool decommission start requests on multi-pool
+deployments.
+
+The admin handler accepts the request shape used by the MinIO-compatible admin
+API, including comma-separated pool targets. An empty target list is rejected.
+Single-pool deployments reject decommission because there is no destination pool.
+On multi-pool deployments, one or more valid target pools are accepted as a
+single queued operation.
+
+### Request Semantics
+
+`POST /v3/pools/decommission` with comma-separated pool targets is treated as a
+queue submission:
+
+- validate all requested pool identifiers before mutating metadata;
+- reject duplicate target pools in the same request;
+- reject active or queued target pools;
+- reject completed decommission targets because completion means the pool can be
+  removed from the deployment configuration;
+- allow failed or canceled targets to be retried;
+- persist queued metadata before starting workers;
+- start only the local-leader prefix of the queue on the receiving node.
+
+The local-leader-prefix rule keeps the active worker on the leader for the pool
+being moved while still allowing a request to contain later targets whose leaders
+are different nodes. Later queued targets are recovered or promoted by the
+leader that owns that target.
+
+### Persisted Metadata Shape
+
+The queue is persisted in pool metadata and decoded with the rest of
+`PoolMeta`. Each pool entry can distinguish:
+
+- `active`: at most one pool currently moving data;
+- `queued`: validated pools waiting for the active entry to finish;
+- `completed`: pools finished successfully;
+- `failed`: pools whose worker reached terminal failure;
+- `canceled`: pools canceled before or during execution.
+
+Legacy metadata without queue fields decodes as a non-queued decommission entry,
+preserving restart behavior for already deployed clusters.
+
+### Serial Scheduling And Recovery
+
+Only one queued entry may own a decommission worker at a time. Startup recovery:
+
+- loads pool metadata before rebalance recovery;
+- resumes the first local non-terminal active/queued entry;
+- skips a durably completed prefix and promotes the next queued entry only after
+  successful completion;
+- treats failed or canceled terminal entries as an automatic-promotion barrier,
+  leaving later queued pools visible but stopped until an operator retries,
+  clears, or otherwise resolves the terminal entry;
+- keeps queued pools out of active worker scheduling until promotion, while still
+  making their future state visible in admin status.
+
+Promotion is persisted before worker execution. If cancellation is already
+requested immediately after promotion, RustFS persists a canceled terminal state
+instead of leaving the promoted pool active without a worker.
+
+### Cancel Semantics
+
+Cancel separates active and queued behavior:
+
+- canceling the active entry requests worker cancellation and persists terminal
+  metadata;
+- canceling a queued entry marks that entry canceled before it becomes active;
+- failed or canceled terminal entries can be cleared explicitly when the operator
+  chooses to abandon the decommission attempt;
+- peer reload failures during cancel must be surfaced in status and logs.
+
+Cancel requests can be accepted on non-leader nodes as remote cancel intent; the
+leader observes the pending cancel and applies it to the active worker.
+
+### Status Response Shape
+
+`GET /v3/pools/list` and `GET /v3/pools/status?pool=...` expose per-pool
+machine-readable decommission state. The `status` field can report `active`,
+`running`, `queued`, `complete`, `failed`, or `canceled`.
+
+When decommission metadata is present, `decommissionInfo` includes:
+
+- queue and terminal flags: `queued`, `complete`, `failed`, `canceled`;
+- progress counters: `objectsDecommissioned`,
+  `objectsDecommissionedFailed`, `bytesDecommissioned`, and
+  `bytesDecommissionedFailed`;
+- current location: `bucket`, `prefix`, and `object`;
+- queue/history lists: `queuedBuckets` and `decommissionedBuckets`;
+- `waitingReason`, currently `queued` for queued entries and
+  `waiting_for_worker` when metadata exists but no worker has started.
+
+This makes queued pools and stalled metadata visible without requiring operators
+to inspect pool metadata files directly.
+
+## MinIO Divergence Decisions
+
+This section records the current product decisions for behavior that is close to
+MinIO but not always byte-for-byte identical.
+
+### Empty Delete Markers
+
+MinIO decommission documentation states that empty delete markers, meaning delete
+markers with no successor object versions, are not transitioned to another pool.
+
+RustFS follows that behavior for decommission when the bucket has no replication
+configuration: a lone remaining delete marker is treated as cleanup-only metadata
+and is skipped. When replication is configured, RustFS intentionally keeps the
+delete marker eligible for movement so delete-marker replication and purge state
+are not lost.
+
+RustFS rebalance uses the same predicate as decommission: skip only a lone delete
+marker without replication. This is intentional even though MinIO's public
+documentation calls out the decommission case more explicitly than the rebalance
+case.
+
+Regression guards:
+
+- `should_skip_decommission_delete_marker_characterizes_empty_marker_without_replication`
+- `should_skip_decommission_delete_marker_characterizes_replication_configured`
+- `test_should_skip_rebalance_delete_marker_characterizes_empty_marker_without_replication`
+- `test_should_skip_rebalance_delete_marker_characterizes_replication_configured`
+
+### Lifecycle-Expired Versions During Cleanup
+
+MinIO decommission ignores versions that are already expired by lifecycle rules.
+RustFS follows that decommission behavior by allowing safely expired versions to
+count toward source cleanup completion.
+
+RustFS rebalance is intentionally stricter. Expired versions do not prove that a
+target pool received an equivalent version, so rebalance cleanup requires actual
+rebalance completion for the source entry instead of treating lifecycle-expired
+versions as moved.
+
+Regression guards:
+
+- `test_should_cleanup_decommission_source_entry_accepts_migrated_and_safely_expired_versions`
+- `test_should_cleanup_decommission_source_entry_accepts_versions_only_safely_expired_by_lifecycle`
+- `test_should_cleanup_rebalance_source_entry_rejects_versions_only_expired_by_lifecycle`
+
+No migration step is required for these decisions because this note documents the
+current RustFS behavior. Changing either decision later requires an operator
+compatibility note and updated characterization tests.
+
+## Regression Guard
+
+The queued multi-pool contract is guarded by:
+
+- `test_contextualized_decommission_start_request_allows_multiple_target_pools`
+- `test_decommission_start_local_leader_allows_remote_queued_pool`
+- `test_local_decommission_queue_prefix_stops_at_remote_leader`
+- `test_pool_meta_queued_decommission_is_not_suspended_until_promoted`
+- `test_pool_meta_promoted_queued_decommission_can_be_canceled`
+- `test_first_resumable_decommission_queue_indices_stops_at_failed_or_canceled_state`
+- `test_first_resumable_decommission_queue_indices_allows_after_completed_prefix`
+- `admin_pool_list_item_exposes_queued_decommission_state`
+
+These tests live in `crates/ecstore/src/pools.rs` and
+`rustfs/src/app/admin_usecase.rs`.

docs/architecture/rebalance-decommission-implementation-plan-index.md

@@ -0,0 +1,76 @@
+# Rebalance and Decommission Implementation Plan Index
+
+> This index is based on `docs/architecture/rebalance-decommission-remediation-plan.md`. The remediation scope is intentionally split into smaller implementation plans because the fixes touch independent risk areas: object-version safety, distributed operation semantics, data movement internals, and operational hardening.
+
+## Why Split the Work
+
+The remediation backlog has fourteen fix blocks. Implementing them in one PR would make review risky and would mix unrelated failure modes. The safer path is:
+
+1. Fix the data semantics that can lose or corrupt object-version meaning.
+2. Fix distributed start/stop/recovery semantics so operators can trust cluster state.
+3. Fix resource and metadata correctness in shared data movement.
+4. Add observability and compatibility hardening.
+
+Each plan below should be reviewed and executed independently unless the plan explicitly says two fixes must share the same implementation.
+
+## Plan Set
+
+| Plan | Fixes | Status | Purpose |
+| --- | --- | --- | --- |
+| `rebalance-decommission-phase1-safety-plan.md` | F01, F02, F03, F04, F05 | Drafted | Protect object-version semantics and cluster operation safety |
+| `rebalance-decommission-phase2-data-movement-plan.md` | F06, F07, F08, F09, F10 | Drafted | Stream multipart migration, preserve metadata, and improve convergence |
+| `rebalance-decommission-phase3-hardening-plan.md` | F11, F12, F13, F14 | Drafted | Improve cleanup reporting, auditability, metadata decoding, and threshold docs |
+| `rebalance-decommission-followup-review-plan.md` | R01-R16 | Reviewed | Close post-implementation review gaps found after F01-F14 |
+
+## Execution Recommendation
+
+Start with `rebalance-decommission-phase1-safety-plan.md`. Phase 1 contains the highest-risk issues and defines semantics that later fixes depend on.
+
+Within Phase 1:
+
+1. F01 should be analyzed first because rebalance delete marker and remote tiered behavior determines whether RustFS should move or skip these versions.
+2. F02 can be implemented independently once the expected decommission overwrite semantics are confirmed.
+3. F03, F04, and F05 can be designed together but should remain separate PRs unless a shared peer-failure helper is introduced.
+
+For a long-running implementation task, use the phase plans as checkpoints:
+
+1. Implement one fix block at a time.
+2. Run the focused tests listed in that block.
+3. Review the diff before moving to the next fix block.
+4. Run the phase-level test matrix before considering the phase complete.
+
+After the initial F01-F14 implementation pass, continue with `rebalance-decommission-followup-review-plan.md`. That follow-up plan is ordered by remaining risk and should be executed before treating the remediation as complete.
+
+## Upstream Change Impact Notes
+
+### 2026-06-17: `ed55857b refactor: move bucket operations contract (#3507)`
+
+This upstream change moved `BucketOperations` from `crates/ecstore/src/store_api/traits.rs` into `crates/storage-api/src/bucket.rs` and re-exported it from `rustfs_storage_api`.
+
+Impact on this remediation plan:
+
+- No F01-F14 risk item is fixed by this upstream change.
+- No planned fix block needs priority changes because of this upstream change.
+- Implementation code that imports or bounds `BucketOperations` must now use `rustfs_storage_api::BucketOperations`.
+- Generic implementations that need the ECStore error type should use an explicit associated error bound such as `BucketOperations<Error = crate::error::Error>`.
+- The only touched planning-relevant files are import/boundary changes in paths such as `crates/ecstore/src/pools.rs`, `crates/ecstore/src/store.rs`, `crates/ecstore/src/set_disk.rs`, and `rustfs/src/admin/handlers/rebalance.rs`; the rebalance/decommission safety logic remains unchanged.
+
+## Review Gates Before Implementation
+
+Before any code patch starts for a fix block:
+
+- Confirm the selected behavior in the corresponding plan.
+- Identify the minimal set of files for that fix.
+- Write or update failing tests first.
+- Keep unrelated refactors out of scope.
+- Run focused tests for the touched crate before broader checks.
+
+## Verification Baseline
+
+For any code PR generated from these plans:
+
+- Run focused `cargo test` commands for touched crates and modules.
+- Run `cargo fmt --all`.
+- Run `cargo fmt --all --check`.
+- Run `make pre-commit` before opening a PR when the implementation is ready.
+- Clean generated build artifacts after build-based verification.

docs/architecture/rebalance-decommission-phase1-safety-plan.md

@@ -0,0 +1,376 @@
+# Rebalance and Decommission Phase 1 Safety Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Fix the highest-risk rebalance/decommission safety defects so object-version semantics and distributed operation state cannot silently diverge.
+
+**Architecture:** Phase 1 keeps the existing ECStore/Rebalance/PoolMeta architecture and introduces narrowly scoped behavior changes. The plan favors fail-closed semantics for unsafe migration, peer propagation, and startup conflicts. Each fix block should be implemented as a separate PR unless a shared helper is explicitly called out.
+
+**Tech Stack:** Rust, Tokio, ECStore, Axum admin handlers, tonic peer RPC, MessagePack metadata, existing RustFS test modules.
+
+---
+
+## Scope
+
+This plan covers:
+
+- F01: rebalance delete marker and remote tiered version migration semantics.
+- F02: decommission `DataMovementOverwriteErr` cleanup safety.
+- F03: decommission `pool_meta` reload as a start barrier.
+- F04: store startup recovery ordering for pool meta and rebalance.
+- F05: rebalance distributed start/stop semantics and `stopping` visibility.
+
+This plan does not cover multipart streaming, checksum preservation, lifecycle-expired cleanup, overwrite convergence, cleanup warning UX, audit fields, or metadata decoding hardening. Those belong to later phase plans.
+
+## Shared Principles
+
+- Treat unknown target equivalence as unsafe.
+- Do not clean a source entry unless the target state is proven complete or the version is intentionally skipped by a documented policy.
+- Do not report distributed admin success when required peers failed.
+- Do not auto-run rebalance and decommission together after restart.
+- Prefer small local helpers near existing logic over broad refactors.
+
+## F01: Rebalance Delete Marker and Remote Tiered Version Safety
+
+### Decision Needed
+
+Remote tiered versions need a product decision before coding:
+
+- **Recommended short-term behavior:** Match MinIO and skip remote tiered versions during rebalance. This avoids changing remote-tier metadata during a balancing operation and removes the immediate source-cleanup data-loss risk.
+- **Later behavior:** Implement explicit cross-pool target metadata movement for remote tiered versions only after tests prove remote references, lifecycle state, and cleanup semantics.
+
+Delete markers should not be skipped. They preserve versioned delete semantics and must be copied to the selected target pool before source cleanup.
+
+### Files
+
+- Modify: `crates/ecstore/src/rebalance.rs`
+- Possibly modify: `crates/ecstore/src/data_movement.rs`
+- Avoid unless necessary: `crates/ecstore/src/set_disk.rs`
+- Test: existing rebalance unit tests in `crates/ecstore/src/rebalance.rs`
+
+### Design
+
+1. Add an ECStore-level helper for rebalance delete marker movement.
+   - Input: source pool index, bucket, object name, source `FileInfo`, version ID.
+   - Behavior: choose a target pool using the same placement rules as normal data movement, excluding source/rebalancing/decommissioned pools.
+   - Write the delete marker metadata to the target pool with the original version ID, mod time, delete marker flag, and replication state.
+   - Return success only when the target write succeeds or an equivalent target delete marker is confirmed.
+
+2. Change the delete marker branch in `migrate_entry_version_with_retry_wait`.
+   - Do not call `SetDisks::delete_object_for_migration` for rebalance delete markers.
+   - Call the ECStore-level helper via the `transfer`/backend abstraction or split delete marker handling out of the `SetDisks` backend path.
+   - Do not set `moved = true` unless the target helper succeeds.
+
+3. Change remote tiered branch.
+   - Short term: return `ignored = true`, `cleanup_ignored = false`, `moved = false` for remote tiered versions so they do not contribute to full source cleanup.
+   - If skipping a remote tiered version prevents source cleanup, record a clear status reason so operators understand why the entry remains.
+
+4. Source cleanup must remain blocked if any version was skipped without cleanup permission.
+
+### Implementation Tasks
+
+- [ ] Write a failing test proving a rebalance delete marker currently does not require target-pool metadata before counting complete.
+- [ ] Write a failing test proving remote tiered versions do not allow source cleanup under the short-term skip policy.
+- [ ] Implement the delete marker target write helper.
+- [ ] Replace the rebalance delete marker source-set branch with the target write helper.
+- [ ] Replace the remote tiered branch with the skip-without-cleanup policy.
+- [ ] Run focused rebalance tests.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs-ecstore rebalance_delete_marker --lib
+cargo test -p rustfs-ecstore remote_tier --lib
+cargo test -p rustfs-ecstore rebalance_entry --lib
+```
+
+### Acceptance Criteria
+
+- Rebalance does not clean a source entry when a delete marker target write fails.
+- Rebalance preserves versioned delete behavior after source cleanup.
+- Remote tiered versions are either safely skipped without source cleanup or explicitly moved with verified target metadata.
+- Existing non-delete-marker rebalance behavior remains unchanged.
+
+### Risks
+
+- The existing `MigrationBackend` trait is source-set oriented. If adapting it becomes invasive, prefer an ECStore-specific path in `rebalance_entry` rather than widening the trait for one branch.
+- Skipping remote tiered versions may leave more source entries behind. That is safer than deleting unverified metadata and can be improved in a later PR.
+
+---
+
+## F02: Decommission `DataMovementOverwriteErr` Cleanup Safety
+
+### Decision
+
+`DataMovementOverwriteErr` must not be cleanup-safe by default. It can count as complete only after explicit target equivalence verification.
+
+### Files
+
+- Modify: `crates/ecstore/src/pools.rs`
+- Possibly modify: `crates/ecstore/src/data_movement.rs`
+- Test: decommission tests in `crates/ecstore/src/pools.rs`
+
+### Design
+
+1. Update delete marker and remote tiered decommission error handling.
+   - Keep object-not-found and version-not-found handling as cleanup-safe only when existing semantics justify it.
+   - Remove `is_err_data_movement_overwrite(&err)` from branches that set `cleanup_ignored = true`.
+
+2. Add a narrow equivalence helper only if an existing target version can be inspected cheaply.
+   - Compare bucket, object, version ID, delete marker state, ETag where applicable, mod time, and key metadata.
+   - If comparison cannot be done reliably in the current code path, treat overwrite as failure for Phase 1.
+
+3. Preserve source entry on unsafe overwrite.
+   - Set `failure = true`.
+   - Include a clear error stage in logs/status.
+   - Do not increment the decommissioned count.
+
+### Implementation Tasks
+
+- [ ] Write a failing unit test for delete marker decommission where `DataMovementOverwriteErr` must not count complete.
+- [ ] Write a failing unit test for remote tiered decommission where `DataMovementOverwriteErr` must not count complete.
+- [ ] Remove `is_err_data_movement_overwrite` from cleanup-safe conditions in those branches.
+- [ ] Add an explicit helper or comment documenting why overwrite is unsafe without equivalence.
+- [ ] Run focused decommission tests.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs-ecstore decommission --lib
+cargo test -p rustfs-ecstore DataMovementOverwriteErr --lib
+```
+
+### Acceptance Criteria
+
+- `DataMovementOverwriteErr` does not set `cleanup_ignored = true` by default.
+- Source cleanup does not run when overwrite equivalence is unknown.
+- Logs make unsafe overwrite distinguishable from not-found cleanup-safe cases.
+
+### Risks
+
+- Tightening this behavior may cause more decommission retries/failures in clusters that previously masked unsafe state. That is intended; status must make the failure actionable.
+
+---
+
+## F03: Decommission Pool Meta Reload Barrier
+
+### Decision
+
+A successful decommission start must mean all required peers have acknowledged the updated pool meta or the API must return an explicit failure/degraded result.
+
+### Files
+
+- Modify: `crates/ecstore/src/pools.rs`
+- Modify: `crates/ecstore/src/notification_sys.rs`
+- Modify: `rustfs/src/admin/handlers/pools.rs`
+- Possibly modify: `crates/ecstore/src/rpc/peer_rest_client.rs`
+- Possibly modify: `rustfs/src/storage/rpc/node_service.rs`
+
+### Design
+
+1. Change `start_decommission` propagation behavior.
+   - After `pool_meta.save`, call `reload_pool_meta`.
+   - If peer reload returns an aggregate error, return that error to the admin handler.
+   - Do not silently proceed.
+
+2. Decide rollback behavior.
+   - Preferred minimal Phase 1: fail the API and leave persisted `pool_meta` in decommission state only if rollback is unsafe or unavailable; status must show reload failure.
+   - Stronger option: persist a reverted pool meta before returning failure. This needs careful lock and persistence review and should not be attempted without tests.
+
+3. Ensure decommission workers do not spawn after reload failure.
+   - If the caller starts workers after `start_decommission` returns, returning `Err` is sufficient.
+   - If any background path can resume from persisted metadata after a failed reload, status must clearly show pending/degraded state.
+
+### Implementation Tasks
+
+- [ ] Write a failing test for `start_decommission` with peer reload failure.
+- [ ] Change reload failure from `warn` to returned error.
+- [ ] Update admin handler error mapping so clients do not receive success.
+- [ ] Add status/log context for peer reload failure.
+- [ ] Run focused pool/decommission tests.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs-ecstore start_decommission --lib
+cargo test -p rustfs-ecstore reload_pool_meta --lib
+cargo test -p rustfs decommission --lib
+```
+
+### Acceptance Criteria
+
+- Admin start decommission does not return success when peer reload fails.
+- Decommission workers do not start after reload failure through the admin path.
+- Failure output includes enough peer context for operators.
+
+### Risks
+
+- Persisted pool meta may already be updated before reload failure is detected. Avoid adding rollback until the exact persistence safety is reviewed. Fail-closed reporting is the minimum safe fix.
+
+---
+
+## F04: Store Init Recovery Ordering
+
+### Decision
+
+Store init must load and install pool meta before deciding whether rebalance can auto-start.
+
+### Files
+
+- Modify: `crates/ecstore/src/store/init.rs`
+- Possibly modify: `crates/ecstore/src/rebalance.rs`
+- Possibly modify: `crates/ecstore/src/pools.rs`
+
+### Design
+
+1. Reorder init sequence.
+   - Initialize boot time.
+   - Load pool meta.
+   - Validate and install pool meta.
+   - Resolve resumable decommission pools.
+   - Load rebalance meta.
+   - Start rebalance only if no decommission is active or resumable.
+
+2. Define conflict behavior.
+   - If active decommission and started rebalance metadata coexist, decommission wins for Phase 1.
+   - Rebalance should not start.
+   - Record or expose a clear deferred/conflict reason if existing status structures can carry it without broad schema changes.
+
+3. Preserve existing happy path.
+   - Clusters with only rebalance metadata should still auto-start rebalance.
+   - Clusters with only decommission metadata should still resume decommission after the existing delay.
+
+### Implementation Tasks
+
+- [ ] Write a failing init test for active decommission plus rebalance metadata.
+- [ ] Move pool meta load/validate/install before `load_rebalance_meta` and `start_rebalance`.
+- [ ] Add a helper such as `should_auto_start_rebalance_after_init`.
+- [ ] Ensure decommission resume still uses the installed pool meta.
+- [ ] Run focused init tests.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs-ecstore init --lib
+cargo test -p rustfs-ecstore rebalance_meta --lib
+cargo test -p rustfs-ecstore decommission --lib
+```
+
+### Acceptance Criteria
+
+- Rebalance does not start when active decommission is loaded from pool meta.
+- Rebalance still starts when no decommission is active.
+- Decommission resume behavior is unchanged except that it no longer races with an already-started rebalance.
+
+### Risks
+
+- Init ordering can affect existing startup failure behavior. Keep error handling and stage names explicit so operators can identify which stage failed.
+
+---
+
+## F05: Rebalance Distributed Start/Stop Semantics
+
+### Decision
+
+Admin start/stop must not report plain success for partial cluster state. If stop remains asynchronous, status must expose that distinction.
+
+### Files
+
+- Modify: `rustfs/src/admin/handlers/rebalance.rs`
+- Modify: `rustfs/src/storage/rpc/node_service.rs`
+- Modify: `crates/ecstore/src/notification_sys.rs`
+- Modify: `crates/ecstore/src/rpc/peer_rest_client.rs`
+- Possibly modify: `crates/ecstore/src/rebalance.rs`
+
+### Design
+
+1. Fix peer RPC stop first.
+   - `node_service::stop_rebalance` must return `success=false` or gRPC error when `store.stop_rebalance().await` fails.
+   - `notification_sys.stop_rebalance` must propagate aggregate peer failure instead of warning and returning `Ok`.
+
+2. Fix start propagation.
+   - Avoid returning success from peer `load_rebalance_meta(start=true)` before local start validation has completed.
+   - If fully synchronous worker startup is too invasive, split semantics:
+     - metadata loaded;
+     - start accepted;
+     - worker running observable through status.
+   - Admin start should fail when any required peer fails load/start acceptance.
+
+3. Define rollback/degraded semantics.
+   - Preferred Phase 1: return failure on propagation failure and attempt local stop if local rebalance was already started.
+   - If rollback fails, return a degraded error and expose status.
+
+4. Add status language for stop.
+   - If cancellation is requested but workers may still be running, status should not claim fully stopped.
+   - Minimal approach: add a `stopping` or equivalent field if existing response types allow it.
+
+### Implementation Tasks
+
+- [ ] Write a failing test for peer stop returning success despite local stop error.
+- [ ] Update `node_service::stop_rebalance` to return failure details.
+- [ ] Write a failing test for `notification_sys.stop_rebalance` aggregate peer failure.
+- [ ] Update `notification_sys.stop_rebalance` to return aggregate error.
+- [ ] Write a failing admin start propagation test.
+- [ ] Update admin start to fail or rollback on propagation failure.
+- [ ] Add status distinction for requested stop versus completed stop if response schema permits.
+- [ ] Run focused admin/RPC/rebalance tests.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs-ecstore stop_rebalance --lib
+cargo test -p rustfs-ecstore load_rebalance_meta --lib
+cargo test -p rustfs rebalance --lib
+```
+
+### Acceptance Criteria
+
+- Peer stop failure is visible to callers.
+- Admin stop fails or reports degraded state on peer failure.
+- Admin start fails or reports degraded state on peer load/start failure.
+- Stop status does not mislead operators while workers are still winding down.
+
+### Risks
+
+- API response schema changes may affect clients. Prefer backward-compatible additions where possible, and keep error responses compatible with existing admin error handling.
+
+---
+
+## Phase 1 Test Matrix
+
+Run these once the individual fix tests pass:
+
+```bash
+cargo test -p rustfs-ecstore rebalance --lib
+cargo test -p rustfs-ecstore decommission --lib
+cargo test -p rustfs-ecstore pools --lib
+cargo test -p rustfs rebalance --lib
+cargo test -p rustfs pools --lib
+cargo fmt --all --check
+```
+
+Before opening a PR with code changes:
+
+```bash
+cargo fmt --all
+cargo fmt --all --check
+make pre-commit
+```
+
+After build-based verification, clean generated build artifacts to avoid unnecessary disk usage.
+
+## Suggested PR Split
+
+1. PR 1: F02 only. Smallest fail-closed fix.
+2. PR 2: F04 only. Startup ordering and conflict tests.
+3. PR 3: F03 only. Decommission peer reload barrier.
+4. PR 4: F05 stop semantics first, then start semantics if review scope remains manageable.
+5. PR 5: F01 delete marker fix and remote tiered skip policy.
+
+F01 is highest risk, but it may need the most design review. F02 and F04 are good first implementation candidates because they reduce safety risk with narrower code changes.
+
+## Open Questions Before Coding
+
+1. For F01 remote tiered versions, should Phase 1 match MinIO and skip them during rebalance, or should RustFS implement cross-pool remote metadata movement now?
+2. For F03 reload failure, should the implementation attempt to rollback persisted pool meta, or fail closed and expose degraded metadata state?
+3. For F05 start propagation, is a backward-compatible degraded error response acceptable, or must the API stay strictly compatible with current success/error shapes?

docs/architecture/rebalance-decommission-phase2-data-movement-plan.md

@@ -0,0 +1,363 @@
+# Rebalance and Decommission Phase 2 Data Movement Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Make shared data movement safe for large objects, metadata-preserving, and resilient to overwrite and cleanup races.
+
+**Architecture:** Phase 2 keeps `crates/ecstore/src/data_movement.rs` as the shared migration core, but tightens its streaming, metadata, and equivalence behavior. Rebalance and decommission should share small helpers where their safety checks are identical, while keeping operation-specific policy decisions in `rebalance.rs` and `pools.rs`.
+
+**Tech Stack:** Rust, Tokio async readers, ECStore object APIs, multipart APIs, lifecycle evaluator, filemeta metadata types, existing crate-local unit tests.
+
+---
+
+## Scope
+
+This plan covers:
+
+- F06: stream multipart data movement instead of buffering whole parts.
+- F07: preserve and verify checksum, replication, version purge, and object-lock metadata.
+- F08: align or explicitly define decommission lifecycle-expired cleanup behavior.
+- F09: handle rebalance overwrite races with explicit target equivalence checks.
+- F10: add final source cleanup verification before prefix deletion.
+
+This plan assumes Phase 1 either has landed or its semantics are stable enough to build on.
+
+## Shared Principles
+
+- Streaming must keep memory bounded by buffer size, not part size.
+- Metadata equivalence should be explicit and testable.
+- A version is complete only when copied, intentionally skipped by policy, or proven equivalent on target.
+- Source cleanup must be the last step and must verify the source version set has not changed.
+- Fail closed when equivalence cannot be proven.
+
+## F06: Stream Multipart Data Movement
+
+### Decision
+
+Multipart migration must not allocate a buffer equal to `part.size`. It should stream each part into `put_object_part`.
+
+### Files
+
+- Modify: `crates/ecstore/src/data_movement.rs`
+- Possibly modify: `crates/ecstore/src/store/multipart.rs`
+- Possibly modify: `crates/ecstore/src/store_api/readers.rs`
+- Test: data movement tests in `crates/ecstore/src/data_movement.rs`
+
+### Design
+
+1. Replace per-part `Vec<u8>` allocation.
+   - Current behavior reads each part with `read_exact` into `chunk`.
+   - New behavior should create a bounded reader over the shared object stream for exactly `part.size` bytes.
+
+2. Preserve part boundaries.
+   - Each `put_object_part` call must receive a reader that ends exactly at the part boundary.
+   - If a part stream ends early, return a staged `read_part` error and abort the multipart upload.
+
+3. Preserve part index behavior.
+   - Continue decoding `part.index`.
+   - Preserve indexed reader behavior so erasure/indexed metadata remains compatible.
+
+4. Preserve abort behavior.
+   - `abort_multipart_upload` must still run if any part upload or complete call fails before completion is marked.
+
+### Implementation Tasks
+
+- [ ] Add a failing data movement test with a fake large multipart stream that panics or fails if code tries to allocate/read the full part into memory.
+- [ ] Add a test proving part boundary reads consume exactly the configured part sizes.
+- [ ] Introduce a bounded async reader helper for a single multipart part.
+- [ ] Replace `vec![0u8; part.size]` and `read_exact` in multipart migration with the bounded reader helper.
+- [ ] Ensure part upload still wraps data in `PutObjReader` with correct size, actual size, and index.
+- [ ] Keep existing abort-on-error behavior and add a focused test for failure during streamed part upload.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs-ecstore data_movement --lib
+cargo test -p rustfs-ecstore multipart --lib
+```
+
+### Acceptance Criteria
+
+- Data movement memory is bounded and not proportional to multipart part size.
+- Multipart part boundaries, sizes, ETags, and indexes remain correct.
+- Failed streamed part upload aborts the temporary multipart upload.
+
+### Risks
+
+- Async reader composition can accidentally over-read into the next part. The part-boundary test is mandatory.
+- If `put_object_part` requires a concrete reader type, introduce the smallest adapter needed rather than rewriting multipart internals.
+
+---
+
+## F07: Preserve and Verify Full Data Movement Metadata
+
+### Decision
+
+Checksum preservation is a confirmed fix. Replication, version purge, and object-lock behavior must be tested first; production changes should follow only where tests reveal loss or mismatch.
+
+### Files
+
+- Modify: `crates/ecstore/src/data_movement.rs`
+- Possibly modify: `crates/ecstore/src/store_api/types.rs`
+- Possibly modify: `crates/ecstore/src/set_disk.rs`
+- Possibly modify: `crates/filemeta/src/fileinfo.rs`
+- Possibly modify: `crates/filemeta/src/replication.rs`
+- Test: data movement tests in `crates/ecstore/src/data_movement.rs`
+
+### Design
+
+1. Define metadata equivalence for data movement.
+   - Required fields: version ID, ETag, size, actual size, mod time, user metadata, storage class, checksum, multipart checksum, replication state, version purge state, object-lock mode/date, legal hold.
+   - Use a helper local to data movement tests first. Promote to production only if F09/F10 need it.
+
+2. Preserve multipart checksum.
+   - When source part checksum exists, populate the corresponding `CompletePart` checksum fields.
+   - Preserve object-level multipart checksum metadata when complete metadata is written.
+
+3. Validate replication state preservation.
+   - Current code copies `user_defined` and `set_disk` reconstructs `replication_state_internal` from metadata.
+   - Add tests to prove this works for replication status and version purge status.
+   - If tests fail, extend `ObjectOptions` or write path metadata handling narrowly.
+
+4. Validate object lock preservation.
+   - Current code copies `user_defined`, which should carry object-lock headers.
+   - Add tests for retention mode/date and legal hold.
+   - If tests fail, fix the metadata copy path without changing normal user copy semantics.
+
+### Implementation Tasks
+
+- [ ] Add a metadata equivalence assertion helper in data movement tests.
+- [ ] Add a failing multipart checksum preservation test.
+- [ ] Populate `CompletePart` checksum fields from source part metadata.
+- [ ] Add single-part checksum preservation test and fix object-level checksum metadata if needed.
+- [ ] Add replication state and version purge state preservation tests.
+- [ ] Add object-lock retention and legal hold preservation tests.
+- [ ] Apply the minimal production fixes required by those tests.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs-ecstore data_movement --lib
+cargo test -p rustfs-ecstore checksum --lib
+cargo test -p rustfs-ecstore replication --lib
+cargo test -p rustfs-ecstore object_lock --lib
+```
+
+### Acceptance Criteria
+
+- Migrated single-part and multipart objects preserve checksum behavior.
+- Migrated multipart parts preserve per-part checksum metadata where RustFS stores it.
+- Replication state and version purge state are equivalent after migration.
+- Object-lock retention and legal hold remain unchanged after migration.
+
+### Risks
+
+- Some metadata may be derived rather than stored directly. Tests should compare externally observable object info, not only internal fields.
+- Avoid broad `ObjectOptions` expansion unless current metadata copy cannot preserve a required field.
+
+---
+
+## F08: Lifecycle-Expired Version Cleanup Semantics
+
+### Decision
+
+RustFS should either align with MinIO by counting lifecycle-expired versions as decommission-complete, or explicitly document and expose retained expired source entries. The recommended behavior is to align with MinIO when lifecycle/object-lock/replication checks say the version is safe to expire.
+
+### Files
+
+- Modify: `crates/ecstore/src/pools.rs`
+- Possibly modify: lifecycle evaluator under `crates/ecstore/src/bucket/lifecycle/`
+- Test: decommission tests in `crates/ecstore/src/pools.rs`
+
+### Design
+
+1. Preserve existing lifecycle safety checks.
+   - Do not bypass object-lock or replication constraints.
+   - Keep `should_skip_lifecycle_for_data_movement` or equivalent evaluator as the source of truth.
+
+2. Count safe expired versions as complete for source cleanup.
+   - Replace `expired == 0 && decommissioned == total_versions` with semantics that allow `decommissioned + expired == total_versions` when expired versions are safe.
+
+3. Keep status honest.
+   - If a version is retained because expiration is not safe, status should show it as remaining, not completed.
+
+### Implementation Tasks
+
+- [ ] Write a failing test where one migrated version plus one lifecycle-expired version should allow source cleanup.
+- [ ] Write a test where object-lock protected expired-looking version must not allow cleanup.
+- [ ] Write a test where replication-pending version must not allow cleanup.
+- [ ] Update source cleanup predicate to count safe expired versions.
+- [ ] Update counters/status if existing fields distinguish expired from decommissioned.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs-ecstore decommission --lib
+cargo test -p rustfs-ecstore lifecycle --lib
+cargo test -p rustfs-ecstore object_lock --lib
+```
+
+### Acceptance Criteria
+
+- Source entry is cleaned when every version is migrated or safely lifecycle-expired.
+- Protected versions still block cleanup.
+- Decommission status remains understandable for migrated, expired, and blocked versions.
+
+### Risks
+
+- Counting expired versions as complete can hide lifecycle evaluator bugs. Tests must include protected negative cases.
+
+---
+
+## F09: Rebalance Overwrite Race Equivalence
+
+### Decision
+
+Unsafe `DataMovementOverwriteErr` remains a failure. Equivalent target overwrite can be counted complete only after explicit comparison.
+
+### Files
+
+- Modify: `crates/ecstore/src/data_movement.rs`
+- Modify: `crates/ecstore/src/rebalance.rs`
+- Possibly modify: `crates/ecstore/src/store/object.rs`
+- Test: data movement and rebalance tests
+
+### Design
+
+1. Define target equivalence helper.
+   - Compare version ID, ETag, size, actual size, mod time, checksum, delete marker state, and selected user metadata.
+   - Reuse the test helper from F07 if appropriate, but production helper must be limited to fields needed for safety.
+
+2. Apply helper only to overwrite race cases.
+   - If `DataMovementOverwriteErr` occurs and target pool differs from source pool, inspect target object/version.
+   - If equivalent, count migration as complete.
+   - If not equivalent or cannot inspect target, return failure.
+
+3. Keep source-equals-target unsafe.
+   - Do not convert source-equals-target into success without independent target proof.
+
+### Implementation Tasks
+
+- [ ] Add a failing rebalance test where equivalent target version exists and overwrite should converge.
+- [ ] Add a failing rebalance test where target version differs and overwrite must fail.
+- [ ] Implement a minimal target equivalence helper.
+- [ ] Wire the helper into data movement overwrite handling for rebalance.
+- [ ] Keep decommission behavior aligned with F02.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs-ecstore DataMovementOverwriteErr --lib
+cargo test -p rustfs-ecstore rebalance --lib
+cargo test -p rustfs-ecstore data_movement --lib
+```
+
+### Acceptance Criteria
+
+- Equivalent overwrite does not block rebalance convergence.
+- Non-equivalent overwrite does not clean source.
+- Logs/status distinguish equivalent overwrite from unsafe overwrite.
+
+### Risks
+
+- Comparing too few fields can mark a corrupt target complete. Prefer strict comparison in Phase 2.
+- Comparing too many volatile fields can prevent convergence. Use tests to calibrate.
+
+---
+
+## F10: Source Cleanup Preflight Verification
+
+### Decision
+
+Before deleting a source prefix, rebalance and decommission should verify the source version set still matches the migrated or intentionally skipped set.
+
+### Files
+
+- Modify: `crates/ecstore/src/rebalance.rs`
+- Modify: `crates/ecstore/src/pools.rs`
+- Possibly modify: `crates/ecstore/src/set_disk.rs`
+- Test: rebalance and decommission cleanup tests
+
+### Design
+
+1. Capture source version identity at scan time.
+   - Version identity should include object name, version ID, delete marker flag, and enough metadata to detect a new or changed version.
+
+2. Re-read before cleanup.
+   - Immediately before `delete_prefix`, read current source metadata.
+   - Compare current identity set to the expected cleanup set.
+
+3. Defer on mismatch.
+   - Rebalance should defer/retry the entry with a clear last error.
+   - Decommission should fail the entry or retry according to existing decommission retry policy.
+
+4. Keep not-found idempotent.
+   - If source entry is already gone, cleanup remains successful.
+
+### Implementation Tasks
+
+- [ ] Add helper to build a stable version identity set from `FileInfoVersions`.
+- [ ] Add rebalance test where source metadata changes between migration and cleanup.
+- [ ] Add decommission test where source metadata changes between migration and cleanup.
+- [ ] Add cleanup preflight before rebalance source `delete_prefix`.
+- [ ] Add cleanup preflight before decommission source `delete_prefix`.
+- [ ] Return deferred/failure status with a clear reason on mismatch.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs-ecstore rebalance_entry --lib
+cargo test -p rustfs-ecstore decommission_entry --lib
+cargo test -p rustfs-ecstore cleanup --lib
+```
+
+### Acceptance Criteria
+
+- Cleanup does not delete a source entry if the version set changed after migration started.
+- Already-deleted source remains idempotent.
+- Mismatch errors are visible in status or last-error fields.
+
+### Risks
+
+- Extra metadata reads can add I/O to hot migration paths. Keep this as a cleanup-only check.
+
+---
+
+## Phase 2 Test Matrix
+
+Run after individual fix tests pass:
+
+```bash
+cargo test -p rustfs-ecstore data_movement --lib
+cargo test -p rustfs-ecstore multipart --lib
+cargo test -p rustfs-ecstore rebalance --lib
+cargo test -p rustfs-ecstore decommission --lib
+cargo test -p rustfs-ecstore lifecycle --lib
+cargo fmt --all --check
+```
+
+Before PR:
+
+```bash
+cargo fmt --all
+cargo fmt --all --check
+make pre-commit
+```
+
+Clean generated build artifacts after build-based verification.
+
+## Suggested PR Split
+
+1. PR 1: F06 streaming multipart migration.
+2. PR 2: F07 checksum and metadata preservation tests/fixes.
+3. PR 3: F08 lifecycle-expired cleanup semantics.
+4. PR 4: F09 overwrite equivalence.
+5. PR 5: F10 cleanup preflight verification.
+
+F06 and F07 may combine only if checksum preservation requires the new streaming reader shape. F09 should wait for F07 if it depends on metadata equivalence helpers.
+
+## Open Questions Before Coding
+
+1. Which metadata fields are considered mandatory for equivalence in Phase 2: strict internal fields or externally observable fields only?
+2. Should F08 fully match MinIO cleanup behavior, or should RustFS keep expired source entries but expose them as residual state?
+3. Should F10 apply to both rebalance and decommission in the same PR, or should rebalance land first as the higher-risk cleanup path?

docs/architecture/rebalance-decommission-phase3-hardening-plan.md

@@ -0,0 +1,320 @@
+# Rebalance and Decommission Phase 3 Hardening Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Improve operator visibility, auditability, metadata robustness, and documented compatibility after the Phase 1 and Phase 2 safety fixes.
+
+**Architecture:** Phase 3 should not change core data movement semantics unless a hardening test exposes a safety bug. It adds bounded status data, structured logs, metadata validation tests, and an explicit decision around rebalance completion tolerance.
+
+**Tech Stack:** Rust, tracing structured logs, admin status DTOs, serde/rmp metadata decoding, existing guardrail scripts, crate-local unit tests.
+
+---
+
+## Scope
+
+This plan covers:
+
+- F11: improve rebalance cleanup failure reporting.
+- F12: add structured audit fields for rebalance/decommission operations.
+- F13: harden persisted rebalance and pool metadata decoding.
+- F14: decide and document rebalance completion threshold behavior.
+
+This plan assumes the safety semantics from Phase 1 and the data movement behavior from Phase 2 are stable.
+
+## Shared Principles
+
+- Hardening must not hide safety failures.
+- Status fields should be bounded to avoid unbounded metadata growth.
+- Logs must be structured, searchable, and free of secrets.
+- Metadata compatibility must be explicit: strict where possible, legacy-compatible where necessary.
+
+## F11: Rebalance Cleanup Failure Reporting
+
+### Decision
+
+Rebalance cleanup failures should not be invisible completion details. Admin status should expose whether completion had cleanup warnings and provide bounded object-level detail.
+
+### Files
+
+- Modify: `crates/ecstore/src/rebalance.rs`
+- Modify: `rustfs/src/admin/handlers/rebalance.rs`
+- Possibly modify: admin response DTOs in the same module or related madmin types
+- Test: rebalance status and metadata tests
+
+### Design
+
+1. Extend cleanup warning metadata.
+   - Keep existing count.
+   - Add a bounded list of recent cleanup failures, for example last 10 entries.
+   - Each entry should include bucket, object, message, and timestamp.
+
+2. Keep metadata bounded.
+   - Do not store every cleanup failure unboundedly in `rebalance.bin`.
+   - Use a ring-buffer style helper or truncate older entries.
+
+3. Make terminal state visible.
+   - If rebalance completes with cleanup warnings, status should show warning count and details.
+   - Avoid reporting a clean completion when source cleanup failed.
+
+### Implementation Tasks
+
+- [ ] Add a cleanup warning entry struct with bounded retention.
+- [ ] Add metadata compatibility defaults for older `rebalance.bin` without the new list.
+- [ ] Update `record_rebalance_cleanup_warning_in_meta` to append bounded entries and update count/last fields.
+- [ ] Update admin status serialization to expose warning count and entries.
+- [ ] Add tests for one warning, multiple warnings, and bounded truncation.
+- [ ] Add legacy metadata decode test.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs-ecstore cleanup_warning --lib
+cargo test -p rustfs-ecstore rebalance_meta --lib
+cargo test -p rustfs rebalance --lib
+```
+
+### Acceptance Criteria
+
+- Multiple cleanup failures are visible through count and bounded details.
+- Legacy rebalance metadata still decodes.
+- Status distinguishes clean completion from completion with cleanup warnings.
+
+### Risks
+
+- Adding status fields may affect clients if response schemas are strict. Prefer additive fields with defaults.
+
+---
+
+## F12: Structured Audit Fields
+
+### Decision
+
+Rebalance and decommission admin operations should have consistent structured logs for success, rejection, propagation failure, and partial/degraded state.
+
+### Files
+
+- Modify: `rustfs/src/admin/handlers/rebalance.rs`
+- Modify: `rustfs/src/admin/handlers/pools.rs`
+- Modify: `crates/ecstore/src/notification_sys.rs`
+- Modify: `crates/ecstore/src/rpc/peer_rest_client.rs`
+- Possibly modify: `scripts/check_logging_guardrails.sh` or related guardrails if logging rules require updates
+
+### Design
+
+1. Define standard fields.
+   - `event`
+   - `component`
+   - `subsystem`
+   - `action`
+   - `result`
+   - `request_id` when available
+   - `actor` or masked access key when available
+   - `remote_addr` when available
+   - `pool_indices` or `rebalance_id`
+   - `peer`
+   - `error`
+
+2. Apply to admin handlers.
+   - Start, stop, cancel, status-affecting operations.
+   - Authorization failures should remain handled by existing auth paths but should have enough context if already logged.
+
+3. Apply to peer propagation.
+   - Peer success and failure should use stable event names.
+   - Partial failure must be warn/error, not info-only.
+
+4. Avoid secret leakage.
+   - Never log raw credentials, signatures, tokens, or full auth headers.
+
+### Implementation Tasks
+
+- [ ] Inventory existing rebalance/decommission log events and field names.
+- [ ] Define a small local helper or convention for masked actor/request fields if one already exists.
+- [ ] Update rebalance admin start/stop/status logs.
+- [ ] Update decommission start/cancel/status logs.
+- [ ] Update notification peer propagation failure logs.
+- [ ] Add log-capture tests for success, partial failure, and rejected operation where feasible.
+- [ ] Run logging guardrail script if the repository has one.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs rebalance --lib
+cargo test -p rustfs pools --lib
+cargo test -p rustfs-ecstore notification --lib
+scripts/check_logging_guardrails.sh
+```
+
+### Acceptance Criteria
+
+- Operators can answer who requested an operation, what resource it targeted, and which peers failed.
+- Partial failures are searchable through stable structured fields.
+- Logs contain no secrets.
+
+### Risks
+
+- Logging changes can be noisy. Keep high-volume per-object logs out of admin audit events.
+
+---
+
+## F13: Persisted Metadata Decode Hardening
+
+### Decision
+
+Persisted rebalance and pool metadata should reject or surface unknown/corrupt fields where compatibility allows. Legacy compatibility must be tested explicitly.
+
+### Files
+
+- Modify: `crates/ecstore/src/rebalance.rs`
+- Modify: `crates/ecstore/src/pools.rs`
+- Possibly add fixtures under an existing test fixture directory if one exists
+- Test: metadata decode tests in `rebalance.rs` and `pools.rs`
+
+### Design
+
+1. Identify persisted structs.
+   - Rebalance metadata and stats structs.
+   - Pool metadata and decommission info structs.
+
+2. Choose strictness per struct.
+   - Use strict unknown-field rejection where metadata is not expected to carry forward-compatible fields.
+   - For legacy-compatible structs, keep decode lenient but log or validate unknown/unsupported state where possible.
+
+3. Add state validation.
+   - Reject conflicting terminal states such as complete plus canceled plus running.
+   - Reject impossible counters or invalid pool indices where existing validation can catch them.
+
+4. Preserve legacy fixtures.
+   - Existing old metadata must still decode if compatibility is required.
+
+### Implementation Tasks
+
+- [ ] List all rebalance and pool metadata structs currently deserialized from persisted bytes.
+- [ ] Add fixture tests for unknown fields, missing critical fields, and conflicting terminal states.
+- [ ] Add strict serde attributes only where fixtures show compatibility is safe.
+- [ ] Add post-decode validation helpers for state conflicts.
+- [ ] Update load paths to surface validation errors with actionable messages.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs-ecstore rebalance_meta --lib
+cargo test -p rustfs-ecstore pool_meta --lib
+cargo test -p rustfs-ecstore metadata --lib
+```
+
+### Acceptance Criteria
+
+- Unknown/corrupt metadata does not silently become an apparently valid operation state.
+- Legacy metadata fixtures continue to decode through documented paths.
+- Conflicting terminal state is rejected or quarantined.
+
+### Risks
+
+- Strict decode can block startup if old metadata contains benign extra fields. Start with tests and post-decode validation before broad strict attributes.
+
+---
+
+## F14: Rebalance Completion Threshold Decision
+
+### Decision
+
+RustFS must explicitly choose whether to match MinIO's tolerance-based rebalance completion or keep strict completion. The recommended default is to match MinIO unless RustFS has a documented reason to move more data.
+
+### Files
+
+- Modify: `crates/ecstore/src/rebalance.rs`
+- Possibly modify: admin docs if present
+- Test: rebalance goal/completion tests in `rebalance.rs`
+
+### Design Options
+
+1. **Match MinIO tolerance.**
+   - Complete when pool free-space ratio is within the MinIO-like tolerance of the goal.
+   - Reduces movement and operational time.
+   - Best for compatibility.
+
+2. **Keep strict behavior and document it.**
+   - No code behavior change.
+   - Add tests proving strict behavior is intentional.
+   - Status/docs should avoid implying MinIO-compatible tolerance.
+
+3. **Configurable tolerance.**
+   - Most flexible but not recommended unless operators need it.
+   - Adds configuration and support burden.
+
+### Recommended Plan
+
+Start with option 1 if compatibility is the goal. If maintainers prefer strictness, implement option 2 with explicit tests and documentation.
+
+### Implementation Tasks
+
+- [ ] Add tests that capture current strict behavior around the completion goal.
+- [ ] Add tests for MinIO-like tolerance behavior.
+- [ ] Confirm maintainers choose MinIO-compatible or strict behavior.
+- [ ] Implement the selected threshold behavior.
+- [ ] Update status/docs comments so completion semantics are clear.
+
+### Focused Test Commands
+
+```bash
+cargo test -p rustfs-ecstore rebalance_goal --lib
+cargo test -p rustfs-ecstore check_if_rebalance_done --lib
+cargo test -p rustfs-ecstore rebalance --lib
+```
+
+### Acceptance Criteria
+
+- Completion threshold behavior is covered by deterministic tests.
+- Operators can understand whether RustFS matches MinIO tolerance.
+- Empty-queue completion path remains valid.
+
+### Risks
+
+- Changing threshold may alter operational expectations for existing users. Document the change if behavior changes.
+
+---
+
+## Phase 3 Test Matrix
+
+Run after individual fix tests pass:
+
+```bash
+cargo test -p rustfs-ecstore rebalance_meta --lib
+cargo test -p rustfs-ecstore pool_meta --lib
+cargo test -p rustfs-ecstore rebalance --lib
+cargo test -p rustfs rebalance --lib
+cargo test -p rustfs pools --lib
+cargo fmt --all --check
+```
+
+If logging guardrails are relevant:
+
+```bash
+scripts/check_logging_guardrails.sh
+```
+
+Before PR:
+
+```bash
+cargo fmt --all
+cargo fmt --all --check
+make pre-commit
+```
+
+Clean generated build artifacts after build-based verification.
+
+## Suggested PR Split
+
+1. PR 1: F11 cleanup warning status.
+2. PR 2: F12 structured audit fields.
+3. PR 3: F13 metadata decode hardening.
+4. PR 4: F14 completion threshold decision and tests.
+
+F11 may depend on Phase 2 cleanup semantics if F10 changes retry behavior. F12 can be done anytime after Phase 1 defines distributed failure semantics.
+
+## Open Questions Before Coding
+
+1. What is the maximum number of cleanup warning entries to keep in persisted rebalance metadata?
+2. Which actor identity is safe and useful to log for admin requests?
+3. Should metadata unknown fields fail startup, warn and continue, or quarantine only the affected operation?
+4. Should RustFS match MinIO's rebalance completion tolerance by default?

docs/architecture/rebalance-decommission-post-remediation-review-plan.md

@@ -0,0 +1,132 @@
+# Rebalance and Decommission Post-Remediation Review
+
+> **Status:** Updated for branch `cxymds/rebalance-decommission-remediation`.
+>
+> **Scope:** Rebalance, decommission, shared data-movement helpers, source cleanup,
+> delete-marker handling, lifecycle expiry, restart/cancel recovery, and CI coverage.
+>
+> **Conclusion:** The main code-level P1/P2 remediation set has been implemented on
+> this branch. The remaining release gate is verification, not another known
+> correctness rewrite: full CI must pass, `make pre-commit` must pass before the
+> final PR state, and the data-movement e2e proof added to CI must complete in the
+> GitHub runner environment.
+
+## Review Corrections
+
+The previous report needed two important corrections.
+
+First, the R28 decommission permit issue is real but was overstated. The worker
+permit was released before entry migration work, so `Workers` did not account for
+entry lifetime and `wk.wait()` could not prove entry work had drained. However,
+the current listing path awaits entry callbacks inline and bucket processing is
+bounded, so the old "unbounded object movement" wording was not supported by the
+code. The fix remains correct because permit lifetime now matches entry lifetime
+and prevents future spawn-based regressions.
+
+Second, SSE-C migration is related to raw data movement but has a distinct
+failure mode. Before the raw migration path, SSE-C objects did not silently
+corrupt through an empty `HeaderMap`; migration reads failed while resolving
+SSE-C material because the customer SSE-C headers were missing. The raw internal
+read path avoids requiring request customer headers for data movement.
+
+## Implemented Remediation
+
+| Area | Current Branch State | Representative Guards |
+| --- | --- | --- |
+| Raw migration reads | Rebalance and decommission use raw data-movement read options instead of normal transformed GET semantics. | `test_rebalance_object_migration_read_opts_are_raw_data_movement`, `test_decommission_object_migration_read_opts_are_raw_data_movement` |
+| Raw write invariants | Data-movement writes preserve source version, ETag, tags, expires, part metadata, and checksums consistently. Single-part raw ETag validation no longer rejects preserved source ETags. | `test_data_movement_put_object_opts_preserves_version_and_etag`, `test_data_movement_opts_preserve_tags_and_expires`, `test_data_movement_single_part_raw_reader_does_not_validate_source_etag` |
+| Terminal decommission recovery | Cancel/failed/complete paths cancel in-memory workers before terminal persistence, and terminal-save failure no longer leaves ghost cancelers. | decommission canceler unit tests in `crates/ecstore/src/pools.rs` |
+| Rebalance operation identity | Rebalance merge/save/stop paths are id-aware and do not let stale operation state clobber a newer operation. | rebalance id-gate and merge tests |
+| Start coordination | Rebalance and decommission starts cross-check persisted peer operation state before starting conflicting work. | start cross-check tests |
+| Decommission worker accounting | Entry permits are held through entry completion. R28 is no longer a pending worker-accounting bug. | decommission worker permit tests |
+| Rebalance rollback | Failed start rollback finalizes metadata instead of leaving `Started` plus `stopping=true` without a worker. | rollback tests |
+| Decommission progress save | Periodic progress-save errors are best-effort; terminal saves remain strict. | progress-save tests |
+| Queue semantics | Multi-pool decommission is queued, local-leader prefix scheduling is supported, completed restart is rejected, completed queue prefixes can advance, failed/canceled entries block automatic promotion, failed/canceled retry preserves bucket progress, and promoted queued pools are canceled if cancellation arrives before work starts. | queue/promotion/retry tests in `pools.rs` |
+| Admin status and query hardening | Pool admin status exposes queued/progress state, dangerous pool/rebalance mutation queries reject unknown or ambiguous parameters, and pool status `by-id` parsing no longer falls back to pool 0. | `admin_pool_list_item`, `pools_handler_tests`, `rebalance_handler_tests` |
+| Cancel/clear operations | Non-leader cancel intent is accepted and failed/canceled terminal decommission can be explicitly cleared. | remote cancel and clear tests |
+| Resume equivalence | Delete-marker replication state, tiered-object metadata, multipart part numbers, tags, expires, and version counts are covered more strictly. | data-movement equivalence tests |
+| Lifecycle expiry | Data movement no longer treats failed lifecycle expiry application as a successful skip. | `resolve_data_movement_lifecycle_expiry_result_rejects_apply_failure` |
+| CI coverage | Existing RustFS e2e delete-marker migration proof is wired into the `e2e-tests` CI job with `--test-threads=1`, reusing the downloaded debug binary. | `.github/workflows/ci.yml` |
+
+## Compatibility Decisions
+
+The current RustFS decommission contract is no longer single-pool only. Multi-pool
+requests are supported as queued operations on multi-pool deployments. The
+current behavior is documented in
+`docs/architecture/decommission-compatibility.md`.
+
+Delete-marker behavior is intentionally characterized rather than guessed:
+
+- a lone delete marker without replication is cleanup-only metadata and can be
+  skipped;
+- delete markers with replication metadata remain eligible for movement;
+- rebalance and decommission share the same predicate.
+
+Lifecycle-expired versions are also explicit:
+
+- decommission can count safely expired versions toward source cleanup only when
+  lifecycle expiry application succeeds;
+- rebalance remains stricter and requires actual data-movement completion for
+  cleanup.
+
+## Remaining Risks and Gates
+
+There are no remaining known P1/P2 code changes in this remediation set, but the
+branch is not final-release-ready until verification completes:
+
+1. `make pre-commit` must pass before the PR is marked ready.
+2. GitHub CI must pass, including the newly wired delete-marker migration e2e
+   proof.
+3. Local full e2e verification was attempted but the machine ran out of disk
+   space while building `rustfs`; this is an environment blocker, not a test
+   assertion failure. The `target/` directory was cleaned afterward.
+4. If product requires stronger end-to-end proof for encrypted and compressed
+   migration, add those scenarios to `crates/e2e_test` before release. Current
+   branch coverage is strongest at the internal option/invariant layer, with CI
+   e2e coverage focused on versioning/delete-marker semantics.
+
+## Focused Verification Run During Implementation
+
+Focused checks were run per task, including:
+
+```bash
+cargo test -p rustfs-ecstore data_movement_single_part_raw_reader --lib
+cargo test -p rustfs-ecstore test_merge_rebalance_meta_preserves_stopping_stop_snapshot --lib
+cargo test -p rustfs-ecstore test_merge_rebalance_pool_stats_clears_stopping_for_terminal_status --lib
+cargo test -p rustfs-ecstore first_resumable_decommission_queue_indices --lib
+cargo test -p rustfs-ecstore test_return_resumable_pools_skips_failed_decommission --lib
+cargo test -p rustfs-ecstore resolve_data_movement_lifecycle_expiry_result --lib
+cargo test -p rustfs-ecstore lifecycle_action_removes_data_movement_version --lib
+cargo test -p rustfs-ecstore test_pool_meta_promoted_queued_decommission_can_be_canceled --lib
+cargo test -p rustfs admin_pool_list_item --lib
+cargo test -p rustfs pools_handler_tests --lib
+cargo test -p rustfs rebalance_handler_tests --lib
+cargo test -p rustfs admin_query_pool_status_by_id --lib
+cargo fmt --all --check
+```
+
+The attempted local e2e command was:
+
+```bash
+cargo test -p e2e_test delete_marker_migration_semantics -- --nocapture
+```
+
+It failed because the local filesystem reached `No space left on device` while
+building the RustFS debug binary. CI now runs the proof in the e2e job after the
+debug binary artifact has been downloaded, and uses `--test-threads=1` to avoid
+parallel test-server startup for that proof.
+
+## Final PR Gate
+
+Before marking the draft PR ready:
+
+```bash
+cargo fmt --all
+cargo fmt --all --check
+make pre-commit
+```
+
+If CI reports failures after the branch is pushed, inspect the failing job logs
+instead of weakening the e2e gate. The delete-marker migration proof was added
+because skipping it would leave the exact versioning regression class invisible
+to CI.

docs/architecture/rebalance-decommission-remediation-plan.md

@@ -0,0 +1,546 @@
+# Rebalance and Decommission Remediation Plan
+
+> This document turns `docs/architecture/expert-review-analysis.md` into an actionable remediation backlog. It is intentionally split into independent fix blocks so each item can be analyzed, assigned, implemented, and verified separately.
+
+## Scope
+
+This plan covers the confirmed and calibrated issues from the rebalance/decommission expert review:
+
+- Critical and high-risk data integrity issues.
+- Distributed state propagation and recovery semantics.
+- Data movement resource usage and metadata preservation.
+- Operational visibility, auditability, and hardening gaps.
+
+This file is not a code patch. Each block below should be expanded into a focused implementation plan before code changes begin.
+
+## Priority Order
+
+| Order | Fix ID | Source | Priority | Main Risk |
+| --- | --- | --- | --- | --- |
+| 1 | F01 | P1.1 | Critical | Rebalance may drop delete marker or remote tiered metadata |
+| 2 | F02 | P1.5 | High | Decommission may treat unsafe overwrite as cleanup-safe |
+| 3 | F03 | P1.4 | High | Retiring pool may remain writable on stale peers |
+| 4 | F04 | P1.2 | High | Rebalance and decommission may both resume after restart |
+| 5 | F05 | P1.3, A4 | High | Admin start/stop may report success for partial cluster state |
+| 6 | F06 | P2.1 | High | Multipart migration can allocate whole parts in memory |
+| 7 | F07 | P2.2, A3 | Medium | Migrated objects may lose checksum or other metadata semantics |
+| 8 | F08 | P2.4 | Medium | Decommission leaves lifecycle-expired source entries behind |
+| 9 | F09 | P2.3 | Medium | Rebalance overwrite races may fail instead of converging |
+| 10 | F10 | A2 | Medium | Source cleanup lacks a final version-set guard |
+| 11 | F11 | A1 | Medium | Rebalance cleanup failures are completion warnings only |
+| 12 | F12 | P3.2 | Low | Admin operations are not sufficiently auditable |
+| 13 | F13 | P3.3 | Low | Persisted metadata accepts unknown or inconsistent fields |
+| 14 | F14 | P3.4 | Low | Rebalance completion threshold differs from MinIO |
+
+## Phase 1: Data Integrity and Safety
+
+### F01: Fix rebalance delete marker and remote tiered version migration
+
+**Source:** P1.1
+**Decision:** Confirmed fix required.
+**Target priority:** Critical.
+
+**Problem:** `rebalance_entry` marks delete markers and remote tiered versions as moved, but those branches operate through the source `SetDisks` path rather than a confirmed cross-pool target write. Source cleanup can then remove the only metadata that preserves tombstone or tiered-object semantics.
+
+**Primary files:**
+- `crates/ecstore/src/rebalance.rs`
+- `crates/ecstore/src/data_movement.rs`
+- `crates/ecstore/src/set_disk.rs`
+- Tests near existing rebalance migration tests in `crates/ecstore/src/rebalance.rs`
+
+**Preferred fix direction:**
+- Route rebalance delete marker movement through an `ECStore`-level data movement path that chooses a non-source target pool and writes the exact version metadata to that target.
+- For remote tiered versions, either:
+  - align with MinIO and skip tiered versions during rebalance, or
+  - implement a target-pool metadata move that preserves the remote tier pointer and lifecycle state.
+- Do not count the version as rebalanced until the target metadata is confirmed.
+
+**Acceptance criteria:**
+- A versioned object with a delete marker remains deleted after rebalance source cleanup.
+- A remote tiered version remains readable or correctly listed after rebalance source cleanup.
+- Rebalance does not mark delete marker or tiered versions as complete when target metadata was not written.
+
+**Required tests:**
+- Multi-pool rebalance test for a versioned object with a delete marker.
+- Multi-pool rebalance test for a remote tiered version.
+- Failure test where target metadata write fails and source cleanup must not happen.
+
+**Dependencies:** None. This is the first safety fix.
+
+---
+
+### F02: Stop treating `DataMovementOverwriteErr` as cleanup-safe in decommission
+
+**Source:** P1.5
+**Decision:** Confirmed fix required.
+**Target priority:** High.
+
+**Problem:** Decommission treats `DataMovementOverwriteErr` like object-not-found/version-not-found and sets `cleanup_ignored = true`. That error only means source and destination pool are the same; it does not prove an equivalent target version exists.
+
+**Primary files:**
+- `crates/ecstore/src/pools.rs`
+- `crates/ecstore/src/data_movement.rs`
+- `crates/ecstore/src/store/object.rs`
+
+**Preferred fix direction:**
+- Remove `DataMovementOverwriteErr` from cleanup-safe branches for delete marker and remote tiered decommission paths.
+- If overwrite occurs because an equivalent target version already exists, perform an explicit equivalence check before counting the version complete.
+- Otherwise return a migration failure and keep the source entry.
+
+**Acceptance criteria:**
+- `DataMovementOverwriteErr` does not increment the decommissioned count unless equivalence is proven.
+- Source cleanup is blocked when target equivalence is unknown.
+- Logs/status clearly state whether overwrite was equivalent-complete or unsafe.
+
+**Required tests:**
+- Delete marker decommission returns `DataMovementOverwriteErr` and does not clean source.
+- Remote tiered decommission returns `DataMovementOverwriteErr` and does not clean source.
+- Equivalent target version exists; overwrite can be counted complete only after metadata equality check passes.
+
+**Dependencies:** F01 clarifies remote/tombstone movement semantics, but F02 can be implemented independently for decommission.
+
+---
+
+### F03: Make decommission `pool_meta` reload a cluster-wide start barrier
+
+**Source:** P1.4
+**Decision:** Confirmed fix required.
+**Target priority:** High.
+
+**Problem:** `start_decommission` saves pool meta and then calls peer reload. Reload failure is logged but the operation still returns success, leaving stale peers able to write into the retiring pool.
+
+**Primary files:**
+- `crates/ecstore/src/pools.rs`
+- `crates/ecstore/src/notification_sys.rs`
+- `crates/ecstore/src/rpc/peer_rest_client.rs`
+- `rustfs/src/admin/handlers/pools.rs`
+- `rustfs/src/storage/rpc/node_service.rs`
+
+**Preferred fix direction:**
+- Treat reload failure as a failed start unless the system has an explicit partial/degraded operation state.
+- Return peer failure information to the admin handler.
+- Do not spawn or resume decommission workers until required peers acknowledge the updated pool meta.
+
+**Acceptance criteria:**
+- Admin decommission start fails or returns a clearly degraded response when peer reload fails.
+- No peer can select the retiring pool for new writes after decommission start is reported successful.
+- Status exposes peer reload failure details.
+
+**Required tests:**
+- Unit test for `start_decommission` with mocked `reload_pool_meta` failure.
+- RPC/admin test showing start does not return plain success on peer failure.
+- Write-placement test verifying retiring pools are skipped only after successful cluster reload.
+
+**Dependencies:** None.
+
+---
+
+### F04: Reorder store init recovery to restore pool meta before rebalance
+
+**Source:** P1.2
+**Decision:** Confirmed fix required.
+**Target priority:** High.
+
+**Problem:** Store init loads and starts rebalance before loading persisted pool meta. If unfinished decommission exists on disk, `start_rebalance` can miss it and both processes may resume.
+
+**Primary files:**
+- `crates/ecstore/src/store/init.rs`
+- `crates/ecstore/src/rebalance.rs`
+- `crates/ecstore/src/pools.rs`
+
+**Preferred fix direction:**
+- Load and install `PoolMeta` before attempting to start rebalance.
+- After pool meta is loaded, explicitly decide:
+  - if decommission is active, do not auto-start rebalance;
+  - if rebalance metadata exists but conflicts with decommission, mark rebalance as blocked/deferred and expose it in status.
+- Keep existing decommission resume delay, but ensure rebalance is not already running when decommission resumes.
+
+**Acceptance criteria:**
+- Restart with active decommission metadata and rebalance metadata does not run both workers.
+- Startup status explains which operation was deferred and why.
+- Existing startup without decommission still resumes rebalance normally.
+
+**Required tests:**
+- Init test with persisted active decommission plus started rebalance metadata.
+- Init test with only rebalance metadata still resumes rebalance.
+- Init test with corrupt or missing pool meta follows existing error policy.
+
+**Dependencies:** None, but coordinate with F05 status semantics.
+
+---
+
+### F05: Make rebalance start/stop distributed semantics explicit and fail-safe
+
+**Source:** P1.3, A4
+**Decision:** Confirmed fix required.
+**Target priority:** High.
+
+**Problem:** Rebalance start can return success even when peer propagation fails or peer background start later fails. Stop can return success while peer stop failed or local stop errors were ignored. Stop is also asynchronous, but admin semantics do not clearly expose `stopping` versus `stopped`.
+
+**Primary files:**
+- `rustfs/src/admin/handlers/rebalance.rs`
+- `rustfs/src/storage/rpc/node_service.rs`
+- `crates/ecstore/src/notification_sys.rs`
+- `crates/ecstore/src/rpc/peer_rest_client.rs`
+- `crates/ecstore/src/rebalance.rs`
+
+**Preferred fix direction:**
+- For start:
+  - propagate rebalance metadata to peers before returning success;
+  - have peer RPC synchronously validate/start or return a structured failure;
+  - on partial failure, rollback local start or report a degraded state.
+- For stop:
+  - never ignore `store.stop_rebalance()` errors;
+  - aggregate peer stop failures and return them to admin;
+  - expose `stop_requested`, `stopping`, and `stopped` in status if worker shutdown remains asynchronous.
+
+**Acceptance criteria:**
+- Admin start does not return plain success when any required peer fails to load/start.
+- Admin stop does not return plain success when any required peer fails to stop.
+- Status can distinguish a requested stop from all workers fully stopped.
+
+**Required tests:**
+- Peer `load_rebalance_meta(true)` failure.
+- Peer background `start_rebalance` failure.
+- Peer `stop_rebalance` local save failure.
+- Stop requested while a worker is in a long migration operation.
+
+**Dependencies:** Coordinate with F04 so startup and admin semantics use the same conflict model.
+
+---
+
+## Phase 2: Data Movement Correctness and Resource Control
+
+### F06: Stream multipart data movement instead of buffering whole parts
+
+**Source:** P2.1
+**Decision:** Confirmed fix required.
+**Target priority:** High.
+
+**Problem:** Multipart migration allocates `Vec<u8>` for each part and reads the whole part into memory. Large parts plus concurrent workers can OOM the process.
+
+**Primary files:**
+- `crates/ecstore/src/data_movement.rs`
+- `crates/ecstore/src/store/multipart.rs`
+- `crates/ecstore/src/store/object.rs`
+
+**Preferred fix direction:**
+- Replace whole-part buffering with bounded streaming.
+- Preserve the current part index and checksum/etag behavior through the streaming reader.
+- Ensure multipart abort still runs on any failed part or complete call.
+
+**Acceptance criteria:**
+- Migrating a large multipart object does not allocate memory proportional to part size.
+- Part ETag, size, actual size, and index handling remain correct.
+- Failed migration aborts the temporary multipart upload.
+
+**Required tests:**
+- Fake large multipart reader proving memory is bounded.
+- Multipart migration preserves part order and ETags.
+- Failure during part upload aborts the multipart upload.
+
+**Dependencies:** Coordinate with F07 for checksum preservation.
+
+---
+
+### F07: Preserve and verify full data movement metadata
+
+**Source:** P2.2, A3
+**Decision:** Confirmed fix required for checksum; additional metadata requires verification and likely fixes.
+**Target priority:** Medium.
+
+**Problem:** Data movement preserves basic metadata but does not fully prove checksum, multipart checksum, replication state, version purge status, or object-lock metadata equivalence after migration.
+
+**Primary files:**
+- `crates/ecstore/src/data_movement.rs`
+- `crates/ecstore/src/store_api/types.rs`
+- `crates/ecstore/src/set_disk.rs`
+- `crates/filemeta/src/fileinfo.rs`
+- `crates/filemeta/src/replication.rs`
+
+**Preferred fix direction:**
+- For multipart migration, populate `CompletePart` checksum fields when source part checksum exists.
+- Preserve or reconstruct object-level checksum metadata.
+- Add migration equivalence checks for:
+  - ETag;
+  - checksum and multipart checksum;
+  - replication status and version purge status;
+  - object lock retention mode/date and legal hold;
+  - version ID and mod time.
+- Only change production metadata handling after tests show a real loss path.
+
+**Acceptance criteria:**
+- Migrated object metadata matches source for the fields above.
+- `GetObjectAttributes` checksum behavior remains correct after migration.
+- Object lock retention/legal hold remains unchanged across migration.
+
+**Required tests:**
+- Single-part object with checksum metadata.
+- Multipart object with per-part checksum metadata.
+- Object with replication state and version purge state.
+- Object with governance/compliance retention and legal hold.
+
+**Dependencies:** F06 should land first or in the same PR if checksum streaming needs shared reader changes.
+
+---
+
+### F08: Align decommission lifecycle-expired version cleanup semantics
+
+**Source:** P2.4
+**Decision:** Confirmed fix required or explicit design decision required.
+**Target priority:** Medium.
+
+**Problem:** Decommission skips lifecycle-expired versions but does not clean the source entry if any expired version exists. MinIO counts expired versions as completed and can clean the source entry.
+
+**Primary files:**
+- `crates/ecstore/src/pools.rs`
+- Lifecycle evaluator code under `crates/ecstore/src/bucket/lifecycle/`
+
+**Preferred fix direction:**
+- Treat lifecycle-expired versions as completed for decommission source cleanup when they are not required to remain accessible.
+- Keep lifecycle/object-lock/replication checks in place before treating a version as expired.
+- If RustFS intentionally keeps expired source versions, expose that state in status and exclude the pool from being considered fully clean.
+
+**Acceptance criteria:**
+- Decommission can clean a source entry when all remaining versions are either migrated or lifecycle-expired.
+- Object-lock or replication-protected versions are not wrongly treated as cleanup-safe.
+- Status accurately reflects residual expired-source entries if they are intentionally retained.
+
+**Required tests:**
+- Bucket with one migrated version plus one lifecycle-expired version.
+- Expired version protected by object lock must not be cleaned.
+- Replication-pending version must not be treated as cleanup-safe.
+
+**Dependencies:** Coordinate with F07 metadata tests for object lock and replication.
+
+---
+
+### F09: Handle rebalance overwrite races using explicit equivalence checks
+
+**Source:** P2.3
+**Decision:** Fix likely required; first confirm exact race cases with tests.
+**Target priority:** Medium.
+
+**Problem:** RustFS treats `DataMovementOverwriteErr` as non-transient during rebalance, while MinIO ignores acceptable overwrite races. A strict failure can prevent rebalance convergence when the target already has an equivalent version.
+
+**Primary files:**
+- `crates/ecstore/src/rebalance.rs`
+- `crates/ecstore/src/data_movement.rs`
+- `crates/ecstore/src/store/object.rs`
+
+**Preferred fix direction:**
+- Keep unsafe source-equals-target cases as failures.
+- If the target version already exists, compare version ID, ETag, size, mod time, checksum, and key metadata.
+- Count as complete only when equivalence passes.
+- Add last-error/status text when overwrite is unsafe.
+
+**Acceptance criteria:**
+- Equivalent target version allows rebalance to continue.
+- Non-equivalent target version fails and does not clean source.
+- Behavior is documented as compatible with MinIO's acceptable overwrite race handling.
+
+**Required tests:**
+- Rebalance overwrite with equivalent target version.
+- Rebalance overwrite with mismatched checksum or metadata.
+- Rebalance overwrite where source and target pool are the same and no equivalent target exists.
+
+**Dependencies:** F07 defines the metadata equivalence fields.
+
+---
+
+### F10: Add source cleanup preflight verification before deleting migrated entries
+
+**Source:** A2
+**Decision:** Fix recommended as defense-in-depth.
+**Target priority:** Medium.
+
+**Problem:** After versions are migrated, source cleanup deletes the source prefix based on the version list read before migration. A final verification would protect against state propagation failures, recovery races, or repair processes changing metadata before cleanup.
+
+**Primary files:**
+- `crates/ecstore/src/rebalance.rs`
+- `crates/ecstore/src/pools.rs`
+- `crates/ecstore/src/set_disk.rs`
+
+**Preferred fix direction:**
+- Before source cleanup, re-read source metadata or a generation marker.
+- Verify the version set scheduled for deletion equals the version set that was migrated or intentionally ignored.
+- If the version set changed, defer/retry the entry instead of deleting the prefix.
+
+**Acceptance criteria:**
+- Cleanup does not delete versions created or discovered after the migration scan.
+- Changed source metadata causes a retry/defer state with a clear status error.
+- The guard applies to rebalance and decommission where source prefix deletion is used.
+
+**Required tests:**
+- Simulate source metadata changing between migration and cleanup.
+- Simulate peer state mismatch where a write lands on a rebalancing/decommissioning pool.
+- Confirm unchanged metadata still allows cleanup.
+
+**Dependencies:** Works best after F03 and F05 reduce state propagation failures.
+
+---
+
+## Phase 3: Cleanup Semantics, Observability, and Hardening
+
+### F11: Improve rebalance cleanup failure handling and reporting
+
+**Source:** A1
+**Decision:** Fix recommended.
+**Target priority:** Medium.
+
+**Problem:** Rebalance source cleanup failure is converted to a warning and the task can still complete. Only a count and last warning are preserved.
+
+**Primary files:**
+- `crates/ecstore/src/rebalance.rs`
+- `rustfs/src/admin/handlers/rebalance.rs`
+- Rebalance status DTOs and serialization paths
+
+**Preferred fix direction:**
+- Preserve a bounded list of cleanup failures, not only the last one.
+- Add a clear terminal or partial terminal state when cleanup warnings exist.
+- Decide whether cleanup failure should:
+  - defer the entry for retry, or
+  - allow completion but mark the pool as `completed_with_cleanup_warnings`.
+
+**Acceptance criteria:**
+- Admin status reports cleanup warning count and representative object list.
+- Operators can identify which objects need manual cleanup or retry.
+- Rebalance terminal state is not misleading when source data remains.
+
+**Required tests:**
+- Multiple cleanup failures preserve count and bounded object details.
+- Status response includes cleanup warning data.
+- Completion state reflects cleanup warnings.
+
+**Dependencies:** Coordinate with F10 if cleanup failures become retryable.
+
+---
+
+### F12: Add structured audit fields for rebalance and decommission operations
+
+**Source:** P3.2
+**Decision:** Fix recommended.
+**Target priority:** Low.
+
+**Problem:** Critical admin operations do not consistently log actor, remote address, request ID, pool indices, peer host, result, and partial failure state.
+
+**Primary files:**
+- `rustfs/src/admin/handlers/rebalance.rs`
+- `rustfs/src/admin/handlers/pools.rs`
+- `crates/ecstore/src/notification_sys.rs`
+- `crates/ecstore/src/rpc/peer_rest_client.rs`
+- Logging guardrail scripts if applicable
+
+**Preferred fix direction:**
+- Standardize structured fields for start, stop, cancel, status-affecting propagation, and peer RPC failures.
+- Mask sensitive actor material.
+- Use warn/error for partial failure rather than info-only logs.
+
+**Acceptance criteria:**
+- Start/stop/cancel logs include actor, request ID, pool/rebalance ID, peer, result, and error when present.
+- Logs do not expose secrets.
+- Partial failures are searchable by stable event fields.
+
+**Required tests:**
+- Log-capture tests for success, rejection, and partial failure.
+- Guardrail check if logging conventions are enforced by script.
+
+**Dependencies:** F03 and F05 define partial-failure semantics.
+
+---
+
+### F13: Harden persisted rebalance and pool metadata decoding
+
+**Source:** P3.3
+**Decision:** Fix recommended after compatibility review.
+**Target priority:** Low.
+
+**Problem:** Persisted rebalance and pool metadata can accept unknown fields silently. This may be acceptable for compatibility, but it weakens corruption and typo detection.
+
+**Primary files:**
+- `crates/ecstore/src/rebalance.rs`
+- `crates/ecstore/src/pools.rs`
+- Metadata fixture tests near existing metadata tests
+
+**Preferred fix direction:**
+- For formats that can be strict, add `deny_unknown_fields`.
+- Where backward compatibility requires leniency, detect and warn on unknown fields during legacy decode.
+- Add explicit validation for conflicting terminal states.
+
+**Acceptance criteria:**
+- Unknown fields either fail decode or produce a warning through a documented compatibility path.
+- Missing critical fields fail safely.
+- Conflicting states such as completed plus running are rejected.
+
+**Required tests:**
+- Unknown field fixture.
+- Missing critical field fixture.
+- Conflicting terminal state fixture.
+- Legacy fixture proving backward compatibility.
+
+**Dependencies:** None.
+
+---
+
+### F14: Decide and document rebalance completion threshold behavior
+
+**Source:** P3.4
+**Decision:** Design decision required.
+**Target priority:** Low.
+
+**Problem:** RustFS uses a stricter rebalance completion goal than MinIO's tolerance-based behavior. This may increase movement work and operational time.
+
+**Primary files:**
+- `crates/ecstore/src/rebalance.rs`
+- Admin status documentation if present
+- Tests around rebalance goal calculation
+
+**Preferred fix direction:**
+- Decide whether RustFS should:
+  - match MinIO tolerance;
+  - keep strict behavior and document it;
+  - make tolerance configurable with a conservative default.
+- Avoid configurability unless a real operational requirement exists.
+
+**Acceptance criteria:**
+- Completion behavior is intentional and covered by tests.
+- Status/ETA reporting does not imply MinIO-compatible tolerance if strict behavior remains.
+- Large-pool behavior is covered by a deterministic unit test.
+
+**Required tests:**
+- Rebalance goal just below target.
+- Rebalance goal within MinIO-like tolerance.
+- Empty queue completion path remains valid.
+
+**Dependencies:** None.
+
+---
+
+## Cross-Cutting Test Matrix
+
+Each fix should add focused tests, but the following end-to-end scenarios should also be covered before enabling production use:
+
+1. Rebalance a versioned object with normal versions, delete marker, and source cleanup.
+2. Rebalance a remote tiered version or verify it is intentionally skipped.
+3. Decommission a pool while peer reload fails.
+4. Restart with both active decommission metadata and rebalance metadata.
+5. Stop rebalance while a large multipart migration is in progress.
+6. Migrate a large multipart object without memory proportional to part size.
+7. Migrate objects with checksum, replication state, retention, and legal hold metadata.
+8. Trigger source cleanup failure and verify admin status exposes it.
+9. Trigger overwrite race with equivalent and non-equivalent target versions.
+10. Decommission lifecycle-expired versions with object-lock and replication constraints.
+
+## Suggested Execution Strategy
+
+1. Start with F01 and F02 because they protect object version semantics.
+2. Implement F03, F04, and F05 before broader rollout because they define distributed operation safety.
+3. Implement F06 before running large-scale data movement tests.
+4. Implement F07 through F11 as compatibility, convergence, and observability hardening.
+5. Keep F12 through F14 as final hardening unless release criteria require them earlier.
+
+Each fix should be developed in a separate PR unless two adjacent fixes share the same core implementation. In particular:
+
+- F01 and F09 should not be merged together unless the equivalence helper is clearly isolated.
+- F06 and F07 may be combined only if streaming part migration and checksum preservation share the same reader changes.
+- F03 and F05 may share peer failure aggregation utilities, but their admin behavior should be tested separately.

rustfs/src/admin/handlers/heal.rs

@@ -34,6 +34,7 @@ use rustfs_utils::path::path_join;
 use s3s::header::{CONTENT_LENGTH, CONTENT_TYPE};
 use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
 use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
 use std::path::PathBuf;
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
 use tokio::sync::mpsc;
@@ -64,21 +65,28 @@ fn extract_heal_init_params(body: &Bytes, uri: &Uri, params: Params<'_, '_>) ->
     validate_heal_target(&hip.bucket, &hip.obj_prefix)?;
 
     if let Some(query) = uri.query() {
-        let params: Vec<&str> = query.split('&').collect();
-        for param in params {
-            let mut parts = param.split('=');
-            if let Some(key) = parts.next() {
-                if key == "clientToken"
-                    && let Some(value) = parts.next()
-                {
-                    hip.client_token = value.to_string();
+        let mut seen = HashSet::with_capacity(3);
+        for (key, value) in url::form_urlencoded::parse(query.as_bytes()) {
+            match key.as_ref() {
+                "clientToken" => {
+                    if !seen.insert("clientToken") {
+                        return Err(s3_error!(InvalidArgument, "duplicate heal query parameter"));
+                    }
+                    hip.client_token = value.into_owned();
                 }
-                if key == "forceStart" && parts.next().is_some() {
-                    hip.force_start = true;
+                "forceStart" => {
+                    if !seen.insert("forceStart") {
+                        return Err(s3_error!(InvalidArgument, "duplicate heal query parameter"));
+                    }
+                    hip.force_start = parse_heal_query_bool(value.as_ref())?;
                 }
-                if key == "forceStop" && parts.next().is_some() {
-                    hip.force_stop = true;
+                "forceStop" => {
+                    if !seen.insert("forceStop") {
+                        return Err(s3_error!(InvalidArgument, "duplicate heal query parameter"));
+                    }
+                    hip.force_stop = parse_heal_query_bool(value.as_ref())?;
                 }
+                _ => return Err(s3_error!(InvalidArgument, "unknown heal query parameter")),
             }
         }
     }
@@ -109,6 +117,14 @@ fn extract_heal_init_params(body: &Bytes, uri: &Uri, params: Params<'_, '_>) ->
     Ok(hip)
 }
 
+fn parse_heal_query_bool(value: &str) -> S3Result<bool> {
+    match value {
+        "true" => Ok(true),
+        "false" => Ok(false),
+        _ => Err(s3_error!(InvalidArgument, "invalid heal query boolean")),
+    }
+}
+
 fn validate_heal_target(bucket: &str, obj_prefix: &str) -> S3Result<()> {
     if bucket.is_empty() && !obj_prefix.is_empty() {
         return Err(s3_error!(InvalidRequest, "invalid bucket name"));
@@ -817,6 +833,28 @@ mod tests {
         assert!(parsed.force_stop);
     }
 
+    #[test]
+    fn test_extract_heal_init_params_rejects_unknown_duplicate_and_invalid_bool() {
+        let mut router = Router::new();
+        router
+            .insert("/rustfs/admin/v3/heal/{bucket}", ())
+            .expect("route should insert");
+
+        for query in [
+            "forceStart=yes",
+            "forceStart=true&forceStart=false",
+            "clientToken=token&unexpected=true",
+        ] {
+            let uri: Uri = format!("/rustfs/admin/v3/heal/test-bucket?{query}")
+                .parse()
+                .expect("uri should parse");
+            let matched = router.at("/rustfs/admin/v3/heal/test-bucket").expect("route should match");
+            let err = extract_heal_init_params(&Bytes::new(), &uri, matched.params)
+                .expect_err("strict heal query should reject malformed input");
+            assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+        }
+    }
+
     #[test]
     fn test_heal_channel_request_preserves_admin_heal_options() {
         let hip = HealInitParams {

rustfs/src/admin/handlers/pools.rs

@@ -12,12 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use http::{HeaderMap, StatusCode};
+use http::{HeaderMap, StatusCode, Uri};
 use matchit::Params;
 use rustfs_policy::policy::action::{Action, AdminAction};
+use rustfs_utils::{
+    MaskedAccessKey,
+    http::{AMZ_REQUEST_ID, REQUEST_ID_HEADER},
+};
 use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
 use serde::Deserialize;
-use serde_urlencoded::from_bytes;
 use tokio_util::sync::CancellationToken;
 use tracing::{error, info, warn};
 
@@ -38,10 +41,114 @@ use std::collections::HashSet;
 
 const LOG_COMPONENT_ADMIN_API: &str = "admin_api";
 const LOG_SUBSYSTEM_POOL_ADMIN: &str = "pool_admin";
+const EVENT_ADMIN_REQUEST_STATE: &str = "admin_request_state";
 const EVENT_ADMIN_REQUEST_REJECTED: &str = "admin_request_rejected";
 const EVENT_ADMIN_REQUEST_FAILED: &str = "admin_request_failed";
 const EVENT_ADMIN_RESPONSE_EMITTED: &str = "admin_response_emitted";
 
+fn admin_request_id(headers: &HeaderMap) -> Option<&str> {
+    headers
+        .get(REQUEST_ID_HEADER)
+        .or_else(|| headers.get(AMZ_REQUEST_ID))
+        .and_then(|value| value.to_str().ok())
+}
+
+fn admin_remote_addr(req: &S3Request<Body>) -> Option<String> {
+    req.extensions
+        .get::<Option<RemoteAddr>>()
+        .and_then(|opt| opt.map(|addr| addr.0.to_string()))
+}
+
+fn log_pool_request_rejected_with_context(operation: &str, reason: &str, request_id: &str, actor: &str, remote_addr: &str) {
+    warn!(
+        event = EVENT_ADMIN_REQUEST_REJECTED,
+        component = LOG_COMPONENT_ADMIN_API,
+        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+        operation,
+        action = operation,
+        result = "rejected",
+        reason,
+        request_id = %request_id,
+        actor = %actor,
+        remote_addr = %remote_addr,
+        "admin request rejected"
+    );
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct PoolAuditContext<'a> {
+    request_id: &'a str,
+    actor: &'a str,
+    remote_addr: &'a str,
+}
+
+impl<'a> PoolAuditContext<'a> {
+    fn new(request_id: &'a str, actor: &'a str, remote_addr: &'a str) -> Self {
+        Self {
+            request_id,
+            actor,
+            remote_addr,
+        }
+    }
+}
+
+fn log_pool_request_rejected_with_audit(operation: &str, reason: &str, audit: PoolAuditContext<'_>) {
+    warn!(
+        event = EVENT_ADMIN_REQUEST_REJECTED,
+        component = LOG_COMPONENT_ADMIN_API,
+        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+        operation,
+        action = operation,
+        result = "rejected",
+        reason,
+        request_id = %audit.request_id,
+        actor = %audit.actor,
+        remote_addr = %audit.remote_addr,
+        "admin request rejected"
+    );
+}
+
+fn log_pool_request_rejected_with_pool_audit(operation: &str, reason: &str, pool: &str, audit: PoolAuditContext<'_>) {
+    warn!(
+        event = EVENT_ADMIN_REQUEST_REJECTED,
+        component = LOG_COMPONENT_ADMIN_API,
+        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+        operation,
+        action = operation,
+        result = "rejected",
+        reason,
+        request_id = %audit.request_id,
+        actor = %audit.actor,
+        remote_addr = %audit.remote_addr,
+        pool,
+        "admin request rejected"
+    );
+}
+
+fn log_pool_request_rejected_with_index_audit(
+    operation: &str,
+    reason: &str,
+    idx: usize,
+    pool_count: usize,
+    audit: PoolAuditContext<'_>,
+) {
+    warn!(
+        event = EVENT_ADMIN_REQUEST_REJECTED,
+        component = LOG_COMPONENT_ADMIN_API,
+        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+        operation,
+        action = operation,
+        result = "rejected",
+        reason,
+        request_id = %audit.request_id,
+        actor = %audit.actor,
+        remote_addr = %audit.remote_addr,
+        pool_index = idx,
+        pool_count,
+        "admin request rejected"
+    );
+}
+
 macro_rules! log_pool_request_rejected {
     ($operation:expr, $reason:expr) => {
         warn!(
@@ -49,6 +156,7 @@ macro_rules! log_pool_request_rejected {
             component = LOG_COMPONENT_ADMIN_API,
             subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
             operation = $operation,
+            action = $operation,
             result = "rejected",
             reason = $reason,
             "admin request rejected"
@@ -56,21 +164,6 @@ macro_rules! log_pool_request_rejected {
     };
 }
 
-macro_rules! log_pool_request_rejected_with_pool {
-    ($operation:expr, $reason:expr, $pool:expr) => {
-        warn!(
-            event = EVENT_ADMIN_REQUEST_REJECTED,
-            component = LOG_COMPONENT_ADMIN_API,
-            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
-            operation = $operation,
-            result = "rejected",
-            reason = $reason,
-            pool = $pool,
-            "admin request rejected"
-        );
-    };
-}
-
 macro_rules! log_pool_request_failed {
     ($operation:expr, $reason:expr, $err:expr) => {
         error!(
@@ -78,6 +171,7 @@ macro_rules! log_pool_request_failed {
             component = LOG_COMPONENT_ADMIN_API,
             subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
             operation = $operation,
+            action = $operation,
             result = "failed",
             reason = $reason,
             error = %$err,
@@ -89,10 +183,11 @@ macro_rules! log_pool_request_failed {
 macro_rules! log_pool_response_emitted {
     ($operation:expr) => {
         info!(
-            event = EVENT_ADMIN_RESPONSE_EMITTED,
+            event = EVENT_ADMIN_REQUEST_STATE,
             component = LOG_COMPONENT_ADMIN_API,
             subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
             operation = $operation,
+            action = $operation,
             result = "success",
             "admin response emitted"
         );
@@ -130,11 +225,20 @@ fn contextualize_admin_pool_api_error(
     }
 }
 
-fn decommission_admin_not_initialized_error(operation: &str) -> S3Error {
-    log_pool_request_failed!(
-        operation_to_event(operation),
-        "object_layer_not_initialized",
-        "object layer not initialized"
+fn decommission_admin_not_initialized_error_with_audit(operation: &str, audit: PoolAuditContext<'_>) -> S3Error {
+    error!(
+        event = EVENT_ADMIN_REQUEST_FAILED,
+        component = LOG_COMPONENT_ADMIN_API,
+        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+        operation = operation_to_event(operation),
+        action = operation_to_event(operation),
+        result = "failed",
+        reason = "object_layer_not_initialized",
+        request_id = %audit.request_id,
+        actor = %audit.actor,
+        remote_addr = %audit.remote_addr,
+        error = "object layer not initialized",
+        "admin request failed"
     );
     S3Error::with_message(S3ErrorCode::InternalError, format!("Failed to {operation}: object layer not initialized"))
 }
@@ -144,36 +248,52 @@ fn pool_admin_missing_credentials_error(operation: &str) -> S3Error {
     S3Error::with_message(S3ErrorCode::InvalidRequest, format!("Failed to {operation}: missing credentials"))
 }
 
+fn pool_admin_missing_credentials_error_with_request(operation: &str, request_id: &str, remote_addr: &str) -> S3Error {
+    warn!(
+        event = EVENT_ADMIN_REQUEST_REJECTED,
+        component = LOG_COMPONENT_ADMIN_API,
+        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+        operation = operation_to_event(operation),
+        action = operation_to_event(operation),
+        result = "rejected",
+        reason = "missing_credentials",
+        request_id = %request_id,
+        remote_addr = %remote_addr,
+        "admin request rejected"
+    );
+    S3Error::with_message(S3ErrorCode::InvalidRequest, format!("Failed to {operation}: missing credentials"))
+}
+
 fn pool_admin_query_parse_error(operation: &str) -> S3Error {
     log_pool_request_rejected!(operation_to_event(operation), "invalid_query_parameters");
     S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Failed to {operation}: invalid query parameters"))
 }
 
-fn pool_admin_pool_parse_error(operation: &str, pool: &str) -> S3Error {
-    log_pool_request_rejected_with_pool!(operation_to_event(operation), "invalid_pool", pool);
+fn pool_admin_query_parse_error_with_audit(operation: &str, audit: PoolAuditContext<'_>) -> S3Error {
+    log_pool_request_rejected_with_audit(operation_to_event(operation), "invalid_query_parameters", audit);
+    S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Failed to {operation}: invalid query parameters"))
+}
+
+fn pool_admin_pool_parse_error_with_audit(operation: &str, pool: &str, audit: PoolAuditContext<'_>) -> S3Error {
+    log_pool_request_rejected_with_pool_audit(operation_to_event(operation), "invalid_pool", pool, audit);
     S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Failed to {operation}: invalid pool `{pool}`"))
 }
 
-fn pool_admin_pool_not_found_error(operation: &str, pool: &str) -> S3Error {
-    log_pool_request_rejected_with_pool!(operation_to_event(operation), "pool_not_found", pool);
+fn pool_admin_pool_not_found_error_with_audit(operation: &str, pool: &str, audit: PoolAuditContext<'_>) -> S3Error {
+    log_pool_request_rejected_with_pool_audit(operation_to_event(operation), "pool_not_found", pool, audit);
     S3Error::with_message(
         S3ErrorCode::InvalidArgument,
         format!("Failed to {operation}: pool `{pool}` was not found"),
     )
 }
 
-fn pool_admin_pool_index_error(operation: &str, idx: usize, pool_count: usize) -> S3Error {
-    warn!(
-        event = EVENT_ADMIN_REQUEST_REJECTED,
-        component = LOG_COMPONENT_ADMIN_API,
-        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
-        operation = operation_to_event(operation),
-        result = "rejected",
-        reason = "pool_index_out_of_range",
-        pool_index = idx,
-        pool_count,
-        "admin request rejected"
-    );
+fn pool_admin_pool_index_error_with_audit(
+    operation: &str,
+    idx: usize,
+    pool_count: usize,
+    audit: PoolAuditContext<'_>,
+) -> S3Error {
+    log_pool_request_rejected_with_index_audit(operation_to_event(operation), "pool_index_out_of_range", idx, pool_count, audit);
     S3Error::with_message(
         S3ErrorCode::InvalidArgument,
         format!("Failed to {operation}: pool index {idx} is out of range for {pool_count} pools"),
@@ -186,6 +306,7 @@ fn operation_to_event(operation: &str) -> &'static str {
         "load pool status" => "query_pool_status",
         "start decommission" => "start_decommission",
         "cancel decommission" => "cancel_decommission",
+        "clear decommission" => "clear_decommission",
         _ => "pool_admin",
     }
 }
@@ -195,15 +316,14 @@ fn parse_pool_idx_by_id(pool: &str, endpoint_count: usize) -> Option<usize> {
     (idx < endpoint_count).then_some(idx)
 }
 
-fn dedup_indices(indices: &[usize]) -> Vec<usize> {
+fn has_duplicate_indices(indices: &[usize]) -> bool {
     let mut seen = HashSet::with_capacity(indices.len());
-    let mut output = Vec::with_capacity(indices.len());
     for idx in indices {
-        if seen.insert(*idx) {
-            output.push(*idx);
+        if !seen.insert(*idx) {
+            return true;
         }
     }
-    output
+    false
 }
 
 pub fn register_pool_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
@@ -231,6 +351,12 @@ pub fn register_pool_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<
         AdminOperation(&CancelDecommission {}),
     )?;
 
+    r.insert(
+        Method::POST,
+        format!("{}{}", ADMIN_PREFIX, "/v3/pools/clear").as_str(),
+        AdminOperation(&ClearDecommission {}),
+    )?;
+
     Ok(())
 }
 
@@ -285,6 +411,52 @@ pub struct StatusPoolQuery {
     pub by_id: String,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum PoolQueryMode {
+    Status,
+    Mutation,
+}
+
+fn parse_status_pool_query(uri: &Uri) -> Result<StatusPoolQuery, ()> {
+    parse_pool_query(uri, PoolQueryMode::Status)
+}
+
+fn parse_mutation_pool_query(uri: &Uri) -> Result<StatusPoolQuery, ()> {
+    parse_pool_query(uri, PoolQueryMode::Mutation)
+}
+
+fn parse_pool_query(uri: &Uri, mode: PoolQueryMode) -> Result<StatusPoolQuery, ()> {
+    let mut parsed = StatusPoolQuery::default();
+    let mut seen = HashSet::with_capacity(2);
+    let Some(query) = uri.query() else {
+        return Ok(parsed);
+    };
+
+    for (key, value) in url::form_urlencoded::parse(query.as_bytes()) {
+        match key.as_ref() {
+            "pool" => {
+                if !seen.insert("pool") {
+                    return Err(());
+                }
+                parsed.pool = value.into_owned();
+            }
+            "by-id" => {
+                if !seen.insert("by-id") {
+                    return Err(());
+                }
+                match value.as_ref() {
+                    "true" | "false" => parsed.by_id = value.into_owned(),
+                    _ => return Err(()),
+                }
+            }
+            _ if mode == PoolQueryMode::Status => {}
+            _ => return Err(()),
+        }
+    }
+
+    Ok(parsed)
+}
+
 pub struct StatusPool {}
 
 #[async_trait::async_trait]
@@ -312,15 +484,7 @@ impl Operation for StatusPool {
         )
         .await?;
 
-        let query = {
-            if let Some(query) = req.uri.query() {
-                let input: StatusPoolQuery =
-                    from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("load pool status"))?;
-                input
-            } else {
-                StatusPoolQuery::default()
-            }
-        };
+        let query = parse_status_pool_query(&req.uri).map_err(|_| pool_admin_query_parse_error("load pool status"))?;
 
         let usecase = DefaultAdminUsecase::from_global();
         let pools_status = usecase
@@ -351,12 +515,31 @@ impl Operation for StartDecommission {
     // POST <endpoint>/<admin-API>/pools/decommission?pool=http://server{1...4}/disk{1...4}
     #[tracing::instrument(skip_all)]
     async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
+        let remote_addr = admin_remote_addr(&req).unwrap_or_default();
+        info!(
+            event = EVENT_ADMIN_REQUEST_STATE,
+            component = LOG_COMPONENT_ADMIN_API,
+            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+            operation = "start_decommission",
+            action = "start_decommission",
+            state = "requested",
+            request_id = %request_id,
+            remote_addr = %remote_addr,
+            "admin pool request state"
+        );
+
         let Some(input_cred) = req.credentials else {
-            return Err(pool_admin_missing_credentials_error("start decommission"));
+            return Err(pool_admin_missing_credentials_error_with_request(
+                "start decommission",
+                &request_id,
+                &remote_addr,
+            ));
         };
 
         let (cred, owner) =
             check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+        let actor = MaskedAccessKey(&input_cred.access_key).to_string();
 
         validate_admin_request(
             &req.headers,
@@ -367,32 +550,51 @@ impl Operation for StartDecommission {
             req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
         )
         .await?;
+        let audit = PoolAuditContext::new(&request_id, &actor, &remote_addr);
 
         let Some(endpoints) = endpoints_from_context() else {
-            log_pool_request_rejected!("start_decommission", "not_implemented");
+            log_pool_request_rejected_with_context("start_decommission", "not_implemented", &request_id, &actor, &remote_addr);
             return Err(s3_error!(NotImplemented));
         };
 
         if endpoints.legacy() {
-            log_pool_request_rejected!("start_decommission", "legacy_endpoints_not_supported");
+            log_pool_request_rejected_with_context(
+                "start_decommission",
+                "legacy_endpoints_not_supported",
+                &request_id,
+                &actor,
+                &remote_addr,
+            );
             return Err(s3_error!(NotImplemented));
         }
 
         let Some(store) = resolve_object_store_handle() else {
-            return Err(decommission_admin_not_initialized_error("start decommission"));
+            return Err(decommission_admin_not_initialized_error_with_audit("start decommission", audit));
         };
 
-        validate_start_decommission_guards(store.is_decommission_running().await, store.is_rebalance_started().await)?;
+        let decommission_running = store.is_decommission_running().await;
+        let rebalance_running = store.is_rebalance_started().await;
+        if decommission_running {
+            log_pool_request_rejected_with_context(
+                "start_decommission",
+                "decommission_already_running",
+                &request_id,
+                &actor,
+                &remote_addr,
+            );
+        } else if rebalance_running {
+            log_pool_request_rejected_with_context(
+                "start_decommission",
+                "rebalance_in_progress",
+                &request_id,
+                &actor,
+                &remote_addr,
+            );
+        }
+        validate_start_decommission_guards(decommission_running, rebalance_running)?;
 
-        let query = {
-            if let Some(query) = req.uri.query() {
-                let input: StatusPoolQuery =
-                    from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("start decommission"))?;
-                input
-            } else {
-                StatusPoolQuery::default()
-            }
-        };
+        let query = parse_mutation_pool_query(&req.uri)
+            .map_err(|_| pool_admin_query_parse_error_with_audit("start decommission", audit))?;
         let is_byid = query.by_id.as_str() == "true";
 
         let pools: Vec<&str> = query.pool.split(",").collect();
@@ -404,33 +606,53 @@ impl Operation for StartDecommission {
             let idx = {
                 if is_byid {
                     parse_pool_idx_by_id(pool, endpoints.as_ref().len())
-                        .ok_or_else(|| pool_admin_pool_parse_error("start decommission", pool))?
+                        .ok_or_else(|| pool_admin_pool_parse_error_with_audit("start decommission", pool, audit))?
                 } else {
                     let Some(idx) = endpoints.get_pool_idx(pool) else {
-                        return Err(pool_admin_pool_parse_error("start decommission", pool));
+                        return Err(pool_admin_pool_parse_error_with_audit("start decommission", pool, audit));
                     };
                     idx
                 }
             };
 
             if idx >= store.pools.len() {
-                return Err(pool_admin_pool_index_error("start decommission", idx, store.pools.len()));
+                return Err(pool_admin_pool_index_error_with_audit(
+                    "start decommission",
+                    idx,
+                    store.pools.len(),
+                    audit,
+                ));
             }
 
             parsed_indices.push(idx);
         }
-        let pools_indices = dedup_indices(&parsed_indices);
+        if has_duplicate_indices(&parsed_indices) {
+            return Err(pool_admin_query_parse_error_with_audit("start decommission", audit));
+        }
+        let pools_indices = parsed_indices;
 
         if !pools_indices.is_empty() {
             let pool_context = format!("pools {:?}", &pools_indices);
             store
-                .decommission(ctx.clone(), pools_indices)
+                .decommission(ctx.clone(), pools_indices.clone())
                 .await
                 .map_err(ApiError::from)
                 .map_err(|err| contextualize_admin_pool_api_error(err, "start decommission", &pool_context))?;
         }
 
-        log_pool_response_emitted!("start_decommission");
+        info!(
+            event = EVENT_ADMIN_RESPONSE_EMITTED,
+            component = LOG_COMPONENT_ADMIN_API,
+            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+            operation = "start_decommission",
+            action = "start_decommission",
+            result = "success",
+            request_id = %request_id,
+            actor = %actor,
+            remote_addr = %remote_addr,
+            pool_indices = ?pools_indices,
+            "admin response emitted"
+        );
         Ok(S3Response::new((StatusCode::OK, Body::default())))
     }
 }
@@ -442,12 +664,31 @@ impl Operation for CancelDecommission {
     // POST <endpoint>/<admin-API>/pools/cancel?pool=http://server{1...4}/disk{1...4}
     #[tracing::instrument(skip_all)]
     async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
+        let remote_addr = admin_remote_addr(&req).unwrap_or_default();
+        info!(
+            event = EVENT_ADMIN_REQUEST_STATE,
+            component = LOG_COMPONENT_ADMIN_API,
+            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+            operation = "cancel_decommission",
+            action = "cancel_decommission",
+            state = "requested",
+            request_id = %request_id,
+            remote_addr = %remote_addr,
+            "admin pool request state"
+        );
+
         let Some(input_cred) = req.credentials else {
-            return Err(pool_admin_missing_credentials_error("cancel decommission"));
+            return Err(pool_admin_missing_credentials_error_with_request(
+                "cancel decommission",
+                &request_id,
+                &remote_addr,
+            ));
         };
 
         let (cred, owner) =
             check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+        let actor = MaskedAccessKey(&input_cred.access_key).to_string();
 
         validate_admin_request(
             &req.headers,
@@ -458,26 +699,26 @@ impl Operation for CancelDecommission {
             req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
         )
         .await?;
+        let audit = PoolAuditContext::new(&request_id, &actor, &remote_addr);
 
         let Some(endpoints) = endpoints_from_context() else {
-            log_pool_request_rejected!("cancel_decommission", "not_implemented");
+            log_pool_request_rejected_with_context("cancel_decommission", "not_implemented", &request_id, &actor, &remote_addr);
             return Err(s3_error!(NotImplemented));
         };
 
         if endpoints.legacy() {
-            log_pool_request_rejected!("cancel_decommission", "legacy_endpoints_not_supported");
+            log_pool_request_rejected_with_context(
+                "cancel_decommission",
+                "legacy_endpoints_not_supported",
+                &request_id,
+                &actor,
+                &remote_addr,
+            );
             return Err(s3_error!(NotImplemented));
         }
 
-        let query = {
-            if let Some(query) = req.uri.query() {
-                let input: StatusPoolQuery =
-                    from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("cancel decommission"))?;
-                input
-            } else {
-                StatusPoolQuery::default()
-            }
-        };
+        let query = parse_mutation_pool_query(&req.uri)
+            .map_err(|_| pool_admin_query_parse_error_with_audit("cancel decommission", audit))?;
 
         let is_byid = query.by_id.as_str() == "true";
 
@@ -490,11 +731,11 @@ impl Operation for CancelDecommission {
         };
 
         let Some(idx) = has_idx else {
-            return Err(pool_admin_pool_not_found_error("cancel decommission", &query.pool));
+            return Err(pool_admin_pool_not_found_error_with_audit("cancel decommission", &query.pool, audit));
         };
 
         let Some(store) = resolve_object_store_handle() else {
-            return Err(decommission_admin_not_initialized_error("cancel decommission"));
+            return Err(decommission_admin_not_initialized_error_with_audit("cancel decommission", audit));
         };
 
         store
@@ -503,7 +744,124 @@ impl Operation for CancelDecommission {
             .map_err(ApiError::from)
             .map_err(|err| contextualize_admin_pool_api_error(err, "cancel decommission", format!("pool {idx}")))?;
 
-        log_pool_response_emitted!("cancel_decommission");
+        info!(
+            event = EVENT_ADMIN_RESPONSE_EMITTED,
+            component = LOG_COMPONENT_ADMIN_API,
+            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+            operation = "cancel_decommission",
+            action = "cancel_decommission",
+            result = "success",
+            request_id = %request_id,
+            actor = %actor,
+            remote_addr = %remote_addr,
+            pool_index = idx,
+            "admin response emitted"
+        );
+        Ok(S3Response::new((StatusCode::OK, Body::default())))
+    }
+}
+
+pub struct ClearDecommission {}
+
+#[async_trait::async_trait]
+impl Operation for ClearDecommission {
+    // POST <endpoint>/<admin-API>/pools/clear?pool=http://server{1...4}/disk{1...4}
+    // Clears failed/canceled decommission metadata only; already moved data is not rolled back.
+    #[tracing::instrument(skip_all)]
+    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
+        let remote_addr = admin_remote_addr(&req).unwrap_or_default();
+        info!(
+            event = EVENT_ADMIN_REQUEST_STATE,
+            component = LOG_COMPONENT_ADMIN_API,
+            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+            operation = "clear_decommission",
+            action = "clear_decommission",
+            state = "requested",
+            request_id = %request_id,
+            remote_addr = %remote_addr,
+            "admin pool request state"
+        );
+
+        let Some(input_cred) = req.credentials else {
+            return Err(pool_admin_missing_credentials_error_with_request(
+                "clear decommission",
+                &request_id,
+                &remote_addr,
+            ));
+        };
+
+        let (cred, owner) =
+            check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+        let actor = MaskedAccessKey(&input_cred.access_key).to_string();
+
+        validate_admin_request(
+            &req.headers,
+            &cred,
+            owner,
+            false,
+            vec![Action::AdminAction(AdminAction::DecommissionAdminAction)],
+            req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
+        )
+        .await?;
+        let audit = PoolAuditContext::new(&request_id, &actor, &remote_addr);
+
+        let Some(endpoints) = endpoints_from_context() else {
+            log_pool_request_rejected_with_context("clear_decommission", "not_implemented", &request_id, &actor, &remote_addr);
+            return Err(s3_error!(NotImplemented));
+        };
+
+        if endpoints.legacy() {
+            log_pool_request_rejected_with_context(
+                "clear_decommission",
+                "legacy_endpoints_not_supported",
+                &request_id,
+                &actor,
+                &remote_addr,
+            );
+            return Err(s3_error!(NotImplemented));
+        }
+
+        let query = parse_mutation_pool_query(&req.uri)
+            .map_err(|_| pool_admin_query_parse_error_with_audit("clear decommission", audit))?;
+
+        let is_byid = query.by_id.as_str() == "true";
+
+        let has_idx = {
+            if is_byid {
+                parse_pool_idx_by_id(&query.pool, endpoints.as_ref().len())
+            } else {
+                endpoints.get_pool_idx(&query.pool)
+            }
+        };
+
+        let Some(idx) = has_idx else {
+            return Err(pool_admin_pool_not_found_error_with_audit("clear decommission", &query.pool, audit));
+        };
+
+        let Some(store) = resolve_object_store_handle() else {
+            return Err(decommission_admin_not_initialized_error_with_audit("clear decommission", audit));
+        };
+
+        store
+            .clear_decommission(idx)
+            .await
+            .map_err(ApiError::from)
+            .map_err(|err| contextualize_admin_pool_api_error(err, "clear decommission", format!("pool {idx}")))?;
+
+        info!(
+            event = EVENT_ADMIN_RESPONSE_EMITTED,
+            component = LOG_COMPONENT_ADMIN_API,
+            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+            operation = "clear_decommission",
+            action = "clear_decommission",
+            result = "success",
+            request_id = %request_id,
+            actor = %actor,
+            remote_addr = %remote_addr,
+            pool_index = idx,
+            "admin response emitted"
+        );
         Ok(S3Response::new((StatusCode::OK, Body::default())))
     }
 }
@@ -511,9 +869,12 @@ impl Operation for CancelDecommission {
 #[cfg(test)]
 mod pools_handler_tests {
     use super::{
-        contextualize_admin_pool_api_error, decommission_admin_not_initialized_error, dedup_indices, parse_pool_idx_by_id,
-        pool_admin_missing_credentials_error, pool_admin_pool_index_error, pool_admin_pool_not_found_error,
-        pool_admin_pool_parse_error, pool_admin_query_parse_error, validate_start_decommission_guards,
+        PoolAuditContext, contextualize_admin_pool_api_error, decommission_admin_not_initialized_error_with_audit,
+        has_duplicate_indices, parse_mutation_pool_query, parse_pool_idx_by_id, parse_status_pool_query,
+        pool_admin_missing_credentials_error, pool_admin_missing_credentials_error_with_request,
+        pool_admin_pool_index_error_with_audit, pool_admin_pool_not_found_error_with_audit,
+        pool_admin_pool_parse_error_with_audit, pool_admin_query_parse_error, pool_admin_query_parse_error_with_audit,
+        validate_start_decommission_guards,
     };
 
     #[test]
@@ -526,6 +887,52 @@ mod pools_handler_tests {
         assert_eq!(parse_pool_idx_by_id("4", 4), None);
     }
 
+    #[test]
+    fn test_parse_status_pool_query_ignores_unknown_but_rejects_duplicate_and_invalid_bool() {
+        let unknown = "/rustfs/admin/v3/pools/status?pool=0&force=true"
+            .parse()
+            .expect("uri should parse");
+        let query = parse_status_pool_query(&unknown).expect("status query should ignore unknown keys");
+        assert_eq!(query.pool, "0");
+
+        let duplicate = "/rustfs/admin/v3/pools/status?pool=0&pool=1"
+            .parse()
+            .expect("uri should parse");
+        assert!(parse_status_pool_query(&duplicate).is_err());
+
+        let invalid_bool = "/rustfs/admin/v3/pools/status?by-id=yes".parse().expect("uri should parse");
+        assert!(parse_status_pool_query(&invalid_bool).is_err());
+    }
+
+    #[test]
+    fn test_parse_mutation_pool_query_rejects_unknown_duplicate_and_invalid_bool() {
+        let unknown = "/rustfs/admin/v3/pools/decommission?pool=0&force=true"
+            .parse()
+            .expect("uri should parse");
+        assert!(parse_mutation_pool_query(&unknown).is_err());
+
+        let duplicate = "/rustfs/admin/v3/pools/decommission?pool=0&pool=1"
+            .parse()
+            .expect("uri should parse");
+        assert!(parse_mutation_pool_query(&duplicate).is_err());
+
+        let invalid_bool = "/rustfs/admin/v3/pools/decommission?by-id=yes"
+            .parse()
+            .expect("uri should parse");
+        assert!(parse_mutation_pool_query(&invalid_bool).is_err());
+    }
+
+    #[test]
+    fn test_parse_status_pool_query_accepts_expected_keys() {
+        let uri = "/rustfs/admin/v3/pools/status?pool=pool-a&by-id=true"
+            .parse()
+            .expect("uri should parse");
+        let query = parse_status_pool_query(&uri).expect("valid query should parse");
+
+        assert_eq!(query.pool, "pool-a");
+        assert_eq!(query.by_id, "true");
+    }
+
     #[test]
     fn test_parse_pool_idx_by_id_rejects_empty_pool_count() {
         assert_eq!(parse_pool_idx_by_id("0", 0), None);
@@ -592,21 +999,14 @@ mod pools_handler_tests {
     }
 
     #[test]
-    fn test_decommission_admin_not_initialized_error_formats_start_context() {
-        let err = decommission_admin_not_initialized_error("start decommission");
+    fn test_decommission_admin_not_initialized_error_with_audit_preserves_response_contract() {
+        let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000");
+        let err = decommission_admin_not_initialized_error_with_audit("start decommission", audit);
 
         assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError);
         assert_eq!(err.message(), Some("Failed to start decommission: object layer not initialized"));
     }
 
-    #[test]
-    fn test_decommission_admin_not_initialized_error_formats_cancel_context() {
-        let err = decommission_admin_not_initialized_error("cancel decommission");
-
-        assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError);
-        assert_eq!(err.message(), Some("Failed to cancel decommission: object layer not initialized"));
-    }
-
     #[test]
     fn test_pool_admin_missing_credentials_error_formats_list_context() {
         let err = pool_admin_missing_credentials_error("list pools");
@@ -623,6 +1023,14 @@ mod pools_handler_tests {
         assert_eq!(err.message(), Some("Failed to start decommission: missing credentials"));
     }
 
+    #[test]
+    fn test_pool_admin_missing_credentials_error_with_request_preserves_response_contract() {
+        let err = pool_admin_missing_credentials_error_with_request("cancel decommission", "req-1", "127.0.0.1:9000");
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest);
+        assert_eq!(err.message(), Some("Failed to cancel decommission: missing credentials"));
+    }
+
     #[test]
     fn test_pool_admin_query_parse_error_formats_status_context() {
         let err = pool_admin_query_parse_error("load pool status");
@@ -632,16 +1040,36 @@ mod pools_handler_tests {
     }
 
     #[test]
-    fn test_pool_admin_pool_parse_error_formats_pool_context() {
-        let err = pool_admin_pool_parse_error("start decommission", "pool-x");
+    fn test_pool_audit_context_keeps_request_actor_and_remote_addr() {
+        let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000");
+
+        assert_eq!(audit.request_id, "req-1");
+        assert_eq!(audit.actor, "access-key");
+        assert_eq!(audit.remote_addr, "127.0.0.1:9000");
+    }
+
+    #[test]
+    fn test_pool_admin_query_parse_error_with_audit_preserves_response_contract() {
+        let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000");
+        let err = pool_admin_query_parse_error_with_audit("start decommission", audit);
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
+        assert_eq!(err.message(), Some("Failed to start decommission: invalid query parameters"));
+    }
+
+    #[test]
+    fn test_pool_admin_pool_parse_error_with_audit_preserves_response_contract() {
+        let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000");
+        let err = pool_admin_pool_parse_error_with_audit("start decommission", "pool-x", audit);
 
         assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
         assert_eq!(err.message(), Some("Failed to start decommission: invalid pool `pool-x`"));
     }
 
     #[test]
-    fn test_pool_admin_pool_index_error_formats_range_context() {
-        let err = pool_admin_pool_index_error("start decommission", 4, 2);
+    fn test_pool_admin_pool_index_error_with_audit_preserves_response_contract() {
+        let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000");
+        let err = pool_admin_pool_index_error_with_audit("start decommission", 4, 2, audit);
 
         assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
         assert_eq!(
@@ -651,21 +1079,23 @@ mod pools_handler_tests {
     }
 
     #[test]
-    fn test_pool_admin_pool_not_found_error_formats_cancel_context() {
-        let err = pool_admin_pool_not_found_error("cancel decommission", "pool-x");
+    fn test_pool_admin_pool_not_found_error_with_audit_preserves_response_contract() {
+        let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000");
+        let err = pool_admin_pool_not_found_error_with_audit("cancel decommission", "pool-x", audit);
 
         assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
         assert_eq!(err.message(), Some("Failed to cancel decommission: pool `pool-x` was not found"));
     }
 
     #[test]
-    fn test_dedup_indices_removes_duplicates_preserving_order() {
-        assert_eq!(dedup_indices(&[0, 2, 1, 2, 3, 0]), vec![0, 2, 1, 3]);
+    fn test_has_duplicate_indices_detects_duplicate_indices() {
+        assert!(has_duplicate_indices(&[0, 2, 1, 2, 3]));
     }
 
     #[test]
-    fn test_dedup_indices_handles_empty_input() {
+    fn test_has_duplicate_indices_allows_unique_and_empty_input() {
         let empty: Vec<usize> = Vec::new();
-        assert!(dedup_indices(&empty).is_empty());
+        assert!(!has_duplicate_indices(&empty));
+        assert!(!has_duplicate_indices(&[0, 2, 1, 3]));
     }
 }

rustfs/src/admin/handlers/rebalance.rs

@@ -13,7 +13,8 @@
 // limitations under the License.
 
 use crate::admin::handlers::storage_compat::{
-    DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, StorageError, get_global_notification_sys,
+    DiskStat, ECStore, NotificationSys, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStopPropagationRecord,
+    StorageError, decode_rebalance_stop_propagation_record, get_global_notification_sys,
 };
 use crate::{
     admin::{
@@ -24,25 +25,154 @@ use crate::{
     auth::{check_key_valid, get_session_token},
     server::{ADMIN_PREFIX, RemoteAddr},
 };
-use http::{HeaderMap, HeaderValue, StatusCode};
+use http::{HeaderMap, HeaderValue, StatusCode, Uri};
 use hyper::Method;
 use matchit::Params;
 use rustfs_policy::policy::action::{Action, AdminAction};
 use rustfs_storage_api::{BucketOperations, BucketOptions, StorageAdminApi};
+use rustfs_utils::{
+    MaskedAccessKey,
+    http::{AMZ_REQUEST_ID, REQUEST_ID_HEADER},
+};
 use s3s::{
     Body, S3Request, S3Response, S3Result,
     header::{CONTENT_LENGTH, CONTENT_TYPE},
     s3_error,
 };
 use serde::{Deserialize, Serialize};
-use std::time::Duration;
+use std::{sync::Arc, time::Duration};
 use time::OffsetDateTime;
-use tracing::info;
+use tracing::{error, info, warn};
 
 const LOG_COMPONENT_ADMIN: &str = "admin";
 const LOG_SUBSYSTEM_REBALANCE: &str = "rebalance";
 const EVENT_ADMIN_REBALANCE_STATE: &str = "admin_rebalance_state";
 
+fn admin_request_id(headers: &HeaderMap) -> Option<&str> {
+    headers
+        .get(REQUEST_ID_HEADER)
+        .or_else(|| headers.get(AMZ_REQUEST_ID))
+        .and_then(|value| value.to_str().ok())
+}
+
+fn admin_remote_addr(req: &S3Request<Body>) -> Option<String> {
+    req.extensions
+        .get::<Option<RemoteAddr>>()
+        .and_then(|opt| opt.map(|addr| addr.0.to_string()))
+}
+
+fn log_rebalance_request_rejected(action: &str, reason: &str, request_id: &str, actor: &str, remote_addr: &str) {
+    warn!(
+        event = EVENT_ADMIN_REBALANCE_STATE,
+        component = LOG_COMPONENT_ADMIN,
+        subsystem = LOG_SUBSYSTEM_REBALANCE,
+        action,
+        result = "rejected",
+        reason,
+        request_id = %request_id,
+        actor = %actor,
+        remote_addr = %remote_addr,
+        "admin rebalance state"
+    );
+}
+
+fn rebalance_query_present(uri: &Uri) -> bool {
+    uri.query().is_some_and(|query| !query.is_empty())
+}
+
+fn rollback_result_label(result: &Result<(), String>) -> &'static str {
+    match result {
+        Ok(_) => "rollback_success",
+        Err(err) if err.contains("peer") => "rollback_partial",
+        Err(_) => "rollback_failed",
+    }
+}
+
+fn rebalance_start_rollback_error(start_err: &str, rollback_result: &Result<(), String>) -> String {
+    match rollback_result {
+        Ok(_) => format!("failed to propagate rebalance start: {start_err}; rollback result: rollback_success"),
+        Err(err) => format!(
+            "failed to propagate rebalance start: {start_err}; rollback result: {}; rollback error: {err}",
+            rollback_result_label(rollback_result)
+        ),
+    }
+}
+
+fn rebalance_rollback_stop_failure_message(rebalance_id: &str, failures: &[String]) -> String {
+    format!("cluster stop_rebalance rollback for {rebalance_id} partial: {}", failures.join("; "))
+}
+
+fn rebalance_rollback_terminal_reload_failure_message(rebalance_id: &str, failures: &[String]) -> String {
+    format!(
+        "cluster terminal rebalance reload rollback for {rebalance_id} partial: {}",
+        failures.join("; ")
+    )
+}
+
+fn rebalance_rollback_failure_message(
+    rebalance_id: &str,
+    stop_failures: &[String],
+    terminal_reload_failures: &[String],
+) -> String {
+    let mut failures = Vec::new();
+    if !stop_failures.is_empty() {
+        failures.push(rebalance_rollback_stop_failure_message(rebalance_id, stop_failures));
+    }
+    if !terminal_reload_failures.is_empty() {
+        failures.push(rebalance_rollback_terminal_reload_failure_message(rebalance_id, terminal_reload_failures));
+    }
+    failures.join("; ")
+}
+
+async fn rollback_cluster_rebalance_start(
+    store: &Arc<ECStore>,
+    notification_sys: Option<&NotificationSys>,
+    rebalance_id: &str,
+) -> Result<(), String> {
+    let stop_attempt_at = OffsetDateTime::now_utc();
+    if let Some(notification_sys) = notification_sys {
+        let stop_failures = notification_sys
+            .stop_rebalance_failures(Some(rebalance_id))
+            .await
+            .map_err(|err| format!("cluster stop_rebalance rollback for {rebalance_id} failed: {err}"))?;
+        let terminal_reload_attempt_at = OffsetDateTime::now_utc();
+        let terminal_reload_failures = match notification_sys.load_rebalance_meta_failures(false).await {
+            Ok(failures) => failures,
+            Err(err) => vec![format!("terminal rebalance reload rollback for {rebalance_id} failed: {err}")],
+        };
+        if !stop_failures.is_empty() || !terminal_reload_failures.is_empty() {
+            let record = RebalanceStopPropagationRecord {
+                stop_attempt_at: Some(stop_attempt_at),
+                stop_failures: stop_failures.clone(),
+                terminal_reload_attempt_at: Some(terminal_reload_attempt_at),
+                terminal_reload_failures: terminal_reload_failures.clone(),
+            };
+            store.record_rebalance_stop_propagation(record).await.map_err(|err| {
+                format!(
+                    "cluster rebalance rollback for {rebalance_id} partial; failed to persist stop propagation: {err}; {}",
+                    rebalance_rollback_failure_message(rebalance_id, &stop_failures, &terminal_reload_failures)
+                )
+            })?;
+            return Err(rebalance_rollback_failure_message(
+                rebalance_id,
+                &stop_failures,
+                &terminal_reload_failures,
+            ));
+        }
+        return Ok(());
+    }
+
+    store
+        .stop_rebalance_for_id(Some(rebalance_id))
+        .await
+        .map_err(|err| format!("local stop_rebalance rollback for {rebalance_id} failed: {err}"))?;
+    store
+        .save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt)
+        .await
+        .map_err(|err| format!("local rollback stop metadata save for {rebalance_id} failed: {err}"))?;
+    Ok(())
+}
+
 pub fn register_rebalance_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
     r.insert(
         Method::POST,
@@ -96,6 +226,8 @@ pub struct RebalancePoolStatus {
     pub id: usize, // Pool index (zero-based)
     #[serde(rename = "status")]
     pub status: String, // Active if rebalance is running, empty otherwise
+    #[serde(rename = "stopping")]
+    pub stopping: bool, // Stop requested but worker terminal acknowledgement not yet persisted
     #[serde(rename = "used")]
     pub used: f64, // Fraction of used space in range 0.0..=1.0
     #[serde(rename = "lastError")]
@@ -106,6 +238,20 @@ pub struct RebalancePoolStatus {
     pub progress: Option<RebalPoolProgress>, // None when rebalance is not running
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct RebalanceStopPropagationStatus {
+    #[serde(rename = "lastAttemptAt", with = "offsetdatetime_rfc3339")]
+    pub last_attempt_at: Option<OffsetDateTime>,
+    #[serde(rename = "failedPeers")]
+    pub failed_peers: Vec<String>,
+    #[serde(rename = "terminalReloadAttemptAt", with = "offsetdatetime_rfc3339")]
+    pub terminal_reload_attempt_at: Option<OffsetDateTime>,
+    #[serde(rename = "terminalReloadFailedPeers")]
+    pub terminal_reload_failed_peers: Vec<String>,
+    #[serde(rename = "pendingTerminalReload")]
+    pub pending_terminal_reload: bool,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct RebalanceAdminStatus {
     pub id: String, // Identifies the ongoing rebalance operation by a UUID
@@ -113,6 +259,8 @@ pub struct RebalanceAdminStatus {
     pub pools: Vec<RebalancePoolStatus>, // Contains all pools, including inactive
     #[serde(rename = "stoppedAt", with = "offsetdatetime_rfc3339")]
     pub stopped_at: Option<OffsetDateTime>, // Optional timestamp when rebalance was stopped
+    #[serde(rename = "stopPropagation")]
+    pub stop_propagation: RebalanceStopPropagationStatus,
 }
 
 fn calculate_rebalance_progress(
@@ -201,6 +349,7 @@ fn build_rebalance_pool_statuses(
             let mut status = RebalancePoolStatus {
                 id: i,
                 status: ps.info.status.to_string(),
+                stopping: ps.info.stopping,
                 used: rebalance_pool_used(disk_stats, i),
                 last_error: ps.info.last_error.clone(),
                 cleanup_warnings: ps.cleanup_warnings.clone(),
@@ -216,18 +365,58 @@ fn build_rebalance_pool_statuses(
         .collect()
 }
 
+fn build_rebalance_stop_propagation_status(meta: &RebalanceMeta) -> RebalanceStopPropagationStatus {
+    let record = meta
+        .pool_stats
+        .iter()
+        .filter_map(|pool_stat| pool_stat.info.last_error.as_deref())
+        .find_map(decode_rebalance_stop_propagation_record);
+
+    if let Some(record) = record {
+        let last_attempt_at = record.stop_attempt_at.or(meta.stopped_at);
+        let terminal_reload_attempt_at = record.terminal_reload_attempt_at;
+        return RebalanceStopPropagationStatus {
+            pending_terminal_reload: last_attempt_at.is_some() && terminal_reload_attempt_at.is_none(),
+            last_attempt_at,
+            failed_peers: record.stop_failures,
+            terminal_reload_attempt_at,
+            terminal_reload_failed_peers: record.terminal_reload_failures,
+        };
+    }
+
+    RebalanceStopPropagationStatus {
+        last_attempt_at: meta.stopped_at,
+        pending_terminal_reload: false,
+        ..Default::default()
+    }
+}
+
+fn build_rebalance_admin_status(now: OffsetDateTime, disk_stats: &[DiskStat], meta: &RebalanceMeta) -> RebalanceAdminStatus {
+    let stop_time = meta.stopped_at;
+    RebalanceAdminStatus {
+        id: meta.id.clone(),
+        stopped_at: meta.stopped_at,
+        pools: build_rebalance_pool_statuses(now, stop_time, meta.percent_free_goal, &meta.pool_stats, disk_stats),
+        stop_propagation: build_rebalance_stop_propagation_status(meta),
+    }
+}
+
 pub struct RebalanceStart {}
 
 #[async_trait::async_trait]
 impl Operation for RebalanceStart {
     #[tracing::instrument(skip_all)]
     async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
+        let remote_addr = admin_remote_addr(&req).unwrap_or_default();
         info!(
             event = EVENT_ADMIN_REBALANCE_STATE,
             component = LOG_COMPONENT_ADMIN,
             subsystem = LOG_SUBSYSTEM_REBALANCE,
             action = "start",
             state = "requested",
+            request_id = %request_id,
+            remote_addr = %remote_addr,
             "admin rebalance state"
         );
 
@@ -237,6 +426,7 @@ impl Operation for RebalanceStart {
 
         let (cred, owner) =
             check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+        let actor = MaskedAccessKey(&input_cred.access_key).to_string();
 
         validate_admin_request(
             &req.headers,
@@ -248,19 +438,27 @@ impl Operation for RebalanceStart {
         )
         .await?;
 
+        if rebalance_query_present(&req.uri) {
+            log_rebalance_request_rejected("start", "invalid_query_parameters", &request_id, &actor, &remote_addr);
+            return Err(s3_error!(InvalidArgument, "rebalance start does not accept query parameters"));
+        }
+
         let Some(store) = resolve_object_store_handle() else {
             return Err(s3_error!(InternalError, "object layer is not initialized"));
         };
 
         if store.pools.len() == 1 {
+            log_rebalance_request_rejected("start", "single_pool_not_supported", &request_id, &actor, &remote_addr);
             return Err(s3_error!(NotImplemented));
         }
 
         if store.is_decommission_running().await {
+            log_rebalance_request_rejected("start", "decommission_in_progress", &request_id, &actor, &remote_addr);
             return Err(s3_error!(InvalidRequest, "cannot start rebalance while decommission is in progress"));
         }
 
         if store.is_rebalance_conflicting_with_decommission().await {
+            log_rebalance_request_rejected("start", "rebalance_already_in_progress", &request_id, &actor, &remote_addr);
             return Err(s3_error!(OperationAborted, "rebalance is already in progress"));
         }
 
@@ -271,24 +469,30 @@ impl Operation for RebalanceStart {
 
         let buckets: Vec<String> = bucket_infos.into_iter().map(|bucket| bucket.name).collect();
 
-        let id = match store.init_rebalance_meta(buckets).await {
+        let id = match store.init_and_start_rebalance(buckets).await {
             Ok(id) => id,
+            Err(StorageError::DecommissionAlreadyRunning) => {
+                log_rebalance_request_rejected("start", "decommission_in_progress", &request_id, &actor, &remote_addr);
+                return Err(s3_error!(InvalidRequest, "cannot start rebalance while decommission is in progress"));
+            }
+            Err(StorageError::RebalanceAlreadyRunning) => {
+                log_rebalance_request_rejected("start", "rebalance_already_in_progress", &request_id, &actor, &remote_addr);
+                return Err(s3_error!(OperationAborted, "rebalance is already in progress"));
+            }
             Err(e) => {
-                return Err(s3_error!(InternalError, "failed to initialize rebalance metadata: {}", e));
+                return Err(s3_error!(InternalError, "failed to start rebalance: {}", e));
             }
         };
 
-        store
-            .start_rebalance()
-            .await
-            .map_err(|e| s3_error!(InternalError, "failed to start rebalance: {}", e))?;
-
         info!(
             event = EVENT_ADMIN_REBALANCE_STATE,
             component = LOG_COMPONENT_ADMIN,
             subsystem = LOG_SUBSYSTEM_REBALANCE,
             action = "start",
             state = "started",
+            request_id = %request_id,
+            actor = %actor,
+            remote_addr = %remote_addr,
             rebalance_id = %id,
             "admin rebalance state"
         );
@@ -299,18 +503,65 @@ impl Operation for RebalanceStart {
                 subsystem = LOG_SUBSYSTEM_REBALANCE,
                 action = "start",
                 state = "propagation_started",
+                request_id = %request_id,
+                actor = %actor,
+                remote_addr = %remote_addr,
+                rebalance_id = %id,
                 "admin rebalance state"
             );
             if let Err(err) = notification_sys.load_rebalance_meta(true).await {
-                info!(
+                error!(
                     event = EVENT_ADMIN_REBALANCE_STATE,
                     component = LOG_COMPONENT_ADMIN,
                     subsystem = LOG_SUBSYSTEM_REBALANCE,
                     action = "start",
                     result = "propagation_failed",
+                    request_id = %request_id,
+                    actor = %actor,
+                    remote_addr = %remote_addr,
+                    rebalance_id = %id,
                     error = %err,
                     "admin rebalance state"
                 );
+
+                let start_err = err.to_string();
+                let rollback_result = rollback_cluster_rebalance_start(&store, Some(notification_sys), &id).await;
+                let rollback_label = rollback_result_label(&rollback_result);
+                match &rollback_result {
+                    Ok(_) => info!(
+                        event = EVENT_ADMIN_REBALANCE_STATE,
+                        component = LOG_COMPONENT_ADMIN,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        action = "start",
+                        result = rollback_label,
+                        request_id = %request_id,
+                        actor = %actor,
+                        remote_addr = %remote_addr,
+                        rebalance_id = %id,
+                        propagation_error = %start_err,
+                        "admin rebalance state"
+                    ),
+                    Err(rollback_err) => error!(
+                        event = EVENT_ADMIN_REBALANCE_STATE,
+                        component = LOG_COMPONENT_ADMIN,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        action = "start",
+                        result = rollback_label,
+                        request_id = %request_id,
+                        actor = %actor,
+                        remote_addr = %remote_addr,
+                        rebalance_id = %id,
+                        propagation_error = %start_err,
+                        rollback_error = %rollback_err,
+                        "admin rebalance state"
+                    ),
+                }
+
+                return Err(s3_error!(
+                    InternalError,
+                    "{}",
+                    rebalance_start_rollback_error(&start_err, &rollback_result)
+                ));
             }
             info!(
                 event = EVENT_ADMIN_REBALANCE_STATE,
@@ -318,6 +569,11 @@ impl Operation for RebalanceStart {
                 subsystem = LOG_SUBSYSTEM_REBALANCE,
                 action = "start",
                 state = "propagation_completed",
+                result = "success",
+                request_id = %request_id,
+                actor = %actor,
+                remote_addr = %remote_addr,
+                rebalance_id = %id,
                 "admin rebalance state"
             );
         }
@@ -340,12 +596,16 @@ pub struct RebalanceStatus {}
 impl Operation for RebalanceStatus {
     #[tracing::instrument(skip_all)]
     async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
+        let remote_addr = admin_remote_addr(&req).unwrap_or_default();
         info!(
             event = EVENT_ADMIN_REBALANCE_STATE,
             component = LOG_COMPONENT_ADMIN,
             subsystem = LOG_SUBSYSTEM_REBALANCE,
             action = "status",
             state = "requested",
+            request_id = %request_id,
+            remote_addr = %remote_addr,
             "admin rebalance state"
         );
 
@@ -355,6 +615,7 @@ impl Operation for RebalanceStatus {
 
         let (cred, owner) =
             check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+        let actor = MaskedAccessKey(&input_cred.access_key).to_string();
 
         validate_admin_request(
             &req.headers,
@@ -383,6 +644,7 @@ impl Operation for RebalanceStatus {
         let mut meta = RebalanceMeta::new();
         if let Err(err) = meta.load(first_pool).await {
             if err == StorageError::ConfigNotFound {
+                log_rebalance_request_rejected("status", "rebalance_not_started", &request_id, &actor, &remote_addr);
                 return Err(s3_error!(NoSuchResource, "pool rebalance is not started"));
             }
 
@@ -401,16 +663,25 @@ impl Operation for RebalanceStatus {
             disk_stats[disk.pool_index as usize].total_space += disk.total_space;
         }
 
-        let stop_time = meta.stopped_at;
         let now = OffsetDateTime::now_utc();
-        let admin_status = RebalanceAdminStatus {
-            id: meta.id.clone(),
-            stopped_at: meta.stopped_at,
-            pools: build_rebalance_pool_statuses(now, stop_time, meta.percent_free_goal, &meta.pool_stats, &disk_stats),
-        };
+        let admin_status = build_rebalance_admin_status(now, &disk_stats, &meta);
 
         let data = serde_json::to_string(&admin_status)
             .map_err(|e| s3_error!(InternalError, "failed to serialize rebalance status response: {}", e))?;
+        info!(
+            event = EVENT_ADMIN_REBALANCE_STATE,
+            component = LOG_COMPONENT_ADMIN,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            action = "status",
+            result = "success",
+            request_id = %request_id,
+            actor = %actor,
+            remote_addr = %remote_addr,
+            rebalance_id = %admin_status.id,
+            pool_count = admin_status.pools.len(),
+            cleanup_warning_count = admin_status.pools.iter().map(|pool| pool.cleanup_warnings.count).sum::<u64>(),
+            "admin rebalance state"
+        );
         let mut header = HeaderMap::new();
         header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
 
@@ -425,12 +696,16 @@ pub struct RebalanceStop {}
 impl Operation for RebalanceStop {
     #[tracing::instrument(skip_all)]
     async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
+        let remote_addr = admin_remote_addr(&req).unwrap_or_default();
         info!(
             event = EVENT_ADMIN_REBALANCE_STATE,
             component = LOG_COMPONENT_ADMIN,
             subsystem = LOG_SUBSYSTEM_REBALANCE,
             action = "stop",
             state = "requested",
+            request_id = %request_id,
+            remote_addr = %remote_addr,
             "admin rebalance state"
         );
 
@@ -440,6 +715,7 @@ impl Operation for RebalanceStop {
 
         let (cred, owner) =
             check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+        let actor = MaskedAccessKey(&input_cred.access_key).to_string();
 
         validate_admin_request(
             &req.headers,
@@ -451,22 +727,37 @@ impl Operation for RebalanceStop {
         )
         .await?;
 
+        if rebalance_query_present(&req.uri) {
+            log_rebalance_request_rejected("stop", "invalid_query_parameters", &request_id, &actor, &remote_addr);
+            return Err(s3_error!(InvalidArgument, "rebalance stop does not accept query parameters"));
+        }
+
         let Some(store) = resolve_object_store_handle() else {
             return Err(s3_error!(InternalError, "object layer is not initialized"));
         };
 
+        store
+            .load_rebalance_meta()
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to load rebalance metadata before stop: {}", e))?;
+        let expected_rebalance_id = store.current_rebalance_id().await;
+
         if !store.is_rebalance_conflicting_with_decommission().await {
+            log_rebalance_request_rejected("stop", "rebalance_not_started", &request_id, &actor, &remote_addr);
             return Err(s3_error!(NoSuchResource, "pool rebalance is not started"));
         }
 
-        if let Some(notification_sys) = get_global_notification_sys() {
-            notification_sys
-                .stop_rebalance()
+        let notification_sys = get_global_notification_sys();
+        let stop_attempt_at = OffsetDateTime::now_utc();
+        let mut stop_failures = Vec::new();
+        if let Some(notification_sys) = notification_sys {
+            stop_failures = notification_sys
+                .stop_rebalance_failures(expected_rebalance_id.as_deref())
                 .await
                 .map_err(|e| s3_error!(InternalError, "failed to stop rebalance via notification system: {}", e))?;
         } else {
             store
-                .stop_rebalance()
+                .stop_rebalance_for_id(expected_rebalance_id.as_deref())
                 .await
                 .map_err(|e| s3_error!(InternalError, "failed to stop rebalance: {}", e))?;
 
@@ -482,36 +773,85 @@ impl Operation for RebalanceStop {
             subsystem = LOG_SUBSYSTEM_REBALANCE,
             action = "stop",
             state = "local_stop_persisted",
+            result = "success",
+            request_id = %request_id,
+            actor = %actor,
+            remote_addr = %remote_addr,
             "admin rebalance state"
         );
-        if let Some(notification_sys) = get_global_notification_sys() {
+
+        let mut terminal_reload_attempt_at = None;
+        let mut terminal_reload_failures = Vec::new();
+        if let Some(notification_sys) = notification_sys {
             info!(
                 event = EVENT_ADMIN_REBALANCE_STATE,
                 component = LOG_COMPONENT_ADMIN,
                 subsystem = LOG_SUBSYSTEM_REBALANCE,
                 action = "stop",
                 state = "propagation_started",
+                request_id = %request_id,
+                actor = %actor,
+                remote_addr = %remote_addr,
                 "admin rebalance state"
             );
-            if let Err(err) = notification_sys.load_rebalance_meta(false).await {
-                info!(
-                    event = EVENT_ADMIN_REBALANCE_STATE,
-                    component = LOG_COMPONENT_ADMIN,
-                    subsystem = LOG_SUBSYSTEM_REBALANCE,
-                    action = "stop",
-                    result = "propagation_failed",
-                    error = %err,
-                    "admin rebalance state"
-                );
+            terminal_reload_attempt_at = Some(OffsetDateTime::now_utc());
+            match notification_sys.load_rebalance_meta_failures(false).await {
+                Ok(failures) => {
+                    terminal_reload_failures = failures;
+                    if terminal_reload_failures.is_empty() {
+                        info!(
+                            event = EVENT_ADMIN_REBALANCE_STATE,
+                            component = LOG_COMPONENT_ADMIN,
+                            subsystem = LOG_SUBSYSTEM_REBALANCE,
+                            action = "stop",
+                            state = "propagation_completed",
+                            result = "success",
+                            request_id = %request_id,
+                            actor = %actor,
+                            remote_addr = %remote_addr,
+                            "admin rebalance state"
+                        );
+                    }
+                }
+                Err(err) => {
+                    terminal_reload_failures.push(format!("terminal rebalance reload propagation failed: {err}"));
+                }
             }
-            info!(
+        }
+
+        if !stop_failures.is_empty() || !terminal_reload_failures.is_empty() {
+            let record = RebalanceStopPropagationRecord {
+                stop_attempt_at: Some(stop_attempt_at),
+                stop_failures: stop_failures.clone(),
+                terminal_reload_attempt_at,
+                terminal_reload_failures: terminal_reload_failures.clone(),
+            };
+            store
+                .record_rebalance_stop_propagation(record)
+                .await
+                .map_err(|e| s3_error!(InternalError, "failed to persist rebalance stop propagation metadata: {}", e))?;
+
+            error!(
                 event = EVENT_ADMIN_REBALANCE_STATE,
                 component = LOG_COMPONENT_ADMIN,
                 subsystem = LOG_SUBSYSTEM_REBALANCE,
                 action = "stop",
-                state = "propagation_completed",
+                result = "propagation_failed",
+                request_id = %request_id,
+                actor = %actor,
+                remote_addr = %remote_addr,
+                stop_failure_count = stop_failures.len(),
+                terminal_reload_failure_count = terminal_reload_failures.len(),
                 "admin rebalance state"
             );
+            let mut failures = Vec::new();
+            failures.extend(stop_failures);
+            failures.extend(terminal_reload_failures);
+            return Err(s3_error!(
+                InternalError,
+                "rebalance stop propagation incomplete after local stop was persisted: {}",
+                failures.join("; ")
+            ));
         }
 
         let mut header = HeaderMap::new();
@@ -558,11 +898,14 @@ mod rebalance_handler_tests {
     use super::build_rebalance_pool_progress;
     use super::calculate_rebalance_progress;
     use super::{
-        RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, build_rebalance_pool_statuses, rebalance_pool_used,
-        rebalance_remaining_buckets, rebalance_used_pct,
+        RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, RebalanceStopPropagationStatus,
+        build_rebalance_admin_status, build_rebalance_pool_statuses, build_rebalance_stop_propagation_status,
+        rebalance_pool_used, rebalance_query_present, rebalance_remaining_buckets, rebalance_rollback_failure_message,
+        rebalance_rollback_stop_failure_message, rebalance_start_rollback_error, rebalance_used_pct, rollback_result_label,
     };
     use crate::admin::handlers::storage_compat::{
-        DiskStat, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceStats,
+        DiskStat, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta,
+        RebalanceStats, RebalanceStopPropagationRecord, encode_rebalance_stop_propagation_record,
     };
     use time::OffsetDateTime;
 
@@ -577,6 +920,76 @@ mod rebalance_handler_tests {
         assert_eq!(eta, 50);
     }
 
+    #[test]
+    fn test_rebalance_start_rollback_error_reports_successful_rollback() {
+        let rollback_result = Ok(());
+        let message = rebalance_start_rollback_error("peer a failed", &rollback_result);
+
+        assert!(message.contains("failed to propagate rebalance start: peer a failed"));
+        assert!(message.contains("rollback result: rollback_success"));
+    }
+
+    #[test]
+    fn test_rebalance_start_rollback_error_reports_partial_peer_rollback() {
+        let rollback_result = Err("peer b stop_rebalance failed: timeout".to_string());
+        let message = rebalance_start_rollback_error("peer a failed", &rollback_result);
+
+        assert_eq!(rollback_result_label(&rollback_result), "rollback_partial");
+        assert!(message.contains("failed to propagate rebalance start: peer a failed"));
+        assert!(message.contains("rollback result: rollback_partial"));
+        assert!(message.contains("rollback error: peer b stop_rebalance failed: timeout"));
+    }
+
+    #[test]
+    fn test_rebalance_rollback_stop_failure_message_lists_failures() {
+        let failures = vec![
+            "peer a stop_rebalance failed: timeout".to_string(),
+            "peer b stop_rebalance failed: unavailable".to_string(),
+        ];
+
+        let message = rebalance_rollback_stop_failure_message("rebalance-id", &failures);
+
+        assert!(message.contains("cluster stop_rebalance rollback for rebalance-id partial"));
+        assert!(message.contains("peer a stop_rebalance failed: timeout"));
+        assert!(message.contains("peer b stop_rebalance failed: unavailable"));
+    }
+
+    #[test]
+    fn test_rebalance_rollback_failure_message_lists_stop_and_terminal_reload_failures() {
+        let stop_failures = vec!["peer a stop_rebalance failed: timeout".to_string()];
+        let terminal_reload_failures = vec!["peer b load_rebalance_meta(start=false) failed: unavailable".to_string()];
+
+        let message = rebalance_rollback_failure_message("rebalance-id", &stop_failures, &terminal_reload_failures);
+
+        assert!(message.contains("cluster stop_rebalance rollback for rebalance-id partial"));
+        assert!(message.contains("peer a stop_rebalance failed: timeout"));
+        assert!(message.contains("cluster terminal rebalance reload rollback for rebalance-id partial"));
+        assert!(message.contains("peer b load_rebalance_meta(start=false) failed: unavailable"));
+    }
+
+    #[test]
+    fn test_rebalance_query_present_detects_non_empty_query() {
+        let uri = "/rustfs/admin/v3/rebalance/start?pool=0".parse().unwrap();
+
+        assert!(rebalance_query_present(&uri));
+    }
+
+    #[test]
+    fn test_rebalance_query_present_allows_no_or_empty_query() {
+        let no_query = "/rustfs/admin/v3/rebalance/start".parse().unwrap();
+        let empty_query = "/rustfs/admin/v3/rebalance/start?".parse().unwrap();
+
+        assert!(!rebalance_query_present(&no_query));
+        assert!(!rebalance_query_present(&empty_query));
+    }
+
+    #[test]
+    fn test_rollback_result_label_reports_non_peer_failure_as_failed() {
+        let rollback_result = Err("local stop_rebalance failed: disk error".to_string());
+
+        assert_eq!(rollback_result_label(&rollback_result), "rollback_failed");
+    }
+
     #[test]
     fn test_calculate_rebalance_progress_stopped_by_end_time() {
         let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
@@ -861,6 +1274,26 @@ mod rebalance_handler_tests {
         assert!(statuses[1].progress.is_some());
     }
 
+    #[test]
+    fn test_build_rebalance_pool_statuses_reports_stopping() {
+        let pool_stats = vec![RebalanceStats {
+            participating: true,
+            info: RebalanceInfo {
+                status: RebalStatus::Started,
+                stopping: true,
+                start_time: Some(OffsetDateTime::from_unix_timestamp(2_000).unwrap()),
+                ..Default::default()
+            },
+            ..Default::default()
+        }];
+
+        let statuses =
+            build_rebalance_pool_statuses(OffsetDateTime::from_unix_timestamp(2_010).unwrap(), None, 0.3, &pool_stats, &[]);
+
+        assert_eq!(statuses[0].status, "Started");
+        assert!(statuses[0].stopping);
+    }
+
     #[test]
     fn test_build_rebalance_pool_statuses_empty_inputs() {
         let statuses = build_rebalance_pool_statuses(
@@ -882,9 +1315,11 @@ mod rebalance_handler_tests {
         let status = RebalanceAdminStatus {
             id: "id-1".to_string(),
             stopped_at: None,
+            stop_propagation: RebalanceStopPropagationStatus::default(),
             pools: vec![RebalancePoolStatus {
                 id: 0,
                 status: "Started".to_string(),
+                stopping: true,
                 used: 0.5,
                 last_error: Some("temporary error".to_string()),
                 cleanup_warnings: RebalanceCleanupWarnings {
@@ -893,6 +1328,12 @@ mod rebalance_handler_tests {
                     last_bucket: Some("bucket-a".to_string()),
                     last_object: Some("obj".to_string()),
                     last_at: Some(OffsetDateTime::from_unix_timestamp(1_001).unwrap()),
+                    entries: vec![RebalanceCleanupWarningEntry {
+                        bucket: "bucket-a".to_string(),
+                        object: "obj".to_string(),
+                        message: "cleanup warning".to_string(),
+                        timestamp: Some(OffsetDateTime::from_unix_timestamp(1_001).unwrap()),
+                    }],
                 },
                 progress: Some(RebalPoolProgress {
                     num_objects: 3,
@@ -911,8 +1352,100 @@ mod rebalance_handler_tests {
         assert!(json.contains("\"remainingBuckets\""));
         assert!(json.contains("\"lastError\""));
         assert!(json.contains("\"cleanupWarnings\""));
+        assert!(json.contains("\"stopping\":true"));
         assert!(json.contains("\"lastMsg\":\"cleanup warning\""));
+        assert!(json.contains("\"entries\""));
+        assert!(json.contains("\"message\":\"cleanup warning\""));
         assert!(json.contains("\"stoppedAt\":null"));
+        assert!(json.contains("\"stopPropagation\""));
+        assert!(json.contains("\"pendingTerminalReload\":false"));
+    }
+
+    #[test]
+    fn test_build_rebalance_admin_status_is_stable_for_same_persisted_meta() {
+        let started = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
+        let disk_stats = vec![
+            DiskStat {
+                total_space: 2_000,
+                available_space: 1_000,
+            },
+            DiskStat {
+                total_space: 2_000,
+                available_space: 1_500,
+            },
+        ];
+        let meta = RebalanceMeta {
+            id: "rebalance-id".to_string(),
+            percent_free_goal: 0.6,
+            pool_stats: vec![
+                RebalanceStats {
+                    participating: true,
+                    init_capacity: 2_000,
+                    init_free_space: 500,
+                    buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
+                    rebalanced_buckets: vec!["bucket-a".to_string()],
+                    bucket: "bucket-b".to_string(),
+                    object: "object.txt".to_string(),
+                    num_objects: 10,
+                    num_versions: 12,
+                    bytes: 300,
+                    info: RebalanceInfo {
+                        status: RebalStatus::Started,
+                        start_time: Some(started),
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                },
+                RebalanceStats {
+                    participating: false,
+                    info: RebalanceInfo {
+                        status: RebalStatus::Completed,
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                },
+            ],
+            ..Default::default()
+        };
+
+        let first = build_rebalance_admin_status(OffsetDateTime::from_unix_timestamp(1_030).unwrap(), &disk_stats, &meta);
+        let second = build_rebalance_admin_status(OffsetDateTime::from_unix_timestamp(1_060).unwrap(), &disk_stats, &meta);
+
+        assert_eq!(first.id, second.id);
+        assert_eq!(first.stopped_at, second.stopped_at);
+        assert_eq!(first.stop_propagation.failed_peers, second.stop_propagation.failed_peers);
+        assert_eq!(first.pools.len(), second.pools.len());
+        for (left, right) in first.pools.iter().zip(second.pools.iter()) {
+            assert_eq!(left.id, right.id);
+            assert_eq!(left.status, right.status);
+            assert_eq!(left.stopping, right.stopping);
+            assert_eq!(left.used, right.used);
+            assert_eq!(left.last_error, right.last_error);
+            assert_eq!(left.cleanup_warnings.count, right.cleanup_warnings.count);
+            assert_eq!(
+                left.progress.as_ref().map(|progress| (
+                    progress.num_objects,
+                    progress.num_versions,
+                    progress.bytes,
+                    progress.remaining_buckets,
+                    progress.bucket.as_str(),
+                    progress.object.as_str()
+                )),
+                right.progress.as_ref().map(|progress| (
+                    progress.num_objects,
+                    progress.num_versions,
+                    progress.bytes,
+                    progress.remaining_buckets,
+                    progress.bucket.as_str(),
+                    progress.object.as_str()
+                ))
+            );
+        }
+
+        assert_ne!(
+            first.pools[0].progress.as_ref().map(|progress| progress.elapsed),
+            second.pools[0].progress.as_ref().map(|progress| progress.elapsed)
+        );
     }
 
     #[test]
@@ -921,9 +1454,14 @@ mod rebalance_handler_tests {
         let status = RebalanceAdminStatus {
             id: "id-2".to_string(),
             stopped_at: Some(stopped),
+            stop_propagation: RebalanceStopPropagationStatus {
+                last_attempt_at: Some(stopped),
+                ..Default::default()
+            },
             pools: vec![RebalancePoolStatus {
                 id: 0,
                 status: "Stopped".to_string(),
+                stopping: false,
                 used: 0.3,
                 last_error: None,
                 cleanup_warnings: RebalanceCleanupWarnings::default(),
@@ -934,5 +1472,45 @@ mod rebalance_handler_tests {
         let json = serde_json::to_string(&status).unwrap();
         assert!(json.contains("\"stoppedAt\""));
         assert!(json.contains("1970-01-01T00:16:40Z"));
+        assert!(json.contains("\"lastAttemptAt\":\"1970-01-01T00:16:40Z\""));
+    }
+
+    #[test]
+    fn test_rebalance_status_exposes_stop_propagation_failures() {
+        let stop_attempt = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
+        let reload_attempt = OffsetDateTime::from_unix_timestamp(1_010).unwrap();
+        let encoded_error = encode_rebalance_stop_propagation_record(&RebalanceStopPropagationRecord {
+            stop_attempt_at: Some(stop_attempt),
+            stop_failures: vec!["peer node-a stop_rebalance failed: timeout".to_string()],
+            terminal_reload_attempt_at: Some(reload_attempt),
+            terminal_reload_failures: vec!["peer node-b load_rebalance_meta(start=false) failed: timeout".to_string()],
+        });
+        let meta = RebalanceMeta {
+            stopped_at: Some(stop_attempt),
+            id: "id-3".to_string(),
+            percent_free_goal: 0.3,
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    stopping: true,
+                    last_error: Some(encoded_error),
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+
+        let status = build_rebalance_stop_propagation_status(&meta);
+
+        assert_eq!(status.last_attempt_at, Some(stop_attempt));
+        assert_eq!(status.terminal_reload_attempt_at, Some(reload_attempt));
+        assert!(!status.pending_terminal_reload);
+        assert_eq!(status.failed_peers, vec!["peer node-a stop_rebalance failed: timeout"]);
+        assert_eq!(
+            status.terminal_reload_failed_peers,
+            vec!["peer node-b load_rebalance_meta(start=false) failed: timeout"]
+        );
     }
 }

rustfs/src/admin/handlers/storage_compat.rs

@@ -16,15 +16,19 @@ pub(crate) use super::super::storage_compat::{
     AdminError, AdminReplicationConfigExt, AdminVersioningConfigExt, CollectMetricsOpts, DailyAllTierStats, DiskStat, ECStore,
     ERR_TIER_ALREADY_EXISTS, ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_CONNECT_ERR,
     ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND,
-    EndpointServerPools, Error, MetricType, PeerRestClient, RUSTFS_META_BUCKET, RebalSaveOpt, RebalanceCleanupWarnings,
-    RebalanceMeta, RebalanceStats, STORAGE_CLASS_SUB_SYS, StorageError, TierConfig, TierCreds, TierType, collect_local_metrics,
-    delete_admin_config, get_global_deployment_id, get_global_endpoints_opt, get_global_notification_sys, get_global_region,
-    global_rustfs_port, init_admin_config_defaults, is_reserved_or_invalid_bucket, load_data_usage_from_backend,
-    read_admin_config, read_admin_config_without_migrate, save_admin_config, save_admin_server_config,
+    EndpointServerPools, Error, MetricType, NotificationSys, PeerRestClient, RUSTFS_META_BUCKET, RebalSaveOpt,
+    RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, STORAGE_CLASS_SUB_SYS, StorageError,
+    TierConfig, TierCreds, TierType, collect_local_metrics, decode_rebalance_stop_propagation_record, delete_admin_config,
+    get_global_deployment_id, get_global_endpoints_opt, get_global_notification_sys, get_global_region, global_rustfs_port,
+    init_admin_config_defaults, is_reserved_or_invalid_bucket, load_data_usage_from_backend, read_admin_config,
+    read_admin_config_without_migrate, save_admin_config, save_admin_server_config,
 };
 
 #[cfg(test)]
-pub(crate) use super::super::storage_compat::{Endpoint, Endpoints, PoolEndpoints, RebalStatus, RebalanceInfo};
+pub(crate) use super::super::storage_compat::{
+    Endpoint, Endpoints, PoolEndpoints, RebalStatus, RebalanceCleanupWarningEntry, RebalanceInfo,
+    encode_rebalance_stop_propagation_record,
+};
 
 pub(crate) mod bucket_target_sys {
     pub(crate) use super::super::super::storage_compat::bucket_target_sys::{BucketTargetError, BucketTargetSys};

rustfs/src/admin/handlers/tier.rs

@@ -43,7 +43,7 @@ use s3s::{
     s3_error,
 };
 use serde_urlencoded::from_bytes;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use time::OffsetDateTime;
 use tracing::{debug, warn};
 
@@ -811,19 +811,42 @@ pub struct ClearTierQuery {
     pub force: String,
 }
 
+fn parse_clear_tier_query(uri: &Uri) -> S3Result<ClearTierQuery> {
+    let mut parsed = ClearTierQuery::default();
+    let mut seen = HashSet::with_capacity(2);
+    let Some(query) = uri.query() else {
+        return Ok(parsed);
+    };
+
+    for (key, value) in url::form_urlencoded::parse(query.as_bytes()) {
+        match key.as_ref() {
+            "rand" => {
+                if !seen.insert("rand") {
+                    return Err(s3_error!(InvalidArgument, "duplicate clear-tier query parameter"));
+                }
+                parsed.rand = Some(value.into_owned());
+            }
+            "force" => {
+                if !seen.insert("force") {
+                    return Err(s3_error!(InvalidArgument, "duplicate clear-tier query parameter"));
+                }
+                match value.as_ref() {
+                    "true" | "false" => parsed.force = value.into_owned(),
+                    _ => return Err(s3_error!(InvalidArgument, "invalid force flag")),
+                }
+            }
+            _ => return Err(s3_error!(InvalidArgument, "unknown clear-tier query parameter")),
+        }
+    }
+
+    Ok(parsed)
+}
+
 pub struct ClearTier {}
 #[async_trait::async_trait]
 impl Operation for ClearTier {
     async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let query = {
-            if let Some(query) = req.uri.query() {
-                let input: ClearTierQuery =
-                    from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
-                input
-            } else {
-                ClearTierQuery::default()
-            }
-        };
+        let query = parse_clear_tier_query(&req.uri)?;
 
         let Some(input_cred) = req.credentials else {
             return Err(s3_error!(InvalidRequest, "authentication required"));
@@ -845,7 +868,9 @@ impl Operation for ClearTier {
         let mut force: bool = false;
         let force_str = query.force;
         if !force_str.is_empty() {
-            force = force_str.parse().unwrap();
+            force = force_str
+                .parse()
+                .map_err(|_e| s3_error!(InvalidArgument, "invalid force flag"))?;
         }
 
         let t = OffsetDateTime::now_utc();
@@ -1050,6 +1075,30 @@ mod tests {
         assert_eq!(mapped.message(), Some("tier verification failed. backend unavailable"));
     }
 
+    #[test]
+    fn parse_clear_tier_query_rejects_unknown_duplicate_and_invalid_force() {
+        for raw in [
+            "/rustfs/admin/v3/tier?rand=token&force=yes",
+            "/rustfs/admin/v3/tier?rand=token&rand=other",
+            "/rustfs/admin/v3/tier?rand=token&unexpected=true",
+        ] {
+            let uri: Uri = raw.parse().expect("uri should parse");
+            let err = parse_clear_tier_query(&uri).expect_err("strict clear-tier query should reject malformed input");
+            assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+        }
+    }
+
+    #[test]
+    fn parse_clear_tier_query_accepts_valid_force() {
+        let uri: Uri = "/rustfs/admin/v3/tier?rand=token&force=true"
+            .parse()
+            .expect("uri should parse");
+        let query = parse_clear_tier_query(&uri).expect("valid clear-tier query should parse");
+
+        assert_eq!(query.rand.as_deref(), Some("token"));
+        assert_eq!(query.force, "true");
+    }
+
     fn sample_daily_stats() -> DailyAllTierStats {
         let mut warm = LastDayTierStats::default();
         warm.add_stats(TierStats {

rustfs/src/admin/route_policy.rs

@@ -236,6 +236,7 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
         RouteRiskLevel::High,
     ),
     admin(HttpMethod::Post, "/rustfs/admin/v3/pools/cancel", DECOMMISSION, RouteRiskLevel::High),
+    admin(HttpMethod::Post, "/rustfs/admin/v3/pools/clear", DECOMMISSION, RouteRiskLevel::High),
     admin(HttpMethod::Post, "/rustfs/admin/v3/rebalance/start", REBALANCE, RouteRiskLevel::High),
     admin(HttpMethod::Get, "/rustfs/admin/v3/rebalance/status", REBALANCE, RouteRiskLevel::Sensitive),
     admin(HttpMethod::Post, "/rustfs/admin/v3/rebalance/stop", REBALANCE, RouteRiskLevel::High),

rustfs/src/admin/route_registration_test.rs

@@ -175,6 +175,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
         admin_route(Method::GET, "/v3/pools/status"),
         admin_route(Method::POST, "/v3/pools/decommission"),
         admin_route(Method::POST, "/v3/pools/cancel"),
+        admin_route(Method::POST, "/v3/pools/clear"),
         admin_route(Method::POST, "/v3/rebalance/start"),
         admin_route(Method::GET, "/v3/rebalance/status"),
         admin_route(Method::POST, "/v3/rebalance/stop"),

rustfs/src/admin/storage_compat.rs

@@ -46,6 +46,7 @@ pub(crate) type RebalSaveOpt = ecstore_rebalance::RebalSaveOpt;
 pub(crate) type RebalanceCleanupWarnings = ecstore_rebalance::RebalanceCleanupWarnings;
 pub(crate) type RebalanceMeta = ecstore_rebalance::RebalanceMeta;
 pub(crate) type RebalanceStats = ecstore_rebalance::RebalanceStats;
+pub(crate) type RebalanceStopPropagationRecord = ecstore_rebalance::RebalanceStopPropagationRecord;
 pub(crate) type StorageError = ecstore_error::StorageError;
 pub(crate) type Error = StorageError;
 pub(crate) type Result<T> = core::result::Result<T, Error>;
@@ -62,8 +63,19 @@ pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints;
 #[cfg(test)]
 pub(crate) type RebalStatus = ecstore_rebalance::RebalStatus;
 #[cfg(test)]
+pub(crate) type RebalanceCleanupWarningEntry = ecstore_rebalance::RebalanceCleanupWarningEntry;
+#[cfg(test)]
 pub(crate) type RebalanceInfo = ecstore_rebalance::RebalanceInfo;
 
+pub(crate) fn decode_rebalance_stop_propagation_record(message: &str) -> Option<RebalanceStopPropagationRecord> {
+    ecstore_rebalance::decode_rebalance_stop_propagation_record(message)
+}
+
+#[cfg(test)]
+pub(crate) fn encode_rebalance_stop_propagation_record(record: &RebalanceStopPropagationRecord) -> String {
+    ecstore_rebalance::encode_rebalance_stop_propagation_record(record)
+}
+
 pub(crate) trait AdminReplicationConfigExt {
     fn filter_target_arns(&self, obj: &replication::ObjectOpts) -> Vec<String>;
     fn has_existing_object_replication(&self, arn: &str) -> (bool, bool);

rustfs/src/app/admin_usecase.rs

@@ -56,7 +56,47 @@ pub struct QueryPoolStatusRequest {
 }
 
 #[derive(Debug, Clone, serde::Serialize)]
-pub struct AdminPoolListItem {
+pub struct AdminPoolDecommissionInfo {
+    #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
+    pub start_time: Option<time::OffsetDateTime>,
+    #[serde(rename = "startSize")]
+    pub start_size: usize,
+    #[serde(rename = "totalSize")]
+    pub total_size: usize,
+    #[serde(rename = "currentSize")]
+    pub current_size: usize,
+    #[serde(rename = "complete")]
+    pub complete: bool,
+    #[serde(rename = "failed")]
+    pub failed: bool,
+    #[serde(rename = "canceled")]
+    pub canceled: bool,
+    #[serde(rename = "queued")]
+    pub queued: bool,
+    #[serde(rename = "queuedBuckets")]
+    pub queued_buckets: Vec<String>,
+    #[serde(rename = "decommissionedBuckets")]
+    pub decommissioned_buckets: Vec<String>,
+    #[serde(rename = "bucket")]
+    pub bucket: String,
+    #[serde(rename = "prefix")]
+    pub prefix: String,
+    #[serde(rename = "object")]
+    pub object: String,
+    #[serde(rename = "objectsDecommissioned")]
+    pub items_decommissioned: usize,
+    #[serde(rename = "objectsDecommissionedFailed")]
+    pub items_decommission_failed: usize,
+    #[serde(rename = "bytesDecommissioned")]
+    pub bytes_done: usize,
+    #[serde(rename = "bytesDecommissionedFailed")]
+    pub bytes_failed: usize,
+    #[serde(rename = "waitingReason")]
+    pub waiting_reason: Option<String>,
+}
+
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct AdminPoolStatus {
     #[serde(rename = "id")]
     pub id: usize,
     #[serde(rename = "cmdline")]
@@ -74,9 +114,11 @@ pub struct AdminPoolListItem {
     #[serde(rename = "status")]
     pub status: String,
     #[serde(rename = "decommissionInfo")]
-    pub decommission: Option<PoolDecommissionInfo>,
+    pub decommission: Option<AdminPoolDecommissionInfo>,
 }
 
+pub type AdminPoolListItem = AdminPoolStatus;
+
 #[derive(Clone, Default)]
 pub struct DefaultAdminUsecase {
     context: Option<Arc<AppContext>>,
@@ -87,6 +129,7 @@ impl DefaultAdminUsecase {
     const POOL_STATUS_CANCELED: &'static str = "canceled";
     const POOL_STATUS_COMPLETE: &'static str = "complete";
     const POOL_STATUS_FAILED: &'static str = "failed";
+    const POOL_STATUS_QUEUED: &'static str = "queued";
     const POOL_STATUS_RUNNING: &'static str = "running";
 
     #[cfg(test)]
@@ -240,7 +283,7 @@ impl DefaultAdminUsecase {
         Ok(pool_statuses.into_iter().map(Self::pool_list_item_from_status).collect())
     }
 
-    pub async fn execute_query_pool_status(&self, req: QueryPoolStatusRequest) -> AdminUsecaseResult<PoolStatus> {
+    pub async fn execute_query_pool_status(&self, req: QueryPoolStatusRequest) -> AdminUsecaseResult<AdminPoolStatus> {
         let Some(endpoints) = self.endpoints() else {
             return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
         };
@@ -250,8 +293,7 @@ impl DefaultAdminUsecase {
         }
 
         let has_idx = if req.by_id {
-            let idx = req.pool.parse::<usize>().unwrap_or_default();
-            if idx < endpoints.as_ref().len() { Some(idx) } else { None }
+            Self::parse_pool_idx_by_id(&req.pool, endpoints.as_ref().len())
         } else {
             endpoints.get_pool_idx(&req.pool)
         };
@@ -265,7 +307,11 @@ impl DefaultAdminUsecase {
             return Err(Self::app_error(S3ErrorCode::InternalError, "Not init"));
         };
 
-        store.status(idx).await.map_err(ApiError::from)
+        store
+            .status(idx)
+            .await
+            .map(Self::pool_list_item_from_status)
+            .map_err(ApiError::from)
     }
 
     fn pool_list_item_from_status(status: PoolStatus) -> AdminPoolListItem {
@@ -279,7 +325,7 @@ impl DefaultAdminUsecase {
         let current_size = decommission.as_ref().map(|info| info.current_size).unwrap_or_default();
         let used_size = total_size.saturating_sub(current_size);
 
-        AdminPoolListItem {
+        AdminPoolStatus {
             id,
             cmd_line,
             last_update,
@@ -288,7 +334,7 @@ impl DefaultAdminUsecase {
             used_size,
             used: Self::used_ratio(total_size, used_size),
             status: Self::pool_list_status(decommission.as_ref()).to_string(),
-            decommission,
+            decommission: decommission.map(Self::admin_decommission_info_from_pool),
         }
     }
 
@@ -297,11 +343,46 @@ impl DefaultAdminUsecase {
             Some(info) if info.complete => Self::POOL_STATUS_COMPLETE,
             Some(info) if info.failed => Self::POOL_STATUS_FAILED,
             Some(info) if info.canceled => Self::POOL_STATUS_CANCELED,
+            Some(info) if info.queued => Self::POOL_STATUS_QUEUED,
             Some(info) if info.start_time.is_some() => Self::POOL_STATUS_RUNNING,
             _ => Self::POOL_STATUS_ACTIVE,
         }
     }
 
+    fn admin_decommission_info_from_pool(info: PoolDecommissionInfo) -> AdminPoolDecommissionInfo {
+        let waiting_reason = Self::decommission_waiting_reason(&info).map(str::to_string);
+        AdminPoolDecommissionInfo {
+            start_time: info.start_time,
+            start_size: info.start_size,
+            total_size: info.total_size,
+            current_size: info.current_size,
+            complete: info.complete,
+            failed: info.failed,
+            canceled: info.canceled,
+            queued: info.queued,
+            queued_buckets: info.queued_buckets,
+            decommissioned_buckets: info.decommissioned_buckets,
+            bucket: info.bucket,
+            prefix: info.prefix,
+            object: info.object,
+            items_decommissioned: info.items_decommissioned,
+            items_decommission_failed: info.items_decommission_failed,
+            bytes_done: info.bytes_done,
+            bytes_failed: info.bytes_failed,
+            waiting_reason,
+        }
+    }
+
+    fn decommission_waiting_reason(info: &PoolDecommissionInfo) -> Option<&'static str> {
+        if info.complete || info.failed || info.canceled || info.start_time.is_some() {
+            return None;
+        }
+        if info.queued {
+            return Some("queued");
+        }
+        Some("waiting_for_worker")
+    }
+
     fn used_ratio(total_size: usize, used_size: usize) -> f64 {
         if total_size == 0 {
             return 0.0;
@@ -310,6 +391,11 @@ impl DefaultAdminUsecase {
         used_size as f64 / total_size as f64
     }
 
+    fn parse_pool_idx_by_id(pool: &str, endpoint_count: usize) -> Option<usize> {
+        let idx = pool.parse::<usize>().ok()?;
+        (idx < endpoint_count).then_some(idx)
+    }
+
     pub async fn execute_collect_dependency_readiness(&self) -> DependencyReadiness {
         collect_runtime_dependency_readiness().await
     }
@@ -346,6 +432,21 @@ mod tests {
         let _ = readiness.iam_ready;
     }
 
+    #[test]
+    fn admin_query_pool_status_by_id_rejects_non_numeric_index() {
+        assert_eq!(DefaultAdminUsecase::parse_pool_idx_by_id("pool-a", 4), None);
+    }
+
+    #[test]
+    fn admin_query_pool_status_by_id_rejects_out_of_range_index() {
+        assert_eq!(DefaultAdminUsecase::parse_pool_idx_by_id("4", 4), None);
+    }
+
+    #[test]
+    fn admin_query_pool_status_by_id_accepts_valid_index() {
+        assert_eq!(DefaultAdminUsecase::parse_pool_idx_by_id("0", 4), Some(0));
+    }
+
     #[test]
     fn admin_pool_list_item_maps_capacity_and_active_status() {
         let now = OffsetDateTime::UNIX_EPOCH;
@@ -437,6 +538,45 @@ mod tests {
         assert_eq!(item.status, "running");
     }
 
+    #[test]
+    fn admin_pool_list_item_exposes_queued_decommission_state() {
+        let item = DefaultAdminUsecase::pool_list_item_from_status(PoolStatus {
+            id: 3,
+            cmd_line: "pool-3".to_string(),
+            last_update: OffsetDateTime::UNIX_EPOCH,
+            decommission: Some(PoolDecommissionInfo {
+                queued: true,
+                queued_buckets: vec!["bucket-a".to_string(), ".rustfs.sys/config".to_string()],
+                decommissioned_buckets: vec!["bucket-done".to_string()],
+                bucket: "bucket-a".to_string(),
+                prefix: "prefix/".to_string(),
+                object: "object.txt".to_string(),
+                items_decommissioned: 7,
+                items_decommission_failed: 1,
+                bytes_done: 1024,
+                bytes_failed: 64,
+                ..Default::default()
+            }),
+        });
+
+        assert_eq!(item.status, "queued");
+        let value = serde_json::to_value(item).expect("admin pool status should serialize");
+        assert_eq!(value["decommissionInfo"]["queued"], true);
+        assert_eq!(
+            value["decommissionInfo"]["queuedBuckets"],
+            serde_json::json!(["bucket-a", ".rustfs.sys/config"])
+        );
+        assert_eq!(value["decommissionInfo"]["decommissionedBuckets"], serde_json::json!(["bucket-done"]));
+        assert_eq!(value["decommissionInfo"]["bucket"], "bucket-a");
+        assert_eq!(value["decommissionInfo"]["prefix"], "prefix/");
+        assert_eq!(value["decommissionInfo"]["object"], "object.txt");
+        assert_eq!(value["decommissionInfo"]["objectsDecommissioned"], 7);
+        assert_eq!(value["decommissionInfo"]["objectsDecommissionedFailed"], 1);
+        assert_eq!(value["decommissionInfo"]["bytesDecommissioned"], 1024);
+        assert_eq!(value["decommissionInfo"]["bytesDecommissionedFailed"], 64);
+        assert_eq!(value["decommissionInfo"]["waitingReason"], "queued");
+    }
+
     #[test]
     fn admin_pool_list_item_maps_terminal_decommission_statuses() {
         let complete = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo {
@@ -451,11 +591,16 @@ mod tests {
             canceled: true,
             ..Default::default()
         }));
+        let queued = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo {
+            queued: true,
+            ..Default::default()
+        }));
         let idle = DefaultAdminUsecase::pool_list_status(None);
 
         assert_eq!(complete, "complete");
         assert_eq!(failed, "failed");
         assert_eq!(canceled, "canceled");
+        assert_eq!(queued, "queued");
         assert_eq!(idle, "active");
     }
 }

rustfs/src/storage/rpc/node_service.rs

@@ -126,6 +126,19 @@ fn background_rebalance_start_error_message(result: crate::storage::rpc::storage
     result.err().map(|err| format!("start_rebalance failed: {err}"))
 }
 
+fn stop_rebalance_response(result: crate::storage::rpc::storage_compat::Result<()>) -> StopRebalanceResponse {
+    match result {
+        Ok(_) => StopRebalanceResponse {
+            success: true,
+            error_info: None,
+        },
+        Err(err) => StopRebalanceResponse {
+            success: false,
+            error_info: Some(err.to_string()),
+        },
+    }
+}
+
 #[path = "bucket.rs"]
 mod bucket;
 #[path = "disk.rs"]
@@ -964,10 +977,16 @@ impl Node for NodeService {
             }));
         };
         match store.reload_pool_meta().await {
-            Ok(_) => Ok(Response::new(ReloadPoolMetaResponse {
-                success: true,
-                error_info: None,
-            })),
+            Ok(_) => match store.spawn_missing_local_decommission_routines().await {
+                Ok(_) => Ok(Response::new(ReloadPoolMetaResponse {
+                    success: true,
+                    error_info: None,
+                })),
+                Err(err) => Ok(Response::new(ReloadPoolMetaResponse {
+                    success: false,
+                    error_info: Some(err.to_string()),
+                })),
+            },
             Err(err) => Ok(Response::new(ReloadPoolMetaResponse {
                 success: false,
                 error_info: Some(err.to_string()),
@@ -975,7 +994,7 @@ impl Node for NodeService {
         }
     }
 
-    async fn stop_rebalance(&self, _request: Request<StopRebalanceRequest>) -> Result<Response<StopRebalanceResponse>, Status> {
+    async fn stop_rebalance(&self, request: Request<StopRebalanceRequest>) -> Result<Response<StopRebalanceResponse>, Status> {
         let Some(store) = resolve_object_store_handle() else {
             return Ok(Response::new(StopRebalanceResponse {
                 success: false,
@@ -983,11 +1002,12 @@ impl Node for NodeService {
             }));
         };
 
-        let _ = store.stop_rebalance().await;
-        Ok(Response::new(StopRebalanceResponse {
-            success: true,
-            error_info: None,
-        }))
+        let expected_rebalance_id = request.into_inner().expected_rebalance_id;
+        let expected_rebalance_id = (!expected_rebalance_id.is_empty()).then_some(expected_rebalance_id);
+
+        Ok(Response::new(stop_rebalance_response(
+            store.stop_rebalance_for_id(expected_rebalance_id.as_deref()).await,
+        )))
     }
 
     #[tracing::instrument(skip_all, fields(start_rebalance))]
@@ -1012,21 +1032,22 @@ impl Node for NodeService {
 
         if start_rebalance {
             log_background_rebalance_task_spawned!(start_rebalance);
-            let store = store.clone();
-            spawn(async move {
-                if let Some(message) = background_rebalance_start_error_message(store.start_rebalance().await) {
-                    error!(
-                        event = EVENT_RPC_BACKGROUND_TASK_FAILED,
-                        component = LOG_COMPONENT_STORAGE,
-                        subsystem = LOG_SUBSYSTEM_REBALANCE,
-                        operation = "start_rebalance",
-                        state = "failed",
-                        start_rebalance,
-                        error = %message,
-                        "node rpc background task failed"
-                    );
-                }
-            });
+            if let Some(message) = background_rebalance_start_error_message(store.start_rebalance().await) {
+                error!(
+                    event = EVENT_RPC_BACKGROUND_TASK_FAILED,
+                    component = LOG_COMPONENT_STORAGE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    operation = "start_rebalance",
+                    state = "failed",
+                    start_rebalance,
+                    error = %message,
+                    "node rpc background task failed"
+                );
+                return Ok(Response::new(LoadRebalanceMetaResponse {
+                    success: false,
+                    error_info: Some(message),
+                }));
+            }
         }
 
         Ok(Response::new(LoadRebalanceMetaResponse {
@@ -2331,7 +2352,9 @@ mod tests {
     async fn test_stop_rebalance() {
         let service = create_test_node_service();
 
-        let request = Request::new(StopRebalanceRequest {});
+        let request = Request::new(StopRebalanceRequest {
+            expected_rebalance_id: String::new(),
+        });
 
         let response = service.stop_rebalance(request).await;
         assert!(response.is_ok());
@@ -2373,6 +2396,22 @@ mod tests {
         assert!(message.contains("boom"));
     }
 
+    #[test]
+    fn test_stop_rebalance_response_reports_local_stop_error() {
+        let response = stop_rebalance_response(Err(crate::storage::rpc::storage_compat::Error::other("boom")));
+
+        assert!(!response.success);
+        assert!(response.error_info.as_deref().is_some_and(|message| message.contains("boom")));
+    }
+
+    #[test]
+    fn test_stop_rebalance_response_reports_success() {
+        let response = stop_rebalance_response(Ok(()));
+
+        assert!(response.success);
+        assert!(response.error_info.is_none());
+    }
+
     #[tokio::test]
     async fn test_load_bucket_metadata_empty_bucket() {
         let service = create_test_node_service();