fix(storage): harden rebalance decommission state (#3515)

rustfs/src/admin/handlers/heal.rs

@@ -34,6 +34,7 @@ use rustfs_utils::path::path_join;
 use s3s::header::{CONTENT_LENGTH, CONTENT_TYPE};
 use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
 use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
 use std::path::PathBuf;
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
 use tokio::sync::mpsc;
@@ -64,21 +65,28 @@ fn extract_heal_init_params(body: &Bytes, uri: &Uri, params: Params<'_, '_>) ->
     validate_heal_target(&hip.bucket, &hip.obj_prefix)?;
 
     if let Some(query) = uri.query() {
-        let params: Vec<&str> = query.split('&').collect();
-        for param in params {
-            let mut parts = param.split('=');
-            if let Some(key) = parts.next() {
-                if key == "clientToken"
-                    && let Some(value) = parts.next()
-                {
-                    hip.client_token = value.to_string();
+        let mut seen = HashSet::with_capacity(3);
+        for (key, value) in url::form_urlencoded::parse(query.as_bytes()) {
+            match key.as_ref() {
+                "clientToken" => {
+                    if !seen.insert("clientToken") {
+                        return Err(s3_error!(InvalidArgument, "duplicate heal query parameter"));
+                    }
+                    hip.client_token = value.into_owned();
                 }
-                if key == "forceStart" && parts.next().is_some() {
-                    hip.force_start = true;
+                "forceStart" => {
+                    if !seen.insert("forceStart") {
+                        return Err(s3_error!(InvalidArgument, "duplicate heal query parameter"));
+                    }
+                    hip.force_start = parse_heal_query_bool(value.as_ref())?;
                 }
-                if key == "forceStop" && parts.next().is_some() {
-                    hip.force_stop = true;
+                "forceStop" => {
+                    if !seen.insert("forceStop") {
+                        return Err(s3_error!(InvalidArgument, "duplicate heal query parameter"));
+                    }
+                    hip.force_stop = parse_heal_query_bool(value.as_ref())?;
                 }
+                _ => return Err(s3_error!(InvalidArgument, "unknown heal query parameter")),
             }
         }
     }
@@ -109,6 +117,14 @@ fn extract_heal_init_params(body: &Bytes, uri: &Uri, params: Params<'_, '_>) ->
     Ok(hip)
 }
 
+fn parse_heal_query_bool(value: &str) -> S3Result<bool> {
+    match value {
+        "true" => Ok(true),
+        "false" => Ok(false),
+        _ => Err(s3_error!(InvalidArgument, "invalid heal query boolean")),
+    }
+}
+
 fn validate_heal_target(bucket: &str, obj_prefix: &str) -> S3Result<()> {
     if bucket.is_empty() && !obj_prefix.is_empty() {
         return Err(s3_error!(InvalidRequest, "invalid bucket name"));
@@ -817,6 +833,28 @@ mod tests {
         assert!(parsed.force_stop);
     }
 
+    #[test]
+    fn test_extract_heal_init_params_rejects_unknown_duplicate_and_invalid_bool() {
+        let mut router = Router::new();
+        router
+            .insert("/rustfs/admin/v3/heal/{bucket}", ())
+            .expect("route should insert");
+
+        for query in [
+            "forceStart=yes",
+            "forceStart=true&forceStart=false",
+            "clientToken=token&unexpected=true",
+        ] {
+            let uri: Uri = format!("/rustfs/admin/v3/heal/test-bucket?{query}")
+                .parse()
+                .expect("uri should parse");
+            let matched = router.at("/rustfs/admin/v3/heal/test-bucket").expect("route should match");
+            let err = extract_heal_init_params(&Bytes::new(), &uri, matched.params)
+                .expect_err("strict heal query should reject malformed input");
+            assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+        }
+    }
+
     #[test]
     fn test_heal_channel_request_preserves_admin_heal_options() {
         let hip = HealInitParams {

rustfs/src/admin/handlers/pools.rs

@@ -12,12 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use http::{HeaderMap, StatusCode};
+use http::{HeaderMap, StatusCode, Uri};
 use matchit::Params;
 use rustfs_policy::policy::action::{Action, AdminAction};
+use rustfs_utils::{
+    MaskedAccessKey,
+    http::{AMZ_REQUEST_ID, REQUEST_ID_HEADER},
+};
 use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
 use serde::Deserialize;
-use serde_urlencoded::from_bytes;
 use tokio_util::sync::CancellationToken;
 use tracing::{error, info, warn};
 
@@ -38,10 +41,114 @@ use std::collections::HashSet;
 
 const LOG_COMPONENT_ADMIN_API: &str = "admin_api";
 const LOG_SUBSYSTEM_POOL_ADMIN: &str = "pool_admin";
+const EVENT_ADMIN_REQUEST_STATE: &str = "admin_request_state";
 const EVENT_ADMIN_REQUEST_REJECTED: &str = "admin_request_rejected";
 const EVENT_ADMIN_REQUEST_FAILED: &str = "admin_request_failed";
 const EVENT_ADMIN_RESPONSE_EMITTED: &str = "admin_response_emitted";
 
+fn admin_request_id(headers: &HeaderMap) -> Option<&str> {
+    headers
+        .get(REQUEST_ID_HEADER)
+        .or_else(|| headers.get(AMZ_REQUEST_ID))
+        .and_then(|value| value.to_str().ok())
+}
+
+fn admin_remote_addr(req: &S3Request<Body>) -> Option<String> {
+    req.extensions
+        .get::<Option<RemoteAddr>>()
+        .and_then(|opt| opt.map(|addr| addr.0.to_string()))
+}
+
+fn log_pool_request_rejected_with_context(operation: &str, reason: &str, request_id: &str, actor: &str, remote_addr: &str) {
+    warn!(
+        event = EVENT_ADMIN_REQUEST_REJECTED,
+        component = LOG_COMPONENT_ADMIN_API,
+        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+        operation,
+        action = operation,
+        result = "rejected",
+        reason,
+        request_id = %request_id,
+        actor = %actor,
+        remote_addr = %remote_addr,
+        "admin request rejected"
+    );
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct PoolAuditContext<'a> {
+    request_id: &'a str,
+    actor: &'a str,
+    remote_addr: &'a str,
+}
+
+impl<'a> PoolAuditContext<'a> {
+    fn new(request_id: &'a str, actor: &'a str, remote_addr: &'a str) -> Self {
+        Self {
+            request_id,
+            actor,
+            remote_addr,
+        }
+    }
+}
+
+fn log_pool_request_rejected_with_audit(operation: &str, reason: &str, audit: PoolAuditContext<'_>) {
+    warn!(
+        event = EVENT_ADMIN_REQUEST_REJECTED,
+        component = LOG_COMPONENT_ADMIN_API,
+        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+        operation,
+        action = operation,
+        result = "rejected",
+        reason,
+        request_id = %audit.request_id,
+        actor = %audit.actor,
+        remote_addr = %audit.remote_addr,
+        "admin request rejected"
+    );
+}
+
+fn log_pool_request_rejected_with_pool_audit(operation: &str, reason: &str, pool: &str, audit: PoolAuditContext<'_>) {
+    warn!(
+        event = EVENT_ADMIN_REQUEST_REJECTED,
+        component = LOG_COMPONENT_ADMIN_API,
+        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+        operation,
+        action = operation,
+        result = "rejected",
+        reason,
+        request_id = %audit.request_id,
+        actor = %audit.actor,
+        remote_addr = %audit.remote_addr,
+        pool,
+        "admin request rejected"
+    );
+}
+
+fn log_pool_request_rejected_with_index_audit(
+    operation: &str,
+    reason: &str,
+    idx: usize,
+    pool_count: usize,
+    audit: PoolAuditContext<'_>,
+) {
+    warn!(
+        event = EVENT_ADMIN_REQUEST_REJECTED,
+        component = LOG_COMPONENT_ADMIN_API,
+        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+        operation,
+        action = operation,
+        result = "rejected",
+        reason,
+        request_id = %audit.request_id,
+        actor = %audit.actor,
+        remote_addr = %audit.remote_addr,
+        pool_index = idx,
+        pool_count,
+        "admin request rejected"
+    );
+}
+
 macro_rules! log_pool_request_rejected {
     ($operation:expr, $reason:expr) => {
         warn!(
@@ -49,6 +156,7 @@ macro_rules! log_pool_request_rejected {
             component = LOG_COMPONENT_ADMIN_API,
             subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
             operation = $operation,
+            action = $operation,
             result = "rejected",
             reason = $reason,
             "admin request rejected"
@@ -56,21 +164,6 @@ macro_rules! log_pool_request_rejected {
     };
 }
 
-macro_rules! log_pool_request_rejected_with_pool {
-    ($operation:expr, $reason:expr, $pool:expr) => {
-        warn!(
-            event = EVENT_ADMIN_REQUEST_REJECTED,
-            component = LOG_COMPONENT_ADMIN_API,
-            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
-            operation = $operation,
-            result = "rejected",
-            reason = $reason,
-            pool = $pool,
-            "admin request rejected"
-        );
-    };
-}
-
 macro_rules! log_pool_request_failed {
     ($operation:expr, $reason:expr, $err:expr) => {
         error!(
@@ -78,6 +171,7 @@ macro_rules! log_pool_request_failed {
             component = LOG_COMPONENT_ADMIN_API,
             subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
             operation = $operation,
+            action = $operation,
             result = "failed",
             reason = $reason,
             error = %$err,
@@ -89,10 +183,11 @@ macro_rules! log_pool_request_failed {
 macro_rules! log_pool_response_emitted {
     ($operation:expr) => {
         info!(
-            event = EVENT_ADMIN_RESPONSE_EMITTED,
+            event = EVENT_ADMIN_REQUEST_STATE,
             component = LOG_COMPONENT_ADMIN_API,
             subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
             operation = $operation,
+            action = $operation,
             result = "success",
             "admin response emitted"
         );
@@ -130,11 +225,20 @@ fn contextualize_admin_pool_api_error(
     }
 }
 
-fn decommission_admin_not_initialized_error(operation: &str) -> S3Error {
-    log_pool_request_failed!(
-        operation_to_event(operation),
-        "object_layer_not_initialized",
-        "object layer not initialized"
+fn decommission_admin_not_initialized_error_with_audit(operation: &str, audit: PoolAuditContext<'_>) -> S3Error {
+    error!(
+        event = EVENT_ADMIN_REQUEST_FAILED,
+        component = LOG_COMPONENT_ADMIN_API,
+        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+        operation = operation_to_event(operation),
+        action = operation_to_event(operation),
+        result = "failed",
+        reason = "object_layer_not_initialized",
+        request_id = %audit.request_id,
+        actor = %audit.actor,
+        remote_addr = %audit.remote_addr,
+        error = "object layer not initialized",
+        "admin request failed"
     );
     S3Error::with_message(S3ErrorCode::InternalError, format!("Failed to {operation}: object layer not initialized"))
 }
@@ -144,36 +248,52 @@ fn pool_admin_missing_credentials_error(operation: &str) -> S3Error {
     S3Error::with_message(S3ErrorCode::InvalidRequest, format!("Failed to {operation}: missing credentials"))
 }
 
+fn pool_admin_missing_credentials_error_with_request(operation: &str, request_id: &str, remote_addr: &str) -> S3Error {
+    warn!(
+        event = EVENT_ADMIN_REQUEST_REJECTED,
+        component = LOG_COMPONENT_ADMIN_API,
+        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+        operation = operation_to_event(operation),
+        action = operation_to_event(operation),
+        result = "rejected",
+        reason = "missing_credentials",
+        request_id = %request_id,
+        remote_addr = %remote_addr,
+        "admin request rejected"
+    );
+    S3Error::with_message(S3ErrorCode::InvalidRequest, format!("Failed to {operation}: missing credentials"))
+}
+
 fn pool_admin_query_parse_error(operation: &str) -> S3Error {
     log_pool_request_rejected!(operation_to_event(operation), "invalid_query_parameters");
     S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Failed to {operation}: invalid query parameters"))
 }
 
-fn pool_admin_pool_parse_error(operation: &str, pool: &str) -> S3Error {
-    log_pool_request_rejected_with_pool!(operation_to_event(operation), "invalid_pool", pool);
+fn pool_admin_query_parse_error_with_audit(operation: &str, audit: PoolAuditContext<'_>) -> S3Error {
+    log_pool_request_rejected_with_audit(operation_to_event(operation), "invalid_query_parameters", audit);
+    S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Failed to {operation}: invalid query parameters"))
+}
+
+fn pool_admin_pool_parse_error_with_audit(operation: &str, pool: &str, audit: PoolAuditContext<'_>) -> S3Error {
+    log_pool_request_rejected_with_pool_audit(operation_to_event(operation), "invalid_pool", pool, audit);
     S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Failed to {operation}: invalid pool `{pool}`"))
 }
 
-fn pool_admin_pool_not_found_error(operation: &str, pool: &str) -> S3Error {
-    log_pool_request_rejected_with_pool!(operation_to_event(operation), "pool_not_found", pool);
+fn pool_admin_pool_not_found_error_with_audit(operation: &str, pool: &str, audit: PoolAuditContext<'_>) -> S3Error {
+    log_pool_request_rejected_with_pool_audit(operation_to_event(operation), "pool_not_found", pool, audit);
     S3Error::with_message(
         S3ErrorCode::InvalidArgument,
         format!("Failed to {operation}: pool `{pool}` was not found"),
     )
 }
 
-fn pool_admin_pool_index_error(operation: &str, idx: usize, pool_count: usize) -> S3Error {
-    warn!(
-        event = EVENT_ADMIN_REQUEST_REJECTED,
-        component = LOG_COMPONENT_ADMIN_API,
-        subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
-        operation = operation_to_event(operation),
-        result = "rejected",
-        reason = "pool_index_out_of_range",
-        pool_index = idx,
-        pool_count,
-        "admin request rejected"
-    );
+fn pool_admin_pool_index_error_with_audit(
+    operation: &str,
+    idx: usize,
+    pool_count: usize,
+    audit: PoolAuditContext<'_>,
+) -> S3Error {
+    log_pool_request_rejected_with_index_audit(operation_to_event(operation), "pool_index_out_of_range", idx, pool_count, audit);
     S3Error::with_message(
         S3ErrorCode::InvalidArgument,
         format!("Failed to {operation}: pool index {idx} is out of range for {pool_count} pools"),
@@ -186,6 +306,7 @@ fn operation_to_event(operation: &str) -> &'static str {
         "load pool status" => "query_pool_status",
         "start decommission" => "start_decommission",
         "cancel decommission" => "cancel_decommission",
+        "clear decommission" => "clear_decommission",
         _ => "pool_admin",
     }
 }
@@ -195,15 +316,14 @@ fn parse_pool_idx_by_id(pool: &str, endpoint_count: usize) -> Option<usize> {
     (idx < endpoint_count).then_some(idx)
 }
 
-fn dedup_indices(indices: &[usize]) -> Vec<usize> {
+fn has_duplicate_indices(indices: &[usize]) -> bool {
     let mut seen = HashSet::with_capacity(indices.len());
-    let mut output = Vec::with_capacity(indices.len());
     for idx in indices {
-        if seen.insert(*idx) {
-            output.push(*idx);
+        if !seen.insert(*idx) {
+            return true;
         }
     }
-    output
+    false
 }
 
 pub fn register_pool_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
@@ -231,6 +351,12 @@ pub fn register_pool_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<
         AdminOperation(&CancelDecommission {}),
     )?;
 
+    r.insert(
+        Method::POST,
+        format!("{}{}", ADMIN_PREFIX, "/v3/pools/clear").as_str(),
+        AdminOperation(&ClearDecommission {}),
+    )?;
+
     Ok(())
 }
 
@@ -285,6 +411,52 @@ pub struct StatusPoolQuery {
     pub by_id: String,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum PoolQueryMode {
+    Status,
+    Mutation,
+}
+
+fn parse_status_pool_query(uri: &Uri) -> Result<StatusPoolQuery, ()> {
+    parse_pool_query(uri, PoolQueryMode::Status)
+}
+
+fn parse_mutation_pool_query(uri: &Uri) -> Result<StatusPoolQuery, ()> {
+    parse_pool_query(uri, PoolQueryMode::Mutation)
+}
+
+fn parse_pool_query(uri: &Uri, mode: PoolQueryMode) -> Result<StatusPoolQuery, ()> {
+    let mut parsed = StatusPoolQuery::default();
+    let mut seen = HashSet::with_capacity(2);
+    let Some(query) = uri.query() else {
+        return Ok(parsed);
+    };
+
+    for (key, value) in url::form_urlencoded::parse(query.as_bytes()) {
+        match key.as_ref() {
+            "pool" => {
+                if !seen.insert("pool") {
+                    return Err(());
+                }
+                parsed.pool = value.into_owned();
+            }
+            "by-id" => {
+                if !seen.insert("by-id") {
+                    return Err(());
+                }
+                match value.as_ref() {
+                    "true" | "false" => parsed.by_id = value.into_owned(),
+                    _ => return Err(()),
+                }
+            }
+            _ if mode == PoolQueryMode::Status => {}
+            _ => return Err(()),
+        }
+    }
+
+    Ok(parsed)
+}
+
 pub struct StatusPool {}
 
 #[async_trait::async_trait]
@@ -312,15 +484,7 @@ impl Operation for StatusPool {
         )
         .await?;
 
-        let query = {
-            if let Some(query) = req.uri.query() {
-                let input: StatusPoolQuery =
-                    from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("load pool status"))?;
-                input
-            } else {
-                StatusPoolQuery::default()
-            }
-        };
+        let query = parse_status_pool_query(&req.uri).map_err(|_| pool_admin_query_parse_error("load pool status"))?;
 
         let usecase = DefaultAdminUsecase::from_global();
         let pools_status = usecase
@@ -351,12 +515,31 @@ impl Operation for StartDecommission {
     // POST <endpoint>/<admin-API>/pools/decommission?pool=http://server{1...4}/disk{1...4}
     #[tracing::instrument(skip_all)]
     async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
+        let remote_addr = admin_remote_addr(&req).unwrap_or_default();
+        info!(
+            event = EVENT_ADMIN_REQUEST_STATE,
+            component = LOG_COMPONENT_ADMIN_API,
+            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+            operation = "start_decommission",
+            action = "start_decommission",
+            state = "requested",
+            request_id = %request_id,
+            remote_addr = %remote_addr,
+            "admin pool request state"
+        );
+
         let Some(input_cred) = req.credentials else {
-            return Err(pool_admin_missing_credentials_error("start decommission"));
+            return Err(pool_admin_missing_credentials_error_with_request(
+                "start decommission",
+                &request_id,
+                &remote_addr,
+            ));
         };
 
         let (cred, owner) =
             check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+        let actor = MaskedAccessKey(&input_cred.access_key).to_string();
 
         validate_admin_request(
             &req.headers,
@@ -367,32 +550,51 @@ impl Operation for StartDecommission {
             req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
         )
         .await?;
+        let audit = PoolAuditContext::new(&request_id, &actor, &remote_addr);
 
         let Some(endpoints) = endpoints_from_context() else {
-            log_pool_request_rejected!("start_decommission", "not_implemented");
+            log_pool_request_rejected_with_context("start_decommission", "not_implemented", &request_id, &actor, &remote_addr);
             return Err(s3_error!(NotImplemented));
         };
 
         if endpoints.legacy() {
-            log_pool_request_rejected!("start_decommission", "legacy_endpoints_not_supported");
+            log_pool_request_rejected_with_context(
+                "start_decommission",
+                "legacy_endpoints_not_supported",
+                &request_id,
+                &actor,
+                &remote_addr,
+            );
             return Err(s3_error!(NotImplemented));
         }
 
         let Some(store) = resolve_object_store_handle() else {
-            return Err(decommission_admin_not_initialized_error("start decommission"));
+            return Err(decommission_admin_not_initialized_error_with_audit("start decommission", audit));
         };
 
-        validate_start_decommission_guards(store.is_decommission_running().await, store.is_rebalance_started().await)?;
+        let decommission_running = store.is_decommission_running().await;
+        let rebalance_running = store.is_rebalance_started().await;
+        if decommission_running {
+            log_pool_request_rejected_with_context(
+                "start_decommission",
+                "decommission_already_running",
+                &request_id,
+                &actor,
+                &remote_addr,
+            );
+        } else if rebalance_running {
+            log_pool_request_rejected_with_context(
+                "start_decommission",
+                "rebalance_in_progress",
+                &request_id,
+                &actor,
+                &remote_addr,
+            );
+        }
+        validate_start_decommission_guards(decommission_running, rebalance_running)?;
 
-        let query = {
-            if let Some(query) = req.uri.query() {
-                let input: StatusPoolQuery =
-                    from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("start decommission"))?;
-                input
-            } else {
-                StatusPoolQuery::default()
-            }
-        };
+        let query = parse_mutation_pool_query(&req.uri)
+            .map_err(|_| pool_admin_query_parse_error_with_audit("start decommission", audit))?;
         let is_byid = query.by_id.as_str() == "true";
 
         let pools: Vec<&str> = query.pool.split(",").collect();
@@ -404,33 +606,53 @@ impl Operation for StartDecommission {
             let idx = {
                 if is_byid {
                     parse_pool_idx_by_id(pool, endpoints.as_ref().len())
-                        .ok_or_else(|| pool_admin_pool_parse_error("start decommission", pool))?
+                        .ok_or_else(|| pool_admin_pool_parse_error_with_audit("start decommission", pool, audit))?
                 } else {
                     let Some(idx) = endpoints.get_pool_idx(pool) else {
-                        return Err(pool_admin_pool_parse_error("start decommission", pool));
+                        return Err(pool_admin_pool_parse_error_with_audit("start decommission", pool, audit));
                     };
                     idx
                 }
             };
 
             if idx >= store.pools.len() {
-                return Err(pool_admin_pool_index_error("start decommission", idx, store.pools.len()));
+                return Err(pool_admin_pool_index_error_with_audit(
+                    "start decommission",
+                    idx,
+                    store.pools.len(),
+                    audit,
+                ));
             }
 
             parsed_indices.push(idx);
         }
-        let pools_indices = dedup_indices(&parsed_indices);
+        if has_duplicate_indices(&parsed_indices) {
+            return Err(pool_admin_query_parse_error_with_audit("start decommission", audit));
+        }
+        let pools_indices = parsed_indices;
 
         if !pools_indices.is_empty() {
             let pool_context = format!("pools {:?}", &pools_indices);
             store
-                .decommission(ctx.clone(), pools_indices)
+                .decommission(ctx.clone(), pools_indices.clone())
                 .await
                 .map_err(ApiError::from)
                 .map_err(|err| contextualize_admin_pool_api_error(err, "start decommission", &pool_context))?;
         }
 
-        log_pool_response_emitted!("start_decommission");
+        info!(
+            event = EVENT_ADMIN_RESPONSE_EMITTED,
+            component = LOG_COMPONENT_ADMIN_API,
+            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+            operation = "start_decommission",
+            action = "start_decommission",
+            result = "success",
+            request_id = %request_id,
+            actor = %actor,
+            remote_addr = %remote_addr,
+            pool_indices = ?pools_indices,
+            "admin response emitted"
+        );
         Ok(S3Response::new((StatusCode::OK, Body::default())))
     }
 }
@@ -442,12 +664,31 @@ impl Operation for CancelDecommission {
     // POST <endpoint>/<admin-API>/pools/cancel?pool=http://server{1...4}/disk{1...4}
     #[tracing::instrument(skip_all)]
     async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
+        let remote_addr = admin_remote_addr(&req).unwrap_or_default();
+        info!(
+            event = EVENT_ADMIN_REQUEST_STATE,
+            component = LOG_COMPONENT_ADMIN_API,
+            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+            operation = "cancel_decommission",
+            action = "cancel_decommission",
+            state = "requested",
+            request_id = %request_id,
+            remote_addr = %remote_addr,
+            "admin pool request state"
+        );
+
         let Some(input_cred) = req.credentials else {
-            return Err(pool_admin_missing_credentials_error("cancel decommission"));
+            return Err(pool_admin_missing_credentials_error_with_request(
+                "cancel decommission",
+                &request_id,
+                &remote_addr,
+            ));
         };
 
         let (cred, owner) =
             check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+        let actor = MaskedAccessKey(&input_cred.access_key).to_string();
 
         validate_admin_request(
             &req.headers,
@@ -458,26 +699,26 @@ impl Operation for CancelDecommission {
             req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
         )
         .await?;
+        let audit = PoolAuditContext::new(&request_id, &actor, &remote_addr);
 
         let Some(endpoints) = endpoints_from_context() else {
-            log_pool_request_rejected!("cancel_decommission", "not_implemented");
+            log_pool_request_rejected_with_context("cancel_decommission", "not_implemented", &request_id, &actor, &remote_addr);
             return Err(s3_error!(NotImplemented));
         };
 
         if endpoints.legacy() {
-            log_pool_request_rejected!("cancel_decommission", "legacy_endpoints_not_supported");
+            log_pool_request_rejected_with_context(
+                "cancel_decommission",
+                "legacy_endpoints_not_supported",
+                &request_id,
+                &actor,
+                &remote_addr,
+            );
             return Err(s3_error!(NotImplemented));
         }
 
-        let query = {
-            if let Some(query) = req.uri.query() {
-                let input: StatusPoolQuery =
-                    from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("cancel decommission"))?;
-                input
-            } else {
-                StatusPoolQuery::default()
-            }
-        };
+        let query = parse_mutation_pool_query(&req.uri)
+            .map_err(|_| pool_admin_query_parse_error_with_audit("cancel decommission", audit))?;
 
         let is_byid = query.by_id.as_str() == "true";
 
@@ -490,11 +731,11 @@ impl Operation for CancelDecommission {
         };
 
         let Some(idx) = has_idx else {
-            return Err(pool_admin_pool_not_found_error("cancel decommission", &query.pool));
+            return Err(pool_admin_pool_not_found_error_with_audit("cancel decommission", &query.pool, audit));
         };
 
         let Some(store) = resolve_object_store_handle() else {
-            return Err(decommission_admin_not_initialized_error("cancel decommission"));
+            return Err(decommission_admin_not_initialized_error_with_audit("cancel decommission", audit));
         };
 
         store
@@ -503,7 +744,124 @@ impl Operation for CancelDecommission {
             .map_err(ApiError::from)
             .map_err(|err| contextualize_admin_pool_api_error(err, "cancel decommission", format!("pool {idx}")))?;
 
-        log_pool_response_emitted!("cancel_decommission");
+        info!(
+            event = EVENT_ADMIN_RESPONSE_EMITTED,
+            component = LOG_COMPONENT_ADMIN_API,
+            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+            operation = "cancel_decommission",
+            action = "cancel_decommission",
+            result = "success",
+            request_id = %request_id,
+            actor = %actor,
+            remote_addr = %remote_addr,
+            pool_index = idx,
+            "admin response emitted"
+        );
+        Ok(S3Response::new((StatusCode::OK, Body::default())))
+    }
+}
+
+pub struct ClearDecommission {}
+
+#[async_trait::async_trait]
+impl Operation for ClearDecommission {
+    // POST <endpoint>/<admin-API>/pools/clear?pool=http://server{1...4}/disk{1...4}
+    // Clears failed/canceled decommission metadata only; already moved data is not rolled back.
+    #[tracing::instrument(skip_all)]
+    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
+        let remote_addr = admin_remote_addr(&req).unwrap_or_default();
+        info!(
+            event = EVENT_ADMIN_REQUEST_STATE,
+            component = LOG_COMPONENT_ADMIN_API,
+            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+            operation = "clear_decommission",
+            action = "clear_decommission",
+            state = "requested",
+            request_id = %request_id,
+            remote_addr = %remote_addr,
+            "admin pool request state"
+        );
+
+        let Some(input_cred) = req.credentials else {
+            return Err(pool_admin_missing_credentials_error_with_request(
+                "clear decommission",
+                &request_id,
+                &remote_addr,
+            ));
+        };
+
+        let (cred, owner) =
+            check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+        let actor = MaskedAccessKey(&input_cred.access_key).to_string();
+
+        validate_admin_request(
+            &req.headers,
+            &cred,
+            owner,
+            false,
+            vec![Action::AdminAction(AdminAction::DecommissionAdminAction)],
+            req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
+        )
+        .await?;
+        let audit = PoolAuditContext::new(&request_id, &actor, &remote_addr);
+
+        let Some(endpoints) = endpoints_from_context() else {
+            log_pool_request_rejected_with_context("clear_decommission", "not_implemented", &request_id, &actor, &remote_addr);
+            return Err(s3_error!(NotImplemented));
+        };
+
+        if endpoints.legacy() {
+            log_pool_request_rejected_with_context(
+                "clear_decommission",
+                "legacy_endpoints_not_supported",
+                &request_id,
+                &actor,
+                &remote_addr,
+            );
+            return Err(s3_error!(NotImplemented));
+        }
+
+        let query = parse_mutation_pool_query(&req.uri)
+            .map_err(|_| pool_admin_query_parse_error_with_audit("clear decommission", audit))?;
+
+        let is_byid = query.by_id.as_str() == "true";
+
+        let has_idx = {
+            if is_byid {
+                parse_pool_idx_by_id(&query.pool, endpoints.as_ref().len())
+            } else {
+                endpoints.get_pool_idx(&query.pool)
+            }
+        };
+
+        let Some(idx) = has_idx else {
+            return Err(pool_admin_pool_not_found_error_with_audit("clear decommission", &query.pool, audit));
+        };
+
+        let Some(store) = resolve_object_store_handle() else {
+            return Err(decommission_admin_not_initialized_error_with_audit("clear decommission", audit));
+        };
+
+        store
+            .clear_decommission(idx)
+            .await
+            .map_err(ApiError::from)
+            .map_err(|err| contextualize_admin_pool_api_error(err, "clear decommission", format!("pool {idx}")))?;
+
+        info!(
+            event = EVENT_ADMIN_RESPONSE_EMITTED,
+            component = LOG_COMPONENT_ADMIN_API,
+            subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
+            operation = "clear_decommission",
+            action = "clear_decommission",
+            result = "success",
+            request_id = %request_id,
+            actor = %actor,
+            remote_addr = %remote_addr,
+            pool_index = idx,
+            "admin response emitted"
+        );
         Ok(S3Response::new((StatusCode::OK, Body::default())))
     }
 }
@@ -511,9 +869,12 @@ impl Operation for CancelDecommission {
 #[cfg(test)]
 mod pools_handler_tests {
     use super::{
-        contextualize_admin_pool_api_error, decommission_admin_not_initialized_error, dedup_indices, parse_pool_idx_by_id,
-        pool_admin_missing_credentials_error, pool_admin_pool_index_error, pool_admin_pool_not_found_error,
-        pool_admin_pool_parse_error, pool_admin_query_parse_error, validate_start_decommission_guards,
+        PoolAuditContext, contextualize_admin_pool_api_error, decommission_admin_not_initialized_error_with_audit,
+        has_duplicate_indices, parse_mutation_pool_query, parse_pool_idx_by_id, parse_status_pool_query,
+        pool_admin_missing_credentials_error, pool_admin_missing_credentials_error_with_request,
+        pool_admin_pool_index_error_with_audit, pool_admin_pool_not_found_error_with_audit,
+        pool_admin_pool_parse_error_with_audit, pool_admin_query_parse_error, pool_admin_query_parse_error_with_audit,
+        validate_start_decommission_guards,
     };
 
     #[test]
@@ -526,6 +887,52 @@ mod pools_handler_tests {
         assert_eq!(parse_pool_idx_by_id("4", 4), None);
     }
 
+    #[test]
+    fn test_parse_status_pool_query_ignores_unknown_but_rejects_duplicate_and_invalid_bool() {
+        let unknown = "/rustfs/admin/v3/pools/status?pool=0&force=true"
+            .parse()
+            .expect("uri should parse");
+        let query = parse_status_pool_query(&unknown).expect("status query should ignore unknown keys");
+        assert_eq!(query.pool, "0");
+
+        let duplicate = "/rustfs/admin/v3/pools/status?pool=0&pool=1"
+            .parse()
+            .expect("uri should parse");
+        assert!(parse_status_pool_query(&duplicate).is_err());
+
+        let invalid_bool = "/rustfs/admin/v3/pools/status?by-id=yes".parse().expect("uri should parse");
+        assert!(parse_status_pool_query(&invalid_bool).is_err());
+    }
+
+    #[test]
+    fn test_parse_mutation_pool_query_rejects_unknown_duplicate_and_invalid_bool() {
+        let unknown = "/rustfs/admin/v3/pools/decommission?pool=0&force=true"
+            .parse()
+            .expect("uri should parse");
+        assert!(parse_mutation_pool_query(&unknown).is_err());
+
+        let duplicate = "/rustfs/admin/v3/pools/decommission?pool=0&pool=1"
+            .parse()
+            .expect("uri should parse");
+        assert!(parse_mutation_pool_query(&duplicate).is_err());
+
+        let invalid_bool = "/rustfs/admin/v3/pools/decommission?by-id=yes"
+            .parse()
+            .expect("uri should parse");
+        assert!(parse_mutation_pool_query(&invalid_bool).is_err());
+    }
+
+    #[test]
+    fn test_parse_status_pool_query_accepts_expected_keys() {
+        let uri = "/rustfs/admin/v3/pools/status?pool=pool-a&by-id=true"
+            .parse()
+            .expect("uri should parse");
+        let query = parse_status_pool_query(&uri).expect("valid query should parse");
+
+        assert_eq!(query.pool, "pool-a");
+        assert_eq!(query.by_id, "true");
+    }
+
     #[test]
     fn test_parse_pool_idx_by_id_rejects_empty_pool_count() {
         assert_eq!(parse_pool_idx_by_id("0", 0), None);
@@ -592,21 +999,14 @@ mod pools_handler_tests {
     }
 
     #[test]
-    fn test_decommission_admin_not_initialized_error_formats_start_context() {
-        let err = decommission_admin_not_initialized_error("start decommission");
+    fn test_decommission_admin_not_initialized_error_with_audit_preserves_response_contract() {
+        let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000");
+        let err = decommission_admin_not_initialized_error_with_audit("start decommission", audit);
 
         assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError);
         assert_eq!(err.message(), Some("Failed to start decommission: object layer not initialized"));
     }
 
-    #[test]
-    fn test_decommission_admin_not_initialized_error_formats_cancel_context() {
-        let err = decommission_admin_not_initialized_error("cancel decommission");
-
-        assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError);
-        assert_eq!(err.message(), Some("Failed to cancel decommission: object layer not initialized"));
-    }
-
     #[test]
     fn test_pool_admin_missing_credentials_error_formats_list_context() {
         let err = pool_admin_missing_credentials_error("list pools");
@@ -623,6 +1023,14 @@ mod pools_handler_tests {
         assert_eq!(err.message(), Some("Failed to start decommission: missing credentials"));
     }
 
+    #[test]
+    fn test_pool_admin_missing_credentials_error_with_request_preserves_response_contract() {
+        let err = pool_admin_missing_credentials_error_with_request("cancel decommission", "req-1", "127.0.0.1:9000");
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest);
+        assert_eq!(err.message(), Some("Failed to cancel decommission: missing credentials"));
+    }
+
     #[test]
     fn test_pool_admin_query_parse_error_formats_status_context() {
         let err = pool_admin_query_parse_error("load pool status");
@@ -632,16 +1040,36 @@ mod pools_handler_tests {
     }
 
     #[test]
-    fn test_pool_admin_pool_parse_error_formats_pool_context() {
-        let err = pool_admin_pool_parse_error("start decommission", "pool-x");
+    fn test_pool_audit_context_keeps_request_actor_and_remote_addr() {
+        let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000");
+
+        assert_eq!(audit.request_id, "req-1");
+        assert_eq!(audit.actor, "access-key");
+        assert_eq!(audit.remote_addr, "127.0.0.1:9000");
+    }
+
+    #[test]
+    fn test_pool_admin_query_parse_error_with_audit_preserves_response_contract() {
+        let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000");
+        let err = pool_admin_query_parse_error_with_audit("start decommission", audit);
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
+        assert_eq!(err.message(), Some("Failed to start decommission: invalid query parameters"));
+    }
+
+    #[test]
+    fn test_pool_admin_pool_parse_error_with_audit_preserves_response_contract() {
+        let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000");
+        let err = pool_admin_pool_parse_error_with_audit("start decommission", "pool-x", audit);
 
         assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
         assert_eq!(err.message(), Some("Failed to start decommission: invalid pool `pool-x`"));
     }
 
     #[test]
-    fn test_pool_admin_pool_index_error_formats_range_context() {
-        let err = pool_admin_pool_index_error("start decommission", 4, 2);
+    fn test_pool_admin_pool_index_error_with_audit_preserves_response_contract() {
+        let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000");
+        let err = pool_admin_pool_index_error_with_audit("start decommission", 4, 2, audit);
 
         assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
         assert_eq!(
@@ -651,21 +1079,23 @@ mod pools_handler_tests {
     }
 
     #[test]
-    fn test_pool_admin_pool_not_found_error_formats_cancel_context() {
-        let err = pool_admin_pool_not_found_error("cancel decommission", "pool-x");
+    fn test_pool_admin_pool_not_found_error_with_audit_preserves_response_contract() {
+        let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000");
+        let err = pool_admin_pool_not_found_error_with_audit("cancel decommission", "pool-x", audit);
 
         assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
         assert_eq!(err.message(), Some("Failed to cancel decommission: pool `pool-x` was not found"));
     }
 
     #[test]
-    fn test_dedup_indices_removes_duplicates_preserving_order() {
-        assert_eq!(dedup_indices(&[0, 2, 1, 2, 3, 0]), vec![0, 2, 1, 3]);
+    fn test_has_duplicate_indices_detects_duplicate_indices() {
+        assert!(has_duplicate_indices(&[0, 2, 1, 2, 3]));
     }
 
     #[test]
-    fn test_dedup_indices_handles_empty_input() {
+    fn test_has_duplicate_indices_allows_unique_and_empty_input() {
         let empty: Vec<usize> = Vec::new();
-        assert!(dedup_indices(&empty).is_empty());
+        assert!(!has_duplicate_indices(&empty));
+        assert!(!has_duplicate_indices(&[0, 2, 1, 3]));
     }
 }

rustfs/src/admin/handlers/rebalance.rs

@@ -13,7 +13,8 @@
 // limitations under the License.
 
 use crate::admin::handlers::storage_compat::{
-    DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, StorageError, get_global_notification_sys,
+    DiskStat, ECStore, NotificationSys, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStopPropagationRecord,
+    StorageError, decode_rebalance_stop_propagation_record, get_global_notification_sys,
 };
 use crate::{
     admin::{
@@ -24,25 +25,154 @@ use crate::{
     auth::{check_key_valid, get_session_token},
     server::{ADMIN_PREFIX, RemoteAddr},
 };
-use http::{HeaderMap, HeaderValue, StatusCode};
+use http::{HeaderMap, HeaderValue, StatusCode, Uri};
 use hyper::Method;
 use matchit::Params;
 use rustfs_policy::policy::action::{Action, AdminAction};
 use rustfs_storage_api::{BucketOperations, BucketOptions, StorageAdminApi};
+use rustfs_utils::{
+    MaskedAccessKey,
+    http::{AMZ_REQUEST_ID, REQUEST_ID_HEADER},
+};
 use s3s::{
     Body, S3Request, S3Response, S3Result,
     header::{CONTENT_LENGTH, CONTENT_TYPE},
     s3_error,
 };
 use serde::{Deserialize, Serialize};
-use std::time::Duration;
+use std::{sync::Arc, time::Duration};
 use time::OffsetDateTime;
-use tracing::info;
+use tracing::{error, info, warn};
 
 const LOG_COMPONENT_ADMIN: &str = "admin";
 const LOG_SUBSYSTEM_REBALANCE: &str = "rebalance";
 const EVENT_ADMIN_REBALANCE_STATE: &str = "admin_rebalance_state";
 
+fn admin_request_id(headers: &HeaderMap) -> Option<&str> {
+    headers
+        .get(REQUEST_ID_HEADER)
+        .or_else(|| headers.get(AMZ_REQUEST_ID))
+        .and_then(|value| value.to_str().ok())
+}
+
+fn admin_remote_addr(req: &S3Request<Body>) -> Option<String> {
+    req.extensions
+        .get::<Option<RemoteAddr>>()
+        .and_then(|opt| opt.map(|addr| addr.0.to_string()))
+}
+
+fn log_rebalance_request_rejected(action: &str, reason: &str, request_id: &str, actor: &str, remote_addr: &str) {
+    warn!(
+        event = EVENT_ADMIN_REBALANCE_STATE,
+        component = LOG_COMPONENT_ADMIN,
+        subsystem = LOG_SUBSYSTEM_REBALANCE,
+        action,
+        result = "rejected",
+        reason,
+        request_id = %request_id,
+        actor = %actor,
+        remote_addr = %remote_addr,
+        "admin rebalance state"
+    );
+}
+
+fn rebalance_query_present(uri: &Uri) -> bool {
+    uri.query().is_some_and(|query| !query.is_empty())
+}
+
+fn rollback_result_label(result: &Result<(), String>) -> &'static str {
+    match result {
+        Ok(_) => "rollback_success",
+        Err(err) if err.contains("peer") => "rollback_partial",
+        Err(_) => "rollback_failed",
+    }
+}
+
+fn rebalance_start_rollback_error(start_err: &str, rollback_result: &Result<(), String>) -> String {
+    match rollback_result {
+        Ok(_) => format!("failed to propagate rebalance start: {start_err}; rollback result: rollback_success"),
+        Err(err) => format!(
+            "failed to propagate rebalance start: {start_err}; rollback result: {}; rollback error: {err}",
+            rollback_result_label(rollback_result)
+        ),
+    }
+}
+
+fn rebalance_rollback_stop_failure_message(rebalance_id: &str, failures: &[String]) -> String {
+    format!("cluster stop_rebalance rollback for {rebalance_id} partial: {}", failures.join("; "))
+}
+
+fn rebalance_rollback_terminal_reload_failure_message(rebalance_id: &str, failures: &[String]) -> String {
+    format!(
+        "cluster terminal rebalance reload rollback for {rebalance_id} partial: {}",
+        failures.join("; ")
+    )
+}
+
+fn rebalance_rollback_failure_message(
+    rebalance_id: &str,
+    stop_failures: &[String],
+    terminal_reload_failures: &[String],
+) -> String {
+    let mut failures = Vec::new();
+    if !stop_failures.is_empty() {
+        failures.push(rebalance_rollback_stop_failure_message(rebalance_id, stop_failures));
+    }
+    if !terminal_reload_failures.is_empty() {
+        failures.push(rebalance_rollback_terminal_reload_failure_message(rebalance_id, terminal_reload_failures));
+    }
+    failures.join("; ")
+}
+
+async fn rollback_cluster_rebalance_start(
+    store: &Arc<ECStore>,
+    notification_sys: Option<&NotificationSys>,
+    rebalance_id: &str,
+) -> Result<(), String> {
+    let stop_attempt_at = OffsetDateTime::now_utc();
+    if let Some(notification_sys) = notification_sys {
+        let stop_failures = notification_sys
+            .stop_rebalance_failures(Some(rebalance_id))
+            .await
+            .map_err(|err| format!("cluster stop_rebalance rollback for {rebalance_id} failed: {err}"))?;
+        let terminal_reload_attempt_at = OffsetDateTime::now_utc();
+        let terminal_reload_failures = match notification_sys.load_rebalance_meta_failures(false).await {
+            Ok(failures) => failures,
+            Err(err) => vec![format!("terminal rebalance reload rollback for {rebalance_id} failed: {err}")],
+        };
+        if !stop_failures.is_empty() || !terminal_reload_failures.is_empty() {
+            let record = RebalanceStopPropagationRecord {
+                stop_attempt_at: Some(stop_attempt_at),
+                stop_failures: stop_failures.clone(),
+                terminal_reload_attempt_at: Some(terminal_reload_attempt_at),
+                terminal_reload_failures: terminal_reload_failures.clone(),
+            };
+            store.record_rebalance_stop_propagation(record).await.map_err(|err| {
+                format!(
+                    "cluster rebalance rollback for {rebalance_id} partial; failed to persist stop propagation: {err}; {}",
+                    rebalance_rollback_failure_message(rebalance_id, &stop_failures, &terminal_reload_failures)
+                )
+            })?;
+            return Err(rebalance_rollback_failure_message(
+                rebalance_id,
+                &stop_failures,
+                &terminal_reload_failures,
+            ));
+        }
+        return Ok(());
+    }
+
+    store
+        .stop_rebalance_for_id(Some(rebalance_id))
+        .await
+        .map_err(|err| format!("local stop_rebalance rollback for {rebalance_id} failed: {err}"))?;
+    store
+        .save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt)
+        .await
+        .map_err(|err| format!("local rollback stop metadata save for {rebalance_id} failed: {err}"))?;
+    Ok(())
+}
+
 pub fn register_rebalance_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
     r.insert(
         Method::POST,
@@ -96,6 +226,8 @@ pub struct RebalancePoolStatus {
     pub id: usize, // Pool index (zero-based)
     #[serde(rename = "status")]
     pub status: String, // Active if rebalance is running, empty otherwise
+    #[serde(rename = "stopping")]
+    pub stopping: bool, // Stop requested but worker terminal acknowledgement not yet persisted
     #[serde(rename = "used")]
     pub used: f64, // Fraction of used space in range 0.0..=1.0
     #[serde(rename = "lastError")]
@@ -106,6 +238,20 @@ pub struct RebalancePoolStatus {
     pub progress: Option<RebalPoolProgress>, // None when rebalance is not running
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct RebalanceStopPropagationStatus {
+    #[serde(rename = "lastAttemptAt", with = "offsetdatetime_rfc3339")]
+    pub last_attempt_at: Option<OffsetDateTime>,
+    #[serde(rename = "failedPeers")]
+    pub failed_peers: Vec<String>,
+    #[serde(rename = "terminalReloadAttemptAt", with = "offsetdatetime_rfc3339")]
+    pub terminal_reload_attempt_at: Option<OffsetDateTime>,
+    #[serde(rename = "terminalReloadFailedPeers")]
+    pub terminal_reload_failed_peers: Vec<String>,
+    #[serde(rename = "pendingTerminalReload")]
+    pub pending_terminal_reload: bool,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct RebalanceAdminStatus {
     pub id: String, // Identifies the ongoing rebalance operation by a UUID
@@ -113,6 +259,8 @@ pub struct RebalanceAdminStatus {
     pub pools: Vec<RebalancePoolStatus>, // Contains all pools, including inactive
     #[serde(rename = "stoppedAt", with = "offsetdatetime_rfc3339")]
     pub stopped_at: Option<OffsetDateTime>, // Optional timestamp when rebalance was stopped
+    #[serde(rename = "stopPropagation")]
+    pub stop_propagation: RebalanceStopPropagationStatus,
 }
 
 fn calculate_rebalance_progress(
@@ -201,6 +349,7 @@ fn build_rebalance_pool_statuses(
             let mut status = RebalancePoolStatus {
                 id: i,
                 status: ps.info.status.to_string(),
+                stopping: ps.info.stopping,
                 used: rebalance_pool_used(disk_stats, i),
                 last_error: ps.info.last_error.clone(),
                 cleanup_warnings: ps.cleanup_warnings.clone(),
@@ -216,18 +365,58 @@ fn build_rebalance_pool_statuses(
         .collect()
 }
 
+fn build_rebalance_stop_propagation_status(meta: &RebalanceMeta) -> RebalanceStopPropagationStatus {
+    let record = meta
+        .pool_stats
+        .iter()
+        .filter_map(|pool_stat| pool_stat.info.last_error.as_deref())
+        .find_map(decode_rebalance_stop_propagation_record);
+
+    if let Some(record) = record {
+        let last_attempt_at = record.stop_attempt_at.or(meta.stopped_at);
+        let terminal_reload_attempt_at = record.terminal_reload_attempt_at;
+        return RebalanceStopPropagationStatus {
+            pending_terminal_reload: last_attempt_at.is_some() && terminal_reload_attempt_at.is_none(),
+            last_attempt_at,
+            failed_peers: record.stop_failures,
+            terminal_reload_attempt_at,
+            terminal_reload_failed_peers: record.terminal_reload_failures,
+        };
+    }
+
+    RebalanceStopPropagationStatus {
+        last_attempt_at: meta.stopped_at,
+        pending_terminal_reload: false,
+        ..Default::default()
+    }
+}
+
+fn build_rebalance_admin_status(now: OffsetDateTime, disk_stats: &[DiskStat], meta: &RebalanceMeta) -> RebalanceAdminStatus {
+    let stop_time = meta.stopped_at;
+    RebalanceAdminStatus {
+        id: meta.id.clone(),
+        stopped_at: meta.stopped_at,
+        pools: build_rebalance_pool_statuses(now, stop_time, meta.percent_free_goal, &meta.pool_stats, disk_stats),
+        stop_propagation: build_rebalance_stop_propagation_status(meta),
+    }
+}
+
 pub struct RebalanceStart {}
 
 #[async_trait::async_trait]
 impl Operation for RebalanceStart {
     #[tracing::instrument(skip_all)]
     async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
+        let remote_addr = admin_remote_addr(&req).unwrap_or_default();
         info!(
             event = EVENT_ADMIN_REBALANCE_STATE,
             component = LOG_COMPONENT_ADMIN,
             subsystem = LOG_SUBSYSTEM_REBALANCE,
             action = "start",
             state = "requested",
+            request_id = %request_id,
+            remote_addr = %remote_addr,
             "admin rebalance state"
         );
 
@@ -237,6 +426,7 @@ impl Operation for RebalanceStart {
 
         let (cred, owner) =
             check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+        let actor = MaskedAccessKey(&input_cred.access_key).to_string();
 
         validate_admin_request(
             &req.headers,
@@ -248,19 +438,27 @@ impl Operation for RebalanceStart {
         )
         .await?;
 
+        if rebalance_query_present(&req.uri) {
+            log_rebalance_request_rejected("start", "invalid_query_parameters", &request_id, &actor, &remote_addr);
+            return Err(s3_error!(InvalidArgument, "rebalance start does not accept query parameters"));
+        }
+
         let Some(store) = resolve_object_store_handle() else {
             return Err(s3_error!(InternalError, "object layer is not initialized"));
         };
 
         if store.pools.len() == 1 {
+            log_rebalance_request_rejected("start", "single_pool_not_supported", &request_id, &actor, &remote_addr);
             return Err(s3_error!(NotImplemented));
         }
 
         if store.is_decommission_running().await {
+            log_rebalance_request_rejected("start", "decommission_in_progress", &request_id, &actor, &remote_addr);
             return Err(s3_error!(InvalidRequest, "cannot start rebalance while decommission is in progress"));
         }
 
         if store.is_rebalance_conflicting_with_decommission().await {
+            log_rebalance_request_rejected("start", "rebalance_already_in_progress", &request_id, &actor, &remote_addr);
             return Err(s3_error!(OperationAborted, "rebalance is already in progress"));
         }
 
@@ -271,24 +469,30 @@ impl Operation for RebalanceStart {
 
         let buckets: Vec<String> = bucket_infos.into_iter().map(|bucket| bucket.name).collect();
 
-        let id = match store.init_rebalance_meta(buckets).await {
+        let id = match store.init_and_start_rebalance(buckets).await {
             Ok(id) => id,
+            Err(StorageError::DecommissionAlreadyRunning) => {
+                log_rebalance_request_rejected("start", "decommission_in_progress", &request_id, &actor, &remote_addr);
+                return Err(s3_error!(InvalidRequest, "cannot start rebalance while decommission is in progress"));
+            }
+            Err(StorageError::RebalanceAlreadyRunning) => {
+                log_rebalance_request_rejected("start", "rebalance_already_in_progress", &request_id, &actor, &remote_addr);
+                return Err(s3_error!(OperationAborted, "rebalance is already in progress"));
+            }
             Err(e) => {
-                return Err(s3_error!(InternalError, "failed to initialize rebalance metadata: {}", e));
+                return Err(s3_error!(InternalError, "failed to start rebalance: {}", e));
             }
         };
 
-        store
-            .start_rebalance()
-            .await
-            .map_err(|e| s3_error!(InternalError, "failed to start rebalance: {}", e))?;
-
         info!(
             event = EVENT_ADMIN_REBALANCE_STATE,
             component = LOG_COMPONENT_ADMIN,
             subsystem = LOG_SUBSYSTEM_REBALANCE,
             action = "start",
             state = "started",
+            request_id = %request_id,
+            actor = %actor,
+            remote_addr = %remote_addr,
             rebalance_id = %id,
             "admin rebalance state"
         );
@@ -299,18 +503,65 @@ impl Operation for RebalanceStart {
                 subsystem = LOG_SUBSYSTEM_REBALANCE,
                 action = "start",
                 state = "propagation_started",
+                request_id = %request_id,
+                actor = %actor,
+                remote_addr = %remote_addr,
+                rebalance_id = %id,
                 "admin rebalance state"
             );
             if let Err(err) = notification_sys.load_rebalance_meta(true).await {
-                info!(
+                error!(
                     event = EVENT_ADMIN_REBALANCE_STATE,
                     component = LOG_COMPONENT_ADMIN,
                     subsystem = LOG_SUBSYSTEM_REBALANCE,
                     action = "start",
                     result = "propagation_failed",
+                    request_id = %request_id,
+                    actor = %actor,
+                    remote_addr = %remote_addr,
+                    rebalance_id = %id,
                     error = %err,
                     "admin rebalance state"
                 );
+
+                let start_err = err.to_string();
+                let rollback_result = rollback_cluster_rebalance_start(&store, Some(notification_sys), &id).await;
+                let rollback_label = rollback_result_label(&rollback_result);
+                match &rollback_result {
+                    Ok(_) => info!(
+                        event = EVENT_ADMIN_REBALANCE_STATE,
+                        component = LOG_COMPONENT_ADMIN,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        action = "start",
+                        result = rollback_label,
+                        request_id = %request_id,
+                        actor = %actor,
+                        remote_addr = %remote_addr,
+                        rebalance_id = %id,
+                        propagation_error = %start_err,
+                        "admin rebalance state"
+                    ),
+                    Err(rollback_err) => error!(
+                        event = EVENT_ADMIN_REBALANCE_STATE,
+                        component = LOG_COMPONENT_ADMIN,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        action = "start",
+                        result = rollback_label,
+                        request_id = %request_id,
+                        actor = %actor,
+                        remote_addr = %remote_addr,
+                        rebalance_id = %id,
+                        propagation_error = %start_err,
+                        rollback_error = %rollback_err,
+                        "admin rebalance state"
+                    ),
+                }
+
+                return Err(s3_error!(
+                    InternalError,
+                    "{}",
+                    rebalance_start_rollback_error(&start_err, &rollback_result)
+                ));
             }
             info!(
                 event = EVENT_ADMIN_REBALANCE_STATE,
@@ -318,6 +569,11 @@ impl Operation for RebalanceStart {
                 subsystem = LOG_SUBSYSTEM_REBALANCE,
                 action = "start",
                 state = "propagation_completed",
+                result = "success",
+                request_id = %request_id,
+                actor = %actor,
+                remote_addr = %remote_addr,
+                rebalance_id = %id,
                 "admin rebalance state"
             );
         }
@@ -340,12 +596,16 @@ pub struct RebalanceStatus {}
 impl Operation for RebalanceStatus {
     #[tracing::instrument(skip_all)]
     async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
+        let remote_addr = admin_remote_addr(&req).unwrap_or_default();
         info!(
             event = EVENT_ADMIN_REBALANCE_STATE,
             component = LOG_COMPONENT_ADMIN,
             subsystem = LOG_SUBSYSTEM_REBALANCE,
             action = "status",
             state = "requested",
+            request_id = %request_id,
+            remote_addr = %remote_addr,
             "admin rebalance state"
         );
 
@@ -355,6 +615,7 @@ impl Operation for RebalanceStatus {
 
         let (cred, owner) =
             check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+        let actor = MaskedAccessKey(&input_cred.access_key).to_string();
 
         validate_admin_request(
             &req.headers,
@@ -383,6 +644,7 @@ impl Operation for RebalanceStatus {
         let mut meta = RebalanceMeta::new();
         if let Err(err) = meta.load(first_pool).await {
             if err == StorageError::ConfigNotFound {
+                log_rebalance_request_rejected("status", "rebalance_not_started", &request_id, &actor, &remote_addr);
                 return Err(s3_error!(NoSuchResource, "pool rebalance is not started"));
             }
 
@@ -401,16 +663,25 @@ impl Operation for RebalanceStatus {
             disk_stats[disk.pool_index as usize].total_space += disk.total_space;
         }
 
-        let stop_time = meta.stopped_at;
         let now = OffsetDateTime::now_utc();
-        let admin_status = RebalanceAdminStatus {
-            id: meta.id.clone(),
-            stopped_at: meta.stopped_at,
-            pools: build_rebalance_pool_statuses(now, stop_time, meta.percent_free_goal, &meta.pool_stats, &disk_stats),
-        };
+        let admin_status = build_rebalance_admin_status(now, &disk_stats, &meta);
 
         let data = serde_json::to_string(&admin_status)
             .map_err(|e| s3_error!(InternalError, "failed to serialize rebalance status response: {}", e))?;
+        info!(
+            event = EVENT_ADMIN_REBALANCE_STATE,
+            component = LOG_COMPONENT_ADMIN,
+            subsystem = LOG_SUBSYSTEM_REBALANCE,
+            action = "status",
+            result = "success",
+            request_id = %request_id,
+            actor = %actor,
+            remote_addr = %remote_addr,
+            rebalance_id = %admin_status.id,
+            pool_count = admin_status.pools.len(),
+            cleanup_warning_count = admin_status.pools.iter().map(|pool| pool.cleanup_warnings.count).sum::<u64>(),
+            "admin rebalance state"
+        );
         let mut header = HeaderMap::new();
         header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
 
@@ -425,12 +696,16 @@ pub struct RebalanceStop {}
 impl Operation for RebalanceStop {
     #[tracing::instrument(skip_all)]
     async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
+        let remote_addr = admin_remote_addr(&req).unwrap_or_default();
         info!(
             event = EVENT_ADMIN_REBALANCE_STATE,
             component = LOG_COMPONENT_ADMIN,
             subsystem = LOG_SUBSYSTEM_REBALANCE,
             action = "stop",
             state = "requested",
+            request_id = %request_id,
+            remote_addr = %remote_addr,
             "admin rebalance state"
         );
 
@@ -440,6 +715,7 @@ impl Operation for RebalanceStop {
 
         let (cred, owner) =
             check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+        let actor = MaskedAccessKey(&input_cred.access_key).to_string();
 
         validate_admin_request(
             &req.headers,
@@ -451,22 +727,37 @@ impl Operation for RebalanceStop {
         )
         .await?;
 
+        if rebalance_query_present(&req.uri) {
+            log_rebalance_request_rejected("stop", "invalid_query_parameters", &request_id, &actor, &remote_addr);
+            return Err(s3_error!(InvalidArgument, "rebalance stop does not accept query parameters"));
+        }
+
         let Some(store) = resolve_object_store_handle() else {
             return Err(s3_error!(InternalError, "object layer is not initialized"));
         };
 
+        store
+            .load_rebalance_meta()
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to load rebalance metadata before stop: {}", e))?;
+        let expected_rebalance_id = store.current_rebalance_id().await;
+
         if !store.is_rebalance_conflicting_with_decommission().await {
+            log_rebalance_request_rejected("stop", "rebalance_not_started", &request_id, &actor, &remote_addr);
             return Err(s3_error!(NoSuchResource, "pool rebalance is not started"));
         }
 
-        if let Some(notification_sys) = get_global_notification_sys() {
-            notification_sys
-                .stop_rebalance()
+        let notification_sys = get_global_notification_sys();
+        let stop_attempt_at = OffsetDateTime::now_utc();
+        let mut stop_failures = Vec::new();
+        if let Some(notification_sys) = notification_sys {
+            stop_failures = notification_sys
+                .stop_rebalance_failures(expected_rebalance_id.as_deref())
                 .await
                 .map_err(|e| s3_error!(InternalError, "failed to stop rebalance via notification system: {}", e))?;
         } else {
             store
-                .stop_rebalance()
+                .stop_rebalance_for_id(expected_rebalance_id.as_deref())
                 .await
                 .map_err(|e| s3_error!(InternalError, "failed to stop rebalance: {}", e))?;
 
@@ -482,36 +773,85 @@ impl Operation for RebalanceStop {
             subsystem = LOG_SUBSYSTEM_REBALANCE,
             action = "stop",
             state = "local_stop_persisted",
+            result = "success",
+            request_id = %request_id,
+            actor = %actor,
+            remote_addr = %remote_addr,
             "admin rebalance state"
         );
-        if let Some(notification_sys) = get_global_notification_sys() {
+
+        let mut terminal_reload_attempt_at = None;
+        let mut terminal_reload_failures = Vec::new();
+        if let Some(notification_sys) = notification_sys {
             info!(
                 event = EVENT_ADMIN_REBALANCE_STATE,
                 component = LOG_COMPONENT_ADMIN,
                 subsystem = LOG_SUBSYSTEM_REBALANCE,
                 action = "stop",
                 state = "propagation_started",
+                request_id = %request_id,
+                actor = %actor,
+                remote_addr = %remote_addr,
                 "admin rebalance state"
             );
-            if let Err(err) = notification_sys.load_rebalance_meta(false).await {
-                info!(
-                    event = EVENT_ADMIN_REBALANCE_STATE,
-                    component = LOG_COMPONENT_ADMIN,
-                    subsystem = LOG_SUBSYSTEM_REBALANCE,
-                    action = "stop",
-                    result = "propagation_failed",
-                    error = %err,
-                    "admin rebalance state"
-                );
+            terminal_reload_attempt_at = Some(OffsetDateTime::now_utc());
+            match notification_sys.load_rebalance_meta_failures(false).await {
+                Ok(failures) => {
+                    terminal_reload_failures = failures;
+                    if terminal_reload_failures.is_empty() {
+                        info!(
+                            event = EVENT_ADMIN_REBALANCE_STATE,
+                            component = LOG_COMPONENT_ADMIN,
+                            subsystem = LOG_SUBSYSTEM_REBALANCE,
+                            action = "stop",
+                            state = "propagation_completed",
+                            result = "success",
+                            request_id = %request_id,
+                            actor = %actor,
+                            remote_addr = %remote_addr,
+                            "admin rebalance state"
+                        );
+                    }
+                }
+                Err(err) => {
+                    terminal_reload_failures.push(format!("terminal rebalance reload propagation failed: {err}"));
+                }
             }
-            info!(
+        }
+
+        if !stop_failures.is_empty() || !terminal_reload_failures.is_empty() {
+            let record = RebalanceStopPropagationRecord {
+                stop_attempt_at: Some(stop_attempt_at),
+                stop_failures: stop_failures.clone(),
+                terminal_reload_attempt_at,
+                terminal_reload_failures: terminal_reload_failures.clone(),
+            };
+            store
+                .record_rebalance_stop_propagation(record)
+                .await
+                .map_err(|e| s3_error!(InternalError, "failed to persist rebalance stop propagation metadata: {}", e))?;
+
+            error!(
                 event = EVENT_ADMIN_REBALANCE_STATE,
                 component = LOG_COMPONENT_ADMIN,
                 subsystem = LOG_SUBSYSTEM_REBALANCE,
                 action = "stop",
-                state = "propagation_completed",
+                result = "propagation_failed",
+                request_id = %request_id,
+                actor = %actor,
+                remote_addr = %remote_addr,
+                stop_failure_count = stop_failures.len(),
+                terminal_reload_failure_count = terminal_reload_failures.len(),
                 "admin rebalance state"
             );
+            let mut failures = Vec::new();
+            failures.extend(stop_failures);
+            failures.extend(terminal_reload_failures);
+            return Err(s3_error!(
+                InternalError,
+                "rebalance stop propagation incomplete after local stop was persisted: {}",
+                failures.join("; ")
+            ));
         }
 
         let mut header = HeaderMap::new();
@@ -558,11 +898,14 @@ mod rebalance_handler_tests {
     use super::build_rebalance_pool_progress;
     use super::calculate_rebalance_progress;
     use super::{
-        RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, build_rebalance_pool_statuses, rebalance_pool_used,
-        rebalance_remaining_buckets, rebalance_used_pct,
+        RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, RebalanceStopPropagationStatus,
+        build_rebalance_admin_status, build_rebalance_pool_statuses, build_rebalance_stop_propagation_status,
+        rebalance_pool_used, rebalance_query_present, rebalance_remaining_buckets, rebalance_rollback_failure_message,
+        rebalance_rollback_stop_failure_message, rebalance_start_rollback_error, rebalance_used_pct, rollback_result_label,
     };
     use crate::admin::handlers::storage_compat::{
-        DiskStat, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceStats,
+        DiskStat, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta,
+        RebalanceStats, RebalanceStopPropagationRecord, encode_rebalance_stop_propagation_record,
     };
     use time::OffsetDateTime;
 
@@ -577,6 +920,76 @@ mod rebalance_handler_tests {
         assert_eq!(eta, 50);
     }
 
+    #[test]
+    fn test_rebalance_start_rollback_error_reports_successful_rollback() {
+        let rollback_result = Ok(());
+        let message = rebalance_start_rollback_error("peer a failed", &rollback_result);
+
+        assert!(message.contains("failed to propagate rebalance start: peer a failed"));
+        assert!(message.contains("rollback result: rollback_success"));
+    }
+
+    #[test]
+    fn test_rebalance_start_rollback_error_reports_partial_peer_rollback() {
+        let rollback_result = Err("peer b stop_rebalance failed: timeout".to_string());
+        let message = rebalance_start_rollback_error("peer a failed", &rollback_result);
+
+        assert_eq!(rollback_result_label(&rollback_result), "rollback_partial");
+        assert!(message.contains("failed to propagate rebalance start: peer a failed"));
+        assert!(message.contains("rollback result: rollback_partial"));
+        assert!(message.contains("rollback error: peer b stop_rebalance failed: timeout"));
+    }
+
+    #[test]
+    fn test_rebalance_rollback_stop_failure_message_lists_failures() {
+        let failures = vec![
+            "peer a stop_rebalance failed: timeout".to_string(),
+            "peer b stop_rebalance failed: unavailable".to_string(),
+        ];
+
+        let message = rebalance_rollback_stop_failure_message("rebalance-id", &failures);
+
+        assert!(message.contains("cluster stop_rebalance rollback for rebalance-id partial"));
+        assert!(message.contains("peer a stop_rebalance failed: timeout"));
+        assert!(message.contains("peer b stop_rebalance failed: unavailable"));
+    }
+
+    #[test]
+    fn test_rebalance_rollback_failure_message_lists_stop_and_terminal_reload_failures() {
+        let stop_failures = vec!["peer a stop_rebalance failed: timeout".to_string()];
+        let terminal_reload_failures = vec!["peer b load_rebalance_meta(start=false) failed: unavailable".to_string()];
+
+        let message = rebalance_rollback_failure_message("rebalance-id", &stop_failures, &terminal_reload_failures);
+
+        assert!(message.contains("cluster stop_rebalance rollback for rebalance-id partial"));
+        assert!(message.contains("peer a stop_rebalance failed: timeout"));
+        assert!(message.contains("cluster terminal rebalance reload rollback for rebalance-id partial"));
+        assert!(message.contains("peer b load_rebalance_meta(start=false) failed: unavailable"));
+    }
+
+    #[test]
+    fn test_rebalance_query_present_detects_non_empty_query() {
+        let uri = "/rustfs/admin/v3/rebalance/start?pool=0".parse().unwrap();
+
+        assert!(rebalance_query_present(&uri));
+    }
+
+    #[test]
+    fn test_rebalance_query_present_allows_no_or_empty_query() {
+        let no_query = "/rustfs/admin/v3/rebalance/start".parse().unwrap();
+        let empty_query = "/rustfs/admin/v3/rebalance/start?".parse().unwrap();
+
+        assert!(!rebalance_query_present(&no_query));
+        assert!(!rebalance_query_present(&empty_query));
+    }
+
+    #[test]
+    fn test_rollback_result_label_reports_non_peer_failure_as_failed() {
+        let rollback_result = Err("local stop_rebalance failed: disk error".to_string());
+
+        assert_eq!(rollback_result_label(&rollback_result), "rollback_failed");
+    }
+
     #[test]
     fn test_calculate_rebalance_progress_stopped_by_end_time() {
         let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
@@ -861,6 +1274,26 @@ mod rebalance_handler_tests {
         assert!(statuses[1].progress.is_some());
     }
 
+    #[test]
+    fn test_build_rebalance_pool_statuses_reports_stopping() {
+        let pool_stats = vec![RebalanceStats {
+            participating: true,
+            info: RebalanceInfo {
+                status: RebalStatus::Started,
+                stopping: true,
+                start_time: Some(OffsetDateTime::from_unix_timestamp(2_000).unwrap()),
+                ..Default::default()
+            },
+            ..Default::default()
+        }];
+
+        let statuses =
+            build_rebalance_pool_statuses(OffsetDateTime::from_unix_timestamp(2_010).unwrap(), None, 0.3, &pool_stats, &[]);
+
+        assert_eq!(statuses[0].status, "Started");
+        assert!(statuses[0].stopping);
+    }
+
     #[test]
     fn test_build_rebalance_pool_statuses_empty_inputs() {
         let statuses = build_rebalance_pool_statuses(
@@ -882,9 +1315,11 @@ mod rebalance_handler_tests {
         let status = RebalanceAdminStatus {
             id: "id-1".to_string(),
             stopped_at: None,
+            stop_propagation: RebalanceStopPropagationStatus::default(),
             pools: vec![RebalancePoolStatus {
                 id: 0,
                 status: "Started".to_string(),
+                stopping: true,
                 used: 0.5,
                 last_error: Some("temporary error".to_string()),
                 cleanup_warnings: RebalanceCleanupWarnings {
@@ -893,6 +1328,12 @@ mod rebalance_handler_tests {
                     last_bucket: Some("bucket-a".to_string()),
                     last_object: Some("obj".to_string()),
                     last_at: Some(OffsetDateTime::from_unix_timestamp(1_001).unwrap()),
+                    entries: vec![RebalanceCleanupWarningEntry {
+                        bucket: "bucket-a".to_string(),
+                        object: "obj".to_string(),
+                        message: "cleanup warning".to_string(),
+                        timestamp: Some(OffsetDateTime::from_unix_timestamp(1_001).unwrap()),
+                    }],
                 },
                 progress: Some(RebalPoolProgress {
                     num_objects: 3,
@@ -911,8 +1352,100 @@ mod rebalance_handler_tests {
         assert!(json.contains("\"remainingBuckets\""));
         assert!(json.contains("\"lastError\""));
         assert!(json.contains("\"cleanupWarnings\""));
+        assert!(json.contains("\"stopping\":true"));
         assert!(json.contains("\"lastMsg\":\"cleanup warning\""));
+        assert!(json.contains("\"entries\""));
+        assert!(json.contains("\"message\":\"cleanup warning\""));
         assert!(json.contains("\"stoppedAt\":null"));
+        assert!(json.contains("\"stopPropagation\""));
+        assert!(json.contains("\"pendingTerminalReload\":false"));
+    }
+
+    #[test]
+    fn test_build_rebalance_admin_status_is_stable_for_same_persisted_meta() {
+        let started = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
+        let disk_stats = vec![
+            DiskStat {
+                total_space: 2_000,
+                available_space: 1_000,
+            },
+            DiskStat {
+                total_space: 2_000,
+                available_space: 1_500,
+            },
+        ];
+        let meta = RebalanceMeta {
+            id: "rebalance-id".to_string(),
+            percent_free_goal: 0.6,
+            pool_stats: vec![
+                RebalanceStats {
+                    participating: true,
+                    init_capacity: 2_000,
+                    init_free_space: 500,
+                    buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
+                    rebalanced_buckets: vec!["bucket-a".to_string()],
+                    bucket: "bucket-b".to_string(),
+                    object: "object.txt".to_string(),
+                    num_objects: 10,
+                    num_versions: 12,
+                    bytes: 300,
+                    info: RebalanceInfo {
+                        status: RebalStatus::Started,
+                        start_time: Some(started),
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                },
+                RebalanceStats {
+                    participating: false,
+                    info: RebalanceInfo {
+                        status: RebalStatus::Completed,
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                },
+            ],
+            ..Default::default()
+        };
+
+        let first = build_rebalance_admin_status(OffsetDateTime::from_unix_timestamp(1_030).unwrap(), &disk_stats, &meta);
+        let second = build_rebalance_admin_status(OffsetDateTime::from_unix_timestamp(1_060).unwrap(), &disk_stats, &meta);
+
+        assert_eq!(first.id, second.id);
+        assert_eq!(first.stopped_at, second.stopped_at);
+        assert_eq!(first.stop_propagation.failed_peers, second.stop_propagation.failed_peers);
+        assert_eq!(first.pools.len(), second.pools.len());
+        for (left, right) in first.pools.iter().zip(second.pools.iter()) {
+            assert_eq!(left.id, right.id);
+            assert_eq!(left.status, right.status);
+            assert_eq!(left.stopping, right.stopping);
+            assert_eq!(left.used, right.used);
+            assert_eq!(left.last_error, right.last_error);
+            assert_eq!(left.cleanup_warnings.count, right.cleanup_warnings.count);
+            assert_eq!(
+                left.progress.as_ref().map(|progress| (
+                    progress.num_objects,
+                    progress.num_versions,
+                    progress.bytes,
+                    progress.remaining_buckets,
+                    progress.bucket.as_str(),
+                    progress.object.as_str()
+                )),
+                right.progress.as_ref().map(|progress| (
+                    progress.num_objects,
+                    progress.num_versions,
+                    progress.bytes,
+                    progress.remaining_buckets,
+                    progress.bucket.as_str(),
+                    progress.object.as_str()
+                ))
+            );
+        }
+
+        assert_ne!(
+            first.pools[0].progress.as_ref().map(|progress| progress.elapsed),
+            second.pools[0].progress.as_ref().map(|progress| progress.elapsed)
+        );
     }
 
     #[test]
@@ -921,9 +1454,14 @@ mod rebalance_handler_tests {
         let status = RebalanceAdminStatus {
             id: "id-2".to_string(),
             stopped_at: Some(stopped),
+            stop_propagation: RebalanceStopPropagationStatus {
+                last_attempt_at: Some(stopped),
+                ..Default::default()
+            },
             pools: vec![RebalancePoolStatus {
                 id: 0,
                 status: "Stopped".to_string(),
+                stopping: false,
                 used: 0.3,
                 last_error: None,
                 cleanup_warnings: RebalanceCleanupWarnings::default(),
@@ -934,5 +1472,45 @@ mod rebalance_handler_tests {
         let json = serde_json::to_string(&status).unwrap();
         assert!(json.contains("\"stoppedAt\""));
         assert!(json.contains("1970-01-01T00:16:40Z"));
+        assert!(json.contains("\"lastAttemptAt\":\"1970-01-01T00:16:40Z\""));
+    }
+
+    #[test]
+    fn test_rebalance_status_exposes_stop_propagation_failures() {
+        let stop_attempt = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
+        let reload_attempt = OffsetDateTime::from_unix_timestamp(1_010).unwrap();
+        let encoded_error = encode_rebalance_stop_propagation_record(&RebalanceStopPropagationRecord {
+            stop_attempt_at: Some(stop_attempt),
+            stop_failures: vec!["peer node-a stop_rebalance failed: timeout".to_string()],
+            terminal_reload_attempt_at: Some(reload_attempt),
+            terminal_reload_failures: vec!["peer node-b load_rebalance_meta(start=false) failed: timeout".to_string()],
+        });
+        let meta = RebalanceMeta {
+            stopped_at: Some(stop_attempt),
+            id: "id-3".to_string(),
+            percent_free_goal: 0.3,
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    stopping: true,
+                    last_error: Some(encoded_error),
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+
+        let status = build_rebalance_stop_propagation_status(&meta);
+
+        assert_eq!(status.last_attempt_at, Some(stop_attempt));
+        assert_eq!(status.terminal_reload_attempt_at, Some(reload_attempt));
+        assert!(!status.pending_terminal_reload);
+        assert_eq!(status.failed_peers, vec!["peer node-a stop_rebalance failed: timeout"]);
+        assert_eq!(
+            status.terminal_reload_failed_peers,
+            vec!["peer node-b load_rebalance_meta(start=false) failed: timeout"]
+        );
     }
 }

rustfs/src/admin/handlers/storage_compat.rs

@@ -16,15 +16,19 @@ pub(crate) use super::super::storage_compat::{
     AdminError, AdminReplicationConfigExt, AdminVersioningConfigExt, CollectMetricsOpts, DailyAllTierStats, DiskStat, ECStore,
     ERR_TIER_ALREADY_EXISTS, ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_CONNECT_ERR,
     ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND,
-    EndpointServerPools, Error, MetricType, PeerRestClient, RUSTFS_META_BUCKET, RebalSaveOpt, RebalanceCleanupWarnings,
-    RebalanceMeta, RebalanceStats, STORAGE_CLASS_SUB_SYS, StorageError, TierConfig, TierCreds, TierType, collect_local_metrics,
-    delete_admin_config, get_global_deployment_id, get_global_endpoints_opt, get_global_notification_sys, get_global_region,
-    global_rustfs_port, init_admin_config_defaults, is_reserved_or_invalid_bucket, load_data_usage_from_backend,
-    read_admin_config, read_admin_config_without_migrate, save_admin_config, save_admin_server_config,
+    EndpointServerPools, Error, MetricType, NotificationSys, PeerRestClient, RUSTFS_META_BUCKET, RebalSaveOpt,
+    RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, STORAGE_CLASS_SUB_SYS, StorageError,
+    TierConfig, TierCreds, TierType, collect_local_metrics, decode_rebalance_stop_propagation_record, delete_admin_config,
+    get_global_deployment_id, get_global_endpoints_opt, get_global_notification_sys, get_global_region, global_rustfs_port,
+    init_admin_config_defaults, is_reserved_or_invalid_bucket, load_data_usage_from_backend, read_admin_config,
+    read_admin_config_without_migrate, save_admin_config, save_admin_server_config,
 };
 
 #[cfg(test)]
-pub(crate) use super::super::storage_compat::{Endpoint, Endpoints, PoolEndpoints, RebalStatus, RebalanceInfo};
+pub(crate) use super::super::storage_compat::{
+    Endpoint, Endpoints, PoolEndpoints, RebalStatus, RebalanceCleanupWarningEntry, RebalanceInfo,
+    encode_rebalance_stop_propagation_record,
+};
 
 pub(crate) mod bucket_target_sys {
     pub(crate) use super::super::super::storage_compat::bucket_target_sys::{BucketTargetError, BucketTargetSys};

rustfs/src/admin/handlers/tier.rs

@@ -43,7 +43,7 @@ use s3s::{
     s3_error,
 };
 use serde_urlencoded::from_bytes;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use time::OffsetDateTime;
 use tracing::{debug, warn};
 
@@ -811,19 +811,42 @@ pub struct ClearTierQuery {
     pub force: String,
 }
 
+fn parse_clear_tier_query(uri: &Uri) -> S3Result<ClearTierQuery> {
+    let mut parsed = ClearTierQuery::default();
+    let mut seen = HashSet::with_capacity(2);
+    let Some(query) = uri.query() else {
+        return Ok(parsed);
+    };
+
+    for (key, value) in url::form_urlencoded::parse(query.as_bytes()) {
+        match key.as_ref() {
+            "rand" => {
+                if !seen.insert("rand") {
+                    return Err(s3_error!(InvalidArgument, "duplicate clear-tier query parameter"));
+                }
+                parsed.rand = Some(value.into_owned());
+            }
+            "force" => {
+                if !seen.insert("force") {
+                    return Err(s3_error!(InvalidArgument, "duplicate clear-tier query parameter"));
+                }
+                match value.as_ref() {
+                    "true" | "false" => parsed.force = value.into_owned(),
+                    _ => return Err(s3_error!(InvalidArgument, "invalid force flag")),
+                }
+            }
+            _ => return Err(s3_error!(InvalidArgument, "unknown clear-tier query parameter")),
+        }
+    }
+
+    Ok(parsed)
+}
+
 pub struct ClearTier {}
 #[async_trait::async_trait]
 impl Operation for ClearTier {
     async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let query = {
-            if let Some(query) = req.uri.query() {
-                let input: ClearTierQuery =
-                    from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
-                input
-            } else {
-                ClearTierQuery::default()
-            }
-        };
+        let query = parse_clear_tier_query(&req.uri)?;
 
         let Some(input_cred) = req.credentials else {
             return Err(s3_error!(InvalidRequest, "authentication required"));
@@ -845,7 +868,9 @@ impl Operation for ClearTier {
         let mut force: bool = false;
         let force_str = query.force;
         if !force_str.is_empty() {
-            force = force_str.parse().unwrap();
+            force = force_str
+                .parse()
+                .map_err(|_e| s3_error!(InvalidArgument, "invalid force flag"))?;
         }
 
         let t = OffsetDateTime::now_utc();
@@ -1050,6 +1075,30 @@ mod tests {
         assert_eq!(mapped.message(), Some("tier verification failed. backend unavailable"));
     }
 
+    #[test]
+    fn parse_clear_tier_query_rejects_unknown_duplicate_and_invalid_force() {
+        for raw in [
+            "/rustfs/admin/v3/tier?rand=token&force=yes",
+            "/rustfs/admin/v3/tier?rand=token&rand=other",
+            "/rustfs/admin/v3/tier?rand=token&unexpected=true",
+        ] {
+            let uri: Uri = raw.parse().expect("uri should parse");
+            let err = parse_clear_tier_query(&uri).expect_err("strict clear-tier query should reject malformed input");
+            assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+        }
+    }
+
+    #[test]
+    fn parse_clear_tier_query_accepts_valid_force() {
+        let uri: Uri = "/rustfs/admin/v3/tier?rand=token&force=true"
+            .parse()
+            .expect("uri should parse");
+        let query = parse_clear_tier_query(&uri).expect("valid clear-tier query should parse");
+
+        assert_eq!(query.rand.as_deref(), Some("token"));
+        assert_eq!(query.force, "true");
+    }
+
     fn sample_daily_stats() -> DailyAllTierStats {
         let mut warm = LastDayTierStats::default();
         warm.add_stats(TierStats {

rustfs/src/admin/route_policy.rs

@@ -236,6 +236,7 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
         RouteRiskLevel::High,
     ),
     admin(HttpMethod::Post, "/rustfs/admin/v3/pools/cancel", DECOMMISSION, RouteRiskLevel::High),
+    admin(HttpMethod::Post, "/rustfs/admin/v3/pools/clear", DECOMMISSION, RouteRiskLevel::High),
     admin(HttpMethod::Post, "/rustfs/admin/v3/rebalance/start", REBALANCE, RouteRiskLevel::High),
     admin(HttpMethod::Get, "/rustfs/admin/v3/rebalance/status", REBALANCE, RouteRiskLevel::Sensitive),
     admin(HttpMethod::Post, "/rustfs/admin/v3/rebalance/stop", REBALANCE, RouteRiskLevel::High),

rustfs/src/admin/route_registration_test.rs

@@ -175,6 +175,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
         admin_route(Method::GET, "/v3/pools/status"),
         admin_route(Method::POST, "/v3/pools/decommission"),
         admin_route(Method::POST, "/v3/pools/cancel"),
+        admin_route(Method::POST, "/v3/pools/clear"),
         admin_route(Method::POST, "/v3/rebalance/start"),
         admin_route(Method::GET, "/v3/rebalance/status"),
         admin_route(Method::POST, "/v3/rebalance/stop"),

rustfs/src/admin/storage_compat.rs

@@ -46,6 +46,7 @@ pub(crate) type RebalSaveOpt = ecstore_rebalance::RebalSaveOpt;
 pub(crate) type RebalanceCleanupWarnings = ecstore_rebalance::RebalanceCleanupWarnings;
 pub(crate) type RebalanceMeta = ecstore_rebalance::RebalanceMeta;
 pub(crate) type RebalanceStats = ecstore_rebalance::RebalanceStats;
+pub(crate) type RebalanceStopPropagationRecord = ecstore_rebalance::RebalanceStopPropagationRecord;
 pub(crate) type StorageError = ecstore_error::StorageError;
 pub(crate) type Error = StorageError;
 pub(crate) type Result<T> = core::result::Result<T, Error>;
@@ -62,8 +63,19 @@ pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints;
 #[cfg(test)]
 pub(crate) type RebalStatus = ecstore_rebalance::RebalStatus;
 #[cfg(test)]
+pub(crate) type RebalanceCleanupWarningEntry = ecstore_rebalance::RebalanceCleanupWarningEntry;
+#[cfg(test)]
 pub(crate) type RebalanceInfo = ecstore_rebalance::RebalanceInfo;
 
+pub(crate) fn decode_rebalance_stop_propagation_record(message: &str) -> Option<RebalanceStopPropagationRecord> {
+    ecstore_rebalance::decode_rebalance_stop_propagation_record(message)
+}
+
+#[cfg(test)]
+pub(crate) fn encode_rebalance_stop_propagation_record(record: &RebalanceStopPropagationRecord) -> String {
+    ecstore_rebalance::encode_rebalance_stop_propagation_record(record)
+}
+
 pub(crate) trait AdminReplicationConfigExt {
     fn filter_target_arns(&self, obj: &replication::ObjectOpts) -> Vec<String>;
     fn has_existing_object_replication(&self, arn: &str) -> (bool, bool);

rustfs/src/app/admin_usecase.rs

@@ -56,7 +56,47 @@ pub struct QueryPoolStatusRequest {
 }
 
 #[derive(Debug, Clone, serde::Serialize)]
-pub struct AdminPoolListItem {
+pub struct AdminPoolDecommissionInfo {
+    #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
+    pub start_time: Option<time::OffsetDateTime>,
+    #[serde(rename = "startSize")]
+    pub start_size: usize,
+    #[serde(rename = "totalSize")]
+    pub total_size: usize,
+    #[serde(rename = "currentSize")]
+    pub current_size: usize,
+    #[serde(rename = "complete")]
+    pub complete: bool,
+    #[serde(rename = "failed")]
+    pub failed: bool,
+    #[serde(rename = "canceled")]
+    pub canceled: bool,
+    #[serde(rename = "queued")]
+    pub queued: bool,
+    #[serde(rename = "queuedBuckets")]
+    pub queued_buckets: Vec<String>,
+    #[serde(rename = "decommissionedBuckets")]
+    pub decommissioned_buckets: Vec<String>,
+    #[serde(rename = "bucket")]
+    pub bucket: String,
+    #[serde(rename = "prefix")]
+    pub prefix: String,
+    #[serde(rename = "object")]
+    pub object: String,
+    #[serde(rename = "objectsDecommissioned")]
+    pub items_decommissioned: usize,
+    #[serde(rename = "objectsDecommissionedFailed")]
+    pub items_decommission_failed: usize,
+    #[serde(rename = "bytesDecommissioned")]
+    pub bytes_done: usize,
+    #[serde(rename = "bytesDecommissionedFailed")]
+    pub bytes_failed: usize,
+    #[serde(rename = "waitingReason")]
+    pub waiting_reason: Option<String>,
+}
+
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct AdminPoolStatus {
     #[serde(rename = "id")]
     pub id: usize,
     #[serde(rename = "cmdline")]
@@ -74,9 +114,11 @@ pub struct AdminPoolListItem {
     #[serde(rename = "status")]
     pub status: String,
     #[serde(rename = "decommissionInfo")]
-    pub decommission: Option<PoolDecommissionInfo>,
+    pub decommission: Option<AdminPoolDecommissionInfo>,
 }
 
+pub type AdminPoolListItem = AdminPoolStatus;
+
 #[derive(Clone, Default)]
 pub struct DefaultAdminUsecase {
     context: Option<Arc<AppContext>>,
@@ -87,6 +129,7 @@ impl DefaultAdminUsecase {
     const POOL_STATUS_CANCELED: &'static str = "canceled";
     const POOL_STATUS_COMPLETE: &'static str = "complete";
     const POOL_STATUS_FAILED: &'static str = "failed";
+    const POOL_STATUS_QUEUED: &'static str = "queued";
     const POOL_STATUS_RUNNING: &'static str = "running";
 
     #[cfg(test)]
@@ -240,7 +283,7 @@ impl DefaultAdminUsecase {
         Ok(pool_statuses.into_iter().map(Self::pool_list_item_from_status).collect())
     }
 
-    pub async fn execute_query_pool_status(&self, req: QueryPoolStatusRequest) -> AdminUsecaseResult<PoolStatus> {
+    pub async fn execute_query_pool_status(&self, req: QueryPoolStatusRequest) -> AdminUsecaseResult<AdminPoolStatus> {
         let Some(endpoints) = self.endpoints() else {
             return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
         };
@@ -250,8 +293,7 @@ impl DefaultAdminUsecase {
         }
 
         let has_idx = if req.by_id {
-            let idx = req.pool.parse::<usize>().unwrap_or_default();
-            if idx < endpoints.as_ref().len() { Some(idx) } else { None }
+            Self::parse_pool_idx_by_id(&req.pool, endpoints.as_ref().len())
         } else {
             endpoints.get_pool_idx(&req.pool)
         };
@@ -265,7 +307,11 @@ impl DefaultAdminUsecase {
             return Err(Self::app_error(S3ErrorCode::InternalError, "Not init"));
         };
 
-        store.status(idx).await.map_err(ApiError::from)
+        store
+            .status(idx)
+            .await
+            .map(Self::pool_list_item_from_status)
+            .map_err(ApiError::from)
     }
 
     fn pool_list_item_from_status(status: PoolStatus) -> AdminPoolListItem {
@@ -279,7 +325,7 @@ impl DefaultAdminUsecase {
         let current_size = decommission.as_ref().map(|info| info.current_size).unwrap_or_default();
         let used_size = total_size.saturating_sub(current_size);
 
-        AdminPoolListItem {
+        AdminPoolStatus {
             id,
             cmd_line,
             last_update,
@@ -288,7 +334,7 @@ impl DefaultAdminUsecase {
             used_size,
             used: Self::used_ratio(total_size, used_size),
             status: Self::pool_list_status(decommission.as_ref()).to_string(),
-            decommission,
+            decommission: decommission.map(Self::admin_decommission_info_from_pool),
         }
     }
 
@@ -297,11 +343,46 @@ impl DefaultAdminUsecase {
             Some(info) if info.complete => Self::POOL_STATUS_COMPLETE,
             Some(info) if info.failed => Self::POOL_STATUS_FAILED,
             Some(info) if info.canceled => Self::POOL_STATUS_CANCELED,
+            Some(info) if info.queued => Self::POOL_STATUS_QUEUED,
             Some(info) if info.start_time.is_some() => Self::POOL_STATUS_RUNNING,
             _ => Self::POOL_STATUS_ACTIVE,
         }
     }
 
+    fn admin_decommission_info_from_pool(info: PoolDecommissionInfo) -> AdminPoolDecommissionInfo {
+        let waiting_reason = Self::decommission_waiting_reason(&info).map(str::to_string);
+        AdminPoolDecommissionInfo {
+            start_time: info.start_time,
+            start_size: info.start_size,
+            total_size: info.total_size,
+            current_size: info.current_size,
+            complete: info.complete,
+            failed: info.failed,
+            canceled: info.canceled,
+            queued: info.queued,
+            queued_buckets: info.queued_buckets,
+            decommissioned_buckets: info.decommissioned_buckets,
+            bucket: info.bucket,
+            prefix: info.prefix,
+            object: info.object,
+            items_decommissioned: info.items_decommissioned,
+            items_decommission_failed: info.items_decommission_failed,
+            bytes_done: info.bytes_done,
+            bytes_failed: info.bytes_failed,
+            waiting_reason,
+        }
+    }
+
+    fn decommission_waiting_reason(info: &PoolDecommissionInfo) -> Option<&'static str> {
+        if info.complete || info.failed || info.canceled || info.start_time.is_some() {
+            return None;
+        }
+        if info.queued {
+            return Some("queued");
+        }
+        Some("waiting_for_worker")
+    }
+
     fn used_ratio(total_size: usize, used_size: usize) -> f64 {
         if total_size == 0 {
             return 0.0;
@@ -310,6 +391,11 @@ impl DefaultAdminUsecase {
         used_size as f64 / total_size as f64
     }
 
+    fn parse_pool_idx_by_id(pool: &str, endpoint_count: usize) -> Option<usize> {
+        let idx = pool.parse::<usize>().ok()?;
+        (idx < endpoint_count).then_some(idx)
+    }
+
     pub async fn execute_collect_dependency_readiness(&self) -> DependencyReadiness {
         collect_runtime_dependency_readiness().await
     }
@@ -346,6 +432,21 @@ mod tests {
         let _ = readiness.iam_ready;
     }
 
+    #[test]
+    fn admin_query_pool_status_by_id_rejects_non_numeric_index() {
+        assert_eq!(DefaultAdminUsecase::parse_pool_idx_by_id("pool-a", 4), None);
+    }
+
+    #[test]
+    fn admin_query_pool_status_by_id_rejects_out_of_range_index() {
+        assert_eq!(DefaultAdminUsecase::parse_pool_idx_by_id("4", 4), None);
+    }
+
+    #[test]
+    fn admin_query_pool_status_by_id_accepts_valid_index() {
+        assert_eq!(DefaultAdminUsecase::parse_pool_idx_by_id("0", 4), Some(0));
+    }
+
     #[test]
     fn admin_pool_list_item_maps_capacity_and_active_status() {
         let now = OffsetDateTime::UNIX_EPOCH;
@@ -437,6 +538,45 @@ mod tests {
         assert_eq!(item.status, "running");
     }
 
+    #[test]
+    fn admin_pool_list_item_exposes_queued_decommission_state() {
+        let item = DefaultAdminUsecase::pool_list_item_from_status(PoolStatus {
+            id: 3,
+            cmd_line: "pool-3".to_string(),
+            last_update: OffsetDateTime::UNIX_EPOCH,
+            decommission: Some(PoolDecommissionInfo {
+                queued: true,
+                queued_buckets: vec!["bucket-a".to_string(), ".rustfs.sys/config".to_string()],
+                decommissioned_buckets: vec!["bucket-done".to_string()],
+                bucket: "bucket-a".to_string(),
+                prefix: "prefix/".to_string(),
+                object: "object.txt".to_string(),
+                items_decommissioned: 7,
+                items_decommission_failed: 1,
+                bytes_done: 1024,
+                bytes_failed: 64,
+                ..Default::default()
+            }),
+        });
+
+        assert_eq!(item.status, "queued");
+        let value = serde_json::to_value(item).expect("admin pool status should serialize");
+        assert_eq!(value["decommissionInfo"]["queued"], true);
+        assert_eq!(
+            value["decommissionInfo"]["queuedBuckets"],
+            serde_json::json!(["bucket-a", ".rustfs.sys/config"])
+        );
+        assert_eq!(value["decommissionInfo"]["decommissionedBuckets"], serde_json::json!(["bucket-done"]));
+        assert_eq!(value["decommissionInfo"]["bucket"], "bucket-a");
+        assert_eq!(value["decommissionInfo"]["prefix"], "prefix/");
+        assert_eq!(value["decommissionInfo"]["object"], "object.txt");
+        assert_eq!(value["decommissionInfo"]["objectsDecommissioned"], 7);
+        assert_eq!(value["decommissionInfo"]["objectsDecommissionedFailed"], 1);
+        assert_eq!(value["decommissionInfo"]["bytesDecommissioned"], 1024);
+        assert_eq!(value["decommissionInfo"]["bytesDecommissionedFailed"], 64);
+        assert_eq!(value["decommissionInfo"]["waitingReason"], "queued");
+    }
+
     #[test]
     fn admin_pool_list_item_maps_terminal_decommission_statuses() {
         let complete = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo {
@@ -451,11 +591,16 @@ mod tests {
             canceled: true,
             ..Default::default()
         }));
+        let queued = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo {
+            queued: true,
+            ..Default::default()
+        }));
         let idle = DefaultAdminUsecase::pool_list_status(None);
 
         assert_eq!(complete, "complete");
         assert_eq!(failed, "failed");
         assert_eq!(canceled, "canceled");
+        assert_eq!(queued, "queued");
         assert_eq!(idle, "active");
     }
 }

rustfs/src/storage/rpc/node_service.rs

@@ -126,6 +126,19 @@ fn background_rebalance_start_error_message(result: crate::storage::rpc::storage
     result.err().map(|err| format!("start_rebalance failed: {err}"))
 }
 
+fn stop_rebalance_response(result: crate::storage::rpc::storage_compat::Result<()>) -> StopRebalanceResponse {
+    match result {
+        Ok(_) => StopRebalanceResponse {
+            success: true,
+            error_info: None,
+        },
+        Err(err) => StopRebalanceResponse {
+            success: false,
+            error_info: Some(err.to_string()),
+        },
+    }
+}
+
 #[path = "bucket.rs"]
 mod bucket;
 #[path = "disk.rs"]
@@ -964,10 +977,16 @@ impl Node for NodeService {
             }));
         };
         match store.reload_pool_meta().await {
-            Ok(_) => Ok(Response::new(ReloadPoolMetaResponse {
-                success: true,
-                error_info: None,
-            })),
+            Ok(_) => match store.spawn_missing_local_decommission_routines().await {
+                Ok(_) => Ok(Response::new(ReloadPoolMetaResponse {
+                    success: true,
+                    error_info: None,
+                })),
+                Err(err) => Ok(Response::new(ReloadPoolMetaResponse {
+                    success: false,
+                    error_info: Some(err.to_string()),
+                })),
+            },
             Err(err) => Ok(Response::new(ReloadPoolMetaResponse {
                 success: false,
                 error_info: Some(err.to_string()),
@@ -975,7 +994,7 @@ impl Node for NodeService {
         }
     }
 
-    async fn stop_rebalance(&self, _request: Request<StopRebalanceRequest>) -> Result<Response<StopRebalanceResponse>, Status> {
+    async fn stop_rebalance(&self, request: Request<StopRebalanceRequest>) -> Result<Response<StopRebalanceResponse>, Status> {
         let Some(store) = resolve_object_store_handle() else {
             return Ok(Response::new(StopRebalanceResponse {
                 success: false,
@@ -983,11 +1002,12 @@ impl Node for NodeService {
             }));
         };
 
-        let _ = store.stop_rebalance().await;
-        Ok(Response::new(StopRebalanceResponse {
-            success: true,
-            error_info: None,
-        }))
+        let expected_rebalance_id = request.into_inner().expected_rebalance_id;
+        let expected_rebalance_id = (!expected_rebalance_id.is_empty()).then_some(expected_rebalance_id);
+
+        Ok(Response::new(stop_rebalance_response(
+            store.stop_rebalance_for_id(expected_rebalance_id.as_deref()).await,
+        )))
     }
 
     #[tracing::instrument(skip_all, fields(start_rebalance))]
@@ -1012,21 +1032,22 @@ impl Node for NodeService {
 
         if start_rebalance {
             log_background_rebalance_task_spawned!(start_rebalance);
-            let store = store.clone();
-            spawn(async move {
-                if let Some(message) = background_rebalance_start_error_message(store.start_rebalance().await) {
-                    error!(
-                        event = EVENT_RPC_BACKGROUND_TASK_FAILED,
-                        component = LOG_COMPONENT_STORAGE,
-                        subsystem = LOG_SUBSYSTEM_REBALANCE,
-                        operation = "start_rebalance",
-                        state = "failed",
-                        start_rebalance,
-                        error = %message,
-                        "node rpc background task failed"
-                    );
-                }
-            });
+            if let Some(message) = background_rebalance_start_error_message(store.start_rebalance().await) {
+                error!(
+                    event = EVENT_RPC_BACKGROUND_TASK_FAILED,
+                    component = LOG_COMPONENT_STORAGE,
+                    subsystem = LOG_SUBSYSTEM_REBALANCE,
+                    operation = "start_rebalance",
+                    state = "failed",
+                    start_rebalance,
+                    error = %message,
+                    "node rpc background task failed"
+                );
+                return Ok(Response::new(LoadRebalanceMetaResponse {
+                    success: false,
+                    error_info: Some(message),
+                }));
+            }
         }
 
         Ok(Response::new(LoadRebalanceMetaResponse {
@@ -2331,7 +2352,9 @@ mod tests {
     async fn test_stop_rebalance() {
         let service = create_test_node_service();
 
-        let request = Request::new(StopRebalanceRequest {});
+        let request = Request::new(StopRebalanceRequest {
+            expected_rebalance_id: String::new(),
+        });
 
         let response = service.stop_rebalance(request).await;
         assert!(response.is_ok());
@@ -2373,6 +2396,22 @@ mod tests {
         assert!(message.contains("boom"));
     }
 
+    #[test]
+    fn test_stop_rebalance_response_reports_local_stop_error() {
+        let response = stop_rebalance_response(Err(crate::storage::rpc::storage_compat::Error::other("boom")));
+
+        assert!(!response.success);
+        assert!(response.error_info.as_deref().is_some_and(|message| message.contains("boom")));
+    }
+
+    #[test]
+    fn test_stop_rebalance_response_reports_success() {
+        let response = stop_rebalance_response(Ok(()));
+
+        assert!(response.success);
+        assert!(response.error_info.is_none());
+    }
+
     #[tokio::test]
     async fn test_load_bucket_metadata_empty_bucket() {
         let service = create_test_node_service();