fix(storage): harden rebalance and decommission state (#3730)

rustfs/src/admin/handlers/pools.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use http::{HeaderMap, StatusCode, Uri};
+use http::{HeaderMap, HeaderValue, StatusCode, Uri};
 use matchit::Params;
 use rustfs_policy::policy::action::{Action, AdminAction};
 use rustfs_utils::{
@@ -26,11 +26,12 @@ use tracing::{error, info, warn};
 
 use crate::{
     admin::{
+        EndpointServerPools, PeerRestClient,
         auth::validate_admin_request,
         router::{AdminOperation, Operation, S3Router},
     },
     app::admin_usecase::{DefaultAdminUsecase, QueryPoolStatusRequest},
-    app::context::{resolve_endpoints_handle, resolve_object_store_handle},
+    app::context::{resolve_endpoints_handle, resolve_notification_system, resolve_object_store_handle},
     auth::{check_key_valid, get_session_token},
     error::ApiError,
     server::{ADMIN_PREFIX, RemoteAddr},
@@ -213,6 +214,84 @@ fn validate_start_decommission_guards(decommission_running: bool, rebalance_runn
     Ok(())
 }
 
+#[cfg(test)]
+fn validate_pool_mutation_leader(
+    endpoints: &EndpointServerPools,
+    idx: usize,
+    operation: &str,
+    audit: PoolAuditContext<'_>,
+) -> s3s::S3Result<()> {
+    let endpoint = endpoints
+        .as_ref()
+        .get(idx)
+        .and_then(|pool| pool.endpoints.as_ref().first())
+        .ok_or_else(|| pool_admin_pool_index_error_with_audit(operation, idx, endpoints.as_ref().len(), audit))?;
+
+    if !endpoint.is_local {
+        log_pool_request_rejected_with_index_audit(
+            operation_to_event(operation),
+            "not_pool_leader",
+            idx,
+            endpoints.as_ref().len(),
+            audit,
+        );
+        return Err(S3Error::with_message(
+            S3ErrorCode::OperationAborted,
+            format!("Failed to {operation}: pool {idx} must be handled by its first endpoint {endpoint}"),
+        ));
+    }
+
+    Ok(())
+}
+
+fn decommission_peer_target(
+    endpoints: &EndpointServerPools,
+    idx: usize,
+    operation: &str,
+    audit: PoolAuditContext<'_>,
+) -> s3s::S3Result<Option<PeerRestClient>> {
+    let endpoint = endpoints
+        .as_ref()
+        .get(idx)
+        .and_then(|pool| pool.endpoints.as_ref().first())
+        .ok_or_else(|| pool_admin_pool_index_error_with_audit(operation, idx, endpoints.as_ref().len(), audit))?;
+
+    if endpoint.is_local {
+        return Ok(None);
+    }
+
+    let grid_host = endpoint.grid_host();
+    let Some(notification_sys) = resolve_notification_system() else {
+        log_pool_request_rejected_with_index_audit(
+            operation_to_event(operation),
+            "notification_sys_not_initialized",
+            idx,
+            endpoints.as_ref().len(),
+            audit,
+        );
+        return Err(S3Error::with_message(
+            S3ErrorCode::OperationAborted,
+            format!("Failed to {operation}: target pool first endpoint is not reachable"),
+        ));
+    };
+
+    let Some(client) = notification_sys.peer_client_for_grid_host(&grid_host) else {
+        log_pool_request_rejected_with_index_audit(
+            operation_to_event(operation),
+            "target_peer_not_found",
+            idx,
+            endpoints.as_ref().len(),
+            audit,
+        );
+        return Err(S3Error::with_message(
+            S3ErrorCode::OperationAborted,
+            format!("Failed to {operation}: target pool first endpoint is not reachable"),
+        ));
+    };
+
+    Ok(Some(client))
+}
+
 fn contextualize_admin_pool_api_error(
     err: crate::error::ApiError,
     operation: &str,
@@ -304,6 +383,7 @@ fn operation_to_event(operation: &str) -> &'static str {
     match operation {
         "list pools" => "list_pools",
         "load pool status" => "query_pool_status",
+        "load decommission status" => "query_decommission_status",
         "start decommission" => "start_decommission",
         "cancel decommission" => "cancel_decommission",
         "clear decommission" => "clear_decommission",
@@ -339,6 +419,12 @@ pub fn register_pool_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<
         AdminOperation(&StatusPool {}),
     )?;
 
+    r.insert(
+        Method::GET,
+        format!("{}{}", ADMIN_PREFIX, "/v3/decommission/status").as_str(),
+        AdminOperation(&StatusDecommission {}),
+    )?;
+
     r.insert(
         Method::POST,
         format!("{}{}", ADMIN_PREFIX, "/v3/pools/decommission").as_str(),
@@ -508,6 +594,62 @@ impl Operation for StatusPool {
     }
 }
 
+pub struct StatusDecommission {}
+
+#[async_trait::async_trait]
+impl Operation for StatusDecommission {
+    // GET <endpoint>/<admin-API>/decommission/status[?pool=http://server{1...4}/disk{1...4}]
+    #[tracing::instrument(skip_all)]
+    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let Some(input_cred) = req.credentials else {
+            return Err(pool_admin_missing_credentials_error("load decommission status"));
+        };
+
+        let (cred, owner) =
+            check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+
+        validate_admin_request(
+            &req.headers,
+            &cred,
+            owner,
+            false,
+            vec![
+                Action::AdminAction(AdminAction::ServerInfoAdminAction),
+                Action::AdminAction(AdminAction::DecommissionAdminAction),
+            ],
+            req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
+        )
+        .await?;
+
+        let query = parse_status_pool_query(&req.uri).map_err(|_| pool_admin_query_parse_error("load decommission status"))?;
+
+        let usecase = DefaultAdminUsecase::from_global();
+        let data = if query.pool.is_empty() {
+            let status = usecase.execute_list_decommission_status().await.map_err(S3Error::from)?;
+            serde_json::to_vec(&status)
+        } else {
+            let status = usecase
+                .execute_query_decommission_status(QueryPoolStatusRequest {
+                    pool: query.pool,
+                    by_id: query.by_id.as_str() == "true",
+                })
+                .await
+                .map_err(S3Error::from)?;
+            serde_json::to_vec(&status)
+        }
+        .map_err(|e| {
+            log_pool_request_failed!("query_decommission_status", "serialize_decommission_status_failed", e);
+            S3Error::with_message(S3ErrorCode::InternalError, "serialize decommission status failed")
+        })?;
+
+        let mut header = HeaderMap::new();
+        header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
+        log_pool_response_emitted!("query_decommission_status");
+
+        Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
+    }
+}
+
 pub struct StartDecommission {}
 
 #[async_trait::async_trait]
@@ -572,27 +714,6 @@ impl Operation for StartDecommission {
             return Err(decommission_admin_not_initialized_error_with_audit("start decommission", audit));
         };
 
-        let decommission_running = store.is_decommission_running().await;
-        let rebalance_running = store.is_rebalance_started().await;
-        if decommission_running {
-            log_pool_request_rejected_with_context(
-                "start_decommission",
-                "decommission_already_running",
-                &request_id,
-                &actor,
-                &remote_addr,
-            );
-        } else if rebalance_running {
-            log_pool_request_rejected_with_context(
-                "start_decommission",
-                "rebalance_in_progress",
-                &request_id,
-                &actor,
-                &remote_addr,
-            );
-        }
-        validate_start_decommission_guards(decommission_running, rebalance_running)?;
-
         let query = parse_mutation_pool_query(&req.uri)
             .map_err(|_| pool_admin_query_parse_error_with_audit("start decommission", audit))?;
         let is_byid = query.by_id.as_str() == "true";
@@ -631,13 +752,49 @@ impl Operation for StartDecommission {
         }
         let pools_indices = parsed_indices;
 
-        if !pools_indices.is_empty() {
+        if let Some(first_idx) = pools_indices.first().copied()
+            && let Some(client) = decommission_peer_target(&endpoints, first_idx, "start decommission", audit)?
+        {
             let pool_context = format!("pools {:?}", &pools_indices);
-            store
-                .decommission(ctx.clone(), pools_indices.clone())
+            client
+                .start_decommission(pools_indices.clone())
                 .await
                 .map_err(ApiError::from)
                 .map_err(|err| contextualize_admin_pool_api_error(err, "start decommission", &pool_context))?;
+        } else {
+            store
+                .load_rebalance_meta()
+                .await
+                .map_err(|e| s3_error!(InternalError, "failed to refresh rebalance metadata before decommission start: {}", e))?;
+            let decommission_running = store.is_decommission_running().await;
+            let rebalance_running = store.is_rebalance_started().await;
+            if decommission_running {
+                log_pool_request_rejected_with_context(
+                    "start_decommission",
+                    "decommission_already_running",
+                    &request_id,
+                    &actor,
+                    &remote_addr,
+                );
+            } else if rebalance_running {
+                log_pool_request_rejected_with_context(
+                    "start_decommission",
+                    "rebalance_in_progress",
+                    &request_id,
+                    &actor,
+                    &remote_addr,
+                );
+            }
+            validate_start_decommission_guards(decommission_running, rebalance_running)?;
+
+            if !pools_indices.is_empty() {
+                let pool_context = format!("pools {:?}", &pools_indices);
+                store
+                    .decommission(ctx.clone(), pools_indices.clone())
+                    .await
+                    .map_err(ApiError::from)
+                    .map_err(|err| contextualize_admin_pool_api_error(err, "start decommission", &pool_context))?;
+            }
         }
 
         info!(
@@ -734,15 +891,23 @@ impl Operation for CancelDecommission {
             return Err(pool_admin_pool_not_found_error_with_audit("cancel decommission", &query.pool, audit));
         };
 
-        let Some(store) = resolve_object_store_handle() else {
-            return Err(decommission_admin_not_initialized_error_with_audit("cancel decommission", audit));
-        };
+        if let Some(client) = decommission_peer_target(&endpoints, idx, "cancel decommission", audit)? {
+            client
+                .decommission_cancel(idx)
+                .await
+                .map_err(ApiError::from)
+                .map_err(|err| contextualize_admin_pool_api_error(err, "cancel decommission", format!("pool {idx}")))?;
+        } else {
+            let Some(store) = resolve_object_store_handle() else {
+                return Err(decommission_admin_not_initialized_error_with_audit("cancel decommission", audit));
+            };
 
-        store
-            .decommission_cancel(idx)
-            .await
-            .map_err(ApiError::from)
-            .map_err(|err| contextualize_admin_pool_api_error(err, "cancel decommission", format!("pool {idx}")))?;
+            store
+                .decommission_cancel(idx)
+                .await
+                .map_err(ApiError::from)
+                .map_err(|err| contextualize_admin_pool_api_error(err, "cancel decommission", format!("pool {idx}")))?;
+        }
 
         info!(
             event = EVENT_ADMIN_RESPONSE_EMITTED,
@@ -839,15 +1004,23 @@ impl Operation for ClearDecommission {
             return Err(pool_admin_pool_not_found_error_with_audit("clear decommission", &query.pool, audit));
         };
 
-        let Some(store) = resolve_object_store_handle() else {
-            return Err(decommission_admin_not_initialized_error_with_audit("clear decommission", audit));
-        };
+        if let Some(client) = decommission_peer_target(&endpoints, idx, "clear decommission", audit)? {
+            client
+                .clear_decommission(idx)
+                .await
+                .map_err(ApiError::from)
+                .map_err(|err| contextualize_admin_pool_api_error(err, "clear decommission", format!("pool {idx}")))?;
+        } else {
+            let Some(store) = resolve_object_store_handle() else {
+                return Err(decommission_admin_not_initialized_error_with_audit("clear decommission", audit));
+            };
 
-        store
-            .clear_decommission(idx)
-            .await
-            .map_err(ApiError::from)
-            .map_err(|err| contextualize_admin_pool_api_error(err, "clear decommission", format!("pool {idx}")))?;
+            store
+                .clear_decommission(idx)
+                .await
+                .map_err(ApiError::from)
+                .map_err(|err| contextualize_admin_pool_api_error(err, "clear decommission", format!("pool {idx}")))?;
+        }
 
         info!(
             event = EVENT_ADMIN_RESPONSE_EMITTED,
@@ -870,12 +1043,26 @@ impl Operation for ClearDecommission {
 mod pools_handler_tests {
     use super::{
         PoolAuditContext, contextualize_admin_pool_api_error, decommission_admin_not_initialized_error_with_audit,
-        has_duplicate_indices, parse_mutation_pool_query, parse_pool_idx_by_id, parse_status_pool_query,
-        pool_admin_missing_credentials_error, pool_admin_missing_credentials_error_with_request,
+        decommission_peer_target, has_duplicate_indices, parse_mutation_pool_query, parse_pool_idx_by_id,
+        parse_status_pool_query, pool_admin_missing_credentials_error, pool_admin_missing_credentials_error_with_request,
         pool_admin_pool_index_error_with_audit, pool_admin_pool_not_found_error_with_audit,
         pool_admin_pool_parse_error_with_audit, pool_admin_query_parse_error, pool_admin_query_parse_error_with_audit,
-        validate_start_decommission_guards,
+        validate_pool_mutation_leader, validate_start_decommission_guards,
     };
+    use crate::admin::{Endpoint, EndpointServerPools, Endpoints, PoolEndpoints};
+
+    fn test_pool_endpoints(is_local: bool) -> EndpointServerPools {
+        let mut endpoint = Endpoint::try_from("http://127.0.0.1:9000/disk").expect("test endpoint should parse");
+        endpoint.is_local = is_local;
+        EndpointServerPools::from(vec![PoolEndpoints {
+            legacy: false,
+            set_count: 1,
+            drives_per_set: 1,
+            cmd_line: "http://127.0.0.1:9000/disk".to_string(),
+            endpoints: Endpoints::from(vec![endpoint]),
+            platform: String::new(),
+        }])
+    }
 
     #[test]
     fn test_parse_pool_idx_by_id_rejects_non_numeric() {
@@ -969,6 +1156,41 @@ mod pools_handler_tests {
         assert!(validate_start_decommission_guards(false, false).is_ok());
     }
 
+    #[test]
+    fn test_validate_pool_mutation_leader_allows_local_first_endpoint() {
+        let endpoints = test_pool_endpoints(true);
+        let audit = PoolAuditContext::new("request", "actor", "remote");
+
+        assert!(validate_pool_mutation_leader(&endpoints, 0, "cancel decommission", audit).is_ok());
+    }
+
+    #[test]
+    fn test_decommission_peer_target_returns_none_for_local_first_endpoint() {
+        let endpoints = test_pool_endpoints(true);
+        let audit = PoolAuditContext::new("request", "actor", "remote");
+
+        let target = decommission_peer_target(&endpoints, 0, "start decommission", audit)
+            .expect("local first endpoint should resolve without peer lookup");
+
+        assert!(target.is_none());
+    }
+
+    #[test]
+    fn test_validate_pool_mutation_leader_rejects_remote_first_endpoint() {
+        let endpoints = test_pool_endpoints(false);
+        let audit = PoolAuditContext::new("request", "actor", "remote");
+
+        let err = validate_pool_mutation_leader(&endpoints, 0, "cancel decommission", audit)
+            .expect_err("remote first endpoint should reject mutation");
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::OperationAborted);
+        assert!(
+            err.message()
+                .expect("rejection should include message")
+                .contains("must be handled by its first endpoint")
+        );
+    }
+
     #[test]
     fn test_contextualize_admin_pool_api_error_preserves_code_and_adds_pool_context() {
         let err = crate::error::ApiError {

rustfs/src/admin/handlers/rebalance.rs

@@ -323,8 +323,8 @@ fn rebalance_used_pct(total: u64, available: u64) -> f64 {
     (total - bounded_available) as f64 / total as f64
 }
 
-fn rebalance_remaining_buckets(buckets: usize, rebalanced_buckets: usize) -> usize {
-    buckets.saturating_sub(rebalanced_buckets)
+fn rebalance_remaining_buckets(buckets: usize, _rebalanced_buckets: usize) -> usize {
+    buckets
 }
 
 fn rebalance_pool_used(disk_stats: &[DiskStat], idx: usize) -> f64 {
@@ -1090,7 +1090,7 @@ mod rebalance_handler_tests {
         assert_eq!(progress.num_objects, 3);
         assert_eq!(progress.num_versions, 5);
         assert_eq!(progress.bytes, 100);
-        assert_eq!(progress.remaining_buckets, 2);
+        assert_eq!(progress.remaining_buckets, 3);
         assert_eq!(progress.bucket, "bucket-b");
         assert_eq!(progress.object, "obj-1");
         assert_eq!(progress.elapsed, 50);
@@ -1157,9 +1157,9 @@ mod rebalance_handler_tests {
     }
 
     #[test]
-    fn test_rebalance_remaining_buckets_is_saturating_sub() {
-        assert_eq!(rebalance_remaining_buckets(10, 7), 3);
-        assert_eq!(rebalance_remaining_buckets(3, 10), 0);
+    fn test_rebalance_remaining_buckets_uses_pending_queue_len() {
+        assert_eq!(rebalance_remaining_buckets(10, 7), 10);
+        assert_eq!(rebalance_remaining_buckets(3, 10), 3);
     }
 
     #[test]
@@ -1227,7 +1227,7 @@ mod rebalance_handler_tests {
         assert_eq!(active.used, 0.5);
         assert_eq!(active.progress.as_ref().unwrap().bucket, "bucket-b");
         assert_eq!(active.progress.as_ref().unwrap().object, "obj-2");
-        assert_eq!(active.progress.as_ref().unwrap().remaining_buckets, 1);
+        assert_eq!(active.progress.as_ref().unwrap().remaining_buckets, 2);
 
         let inactive = &statuses[1];
         assert_eq!(inactive.id, 1);

rustfs/src/admin/route_policy.rs

@@ -235,6 +235,12 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
         DECOMMISSION,
         RouteRiskLevel::High,
     ),
+    admin(
+        HttpMethod::Get,
+        "/rustfs/admin/v3/decommission/status",
+        DECOMMISSION,
+        RouteRiskLevel::Sensitive,
+    ),
     admin(HttpMethod::Post, "/rustfs/admin/v3/pools/cancel", DECOMMISSION, RouteRiskLevel::High),
     admin(HttpMethod::Post, "/rustfs/admin/v3/pools/clear", DECOMMISSION, RouteRiskLevel::High),
     admin(HttpMethod::Post, "/rustfs/admin/v3/rebalance/start", REBALANCE, RouteRiskLevel::High),

rustfs/src/admin/route_registration_test.rs

@@ -173,6 +173,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
         admin_route(Method::GET, "/v3/metrics"),
         admin_route(Method::GET, "/v3/pools/list"),
         admin_route(Method::GET, "/v3/pools/status"),
+        admin_route(Method::GET, "/v3/decommission/status"),
         admin_route(Method::POST, "/v3/pools/decommission"),
         admin_route(Method::POST, "/v3/pools/cancel"),
         admin_route(Method::POST, "/v3/pools/clear"),
@@ -1036,6 +1037,7 @@ fn test_register_routes_cover_representative_admin_paths() {
     assert_route(&router, Method::GET, &admin_path("/v3/metrics"));
 
     assert_route(&router, Method::GET, &admin_path("/v3/pools/list"));
+    assert_route(&router, Method::GET, &admin_path("/v3/decommission/status"));
     assert_route(&router, Method::POST, &admin_path("/v3/rebalance/start"));
     assert_route(&router, Method::GET, &admin_path("/v3/rebalance/status"));
     assert_route(&router, Method::POST, &admin_path("/v3/heal/"));

rustfs/src/app/admin_usecase.rs

@@ -17,7 +17,7 @@
 use super::ECStore;
 use super::EndpointServerPools;
 use super::get_server_info;
-use super::{PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free};
+use super::{PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free};
 use super::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend};
 use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
 use crate::capacity::resolve_admin_used_capacity;
@@ -81,6 +81,8 @@ pub struct AdminPoolDecommissionInfo {
     pub prefix: String,
     #[serde(rename = "object")]
     pub object: String,
+    #[serde(rename = "stage")]
+    pub stage: String,
     #[serde(rename = "objectsDecommissioned")]
     pub items_decommissioned: usize,
     #[serde(rename = "objectsDecommissionedFailed")]
@@ -111,24 +113,58 @@ pub struct AdminPoolStatus {
     pub used: f64,
     #[serde(rename = "status")]
     pub status: String,
+    #[serde(rename = "decommissionStatus")]
+    pub decommission_status: String,
+    #[serde(rename = "rebalanceStatus")]
+    pub rebalance_status: String,
     #[serde(rename = "decommissionInfo")]
     pub decommission: Option<AdminPoolDecommissionInfo>,
 }
 
 pub type AdminPoolListItem = AdminPoolStatus;
 
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct AdminDecommissionPoolStatus {
+    #[serde(rename = "id")]
+    pub id: usize,
+    #[serde(rename = "cmdline")]
+    pub cmd_line: String,
+    #[serde(rename = "status")]
+    pub status: String,
+    #[serde(rename = "poolStatus")]
+    pub pool_status: String,
+    #[serde(rename = "decommissionInfo")]
+    pub decommission: Option<AdminPoolDecommissionInfo>,
+}
+
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct AdminDecommissionStatus {
+    #[serde(rename = "pools")]
+    pub pools: Vec<AdminDecommissionPoolStatus>,
+}
+
 #[derive(Clone, Default)]
 pub struct DefaultAdminUsecase {
     context: Option<Arc<AppContext>>,
 }
 
 impl DefaultAdminUsecase {
-    const POOL_STATUS_ACTIVE: &'static str = "active";
     const POOL_STATUS_CANCELED: &'static str = "canceled";
     const POOL_STATUS_COMPLETE: &'static str = "complete";
     const POOL_STATUS_FAILED: &'static str = "failed";
     const POOL_STATUS_QUEUED: &'static str = "queued";
     const POOL_STATUS_RUNNING: &'static str = "running";
+    const POOL_STATUS_UNKNOWN: &'static str = "unknown";
+    const POOL_STATE_ACTIVE: &'static str = "active";
+    const POOL_STATE_BLOCKED: &'static str = "blocked";
+    const POOL_STATE_DECOMMISSIONED: &'static str = "decommissioned";
+    const POOL_STATE_DECOMMISSIONING: &'static str = "decommissioning";
+    const REBALANCE_STATUS_COMPLETED: &'static str = "completed";
+    const REBALANCE_STATUS_FAILED: &'static str = "failed";
+    const REBALANCE_STATUS_NONE: &'static str = "none";
+    const REBALANCE_STATUS_STARTED: &'static str = "started";
+    const REBALANCE_STATUS_STOPPING: &'static str = "stopping";
+    const REBALANCE_STATUS_STOPPED: &'static str = "stopped";
 
     #[cfg(test)]
     pub fn without_context() -> Self {
@@ -277,19 +313,19 @@ impl DefaultAdminUsecase {
     }
 
     pub async fn execute_list_pools(&self) -> AdminUsecaseResult<Vec<AdminPoolListItem>> {
+        let Some(store) = self.object_store() else {
+            return Err(Self::app_error(S3ErrorCode::InternalError, "Not init"));
+        };
         let pool_statuses = self.execute_list_pool_statuses().await?;
-        Ok(pool_statuses.into_iter().map(Self::pool_list_item_from_status).collect())
+        let mut items = Vec::with_capacity(pool_statuses.len());
+        for status in pool_statuses {
+            let rebalance_status = store.pool_rebalance_status(status.id).await;
+            items.push(Self::pool_list_item_from_status(status, rebalance_status));
+        }
+        Ok(items)
     }
 
-    pub async fn execute_query_pool_status(&self, req: QueryPoolStatusRequest) -> AdminUsecaseResult<AdminPoolStatus> {
-        let Some(endpoints) = self.endpoints() else {
-            return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
-        };
-
-        if endpoints.legacy() {
-            return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
-        }
-
+    fn resolve_pool_index(&self, req: &QueryPoolStatusRequest, endpoints: &EndpointServerPools) -> AdminUsecaseResult<usize> {
         let has_idx = if req.by_id {
             Self::parse_pool_idx_by_id(&req.pool, endpoints.as_ref().len())
         } else {
@@ -301,18 +337,62 @@ impl DefaultAdminUsecase {
             return Err(Self::app_error_default(S3ErrorCode::InvalidArgument));
         };
 
+        Ok(idx)
+    }
+
+    pub async fn execute_query_pool_status(&self, req: QueryPoolStatusRequest) -> AdminUsecaseResult<AdminPoolStatus> {
+        let Some(endpoints) = self.endpoints() else {
+            return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
+        };
+
+        if endpoints.legacy() {
+            return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
+        }
+
+        let idx = self.resolve_pool_index(&req, &endpoints)?;
+
         let Some(store) = self.object_store() else {
             return Err(Self::app_error(S3ErrorCode::InternalError, "Not init"));
         };
 
-        store
-            .status(idx)
-            .await
-            .map(Self::pool_list_item_from_status)
-            .map_err(ApiError::from)
+        let status = store.status(idx).await.map_err(ApiError::from)?;
+        let rebalance_status = store.pool_rebalance_status(idx).await;
+        Ok(Self::pool_list_item_from_status(status, rebalance_status))
     }
 
-    fn pool_list_item_from_status(status: PoolStatus) -> AdminPoolListItem {
+    pub async fn execute_list_decommission_status(&self) -> AdminUsecaseResult<AdminDecommissionStatus> {
+        let pool_statuses = self.execute_list_pool_statuses().await?;
+        Ok(AdminDecommissionStatus {
+            pools: pool_statuses
+                .into_iter()
+                .map(Self::decommission_pool_status_from_status)
+                .collect(),
+        })
+    }
+
+    pub async fn execute_query_decommission_status(
+        &self,
+        req: QueryPoolStatusRequest,
+    ) -> AdminUsecaseResult<AdminDecommissionPoolStatus> {
+        let Some(endpoints) = self.endpoints() else {
+            return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
+        };
+
+        if endpoints.legacy() {
+            return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
+        }
+
+        let idx = self.resolve_pool_index(&req, &endpoints)?;
+
+        let Some(store) = self.object_store() else {
+            return Err(Self::app_error(S3ErrorCode::InternalError, "Not init"));
+        };
+
+        let status = store.status(idx).await.map_err(ApiError::from)?;
+        Ok(Self::decommission_pool_status_from_status(status))
+    }
+
+    fn pool_list_item_from_status(status: PoolStatus, rebalance_status: (RebalStatus, bool)) -> AdminPoolListItem {
         let PoolStatus {
             id,
             cmd_line,
@@ -322,6 +402,7 @@ impl DefaultAdminUsecase {
         let total_size = decommission.as_ref().map(|info| info.total_size).unwrap_or_default();
         let current_size = decommission.as_ref().map(|info| info.current_size).unwrap_or_default();
         let used_size = total_size.saturating_sub(current_size);
+        let pool_state = Self::pool_lifecycle_state(decommission.as_ref());
 
         AdminPoolStatus {
             id,
@@ -331,19 +412,64 @@ impl DefaultAdminUsecase {
             current_size,
             used_size,
             used: Self::used_ratio(total_size, used_size),
-            status: Self::pool_list_status(decommission.as_ref()).to_string(),
+            status: pool_state.to_string(),
+            decommission_status: Self::pool_decommission_status(decommission.as_ref()).to_string(),
+            rebalance_status: Self::pool_rebalance_status(rebalance_status).to_string(),
             decommission: decommission.map(Self::admin_decommission_info_from_pool),
         }
     }
 
-    fn pool_list_status(decommission: Option<&PoolDecommissionInfo>) -> &'static str {
+    fn pool_lifecycle_state(decommission: Option<&PoolDecommissionInfo>) -> &'static str {
+        match decommission {
+            Some(info) if info.complete => Self::POOL_STATE_DECOMMISSIONED,
+            Some(info) if info.failed || info.canceled => Self::POOL_STATE_BLOCKED,
+            Some(_) => Self::POOL_STATE_DECOMMISSIONING,
+            None => Self::POOL_STATE_ACTIVE,
+        }
+    }
+
+    fn pool_decommission_status(decommission: Option<&PoolDecommissionInfo>) -> &'static str {
         match decommission {
             Some(info) if info.complete => Self::POOL_STATUS_COMPLETE,
             Some(info) if info.failed => Self::POOL_STATUS_FAILED,
             Some(info) if info.canceled => Self::POOL_STATUS_CANCELED,
             Some(info) if info.queued => Self::POOL_STATUS_QUEUED,
             Some(info) if info.start_time.is_some() => Self::POOL_STATUS_RUNNING,
-            _ => Self::POOL_STATUS_ACTIVE,
+            Some(_) => Self::POOL_STATUS_UNKNOWN,
+            None => Self::REBALANCE_STATUS_NONE,
+        }
+    }
+
+    fn pool_rebalance_status((status, stopping): (RebalStatus, bool)) -> &'static str {
+        if stopping {
+            return Self::REBALANCE_STATUS_STOPPING;
+        }
+
+        match status {
+            RebalStatus::None => Self::REBALANCE_STATUS_NONE,
+            RebalStatus::Started => Self::REBALANCE_STATUS_STARTED,
+            RebalStatus::Completed => Self::REBALANCE_STATUS_COMPLETED,
+            RebalStatus::Stopped => Self::REBALANCE_STATUS_STOPPED,
+            RebalStatus::Failed => Self::REBALANCE_STATUS_FAILED,
+        }
+    }
+
+    fn decommission_pool_status_from_status(status: PoolStatus) -> AdminDecommissionPoolStatus {
+        let PoolStatus {
+            id,
+            cmd_line,
+            decommission,
+            ..
+        } = status;
+        let pool_status = Self::pool_lifecycle_state(decommission.as_ref()).to_string();
+        let status = Self::pool_decommission_status(decommission.as_ref()).to_string();
+
+        AdminDecommissionPoolStatus {
+            id,
+            cmd_line,
+            status,
+            pool_status,
+            decommission: decommission.map(Self::admin_decommission_info_from_pool),
         }
     }
 
@@ -363,6 +489,7 @@ impl DefaultAdminUsecase {
             bucket: info.bucket,
             prefix: info.prefix,
             object: info.object,
+            stage: info.stage,
             items_decommissioned: info.items_decommissioned,
             items_decommission_failed: info.items_decommission_failed,
             bytes_done: info.bytes_done,
@@ -446,7 +573,7 @@ mod tests {
     }
 
     #[test]
-    fn admin_pool_list_item_maps_capacity_and_active_status() {
+    fn admin_pool_list_item_maps_capacity_and_unknown_decommission_status() {
         let now = OffsetDateTime::UNIX_EPOCH;
         let pool = PoolStatus {
             id: 2,
@@ -459,24 +586,29 @@ mod tests {
             }),
         };
 
-        let item = DefaultAdminUsecase::pool_list_item_from_status(pool);
+        let item = DefaultAdminUsecase::pool_list_item_from_status(pool, (RebalStatus::None, false));
 
         assert_eq!(item.id, 2);
         assert_eq!(item.total_size, 1_000);
         assert_eq!(item.current_size, 250);
         assert_eq!(item.used_size, 750);
         assert!((item.used - 0.75).abs() < f64::EPSILON);
-        assert_eq!(item.status, "active");
+        assert_eq!(item.status, "decommissioning");
+        assert_eq!(item.decommission_status, "unknown");
+        assert_eq!(item.rebalance_status, "none");
     }
 
     #[test]
     fn admin_pool_list_item_serializes_admin_api_fields() {
-        let item = DefaultAdminUsecase::pool_list_item_from_status(PoolStatus {
-            id: 1,
-            cmd_line: "pool-1".to_string(),
-            last_update: OffsetDateTime::UNIX_EPOCH,
-            decommission: None,
-        });
+        let item = DefaultAdminUsecase::pool_list_item_from_status(
+            PoolStatus {
+                id: 1,
+                cmd_line: "pool-1".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: None,
+            },
+            (RebalStatus::Completed, false),
+        );
 
         let value = serde_json::to_value(item).unwrap();
 
@@ -491,6 +623,8 @@ mod tests {
                 "usedSize": 0,
                 "used": 0.0,
                 "status": "active",
+                "decommissionStatus": "none",
+                "rebalanceStatus": "completed",
                 "decommissionInfo": null
             })
         );
@@ -509,7 +643,7 @@ mod tests {
             }),
         };
 
-        let item = DefaultAdminUsecase::pool_list_item_from_status(pool);
+        let item = DefaultAdminUsecase::pool_list_item_from_status(pool, (RebalStatus::None, false));
 
         assert_eq!(item.total_size, 100);
         assert_eq!(item.current_size, 150);
@@ -531,33 +665,40 @@ mod tests {
             }),
         };
 
-        let item = DefaultAdminUsecase::pool_list_item_from_status(pool);
+        let item = DefaultAdminUsecase::pool_list_item_from_status(pool, (RebalStatus::Started, false));
 
-        assert_eq!(item.status, "running");
+        assert_eq!(item.status, "decommissioning");
+        assert_eq!(item.decommission_status, "running");
+        assert_eq!(item.rebalance_status, "started");
     }
 
     #[test]
     fn admin_pool_list_item_exposes_queued_decommission_state() {
-        let item = DefaultAdminUsecase::pool_list_item_from_status(PoolStatus {
-            id: 3,
-            cmd_line: "pool-3".to_string(),
-            last_update: OffsetDateTime::UNIX_EPOCH,
-            decommission: Some(PoolDecommissionInfo {
-                queued: true,
-                queued_buckets: vec!["bucket-a".to_string(), ".rustfs.sys/config".to_string()],
-                decommissioned_buckets: vec!["bucket-done".to_string()],
-                bucket: "bucket-a".to_string(),
-                prefix: "prefix/".to_string(),
-                object: "object.txt".to_string(),
-                items_decommissioned: 7,
-                items_decommission_failed: 1,
-                bytes_done: 1024,
-                bytes_failed: 64,
-                ..Default::default()
-            }),
-        });
+        let item = DefaultAdminUsecase::pool_list_item_from_status(
+            PoolStatus {
+                id: 3,
+                cmd_line: "pool-3".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: Some(PoolDecommissionInfo {
+                    queued: true,
+                    queued_buckets: vec!["bucket-a".to_string(), ".rustfs.sys/config".to_string()],
+                    decommissioned_buckets: vec!["bucket-done".to_string()],
+                    bucket: "bucket-a".to_string(),
+                    prefix: "prefix/".to_string(),
+                    object: "object.txt".to_string(),
+                    stage: "migrate_object".to_string(),
+                    items_decommissioned: 7,
+                    items_decommission_failed: 1,
+                    bytes_done: 1024,
+                    bytes_failed: 64,
+                    ..Default::default()
+                }),
+            },
+            (RebalStatus::None, false),
+        );
 
-        assert_eq!(item.status, "queued");
+        assert_eq!(item.status, "decommissioning");
+        assert_eq!(item.decommission_status, "queued");
         let value = serde_json::to_value(item).expect("admin pool status should serialize");
         assert_eq!(value["decommissionInfo"]["queued"], true);
         assert_eq!(
@@ -568,6 +709,7 @@ mod tests {
         assert_eq!(value["decommissionInfo"]["bucket"], "bucket-a");
         assert_eq!(value["decommissionInfo"]["prefix"], "prefix/");
         assert_eq!(value["decommissionInfo"]["object"], "object.txt");
+        assert_eq!(value["decommissionInfo"]["stage"], "migrate_object");
         assert_eq!(value["decommissionInfo"]["objectsDecommissioned"], 7);
         assert_eq!(value["decommissionInfo"]["objectsDecommissionedFailed"], 1);
         assert_eq!(value["decommissionInfo"]["bytesDecommissioned"], 1024);
@@ -577,28 +719,128 @@ mod tests {
 
     #[test]
     fn admin_pool_list_item_maps_terminal_decommission_statuses() {
-        let complete = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo {
+        let complete = DefaultAdminUsecase::pool_decommission_status(Some(&PoolDecommissionInfo {
             complete: true,
             ..Default::default()
         }));
-        let failed = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo {
+        let failed = DefaultAdminUsecase::pool_decommission_status(Some(&PoolDecommissionInfo {
             failed: true,
             ..Default::default()
         }));
-        let canceled = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo {
+        let canceled = DefaultAdminUsecase::pool_decommission_status(Some(&PoolDecommissionInfo {
             canceled: true,
             ..Default::default()
         }));
-        let queued = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo {
+        let queued = DefaultAdminUsecase::pool_decommission_status(Some(&PoolDecommissionInfo {
             queued: true,
             ..Default::default()
         }));
-        let idle = DefaultAdminUsecase::pool_list_status(None);
+        let idle = DefaultAdminUsecase::pool_decommission_status(None);
 
         assert_eq!(complete, "complete");
         assert_eq!(failed, "failed");
         assert_eq!(canceled, "canceled");
         assert_eq!(queued, "queued");
-        assert_eq!(idle, "active");
+        assert_eq!(idle, "none");
+    }
+
+    #[test]
+    fn admin_pool_list_item_keeps_rebalance_failure_separate_from_pool_state() {
+        let item = DefaultAdminUsecase::pool_list_item_from_status(
+            PoolStatus {
+                id: 1,
+                cmd_line: "pool-1".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: None,
+            },
+            (RebalStatus::Failed, false),
+        );
+
+        assert_eq!(item.status, "active");
+        assert_eq!(item.decommission_status, "none");
+        assert_eq!(item.rebalance_status, "failed");
+    }
+
+    #[test]
+    fn admin_pool_list_item_maps_stopping_rebalance_status() {
+        let item = DefaultAdminUsecase::pool_list_item_from_status(
+            PoolStatus {
+                id: 1,
+                cmd_line: "pool-1".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: None,
+            },
+            (RebalStatus::Started, true),
+        );
+
+        assert_eq!(item.status, "active");
+        assert_eq!(item.decommission_status, "none");
+        assert_eq!(item.rebalance_status, "stopping");
+    }
+
+    #[test]
+    fn admin_pool_lifecycle_state_distinguishes_decommission_terminal_states() {
+        let complete = DefaultAdminUsecase::pool_lifecycle_state(Some(&PoolDecommissionInfo {
+            complete: true,
+            ..Default::default()
+        }));
+        let failed = DefaultAdminUsecase::pool_lifecycle_state(Some(&PoolDecommissionInfo {
+            failed: true,
+            ..Default::default()
+        }));
+        let canceled = DefaultAdminUsecase::pool_lifecycle_state(Some(&PoolDecommissionInfo {
+            canceled: true,
+            ..Default::default()
+        }));
+
+        assert_eq!(complete, "decommissioned");
+        assert_eq!(failed, "blocked");
+        assert_eq!(canceled, "blocked");
+    }
+
+    #[test]
+    fn admin_decommission_status_serializes_task_status_and_pool_status() {
+        let item = DefaultAdminUsecase::decommission_pool_status_from_status(PoolStatus {
+            id: 3,
+            cmd_line: "pool-3".to_string(),
+            last_update: OffsetDateTime::UNIX_EPOCH,
+            decommission: Some(PoolDecommissionInfo {
+                failed: true,
+                ..Default::default()
+            }),
+        });
+
+        let value = serde_json::to_value(item).expect("decommission status should serialize");
+
+        assert_eq!(
+            value,
+            serde_json::json!({
+                "id": 3,
+                "cmdline": "pool-3",
+                "status": "failed",
+                "poolStatus": "blocked",
+                "decommissionInfo": {
+                    "startTime": null,
+                    "startSize": 0,
+                    "totalSize": 0,
+                    "currentSize": 0,
+                    "complete": false,
+                    "failed": true,
+                    "canceled": false,
+                    "queued": false,
+                    "queuedBuckets": [],
+                    "decommissionedBuckets": [],
+                    "bucket": "",
+                    "prefix": "",
+                    "object": "",
+                    "stage": "",
+                    "objectsDecommissioned": 0,
+                    "objectsDecommissionedFailed": 0,
+                    "bytesDecommissioned": 0,
+                    "bytesDecommissionedFailed": 0,
+                    "waitingReason": null
+                }
+            })
+        );
     }
 }

rustfs/src/app/mod.rs

@@ -105,6 +105,7 @@ pub(crate) type ObjectInfo = <ECStore as rustfs_storage_api::ObjectOperations>::
 pub(crate) type ObjectOptions = <ECStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
 pub(crate) type PoolDecommissionInfo = ecstore_capacity::PoolDecommissionInfo;
 pub(crate) type PoolStatus = ecstore_capacity::PoolStatus;
+pub(crate) type RebalStatus = crate::storage::ecstore_rebalance::RebalStatus;
 pub(crate) type StorageError = crate::storage::StorageError;
 pub(crate) type Error = StorageError;
 pub(crate) type TierConfigMgr = crate::storage::TierConfigMgr;

rustfs/src/storage/rpc/node_service.rs

@@ -13,11 +13,11 @@
 // limitations under the License.
 
 use super::super::{
-    CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, FileInfoVersions, LocalPeerS3Client, MetricType,
-    PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG,
-    SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StoragePeerS3ClientExt as _, UpdateMetadataOpts, all_local_disk_path,
-    collect_local_metrics, find_local_disk_by_ref, get_local_server_property, load_bucket_metadata,
-    reload_transition_tier_config, resolve_object_store_handle, set_bucket_metadata,
+    CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, ECStore, Error, FileInfoVersions,
+    LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions,
+    SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StoragePeerS3ClientExt as _,
+    UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, get_local_server_property,
+    load_bucket_metadata, reload_transition_tier_config, resolve_object_store_handle, set_bucket_metadata,
 };
 use crate::admin::service::{
     config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot},
@@ -46,6 +46,7 @@ use std::{collections::HashMap, io::Cursor, pin::Pin, sync::Arc};
 use tokio::spawn;
 use tokio::sync::mpsc;
 use tokio_stream::wrappers::ReceiverStream;
+use tokio_util::sync::CancellationToken;
 use tonic::{Request, Response, Status, Streaming};
 use tracing::{debug, error, info, warn};
 
@@ -140,6 +141,23 @@ fn stop_rebalance_response(result: super::super::Result<()>) -> StopRebalanceRes
     }
 }
 
+fn ensure_rpc_decommission_local_leader(store: &ECStore, idx: usize) -> super::super::Result<()> {
+    let endpoints = store.endpoints();
+    let endpoint = endpoints
+        .as_ref()
+        .get(idx)
+        .and_then(|pool| pool.endpoints.as_ref().first())
+        .ok_or_else(|| Error::other(format!("invalid decommission pool index {idx} for {} pools", endpoints.as_ref().len())))?;
+
+    if !endpoint.is_local {
+        return Err(Error::other(format!(
+            "decommission for pool {idx} must run on the pool first endpoint {endpoint}"
+        )));
+    }
+
+    Ok(())
+}
+
 #[path = "bucket.rs"]
 mod bucket;
 #[path = "disk.rs"]
@@ -1057,6 +1075,101 @@ impl Node for NodeService {
         }))
     }
 
+    async fn start_decommission(
+        &self,
+        request: Request<StartDecommissionRequest>,
+    ) -> Result<Response<StartDecommissionResponse>, Status> {
+        let Some(store) = resolve_object_store_handle() else {
+            return Ok(Response::new(StartDecommissionResponse {
+                success: false,
+                error_info: Some("errServerNotInitialized".to_string()),
+            }));
+        };
+
+        let mut indices = Vec::with_capacity(request.get_ref().pool_indices.len());
+        for idx in request.into_inner().pool_indices {
+            indices.push(
+                usize::try_from(idx)
+                    .map_err(|_| Status::invalid_argument(format!("decommission pool index {idx} exceeds local range")))?,
+            );
+        }
+
+        match store.decommission(CancellationToken::new(), indices).await {
+            Ok(()) => Ok(Response::new(StartDecommissionResponse {
+                success: true,
+                error_info: None,
+            })),
+            Err(err) => Ok(Response::new(StartDecommissionResponse {
+                success: false,
+                error_info: Some(err.to_string()),
+            })),
+        }
+    }
+
+    async fn cancel_decommission(
+        &self,
+        request: Request<CancelDecommissionRequest>,
+    ) -> Result<Response<CancelDecommissionResponse>, Status> {
+        let Some(store) = resolve_object_store_handle() else {
+            return Ok(Response::new(CancelDecommissionResponse {
+                success: false,
+                error_info: Some("errServerNotInitialized".to_string()),
+            }));
+        };
+
+        let idx = usize::try_from(request.into_inner().pool_index)
+            .map_err(|_| Status::invalid_argument("decommission pool index exceeds local range"))?;
+        if let Err(err) = ensure_rpc_decommission_local_leader(&store, idx) {
+            return Ok(Response::new(CancelDecommissionResponse {
+                success: false,
+                error_info: Some(err.to_string()),
+            }));
+        }
+
+        match store.decommission_cancel(idx).await {
+            Ok(()) => Ok(Response::new(CancelDecommissionResponse {
+                success: true,
+                error_info: None,
+            })),
+            Err(err) => Ok(Response::new(CancelDecommissionResponse {
+                success: false,
+                error_info: Some(err.to_string()),
+            })),
+        }
+    }
+
+    async fn clear_decommission(
+        &self,
+        request: Request<ClearDecommissionRequest>,
+    ) -> Result<Response<ClearDecommissionResponse>, Status> {
+        let Some(store) = resolve_object_store_handle() else {
+            return Ok(Response::new(ClearDecommissionResponse {
+                success: false,
+                error_info: Some("errServerNotInitialized".to_string()),
+            }));
+        };
+
+        let idx = usize::try_from(request.into_inner().pool_index)
+            .map_err(|_| Status::invalid_argument("decommission pool index exceeds local range"))?;
+        if let Err(err) = ensure_rpc_decommission_local_leader(&store, idx) {
+            return Ok(Response::new(ClearDecommissionResponse {
+                success: false,
+                error_info: Some(err.to_string()),
+            }));
+        }
+
+        match store.clear_decommission(idx).await {
+            Ok(()) => Ok(Response::new(ClearDecommissionResponse {
+                success: true,
+                error_info: None,
+            })),
+            Err(err) => Ok(Response::new(ClearDecommissionResponse {
+                success: false,
+                error_info: Some(err.to_string()),
+            })),
+        }
+    }
+
     async fn load_transition_tier_config(
         &self,
         _request: Request<LoadTransitionTierConfigRequest>,