feat(ecstore): implement decommission and rebalance (#2281)

Co-authored-by: weisd <im@weisd.in>
Co-authored-by: houseme <housemecn@gmail.com>

rustfs/src/admin/handlers/pools.rs

@@ -35,10 +35,85 @@ use crate::{
 use hyper::Method;
 use rustfs_ecstore::new_object_layer_fn;
 
+use std::collections::HashSet;
+
 fn endpoints_from_context() -> Option<rustfs_ecstore::endpoints::EndpointServerPools> {
     resolve_endpoints_handle()
 }
 
+fn validate_start_decommission_guards(decommission_running: bool, rebalance_running: bool) -> s3s::S3Result<()> {
+    if decommission_running {
+        return Err(s3_error!(InvalidRequest, "DecommissionAlreadyRunning"));
+    }
+
+    if rebalance_running {
+        return Err(S3Error::with_message(
+            S3ErrorCode::OperationAborted,
+            "Decommission cannot be started, rebalance is already in progress".to_string(),
+        ));
+    }
+
+    Ok(())
+}
+
+fn contextualize_admin_pool_api_error(
+    err: crate::error::ApiError,
+    operation: &str,
+    pool_context: impl std::fmt::Display,
+) -> crate::error::ApiError {
+    crate::error::ApiError {
+        code: err.code,
+        message: format!("admin {operation} failed for {pool_context}: {}", err.message),
+        source: err.source,
+    }
+}
+
+fn decommission_admin_not_initialized_error(operation: &str) -> S3Error {
+    S3Error::with_message(S3ErrorCode::InternalError, format!("Failed to {operation}: object layer not initialized"))
+}
+
+fn pool_admin_missing_credentials_error(operation: &str) -> S3Error {
+    S3Error::with_message(S3ErrorCode::InvalidRequest, format!("Failed to {operation}: missing credentials"))
+}
+
+fn pool_admin_query_parse_error(operation: &str) -> S3Error {
+    S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Failed to {operation}: invalid query parameters"))
+}
+
+fn pool_admin_pool_parse_error(operation: &str, pool: &str) -> S3Error {
+    S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Failed to {operation}: invalid pool `{pool}`"))
+}
+
+fn pool_admin_pool_not_found_error(operation: &str, pool: &str) -> S3Error {
+    S3Error::with_message(
+        S3ErrorCode::InvalidArgument,
+        format!("Failed to {operation}: pool `{pool}` was not found"),
+    )
+}
+
+fn pool_admin_pool_index_error(operation: &str, idx: usize, pool_count: usize) -> S3Error {
+    S3Error::with_message(
+        S3ErrorCode::InvalidArgument,
+        format!("Failed to {operation}: pool index {idx} is out of range for {pool_count} pools"),
+    )
+}
+
+fn parse_pool_idx_by_id(pool: &str, endpoint_count: usize) -> Option<usize> {
+    let idx = pool.parse::<usize>().ok()?;
+    (idx < endpoint_count).then_some(idx)
+}
+
+fn dedup_indices(indices: &[usize]) -> Vec<usize> {
+    let mut seen = HashSet::with_capacity(indices.len());
+    let mut output = Vec::with_capacity(indices.len());
+    for idx in indices {
+        if seen.insert(*idx) {
+            output.push(*idx);
+        }
+    }
+    output
+}
+
 pub fn register_pool_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
     r.insert(
         Method::GET,
@@ -77,7 +152,7 @@ impl Operation for ListPools {
         warn!("handle ListPools");
 
         let Some(input_cred) = req.credentials else {
-            return Err(s3_error!(InvalidRequest, "get cred failed"));
+            return Err(pool_admin_missing_credentials_error("list pools"));
         };
 
         let (cred, owner) =
@@ -127,7 +202,7 @@ impl Operation for StatusPool {
         warn!("handle StatusPool");
 
         let Some(input_cred) = req.credentials else {
-            return Err(s3_error!(InvalidRequest, "get cred failed"));
+            return Err(pool_admin_missing_credentials_error("load pool status"));
         };
 
         let (cred, owner) =
@@ -149,7 +224,7 @@ impl Operation for StatusPool {
         let query = {
             if let Some(query) = req.uri.query() {
                 let input: StatusPoolQuery =
-                    from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
+                    from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("load pool status"))?;
                 input
             } else {
                 StatusPoolQuery::default()
@@ -185,7 +260,7 @@ impl Operation for StartDecommission {
         warn!("handle StartDecommission");
 
         let Some(input_cred) = req.credentials else {
-            return Err(s3_error!(InvalidRequest, "get cred failed"));
+            return Err(pool_admin_missing_credentials_error("start decommission"));
         };
 
         let (cred, owner) =
@@ -210,27 +285,15 @@ impl Operation for StartDecommission {
         }
 
         let Some(store) = new_object_layer_fn() else {
-            return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
+            return Err(decommission_admin_not_initialized_error("start decommission"));
         };
 
-        if store.is_decommission_running().await {
-            return Err(S3Error::with_message(
-                S3ErrorCode::InvalidRequest,
-                "DecommissionAlreadyRunning".to_string(),
-            ));
-        }
-
-        if store.is_rebalance_started().await {
-            return Err(S3Error::with_message(
-                S3ErrorCode::OperationAborted,
-                "Decommission cannot be started, rebalance is already in progress".to_string(),
-            ));
-        }
+        validate_start_decommission_guards(store.is_decommission_running().await, store.is_rebalance_started().await)?;
 
         let query = {
             if let Some(query) = req.uri.query() {
                 let input: StatusPoolQuery =
-                    from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
+                    from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("start decommission"))?;
                 input
             } else {
                 StatusPoolQuery::default()
@@ -239,40 +302,38 @@ impl Operation for StartDecommission {
         let is_byid = query.by_id.as_str() == "true";
 
         let pools: Vec<&str> = query.pool.split(",").collect();
-        let mut pools_indices = Vec::with_capacity(pools.len());
+        let mut parsed_indices = Vec::with_capacity(pools.len());
 
         let ctx = CancellationToken::new();
 
         for pool in pools.iter() {
             let idx = {
                 if is_byid {
-                    pool.parse::<usize>()
-                        .map_err(|_e| s3_error!(InvalidArgument, "pool parse failed"))?
+                    parse_pool_idx_by_id(pool, endpoints.as_ref().len())
+                        .ok_or_else(|| pool_admin_pool_parse_error("start decommission", pool))?
                 } else {
                     let Some(idx) = endpoints.get_pool_idx(pool) else {
-                        return Err(s3_error!(InvalidArgument, "pool parse failed"));
+                        return Err(pool_admin_pool_parse_error("start decommission", pool));
                     };
                     idx
                 }
             };
 
-            let mut has_found = None;
-            for (i, pool) in store.pools.iter().enumerate() {
-                if i == idx {
-                    has_found = Some(pool.clone());
-                    break;
-                }
+            if idx >= store.pools.len() {
+                return Err(pool_admin_pool_index_error("start decommission", idx, store.pools.len()));
             }
 
-            let Some(_p) = has_found else {
-                return Err(s3_error!(InvalidArgument));
-            };
-
-            pools_indices.push(idx);
+            parsed_indices.push(idx);
         }
+        let pools_indices = dedup_indices(&parsed_indices);
 
         if !pools_indices.is_empty() {
-            store.decommission(ctx.clone(), pools_indices).await.map_err(ApiError::from)?;
+            let pool_context = format!("pools {:?}", &pools_indices);
+            store
+                .decommission(ctx.clone(), pools_indices)
+                .await
+                .map_err(ApiError::from)
+                .map_err(|err| contextualize_admin_pool_api_error(err, "start decommission", &pool_context))?;
         }
 
         Ok(S3Response::new((StatusCode::OK, Body::default())))
@@ -289,7 +350,7 @@ impl Operation for CancelDecommission {
         warn!("handle CancelDecommission");
 
         let Some(input_cred) = req.credentials else {
-            return Err(s3_error!(InvalidRequest, "get cred failed"));
+            return Err(pool_admin_missing_credentials_error("cancel decommission"));
         };
 
         let (cred, owner) =
@@ -316,7 +377,7 @@ impl Operation for CancelDecommission {
         let query = {
             if let Some(query) = req.uri.query() {
                 let input: StatusPoolQuery =
-                    from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
+                    from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("cancel decommission"))?;
                 input
             } else {
                 StatusPoolQuery::default()
@@ -327,8 +388,7 @@ impl Operation for CancelDecommission {
 
         let has_idx = {
             if is_byid {
-                let a = query.pool.parse::<usize>().unwrap_or_default();
-                if a < endpoints.as_ref().len() { Some(a) } else { None }
+                parse_pool_idx_by_id(&query.pool, endpoints.as_ref().len())
             } else {
                 endpoints.get_pool_idx(&query.pool)
             }
@@ -336,15 +396,181 @@ impl Operation for CancelDecommission {
 
         let Some(idx) = has_idx else {
             warn!("specified pool {} not found, please specify a valid pool", &query.pool);
-            return Err(s3_error!(InvalidArgument));
+            return Err(pool_admin_pool_not_found_error("cancel decommission", &query.pool));
         };
 
         let Some(store) = new_object_layer_fn() else {
-            return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
+            return Err(decommission_admin_not_initialized_error("cancel decommission"));
         };
 
-        store.decommission_cancel(idx).await.map_err(ApiError::from)?;
+        store
+            .decommission_cancel(idx)
+            .await
+            .map_err(ApiError::from)
+            .map_err(|err| contextualize_admin_pool_api_error(err, "cancel decommission", format!("pool {idx}")))?;
 
         Ok(S3Response::new((StatusCode::OK, Body::default())))
     }
 }
+
+#[cfg(test)]
+mod pools_handler_tests {
+    use super::{
+        contextualize_admin_pool_api_error, decommission_admin_not_initialized_error, dedup_indices, parse_pool_idx_by_id,
+        pool_admin_missing_credentials_error, pool_admin_pool_index_error, pool_admin_pool_not_found_error,
+        pool_admin_pool_parse_error, pool_admin_query_parse_error, validate_start_decommission_guards,
+    };
+
+    #[test]
+    fn test_parse_pool_idx_by_id_rejects_non_numeric() {
+        assert_eq!(parse_pool_idx_by_id("invalid", 4), None);
+    }
+
+    #[test]
+    fn test_parse_pool_idx_by_id_rejects_out_of_range() {
+        assert_eq!(parse_pool_idx_by_id("4", 4), None);
+    }
+
+    #[test]
+    fn test_parse_pool_idx_by_id_rejects_empty_pool_count() {
+        assert_eq!(parse_pool_idx_by_id("0", 0), None);
+    }
+
+    #[test]
+    fn test_parse_pool_idx_by_id_accepts_valid_index() {
+        assert_eq!(parse_pool_idx_by_id("2", 4), Some(2));
+    }
+
+    #[test]
+    fn test_validate_start_decommission_guards_rejects_decommission_running() {
+        let err = validate_start_decommission_guards(true, false).expect_err("decommission running should be rejected");
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest);
+        assert_eq!(err.message(), Some("DecommissionAlreadyRunning"));
+    }
+
+    #[test]
+    fn test_validate_start_decommission_guards_rejects_rebalance_running() {
+        let err = validate_start_decommission_guards(false, true).expect_err("rebalance running should be rejected");
+        assert_eq!(err.code(), &s3s::S3ErrorCode::OperationAborted);
+        assert_eq!(err.message(), Some("Decommission cannot be started, rebalance is already in progress"));
+    }
+
+    #[test]
+    fn test_validate_start_decommission_guards_prefers_decommission_over_rebalance() {
+        let err = validate_start_decommission_guards(true, true).expect_err("decommission should be checked before rebalance");
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest);
+        assert_eq!(err.message(), Some("DecommissionAlreadyRunning"));
+    }
+
+    #[test]
+    fn test_validate_start_decommission_guards_allows_when_idle() {
+        assert!(validate_start_decommission_guards(false, false).is_ok());
+    }
+
+    #[test]
+    fn test_contextualize_admin_pool_api_error_preserves_code_and_adds_pool_context() {
+        let err = crate::error::ApiError {
+            code: s3s::S3ErrorCode::InvalidRequest,
+            message: "decommission already running".to_string(),
+            source: None,
+        };
+
+        let err = contextualize_admin_pool_api_error(err, "start decommission", "pools [1, 3]");
+
+        assert_eq!(err.code, s3s::S3ErrorCode::InvalidRequest);
+        assert_eq!(
+            err.message,
+            "admin start decommission failed for pools [1, 3]: decommission already running"
+        );
+    }
+
+    #[test]
+    fn test_contextualize_admin_pool_api_error_preserves_source() {
+        let err = contextualize_admin_pool_api_error(
+            crate::error::ApiError::other(std::io::Error::other("boom")),
+            "cancel decommission",
+            "pool 2",
+        );
+
+        assert!(err.message.contains("admin cancel decommission failed for pool 2"));
+        assert!(err.source.is_some());
+    }
+
+    #[test]
+    fn test_decommission_admin_not_initialized_error_formats_start_context() {
+        let err = decommission_admin_not_initialized_error("start decommission");
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError);
+        assert_eq!(err.message(), Some("Failed to start decommission: object layer not initialized"));
+    }
+
+    #[test]
+    fn test_decommission_admin_not_initialized_error_formats_cancel_context() {
+        let err = decommission_admin_not_initialized_error("cancel decommission");
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError);
+        assert_eq!(err.message(), Some("Failed to cancel decommission: object layer not initialized"));
+    }
+
+    #[test]
+    fn test_pool_admin_missing_credentials_error_formats_list_context() {
+        let err = pool_admin_missing_credentials_error("list pools");
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest);
+        assert_eq!(err.message(), Some("Failed to list pools: missing credentials"));
+    }
+
+    #[test]
+    fn test_pool_admin_missing_credentials_error_formats_decommission_context() {
+        let err = pool_admin_missing_credentials_error("start decommission");
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest);
+        assert_eq!(err.message(), Some("Failed to start decommission: missing credentials"));
+    }
+
+    #[test]
+    fn test_pool_admin_query_parse_error_formats_status_context() {
+        let err = pool_admin_query_parse_error("load pool status");
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
+        assert_eq!(err.message(), Some("Failed to load pool status: invalid query parameters"));
+    }
+
+    #[test]
+    fn test_pool_admin_pool_parse_error_formats_pool_context() {
+        let err = pool_admin_pool_parse_error("start decommission", "pool-x");
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
+        assert_eq!(err.message(), Some("Failed to start decommission: invalid pool `pool-x`"));
+    }
+
+    #[test]
+    fn test_pool_admin_pool_index_error_formats_range_context() {
+        let err = pool_admin_pool_index_error("start decommission", 4, 2);
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
+        assert_eq!(
+            err.message(),
+            Some("Failed to start decommission: pool index 4 is out of range for 2 pools")
+        );
+    }
+
+    #[test]
+    fn test_pool_admin_pool_not_found_error_formats_cancel_context() {
+        let err = pool_admin_pool_not_found_error("cancel decommission", "pool-x");
+
+        assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
+        assert_eq!(err.message(), Some("Failed to cancel decommission: pool `pool-x` was not found"));
+    }
+
+    #[test]
+    fn test_dedup_indices_removes_duplicates_preserving_order() {
+        assert_eq!(dedup_indices(&[0, 2, 1, 2, 3, 0]), vec![0, 2, 1, 3]);
+    }
+
+    #[test]
+    fn test_dedup_indices_handles_empty_input() {
+        let empty: Vec<usize> = Vec::new();
+        assert!(dedup_indices(&empty).is_empty());
+    }
+}

rustfs/src/admin/handlers/rebalance.rs

@@ -20,7 +20,7 @@ use crate::{
     auth::{check_key_valid, get_session_token},
     server::{ADMIN_PREFIX, RemoteAddr},
 };
-use http::{HeaderMap, StatusCode};
+use http::{HeaderMap, HeaderValue, StatusCode};
 use hyper::Method;
 use matchit::Params;
 use rustfs_ecstore::rebalance::RebalanceMeta;
@@ -79,6 +79,8 @@ pub struct RebalPoolProgress {
     pub num_versions: u64,
     #[serde(rename = "bytes")]
     pub bytes: u64,
+    #[serde(rename = "remainingBuckets")]
+    pub remaining_buckets: usize,
     #[serde(rename = "bucket")]
     pub bucket: String,
     #[serde(rename = "object")]
@@ -96,7 +98,9 @@ pub struct RebalancePoolStatus {
     #[serde(rename = "status")]
     pub status: String, // Active if rebalance is running, empty otherwise
     #[serde(rename = "used")]
-    pub used: f64, // Percentage used space
+    pub used: f64, // Fraction of used space in range 0.0..=1.0
+    #[serde(rename = "lastError")]
+    pub last_error: Option<String>, // Last rebalance error message for this pool
     #[serde(rename = "progress")]
     pub progress: Option<RebalPoolProgress>, // None when rebalance is not running
 }
@@ -110,6 +114,106 @@ pub struct RebalanceAdminStatus {
     pub stopped_at: Option<OffsetDateTime>, // Optional timestamp when rebalance was stopped
 }
 
+fn calculate_rebalance_progress(
+    now: OffsetDateTime,
+    start_time: Option<OffsetDateTime>,
+    terminal_time: Option<OffsetDateTime>,
+    bytes: u64,
+    target_bytes: f64,
+) -> Option<(u64, u64)> {
+    let start = start_time?;
+    let reference = terminal_time.unwrap_or(now);
+    let elapsed_secs = (reference - start).whole_seconds().max(0) as u64;
+
+    if terminal_time.is_some() {
+        return Some((elapsed_secs, 0));
+    }
+
+    if !target_bytes.is_finite() || bytes == 0 || target_bytes <= bytes as f64 {
+        return Some((elapsed_secs, 0));
+    }
+
+    let remaining = target_bytes - bytes as f64;
+    if remaining <= 0.0 {
+        return Some((elapsed_secs, 0));
+    }
+
+    let eta_secs_f64 = remaining * elapsed_secs as f64 / bytes as f64;
+    let eta_secs = Duration::try_from_secs_f64(eta_secs_f64).map_or(0, |duration| duration.as_secs());
+    Some((elapsed_secs, eta_secs))
+}
+
+fn build_rebalance_pool_progress(
+    now: OffsetDateTime,
+    stop_time: Option<OffsetDateTime>,
+    percent_free_goal: f64,
+    ps: &rustfs_ecstore::rebalance::RebalanceStats,
+) -> Option<RebalPoolProgress> {
+    let total_bytes_to_rebal = ps.init_capacity as f64 * percent_free_goal - ps.init_free_space as f64;
+    let terminal_time = ps.info.end_time.or(stop_time);
+    let (elapsed, eta) = calculate_rebalance_progress(now, ps.info.start_time, terminal_time, ps.bytes, total_bytes_to_rebal)?;
+
+    Some(RebalPoolProgress {
+        num_objects: ps.num_objects,
+        num_versions: ps.num_versions,
+        bytes: ps.bytes,
+        remaining_buckets: rebalance_remaining_buckets(ps.buckets.len(), ps.rebalanced_buckets.len()),
+        bucket: ps.bucket.clone(),
+        object: ps.object.clone(),
+        elapsed,
+        eta,
+    })
+}
+
+fn rebalance_used_pct(total: u64, available: u64) -> f64 {
+    if total == 0 {
+        return 0.0;
+    }
+
+    let bounded_available = available.min(total);
+    (total - bounded_available) as f64 / total as f64
+}
+
+fn rebalance_remaining_buckets(buckets: usize, rebalanced_buckets: usize) -> usize {
+    buckets.saturating_sub(rebalanced_buckets)
+}
+
+fn rebalance_pool_used(disk_stats: &[DiskStat], idx: usize) -> f64 {
+    let (total_space, available_space) = disk_stats
+        .get(idx)
+        .map(|stat| (stat.total_space, stat.available_space))
+        .unwrap_or((0, 0));
+    rebalance_used_pct(total_space, available_space)
+}
+
+fn build_rebalance_pool_statuses(
+    now: OffsetDateTime,
+    stop_time: Option<OffsetDateTime>,
+    percent_free_goal: f64,
+    pool_stats: &[rustfs_ecstore::rebalance::RebalanceStats],
+    disk_stats: &[DiskStat],
+) -> Vec<RebalancePoolStatus> {
+    pool_stats
+        .iter()
+        .enumerate()
+        .map(|(i, ps)| {
+            let mut status = RebalancePoolStatus {
+                id: i,
+                status: ps.info.status.to_string(),
+                used: rebalance_pool_used(disk_stats, i),
+                last_error: ps.info.last_error.clone(),
+                progress: None,
+            };
+
+            if ps.participating {
+                status.progress = build_rebalance_pool_progress(now, stop_time, percent_free_goal, ps);
+            }
+
+            status
+        })
+        .collect()
+}
+
 pub struct RebalanceStart {}
 
 #[async_trait::async_trait]
@@ -119,7 +223,7 @@ impl Operation for RebalanceStart {
         warn!("handle RebalanceStart");
 
         let Some(input_cred) = req.credentials else {
-            return Err(s3_error!(InvalidRequest, "get cred failed"));
+            return Err(s3_error!(InvalidRequest, "Failed to start rebalance: missing credentials"));
         };
 
         let (cred, owner) =
@@ -136,7 +240,7 @@ impl Operation for RebalanceStart {
         .await?;
 
         let Some(store) = new_object_layer_fn() else {
-            return Err(s3_error!(InternalError, "Not init"));
+            return Err(s3_error!(InternalError, "Failed to start rebalance: object layer not initialized"));
         };
 
         if store.pools.len() == 1 {
@@ -150,38 +254,44 @@ impl Operation for RebalanceStart {
             ));
         }
 
-        if store.is_rebalance_started().await {
+        if store.is_rebalance_conflicting_with_decommission().await {
             return Err(s3_error!(OperationAborted, "Rebalance already in progress"));
         }
 
         let bucket_infos = store
             .list_bucket(&BucketOptions::default())
             .await
-            .map_err(|e| s3_error!(InternalError, "Failed to list buckets: {}", e))?;
+            .map_err(|e| s3_error!(InternalError, "Failed to list buckets for rebalance: {}", e))?;
 
         let buckets: Vec<String> = bucket_infos.into_iter().map(|bucket| bucket.name).collect();
 
         let id = match store.init_rebalance_meta(buckets).await {
             Ok(id) => id,
             Err(e) => {
-                return Err(s3_error!(InternalError, "Failed to init rebalance meta: {}", e));
+                return Err(s3_error!(InternalError, "Failed to initialize rebalance metadata: {}", e));
             }
         };
 
-        store.start_rebalance().await;
+        store
+            .start_rebalance()
+            .await
+            .map_err(|e| s3_error!(InternalError, "Failed to start rebalance: {}", e))?;
 
         warn!("Rebalance started with id: {}", id);
         if let Some(notification_sys) = get_global_notification_sys() {
             warn!("RebalanceStart Loading rebalance meta start");
-            notification_sys.load_rebalance_meta(true).await;
+            if let Err(err) = notification_sys.load_rebalance_meta(true).await {
+                warn!("rebalance start propagation failed after local state update: {err}");
+            }
             warn!("RebalanceStart Loading rebalance meta done");
         }
 
         let resp = RebalanceResp { id };
-        let data = serde_json::to_string(&resp).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?;
+        let data = serde_json::to_string(&resp)
+            .map_err(|e| s3_error!(InternalError, "Failed to serialize rebalance start response: {}", e))?;
 
         let mut header = HeaderMap::new();
-        header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
+        header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
 
         Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
     }
@@ -197,7 +307,7 @@ impl Operation for RebalanceStatus {
         warn!("handle RebalanceStatus");
 
         let Some(input_cred) = req.credentials else {
-            return Err(s3_error!(InvalidRequest, "get cred failed"));
+            return Err(s3_error!(InvalidRequest, "Failed to load rebalance status: missing credentials"));
         };
 
         let (cred, owner) =
@@ -214,16 +324,26 @@ impl Operation for RebalanceStatus {
         .await?;
 
         let Some(store) = new_object_layer_fn() else {
-            return Err(s3_error!(InternalError, "Not init"));
+            return Err(s3_error!(InternalError, "Failed to load rebalance status: object layer not initialized"));
         };
 
+        if store.pools.is_empty() {
+            return Err(s3_error!(InternalError, "Failed to load rebalance status: no storage pools available"));
+        }
+
+        let first_pool = store
+            .pools
+            .first()
+            .cloned()
+            .ok_or_else(|| s3_error!(InternalError, "Failed to load rebalance status: no storage pools available"))?;
+
         let mut meta = RebalanceMeta::new();
-        if let Err(err) = meta.load(store.pools[0].clone()).await {
+        if let Err(err) = meta.load(first_pool).await {
             if err == StorageError::ConfigNotFound {
                 return Err(s3_error!(NoSuchResource, "Pool rebalance is not started"));
             }
 
-            return Err(s3_error!(InternalError, "Failed to load rebalance meta: {}", err));
+            return Err(s3_error!(InternalError, "Failed to load rebalance metadata from pool 0: {}", err));
         }
 
         // Compute disk usage percentage
@@ -238,68 +358,18 @@ impl Operation for RebalanceStatus {
             disk_stats[disk.pool_index as usize].total_space += disk.total_space;
         }
 
-        let mut stop_time = meta.stopped_at;
-        let mut admin_status = RebalanceAdminStatus {
+        let stop_time = meta.stopped_at;
+        let now = OffsetDateTime::now_utc();
+        let admin_status = RebalanceAdminStatus {
             id: meta.id.clone(),
             stopped_at: meta.stopped_at,
-            pools: vec![RebalancePoolStatus::default(); meta.pool_stats.len()],
+            pools: build_rebalance_pool_statuses(now, stop_time, meta.percent_free_goal, &meta.pool_stats, &disk_stats),
         };
 
-        for (i, ps) in meta.pool_stats.iter().enumerate() {
-            admin_status.pools[i] = RebalancePoolStatus {
-                id: i,
-                status: ps.info.status.to_string(),
-                used: (disk_stats[i].total_space - disk_stats[i].available_space) as f64 / disk_stats[i].total_space as f64,
-                progress: None,
-            };
-
-            if !ps.participating {
-                continue;
-            }
-
-            // Calculate total bytes to be rebalanced
-            let total_bytes_to_rebal = ps.init_capacity as f64 * meta.percent_free_goal - ps.init_free_space as f64;
-
-            let mut elapsed = if let Some(start_time) = ps.info.start_time {
-                let now = OffsetDateTime::now_utc();
-                now - start_time
-            } else {
-                return Err(s3_error!(InternalError, "Start time is not available"));
-            };
-
-            let mut eta = if ps.bytes > 0 {
-                Duration::from_secs_f64(total_bytes_to_rebal * elapsed.as_seconds_f64() / ps.bytes as f64)
-            } else {
-                Duration::ZERO
-            };
-
-            if ps.info.end_time.is_some() {
-                stop_time = ps.info.end_time;
-            }
-
-            if let Some(stopped_at) = stop_time {
-                if let Some(start_time) = ps.info.start_time {
-                    elapsed = stopped_at - start_time;
-                }
-
-                eta = Duration::ZERO;
-            }
-
-            admin_status.pools[i].progress = Some(RebalPoolProgress {
-                num_objects: ps.num_objects,
-                num_versions: ps.num_versions,
-                bytes: ps.bytes,
-                bucket: ps.bucket.clone(),
-                object: ps.object.clone(),
-                elapsed: elapsed.whole_seconds() as u64,
-                eta: eta.as_secs(),
-            });
-        }
-
-        let data =
-            serde_json::to_string(&admin_status).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?;
+        let data = serde_json::to_string(&admin_status)
+            .map_err(|e| s3_error!(InternalError, "Failed to serialize rebalance status response: {}", e))?;
         let mut header = HeaderMap::new();
-        header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
+        header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
 
         Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
     }
@@ -315,7 +385,7 @@ impl Operation for RebalanceStop {
         warn!("handle RebalanceStop");
 
         let Some(input_cred) = req.credentials else {
-            return Err(s3_error!(InvalidRequest, "get cred failed"));
+            return Err(s3_error!(InvalidRequest, "Failed to stop rebalance: missing credentials"));
         };
 
         let (cred, owner) =
@@ -332,28 +402,42 @@ impl Operation for RebalanceStop {
         .await?;
 
         let Some(store) = new_object_layer_fn() else {
-            return Err(s3_error!(InternalError, "Not init"));
+            return Err(s3_error!(InternalError, "Failed to stop rebalance: object layer not initialized"));
         };
 
-        if let Some(notification_sys) = get_global_notification_sys() {
-            notification_sys.stop_rebalance().await;
+        if !store.is_rebalance_conflicting_with_decommission().await {
+            return Err(s3_error!(NoSuchResource, "Pool rebalance is not started"));
         }
 
-        store
-            .save_rebalance_stats(0, RebalSaveOpt::StoppedAt)
-            .await
-            .map_err(|e| s3_error!(InternalError, "Failed to stop rebalance: {}", e))?;
+        if let Some(notification_sys) = get_global_notification_sys() {
+            notification_sys
+                .stop_rebalance()
+                .await
+                .map_err(|e| s3_error!(InternalError, "Failed to stop rebalance via notification system: {}", e))?;
+        } else {
+            store
+                .stop_rebalance()
+                .await
+                .map_err(|e| s3_error!(InternalError, "Failed to stop rebalance: {}", e))?;
+
+            store
+                .save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt)
+                .await
+                .map_err(|e| s3_error!(InternalError, "Failed to persist rebalance stop metadata: {}", e))?;
+        }
 
         warn!("handle RebalanceStop save_rebalance_stats done ");
         if let Some(notification_sys) = get_global_notification_sys() {
             warn!("handle RebalanceStop notification_sys load_rebalance_meta");
-            notification_sys.load_rebalance_meta(false).await;
+            if let Err(err) = notification_sys.load_rebalance_meta(false).await {
+                warn!("rebalance stop propagation failed after local state update: {err}");
+            }
             warn!("handle RebalanceStop notification_sys load_rebalance_meta done");
         }
 
         let mut header = HeaderMap::new();
-        header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
-        header.insert(CONTENT_LENGTH, "0".parse().unwrap());
+        header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
+        header.insert(CONTENT_LENGTH, HeaderValue::from_static("0"));
         Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header))
     }
 }
@@ -389,3 +473,372 @@ mod offsetdatetime_rfc3339 {
         }
     }
 }
+
+#[cfg(test)]
+mod rebalance_handler_tests {
+    use super::build_rebalance_pool_progress;
+    use super::calculate_rebalance_progress;
+    use super::{
+        RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, build_rebalance_pool_statuses, rebalance_pool_used,
+        rebalance_remaining_buckets, rebalance_used_pct,
+    };
+    use rustfs_ecstore::rebalance::{DiskStat, RebalStatus, RebalanceInfo, RebalanceStats};
+    use time::OffsetDateTime;
+
+    #[test]
+    fn test_calculate_rebalance_progress_running() {
+        let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
+        let now = OffsetDateTime::from_unix_timestamp(1_050).unwrap();
+
+        let (elapsed, eta) = calculate_rebalance_progress(now, Some(start), None, 100, 200.0).unwrap();
+
+        assert_eq!(elapsed, 50);
+        assert_eq!(eta, 50);
+    }
+
+    #[test]
+    fn test_calculate_rebalance_progress_stopped_by_end_time() {
+        let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
+        let terminal = OffsetDateTime::from_unix_timestamp(1_120).unwrap();
+
+        let (elapsed, eta) = calculate_rebalance_progress(
+            OffsetDateTime::from_unix_timestamp(1_200).unwrap(),
+            Some(start),
+            Some(terminal),
+            100,
+            200.0,
+        )
+        .unwrap();
+
+        assert_eq!(elapsed, 120);
+        assert_eq!(eta, 0);
+    }
+
+    #[test]
+    fn test_calculate_rebalance_progress_invalid_target_is_zero_eta() {
+        let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
+        let now = OffsetDateTime::from_unix_timestamp(1_010).unwrap();
+
+        let (elapsed, eta) = calculate_rebalance_progress(now, Some(start), None, 100, f64::NAN).unwrap();
+
+        assert_eq!(elapsed, 10);
+        assert_eq!(eta, 0);
+    }
+
+    #[test]
+    fn test_calculate_rebalance_progress_negative_target_is_zero_eta() {
+        let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
+        let now = OffsetDateTime::from_unix_timestamp(1_010).unwrap();
+
+        let (elapsed, eta) = calculate_rebalance_progress(now, Some(start), None, 100, -10.0).unwrap();
+
+        assert_eq!(elapsed, 10);
+        assert_eq!(eta, 0);
+    }
+
+    #[test]
+    fn test_calculate_rebalance_progress_overflow_eta_is_zero() {
+        let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
+        let now = OffsetDateTime::from_unix_timestamp(1_010).unwrap();
+
+        let (elapsed, eta) = calculate_rebalance_progress(now, Some(start), None, 1, f64::MAX).unwrap();
+
+        assert_eq!(elapsed, 10);
+        assert_eq!(eta, 0);
+    }
+
+    #[test]
+    fn test_calculate_rebalance_progress_no_start_time() {
+        assert!(
+            calculate_rebalance_progress(OffsetDateTime::from_unix_timestamp(1_000).unwrap(), None, None, 1, 100.0).is_none()
+        );
+    }
+
+    #[test]
+    fn test_build_rebalance_pool_progress_returns_none_without_start_time() {
+        let ps = RebalanceStats {
+            participating: true,
+            info: RebalanceInfo {
+                status: RebalStatus::Started,
+                start_time: None,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        let progress = build_rebalance_pool_progress(OffsetDateTime::from_unix_timestamp(1_000).unwrap(), None, 0.3, &ps);
+        assert!(progress.is_none());
+    }
+
+    #[test]
+    fn test_build_rebalance_pool_progress_maps_fields_and_eta() {
+        let ps = RebalanceStats {
+            init_capacity: 1_000,
+            init_free_space: 200,
+            buckets: vec!["bucket-a".to_string(), "bucket-b".to_string(), "bucket-c".to_string()],
+            rebalanced_buckets: vec!["bucket-a".to_string()],
+            bucket: "bucket-b".to_string(),
+            object: "obj-1".to_string(),
+            num_objects: 3,
+            num_versions: 5,
+            bytes: 100,
+            participating: true,
+            info: RebalanceInfo {
+                status: RebalStatus::Started,
+                start_time: Some(OffsetDateTime::from_unix_timestamp(1_000).unwrap()),
+                ..Default::default()
+            },
+        };
+
+        let progress = build_rebalance_pool_progress(OffsetDateTime::from_unix_timestamp(1_050).unwrap(), None, 0.3, &ps)
+            .expect("progress should be generated");
+        assert_eq!(progress.num_objects, 3);
+        assert_eq!(progress.num_versions, 5);
+        assert_eq!(progress.bytes, 100);
+        assert_eq!(progress.remaining_buckets, 2);
+        assert_eq!(progress.bucket, "bucket-b");
+        assert_eq!(progress.object, "obj-1");
+        assert_eq!(progress.elapsed, 50);
+        assert_eq!(progress.eta, 0);
+    }
+
+    #[test]
+    fn test_build_rebalance_pool_progress_stopped_uses_stop_time() {
+        let ps = RebalanceStats {
+            init_capacity: 1_000,
+            init_free_space: 200,
+            info: RebalanceInfo {
+                status: RebalStatus::Started,
+                start_time: Some(OffsetDateTime::from_unix_timestamp(1_000).unwrap()),
+                ..Default::default()
+            },
+            participating: true,
+            ..Default::default()
+        };
+
+        let stop_time = OffsetDateTime::from_unix_timestamp(1_200).unwrap();
+        let progress = build_rebalance_pool_progress(stop_time, Some(stop_time), 0.3, &ps).expect("progress should be generated");
+
+        assert_eq!(progress.elapsed, 200);
+        assert_eq!(progress.eta, 0);
+    }
+
+    #[test]
+    fn test_build_rebalance_pool_progress_prefers_info_end_time_over_stop_time() {
+        let ps = RebalanceStats {
+            init_capacity: 1_000,
+            init_free_space: 200,
+            info: RebalanceInfo {
+                status: RebalStatus::Started,
+                start_time: Some(OffsetDateTime::from_unix_timestamp(1_000).unwrap()),
+                end_time: Some(OffsetDateTime::from_unix_timestamp(1_180).unwrap()),
+                ..Default::default()
+            },
+            participating: true,
+            ..Default::default()
+        };
+
+        let progress = build_rebalance_pool_progress(
+            OffsetDateTime::from_unix_timestamp(1_300).unwrap(),
+            Some(OffsetDateTime::from_unix_timestamp(1_250).unwrap()),
+            0.3,
+            &ps,
+        )
+        .expect("progress should be generated");
+
+        assert_eq!(progress.elapsed, 180);
+        assert_eq!(progress.eta, 0);
+    }
+
+    #[test]
+    fn test_rebalance_used_pct_normal_and_zero_total() {
+        assert_eq!(rebalance_used_pct(1_000, 650), 0.35);
+        assert_eq!(rebalance_used_pct(0, 0), 0.0);
+    }
+
+    #[test]
+    fn test_rebalance_used_pct_clamps_available_over_total() {
+        assert_eq!(rebalance_used_pct(1_000, 1_500), 0.0);
+    }
+
+    #[test]
+    fn test_rebalance_remaining_buckets_is_saturating_sub() {
+        assert_eq!(rebalance_remaining_buckets(10, 7), 3);
+        assert_eq!(rebalance_remaining_buckets(3, 10), 0);
+    }
+
+    #[test]
+    fn test_rebalance_pool_used_defaults_to_zero_when_disk_stat_missing() {
+        let disk_stats: Vec<DiskStat> = vec![];
+        assert_eq!(rebalance_pool_used(&disk_stats, 0), 0.0);
+    }
+
+    #[test]
+    fn test_build_rebalance_pool_statuses_tracks_progress_for_participants() {
+        let pool_stats = vec![
+            RebalanceStats {
+                participating: true,
+                init_capacity: 1_000,
+                init_free_space: 200,
+                num_objects: 2,
+                num_versions: 2,
+                bytes: 100,
+                buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
+                rebalanced_buckets: vec!["bucket-a".to_string()],
+                bucket: "bucket-b".to_string(),
+                object: "obj-2".to_string(),
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    start_time: Some(OffsetDateTime::from_unix_timestamp(1_000).unwrap()),
+                    ..Default::default()
+                },
+            },
+            RebalanceStats {
+                participating: false,
+                info: RebalanceInfo {
+                    status: RebalStatus::Completed,
+                    start_time: Some(OffsetDateTime::from_unix_timestamp(1_000).unwrap()),
+                    ..Default::default()
+                },
+                ..Default::default()
+            },
+        ];
+
+        let disk_stats = vec![
+            DiskStat {
+                total_space: 1_000,
+                available_space: 500,
+            },
+            DiskStat {
+                total_space: 0,
+                available_space: 0,
+            },
+        ];
+
+        let statuses = build_rebalance_pool_statuses(
+            OffsetDateTime::from_unix_timestamp(1_050).unwrap(),
+            None,
+            0.3,
+            &pool_stats,
+            &disk_stats,
+        );
+
+        assert_eq!(statuses.len(), 2);
+
+        let active = &statuses[0];
+        assert_eq!(active.id, 0);
+        assert_eq!(active.status, "Started");
+        assert_eq!(active.used, 0.5);
+        assert_eq!(active.progress.as_ref().unwrap().bucket, "bucket-b");
+        assert_eq!(active.progress.as_ref().unwrap().object, "obj-2");
+        assert_eq!(active.progress.as_ref().unwrap().remaining_buckets, 1);
+
+        let inactive = &statuses[1];
+        assert_eq!(inactive.id, 1);
+        assert_eq!(inactive.status, "Completed");
+        assert_eq!(inactive.used, 0.0);
+        assert!(inactive.progress.is_none());
+    }
+
+    #[test]
+    fn test_build_rebalance_pool_statuses_uses_zero_used_for_missing_disk_stats() {
+        let pool_stats = vec![
+            RebalanceStats {
+                participating: false,
+                info: RebalanceInfo {
+                    status: RebalStatus::Completed,
+                    ..Default::default()
+                },
+                ..Default::default()
+            },
+            RebalanceStats {
+                participating: true,
+                init_capacity: 2_000,
+                init_free_space: 400,
+                num_objects: 1,
+                num_versions: 1,
+                bytes: 10,
+                buckets: vec!["bucket-a".to_string()],
+                rebalanced_buckets: vec![],
+                bucket: "bucket-a".to_string(),
+                object: "obj".to_string(),
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    start_time: Some(OffsetDateTime::from_unix_timestamp(2_000).unwrap()),
+                    ..Default::default()
+                },
+            },
+        ];
+
+        let statuses =
+            build_rebalance_pool_statuses(OffsetDateTime::from_unix_timestamp(2_010).unwrap(), None, 0.3, &pool_stats, &[]);
+
+        assert_eq!(statuses[1].used, 0.0);
+        assert!(statuses[1].progress.is_some());
+    }
+
+    #[test]
+    fn test_build_rebalance_pool_statuses_empty_inputs() {
+        let statuses = build_rebalance_pool_statuses(
+            OffsetDateTime::from_unix_timestamp(2_000).unwrap(),
+            None,
+            0.3,
+            &[],
+            &[DiskStat {
+                total_space: 1_000,
+                available_space: 500,
+            }],
+        );
+
+        assert!(statuses.is_empty());
+    }
+
+    #[test]
+    fn test_rebalance_status_serializes_new_fields() {
+        let status = RebalanceAdminStatus {
+            id: "id-1".to_string(),
+            stopped_at: None,
+            pools: vec![RebalancePoolStatus {
+                id: 0,
+                status: "Started".to_string(),
+                used: 0.5,
+                last_error: Some("temporary error".to_string()),
+                progress: Some(RebalPoolProgress {
+                    num_objects: 3,
+                    num_versions: 5,
+                    bytes: 1024,
+                    remaining_buckets: 2,
+                    bucket: "bucket-a".to_string(),
+                    object: "obj".to_string(),
+                    elapsed: 10,
+                    eta: 20,
+                }),
+            }],
+        };
+
+        let json = serde_json::to_string(&status).unwrap();
+        assert!(json.contains("\"remainingBuckets\""));
+        assert!(json.contains("\"lastError\""));
+        assert!(json.contains("\"stoppedAt\":null"));
+    }
+
+    #[test]
+    fn test_rebalance_status_serializes_stopped_at_when_present() {
+        let stopped = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
+        let status = RebalanceAdminStatus {
+            id: "id-2".to_string(),
+            stopped_at: Some(stopped),
+            pools: vec![RebalancePoolStatus {
+                id: 0,
+                status: "Stopped".to_string(),
+                used: 0.3,
+                last_error: None,
+                progress: None,
+            }],
+        };
+
+        let json = serde_json::to_string(&status).unwrap();
+        assert!(json.contains("\"stoppedAt\""));
+        assert!(json.contains("1970-01-01T00:16:40Z"));
+    }
+}

rustfs/src/error.rs

@@ -217,6 +217,9 @@ impl From<StorageError> for ApiError {
             StorageError::BucketExists(_) => S3ErrorCode::BucketAlreadyOwnedByYou,
             StorageError::StorageFull => S3ErrorCode::ServiceUnavailable,
             StorageError::SlowDown => S3ErrorCode::SlowDown,
+            StorageError::DecommissionNotStarted => S3ErrorCode::InvalidRequest,
+            StorageError::DecommissionAlreadyRunning => S3ErrorCode::InvalidRequest,
+            StorageError::RebalanceAlreadyRunning => S3ErrorCode::InvalidRequest,
             StorageError::PrefixAccessDenied(_, _) => S3ErrorCode::AccessDenied,
             StorageError::InvalidUploadIDKeyCombination(_, _) => S3ErrorCode::InvalidArgument,
             StorageError::MalformedUploadID(_) => S3ErrorCode::InvalidArgument,
@@ -410,6 +413,9 @@ mod tests {
             (StorageError::BucketExists("test".into()), S3ErrorCode::BucketAlreadyOwnedByYou),
             (StorageError::StorageFull, S3ErrorCode::ServiceUnavailable),
             (StorageError::SlowDown, S3ErrorCode::SlowDown),
+            (StorageError::DecommissionNotStarted, S3ErrorCode::InvalidRequest),
+            (StorageError::DecommissionAlreadyRunning, S3ErrorCode::InvalidRequest),
+            (StorageError::RebalanceAlreadyRunning, S3ErrorCode::InvalidRequest),
             (StorageError::PrefixAccessDenied("test".into(), "test".into()), S3ErrorCode::AccessDenied),
             (StorageError::ObjectNotFound("test".into(), "test".into()), S3ErrorCode::NoSuchKey),
             (StorageError::ConfigNotFound, S3ErrorCode::NoSuchKey),

rustfs/src/storage/rpc/node_service.rs

@@ -52,6 +52,10 @@ use tracing::{debug, error, info, warn};
 
 type ResponseStream<T> = Pin<Box<dyn Stream<Item = Result<T, Status>> + Send>>;
 
+fn background_rebalance_start_error_message(result: rustfs_ecstore::error::Result<()>) -> Option<String> {
+    result.err().map(|err| format!("start_rebalance failed: {err}"))
+}
+
 #[path = "bucket.rs"]
 mod bucket;
 #[path = "disk.rs"]
@@ -850,7 +854,9 @@ impl Node for NodeService {
             warn!("start rebalance");
             let store = store.clone();
             spawn(async move {
-                store.start_rebalance().await;
+                if let Some(message) = background_rebalance_start_error_message(store.start_rebalance().await) {
+                    error!("{message}");
+                }
             });
         }
 
@@ -1992,6 +1998,20 @@ mod tests {
         assert!(load_response.error_info.unwrap().contains("errServerNotInitialized"));
     }
 
+    #[test]
+    fn test_background_rebalance_start_error_message_ignores_success() {
+        assert!(background_rebalance_start_error_message(Ok(())).is_none());
+    }
+
+    #[test]
+    fn test_background_rebalance_start_error_message_formats_error() {
+        let message = background_rebalance_start_error_message(Err(rustfs_ecstore::error::Error::other("boom")))
+            .expect("background rebalance start failure should be formatted");
+
+        assert!(message.contains("start_rebalance failed"));
+        assert!(message.contains("boom"));
+    }
+
     #[tokio::test]
     async fn test_load_bucket_metadata_empty_bucket() {
         let service = create_test_node_service();