From 950cce6f579542c0b3727d9d832f2051872d310d Mon Sep 17 00:00:00 2001 From: overtrue Date: Sun, 6 Sep 2026 15:21:02 +0800 Subject: [PATCH] fix(admin): preserve decommission readiness error --- crates/ecstore/src/api/mod.rs | 2 +- crates/ecstore/src/core/pools.rs | 2 +- rustfs/src/admin/handlers/pools.rs | 53 +++++++++++++++++++++++++++--- rustfs/src/storage/storage_api.rs | 2 +- rustfs/src/storage_api.rs | 2 ++ 5 files changed, 54 insertions(+), 7 deletions(-) diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index e0cc06d26..db6ad5b48 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -293,7 +293,7 @@ pub mod cache { pub mod capacity { pub use crate::core::pools::{ DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free, - path2_bucket_object, path2_bucket_object_with_base_path, + is_pool_activation_fleet_proof_error, path2_bucket_object, path2_bucket_object_with_base_path, }; pub use crate::store::utils::is_reserved_or_invalid_bucket; } diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index c4235bd40..c9733e787 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -3385,7 +3385,7 @@ pub(crate) async fn acquire_pool_activation_fleet_proof( .ok_or_else(|| Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED)) } -pub(crate) fn is_pool_activation_fleet_proof_error(err: &Error) -> bool { +pub fn is_pool_activation_fleet_proof_error(err: &Error) -> bool { // Save-stage helpers add context by formatting the original error, so the // marker may be nested in the display string. Restrict matching to the // `Error::other` I/O shape used by this activation path. diff --git a/rustfs/src/admin/handlers/pools.rs b/rustfs/src/admin/handlers/pools.rs index c506f2e3d..22ec2a20d 100644 --- a/rustfs/src/admin/handlers/pools.rs +++ b/rustfs/src/admin/handlers/pools.rs @@ -63,6 +63,7 @@ const EVENT_ADMIN_REQUEST_STATE: &str = "admin_request_state"; const EVENT_ADMIN_REQUEST_REJECTED: &str = "admin_request_rejected"; const EVENT_ADMIN_REQUEST_FAILED: &str = "admin_request_failed"; const EVENT_ADMIN_RESPONSE_EMITTED: &str = "admin_response_emitted"; +const POOL_ACTIVATION_FLEET_PROOF_REQUIRED: &str = "pool activation requires a live fleet capability proof"; fn admin_request_id(headers: &HeaderMap) -> Option<&str> { headers @@ -321,6 +322,17 @@ fn contextualize_admin_pool_api_error( } } +fn decommission_start_api_error(err: crate::storage_api::error::StorageError) -> ApiError { + if crate::storage_api::capacity::is_pool_activation_fleet_proof_error(&err) { + return ApiError { + code: S3ErrorCode::InternalError, + message: POOL_ACTIVATION_FLEET_PROOF_REQUIRED.to_string(), + source: Some(Box::new(err)), + }; + } + ApiError::from(err) +} + fn decommission_admin_not_initialized_error_with_audit(operation: &str, audit: PoolAuditContext<'_>) -> S3Error { error!( event = EVENT_ADMIN_REQUEST_FAILED, @@ -790,7 +802,24 @@ impl Operation for StartDecommission { store .decommission(ctx.clone(), pools_indices.clone()) .await - .map_err(ApiError::from) + .map_err(|err| { + error!( + event = EVENT_ADMIN_REQUEST_FAILED, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_POOL_ADMIN, + operation = "start_decommission", + action = "start_decommission", + result = "failed", + reason = "storage_decommission_failed", + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + pool_indices = ?pools_indices, + error = %err, + "admin request failed" + ); + decommission_start_api_error(err) + }) .map_err(|err| contextualize_admin_pool_api_error(err, "start decommission", &pool_context))?; } } @@ -1018,9 +1047,10 @@ impl Operation for ClearDecommission { #[cfg(test)] mod pools_handler_tests { use super::{ - AdminPoolStatus, Body, CancelDecommission, ClearDecommission, HeaderMap, ListPools, Method, Operation, Params, - PoolAuditContext, S3ErrorCode, S3Request, StartDecommission, StatusDecommission, StatusPool, Uri, - contextualize_admin_pool_api_error, decommission_admin_not_initialized_error_with_audit, decommission_peer_target, + AdminPoolStatus, Body, CancelDecommission, ClearDecommission, HeaderMap, ListPools, Method, Operation, + POOL_ACTIVATION_FLEET_PROOF_REQUIRED, Params, PoolAuditContext, S3ErrorCode, S3Request, StartDecommission, + StatusDecommission, StatusPool, Uri, contextualize_admin_pool_api_error, + decommission_admin_not_initialized_error_with_audit, decommission_peer_target, decommission_start_api_error, has_duplicate_indices, parse_mutation_pool_query, parse_pool_idx_by_id, parse_status_pool_query, pool_admin_missing_credentials_error, pool_admin_missing_credentials_error_with_request, pool_admin_pool_index_error_with_audit, pool_admin_pool_not_found_error_with_audit, @@ -1209,6 +1239,21 @@ mod pools_handler_tests { ); } + #[test] + fn test_decommission_start_api_error_preserves_fleet_proof_retry_marker() { + let err = crate::storage_api::error::StorageError::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED); + + let err = decommission_start_api_error(err); + + assert_eq!(err.code, s3s::S3ErrorCode::InternalError); + assert_eq!(err.message, POOL_ACTIVATION_FLEET_PROOF_REQUIRED); + assert!(err.source.is_some()); + + let unrelated = decommission_start_api_error(crate::storage_api::error::StorageError::other("disk read failed")); + assert_eq!(unrelated.code, s3s::S3ErrorCode::InternalError); + assert_eq!(unrelated.message, "We encountered an internal error, please try again."); + } + #[test] fn test_contextualize_admin_pool_api_error_preserves_source() { let err = contextualize_admin_pool_api_error( diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index e35cf812d..33122be9d 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -417,7 +417,7 @@ pub(crate) mod ecstore_bucket { pub(crate) mod ecstore_capacity { pub(crate) use rustfs_ecstore::api::capacity::{ DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free, - is_reserved_or_invalid_bucket, + is_pool_activation_fleet_proof_error, is_reserved_or_invalid_bucket, }; } diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index 9abdbd5fd..e74cb689a 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -18,6 +18,8 @@ use rustfs_storage_api as storage_contracts; pub(crate) mod capacity { + pub(crate) use crate::storage::storage_api::ecstore_capacity::is_pool_activation_fleet_proof_error; + pub(crate) mod service { pub(crate) use crate::storage::storage_api::{all_local_disk, disk_drive_path, disk_endpoint}; }