mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 11:56:38 +00:00
fix(ecstore): classify decommission stage failures by type, not by message (#6269)
T2 of backlog#1827. `data_movement_stage_error` flattened every stage failure into `Error::other(format!(...))`, discarding the typed error. The cost was visible in tree: `is_decommission_target_capacity_error` had to match rendered text —
let message = err.to_string();
message.contains(&disk_full) || message.contains(&storage_full)
— to notice that the destination pool had filled up, and `is_decommission_copy_cleanup_safe_error` could not see a not-found that surfaced from inside a stage at all.
The wrapper now carries what it wrapped. `DataMovementStageError` renders the same string and returns the original through `source()`; `Error::other` boxes it through `std::io::Error`, so `data_movement_stage_source` recovers it by downcast. Both classifiers unwrap before matching, keeping their substring paths for errors that arrive through some other wrapper.
The rendered message is unchanged, which a test now pins against the exact string the old `format!` produced rather than against a `contains`. Three more cover the round trip for `DiskFull`, `StorageFull`, `FileNotFound` and `SlowDown`, that unrelated errors are not mistaken for stage wrappers, and — the case the issue names — that a not-found surfacing from inside a stage is judged cleanup-safe by the decommission loop exactly as a direct one is.
Refs backlog#1827
This commit is contained in:
@@ -1041,7 +1041,13 @@ fn should_count_decommission_version_complete(ignore: bool, cleanup_ignored: boo
|
||||
fn is_decommission_copy_cleanup_safe_error(err: &Error) -> bool {
|
||||
// DataMovementOverwriteErr only means source and destination pool resolved to
|
||||
// the same pool. Without a target equivalence check it is not cleanup-safe.
|
||||
is_err_object_not_found(err) || is_err_version_not_found(err)
|
||||
if is_err_object_not_found(err) || is_err_version_not_found(err) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// A not-found surfacing from inside a data-movement stage is the same
|
||||
// condition once the wrapper is unwrapped (backlog#1827 T2).
|
||||
crate::data_movement::data_movement_stage_source(err).is_some_and(is_decommission_copy_cleanup_safe_error)
|
||||
}
|
||||
|
||||
fn is_decommission_target_capacity_error(err: &Error) -> bool {
|
||||
@@ -1049,6 +1055,13 @@ fn is_decommission_target_capacity_error(err: &Error) -> bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
// A stage failure keeps the error it wrapped, so classify by type rather
|
||||
// than by the rendered message (backlog#1827 T2). The substring fallback
|
||||
// stays for errors that reached here through some other wrapper.
|
||||
if let Some(source) = crate::data_movement::data_movement_stage_source(err) {
|
||||
return is_decommission_target_capacity_error(source);
|
||||
}
|
||||
|
||||
let message = err.to_string();
|
||||
let disk_full = Error::DiskFull.to_string();
|
||||
let storage_full = Error::StorageFull.to_string();
|
||||
@@ -4427,6 +4440,36 @@ mod tests {
|
||||
assert!(is_decommission_target_capacity_error(&Error::StorageFull));
|
||||
}
|
||||
|
||||
/// The decommission loop classifies errors that came back through a
|
||||
/// data-movement stage wrapper. Before backlog#1827 T2 the wrapper flattened
|
||||
/// everything into `Error::other(String)`, so these two classifiers had to
|
||||
/// match on rendered text; now the wrapped error is recoverable by type.
|
||||
#[test]
|
||||
fn decommission_classifiers_see_through_a_stage_wrapper() {
|
||||
let wrap = |inner: Error| {
|
||||
crate::data_movement::data_movement_stage_error_for_test(
|
||||
"decommission_object",
|
||||
"put_object",
|
||||
"bucket-a",
|
||||
"object-a",
|
||||
inner,
|
||||
)
|
||||
};
|
||||
|
||||
// Capacity: the target pool filling up must still stop the loop.
|
||||
assert!(is_decommission_target_capacity_error(&wrap(Error::DiskFull)));
|
||||
assert!(is_decommission_target_capacity_error(&wrap(Error::StorageFull)));
|
||||
assert!(!is_decommission_target_capacity_error(&wrap(Error::SlowDown)));
|
||||
|
||||
// Cleanup safety: a not-found surfacing from inside a stage is the same
|
||||
// condition as one surfacing directly, so the source entry stays
|
||||
// eligible for cleanup.
|
||||
let not_found = Error::ObjectNotFound("bucket-a".to_string(), "object-a".to_string());
|
||||
assert!(is_decommission_copy_cleanup_safe_error(¬_found));
|
||||
assert!(is_decommission_copy_cleanup_safe_error(&wrap(not_found)));
|
||||
assert!(!is_decommission_copy_cleanup_safe_error(&wrap(Error::SlowDown)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
|
||||
let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
|
||||
|
||||
Reference in New Issue
Block a user