gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-19 11:06:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(rebalance): defer changed source cleanup (#5829)

Browse Source
This commit is contained in:
cxymds
2026-08-08 10:45:34 +08:00
committed by GitHub
parent cb93ac5df1
commit f5463f4aa8
8 changed files with 347 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/services/rebalance/entry.rs
+78 -40
View File
@@ -18,8 +18,8 @@ use super::meta::{
};
use super::migration::migrate_entry_version;
use super::worker::{
RebalanceEntryTask, load_rebalance_bucket_configs, rebalance_max_attempts, resolve_rebalance_bucket_error,
resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
RebalanceEntryCleanupResult, RebalanceEntryTask, load_rebalance_bucket_configs, rebalance_max_attempts,
resolve_rebalance_bucket_error, resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
resolve_rebalance_migrate_result_error, resolve_rebalance_stats_update_result, resolve_rebalance_worker_result,
run_rebalance_listing_with_retry, should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete,
should_defer_rebalance_entry_failure, should_skip_rebalance_delete_marker, wait_rebalance_entry_tasks,
@@ -50,11 +50,14 @@ impl ECStore {
bucket: &str,
object: &str,
stats_updates: &[&FileInfo],
cleanup: impl std::future::Future<Output = Result<ObjectInfo>>,
) -> Result<Option<String>> {
cleanup: impl std::future::Future<Output = std::result::Result<ObjectInfo, data_movement::SourceCleanupError>>,
) -> Result<RebalanceEntryCleanupResult> {
// Persisted stats can complete a pool on restart, so source cleanup must resolve first.
let cleanup_warning = resolve_rebalance_entry_cleanup_delete_result(cleanup.await, bucket, object)?;
if let Some(message) = cleanup_warning.as_ref()
let cleanup_result = resolve_rebalance_entry_cleanup_delete_result(cleanup.await, bucket, object);
let RebalanceEntryCleanupResult::Completed { warning } = cleanup_result else {
return Ok(cleanup_result);
};
if let Some(message) = warning.as_ref()
&& let Err(err) = self
.record_rebalance_cleanup_warning(pool_index, bucket, object, message.clone())
.await
@@ -80,7 +83,7 @@ impl ECStore {
object,
)?;
Ok(cleanup_warning)
Ok(RebalanceEntryCleanupResult::Completed { warning })
}
#[allow(unused_assignments)]
@@ -255,7 +258,7 @@ impl ECStore {
);
if should_defer_rebalance_entry_failure(&err) {
let deferred_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} {err}");
warn!(
debug!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
@@ -300,7 +303,7 @@ impl ECStore {
}
if should_cleanup_rebalance_source_entry(rebalanced, fivs.versions.len(), expired) {
let cleanup_warning = self
let cleanup_result = self
.finish_rebalance_entry_after_cleanup(
pool_index,
bucket.as_str(),
@@ -316,30 +319,47 @@ impl ECStore {
),
)
.await?;
if let Some(message) = cleanup_warning {
warn!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
object = %entry.name,
stage = "cleanup_source",
cleanup_status = "failed_ignored",
error = %message,
"Ignored rebalance source cleanup failure"
);
} else {
debug!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
object = %entry.name,
state = "source_deleted",
"Deleted rebalance source entry"
);
match cleanup_result {
RebalanceEntryCleanupResult::Deferred { last_error } => {
debug!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
object = %entry.name,
state = "deferred",
error = %last_error,
"Deferred rebalance entry after source cleanup conflict"
);
return Ok(RebalanceEntryOutcome::Deferred { last_error });
}
RebalanceEntryCleanupResult::Completed { warning: Some(message) } => {
warn!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
object = %entry.name,
stage = "cleanup_source",
cleanup_status = "failed_ignored",
error = %message,
"Ignored rebalance source cleanup failure"
);
}
RebalanceEntryCleanupResult::Completed { warning: None } => {
debug!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
object = %entry.name,
state = "source_deleted",
"Deleted rebalance source entry"
);
}
}
} else if rebalanced != fivs.versions.len() || expired > 0 {
warn!(
@@ -655,13 +675,12 @@ mod tests {
);
release_cleanup.send(()).expect("cleanup waiter should remain alive");
assert!(
assert_eq!(
finish
.await
.expect("finish task should not panic")
.expect("finish should succeed")
.is_none(),
"successful cleanup should not produce a warning"
.expect("finish should succeed"),
RebalanceEntryCleanupResult::Completed { warning: None }
);
assert!(
store
@@ -680,14 +699,33 @@ mod tests {
let mut meta = store.rebalance_meta.write().await;
meta.as_mut().expect("rebalance metadata should exist").pool_stats[0].bytes = 0;
}
let warning = store
.finish_rebalance_entry_after_cleanup(0, "bucket", "object.bin", &[&warning_version], async { Err(Error::SlowDown) })
let warning_result = store
.finish_rebalance_entry_after_cleanup(0, "bucket", "object.bin", &[&warning_version], async {
Err(Error::SlowDown.into())
})
.await
.expect("cleanup warnings should not fail the completed migration");
assert!(warning.is_some(), "cleanup failure should return a warning");
assert!(matches!(warning_result, RebalanceEntryCleanupResult::Completed { warning: Some(_) }));
let meta = store.rebalance_meta.read().await;
let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
assert_eq!(pool_stats.cleanup_warnings.count, 1, "cleanup warning must block pool completion");
assert!(pool_stats.bytes > 0, "completed migration bytes should still be recorded");
drop(meta);
{
let mut meta = store.rebalance_meta.write().await;
meta.as_mut().expect("rebalance metadata should exist").pool_stats[0].bytes = 0;
}
let deferred = store
.finish_rebalance_entry_after_cleanup(0, "bucket", "object.bin", &[&warning_version], async {
Err(data_movement::SourceCleanupError::SourceChanged)
})
.await
.expect("source changes should defer cleanup without failing the worker");
assert!(matches!(deferred, RebalanceEntryCleanupResult::Deferred { .. }));
let meta = store.rebalance_meta.read().await;
let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
assert_eq!(pool_stats.bytes, 0, "deferred cleanup must not commit completion stats");
assert_eq!(pool_stats.cleanup_warnings.count, 1, "deferred cleanup must not add a permanent warning");
}
}
crates/ecstore/src/services/rebalance/mod.rs
+2
View File
@@ -27,12 +27,14 @@ const REBAL_META_FMT: u16 = 1; // Replace with actual format value
const REBAL_META_VER: u16 = 1; // Replace with actual version value
pub(crate) const REBAL_META_NAME: &str = "rebalance.bin";
const DEFAULT_REBALANCE_MAX_ATTEMPTS: usize = 3;
pub(crate) const REBALANCE_SOURCE_CLEANUP_MAX_DEFERS: usize = 3;
const REBALANCE_MAX_ATTEMPTS_ENV: &str = "RUSTFS_REBALANCE_MAX_ATTEMPTS";
const REBALANCE_STOP_PROPAGATION_ERROR_PREFIX: &str = "rebalance stop propagation incomplete: ";
const REBALANCE_LISTING_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
const REBALANCE_MIGRATION_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
const REBALANCE_MIGRATION_LOCK_RETRY_CAP: Duration = Duration::from_secs(10);
const REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX: &str = "deferred transient rebalance entry failure:";
pub(crate) const REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX: &str = "deferred rebalance source cleanup conflict:";
const REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT: usize = 10;
mod control;
crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs
+58 -18
View File
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX;
use super::control::validate_rebalance_disk_stats_coverage;
use super::meta::{
RebalanceMetaMergeOutcome, RebalanceTerminalEvent, apply_rebalance_save_option, apply_rebalance_terminal_event,
@@ -33,23 +32,27 @@ use super::migration::{
MigrationBackend, MigrationVersionResult, migrate_entry_version, migrate_entry_version_with_retry_wait,
rebalance_delete_marker_opts,
};
use super::runtime::{should_fail_repeated_rebalance_bucket_defer, source_cleanup_defer_attempt};
use super::worker::{
ensure_rebalance_listing_disks_available, is_transient_rebalance_error, parse_rebalance_max_attempts,
rebalance_listing_retry_delay, rebalance_migration_retry_delay, resolve_load_rebalance_stats_update_result,
resolve_rebalance_bucket_error, resolve_rebalance_bucket_result, resolve_rebalance_entry_cleanup_delete_result,
resolve_rebalance_file_info_versions_result, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
resolve_rebalance_migrate_result_error, resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result,
resolve_rebalance_stats_update_result, resolve_rebalance_terminal_error, resolve_rebalance_worker_result,
run_rebalance_listing_with_retry, send_rebalance_done_signal, should_cleanup_rebalance_source_entry,
should_count_rebalance_version_complete, should_defer_rebalance_entry_failure, should_retry_rebalance_listing,
should_skip_rebalance_delete_marker, wait_rebalance_entry_tasks, wait_rebalance_listing_retry, with_rebalance_entry_context,
RebalanceEntryCleanupResult, ensure_rebalance_listing_disks_available, is_transient_rebalance_error,
parse_rebalance_max_attempts, rebalance_listing_retry_delay, rebalance_migration_retry_delay,
resolve_load_rebalance_stats_update_result, resolve_rebalance_bucket_error, resolve_rebalance_bucket_result,
resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, resolve_rebalance_migrate_result_error,
resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result, resolve_rebalance_stats_update_result,
resolve_rebalance_terminal_error, resolve_rebalance_worker_result, run_rebalance_listing_with_retry,
send_rebalance_done_signal, should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete,
should_defer_rebalance_entry_failure, should_retry_rebalance_listing, should_skip_rebalance_delete_marker,
wait_rebalance_entry_tasks, wait_rebalance_listing_retry, with_rebalance_entry_context,
};
use super::{
DiskStat, GetObjectReader, ObjectInfo, ObjectOptions, RebalSaveOpt, RebalStatus, RebalanceBucketConfigs,
RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta, RebalanceStats,
};
use super::{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX};
use crate::bucket::replication::{ReplicationState, ReplicationStatusType, replication_state_to_filemeta};
use crate::data_movement;
use crate::data_movement::SourceCleanupError;
use crate::data_usage::DATA_USAGE_CACHE_NAME;
use crate::disk::RUSTFS_META_BUCKET;
use crate::disk::error::DiskError;
@@ -1665,26 +1668,63 @@ fn test_resolve_rebalance_meta_load_result_wraps_error_context() {
#[test]
fn test_resolve_rebalance_entry_cleanup_delete_result_passthrough() {
let result = resolve_rebalance_entry_cleanup_delete_result(Ok(ObjectInfo::default()), "bucket-a", "obj.txt");
assert_eq!(result.expect("successful cleanup should pass through"), None);
assert_eq!(result, RebalanceEntryCleanupResult::Completed { warning: None });
}
#[test]
fn test_resolve_rebalance_entry_cleanup_delete_result_ignores_not_found() {
let result = resolve_rebalance_entry_cleanup_delete_result(
Err(Error::ObjectNotFound("bucket-a".to_string(), "obj.txt".to_string())),
Err(Error::ObjectNotFound("bucket-a".to_string(), "obj.txt".to_string()).into()),
"bucket-a",
"obj.txt",
);
assert_eq!(result.expect("missing cleanup source should be ignored"), None);
assert_eq!(result, RebalanceEntryCleanupResult::Completed { warning: None });
}
#[test]
fn test_resolve_rebalance_entry_cleanup_delete_result_returns_warning_for_failures() {
let warning = resolve_rebalance_entry_cleanup_delete_result(Err(Error::SlowDown), "bucket-a", "obj.txt")
.expect("cleanup delete failures should be downgraded to warnings")
.expect("cleanup delete failure should return warning");
let message = warning.as_str();
assert!(message.contains("rebalance cleanup delete failed for bucket-a/obj.txt"));
let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::SlowDown.into()), "bucket-a", "obj.txt");
assert!(matches!(
result,
RebalanceEntryCleanupResult::Completed { warning: Some(ref message) }
if message.contains("rebalance cleanup delete failed for bucket-a/obj.txt")
));
}
#[test]
fn test_resolve_rebalance_entry_cleanup_delete_result_defers_source_change() {
let result = resolve_rebalance_entry_cleanup_delete_result(Err(SourceCleanupError::SourceChanged), "bucket-a", "obj.txt");
assert!(matches!(
result,
RebalanceEntryCleanupResult::Deferred { ref last_error }
if last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX)
&& last_error.contains("source changed during cleanup preflight for bucket-a/obj.txt")
));
}
#[test]
fn test_resolve_rebalance_entry_cleanup_delete_result_does_not_defer_other_precondition_failure() {
let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::PreconditionFailed.into()), "bucket-a", "obj.txt");
assert!(matches!(
result,
RebalanceEntryCleanupResult::Completed { warning: Some(ref message) }
if message.contains("rebalance cleanup delete failed for bucket-a/obj.txt")
));
}
#[test]
fn test_source_cleanup_defer_does_not_fail_repeated_bucket_retry() {
let mut deferred_buckets = std::collections::HashSet::new();
assert!(!should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, "bucket-a", true));
assert!(!should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, "bucket-a", true));
assert!(!should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, "bucket-b", false));
assert!(should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, "bucket-b", false));
let mut source_attempts = std::collections::HashMap::new();
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 1);
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 2);
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 3);
}
#[test]
crates/ecstore/src/services/rebalance/runtime.rs
+62 -3
View File
@@ -10,12 +10,14 @@ use super::worker::{
resolve_rebalance_terminal_error, send_rebalance_done_signal,
};
use super::{
EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, RebalSaveOpt, RebalStatus,
EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalSaveOpt, RebalStatus,
RebalanceBucketOutcome,
};
use crate::error::{Error, Result};
use crate::runtime::sources as runtime_sources;
use crate::store::ECStore;
use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;
use time::OffsetDateTime;
@@ -23,6 +25,20 @@ use tokio::time::{Duration, Instant};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
pub(super) fn should_fail_repeated_rebalance_bucket_defer(
deferred_buckets: &mut HashSet<String>,
bucket: &str,
source_cleanup_deferred: bool,
) -> bool {
!source_cleanup_deferred && !deferred_buckets.insert(bucket.to_string())
}
pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<String, usize>, bucket: &str) -> usize {
let attempts = deferred_attempts.entry(bucket.to_string()).or_default();
*attempts = attempts.saturating_add(1);
*attempts
}
impl ECStore {
#[tracing::instrument(skip_all)]
pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
@@ -298,6 +314,7 @@ impl ECStore {
);
let mut final_result: Result<()> = Ok(());
let mut deferred_buckets = HashSet::new();
let mut source_cleanup_deferred_attempts = HashMap::new();
loop {
if rx.is_cancelled() {
@@ -375,7 +392,8 @@ impl ECStore {
};
if let RebalanceBucketOutcome::Deferred { last_error } = outcome {
if !deferred_buckets.insert(bucket.clone()) {
let source_cleanup_deferred = last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX);
if should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, &bucket, source_cleanup_deferred) {
let err = Error::other(format!(
"rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}"
));
@@ -396,6 +414,11 @@ impl ECStore {
break;
}
let source_cleanup_attempt = if source_cleanup_deferred {
source_cleanup_defer_attempt(&mut source_cleanup_deferred_attempts, &bucket)
} else {
0
};
warn!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
@@ -406,7 +429,10 @@ impl ECStore {
error = %last_error,
"Deferred rebalance bucket after transient object failures"
);
if let Err(err) = self.defer_rebalance_bucket(pool_index, bucket.clone(), last_error).await {
if let Err(err) = self
.defer_rebalance_bucket(pool_index, bucket.clone(), last_error.clone())
.await
{
error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
@@ -423,6 +449,38 @@ impl ECStore {
));
break;
}
if source_cleanup_deferred {
if source_cleanup_attempt >= super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS {
let err = Error::other(format!(
"rebalance bucket {bucket} source cleanup remained unstable after {} deferrals: {last_error}",
super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
));
warn!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "source_cleanup_defer_limit",
error = ?err,
"Rebalance bucket failed after repeated source cleanup conflicts"
);
final_result = Err(resolve_rebalance_terminal_error(
err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
));
break;
}
if let Err(err) =
super::worker::wait_rebalance_listing_retry(&rx, REBALANCE_LISTING_RETRY_BASE_DELAY).await
{
final_result = Err(resolve_rebalance_terminal_error(
err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
));
break;
}
}
continue;
}
@@ -435,6 +493,7 @@ impl ECStore {
state = "completed",
"Completed rebalance bucket"
);
source_cleanup_deferred_attempts.remove(&bucket);
if let Err(err) = self.bucket_rebalance_done(pool_index, bucket).await {
error!(
event = EVENT_REBALANCE_BUCKET,
crates/ecstore/src/services/rebalance/worker.rs
+23 -6
View File
@@ -2,10 +2,12 @@ use super::migration::MigrationVersionResult;
use super::{
DEFAULT_REBALANCE_MAX_ATTEMPTS, EVENT_REBALANCE_LISTING, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_MAX_ATTEMPTS_ENV, REBALANCE_MIGRATION_LOCK_RETRY_CAP,
REBALANCE_MIGRATION_RETRY_BASE_DELAY, RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome, Result,
REBALANCE_MIGRATION_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalanceBucketConfigs,
RebalanceBucketOutcome, RebalanceEntryOutcome, Result,
};
use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
use crate::core::pools::ListCallback;
use crate::data_movement::SourceCleanupError;
use crate::disk::error::DiskError;
use crate::error::{
Error, is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found, is_network_or_host_down,
@@ -36,6 +38,12 @@ pub(super) fn resolve_rebalance_worker_result<T>(
pub(super) type RebalanceEntryTask = tokio::task::JoinHandle<Result<RebalanceEntryOutcome>>;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) enum RebalanceEntryCleanupResult {
Completed { warning: Option<String> },
Deferred { last_error: String },
}
pub(super) async fn wait_rebalance_entry_tasks(
set_idx: usize,
tasks: Arc<tokio::sync::Mutex<Vec<RebalanceEntryTask>>>,
@@ -145,14 +153,23 @@ where
}
pub(super) fn resolve_rebalance_entry_cleanup_delete_result(
result: Result<crate::object_api::ObjectInfo>,
result: std::result::Result<crate::object_api::ObjectInfo, SourceCleanupError>,
bucket: &str,
object_name: &str,
) -> Result<Option<String>> {
) -> RebalanceEntryCleanupResult {
match result {
Ok(_) => Ok(None),
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
Err(err) => Ok(Some(format!("rebalance cleanup delete failed for {bucket}/{object_name}: {err}"))),
Ok(_) => RebalanceEntryCleanupResult::Completed { warning: None },
Err(SourceCleanupError::Storage(err)) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
RebalanceEntryCleanupResult::Completed { warning: None }
}
Err(SourceCleanupError::SourceChanged) => RebalanceEntryCleanupResult::Deferred {
last_error: format!(
"{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} source changed during cleanup preflight for {bucket}/{object_name}"
),
},
Err(SourceCleanupError::Storage(err)) => RebalanceEntryCleanupResult::Completed {
warning: Some(format!("rebalance cleanup delete failed for {bucket}/{object_name}: {err}")),
},
}
}