gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-02 18:28:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): finalize decommission capacity recovery (#6955)

Browse Source
This commit is contained in:
Zhengchao An
2026-08-31 18:09:09 +08:00
committed by GitHub
parent 9a22cb85f3
commit 9d4ccb7884
10 changed files with 280 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/core/pools.rs
+148 -26
View File
@@ -128,6 +128,7 @@ const METRIC_DECOMMISSION_CAPACITY_PREDICTION_ABSOLUTE_ERROR_BYTES: &str =
const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
pub(crate) const DECOMMISSION_ENTRY_MAX_ATTEMPTS: usize = 3;
const DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS: usize = 12;
const DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(100);
pub(crate) const DECOMMISSION_VERSION_COPY_ATTEMPTS: usize = 3;
const DECOMMISSION_COPY_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(50);
@@ -927,6 +928,16 @@ fn is_decommission_capacity_blocked_error(err: &Error) -> bool {
data_movement::data_movement_stage_source(err).is_some_and(is_decommission_capacity_blocked_error)
}
fn is_decommission_capacity_intent_conflict(err: &Error) -> bool {
if matches!(err, Error::DecommissionCapacityBlocked { message } if message.contains("unresolved target capacity intent")) {
return true;
}
if data_movement::data_movement_stage_source(err).is_some_and(is_decommission_capacity_intent_conflict) {
return true;
}
err.to_string().contains("unresolved target capacity intent")
}
fn validate_decommission_capacity_reservation(reservation: Option<&DecommissionCapacityReservation>) -> Result<()> {
let Some(reservation) = reservation else {
return Ok(());
@@ -1608,9 +1619,9 @@ fn reserve_decommission_target_pending(
pool.last_update = now;
return Ok(additional);
}
_ => {
pending_mutation_id => {
return Err(decommission_capacity_blocked_error(format!(
"source pool {source_pool_index} target pool {target_pool_index} has an unresolved target capacity intent"
"source pool {source_pool_index} target pool {target_pool_index} has an unresolved target capacity intent {pending_mutation_id:?} while mutation {mutation_id} is waiting"
)));
}
}
@@ -7789,6 +7800,19 @@ fn decommission_remote_tiered_opts(
}
}
fn decommission_capacity_version_mutation_id(
owner: DecommissionCapacityOwner,
bucket: &str,
version: &rustfs_filemeta::FileInfo,
) -> uuid::Uuid {
let version_id = if version.deleted && version.version_id.is_none() {
Some(uuid::Uuid::nil().to_string())
} else {
version.version_id.map(|version_id| version_id.to_string())
};
decommission_capacity_mutation_id(owner, bucket, &version.name, version_id.as_deref(), version.deleted, version.mod_time)
}
fn decommission_capacity_owned_opts(mut opts: ObjectOptions, capacity_owner: Option<DecommissionCapacityOwner>) -> ObjectOptions {
if let Some(capacity_owner) = capacity_owner {
capacity_owner.apply_to(&mut opts);
@@ -9162,13 +9186,15 @@ impl ECStore {
) -> Result<Option<DecommissionCapacityOwner>> {
let pool_meta = self.pool_meta.read().await;
ensure_decommission_generation(&pool_meta, idx, generation)?;
let reservation = pool_meta
let Some(reservation) = pool_meta
.pools
.get(idx)
.and_then(|pool| pool.decommission.as_ref())
.and_then(|info| info.capacity_reservation.as_ref())
.filter(|reservation| reservation.lease_active_at(OffsetDateTime::now_utc()))
.ok_or_else(|| decommission_capacity_blocked_error(format!("source pool {idx} has no active reservation")))?;
else {
return Ok(None);
};
Ok(Some(DecommissionCapacityOwner {
source_pool_index: idx,
operation_id: reservation.operation_id,
@@ -10320,29 +10346,65 @@ impl ECStore {
expected_bucket_incarnation_id: Option<uuid::Uuid>,
source_changed_exhaustions: Arc<AtomicUsize>,
) -> Result<()> {
let uses_capacity_ledger = self
.pool_meta
.read()
.await
.pools
.get(idx)
.and_then(|pool| pool.decommission.as_ref())
.and_then(|info| info.capacity_reservation.as_ref())
.is_some_and(DecommissionCapacityReservation::active);
let mut counted_versions = HashSet::new();
for entry_attempt in 1..=DECOMMISSION_ENTRY_MAX_ATTEMPTS {
match self
.decommission_entry_attempt(
rx.clone(),
idx,
generation,
entry.clone(),
bucket.clone(),
Arc::clone(&set),
lifecycle_config.clone(),
object_lock_config.clone(),
replication_config.clone(),
expected_bucket_incarnation_id,
entry_attempt,
source_changed_exhaustions.as_ref(),
&mut counted_versions,
)
.await?
{
DecommissionEntryAttemptOutcome::Complete => return Ok(()),
DecommissionEntryAttemptOutcome::SourceChanged => {
let attempt_result = {
let mut conflict_attempt = 0;
loop {
let result = {
let _capacity_entry_guard = if uses_capacity_ledger {
Some(tokio::select! {
biased;
_ = rx.cancelled() => return decommission_cancel_signal_result(true),
guard = self.decommission_capacity_entry_gate.lock() => guard,
})
} else {
None
};
self.decommission_entry_attempt(
rx.clone(),
idx,
generation,
entry.clone(),
bucket.clone(),
Arc::clone(&set),
lifecycle_config.clone(),
object_lock_config.clone(),
replication_config.clone(),
expected_bucket_incarnation_id,
entry_attempt,
source_changed_exhaustions.as_ref(),
&mut counted_versions,
)
.await
};
if result.as_ref().is_err_and(is_decommission_capacity_intent_conflict)
&& conflict_attempt < DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS
{
conflict_attempt += 1;
let retry_delay =
decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, conflict_attempt);
if wait_decommission_retry_backoff(&rx, retry_delay).await {
decommission_cancel_signal_result(rx.is_cancelled())?;
}
continue;
}
break result;
}
};
match attempt_result {
Ok(DecommissionEntryAttemptOutcome::Complete) => return Ok(()),
Ok(DecommissionEntryAttemptOutcome::SourceChanged) => {
let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, entry_attempt);
warn!(
event = EVENT_DECOMMISSION_ENTRY,
@@ -10361,6 +10423,7 @@ impl ECStore {
decommission_cancel_signal_result(rx.is_cancelled())?;
}
}
Err(err) => return Err(err),
}
}
@@ -10434,8 +10497,34 @@ impl ECStore {
let mut fivs = load_decommission_entry_exact_versions(&set, &entry, &bucket, "file_info_versions").await?;
fivs.versions
.sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
let pending_mutations = if let Some(owner) = capacity_owner {
self.pool_meta
.read()
.await
.pools
.get(owner.source_pool_index)
.and_then(|pool| pool.decommission.as_ref())
.and_then(|info| info.capacity_reservation.as_ref())
.filter(|reservation| reservation.admits_cleanup_owner(owner))
.map(|reservation| {
reservation
.targets
.iter()
.filter_map(|target| target.pending_mutation_id)
.collect::<HashSet<_>>()
})
.unwrap_or_default()
} else {
HashSet::new()
};
fivs.versions.sort_by_key(|version| {
let mutation_id = capacity_owner.map(|owner| decommission_capacity_version_mutation_id(owner, &bucket, version));
(
mutation_id.is_none_or(|mutation_id| !pending_mutations.contains(&mutation_id)),
version.mod_time.is_none(),
std::cmp::Reverse(version.mod_time),
)
});
let mut decommissioned: usize = 0;
let mut expired: usize = 0;
@@ -15353,6 +15442,18 @@ mod tests {
assert!(is_decommission_target_capacity_error(&storage_full));
}
#[test]
fn decommission_capacity_intent_conflict_accepts_context_wrapped_error() {
let err = with_decommission_entry_context(
"migrate_object",
"bucket",
"object",
decommission_capacity_blocked_error("target has an unresolved target capacity intent"),
);
assert!(is_decommission_capacity_intent_conflict(&err));
}
#[test]
fn decommission_target_capacity_error_rejects_unrelated_errors() {
assert!(!is_decommission_target_capacity_error(&Error::SlowDown));
@@ -15429,6 +15530,7 @@ mod tests {
#[test]
fn decommission_delete_marker_opts_preserves_suspended_null_version() {
let version = rustfs_filemeta::FileInfo {
name: "object".to_string(),
deleted: true,
..Default::default()
};
@@ -15437,6 +15539,25 @@ mod tests {
assert!(!opts.versioned);
assert!(opts.version_suspended);
assert_eq!(opts.version_id.as_deref(), Some(uuid::Uuid::nil().to_string().as_str()));
let owner = DecommissionCapacityOwner {
source_pool_index: 7,
operation_id: uuid::Uuid::new_v4(),
generation: 1,
owner_nonce: uuid::Uuid::new_v4(),
mutation_id: None,
};
assert_eq!(
decommission_capacity_version_mutation_id(owner, "bucket", &version),
decommission_capacity_mutation_id(
owner,
"bucket",
&version.name,
opts.version_id.as_deref(),
opts.delete_marker,
opts.mod_time,
)
);
}
#[test]
@@ -16549,6 +16670,7 @@ mod pools_tests {
decommission_cancelers: tokio::sync::RwLock::new(cancelers),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: tokio::sync::Mutex::new(super::PoolMetaWriteState::for_test_bootstrap()),
decommission_capacity_entry_gate: tokio::sync::Mutex::default(),
ctx,
bucket_fence_registry: Arc::default(),
})