gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-29 00:17:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): repair decommission pool quorum (#2847)

Browse Source 
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
2026-05-07 19:14:11 +08:00
committed by GitHub
parent 5431b9273d
commit 2a0fbb8d77
3 changed files with 225 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/rebalance.rs
+77 -6
View File
@@ -780,6 +780,7 @@ impl ECStore {
let cancel_tx = CancellationToken::new();
let rx = cancel_tx.clone();
let mut meta_to_save = None;
{
let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -792,11 +793,19 @@ impl ECStore {
info!("start_rebalance: already in progress, skip duplicate start");
return Ok(());
}
if complete_rebalance_pools_at_goal(meta, OffsetDateTime::now_utc()) {
meta_to_save = Some(meta.clone());
}
meta.cancel = Some(cancel_tx);
drop(rebalance_meta);
}
if let Some(meta) = meta_to_save {
let pool = clone_first_arc(self.pools.as_slice(), "start_rebalance: no pools available")?;
resolve_rebalance_meta_save_result(meta.save(pool).await, "start_rebalance complete pools at goal")?;
}
let participants = if let Some(ref meta) = *self.rebalance_meta.read().await {
resolve_rebalance_participants(meta.pool_stats.as_slice(), self.pools.len())
} else {
@@ -1126,6 +1135,30 @@ fn should_pool_participate(init_free_space: u64, init_capacity: u64, percent_fre
init_capacity > 0 && percent_free_ratio(init_free_space, init_capacity) < percent_free_goal
}
fn complete_rebalance_pools_at_goal(meta: &mut RebalanceMeta, now: OffsetDateTime) -> bool {
let mut changed = false;
for pool_stat in meta.pool_stats.iter_mut() {
if !is_rebalance_pool_started(pool_stat) {
continue;
}
if rebalance_goal_reached(
pool_stat.init_free_space,
pool_stat.init_capacity,
pool_stat.bytes,
meta.percent_free_goal,
) {
pool_stat.info.status = RebalStatus::Completed;
pool_stat.info.end_time = Some(now);
pool_stat.info.last_error = None;
changed = true;
}
}
changed
}
fn resolve_rebalance_worker_result(
set_idx: usize,
worker_result: std::result::Result<Result<()>, tokio::task::JoinError>,
@@ -1838,12 +1871,12 @@ mod rebalance_unit_tests {
GetObjectReader, HTTPRangeSpec, MigrationBackend, MigrationVersionResult, ObjectInfo, ObjectOptions, RebalSaveOpt,
RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceTerminalEvent, apply_rebalance_save_option,
apply_rebalance_terminal_event, apply_stopped_at, classify_rebalance_terminal_event, clone_arc_by_index, clone_first_arc,
clone_rebalance_pool_stats, ensure_rebalance_listing_disks_available, ensure_rebalance_not_decommissioning,
ensure_valid_rebalance_pool_index, is_rebalance_stopped_terminal_event, load_rebalance_bucket_configs,
mark_rebalance_bucket_done, migrate_entry_version, next_rebal_bucket_from_stat, rebalance_delete_marker_opts,
rebalance_meta_load_no_data_error, rebalance_meta_load_unknown_format_error, rebalance_meta_load_unknown_version_error,
resolve_load_rebalance_stats_update_result, resolve_next_rebalance_bucket, resolve_rebalance_bucket_error,
resolve_rebalance_bucket_result, resolve_rebalance_entry_cleanup_delete_result,
clone_rebalance_pool_stats, complete_rebalance_pools_at_goal, ensure_rebalance_listing_disks_available,
ensure_rebalance_not_decommissioning, ensure_valid_rebalance_pool_index, is_rebalance_stopped_terminal_event,
load_rebalance_bucket_configs, mark_rebalance_bucket_done, migrate_entry_version, next_rebal_bucket_from_stat,
rebalance_delete_marker_opts, rebalance_meta_load_no_data_error, rebalance_meta_load_unknown_format_error,
rebalance_meta_load_unknown_version_error, resolve_load_rebalance_stats_update_result, resolve_next_rebalance_bucket,
resolve_rebalance_bucket_error, resolve_rebalance_bucket_result, resolve_rebalance_entry_cleanup_delete_result,
resolve_rebalance_file_info_versions_result, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
resolve_rebalance_migrate_result_error, resolve_rebalance_optional_bucket_config_result, resolve_rebalance_participants,
resolve_rebalance_save_task_result, resolve_rebalance_stats_update_result, resolve_rebalance_terminal_error,
@@ -3248,6 +3281,44 @@ mod rebalance_unit_tests {
assert!(!should_pool_participate(300, 1_000, 0.3));
}
#[test]
fn test_complete_rebalance_pools_at_goal_marks_started_participants_completed() {
let now = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
let mut meta = RebalanceMeta {
percent_free_goal: 0.5,
pool_stats: vec![
RebalanceStats {
participating: true,
init_free_space: 400,
init_capacity: 1_000,
bytes: 50,
info: RebalanceInfo {
status: RebalStatus::Started,
..Default::default()
},
..Default::default()
},
RebalanceStats {
participating: true,
init_free_space: 100,
init_capacity: 1_000,
bytes: 0,
info: RebalanceInfo {
status: RebalStatus::Started,
..Default::default()
},
..Default::default()
},
],
..Default::default()
};
assert!(complete_rebalance_pools_at_goal(&mut meta, now));
assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Completed);
assert_eq!(meta.pool_stats[0].info.end_time, Some(now));
assert_eq!(meta.pool_stats[1].info.status, RebalStatus::Started);
}
#[test]
fn test_should_skip_start_rebalance_only_when_running_and_cancel_attached() {
assert!(should_skip_start_rebalance(true, true));