gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-17 10:17:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(rebalance): fence peers before activation (#5842)

Browse Source
This commit is contained in:
cxymds
2026-08-08 19:55:50 +08:00
committed by GitHub
parent c2e23411e8
commit a7de957eb8
4 changed files with 374 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/services/rebalance/control.rs
+43 -10
View File
@@ -327,16 +327,8 @@ impl ECStore {
#[tracing::instrument(skip(self, bucktes))]
pub async fn init_and_start_rebalance(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
let _start_guard = self.start_gate.lock().await;
let decommission_running = self.is_decommission_running().await;
{
let rebalance_meta = self.rebalance_meta.read().await;
validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
}
let id = self.init_rebalance_meta(bucktes).await?;
if let Err(start_err) = self.start_rebalance().await {
let id = self.init_rebalance_start(bucktes).await?;
if let Err(start_err) = self.start_rebalance_for_id(&id).await {
if let Err(rollback_err) = self
.rollback_rebalance_start_without_worker_for_id(Some(&id), start_err.to_string())
.await
@@ -354,6 +346,47 @@ impl ECStore {
Ok(id)
}
#[tracing::instrument(skip(self, bucktes))]
pub async fn init_rebalance_start(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
let _start_guard = self.start_gate.lock().await;
let decommission_running = self.is_decommission_running().await;
{
let rebalance_meta = self.rebalance_meta.read().await;
validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
}
self.init_rebalance_meta(bucktes).await
}
#[tracing::instrument(skip(self))]
pub async fn start_rebalance_for_id(self: &Arc<Self>, expected_id: &str) -> Result<()> {
let _start_guard = self.start_gate.lock().await;
{
let rebalance_meta = self.rebalance_meta.read().await;
let Some(meta) = rebalance_meta.as_ref() else {
return Err(Error::ConfigNotFound);
};
if meta.id != expected_id {
return Err(Error::other(format!(
"rebalance metadata changed before start: expected {expected_id}, found {}",
meta.id
)));
}
if meta.stopped_at.is_some() {
return Err(Error::other(format!("rebalance {expected_id} was stopped before start")));
}
}
self.start_rebalance().await
}
pub async fn rollback_rebalance_start_for_id(self: &Arc<Self>, expected_id: Option<&str>, start_error: String) -> Result<()> {
self.rollback_rebalance_start_without_worker_for_id(expected_id, start_error)
.await
}
#[tracing::instrument(skip(self, fi))]
pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> {
self.update_pool_stats_batch(pool_index, bucket, &[fi]).await
crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs
+67 -15
View File
@@ -2584,21 +2584,7 @@ async fn test_init_and_start_rebalance_rejects_second_start_after_gate() {
}],
..Default::default()
};
let endpoint_pools: crate::layout::endpoints::EndpointServerPools = Vec::new().into();
let store = Arc::new(crate::store::ECStore {
id: uuid::Uuid::new_v4(),
disk_map: std::collections::HashMap::new(),
pools: Vec::new(),
peer_sys: crate::cluster::rpc::S3PeerSys::new(&endpoint_pools),
pool_meta: tokio::sync::RwLock::new(crate::core::pools::PoolMeta::default()),
rebalance_meta: tokio::sync::RwLock::new(Some(active_meta)),
decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: tokio::sync::Mutex::new(()),
ctx: crate::runtime::instance::bootstrap_ctx(),
bucket_fence_registry: std::sync::Arc::default(),
});
let store = test_store_with_rebalance_meta(active_meta);
let err = store
.init_and_start_rebalance(vec!["bucket".to_string()])
@@ -2608,6 +2594,72 @@ async fn test_init_and_start_rebalance_rejects_second_start_after_gate() {
assert!(matches!(err, Error::RebalanceAlreadyRunning));
}
#[tokio::test]
async fn test_start_rebalance_for_id_rejects_changed_metadata() {
let meta = RebalanceMeta {
id: "rebalance-a".to_string(),
pool_stats: vec![RebalanceStats {
participating: true,
info: RebalanceInfo {
status: RebalStatus::Started,
..Default::default()
},
..Default::default()
}],
..Default::default()
};
let store = test_store_with_rebalance_meta(meta);
let err = store
.start_rebalance_for_id("rebalance-b")
.await
.expect_err("staged start must not start changed metadata");
assert!(err.to_string().contains("rebalance metadata changed before start"));
}
#[tokio::test]
async fn test_start_rebalance_for_id_rejects_stopped_metadata() {
let meta = RebalanceMeta {
id: "rebalance-a".to_string(),
stopped_at: Some(OffsetDateTime::now_utc()),
pool_stats: vec![RebalanceStats {
participating: true,
info: RebalanceInfo {
status: RebalStatus::Started,
..Default::default()
},
..Default::default()
}],
..Default::default()
};
let store = test_store_with_rebalance_meta(meta);
let err = store
.start_rebalance_for_id("rebalance-a")
.await
.expect_err("staged start must not restart stopped metadata");
assert!(err.to_string().contains("was stopped before start"));
}
fn test_store_with_rebalance_meta(meta: RebalanceMeta) -> Arc<crate::store::ECStore> {
let endpoint_pools: crate::layout::endpoints::EndpointServerPools = Vec::new().into();
Arc::new(crate::store::ECStore {
id: uuid::Uuid::new_v4(),
disk_map: std::collections::HashMap::new(),
pools: Vec::new(),
peer_sys: crate::cluster::rpc::S3PeerSys::new(&endpoint_pools),
pool_meta: tokio::sync::RwLock::new(crate::core::pools::PoolMeta::default()),
rebalance_meta: tokio::sync::RwLock::new(Some(meta)),
decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: tokio::sync::Mutex::new(()),
ctx: crate::runtime::instance::bootstrap_ctx(),
bucket_fence_registry: std::sync::Arc::default(),
})
}
#[test]
fn test_percent_free_ratio_zero_capacity_is_zero() {
assert_eq!(percent_free_ratio(100, 0), 0.0);