mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-20 03:22:18 +00:00
fix(storage): harden rebalance decommission state (#3515)
This commit is contained in:
cxymds
@@ -32,10 +32,13 @@ use std::hash::{Hash, Hasher};
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::time::timeout;
|
||||
use tracing::{error, warn};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
/// After this many consecutive admin-call failures, mark the peer as offline.
|
||||
const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3;
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification";
|
||||
const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation";
|
||||
|
||||
/// Cached result from the last successful admin call to a peer.
|
||||
struct PeerAdminCache {
|
||||
@@ -472,6 +475,15 @@ impl NotificationSys {
|
||||
let host = client.grid_host.clone();
|
||||
futures.push(async move { client.reload_pool_meta().await.map_err(|err| (host, err)) });
|
||||
} else {
|
||||
warn!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
action = "reload_pool_meta",
|
||||
result = "peer_unreachable",
|
||||
peer_index = idx,
|
||||
"notification peer propagation"
|
||||
);
|
||||
failures.push(format!("peer[{idx}] reload_pool_meta failed: peer is not reachable"));
|
||||
}
|
||||
}
|
||||
@@ -479,7 +491,16 @@ impl NotificationSys {
|
||||
for result in join_all(futures).await {
|
||||
if let Err((host, err)) = result {
|
||||
let failure = format!("peer {host} reload_pool_meta failed: {err}");
|
||||
error!("notification reload_pool_meta err {}", failure);
|
||||
error!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
action = "reload_pool_meta",
|
||||
result = "peer_failed",
|
||||
peer = %host,
|
||||
error = %err,
|
||||
"notification peer propagation"
|
||||
);
|
||||
failures.push(failure);
|
||||
}
|
||||
}
|
||||
@@ -489,87 +510,187 @@ impl NotificationSys {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub async fn load_rebalance_meta(&self, start: bool) -> Result<()> {
|
||||
let failures = self.load_rebalance_meta_failures(start).await?;
|
||||
aggregate_notification_failures("load_rebalance_meta", failures)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub async fn load_rebalance_meta_failures(&self, start: bool) -> Result<Vec<String>> {
|
||||
let operation = format!("load_rebalance_meta(start={start})");
|
||||
let mut failures = Vec::new();
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for (idx, client) in self.peer_clients.iter().enumerate() {
|
||||
if let Some(client) = client {
|
||||
warn!(
|
||||
"notification load_rebalance_meta start: {}, index: {}, client: {:?}",
|
||||
start, idx, client.host
|
||||
);
|
||||
let host = client.grid_host.clone();
|
||||
futures.push(async move { client.load_rebalance_meta(start).await.map_err(|err| (host, err)) });
|
||||
futures.push(async move {
|
||||
let result = client.load_rebalance_meta(start).await;
|
||||
(host, result)
|
||||
});
|
||||
} else {
|
||||
warn!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
action = "load_rebalance_meta",
|
||||
result = "peer_unreachable",
|
||||
peer_index = idx,
|
||||
start_rebalance = start,
|
||||
"notification peer propagation"
|
||||
);
|
||||
failures.push(format!("peer[{idx}] {operation} failed: peer is not reachable"));
|
||||
}
|
||||
}
|
||||
|
||||
for result in join_all(futures).await {
|
||||
if let Err((host, err)) = result {
|
||||
for (host, result) in join_all(futures).await {
|
||||
if let Err(err) = result {
|
||||
let failure = format!("peer {host} {operation} failed: {err}");
|
||||
error!("notification load_rebalance_meta err {}", failure);
|
||||
error!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
action = "load_rebalance_meta",
|
||||
result = "peer_failed",
|
||||
peer = %host,
|
||||
start_rebalance = start,
|
||||
error = %err,
|
||||
"notification peer propagation"
|
||||
);
|
||||
failures.push(failure);
|
||||
} else {
|
||||
warn!("notification load_rebalance_meta success");
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
action = "load_rebalance_meta",
|
||||
result = "peer_success",
|
||||
peer = %host,
|
||||
start_rebalance = start,
|
||||
"notification peer propagation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
aggregate_notification_failures("load_rebalance_meta", failures)
|
||||
Ok(failures)
|
||||
}
|
||||
|
||||
pub async fn stop_rebalance(&self) -> Result<()> {
|
||||
warn!("notification stop_rebalance start");
|
||||
pub async fn stop_rebalance(&self, expected_rebalance_id: Option<&str>) -> Result<()> {
|
||||
let failures = self.stop_rebalance_failures(expected_rebalance_id).await?;
|
||||
aggregate_notification_failures("stop_rebalance", failures)
|
||||
}
|
||||
|
||||
pub async fn stop_rebalance_failures(&self, expected_rebalance_id: Option<&str>) -> Result<Vec<String>> {
|
||||
info!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
action = "stop_rebalance",
|
||||
state = "started",
|
||||
"notification peer propagation"
|
||||
);
|
||||
let Some(store) = resolve_object_store_handle() else {
|
||||
error!("stop_rebalance: not init");
|
||||
error!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
action = "stop_rebalance",
|
||||
result = "failed",
|
||||
reason = "object_layer_not_initialized",
|
||||
"notification peer propagation"
|
||||
);
|
||||
return Err(Error::other("stop_rebalance: object layer not initialized"));
|
||||
};
|
||||
|
||||
// warn!("notification stop_rebalance load_rebalance_meta");
|
||||
// self.load_rebalance_meta(false).await;
|
||||
// warn!("notification stop_rebalance load_rebalance_meta done");
|
||||
|
||||
let mut failures = Vec::new();
|
||||
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for (idx, client) in self.peer_clients.iter().enumerate() {
|
||||
if let Some(client) = client {
|
||||
let host = client.grid_host.clone();
|
||||
futures.push(async move { client.stop_rebalance().await.map_err(|err| (host, err)) });
|
||||
futures.push(async move {
|
||||
let result = client.stop_rebalance(expected_rebalance_id).await;
|
||||
(host, result)
|
||||
});
|
||||
} else {
|
||||
warn!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
action = "stop_rebalance",
|
||||
result = "peer_unreachable",
|
||||
peer_index = idx,
|
||||
"notification peer propagation"
|
||||
);
|
||||
failures.push(format!("peer[{idx}] stop_rebalance failed: peer is not reachable"));
|
||||
}
|
||||
}
|
||||
|
||||
for result in join_all(futures).await {
|
||||
if let Err((host, err)) = result {
|
||||
for (host, result) in join_all(futures).await {
|
||||
if let Err(err) = result {
|
||||
let failure = format!("peer {host} stop_rebalance failed: {err}");
|
||||
error!("notification stop_rebalance err {}", failure);
|
||||
error!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
action = "stop_rebalance",
|
||||
result = "peer_failed",
|
||||
peer = %host,
|
||||
error = %err,
|
||||
"notification peer propagation"
|
||||
);
|
||||
failures.push(failure);
|
||||
} else {
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
action = "stop_rebalance",
|
||||
result = "peer_success",
|
||||
peer = %host,
|
||||
"notification peer propagation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
warn!("notification stop_rebalance stop_rebalance start");
|
||||
match store.stop_rebalance().await {
|
||||
match store.stop_rebalance_for_id(expected_rebalance_id).await {
|
||||
Ok(_) => {
|
||||
if let Err(err) = store.save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt).await {
|
||||
error!("notification stop_rebalance local save err {:?}", err);
|
||||
error!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
action = "stop_rebalance",
|
||||
result = "local_save_failed",
|
||||
error = %err,
|
||||
"notification peer propagation"
|
||||
);
|
||||
return Err(Error::other(format!(
|
||||
"local stop_rebalance save_rebalance_stats(stopped_at) failed: {err}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
error!("notification stop_rebalance local stop err {:?}", err);
|
||||
error!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
action = "stop_rebalance",
|
||||
result = "local_stop_failed",
|
||||
error = %err,
|
||||
"notification peer propagation"
|
||||
);
|
||||
return Err(Error::other(format!("local stop_rebalance stop failed: {err}")));
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(err) = aggregate_notification_failures("stop_rebalance", failures) {
|
||||
warn!("{err}");
|
||||
}
|
||||
warn!("notification stop_rebalance stop_rebalance done");
|
||||
Ok(())
|
||||
info!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
action = "stop_rebalance",
|
||||
result = if failures.is_empty() { "success" } else { "partial_failure" },
|
||||
"notification peer propagation"
|
||||
);
|
||||
Ok(failures)
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str) -> Result<()> {
|
||||
@@ -1094,6 +1215,53 @@ mod tests {
|
||||
assert!(msg.contains("local save failed"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_rebalance_meta_aggregate_failures_return_error() {
|
||||
let err = aggregate_notification_failures(
|
||||
"load_rebalance_meta(start=true)",
|
||||
vec!["peer[0] load_rebalance_meta failed: peer is not reachable".to_string()],
|
||||
)
|
||||
.expect_err("load_rebalance_meta peer failures must be returned");
|
||||
|
||||
let msg = err.to_string();
|
||||
assert!(msg.contains("load_rebalance_meta(start=true)"));
|
||||
assert!(msg.contains("1 failure(s)"));
|
||||
assert!(msg.contains("peer[0]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_rebalance_aggregate_failures_return_error() {
|
||||
let err = aggregate_notification_failures(
|
||||
"stop_rebalance",
|
||||
vec!["peer[0] stop_rebalance failed: peer is not reachable".to_string()],
|
||||
)
|
||||
.expect_err("stop_rebalance peer failures must be returned");
|
||||
|
||||
let msg = err.to_string();
|
||||
assert!(msg.contains("stop_rebalance"));
|
||||
assert!(msg.contains("1 failure(s)"));
|
||||
assert!(msg.contains("peer[0]"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reload_pool_meta_reports_unreachable_peers() {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: Vec::new(),
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
};
|
||||
|
||||
let err = sys
|
||||
.reload_pool_meta()
|
||||
.await
|
||||
.expect_err("unreachable peers should fail pool metadata reload");
|
||||
|
||||
let msg = err.to_string();
|
||||
assert!(msg.contains("reload_pool_meta"));
|
||||
assert!(msg.contains("1 failure(s)"));
|
||||
assert!(msg.contains("peer[0]"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_bucket_metadata_reports_unreachable_peers() {
|
||||
let sys = NotificationSys {
|
||||
|
||||
Reference in New Issue
Block a user