gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-29 00:17:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(storage): harden rebalance decommission state (#3515)

Browse Source
This commit is contained in:
cxymds
2026-06-22 12:03:13 +08:00
committed by GitHub
parent 2dcc32db5a
commit 2f25cf606e
43 changed files with 10151 additions and 835 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/rebalance/control.rs
+98 -4
View File
@@ -2,7 +2,9 @@ use super::meta::{
clone_first_arc, clone_rebalance_pool_stats, defer_bucket_in_rebalance_queue, ensure_valid_rebalance_pool_index,
invalid_rebalance_pool_index_error, is_rebalance_conflicting_with_decommission, mark_rebalance_bucket_done,
merge_rebalance_meta, percent_free_ratio, rebalance_metadata_not_initialized_error, record_rebalance_cleanup_warning_in_meta,
resolve_next_rebalance_bucket, should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot,
record_rebalance_stop_propagation_snapshot, resolve_next_rebalance_bucket, rollback_rebalance_start_meta_snapshot_for_id,
should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot_for_id,
validate_init_rebalance_state,
};
use super::worker::{
rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result, resolve_rebalance_meta_load_result,
@@ -10,7 +12,8 @@ use super::worker::{
};
use super::{
DiskStat, EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats,
RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
encode_rebalance_stop_propagation_record,
};
use crate::error::{Error, Result};
use crate::object_api::ObjectOptions;
@@ -247,6 +250,35 @@ impl ECStore {
Ok(id)
}
#[tracing::instrument(skip(self, bucktes))]
pub async fn init_and_start_rebalance(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
let _start_guard = self.start_gate.lock().await;
let decommission_running = self.is_decommission_running().await;
{
let rebalance_meta = self.rebalance_meta.read().await;
validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
}
let id = self.init_rebalance_meta(bucktes).await?;
if let Err(start_err) = self.start_rebalance().await {
if let Err(rollback_err) = self
.rollback_rebalance_start_without_worker_for_id(Some(&id), start_err.to_string())
.await
{
return Err(Error::other(format!(
"failed to start rebalance after metadata initialized for {id}: {start_err}; rollback failed: {rollback_err}"
)));
}
return Err(Error::other(format!(
"failed to start rebalance after metadata initialized for {id}; local metadata was finalized as failed: {start_err}"
)));
}
Ok(id)
}
#[tracing::instrument(skip(self, fi))]
pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> {
self.update_pool_stats_batch(pool_index, bucket, &[fi]).await
@@ -389,12 +421,24 @@ impl ECStore {
false
}
pub async fn current_rebalance_id(&self) -> Option<String> {
let rebalance_meta = self.rebalance_meta.read().await;
rebalance_meta
.as_ref()
.and_then(|meta| (!meta.id.is_empty()).then(|| meta.id.clone()))
}
#[tracing::instrument(skip(self))]
pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> {
self.stop_rebalance_for_id(None).await
}
#[tracing::instrument(skip(self))]
pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> {
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
stop_rebalance_meta_snapshot(rebalance_meta.as_mut(), OffsetDateTime::now_utc())
};
stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id)
}?;
if let Some(meta_to_save) = meta_to_save {
let pool = clone_first_arc(self.pools.as_slice(), "stop_rebalance: no pools available")?;
@@ -407,4 +451,54 @@ impl ECStore {
Ok(())
}
async fn rollback_rebalance_start_without_worker_for_id(
self: &Arc<Self>,
expected_id: Option<&str>,
start_error: String,
) -> Result<()> {
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
rollback_rebalance_start_meta_snapshot_for_id(
rebalance_meta.as_mut(),
OffsetDateTime::now_utc(),
expected_id,
start_error,
)
};
if let Some(meta_to_save) = meta_to_save {
let pool = clone_first_arc(self.pools.as_slice(), "rollback_rebalance_start: no pools available")?;
resolve_rebalance_meta_save_result(
self.save_rebalance_meta_with_merge(pool, &meta_to_save, "rollback_rebalance_start")
.await,
"rollback_rebalance_start",
)?;
}
Ok(())
}
pub async fn record_rebalance_stop_propagation(self: &Arc<Self>, record: RebalanceStopPropagationRecord) -> Result<()> {
if !record.has_failures() {
return Ok(());
}
let encoded_error = encode_rebalance_stop_propagation_record(&record);
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
record_rebalance_stop_propagation_snapshot(rebalance_meta.as_mut(), encoded_error, OffsetDateTime::now_utc())
};
if let Some(meta_to_save) = meta_to_save {
let pool = clone_first_arc(self.pools.as_slice(), "record_rebalance_stop_propagation: no pools available")?;
resolve_rebalance_meta_save_result(
self.save_rebalance_meta_with_merge(pool, &meta_to_save, "record_rebalance_stop_propagation")
.await,
"record_rebalance_stop_propagation",
)?;
}
Ok(())
}
}
crates/ecstore/src/rebalance/entry.rs
+1 -1
View File
@@ -122,7 +122,7 @@ impl ECStore {
true,
&crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Rebal,
)
.await
.await?
{
expired += 1;
debug!(
crates/ecstore/src/rebalance/meta.rs
+192 -25
View File
@@ -1,8 +1,9 @@
use super::{
EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, Error, GetObjectReader, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
ObjectInfo, ObjectOptions, PutObjReader, REBAL_META_FMT, REBAL_META_NAME, REBAL_META_VER,
REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, RebalSaveOpt, RebalStatus, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats,
Result,
REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT, REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_STOP_PROPAGATION_ERROR_PREFIX,
RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats,
RebalanceStopPropagationRecord, Result,
};
use crate::config::com::{read_config_with_metadata, save_config_with_opts};
use crate::error::is_err_operation_canceled;
@@ -197,15 +198,15 @@ pub(super) fn is_rebalance_pool_started(pool_stat: &RebalanceStats) -> bool {
pool_stat.participating && pool_stat.info.status == RebalStatus::Started
}
pub(super) fn is_rebalance_in_progress(meta: &RebalanceMeta) -> bool {
if meta.stopped_at.is_some() {
return false;
}
meta.pool_stats.iter().any(is_rebalance_pool_started)
pub(super) fn is_rebalance_pool_active(pool_stat: &RebalanceStats) -> bool {
is_rebalance_pool_started(pool_stat) || pool_stat.info.stopping
}
pub(super) fn is_rebalance_conflicting_with_decommission(meta: &RebalanceMeta) -> bool {
pub(super) fn is_rebalance_in_progress(meta: &RebalanceMeta) -> bool {
meta.pool_stats.iter().any(is_rebalance_pool_active)
}
pub(crate) fn is_rebalance_conflicting_with_decommission(meta: &RebalanceMeta) -> bool {
is_rebalance_in_progress(meta)
}
@@ -224,6 +225,30 @@ pub(super) fn rebalance_meta_load_unknown_format_error(fmt: u16) -> Error {
pub(super) fn rebalance_meta_load_unknown_version_error(ver: u16) -> Error {
Error::other(format!("rebalance metadata load failed: unknown version {ver}"))
}
pub fn encode_rebalance_stop_propagation_record(record: &RebalanceStopPropagationRecord) -> String {
match serde_json::to_string(record) {
Ok(payload) => format!("{REBALANCE_STOP_PROPAGATION_ERROR_PREFIX}{payload}"),
Err(err) => {
let payload = serde_json::json!({
"encodeError": err.to_string(),
"stopFailures": [],
"terminalReloadFailures": [],
});
format!("{REBALANCE_STOP_PROPAGATION_ERROR_PREFIX}{payload}")
}
}
}
pub fn decode_rebalance_stop_propagation_record(message: &str) -> Option<RebalanceStopPropagationRecord> {
let payload = message.strip_prefix(REBALANCE_STOP_PROPAGATION_ERROR_PREFIX)?;
serde_json::from_str(payload).ok()
}
fn is_rebalance_stop_propagation_error(message: Option<&str>) -> bool {
message.is_some_and(|message| message.starts_with(REBALANCE_STOP_PROPAGATION_ERROR_PREFIX))
}
pub(super) fn rebalance_goal_reached(init_free_space: u64, init_capacity: u64, bytes: u64, percent_free_goal: f64) -> bool {
if init_capacity == 0 {
return false;
@@ -384,10 +409,17 @@ pub(super) fn record_rebalance_cleanup_warning_in_meta(
};
pool_stat.cleanup_warnings.count = pool_stat.cleanup_warnings.count.saturating_add(1);
pool_stat.cleanup_warnings.last_message = Some(message);
pool_stat.cleanup_warnings.last_message = Some(message.clone());
pool_stat.cleanup_warnings.last_bucket = Some(bucket.to_string());
pool_stat.cleanup_warnings.last_object = Some(object.to_string());
pool_stat.cleanup_warnings.last_at = Some(now);
pool_stat.cleanup_warnings.entries.push(RebalanceCleanupWarningEntry {
bucket: bucket.to_string(),
object: object.to_string(),
message,
timestamp: Some(now),
});
truncate_rebalance_cleanup_warning_entries(&mut pool_stat.cleanup_warnings.entries);
meta.last_refreshed_at = Some(now);
Ok(())
}
@@ -572,6 +604,17 @@ pub(super) fn validate_start_rebalance_state(decommission_running: bool, meta_lo
Ok(())
}
pub(super) fn validate_init_rebalance_state(decommission_running: bool, current_meta: Option<&RebalanceMeta>) -> Result<()> {
if !ensure_rebalance_not_decommissioning(decommission_running) {
return Err(Error::DecommissionAlreadyRunning);
}
if current_meta.is_some_and(is_rebalance_in_progress) {
return Err(Error::RebalanceAlreadyRunning);
}
Ok(())
}
pub(super) fn should_skip_start_rebalance(cancel_attached: bool, in_progress: bool) -> bool {
cancel_attached && in_progress
}
@@ -654,6 +697,31 @@ pub(super) fn merge_rebalance_cleanup_warnings(remote: &mut RebalanceCleanupWarn
remote.last_object = local.last_object.clone();
remote.last_at = local.last_at;
}
merge_rebalance_cleanup_warning_entries(&mut remote.entries, &local.entries);
let retained_entries = u64::try_from(remote.entries.len()).unwrap_or(u64::MAX);
remote.count = remote.count.max(retained_entries);
}
pub(super) fn merge_rebalance_cleanup_warning_entries(
remote: &mut Vec<RebalanceCleanupWarningEntry>,
local: &[RebalanceCleanupWarningEntry],
) {
for entry in local {
if !remote.iter().any(|existing| existing == entry) {
remote.push(entry.clone());
}
}
remote.sort_by_key(|entry| entry.timestamp);
truncate_rebalance_cleanup_warning_entries(remote);
}
fn truncate_rebalance_cleanup_warning_entries(entries: &mut Vec<RebalanceCleanupWarningEntry>) {
if entries.len() > REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT {
let remove_count = entries.len() - REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT;
entries.drain(0..remove_count);
}
}
pub(super) fn should_replace_rebalance_cleanup_warning(
@@ -667,6 +735,16 @@ pub(super) fn should_replace_rebalance_cleanup_warning(
}
}
pub(super) fn merge_rebalance_stop_propagation_error(remote: Option<String>, local: Option<String>) -> Option<String> {
if is_rebalance_stop_propagation_error(local.as_deref()) {
local
} else if is_rebalance_stop_propagation_error(remote.as_deref()) {
remote
} else {
None
}
}
pub(super) fn merge_rebalance_pool_stats(remote: &mut RebalanceStats, local: &RebalanceStats) {
remote.init_free_space = remote.init_free_space.max(local.init_free_space);
remote.init_capacity = remote.init_capacity.max(local.init_capacity);
@@ -702,19 +780,23 @@ pub(super) fn merge_rebalance_pool_stats(remote: &mut RebalanceStats, local: &Re
match local.info.status {
RebalStatus::Failed => {
remote.info.status = RebalStatus::Failed;
remote.info.stopping = false;
remote.info.end_time = local.info.end_time.or(remote.info.end_time);
remote.info.last_error = local.info.last_error.clone().or_else(|| remote.info.last_error.clone());
}
RebalStatus::Stopped => {
if remote.info.status != RebalStatus::Failed {
remote.info.status = RebalStatus::Stopped;
remote.info.stopping = false;
remote.info.end_time = local.info.end_time.or(remote.info.end_time);
remote.info.last_error = None;
remote.info.last_error =
merge_rebalance_stop_propagation_error(remote.info.last_error.clone(), local.info.last_error.clone());
}
}
RebalStatus::Completed => {
if !matches!(remote.info.status, RebalStatus::Failed | RebalStatus::Stopped) {
remote.info.status = RebalStatus::Completed;
remote.info.stopping = false;
remote.info.end_time = local.info.end_time.or(remote.info.end_time);
remote.info.last_error = None;
}
@@ -722,6 +804,7 @@ pub(super) fn merge_rebalance_pool_stats(remote: &mut RebalanceStats, local: &Re
RebalStatus::Started => {
if !is_rebalance_terminal_status(remote.info.status) {
remote.info.status = RebalStatus::Started;
remote.info.stopping |= local.info.stopping;
remote.info.last_error = local.info.last_error.clone();
}
}
@@ -736,7 +819,6 @@ pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &Rebalance
}
if !local.id.is_empty() && remote.id != local.id {
*remote = local.clone();
return;
}
@@ -761,35 +843,120 @@ pub(super) fn mark_started_rebalance_pools_stopped(meta: &mut RebalanceMeta, sto
for pool_stat in meta.pool_stats.iter_mut() {
if pool_stat.info.status == RebalStatus::Started {
pool_stat.info.status = RebalStatus::Stopped;
pool_stat.info.stopping = false;
pool_stat.info.end_time.get_or_insert(stop_time);
if !is_rebalance_stop_propagation_error(pool_stat.info.last_error.as_deref()) {
pool_stat.info.last_error = None;
}
}
}
}
pub(super) fn mark_started_rebalance_pools_stopping(meta: &mut RebalanceMeta) {
for pool_stat in meta.pool_stats.iter_mut() {
if pool_stat.info.status == RebalStatus::Started {
pool_stat.info.stopping = true;
pool_stat.info.last_error = None;
}
}
}
pub(super) fn apply_stopped_at(meta: &mut RebalanceMeta, now: OffsetDateTime) {
meta.stopped_at = Some(now);
mark_started_rebalance_pools_stopped(meta, now);
meta.stopped_at.get_or_insert(now);
mark_started_rebalance_pools_stopping(meta);
}
pub(super) fn clear_rebalance_cancel_token(meta: Option<&mut RebalanceMeta>) -> bool {
let Some(meta) = meta else {
return false;
};
if let Some(cancel_tx) = meta.cancel.take() {
cancel_tx.cancel();
return true;
}
false
}
pub(super) fn stop_rebalance_state(meta: &mut RebalanceMeta, now: OffsetDateTime) {
if let Some(cancel_tx) = meta.cancel.take() {
cancel_tx.cancel();
}
let stop_time = meta.stopped_at.unwrap_or(now);
clear_rebalance_cancel_token(Some(meta));
if meta.stopped_at.is_none() && is_rebalance_in_progress(meta) {
meta.stopped_at = Some(stop_time);
}
if meta.stopped_at.is_some() {
mark_started_rebalance_pools_stopped(meta, stop_time);
apply_stopped_at(meta, now);
} else if meta.stopped_at.is_some() {
mark_started_rebalance_pools_stopping(meta);
}
}
pub(super) fn stop_rebalance_meta_snapshot_for_id(
meta: Option<&mut RebalanceMeta>,
now: OffsetDateTime,
expected_id: Option<&str>,
) -> Result<Option<RebalanceMeta>> {
let Some(meta) = meta else {
return Ok(None);
};
if let Some(expected_id) = expected_id
&& !expected_id.is_empty()
&& meta.id != expected_id
{
return Err(Error::other(format!(
"rebalance stop id mismatch: expected {expected_id}, found {}",
meta.id
)));
}
stop_rebalance_state(meta, now);
meta.last_refreshed_at = Some(now);
Ok(Some(meta.clone()))
}
pub(super) fn rollback_rebalance_start_meta_snapshot_for_id(
meta: Option<&mut RebalanceMeta>,
now: OffsetDateTime,
expected_id: Option<&str>,
start_error: String,
) -> Option<RebalanceMeta> {
meta.and_then(|meta| {
if let Some(expected_id) = expected_id
&& !expected_id.is_empty()
&& meta.id != expected_id
{
return None;
}
clear_rebalance_cancel_token(Some(meta));
meta.stopped_at.get_or_insert(now);
meta.last_refreshed_at = Some(now);
for pool_stat in meta.pool_stats.iter_mut() {
if pool_stat.info.status == RebalStatus::Started {
pool_stat.info.status = RebalStatus::Failed;
pool_stat.info.stopping = false;
pool_stat.info.end_time.get_or_insert(now);
pool_stat.info.last_error = Some(start_error.clone());
}
}
Some(meta.clone())
})
}
pub(super) fn stop_rebalance_meta_snapshot(meta: Option<&mut RebalanceMeta>, now: OffsetDateTime) -> Option<RebalanceMeta> {
let meta = meta?;
stop_rebalance_state(meta, now);
meta.last_refreshed_at = Some(now);
Some(meta.clone())
}
pub(super) fn record_rebalance_stop_propagation_snapshot(
meta: Option<&mut RebalanceMeta>,
encoded_error: String,
now: OffsetDateTime,
) -> Option<RebalanceMeta> {
meta.map(|meta| {
stop_rebalance_state(meta, now);
for pool_stat in meta.pool_stats.iter_mut() {
if pool_stat.participating || pool_stat.info.stopping || pool_stat.info.status == RebalStatus::Stopped {
pool_stat.info.last_error = Some(encoded_error.clone());
}
}
meta.last_refreshed_at = Some(now);
meta.clone()
})
crates/ecstore/src/rebalance/rebalance_unit_tests.rs
+114 -19
View File
@@ -25,7 +25,7 @@ use super::meta::{
record_rebalance_cleanup_warning_in_meta, remove_rebalanced_buckets_from_queue, resolve_next_rebalance_bucket,
resolve_rebalance_participants, should_accept_rebalance_stats_update, should_ignore_rebalance_data_usage_cache,
should_pool_participate, should_preserve_rebalance_stopped_state, should_skip_start_rebalance, stop_rebalance_meta_snapshot,
stop_rebalance_state, take_bucket_from_rebalance_queue, validate_start_rebalance_state,
stop_rebalance_state, take_bucket_from_rebalance_queue, validate_init_rebalance_state, validate_start_rebalance_state,
};
use super::migration::{
MigrationBackend, MigrationVersionResult, migrate_entry_version, migrate_entry_version_with_retry_wait,
@@ -1199,6 +1199,7 @@ fn test_merge_rebalance_meta_preserves_updates_from_multiple_pools() {
last_bucket: Some("bucket-a".to_string()),
last_object: Some("local-object".to_string()),
last_at: Some(warning_at),
entries: Vec::new(),
},
..Default::default()
},
@@ -1242,7 +1243,7 @@ fn test_merge_rebalance_meta_does_not_overwrite_failed_with_started_stats() {
pool_stats: vec![RebalanceStats {
participating: true,
info: RebalanceInfo {
status: RebalStatus::Started,
status: RebalStatus::Stopped,
..Default::default()
},
num_versions: 8,
@@ -1307,7 +1308,7 @@ fn test_merge_rebalance_meta_preserves_failed_status_over_stopped() {
stopped_at: Some(stopped_at),
pool_stats: vec![RebalanceStats {
info: RebalanceInfo {
status: RebalStatus::Stopped,
status: RebalStatus::Started,
end_time: Some(stopped_at),
..Default::default()
},
@@ -1812,7 +1813,7 @@ fn test_should_accept_rebalance_stats_update_rejects_stopped_meta() {
stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
pool_stats: vec![RebalanceStats {
info: RebalanceInfo {
status: RebalStatus::Started,
status: RebalStatus::Stopped,
..Default::default()
},
..Default::default()
@@ -2173,6 +2174,93 @@ fn test_validate_start_rebalance_state_allows_loaded_meta() {
validate_start_rebalance_state(false, true).expect("loaded rebalance meta should allow start");
}
#[test]
fn test_validate_init_rebalance_state_rejects_running_decommission() {
let err = validate_init_rebalance_state(true, None).expect_err("running decommission should block rebalance init");
assert!(matches!(err, Error::DecommissionAlreadyRunning));
}
#[test]
fn test_validate_init_rebalance_state_rejects_active_rebalance() {
let now = OffsetDateTime::now_utc();
let meta = RebalanceMeta {
pool_stats: vec![RebalanceStats {
participating: true,
info: RebalanceInfo {
start_time: Some(now),
status: RebalStatus::Started,
..Default::default()
},
..Default::default()
}],
..Default::default()
};
let err = validate_init_rebalance_state(false, Some(&meta)).expect_err("active rebalance should block rebalance init");
assert!(matches!(err, Error::RebalanceAlreadyRunning));
}
#[test]
fn test_validate_init_rebalance_state_allows_terminal_or_missing_rebalance() {
let completed = RebalanceMeta {
pool_stats: vec![RebalanceStats {
participating: true,
info: RebalanceInfo {
status: RebalStatus::Completed,
..Default::default()
},
..Default::default()
}],
..Default::default()
};
validate_init_rebalance_state(false, None).expect("missing rebalance meta should allow init");
validate_init_rebalance_state(false, Some(&completed)).expect("terminal rebalance meta should allow init");
}
#[tokio::test]
async fn test_init_and_start_rebalance_rejects_second_start_after_gate() {
let now = OffsetDateTime::now_utc();
let active_meta = RebalanceMeta {
pool_stats: vec![RebalanceStats {
participating: true,
info: RebalanceInfo {
start_time: Some(now),
status: RebalStatus::Started,
..Default::default()
},
..Default::default()
}],
..Default::default()
};
let endpoint_pools: crate::endpoints::EndpointServerPools = Vec::new().into();
let store = Arc::new(crate::store::ECStore {
id: uuid::Uuid::new_v4(),
disk_map: std::collections::HashMap::new(),
pools: Vec::new(),
peer_sys: crate::rpc::S3PeerSys::new(&endpoint_pools),
pool_meta: tokio::sync::RwLock::new(crate::pools::PoolMeta::default()),
rebalance_meta: tokio::sync::RwLock::new(Some(active_meta)),
decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: tokio::sync::Mutex::new(()),
local_disk_map: crate::global::GLOBAL_LOCAL_DISK_MAP.clone(),
local_disk_id_map: crate::global::GLOBAL_LOCAL_DISK_ID_MAP.clone(),
local_disk_set_drives: crate::global::GLOBAL_LOCAL_DISK_SET_DRIVES.clone(),
tier_config_mgr: crate::tier::tier::TierConfigMgr::new(),
event_notifier: crate::event_notification::EventNotifier::new(),
bucket_monitor: std::sync::OnceLock::new(),
});
let err = store
.init_and_start_rebalance(vec!["bucket".to_string()])
.await
.expect_err("second rebalance start should be rejected before metadata init");
assert!(matches!(err, Error::RebalanceAlreadyRunning));
}
#[test]
fn test_percent_free_ratio_zero_capacity_is_zero() {
assert_eq!(percent_free_ratio(100, 0), 0.0);
@@ -2412,7 +2500,7 @@ fn test_resolve_rebalance_participants_respects_runtime_pool_count() {
RebalanceStats {
participating: false,
info: RebalanceInfo {
status: RebalStatus::Started,
status: RebalStatus::Stopped,
start_time: Some(now),
..Default::default()
},
@@ -2421,7 +2509,7 @@ fn test_resolve_rebalance_participants_respects_runtime_pool_count() {
RebalanceStats {
participating: true,
info: RebalanceInfo {
status: RebalStatus::Started,
status: RebalStatus::Stopped,
start_time: Some(now),
..Default::default()
},
@@ -2542,7 +2630,7 @@ fn test_is_rebalance_conflicting_with_decommission_false_when_stopped() {
participating: true,
info: RebalanceInfo {
start_time: Some(now),
status: RebalStatus::Started,
status: RebalStatus::Stopped,
..Default::default()
},
..Default::default()
@@ -2562,7 +2650,7 @@ fn test_is_rebalance_in_progress_stopped_takes_precedence() {
participating: true,
info: RebalanceInfo {
start_time: Some(now),
status: RebalStatus::Started,
status: RebalStatus::Stopped,
..Default::default()
},
..Default::default()
@@ -2871,6 +2959,7 @@ fn test_complete_rebalance_pools_with_empty_queue_preserves_cleanup_warnings() {
last_bucket: Some("bucket-a".to_string()),
last_object: Some("obj.txt".to_string()),
last_at: Some(warning_at),
entries: Vec::new(),
},
..Default::default()
}],
@@ -2916,8 +3005,9 @@ fn test_apply_stopped_at_transitions_started_pools_only() {
apply_stopped_at(&mut meta, now);
assert_eq!(meta.stopped_at, Some(now));
assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
assert_eq!(meta.pool_stats[0].info.end_time, Some(now));
assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started);
assert!(meta.pool_stats[0].info.stopping);
assert_eq!(meta.pool_stats[0].info.end_time, None);
assert_eq!(meta.pool_stats[0].info.last_error, None);
assert_eq!(meta.pool_stats[1].info.status, RebalStatus::Failed);
@@ -2948,8 +3038,9 @@ fn test_stop_rebalance_state_cancels_token_and_marks_stopped_when_in_progress()
assert!(cancel_clone.is_cancelled());
assert!(meta.cancel.is_none());
assert_eq!(meta.stopped_at, Some(now));
assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
assert_eq!(meta.pool_stats[0].info.end_time, Some(now));
assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started);
assert!(meta.pool_stats[0].info.stopping);
assert_eq!(meta.pool_stats[0].info.end_time, None);
}
#[test]
@@ -3005,8 +3096,9 @@ fn test_stop_rebalance_state_normalizes_started_pool_when_stopped_at_already_set
assert!(cancel_clone.is_cancelled());
assert!(meta.cancel.is_none());
assert_eq!(meta.stopped_at, Some(stopped_at));
assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
assert_eq!(meta.pool_stats[0].info.end_time, Some(stopped_at));
assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started);
assert!(meta.pool_stats[0].info.stopping);
assert_eq!(meta.pool_stats[0].info.end_time, None);
assert_eq!(meta.pool_stats[0].info.last_error, None);
}
@@ -3040,13 +3132,15 @@ fn test_stop_rebalance_meta_snapshot_stops_meta_and_returns_snapshot() {
assert!(meta.cancel.is_none());
assert_eq!(meta.stopped_at, Some(now));
assert_eq!(meta.last_refreshed_at, Some(now));
assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started);
assert!(meta.pool_stats[0].info.stopping);
assert!(snapshot.cancel.is_none());
assert_eq!(snapshot.stopped_at, Some(now));
assert_eq!(snapshot.last_refreshed_at, Some(now));
assert_eq!(snapshot.pool_stats[0].info.status, RebalStatus::Stopped);
assert_eq!(snapshot.pool_stats[0].info.end_time, Some(now));
assert_eq!(snapshot.pool_stats[0].info.status, RebalStatus::Started);
assert!(snapshot.pool_stats[0].info.stopping);
assert_eq!(snapshot.pool_stats[0].info.end_time, None);
}
#[test]
@@ -3108,8 +3202,9 @@ fn test_apply_rebalance_save_option_stopped_at_updates_refresh_and_statuses() {
assert_eq!(meta.stopped_at, Some(now));
assert_eq!(meta.last_refreshed_at, Some(now));
assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
assert_eq!(meta.pool_stats[0].info.end_time, Some(now));
assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started);
assert!(meta.pool_stats[0].info.stopping);
assert_eq!(meta.pool_stats[0].info.end_time, None);
assert!(meta.pool_stats[0].info.last_error.is_none());
assert_eq!(meta.pool_stats[1].info.status, RebalStatus::Failed);
assert_eq!(meta.pool_stats[1].info.last_error.as_deref(), Some("previous failure"));
crates/ecstore/src/rebalance/runtime.rs
+1
View File
@@ -225,6 +225,7 @@ impl ECStore {
"Preserved stopped rebalance status"
);
} else {
pool_stat.info.stopping = false;
apply_rebalance_terminal_event(
&mut pool_stat.info.status,
&mut pool_stat.info.end_time,
crates/ecstore/src/rebalance/types.rs
+40
View File
@@ -4,6 +4,7 @@ use time::OffsetDateTime;
use tokio_util::sync::CancellationToken;
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RebalanceStats {
#[serde(rename = "ifs")]
pub init_free_space: u64, // Pool free space at the start of rebalance
@@ -70,6 +71,7 @@ pub enum RebalSaveOpt {
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RebalanceInfo {
#[serde(rename = "startTs")]
pub start_time: Option<OffsetDateTime>, // Time at which rebalance-start was issued
@@ -79,9 +81,25 @@ pub struct RebalanceInfo {
pub last_error: Option<String>, // Last rebalance error message
#[serde(rename = "status")]
pub status: RebalStatus, // Current state of rebalance operation
#[serde(rename = "stopping", default)]
pub stopping: bool, // True after stop is requested and before worker terminal acknowledgement
}
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct RebalanceCleanupWarningEntry {
#[serde(rename = "bucket", default)]
pub bucket: String,
#[serde(rename = "object", default)]
pub object: String,
#[serde(rename = "message", default)]
pub message: String,
#[serde(rename = "timestamp", default)]
pub timestamp: Option<OffsetDateTime>,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct RebalanceCleanupWarnings {
#[serde(rename = "count", default)]
pub count: u64,
@@ -93,6 +111,27 @@ pub struct RebalanceCleanupWarnings {
pub last_object: Option<String>,
#[serde(rename = "lastAt", default)]
pub last_at: Option<OffsetDateTime>,
#[serde(rename = "entries", default)]
pub entries: Vec<RebalanceCleanupWarningEntry>,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct RebalanceStopPropagationRecord {
#[serde(rename = "stopAttemptAt", default)]
pub stop_attempt_at: Option<OffsetDateTime>,
#[serde(rename = "stopFailures", default)]
pub stop_failures: Vec<String>,
#[serde(rename = "terminalReloadAttemptAt", default)]
pub terminal_reload_attempt_at: Option<OffsetDateTime>,
#[serde(rename = "terminalReloadFailures", default)]
pub terminal_reload_failures: Vec<String>,
}
impl RebalanceStopPropagationRecord {
pub fn has_failures(&self) -> bool {
!self.stop_failures.is_empty() || !self.terminal_reload_failures.is_empty()
}
}
#[allow(dead_code)]
@@ -103,6 +142,7 @@ pub struct DiskStat {
}
#[derive(Debug, Default, Serialize, Deserialize, Clone)]
#[serde(deny_unknown_fields)]
pub struct RebalanceMeta {
#[serde(skip)]
pub cancel: Option<CancellationToken>, // To be invoked on rebalance-stop