mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-03 18:55:39 +00:00
refactor: consolidate ecstore owner module layout (#3934)
* refactor: shrink ecstore root owner facades * refactor: remove ecstore core store root shims * refactor: move ecstore erasure owner modules * refactor: remove ecstore root rpc facade * refactor: move ecstore services domain modules
This commit is contained in:
Zhengchao An
@@ -0,0 +1,780 @@
|
||||
use super::meta::{
|
||||
RebalanceMetaMergeOutcome, clone_first_arc, clone_rebalance_pool_stats, defer_bucket_in_rebalance_queue,
|
||||
ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error, is_rebalance_actively_running,
|
||||
is_rebalance_conflicting_with_decommission, mark_rebalance_bucket_done, merge_rebalance_meta, percent_free_ratio,
|
||||
rebalance_metadata_not_initialized_error, record_rebalance_cleanup_warning_in_meta,
|
||||
record_rebalance_stop_propagation_snapshot, resolve_next_rebalance_bucket, rollback_rebalance_start_meta_snapshot_for_id,
|
||||
should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot_for_id,
|
||||
validate_init_rebalance_state,
|
||||
};
|
||||
use super::worker::{
|
||||
rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result, resolve_rebalance_meta_load_result,
|
||||
resolve_rebalance_meta_save_result,
|
||||
};
|
||||
use super::{
|
||||
DiskStat, EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
|
||||
RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
|
||||
encode_rebalance_stop_propagation_record,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::storage_api_contracts::{
|
||||
admin::StorageAdminApi, namespace::NamespaceLocking as StorageNamespaceLocking, object::EcstoreObjectIO,
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{debug, info};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(super) fn validate_rebalance_disk_stats_coverage(disk_stats: &[DiskStat]) -> Result<()> {
|
||||
for (idx, disk_stat) in disk_stats.iter().enumerate() {
|
||||
if disk_stat.total_space == 0 {
|
||||
return Err(Error::other(format!(
|
||||
"rebalance storage info is incomplete: pool {idx} has no reported capacity"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn pool_rebalance_status_from_meta(meta: Option<&RebalanceMeta>, pool_index: usize) -> (RebalStatus, bool) {
|
||||
meta.and_then(|meta| meta.pool_stats.get(pool_index))
|
||||
.filter(|pool_stat| pool_stat.participating)
|
||||
.map(|pool_stat| (pool_stat.info.status, pool_stat.info.stopping))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) {
|
||||
if persisted.id.is_empty() && persisted.pool_stats.is_empty() {
|
||||
clear_rebalance_status_refresh(current);
|
||||
return;
|
||||
}
|
||||
|
||||
match current.as_mut() {
|
||||
Some(current_meta) => {
|
||||
if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict
|
||||
&& !is_rebalance_actively_running(current_meta)
|
||||
{
|
||||
*current = Some(persisted);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
*current = Some(persisted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) {
|
||||
if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) {
|
||||
*current = None;
|
||||
}
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
pub(super) async fn save_rebalance_meta_with_merge<S>(
|
||||
&self,
|
||||
pool: Arc<S>,
|
||||
local_snapshot: &RebalanceMeta,
|
||||
stage: &str,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
|
||||
let _guard = ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(rebalance_meta_lock_error)?;
|
||||
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut merged = RebalanceMeta::new();
|
||||
match merged.load_with_opts(pool.clone(), opts.clone()).await {
|
||||
Ok(()) => {
|
||||
if merge_rebalance_meta(&mut merged, local_snapshot) == RebalanceMetaMergeOutcome::RejectedActiveConflict {
|
||||
return Err(Error::RebalanceAlreadyRunning);
|
||||
}
|
||||
}
|
||||
Err(Error::ConfigNotFound) => {
|
||||
merged = local_snapshot.clone();
|
||||
}
|
||||
Err(err) => return Err(Error::other(format!("rebalance meta load before save failed during {stage}: {err}"))),
|
||||
}
|
||||
|
||||
merged.save_with_opts(pool, opts).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub async fn load_rebalance_meta(&self) -> Result<()> {
|
||||
let mut meta = RebalanceMeta::new();
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "metadata_loading",
|
||||
"Loading rebalance metadata"
|
||||
);
|
||||
let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?;
|
||||
if resolve_rebalance_meta_load_result(meta.load(pool).await)? {
|
||||
{
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
|
||||
*rebalance_meta = Some(meta);
|
||||
|
||||
drop(rebalance_meta);
|
||||
}
|
||||
|
||||
resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?;
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "metadata_loaded",
|
||||
"Loaded rebalance metadata"
|
||||
);
|
||||
} else {
|
||||
{
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
*rebalance_meta = None;
|
||||
}
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "metadata_missing",
|
||||
reason = "rebalance_not_started",
|
||||
"Rebalance metadata not found"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub async fn refresh_rebalance_status_meta(&self) -> Result<()> {
|
||||
let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?;
|
||||
let mut persisted = RebalanceMeta::new();
|
||||
match persisted.load(pool).await {
|
||||
Ok(()) => {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
merge_rebalance_status_refresh(&mut rebalance_meta, persisted);
|
||||
}
|
||||
Err(Error::ConfigNotFound) => {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
clear_rebalance_status_refresh(&mut rebalance_meta);
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}")));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub async fn update_rebalance_stats(&self) -> Result<()> {
|
||||
let mut ok = false;
|
||||
|
||||
let pool_stats = {
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
clone_rebalance_pool_stats(rebalance_meta.as_ref())?
|
||||
};
|
||||
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_count = pool_stats.len(),
|
||||
"Refreshing rebalance stats snapshot"
|
||||
);
|
||||
|
||||
for i in 0..self.pools.len() {
|
||||
if pool_stats.get(i).is_none() {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index = i,
|
||||
state = "pool_stat_missing",
|
||||
"Adding missing rebalance pool stats entry"
|
||||
);
|
||||
if let Some(meta) = rebalance_meta.as_mut() {
|
||||
meta.pool_stats.push(RebalanceStats::default());
|
||||
}
|
||||
ok = true;
|
||||
drop(rebalance_meta);
|
||||
}
|
||||
}
|
||||
|
||||
if ok {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "metadata_saving",
|
||||
"Saving rebalance metadata after stats refresh"
|
||||
);
|
||||
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
if let Some(meta) = rebalance_meta.as_ref() {
|
||||
let pool = clone_first_arc(&self.pools, "update_rebalance_stats: no pools available")?;
|
||||
resolve_rebalance_meta_save_result(
|
||||
self.save_rebalance_meta_with_merge(pool, meta, "update_rebalance_stats")
|
||||
.await,
|
||||
"update_rebalance_stats",
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub async fn init_rebalance_meta(&self, bucktes: Vec<String>) -> Result<String> {
|
||||
info!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "initializing",
|
||||
bucket_count = bucktes.len(),
|
||||
"Initializing rebalance metadata"
|
||||
);
|
||||
let si = StorageAdminApi::storage_info(self).await;
|
||||
|
||||
let mut disk_stats = vec![DiskStat::default(); self.pools.len()];
|
||||
|
||||
let mut total_cap = 0;
|
||||
let mut total_free = 0;
|
||||
for disk in si.disks.iter() {
|
||||
if disk.pool_index < 0 || disk_stats.len() <= disk.pool_index as usize {
|
||||
continue;
|
||||
}
|
||||
|
||||
total_cap += disk.total_space;
|
||||
total_free += disk.available_space;
|
||||
|
||||
disk_stats[disk.pool_index as usize].total_space += disk.total_space;
|
||||
disk_stats[disk.pool_index as usize].available_space += disk.available_space;
|
||||
}
|
||||
|
||||
let percent_free_goal = percent_free_ratio(total_free, total_cap);
|
||||
validate_rebalance_disk_stats_coverage(&disk_stats)?;
|
||||
|
||||
let mut pool_stats = Vec::with_capacity(self.pools.len());
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
|
||||
for disk_stat in disk_stats.iter() {
|
||||
let mut pool_stat = RebalanceStats {
|
||||
init_free_space: disk_stat.available_space,
|
||||
init_capacity: disk_stat.total_space,
|
||||
buckets: bucktes.clone(),
|
||||
rebalanced_buckets: Vec::with_capacity(bucktes.len()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if should_pool_participate(disk_stat.available_space, disk_stat.total_space, percent_free_goal) {
|
||||
pool_stat.participating = true;
|
||||
pool_stat.info = RebalanceInfo {
|
||||
start_time: Some(now),
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
|
||||
pool_stats.push(pool_stat);
|
||||
}
|
||||
|
||||
let meta = RebalanceMeta {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
percent_free_goal,
|
||||
pool_stats,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let pool = clone_first_arc(&self.pools, "init_rebalance_meta: no pools available")?;
|
||||
resolve_rebalance_meta_save_result(
|
||||
self.save_rebalance_meta_with_merge(pool, &meta, "init_rebalance_meta").await,
|
||||
"init_rebalance_meta",
|
||||
)?;
|
||||
|
||||
info!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "metadata_initialized",
|
||||
bucket_count = bucktes.len(),
|
||||
"Rebalance metadata initialized"
|
||||
);
|
||||
|
||||
let id = meta.id.clone();
|
||||
|
||||
{
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
*rebalance_meta = Some(meta);
|
||||
drop(rebalance_meta);
|
||||
}
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, bucktes))]
|
||||
pub async fn init_and_start_rebalance(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
|
||||
let _start_guard = self.start_gate.lock().await;
|
||||
|
||||
let decommission_running = self.is_decommission_running().await;
|
||||
{
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
|
||||
}
|
||||
|
||||
let id = self.init_rebalance_meta(bucktes).await?;
|
||||
if let Err(start_err) = self.start_rebalance().await {
|
||||
if let Err(rollback_err) = self
|
||||
.rollback_rebalance_start_without_worker_for_id(Some(&id), start_err.to_string())
|
||||
.await
|
||||
{
|
||||
return Err(Error::other(format!(
|
||||
"failed to start rebalance after metadata initialized for {id}: {start_err}; rollback failed: {rollback_err}"
|
||||
)));
|
||||
}
|
||||
|
||||
return Err(Error::other(format!(
|
||||
"failed to start rebalance after metadata initialized for {id}; local metadata was finalized as failed: {start_err}"
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, fi))]
|
||||
pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> {
|
||||
self.update_pool_stats_batch(pool_index, bucket, &[fi]).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, versions))]
|
||||
pub async fn update_pool_stats_batch(&self, pool_index: usize, bucket: String, versions: &[&FileInfo]) -> Result<()> {
|
||||
if versions.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
if let Some(meta) = rebalance_meta.as_mut() {
|
||||
if !should_accept_rebalance_stats_update(meta, pool_index) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
|
||||
pool_stat.update_batch(bucket, versions);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub async fn next_rebal_bucket(&self, pool_index: usize) -> Result<Option<String>> {
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
has_meta = rebalance_meta.is_some(),
|
||||
state = "next_bucket_lookup",
|
||||
"Rebalance next bucket lookup"
|
||||
);
|
||||
resolve_next_rebalance_bucket(rebalance_meta.as_ref(), pool_index)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub async fn bucket_rebalance_done(&self, pool_index: usize, bucket: String) -> Result<()> {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
mark_rebalance_bucket_done(rebalance_meta.as_mut(), pool_index, &bucket)
|
||||
}
|
||||
|
||||
pub(super) async fn record_rebalance_cleanup_warning(
|
||||
&self,
|
||||
pool_index: usize,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
message: String,
|
||||
) -> Result<()> {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
record_rebalance_cleanup_warning_in_meta(
|
||||
rebalance_meta.as_mut(),
|
||||
pool_index,
|
||||
bucket,
|
||||
object,
|
||||
message,
|
||||
OffsetDateTime::now_utc(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) async fn defer_rebalance_bucket(&self, pool_index: usize, bucket: String, last_error: String) -> Result<()> {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
let Some(meta) = rebalance_meta.as_mut() else {
|
||||
return Err(rebalance_metadata_not_initialized_error("defer rebalance bucket"));
|
||||
};
|
||||
let pool_count = meta.pool_stats.len();
|
||||
ensure_valid_rebalance_pool_index(pool_count, pool_index)?;
|
||||
let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) else {
|
||||
return Err(invalid_rebalance_pool_index_error(pool_index, pool_count));
|
||||
};
|
||||
|
||||
defer_bucket_in_rebalance_queue(pool_stat, &bucket)?;
|
||||
pool_stat.info.last_error = Some(last_error);
|
||||
meta.last_refreshed_at = Some(OffsetDateTime::now_utc());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn is_rebalance_started(&self) -> bool {
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
if let Some(meta) = rebalance_meta.as_ref() {
|
||||
meta.pool_stats.iter().enumerate().for_each(|(i, v)| {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index = i,
|
||||
participating = v.participating,
|
||||
status = ?v.info.status,
|
||||
state = "status_inspected",
|
||||
"Rebalance status inspected"
|
||||
);
|
||||
});
|
||||
|
||||
let started = is_rebalance_conflicting_with_decommission(meta);
|
||||
if started {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "running",
|
||||
"Rebalance is running"
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "not_running",
|
||||
"Rebalance is not running"
|
||||
);
|
||||
false
|
||||
}
|
||||
|
||||
pub async fn is_rebalance_conflicting_with_decommission(&self) -> bool {
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
rebalance_meta
|
||||
.as_ref()
|
||||
.is_some_and(is_rebalance_conflicting_with_decommission)
|
||||
}
|
||||
|
||||
pub async fn is_pool_rebalancing(&self, pool_index: usize) -> bool {
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
if let Some(ref meta) = *rebalance_meta {
|
||||
if meta.stopped_at.is_some() {
|
||||
return false;
|
||||
}
|
||||
|
||||
if let Some(pool_stat) = meta.pool_stats.get(pool_index) {
|
||||
return pool_stat.participating && pool_stat.info.status == RebalStatus::Started;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
pub async fn pool_rebalance_status(&self, pool_index: usize) -> (RebalStatus, bool) {
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
pool_rebalance_status_from_meta(rebalance_meta.as_ref(), pool_index)
|
||||
}
|
||||
|
||||
pub async fn current_rebalance_id(&self) -> Option<String> {
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
rebalance_meta
|
||||
.as_ref()
|
||||
.and_then(|meta| (!meta.id.is_empty()).then(|| meta.id.clone()))
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> {
|
||||
self.stop_rebalance_for_id(None).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> {
|
||||
let meta_to_save = {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id)
|
||||
}?;
|
||||
|
||||
if let Some(meta_to_save) = meta_to_save {
|
||||
let pool = clone_first_arc(self.pools.as_slice(), "stop_rebalance: no pools available")?;
|
||||
resolve_rebalance_meta_save_result(
|
||||
self.save_rebalance_meta_with_merge(pool, &meta_to_save, "stop_rebalance")
|
||||
.await,
|
||||
"stop_rebalance",
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn rollback_rebalance_start_without_worker_for_id(
|
||||
self: &Arc<Self>,
|
||||
expected_id: Option<&str>,
|
||||
start_error: String,
|
||||
) -> Result<()> {
|
||||
let meta_to_save = {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
rollback_rebalance_start_meta_snapshot_for_id(
|
||||
rebalance_meta.as_mut(),
|
||||
OffsetDateTime::now_utc(),
|
||||
expected_id,
|
||||
start_error,
|
||||
)
|
||||
};
|
||||
|
||||
if let Some(meta_to_save) = meta_to_save {
|
||||
let pool = clone_first_arc(self.pools.as_slice(), "rollback_rebalance_start: no pools available")?;
|
||||
resolve_rebalance_meta_save_result(
|
||||
self.save_rebalance_meta_with_merge(pool, &meta_to_save, "rollback_rebalance_start")
|
||||
.await,
|
||||
"rollback_rebalance_start",
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn record_rebalance_stop_propagation(self: &Arc<Self>, record: RebalanceStopPropagationRecord) -> Result<()> {
|
||||
if !record.has_failures() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let encoded_error = encode_rebalance_stop_propagation_record(&record);
|
||||
let meta_to_save = {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
record_rebalance_stop_propagation_snapshot(rebalance_meta.as_mut(), encoded_error, OffsetDateTime::now_utc())
|
||||
};
|
||||
|
||||
if let Some(meta_to_save) = meta_to_save {
|
||||
let pool = clone_first_arc(self.pools.as_slice(), "record_rebalance_stop_propagation: no pools available")?;
|
||||
resolve_rebalance_meta_save_result(
|
||||
self.save_rebalance_meta_with_merge(pool, &meta_to_save, "record_rebalance_stop_propagation")
|
||||
.await,
|
||||
"record_rebalance_stop_propagation",
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn pool_rebalance_status_ignores_non_participating_pool_state() {
|
||||
let meta = RebalanceMeta {
|
||||
pool_stats: vec![
|
||||
RebalanceStats {
|
||||
participating: false,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
stopping: true,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 0), (RebalStatus::None, false));
|
||||
assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 1), (RebalStatus::Started, false));
|
||||
assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 2), (RebalStatus::None, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_status_refresh_applies_persisted_terminal_state() {
|
||||
let rebalance_id = "rebalance-id".to_string();
|
||||
let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
|
||||
let mut current = Some(RebalanceMeta {
|
||||
id: rebalance_id.clone(),
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
start_time: Some(now),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
});
|
||||
let persisted = RebalanceMeta {
|
||||
id: rebalance_id,
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Completed,
|
||||
start_time: Some(now),
|
||||
end_time: Some(now),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
merge_rebalance_status_refresh(&mut current, persisted);
|
||||
|
||||
let refreshed = current.as_ref().expect("refresh should keep rebalance metadata");
|
||||
assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
|
||||
assert_eq!(refreshed.pool_stats[0].info.end_time, Some(now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_status_refresh_preserves_runtime_cancel_token() {
|
||||
let rebalance_id = "rebalance-id".to_string();
|
||||
let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
|
||||
let mut current = Some(RebalanceMeta {
|
||||
id: rebalance_id.clone(),
|
||||
cancel: Some(tokio_util::sync::CancellationToken::new()),
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
start_time: Some(now),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
});
|
||||
let persisted = RebalanceMeta {
|
||||
id: rebalance_id,
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
start_time: Some(now),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
merge_rebalance_status_refresh(&mut current, persisted);
|
||||
|
||||
assert!(
|
||||
current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(),
|
||||
"status refresh must not drop the runtime cancellation token"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_status_refresh_preserves_local_active_different_id_conflict() {
|
||||
let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
|
||||
let mut current = Some(RebalanceMeta {
|
||||
id: "old-active-id".to_string(),
|
||||
cancel: Some(tokio_util::sync::CancellationToken::new()),
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
start_time: Some(now),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
});
|
||||
let persisted = RebalanceMeta {
|
||||
id: "new-terminal-id".to_string(),
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Completed,
|
||||
start_time: Some(now),
|
||||
end_time: Some(now),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
merge_rebalance_status_refresh(&mut current, persisted);
|
||||
|
||||
let refreshed = current.as_ref().expect("local active metadata should remain visible");
|
||||
assert_eq!(refreshed.id, "old-active-id");
|
||||
assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Started);
|
||||
assert!(
|
||||
refreshed.cancel.is_some(),
|
||||
"status refresh must not drop a live runtime cancellation token"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_status_refresh_replaces_stale_memory_without_runtime_token() {
|
||||
let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
|
||||
let mut current = Some(RebalanceMeta {
|
||||
id: "old-stale-id".to_string(),
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
start_time: Some(now),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
});
|
||||
let persisted = RebalanceMeta {
|
||||
id: "new-terminal-id".to_string(),
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Completed,
|
||||
start_time: Some(now),
|
||||
end_time: Some(now),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
merge_rebalance_status_refresh(&mut current, persisted);
|
||||
|
||||
let refreshed = current.as_ref().expect("persisted metadata should replace stale memory");
|
||||
assert_eq!(refreshed.id, "new-terminal-id");
|
||||
assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
|
||||
assert!(refreshed.cancel.is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,533 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::meta::{
|
||||
clone_arc_by_index, ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error,
|
||||
rebalance_metadata_not_initialized_error, should_ignore_rebalance_data_usage_cache,
|
||||
};
|
||||
use super::migration::migrate_entry_version;
|
||||
use super::worker::{
|
||||
RebalanceEntryTask, load_rebalance_bucket_configs, rebalance_max_attempts, resolve_rebalance_bucket_error,
|
||||
resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
|
||||
resolve_rebalance_migrate_result_error, resolve_rebalance_stats_update_result, resolve_rebalance_worker_result,
|
||||
run_rebalance_listing_with_retry, should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete,
|
||||
should_defer_rebalance_entry_failure, should_skip_rebalance_delete_marker, wait_rebalance_entry_tasks,
|
||||
with_rebalance_entry_context,
|
||||
};
|
||||
use super::{
|
||||
EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_ENTRY, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
|
||||
REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome,
|
||||
};
|
||||
use crate::core::pools::ListCallback;
|
||||
use crate::data_movement;
|
||||
use crate::data_movement::backpressure::{self, DataMovementOperation};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::{GetObjectReader, ObjectOptions};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::object::ObjectOperations as _;
|
||||
use crate::store::ECStore;
|
||||
use rustfs_filemeta::MetaCacheEntry;
|
||||
use rustfs_utils::path::encode_dir_object;
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
impl ECStore {
|
||||
#[allow(unused_assignments)]
|
||||
#[tracing::instrument(skip(self, set))]
|
||||
async fn rebalance_entry(
|
||||
self: Arc<Self>,
|
||||
bucket: String,
|
||||
pool_index: usize,
|
||||
entry: MetaCacheEntry,
|
||||
set: Arc<SetDisks>,
|
||||
bucket_configs: Arc<RebalanceBucketConfigs>,
|
||||
// wk: Arc<Workers>,
|
||||
) -> Result<RebalanceEntryOutcome> {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
pool_index,
|
||||
state = "started",
|
||||
"Starting rebalance entry"
|
||||
);
|
||||
|
||||
// defer!(|| async {
|
||||
// warn!("rebalance_entry: defer give worker start");
|
||||
// wk.give().await;
|
||||
// warn!("rebalance_entry: defer give worker done");
|
||||
// });
|
||||
|
||||
if entry.is_dir() {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
pool_index,
|
||||
state = "skipped",
|
||||
reason = "directory_entry",
|
||||
"Skipped rebalance entry"
|
||||
);
|
||||
return Ok(RebalanceEntryOutcome::Completed);
|
||||
}
|
||||
|
||||
if self.check_if_rebalance_done(pool_index).await {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
state = "skipped",
|
||||
reason = "pool_completed",
|
||||
"Skipped rebalance entry"
|
||||
);
|
||||
return Ok(RebalanceEntryOutcome::Completed);
|
||||
}
|
||||
|
||||
let mut fivs =
|
||||
resolve_rebalance_file_info_versions_result(entry.file_info_versions(&bucket), bucket.as_str(), entry.name.as_str())?;
|
||||
|
||||
fivs.versions
|
||||
.sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
|
||||
|
||||
let mut rebalanced: usize = 0;
|
||||
let mut expired: usize = 0;
|
||||
let mut stats_updates = Vec::with_capacity(fivs.versions.len());
|
||||
for version in fivs.versions.iter() {
|
||||
if crate::core::pools::should_skip_lifecycle_for_data_movement(
|
||||
self.clone(),
|
||||
&bucket,
|
||||
version,
|
||||
bucket_configs.lifecycle_config.as_ref(),
|
||||
bucket_configs.lock_retention.clone(),
|
||||
bucket_configs.replication_config.clone(),
|
||||
true,
|
||||
&crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Rebal,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
expired += 1;
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
object = %version.name,
|
||||
state = "skipped",
|
||||
reason = "expired_by_lifecycle",
|
||||
"Skipped rebalance version"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
let remaining_versions = fivs.versions.len() - expired;
|
||||
if should_skip_rebalance_delete_marker(version, remaining_versions, bucket_configs.replication_config.is_some()) {
|
||||
rebalanced += 1;
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
object = %version.name,
|
||||
state = "skipped",
|
||||
reason = "last_delete_marker_without_replication",
|
||||
"Skipped rebalance version"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
let version_id = version.version_id.map(|v| v.to_string());
|
||||
let mut transfer = |src_pool_idx: usize, bucket: String, rd: GetObjectReader| {
|
||||
let store = self.clone();
|
||||
async move { store.rebalance_object(src_pool_idx, bucket, rd).await }
|
||||
};
|
||||
let result = migrate_entry_version(
|
||||
set.as_ref(),
|
||||
bucket.clone(),
|
||||
pool_index,
|
||||
version,
|
||||
version_id.clone(),
|
||||
rebalance_max_attempts(),
|
||||
should_ignore_rebalance_data_usage_cache(bucket.as_str()),
|
||||
&mut transfer,
|
||||
)
|
||||
.await;
|
||||
|
||||
if result.ignored {
|
||||
if should_count_rebalance_version_complete(&result) {
|
||||
rebalanced += 1;
|
||||
}
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
object = %version.name,
|
||||
state = "skipped",
|
||||
reason = "already_deleted",
|
||||
"Skipped rebalance version"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if result.failed {
|
||||
let err = resolve_rebalance_migrate_result_error(
|
||||
result.error,
|
||||
pool_index,
|
||||
bucket.as_str(),
|
||||
version.name.as_str(),
|
||||
version_id.as_deref(),
|
||||
);
|
||||
error!(
|
||||
"rebalance_entry {} Error rebalancing entry {}/{:?}: {:?}",
|
||||
&bucket, &version.name, &version.version_id, err
|
||||
);
|
||||
if should_defer_rebalance_entry_failure(&err) {
|
||||
let deferred_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} {err}");
|
||||
warn!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
object = %version.name,
|
||||
state = "deferred",
|
||||
error = %err,
|
||||
"Deferred rebalance entry after transient migration failure"
|
||||
);
|
||||
if let Err(stats_err) = self.update_rebalance_last_error(pool_index, deferred_error.clone()).await {
|
||||
error!(
|
||||
"rebalance_entry {} failed to record deferred transient failure for {}: {}",
|
||||
&bucket, &entry.name, stats_err
|
||||
);
|
||||
}
|
||||
return Ok(RebalanceEntryOutcome::Deferred {
|
||||
last_error: deferred_error,
|
||||
});
|
||||
}
|
||||
let entry_err =
|
||||
with_rebalance_entry_context(result.stage.unwrap_or("migrate"), bucket.as_str(), version.name.as_str(), err);
|
||||
|
||||
if !stats_updates.is_empty()
|
||||
&& let Err(stats_err) = self
|
||||
.update_pool_stats_batch(pool_index, bucket.clone(), stats_updates.as_slice())
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
"rebalance_entry {} failed to update stats before returning migration error for {}: {}",
|
||||
&bucket, &entry.name, stats_err
|
||||
);
|
||||
}
|
||||
|
||||
return Err(entry_err);
|
||||
}
|
||||
|
||||
stats_updates.push(version);
|
||||
if should_count_rebalance_version_complete(&result) {
|
||||
rebalanced += 1;
|
||||
}
|
||||
}
|
||||
|
||||
resolve_rebalance_stats_update_result(
|
||||
self.update_pool_stats_batch(pool_index, bucket.clone(), stats_updates.as_slice())
|
||||
.await,
|
||||
pool_index,
|
||||
bucket.as_str(),
|
||||
entry.name.as_str(),
|
||||
)?;
|
||||
|
||||
if should_cleanup_rebalance_source_entry(rebalanced, fivs.versions.len()) {
|
||||
let cleanup_warning = resolve_rebalance_entry_cleanup_delete_result(
|
||||
set.delete_object(
|
||||
bucket.as_str(),
|
||||
&encode_dir_object(&entry.name),
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
bucket.as_str(),
|
||||
entry.name.as_str(),
|
||||
)?;
|
||||
if let Some(message) = cleanup_warning {
|
||||
warn!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
stage = "cleanup_source",
|
||||
cleanup_status = "failed_ignored",
|
||||
error = %message,
|
||||
"Ignored rebalance source cleanup failure"
|
||||
);
|
||||
if let Err(err) = self
|
||||
.record_rebalance_cleanup_warning(pool_index, bucket.as_str(), entry.name.as_str(), message)
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
stage = "cleanup_source",
|
||||
error = ?err,
|
||||
"Failed to record rebalance source cleanup warning"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
state = "source_deleted",
|
||||
"Deleted rebalance source entry"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(RebalanceEntryOutcome::Completed)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, rd))]
|
||||
async fn rebalance_object(self: Arc<Self>, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> {
|
||||
data_movement::migrate_object(self, pool_idx, bucket, rd, "rebalance_object").await
|
||||
}
|
||||
|
||||
async fn update_rebalance_last_error(&self, pool_idx: usize, message: String) -> Result<()> {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
let Some(meta) = rebalance_meta.as_mut() else {
|
||||
return Err(rebalance_metadata_not_initialized_error("record rebalance last error"));
|
||||
};
|
||||
let pool_count = meta.pool_stats.len();
|
||||
ensure_valid_rebalance_pool_index(pool_count, pool_idx)?;
|
||||
let Some(pool_stat) = meta.pool_stats.get_mut(pool_idx) else {
|
||||
return Err(invalid_rebalance_pool_index_error(pool_idx, pool_count));
|
||||
};
|
||||
|
||||
pool_stat.info.last_error = Some(message);
|
||||
meta.last_refreshed_at = Some(OffsetDateTime::now_utc());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, rx))]
|
||||
pub(super) async fn rebalance_bucket(
|
||||
self: &Arc<Self>,
|
||||
rx: CancellationToken,
|
||||
bucket: String,
|
||||
pool_index: usize,
|
||||
) -> Result<RebalanceBucketOutcome> {
|
||||
ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?;
|
||||
|
||||
// Placeholder for actual bucket rebalance logic
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "entry_scan_started",
|
||||
"Rebalance bucket entry scan started"
|
||||
);
|
||||
|
||||
// TODO: other config
|
||||
// if bucket != RUSTFS_META_BUCKET{
|
||||
|
||||
// }
|
||||
|
||||
let pool = clone_arc_by_index(self.pools.as_slice(), pool_index, "invalid rebalance pool index")?;
|
||||
let bucket_configs = Arc::new(load_rebalance_bucket_configs(&bucket).await?);
|
||||
|
||||
let mut jobs = Vec::new();
|
||||
let entry_error = Arc::new(tokio::sync::Mutex::new(None::<Error>));
|
||||
let entry_workers = Arc::new(tokio::sync::Semaphore::new(pool.disk_set.len().max(1)));
|
||||
|
||||
for (set_idx, set) in pool.disk_set.iter().enumerate() {
|
||||
let entry_tasks = Arc::new(tokio::sync::Mutex::new(Vec::<RebalanceEntryTask>::new()));
|
||||
let rebalance_entry: ListCallback = Arc::new({
|
||||
let this = Arc::clone(self);
|
||||
let bucket = bucket.clone();
|
||||
let entry_error = entry_error.clone();
|
||||
let callback_rx = rx.clone();
|
||||
let set = set.clone();
|
||||
let bucket_configs = bucket_configs.clone();
|
||||
let entry_tasks = entry_tasks.clone();
|
||||
let entry_workers = entry_workers.clone();
|
||||
move |entry: MetaCacheEntry| {
|
||||
let this = this.clone();
|
||||
let bucket = bucket.clone();
|
||||
let entry_error = entry_error.clone();
|
||||
let callback_rx = callback_rx.clone();
|
||||
let set = set.clone();
|
||||
let bucket_configs = bucket_configs.clone();
|
||||
let entry_tasks = entry_tasks.clone();
|
||||
let entry_workers = entry_workers.clone();
|
||||
Box::pin(async move {
|
||||
if callback_rx.is_cancelled() {
|
||||
return;
|
||||
}
|
||||
if entry_error.lock().await.is_some() {
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(err) = backpressure::wait_for_data_movement_admission(
|
||||
DataMovementOperation::Rebalance,
|
||||
pool_index,
|
||||
&callback_rx,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if matches!(err, Error::OperationCanceled) {
|
||||
return;
|
||||
}
|
||||
error!("rebalance_entry: data movement admission failed: {err}");
|
||||
let mut first_err = entry_error.lock().await;
|
||||
if first_err.is_none() {
|
||||
*first_err = Some(err);
|
||||
callback_rx.cancel();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let permit = tokio::select! {
|
||||
_ = callback_rx.cancelled() => return,
|
||||
permit = entry_workers.clone().acquire_owned() => match permit {
|
||||
Ok(permit) => permit,
|
||||
Err(err) => {
|
||||
error!("rebalance_entry: worker semaphore closed: {err}");
|
||||
return;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
if entry_error.lock().await.is_some() {
|
||||
return;
|
||||
}
|
||||
|
||||
let task = tokio::spawn(async move {
|
||||
let _permit = permit;
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
set_index = set_idx,
|
||||
state = "task_started",
|
||||
"Started rebalance entry task"
|
||||
);
|
||||
let result = this.rebalance_entry(bucket, pool_index, entry, set, bucket_configs).await;
|
||||
if let Err(err) = &result {
|
||||
error!("rebalance_entry: rebalance entry failed: {err}");
|
||||
let mut first_err = entry_error.lock().await;
|
||||
if first_err.is_none() {
|
||||
*first_err = Some(err.clone());
|
||||
callback_rx.cancel();
|
||||
}
|
||||
}
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
set_index = set_idx,
|
||||
state = "task_completed",
|
||||
"Completed rebalance entry task"
|
||||
);
|
||||
result
|
||||
});
|
||||
|
||||
entry_tasks.lock().await.push(task);
|
||||
})
|
||||
}
|
||||
});
|
||||
|
||||
let set = set.clone();
|
||||
let rx = rx.clone();
|
||||
let bucket = bucket.clone();
|
||||
let entry_tasks = entry_tasks.clone();
|
||||
|
||||
let job = tokio::spawn(async move {
|
||||
let list_result =
|
||||
run_rebalance_listing_with_retry(set, rx, bucket.clone(), rebalance_entry, set_idx, rebalance_max_attempts())
|
||||
.await;
|
||||
let entry_result = wait_rebalance_entry_tasks(set_idx, entry_tasks).await;
|
||||
let result = list_result.and(entry_result);
|
||||
if let Err(err) = &result {
|
||||
error!("Rebalance worker {} error: {}", set_idx, err);
|
||||
} else {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
set_index = set_idx,
|
||||
state = "worker_completed",
|
||||
"Completed rebalance worker"
|
||||
);
|
||||
}
|
||||
result
|
||||
});
|
||||
|
||||
jobs.push((set_idx, job));
|
||||
}
|
||||
|
||||
let mut worker_error: Option<Error> = None;
|
||||
let mut deferred_error: Option<String> = None;
|
||||
for (set_idx, job) in jobs {
|
||||
match resolve_rebalance_worker_result(set_idx, job.await) {
|
||||
Ok(Some(last_error)) if deferred_error.is_none() => {
|
||||
deferred_error = Some(last_error);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(err) if worker_error.is_none() => {
|
||||
worker_error = Some(err);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
let entry_error = entry_error.lock().await.clone();
|
||||
resolve_rebalance_bucket_error(entry_error, worker_error)?;
|
||||
if let Some(last_error) = deferred_error {
|
||||
return Ok(RebalanceBucketOutcome::Deferred { last_error });
|
||||
}
|
||||
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "completed",
|
||||
"Finished rebalance bucket"
|
||||
);
|
||||
Ok(RebalanceBucketOutcome::Completed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,988 @@
|
||||
use super::{
|
||||
EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, Error, GetObjectReader, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
|
||||
ObjectInfo, ObjectOptions, PutObjReader, REBAL_META_FMT, REBAL_META_NAME, REBAL_META_VER,
|
||||
REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT, REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_STOP_PROPAGATION_ERROR_PREFIX,
|
||||
RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats,
|
||||
RebalanceStopPropagationRecord, Result,
|
||||
};
|
||||
use crate::config::com::{read_config_with_metadata, save_config_with_opts};
|
||||
use crate::error::is_err_operation_canceled;
|
||||
use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec};
|
||||
use http::HeaderMap;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use std::{collections::HashSet, fmt, io::Cursor, sync::Arc};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{debug, info};
|
||||
|
||||
impl RebalanceStats {
|
||||
pub fn update(&mut self, bucket: String, fi: &FileInfo) {
|
||||
if fi.is_latest {
|
||||
self.num_objects += 1;
|
||||
}
|
||||
|
||||
self.num_versions += 1;
|
||||
let on_disk_size = if fi.deleted || fi.erasure.data_blocks == 0 || fi.size <= 0 {
|
||||
0
|
||||
} else {
|
||||
let data_blocks = fi.erasure.data_blocks as i64;
|
||||
let total_blocks = fi.erasure.data_blocks.saturating_add(fi.erasure.parity_blocks) as i64;
|
||||
fi.size
|
||||
.saturating_mul(total_blocks)
|
||||
.checked_div(data_blocks)
|
||||
.unwrap_or(0)
|
||||
.max(0) as u64
|
||||
};
|
||||
self.bytes = self.bytes.saturating_add(on_disk_size);
|
||||
self.bucket = bucket;
|
||||
self.object = fi.name.clone();
|
||||
}
|
||||
|
||||
pub fn update_batch(&mut self, bucket: String, versions: &[&FileInfo]) {
|
||||
for version in versions {
|
||||
self.update(bucket.clone(), version);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for RebalStatus {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let status = match self {
|
||||
RebalStatus::None => "None",
|
||||
RebalStatus::Started => "Started",
|
||||
RebalStatus::Completed => "Completed",
|
||||
RebalStatus::Stopped => "Stopped",
|
||||
RebalStatus::Failed => "Failed",
|
||||
};
|
||||
write!(f, "{status}")
|
||||
}
|
||||
}
|
||||
|
||||
impl From<u8> for RebalStatus {
|
||||
fn from(value: u8) -> Self {
|
||||
match value {
|
||||
1 => RebalStatus::Started,
|
||||
2 => RebalStatus::Completed,
|
||||
3 => RebalStatus::Stopped,
|
||||
4 => RebalStatus::Failed,
|
||||
_ => RebalStatus::None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RebalanceMeta {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
pub async fn load<S>(&mut self, store: Arc<S>) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
self.load_with_opts(store, ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
pub async fn load_with_opts<S>(&mut self, store: Arc<S>, opts: ObjectOptions) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
let (data, _) = read_config_with_metadata(store, REBAL_META_NAME, &opts).await?;
|
||||
if data.is_empty() {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "metadata_empty",
|
||||
"Rebalance metadata is empty"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
if data.len() <= 4 {
|
||||
return Err(rebalance_meta_load_no_data_error());
|
||||
}
|
||||
|
||||
// Read header
|
||||
match u16::from_le_bytes([data[0], data[1]]) {
|
||||
REBAL_META_FMT => {}
|
||||
fmt => return Err(rebalance_meta_load_unknown_format_error(fmt)),
|
||||
}
|
||||
match u16::from_le_bytes([data[2], data[3]]) {
|
||||
REBAL_META_VER => {}
|
||||
ver => return Err(rebalance_meta_load_unknown_version_error(ver)),
|
||||
}
|
||||
|
||||
let meta: Self = rmp_serde::from_read(Cursor::new(&data[4..]))?;
|
||||
*self = meta;
|
||||
|
||||
self.last_refreshed_at = Some(OffsetDateTime::now_utc());
|
||||
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "metadata_loaded",
|
||||
"Loaded rebalance metadata"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn save<S>(&self, store: Arc<S>) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
self.save_with_opts(store, ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
pub async fn save_with_opts<S>(&self, store: Arc<S>, opts: ObjectOptions) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
if self.pool_stats.is_empty() {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "metadata_save_skipped",
|
||||
reason = "no_pool_stats",
|
||||
"Skipped rebalance metadata save"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut data = Vec::new();
|
||||
|
||||
// Initialize the header
|
||||
data.extend(&REBAL_META_FMT.to_le_bytes());
|
||||
data.extend(&REBAL_META_VER.to_le_bytes());
|
||||
|
||||
let msg = rmp_serde::to_vec(self)?;
|
||||
data.extend(msg);
|
||||
|
||||
save_config_with_opts(store, REBAL_META_NAME, data, &opts).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_rebalance_pool_started(pool_stat: &RebalanceStats) -> bool {
|
||||
pool_stat.participating && pool_stat.info.status == RebalStatus::Started
|
||||
}
|
||||
|
||||
pub(super) fn is_rebalance_pool_active(pool_stat: &RebalanceStats) -> bool {
|
||||
is_rebalance_pool_started(pool_stat) || pool_stat.info.stopping
|
||||
}
|
||||
|
||||
pub(super) fn is_rebalance_in_progress(meta: &RebalanceMeta) -> bool {
|
||||
meta.pool_stats.iter().any(is_rebalance_pool_active)
|
||||
}
|
||||
|
||||
pub(crate) fn is_rebalance_conflicting_with_decommission(meta: &RebalanceMeta) -> bool {
|
||||
is_rebalance_in_progress(meta)
|
||||
}
|
||||
|
||||
pub(super) fn first_rebalance_bucket(pool_stat: &RebalanceStats) -> Option<String> {
|
||||
pool_stat.buckets.first().cloned()
|
||||
}
|
||||
|
||||
pub(super) fn rebalance_meta_load_no_data_error() -> Error {
|
||||
Error::other("rebalance metadata load failed: metadata payload is too short")
|
||||
}
|
||||
|
||||
pub(super) fn rebalance_meta_load_unknown_format_error(fmt: u16) -> Error {
|
||||
Error::other(format!("rebalance metadata load failed: unknown format {fmt}"))
|
||||
}
|
||||
|
||||
pub(super) fn rebalance_meta_load_unknown_version_error(ver: u16) -> Error {
|
||||
Error::other(format!("rebalance metadata load failed: unknown version {ver}"))
|
||||
}
|
||||
|
||||
pub fn encode_rebalance_stop_propagation_record(record: &RebalanceStopPropagationRecord) -> String {
|
||||
match serde_json::to_string(record) {
|
||||
Ok(payload) => format!("{REBALANCE_STOP_PROPAGATION_ERROR_PREFIX}{payload}"),
|
||||
Err(err) => {
|
||||
let payload = serde_json::json!({
|
||||
"encodeError": err.to_string(),
|
||||
"stopFailures": [],
|
||||
"terminalReloadFailures": [],
|
||||
});
|
||||
format!("{REBALANCE_STOP_PROPAGATION_ERROR_PREFIX}{payload}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode_rebalance_stop_propagation_record(message: &str) -> Option<RebalanceStopPropagationRecord> {
|
||||
let payload = message.strip_prefix(REBALANCE_STOP_PROPAGATION_ERROR_PREFIX)?;
|
||||
serde_json::from_str(payload).ok()
|
||||
}
|
||||
|
||||
fn is_rebalance_stop_propagation_error(message: Option<&str>) -> bool {
|
||||
message.is_some_and(|message| message.starts_with(REBALANCE_STOP_PROPAGATION_ERROR_PREFIX))
|
||||
}
|
||||
|
||||
pub(super) fn rebalance_goal_reached(init_free_space: u64, init_capacity: u64, bytes: u64, percent_free_goal: f64) -> bool {
|
||||
if init_capacity == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let pfi = (init_free_space + bytes) as f64 / init_capacity as f64;
|
||||
pfi + f64::EPSILON >= percent_free_goal
|
||||
}
|
||||
|
||||
pub(super) fn percent_free_ratio(total_free: u64, total_cap: u64) -> f64 {
|
||||
if total_cap == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
total_free as f64 / total_cap as f64
|
||||
}
|
||||
|
||||
pub(super) fn next_rebal_bucket_from_stat(pool_stat: &RebalanceStats) -> Option<String> {
|
||||
if pool_stat.buckets.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
first_rebalance_bucket(pool_stat)
|
||||
}
|
||||
|
||||
pub(super) fn rebalance_metadata_not_initialized_error(operation: &str) -> Error {
|
||||
Error::other(format!("failed to {operation}: rebalance metadata not initialized"))
|
||||
}
|
||||
|
||||
pub(super) fn invalid_rebalance_pool_index_error(pool_index: usize, pool_count: usize) -> Error {
|
||||
Error::other(format!("invalid rebalance pool index {pool_index} for {pool_count} pools"))
|
||||
}
|
||||
|
||||
pub(super) fn clone_rebalance_pool_stats(meta: Option<&RebalanceMeta>) -> Result<Vec<RebalanceStats>> {
|
||||
let Some(meta) = meta else {
|
||||
return Err(rebalance_metadata_not_initialized_error("clone rebalance pool stats"));
|
||||
};
|
||||
Ok(meta.pool_stats.clone())
|
||||
}
|
||||
|
||||
pub(super) fn should_accept_rebalance_stats_update(meta: &RebalanceMeta, pool_index: usize) -> bool {
|
||||
if meta.stopped_at.is_some() {
|
||||
return false;
|
||||
}
|
||||
|
||||
meta.pool_stats
|
||||
.get(pool_index)
|
||||
.is_some_and(|pool_stat| pool_stat.info.status == RebalStatus::Started)
|
||||
}
|
||||
|
||||
pub(super) fn resolve_next_rebalance_bucket(meta: Option<&RebalanceMeta>, pool_index: usize) -> Result<Option<String>> {
|
||||
let Some(meta) = meta else {
|
||||
return Err(rebalance_metadata_not_initialized_error("resolve next rebalance bucket"));
|
||||
};
|
||||
|
||||
ensure_valid_rebalance_pool_index(meta.pool_stats.len(), pool_index)?;
|
||||
let Some(pool_stat) = meta.pool_stats.get(pool_index) else {
|
||||
return Err(invalid_rebalance_pool_index_error(pool_index, meta.pool_stats.len()));
|
||||
};
|
||||
|
||||
if pool_stat.info.status == RebalStatus::Completed || !pool_stat.participating {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "unavailable",
|
||||
reason = "completed_or_not_participating",
|
||||
"No rebalance bucket available"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if pool_stat.buckets.is_empty() {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "unavailable",
|
||||
reason = "bucket_queue_empty",
|
||||
"No rebalance bucket available"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if let Some(bucket) = next_rebal_bucket_from_stat(pool_stat) {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "selected",
|
||||
"Selected rebalance bucket"
|
||||
);
|
||||
return Ok(Some(bucket));
|
||||
}
|
||||
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "unavailable",
|
||||
reason = "selection_returned_none",
|
||||
"No rebalance bucket available"
|
||||
);
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub(super) fn mark_rebalance_bucket_done(meta: Option<&mut RebalanceMeta>, pool_index: usize, bucket: &str) -> Result<()> {
|
||||
let Some(meta) = meta else {
|
||||
return Err(rebalance_metadata_not_initialized_error("mark rebalance bucket done"));
|
||||
};
|
||||
|
||||
ensure_valid_rebalance_pool_index(meta.pool_stats.len(), pool_index)?;
|
||||
let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) else {
|
||||
return Err(invalid_rebalance_pool_index_error(pool_index, meta.pool_stats.len()));
|
||||
};
|
||||
|
||||
if take_bucket_from_rebalance_queue(pool_stat, bucket) {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "queue_removed",
|
||||
remaining_bucket_count = pool_stat.buckets.len(),
|
||||
"Removed bucket from rebalance queue"
|
||||
);
|
||||
if has_deferred_rebalance_error(pool_stat) {
|
||||
pool_stat.info.last_error = None;
|
||||
}
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::other(format!(
|
||||
"failed to mark rebalance bucket done: bucket {bucket} was not queued for pool {pool_index}"
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_rebalance_cleanup_warning_in_meta(
|
||||
meta: Option<&mut RebalanceMeta>,
|
||||
pool_index: usize,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
message: String,
|
||||
now: OffsetDateTime,
|
||||
) -> Result<()> {
|
||||
let Some(meta) = meta else {
|
||||
return Err(rebalance_metadata_not_initialized_error("record rebalance cleanup warning"));
|
||||
};
|
||||
|
||||
ensure_valid_rebalance_pool_index(meta.pool_stats.len(), pool_index)?;
|
||||
let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) else {
|
||||
return Err(invalid_rebalance_pool_index_error(pool_index, meta.pool_stats.len()));
|
||||
};
|
||||
|
||||
pool_stat.cleanup_warnings.count = pool_stat.cleanup_warnings.count.saturating_add(1);
|
||||
pool_stat.cleanup_warnings.last_message = Some(message.clone());
|
||||
pool_stat.cleanup_warnings.last_bucket = Some(bucket.to_string());
|
||||
pool_stat.cleanup_warnings.last_object = Some(object.to_string());
|
||||
pool_stat.cleanup_warnings.last_at = Some(now);
|
||||
pool_stat.cleanup_warnings.entries.push(RebalanceCleanupWarningEntry {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
message,
|
||||
timestamp: Some(now),
|
||||
});
|
||||
truncate_rebalance_cleanup_warning_entries(&mut pool_stat.cleanup_warnings.entries);
|
||||
meta.last_refreshed_at = Some(now);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn take_bucket_from_rebalance_queue(pool_stat: &mut RebalanceStats, bucket: &str) -> bool {
|
||||
let mut found = false;
|
||||
pool_stat.buckets.retain(|name| {
|
||||
if name == bucket {
|
||||
found = true;
|
||||
pool_stat.rebalanced_buckets.push(name.clone());
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
|
||||
found
|
||||
}
|
||||
|
||||
pub(super) fn defer_bucket_in_rebalance_queue(pool_stat: &mut RebalanceStats, bucket: &str) -> Result<()> {
|
||||
let Some(pos) = pool_stat.buckets.iter().position(|name| name == bucket) else {
|
||||
return Err(Error::other(format!("failed to defer rebalance bucket {bucket}: bucket was not queued")));
|
||||
};
|
||||
|
||||
let bucket = pool_stat.buckets.remove(pos);
|
||||
pool_stat.buckets.push(bucket);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn should_pool_participate(init_free_space: u64, init_capacity: u64, percent_free_goal: f64) -> bool {
|
||||
init_capacity > 0 && percent_free_ratio(init_free_space, init_capacity) < percent_free_goal
|
||||
}
|
||||
|
||||
pub(super) fn complete_rebalance_pools_at_goal(meta: &mut RebalanceMeta, now: OffsetDateTime) -> bool {
|
||||
let mut changed = false;
|
||||
|
||||
for pool_stat in meta.pool_stats.iter_mut() {
|
||||
if !is_rebalance_pool_started(pool_stat)
|
||||
|| has_deferred_rebalance_error(pool_stat)
|
||||
|| has_rebalance_cleanup_warnings(pool_stat)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if rebalance_goal_reached(
|
||||
pool_stat.init_free_space,
|
||||
pool_stat.init_capacity,
|
||||
pool_stat.bytes,
|
||||
meta.percent_free_goal,
|
||||
) {
|
||||
pool_stat.info.status = RebalStatus::Completed;
|
||||
pool_stat.info.end_time = Some(now);
|
||||
pool_stat.info.last_error = None;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
changed
|
||||
}
|
||||
|
||||
pub(super) fn complete_rebalance_pools_with_empty_queue(meta: &mut RebalanceMeta, now: OffsetDateTime) -> bool {
|
||||
let mut changed = false;
|
||||
|
||||
for pool_stat in meta.pool_stats.iter_mut() {
|
||||
if !is_rebalance_pool_started(pool_stat) || !pool_stat.buckets.is_empty() || has_rebalance_cleanup_warnings(pool_stat) {
|
||||
continue;
|
||||
}
|
||||
|
||||
pool_stat.info.status = RebalStatus::Completed;
|
||||
pool_stat.info.end_time = Some(now);
|
||||
pool_stat.info.last_error = None;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
changed
|
||||
}
|
||||
|
||||
pub(super) fn has_deferred_rebalance_error(pool_stat: &RebalanceStats) -> bool {
|
||||
pool_stat
|
||||
.info
|
||||
.last_error
|
||||
.as_deref()
|
||||
.is_some_and(|last_error| last_error.starts_with(REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX))
|
||||
}
|
||||
|
||||
pub(super) fn has_rebalance_cleanup_warnings(pool_stat: &RebalanceStats) -> bool {
|
||||
pool_stat.cleanup_warnings.count > 0
|
||||
}
|
||||
|
||||
pub(super) fn clone_first_arc<T>(values: &[Arc<T>], err_msg: &str) -> Result<Arc<T>> {
|
||||
values.first().cloned().ok_or_else(|| Error::other(err_msg))
|
||||
}
|
||||
|
||||
pub(super) fn clone_arc_by_index<T>(values: &[Arc<T>], idx: usize, err_prefix: &str) -> Result<Arc<T>> {
|
||||
values
|
||||
.get(idx)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::other(format!("{err_prefix}: {idx}")))
|
||||
}
|
||||
|
||||
pub(super) fn ensure_valid_rebalance_pool_index(pool_count: usize, idx: usize) -> Result<()> {
|
||||
if idx >= pool_count {
|
||||
return Err(invalid_rebalance_pool_index_error(idx, pool_count));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) enum RebalanceTerminalEvent {
|
||||
Completed { msg: String },
|
||||
Stopped { msg: String },
|
||||
Failed { msg: String, last_error: String },
|
||||
ChannelClosed { msg: String, last_error: String },
|
||||
}
|
||||
|
||||
impl RebalanceTerminalEvent {
|
||||
pub(super) fn message(&self) -> &str {
|
||||
match self {
|
||||
RebalanceTerminalEvent::Completed { msg }
|
||||
| RebalanceTerminalEvent::Stopped { msg }
|
||||
| RebalanceTerminalEvent::Failed { msg, .. }
|
||||
| RebalanceTerminalEvent::ChannelClosed { msg, .. } => msg,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn apply_rebalance_terminal_event(
|
||||
status: &mut RebalStatus,
|
||||
end_time: &mut Option<OffsetDateTime>,
|
||||
last_error: &mut Option<String>,
|
||||
terminal_event: RebalanceTerminalEvent,
|
||||
now: OffsetDateTime,
|
||||
) {
|
||||
match terminal_event {
|
||||
RebalanceTerminalEvent::Completed { .. } => {
|
||||
*status = RebalStatus::Completed;
|
||||
*end_time = Some(now);
|
||||
*last_error = None;
|
||||
}
|
||||
RebalanceTerminalEvent::Stopped { .. } => {
|
||||
*status = RebalStatus::Stopped;
|
||||
*end_time = Some(now);
|
||||
*last_error = None;
|
||||
}
|
||||
RebalanceTerminalEvent::Failed { last_error: err, .. }
|
||||
| RebalanceTerminalEvent::ChannelClosed { last_error: err, .. } => {
|
||||
*status = RebalStatus::Failed;
|
||||
*end_time = Some(now);
|
||||
*last_error = Some(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn classify_rebalance_terminal_event(signal: Option<Result<()>>, now: OffsetDateTime) -> RebalanceTerminalEvent {
|
||||
match signal {
|
||||
Some(Ok(())) => RebalanceTerminalEvent::Completed {
|
||||
msg: format!("Rebalance completed at {now:?}"),
|
||||
},
|
||||
Some(Err(err)) => {
|
||||
if is_err_operation_canceled(&err) {
|
||||
RebalanceTerminalEvent::Stopped {
|
||||
msg: format!("Rebalance stopped at {now:?}"),
|
||||
}
|
||||
} else {
|
||||
RebalanceTerminalEvent::Failed {
|
||||
msg: format!("Rebalance failed at {now:?} with err {err:?}"),
|
||||
last_error: err.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
None => RebalanceTerminalEvent::ChannelClosed {
|
||||
msg: format!("Rebalance save task channel closed unexpectedly at {now:?}"),
|
||||
last_error: format!("rebalance save channel closed before terminal event at {now:?}"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn ensure_rebalance_not_decommissioning(decommission_running: bool) -> bool {
|
||||
!decommission_running
|
||||
}
|
||||
|
||||
pub(super) fn validate_start_rebalance_state(decommission_running: bool, meta_loaded: bool) -> Result<()> {
|
||||
if !ensure_rebalance_not_decommissioning(decommission_running) {
|
||||
return Err(Error::DecommissionAlreadyRunning);
|
||||
}
|
||||
if !meta_loaded {
|
||||
return Err(Error::ConfigNotFound);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn validate_init_rebalance_state(decommission_running: bool, current_meta: Option<&RebalanceMeta>) -> Result<()> {
|
||||
if !ensure_rebalance_not_decommissioning(decommission_running) {
|
||||
return Err(Error::DecommissionAlreadyRunning);
|
||||
}
|
||||
if current_meta.is_some_and(|meta| !is_rebalance_meta_replaceable_for_new_id(meta)) {
|
||||
return Err(Error::RebalanceAlreadyRunning);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn should_skip_start_rebalance(cancel_attached: bool, in_progress: bool) -> bool {
|
||||
cancel_attached && in_progress
|
||||
}
|
||||
|
||||
pub(super) fn is_rebalance_stopped_terminal_event(terminal_event: &RebalanceTerminalEvent) -> bool {
|
||||
matches!(terminal_event, RebalanceTerminalEvent::Stopped { .. })
|
||||
}
|
||||
|
||||
pub(super) fn should_preserve_rebalance_stopped_state(
|
||||
meta_stopped: bool,
|
||||
status: RebalStatus,
|
||||
terminal_event: &RebalanceTerminalEvent,
|
||||
) -> bool {
|
||||
(meta_stopped || status == RebalStatus::Stopped) && !is_rebalance_stopped_terminal_event(terminal_event)
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_participants(pool_stats: &[RebalanceStats], pool_count: usize) -> Vec<bool> {
|
||||
let mut participants = vec![false; pool_count];
|
||||
|
||||
for (idx, pool_stat) in pool_stats.iter().enumerate() {
|
||||
if idx >= participants.len() {
|
||||
break;
|
||||
}
|
||||
|
||||
if pool_stat.info.status == RebalStatus::Started {
|
||||
participants[idx] = pool_stat.participating;
|
||||
}
|
||||
}
|
||||
|
||||
participants
|
||||
}
|
||||
|
||||
pub(super) fn is_rebalance_actively_running(meta: &RebalanceMeta) -> bool {
|
||||
meta.cancel.is_some() && is_rebalance_in_progress(meta)
|
||||
}
|
||||
|
||||
pub(super) fn should_ignore_rebalance_data_usage_cache(bucket: &str) -> bool {
|
||||
bucket == crate::disk::RUSTFS_META_BUCKET
|
||||
}
|
||||
|
||||
pub(super) fn apply_rebalance_save_option(meta: &mut RebalanceMeta, pool_idx: usize, opt: RebalSaveOpt, now: OffsetDateTime) {
|
||||
match opt {
|
||||
RebalSaveOpt::Stats => {
|
||||
if pool_idx >= meta.pool_stats.len() {
|
||||
info!("save_rebalance_stats: pool_idx {pool_idx} out of range for pool_stats");
|
||||
}
|
||||
}
|
||||
RebalSaveOpt::StoppedAt => {
|
||||
apply_stopped_at(meta, now);
|
||||
}
|
||||
}
|
||||
|
||||
meta.last_refreshed_at = Some(now);
|
||||
}
|
||||
|
||||
pub(super) fn is_rebalance_terminal_status(status: RebalStatus) -> bool {
|
||||
matches!(status, RebalStatus::Completed | RebalStatus::Stopped | RebalStatus::Failed)
|
||||
}
|
||||
|
||||
pub(super) fn merge_rebalance_bucket_lists(remote: &mut Vec<String>, local: &[String]) {
|
||||
let mut existing: HashSet<String> = remote.iter().cloned().collect();
|
||||
for bucket in local {
|
||||
if existing.insert(bucket.clone()) {
|
||||
remote.push(bucket.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn remove_rebalanced_buckets_from_queue(pool_stat: &mut RebalanceStats) {
|
||||
let rebalanced_buckets: HashSet<String> = pool_stat.rebalanced_buckets.iter().cloned().collect();
|
||||
pool_stat.buckets.retain(|bucket| !rebalanced_buckets.contains(bucket));
|
||||
}
|
||||
|
||||
pub(super) fn merge_rebalance_cleanup_warnings(remote: &mut RebalanceCleanupWarnings, local: &RebalanceCleanupWarnings) {
|
||||
remote.count = remote.count.max(local.count);
|
||||
|
||||
if should_replace_rebalance_cleanup_warning(remote.last_at, local.last_at) {
|
||||
remote.last_message = local.last_message.clone();
|
||||
remote.last_bucket = local.last_bucket.clone();
|
||||
remote.last_object = local.last_object.clone();
|
||||
remote.last_at = local.last_at;
|
||||
}
|
||||
|
||||
merge_rebalance_cleanup_warning_entries(&mut remote.entries, &local.entries);
|
||||
let retained_entries = u64::try_from(remote.entries.len()).unwrap_or(u64::MAX);
|
||||
remote.count = remote.count.max(retained_entries);
|
||||
}
|
||||
|
||||
pub(super) fn merge_rebalance_cleanup_warning_entries(
|
||||
remote: &mut Vec<RebalanceCleanupWarningEntry>,
|
||||
local: &[RebalanceCleanupWarningEntry],
|
||||
) {
|
||||
for entry in local {
|
||||
if !remote.iter().any(|existing| existing == entry) {
|
||||
remote.push(entry.clone());
|
||||
}
|
||||
}
|
||||
|
||||
remote.sort_by_key(|entry| entry.timestamp);
|
||||
truncate_rebalance_cleanup_warning_entries(remote);
|
||||
}
|
||||
|
||||
fn truncate_rebalance_cleanup_warning_entries(entries: &mut Vec<RebalanceCleanupWarningEntry>) {
|
||||
if entries.len() > REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT {
|
||||
let remove_count = entries.len() - REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT;
|
||||
entries.drain(0..remove_count);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn should_replace_rebalance_cleanup_warning(
|
||||
remote_at: Option<OffsetDateTime>,
|
||||
local_at: Option<OffsetDateTime>,
|
||||
) -> bool {
|
||||
match (remote_at, local_at) {
|
||||
(_, None) => false,
|
||||
(None, Some(_)) => true,
|
||||
(Some(remote), Some(local)) => local >= remote,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn merge_rebalance_stop_propagation_error(remote: Option<String>, local: Option<String>) -> Option<String> {
|
||||
if is_rebalance_stop_propagation_error(local.as_deref()) {
|
||||
local
|
||||
} else if is_rebalance_stop_propagation_error(remote.as_deref()) {
|
||||
remote
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn merge_rebalance_pool_stats(remote: &mut RebalanceStats, local: &RebalanceStats) {
|
||||
remote.init_free_space = remote.init_free_space.max(local.init_free_space);
|
||||
remote.init_capacity = remote.init_capacity.max(local.init_capacity);
|
||||
remote.participating |= local.participating;
|
||||
|
||||
merge_rebalance_bucket_lists(&mut remote.buckets, &local.buckets);
|
||||
merge_rebalance_bucket_lists(&mut remote.rebalanced_buckets, &local.rebalanced_buckets);
|
||||
remove_rebalanced_buckets_from_queue(remote);
|
||||
|
||||
let local_is_newer = local.num_versions >= remote.num_versions;
|
||||
remote.num_objects = remote.num_objects.max(local.num_objects);
|
||||
remote.num_versions = remote.num_versions.max(local.num_versions);
|
||||
remote.bytes = remote.bytes.max(local.bytes);
|
||||
merge_rebalance_cleanup_warnings(&mut remote.cleanup_warnings, &local.cleanup_warnings);
|
||||
|
||||
if local_is_newer {
|
||||
remote.bucket = local.bucket.clone();
|
||||
remote.object = local.object.clone();
|
||||
}
|
||||
|
||||
if remote.info.start_time.is_none() {
|
||||
remote.info.start_time = local.info.start_time;
|
||||
}
|
||||
|
||||
if is_rebalance_terminal_status(remote.info.status) && local.info.status == RebalStatus::Started {
|
||||
return;
|
||||
}
|
||||
|
||||
if remote.info.status == RebalStatus::Stopped && matches!(local.info.status, RebalStatus::Started | RebalStatus::Completed) {
|
||||
return;
|
||||
}
|
||||
|
||||
match local.info.status {
|
||||
RebalStatus::Failed => {
|
||||
remote.info.status = RebalStatus::Failed;
|
||||
remote.info.stopping = false;
|
||||
remote.info.end_time = local.info.end_time.or(remote.info.end_time);
|
||||
remote.info.last_error = local.info.last_error.clone().or_else(|| remote.info.last_error.clone());
|
||||
}
|
||||
RebalStatus::Stopped => {
|
||||
if remote.info.status != RebalStatus::Failed {
|
||||
remote.info.status = RebalStatus::Stopped;
|
||||
remote.info.stopping = false;
|
||||
remote.info.end_time = local.info.end_time.or(remote.info.end_time);
|
||||
remote.info.last_error =
|
||||
merge_rebalance_stop_propagation_error(remote.info.last_error.clone(), local.info.last_error.clone());
|
||||
}
|
||||
}
|
||||
RebalStatus::Completed => {
|
||||
if !matches!(remote.info.status, RebalStatus::Failed | RebalStatus::Stopped) {
|
||||
remote.info.status = RebalStatus::Completed;
|
||||
remote.info.stopping = false;
|
||||
remote.info.end_time = local.info.end_time.or(remote.info.end_time);
|
||||
remote.info.last_error = None;
|
||||
}
|
||||
}
|
||||
RebalStatus::Started => {
|
||||
if !is_rebalance_terminal_status(remote.info.status) {
|
||||
remote.info.status = RebalStatus::Started;
|
||||
remote.info.stopping |= local.info.stopping;
|
||||
remote.info.last_error = local.info.last_error.clone();
|
||||
}
|
||||
}
|
||||
RebalStatus::None => {}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum RebalanceMetaMergeOutcome {
|
||||
Merged,
|
||||
Replaced,
|
||||
RejectedActiveConflict,
|
||||
}
|
||||
|
||||
pub(super) fn is_rebalance_meta_replaceable_for_new_id(meta: &RebalanceMeta) -> bool {
|
||||
meta.stopped_at.is_some() || !is_rebalance_in_progress(meta)
|
||||
}
|
||||
|
||||
pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &RebalanceMeta) -> RebalanceMetaMergeOutcome {
|
||||
if remote.id.is_empty() {
|
||||
*remote = local.clone();
|
||||
return RebalanceMetaMergeOutcome::Replaced;
|
||||
}
|
||||
|
||||
if !local.id.is_empty() && remote.id != local.id {
|
||||
if is_rebalance_meta_replaceable_for_new_id(remote) {
|
||||
*remote = local.clone();
|
||||
return RebalanceMetaMergeOutcome::Replaced;
|
||||
}
|
||||
return RebalanceMetaMergeOutcome::RejectedActiveConflict;
|
||||
}
|
||||
|
||||
remote.percent_free_goal = local.percent_free_goal;
|
||||
remote.last_refreshed_at = Some(OffsetDateTime::now_utc());
|
||||
if remote.stopped_at.is_none() {
|
||||
remote.stopped_at = local.stopped_at;
|
||||
}
|
||||
|
||||
if remote.pool_stats.len() < local.pool_stats.len() {
|
||||
remote.pool_stats.resize_with(local.pool_stats.len(), RebalanceStats::default);
|
||||
}
|
||||
|
||||
for (idx, local_pool_stat) in local.pool_stats.iter().enumerate() {
|
||||
if let Some(remote_pool_stat) = remote.pool_stats.get_mut(idx) {
|
||||
merge_rebalance_pool_stats(remote_pool_stat, local_pool_stat);
|
||||
}
|
||||
}
|
||||
|
||||
RebalanceMetaMergeOutcome::Merged
|
||||
}
|
||||
|
||||
pub(super) fn mark_started_rebalance_pools_stopped(meta: &mut RebalanceMeta, stop_time: OffsetDateTime) {
|
||||
for pool_stat in meta.pool_stats.iter_mut() {
|
||||
if pool_stat.info.status == RebalStatus::Started {
|
||||
pool_stat.info.status = RebalStatus::Stopped;
|
||||
pool_stat.info.stopping = false;
|
||||
pool_stat.info.end_time.get_or_insert(stop_time);
|
||||
if !is_rebalance_stop_propagation_error(pool_stat.info.last_error.as_deref()) {
|
||||
pool_stat.info.last_error = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn mark_started_rebalance_pools_stopping(meta: &mut RebalanceMeta) {
|
||||
for pool_stat in meta.pool_stats.iter_mut() {
|
||||
if pool_stat.info.status == RebalStatus::Started {
|
||||
pool_stat.info.stopping = true;
|
||||
pool_stat.info.last_error = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn apply_stopped_at(meta: &mut RebalanceMeta, now: OffsetDateTime) {
|
||||
meta.stopped_at.get_or_insert(now);
|
||||
mark_started_rebalance_pools_stopping(meta);
|
||||
}
|
||||
|
||||
pub(super) fn clear_rebalance_cancel_token(meta: Option<&mut RebalanceMeta>) -> bool {
|
||||
let Some(meta) = meta else {
|
||||
return false;
|
||||
};
|
||||
if let Some(cancel_tx) = meta.cancel.take() {
|
||||
cancel_tx.cancel();
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub(super) fn stop_rebalance_state(meta: &mut RebalanceMeta, now: OffsetDateTime) {
|
||||
clear_rebalance_cancel_token(Some(meta));
|
||||
if meta.stopped_at.is_none() && is_rebalance_in_progress(meta) {
|
||||
apply_stopped_at(meta, now);
|
||||
} else if meta.stopped_at.is_some() {
|
||||
mark_started_rebalance_pools_stopping(meta);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn stop_rebalance_meta_snapshot_for_id(
|
||||
meta: Option<&mut RebalanceMeta>,
|
||||
now: OffsetDateTime,
|
||||
expected_id: Option<&str>,
|
||||
) -> Result<Option<RebalanceMeta>> {
|
||||
let Some(meta) = meta else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if let Some(expected_id) = expected_id
|
||||
&& !expected_id.is_empty()
|
||||
&& meta.id != expected_id
|
||||
{
|
||||
return Err(Error::other(format!(
|
||||
"rebalance stop id mismatch: expected {expected_id}, found {}",
|
||||
meta.id
|
||||
)));
|
||||
}
|
||||
|
||||
stop_rebalance_state(meta, now);
|
||||
meta.last_refreshed_at = Some(now);
|
||||
Ok(Some(meta.clone()))
|
||||
}
|
||||
|
||||
pub(super) fn rollback_rebalance_start_meta_snapshot_for_id(
|
||||
meta: Option<&mut RebalanceMeta>,
|
||||
now: OffsetDateTime,
|
||||
expected_id: Option<&str>,
|
||||
start_error: String,
|
||||
) -> Option<RebalanceMeta> {
|
||||
meta.and_then(|meta| {
|
||||
if let Some(expected_id) = expected_id
|
||||
&& !expected_id.is_empty()
|
||||
&& meta.id != expected_id
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
clear_rebalance_cancel_token(Some(meta));
|
||||
meta.stopped_at.get_or_insert(now);
|
||||
meta.last_refreshed_at = Some(now);
|
||||
for pool_stat in meta.pool_stats.iter_mut() {
|
||||
if pool_stat.info.status == RebalStatus::Started {
|
||||
pool_stat.info.status = RebalStatus::Failed;
|
||||
pool_stat.info.stopping = false;
|
||||
pool_stat.info.end_time.get_or_insert(now);
|
||||
pool_stat.info.last_error = Some(start_error.clone());
|
||||
}
|
||||
}
|
||||
Some(meta.clone())
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn stop_rebalance_meta_snapshot(meta: Option<&mut RebalanceMeta>, now: OffsetDateTime) -> Option<RebalanceMeta> {
|
||||
let meta = meta?;
|
||||
stop_rebalance_state(meta, now);
|
||||
meta.last_refreshed_at = Some(now);
|
||||
Some(meta.clone())
|
||||
}
|
||||
|
||||
pub(super) fn record_rebalance_stop_propagation_snapshot(
|
||||
meta: Option<&mut RebalanceMeta>,
|
||||
encoded_error: String,
|
||||
now: OffsetDateTime,
|
||||
) -> Option<RebalanceMeta> {
|
||||
meta.map(|meta| {
|
||||
for pool_stat in meta.pool_stats.iter_mut() {
|
||||
if pool_stat.participating || pool_stat.info.stopping || pool_stat.info.status == RebalStatus::Stopped {
|
||||
pool_stat.info.last_error = Some(encoded_error.clone());
|
||||
}
|
||||
}
|
||||
meta.last_refreshed_at = Some(now);
|
||||
meta.clone()
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
use super::worker::{is_transient_rebalance_error, rebalance_migration_retry_delay, sleep_rebalance_migration_retry};
|
||||
use crate::data_usage::DATA_USAGE_CACHE_NAME;
|
||||
use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::{
|
||||
object::{ObjectIO, ObjectOperations as _},
|
||||
range::HTTPRangeSpec,
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_utils::path::encode_dir_object;
|
||||
use std::future::Future;
|
||||
use tokio::time::Duration;
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub(crate) struct MigrationVersionResult {
|
||||
pub moved: bool,
|
||||
pub ignored: bool,
|
||||
pub cleanup_ignored: bool,
|
||||
pub failed: bool,
|
||||
pub stage: Option<&'static str>,
|
||||
pub error: Option<Error>,
|
||||
}
|
||||
|
||||
pub(super) fn rebalance_delete_marker_opts(version: &FileInfo, version_id: Option<String>, src_pool_idx: usize) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id,
|
||||
mod_time: version.mod_time,
|
||||
src_pool_idx,
|
||||
data_movement: true,
|
||||
delete_marker: true,
|
||||
skip_decommissioned: true,
|
||||
delete_replication: version.replication_state_internal.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn rebalance_remote_tiered_opts(version: &FileInfo, version_id: Option<String>, src_pool_idx: usize) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
versioned: version_id.is_some(),
|
||||
version_id,
|
||||
mod_time: version.mod_time,
|
||||
user_defined: version.metadata.clone(),
|
||||
src_pool_idx,
|
||||
data_movement: true,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait MigrationBackend: Send + Sync {
|
||||
async fn get_object_reader_for_migration(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
h: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader>;
|
||||
|
||||
async fn delete_object_for_migration(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo>;
|
||||
|
||||
async fn move_remote_version_for_migration(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
fi: &FileInfo,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MigrationBackend for SetDisks {
|
||||
async fn get_object_reader_for_migration(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
h: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
self.get_object_reader(bucket, object, range, h, opts).await
|
||||
}
|
||||
|
||||
async fn delete_object_for_migration(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
|
||||
self.delete_object(bucket, object, opts).await
|
||||
}
|
||||
|
||||
async fn move_remote_version_for_migration(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
fi: &FileInfo,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
self.decommission_tiered_object(bucket, object, fi, opts).await
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn migrate_entry_version<Backend, F, Fut>(
|
||||
set: &Backend,
|
||||
bucket: String,
|
||||
pool_index: usize,
|
||||
version: &FileInfo,
|
||||
version_id: Option<String>,
|
||||
max_attempts: usize,
|
||||
ignore_data_usage_cache: bool,
|
||||
transfer: F,
|
||||
) -> MigrationVersionResult
|
||||
where
|
||||
Backend: MigrationBackend + ?Sized,
|
||||
F: FnMut(usize, String, GetObjectReader) -> Fut + Send,
|
||||
Fut: Future<Output = Result<()>> + Send,
|
||||
{
|
||||
migrate_entry_version_with_retry_wait(
|
||||
set,
|
||||
bucket,
|
||||
pool_index,
|
||||
version,
|
||||
version_id,
|
||||
max_attempts,
|
||||
ignore_data_usage_cache,
|
||||
transfer,
|
||||
sleep_rebalance_migration_retry,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn migrate_entry_version_with_retry_wait<Backend, F, Fut, W, WFut>(
|
||||
set: &Backend,
|
||||
bucket: String,
|
||||
pool_index: usize,
|
||||
version: &FileInfo,
|
||||
version_id: Option<String>,
|
||||
max_attempts: usize,
|
||||
ignore_data_usage_cache: bool,
|
||||
mut transfer: F,
|
||||
mut wait_retry: W,
|
||||
) -> MigrationVersionResult
|
||||
where
|
||||
Backend: MigrationBackend + ?Sized,
|
||||
F: FnMut(usize, String, GetObjectReader) -> Fut + Send,
|
||||
Fut: Future<Output = Result<()>> + Send,
|
||||
W: FnMut(Duration) -> WFut + Send,
|
||||
WFut: Future<Output = ()> + Send,
|
||||
{
|
||||
let max_attempts = max_attempts.max(1);
|
||||
|
||||
if ignore_data_usage_cache && bucket == crate::disk::RUSTFS_META_BUCKET && version.name.contains(DATA_USAGE_CACHE_NAME) {
|
||||
return MigrationVersionResult {
|
||||
moved: false,
|
||||
ignored: true,
|
||||
cleanup_ignored: false,
|
||||
failed: false,
|
||||
stage: None,
|
||||
error: None,
|
||||
};
|
||||
}
|
||||
|
||||
if version.is_remote() {
|
||||
if let Err(err) = set
|
||||
.move_remote_version_for_migration(
|
||||
&bucket,
|
||||
&version.name,
|
||||
version,
|
||||
&rebalance_remote_tiered_opts(version, version_id, pool_index),
|
||||
)
|
||||
.await
|
||||
{
|
||||
if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
|
||||
return MigrationVersionResult {
|
||||
moved: false,
|
||||
ignored: true,
|
||||
cleanup_ignored: true,
|
||||
failed: false,
|
||||
stage: Some("move_remote_version"),
|
||||
error: None,
|
||||
};
|
||||
}
|
||||
|
||||
return MigrationVersionResult {
|
||||
moved: false,
|
||||
ignored: false,
|
||||
cleanup_ignored: false,
|
||||
failed: true,
|
||||
stage: Some("move_remote_version"),
|
||||
error: Some(err),
|
||||
};
|
||||
}
|
||||
|
||||
return MigrationVersionResult {
|
||||
moved: true,
|
||||
ignored: false,
|
||||
cleanup_ignored: false,
|
||||
failed: false,
|
||||
stage: None,
|
||||
error: None,
|
||||
};
|
||||
}
|
||||
|
||||
if version.deleted {
|
||||
if let Err(err) = set
|
||||
.delete_object_for_migration(&bucket, &version.name, rebalance_delete_marker_opts(version, version_id, pool_index))
|
||||
.await
|
||||
{
|
||||
if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
|
||||
return MigrationVersionResult {
|
||||
moved: false,
|
||||
ignored: true,
|
||||
cleanup_ignored: true,
|
||||
failed: false,
|
||||
stage: Some("delete_marker"),
|
||||
error: None,
|
||||
};
|
||||
}
|
||||
|
||||
return MigrationVersionResult {
|
||||
moved: false,
|
||||
ignored: false,
|
||||
cleanup_ignored: false,
|
||||
failed: true,
|
||||
stage: Some("delete_marker"),
|
||||
error: Some(err),
|
||||
};
|
||||
}
|
||||
|
||||
return MigrationVersionResult {
|
||||
moved: true,
|
||||
ignored: false,
|
||||
cleanup_ignored: false,
|
||||
failed: false,
|
||||
stage: None,
|
||||
error: None,
|
||||
};
|
||||
}
|
||||
|
||||
let mut last_error: Option<Error> = None;
|
||||
for attempt in 0..max_attempts {
|
||||
let rd = match set
|
||||
.get_object_reader_for_migration(
|
||||
&bucket,
|
||||
&encode_dir_object(&version.name),
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: version_id.clone(),
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(rd) => rd,
|
||||
Err(err) => {
|
||||
if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
|
||||
return MigrationVersionResult {
|
||||
moved: false,
|
||||
ignored: true,
|
||||
cleanup_ignored: true,
|
||||
failed: false,
|
||||
stage: Some("read_source"),
|
||||
error: None,
|
||||
};
|
||||
}
|
||||
|
||||
last_error = Some(err);
|
||||
let Some(err) = last_error.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
if attempt + 1 >= max_attempts || !is_transient_rebalance_error(err) {
|
||||
return MigrationVersionResult {
|
||||
moved: false,
|
||||
ignored: false,
|
||||
cleanup_ignored: false,
|
||||
failed: true,
|
||||
stage: Some("read_source"),
|
||||
error: last_error,
|
||||
};
|
||||
}
|
||||
|
||||
wait_retry(rebalance_migration_retry_delay(attempt, err)).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(err) = transfer(pool_index, bucket.clone(), rd).await {
|
||||
if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
|
||||
return MigrationVersionResult {
|
||||
moved: false,
|
||||
ignored: true,
|
||||
cleanup_ignored: true,
|
||||
failed: false,
|
||||
stage: Some("write_target"),
|
||||
error: None,
|
||||
};
|
||||
}
|
||||
|
||||
last_error = Some(err);
|
||||
let Some(err) = last_error.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
if attempt + 1 >= max_attempts || !is_transient_rebalance_error(err) {
|
||||
return MigrationVersionResult {
|
||||
moved: false,
|
||||
ignored: false,
|
||||
cleanup_ignored: false,
|
||||
failed: true,
|
||||
stage: Some("write_target"),
|
||||
error: last_error,
|
||||
};
|
||||
}
|
||||
|
||||
wait_retry(rebalance_migration_retry_delay(attempt, err)).await;
|
||||
continue;
|
||||
}
|
||||
|
||||
return MigrationVersionResult {
|
||||
moved: true,
|
||||
ignored: false,
|
||||
cleanup_ignored: false,
|
||||
failed: false,
|
||||
stage: None,
|
||||
error: None,
|
||||
};
|
||||
}
|
||||
|
||||
MigrationVersionResult {
|
||||
moved: false,
|
||||
ignored: false,
|
||||
cleanup_ignored: false,
|
||||
failed: true,
|
||||
stage: Some("migrate"),
|
||||
error: last_error,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use tokio::time::Duration;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_REBALANCE: &str = "rebalance";
|
||||
const EVENT_REBALANCE_STATE: &str = "rebalance_state";
|
||||
const EVENT_REBALANCE_BUCKET: &str = "rebalance_bucket";
|
||||
const EVENT_REBALANCE_ENTRY: &str = "rebalance_entry";
|
||||
const EVENT_REBALANCE_LISTING: &str = "rebalance_listing";
|
||||
|
||||
const REBAL_META_FMT: u16 = 1; // Replace with actual format value
|
||||
const REBAL_META_VER: u16 = 1; // Replace with actual version value
|
||||
pub(crate) const REBAL_META_NAME: &str = "rebalance.bin";
|
||||
const DEFAULT_REBALANCE_MAX_ATTEMPTS: usize = 3;
|
||||
const REBALANCE_MAX_ATTEMPTS_ENV: &str = "RUSTFS_REBALANCE_MAX_ATTEMPTS";
|
||||
const REBALANCE_STOP_PROPAGATION_ERROR_PREFIX: &str = "rebalance stop propagation incomplete: ";
|
||||
const REBALANCE_LISTING_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
|
||||
const REBALANCE_MIGRATION_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
|
||||
const REBALANCE_MIGRATION_LOCK_RETRY_CAP: Duration = Duration::from_secs(10);
|
||||
const REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX: &str = "deferred transient rebalance entry failure:";
|
||||
const REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT: usize = 10;
|
||||
|
||||
mod control;
|
||||
mod entry;
|
||||
mod meta;
|
||||
mod migration;
|
||||
mod runtime;
|
||||
mod types;
|
||||
mod worker;
|
||||
|
||||
pub(crate) use meta::is_rebalance_conflicting_with_decommission;
|
||||
pub use meta::{decode_rebalance_stop_propagation_record, encode_rebalance_stop_propagation_record};
|
||||
pub use types::{
|
||||
DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta,
|
||||
RebalanceStats, RebalanceStopPropagationRecord,
|
||||
};
|
||||
use types::{RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome};
|
||||
|
||||
#[cfg(test)]
|
||||
mod rebalance_unit_tests;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,586 @@
|
||||
use super::meta::{
|
||||
apply_rebalance_save_option, apply_rebalance_terminal_event, classify_rebalance_terminal_event, clone_first_arc,
|
||||
complete_rebalance_pools_at_goal, complete_rebalance_pools_with_empty_queue, ensure_valid_rebalance_pool_index,
|
||||
has_deferred_rebalance_error, has_rebalance_cleanup_warnings, is_rebalance_in_progress, rebalance_goal_reached,
|
||||
resolve_rebalance_participants, should_preserve_rebalance_stopped_state, should_skip_start_rebalance,
|
||||
validate_start_rebalance_state,
|
||||
};
|
||||
use super::worker::{
|
||||
resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result, resolve_rebalance_save_task_result,
|
||||
resolve_rebalance_terminal_error, send_rebalance_done_signal,
|
||||
};
|
||||
use super::{
|
||||
EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, RebalSaveOpt, RebalStatus,
|
||||
RebalanceBucketOutcome,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::store::ECStore;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
impl ECStore {
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
|
||||
info!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "starting",
|
||||
"Starting rebalance"
|
||||
);
|
||||
let decommission_running = self.is_decommission_running().await;
|
||||
// let rebalance_meta = self.rebalance_meta.read().await;
|
||||
|
||||
let cancel_tx = CancellationToken::new();
|
||||
let rx = cancel_tx.clone();
|
||||
let mut meta_to_save = None;
|
||||
|
||||
{
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
validate_start_rebalance_state(decommission_running, rebalance_meta.is_some())?;
|
||||
|
||||
let Some(meta) = rebalance_meta.as_mut() else {
|
||||
return Err(Error::ConfigNotFound);
|
||||
};
|
||||
if should_skip_start_rebalance(meta.cancel.is_some(), is_rebalance_in_progress(meta)) {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "start_skipped",
|
||||
reason = "already_in_progress",
|
||||
"Skipped duplicate rebalance start"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
let now = OffsetDateTime::now_utc();
|
||||
if complete_rebalance_pools_at_goal(meta, now) {
|
||||
meta_to_save = Some(meta.clone());
|
||||
}
|
||||
if complete_rebalance_pools_with_empty_queue(meta, now) {
|
||||
meta_to_save = Some(meta.clone());
|
||||
}
|
||||
meta.cancel = Some(cancel_tx);
|
||||
|
||||
drop(rebalance_meta);
|
||||
}
|
||||
|
||||
if let Some(meta) = meta_to_save {
|
||||
let pool = clone_first_arc(self.pools.as_slice(), "start_rebalance: no pools available")?;
|
||||
resolve_rebalance_meta_save_result(
|
||||
self.save_rebalance_meta_with_merge(pool, &meta, "start_rebalance complete pools at goal")
|
||||
.await,
|
||||
"start_rebalance complete pools at goal",
|
||||
)?;
|
||||
}
|
||||
|
||||
let participants = if let Some(ref meta) = *self.rebalance_meta.read().await {
|
||||
resolve_rebalance_participants(meta.pool_stats.as_slice(), self.pools.len())
|
||||
} else {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "start_skipped",
|
||||
reason = "metadata_missing",
|
||||
"Skipped rebalance start because metadata is unavailable"
|
||||
);
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
if !participants.iter().any(|participating| *participating) {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "start_skipped",
|
||||
reason = "no_participants",
|
||||
"Skipped rebalance start because no pools are participating"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut workers_started = 0usize;
|
||||
for (idx, participating) in participants.iter().enumerate() {
|
||||
if !*participating {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index = idx,
|
||||
state = "pool_skipped",
|
||||
reason = "not_participating",
|
||||
"Skipped rebalance pool"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if !runtime_sources::endpoint_pool_is_local(idx) {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index = idx,
|
||||
state = "pool_skipped",
|
||||
reason = "not_local",
|
||||
"Skipped non-local rebalance pool"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
let pool_idx = idx;
|
||||
let store = self.clone();
|
||||
let rx_clone = rx.clone();
|
||||
workers_started += 1;
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx).await {
|
||||
error!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index = pool_idx,
|
||||
state = "pool_failed",
|
||||
error = %err,
|
||||
"Rebalance pool failed"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index = pool_idx,
|
||||
state = "completed",
|
||||
"Rebalance pool completed"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if workers_started == 0 {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "start_skipped",
|
||||
reason = "no_local_participants",
|
||||
"Skipped rebalance start because no local pools are participating"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
info!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "started",
|
||||
worker_count = workers_started,
|
||||
"Rebalance started"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, rx))]
|
||||
async fn rebalance_buckets(self: &Arc<Self>, rx: CancellationToken, pool_index: usize) -> Result<()> {
|
||||
ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?;
|
||||
|
||||
let (done_tx, mut done_rx) = tokio::sync::mpsc::channel::<Result<()>>(1);
|
||||
|
||||
// Save rebalance metadata periodically
|
||||
let store = self.clone();
|
||||
let save_task = tokio::spawn(async move {
|
||||
let mut timer = tokio::time::interval_at(Instant::now() + Duration::from_secs(30), Duration::from_secs(10));
|
||||
let mut msg: String;
|
||||
let mut quit = false;
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
result = done_rx.recv() => {
|
||||
quit = true;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let terminal_event = classify_rebalance_terminal_event(result, now);
|
||||
msg = terminal_event.message().to_string();
|
||||
let mut rebalance_meta = store.rebalance_meta.write().await;
|
||||
if let Some(meta) = rebalance_meta.as_mut() {
|
||||
let meta_stopped = meta.stopped_at.is_some();
|
||||
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
|
||||
if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
|
||||
&& has_rebalance_cleanup_warnings(pool_stat)
|
||||
{
|
||||
pool_stat.info.stopping = false;
|
||||
pool_stat.info.status = RebalStatus::Failed;
|
||||
pool_stat.info.end_time = Some(now);
|
||||
pool_stat.info.last_error = Some(
|
||||
pool_stat
|
||||
.cleanup_warnings
|
||||
.last_message
|
||||
.clone()
|
||||
.unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
|
||||
);
|
||||
} else if should_preserve_rebalance_stopped_state(
|
||||
meta_stopped,
|
||||
pool_stat.info.status,
|
||||
&terminal_event,
|
||||
) {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "stopped_preserved",
|
||||
"Preserved stopped rebalance status"
|
||||
);
|
||||
} else {
|
||||
pool_stat.info.stopping = false;
|
||||
apply_rebalance_terminal_event(
|
||||
&mut pool_stat.info.status,
|
||||
&mut pool_stat.info.end_time,
|
||||
&mut pool_stat.info.last_error,
|
||||
terminal_event,
|
||||
now,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = timer.tick() => {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
msg = format!("Saving rebalance metadata at {now:?}");
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await {
|
||||
let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
|
||||
error!("{} err: {:?}", msg, wrapped);
|
||||
if quit {
|
||||
return Err(wrapped);
|
||||
}
|
||||
} else {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "metadata_saved",
|
||||
message = %msg,
|
||||
"Saved rebalance metadata"
|
||||
);
|
||||
}
|
||||
|
||||
if quit {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "save_task_exiting",
|
||||
message = %msg,
|
||||
"Exiting rebalance save task"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
timer.reset();
|
||||
}
|
||||
});
|
||||
|
||||
info!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "pool_started",
|
||||
"Rebalance worker started"
|
||||
);
|
||||
let mut final_result: Result<()> = Ok(());
|
||||
let mut deferred_buckets = HashSet::new();
|
||||
|
||||
loop {
|
||||
if rx.is_cancelled() {
|
||||
info!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "pool_stopped",
|
||||
reason = "cancelled",
|
||||
"Stopped rebalance worker"
|
||||
);
|
||||
let err = Error::OperationCanceled;
|
||||
final_result = Err(resolve_rebalance_terminal_error(
|
||||
err.clone(),
|
||||
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
|
||||
));
|
||||
break;
|
||||
}
|
||||
|
||||
let next_bucket = match self.next_rebal_bucket(pool_index).await {
|
||||
Ok(bucket) => bucket,
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "next_bucket_failed",
|
||||
error = ?err,
|
||||
"Rebalance next bucket lookup failed"
|
||||
);
|
||||
final_result = Err(resolve_rebalance_terminal_error(
|
||||
err.clone(),
|
||||
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
|
||||
));
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(bucket) = next_bucket {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "started",
|
||||
"Starting rebalance bucket"
|
||||
);
|
||||
|
||||
let outcome = match resolve_rebalance_bucket_result(
|
||||
self.rebalance_bucket(rx.clone(), bucket.clone(), pool_index).await,
|
||||
pool_index,
|
||||
&bucket,
|
||||
) {
|
||||
Ok(outcome) => outcome,
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "bucket_failed",
|
||||
error = ?err,
|
||||
"Rebalance bucket failed"
|
||||
);
|
||||
final_result = Err(resolve_rebalance_terminal_error(
|
||||
err.clone(),
|
||||
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
|
||||
));
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
if let RebalanceBucketOutcome::Deferred { last_error } = outcome {
|
||||
if !deferred_buckets.insert(bucket.clone()) {
|
||||
let err = Error::other(format!(
|
||||
"rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}"
|
||||
));
|
||||
error!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "bucket_deferred_repeatedly",
|
||||
error = ?err,
|
||||
"Rebalance bucket failed after repeated deferral"
|
||||
);
|
||||
final_result = Err(resolve_rebalance_terminal_error(
|
||||
err.clone(),
|
||||
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
|
||||
));
|
||||
break;
|
||||
}
|
||||
|
||||
warn!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "deferred",
|
||||
error = %last_error,
|
||||
"Deferred rebalance bucket after transient object failures"
|
||||
);
|
||||
if let Err(err) = self.defer_rebalance_bucket(pool_index, bucket.clone(), last_error).await {
|
||||
error!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "defer_failed",
|
||||
error = ?err,
|
||||
"Rebalance bucket defer failed"
|
||||
);
|
||||
final_result = Err(resolve_rebalance_terminal_error(
|
||||
err.clone(),
|
||||
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
|
||||
));
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "completed",
|
||||
"Completed rebalance bucket"
|
||||
);
|
||||
if let Err(err) = self.bucket_rebalance_done(pool_index, bucket).await {
|
||||
error!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "bucket_done_mark_failed",
|
||||
error = ?err,
|
||||
"Rebalance bucket completion mark failed"
|
||||
);
|
||||
final_result = Err(resolve_rebalance_terminal_error(
|
||||
err.clone(),
|
||||
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
|
||||
));
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "idle",
|
||||
reason = "no_bucket_to_rebalance",
|
||||
"No rebalance bucket available"
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "pool_done",
|
||||
"Rebalance worker finished"
|
||||
);
|
||||
|
||||
if final_result.is_ok()
|
||||
&& let Err(err) = send_rebalance_done_signal(&done_tx, Ok(()), pool_index).await
|
||||
{
|
||||
final_result = Err(err);
|
||||
}
|
||||
drop(done_tx);
|
||||
if let Err(err) = resolve_rebalance_save_task_result(pool_index, save_task.await)
|
||||
&& final_result.is_ok()
|
||||
{
|
||||
final_result = Err(err);
|
||||
}
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "pool_result_returned",
|
||||
"Rebalance worker result returned"
|
||||
);
|
||||
final_result
|
||||
}
|
||||
|
||||
pub(super) async fn check_if_rebalance_done(&self, pool_index: usize) -> bool {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
|
||||
if let Some(meta) = rebalance_meta.as_mut()
|
||||
&& let Some(pool_stat) = meta.pool_stats.get_mut(pool_index)
|
||||
{
|
||||
// Check if the pool's rebalance status is already completed
|
||||
if pool_stat.info.status == RebalStatus::Completed {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "already_completed",
|
||||
"Rebalance pool is already completed"
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Mark pool rebalance as done only after it reaches the PercentFreeGoal.
|
||||
let pfi = if pool_stat.init_capacity == 0 {
|
||||
0.0
|
||||
} else {
|
||||
(pool_stat.init_free_space + pool_stat.bytes) as f64 / pool_stat.init_capacity as f64
|
||||
};
|
||||
|
||||
if !has_deferred_rebalance_error(pool_stat)
|
||||
&& !has_rebalance_cleanup_warnings(pool_stat)
|
||||
&& rebalance_goal_reached(
|
||||
pool_stat.init_free_space,
|
||||
pool_stat.init_capacity,
|
||||
pool_stat.bytes,
|
||||
meta.percent_free_goal,
|
||||
)
|
||||
{
|
||||
pool_stat.info.status = RebalStatus::Completed;
|
||||
pool_stat.info.end_time = Some(OffsetDateTime::now_utc());
|
||||
info!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "completed",
|
||||
percent_free = pfi,
|
||||
"Marked rebalance pool completed"
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
|
||||
let meta_to_save = {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
let Some(meta) = rebalance_meta.as_mut() else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
apply_rebalance_save_option(meta, pool_idx, opt, now);
|
||||
meta.clone()
|
||||
};
|
||||
|
||||
let pool = clone_first_arc(&self.pools, "save_rebalance_stats: no pools available")?;
|
||||
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index = pool_idx,
|
||||
save_opt = ?opt,
|
||||
state = "metadata_save_requested",
|
||||
"Rebalance metadata save requested"
|
||||
);
|
||||
let stage = format!("save_rebalance_stats for pool {pool_idx} opt {opt:?}");
|
||||
resolve_rebalance_meta_save_result(
|
||||
self.save_rebalance_meta_with_merge(pool, &meta_to_save, stage.as_str()).await,
|
||||
stage.as_str(),
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RebalanceStats {
|
||||
#[serde(rename = "ifs")]
|
||||
pub init_free_space: u64, // Pool free space at the start of rebalance
|
||||
#[serde(rename = "ic")]
|
||||
pub init_capacity: u64, // Pool capacity at the start of rebalance
|
||||
#[serde(rename = "bus")]
|
||||
pub buckets: Vec<String>, // Buckets being rebalanced or to be rebalanced
|
||||
#[serde(rename = "rbs")]
|
||||
pub rebalanced_buckets: Vec<String>, // Buckets rebalanced
|
||||
#[serde(rename = "bu")]
|
||||
pub bucket: String, // Last rebalanced bucket
|
||||
#[serde(rename = "ob")]
|
||||
pub object: String, // Last rebalanced object
|
||||
#[serde(rename = "no")]
|
||||
pub num_objects: u64, // Number of objects rebalanced
|
||||
#[serde(rename = "nv")]
|
||||
pub num_versions: u64, // Number of versions rebalanced
|
||||
#[serde(rename = "bs")]
|
||||
pub bytes: u64, // Number of bytes rebalanced
|
||||
#[serde(rename = "par")]
|
||||
pub participating: bool, // Whether the pool is participating in rebalance
|
||||
#[serde(rename = "inf")]
|
||||
pub info: RebalanceInfo, // Rebalance operation info
|
||||
#[serde(rename = "cw", default)]
|
||||
pub cleanup_warnings: RebalanceCleanupWarnings,
|
||||
}
|
||||
|
||||
pub type RStats = Vec<Arc<RebalanceStats>>;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct RebalanceBucketConfigs {
|
||||
pub(super) lifecycle_config: Option<s3s::dto::BucketLifecycleConfiguration>,
|
||||
pub(super) lock_retention: Option<s3s::dto::DefaultRetention>,
|
||||
pub(super) replication_config: Option<(s3s::dto::ReplicationConfiguration, OffsetDateTime)>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(super) enum RebalanceBucketOutcome {
|
||||
Completed,
|
||||
Deferred { last_error: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(super) enum RebalanceEntryOutcome {
|
||||
Completed,
|
||||
Deferred { last_error: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
pub enum RebalStatus {
|
||||
#[default]
|
||||
None,
|
||||
Started,
|
||||
Completed,
|
||||
Stopped,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
pub enum RebalSaveOpt {
|
||||
#[default]
|
||||
Stats,
|
||||
StoppedAt,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RebalanceInfo {
|
||||
#[serde(rename = "startTs")]
|
||||
pub start_time: Option<OffsetDateTime>, // Time at which rebalance-start was issued
|
||||
#[serde(rename = "stopTs")]
|
||||
pub end_time: Option<OffsetDateTime>, // Time at which rebalance operation completed or rebalance-stop was called
|
||||
#[serde(rename = "err")]
|
||||
pub last_error: Option<String>, // Last rebalance error message
|
||||
#[serde(rename = "status")]
|
||||
pub status: RebalStatus, // Current state of rebalance operation
|
||||
#[serde(rename = "stopping", default)]
|
||||
pub stopping: bool, // True after stop is requested and before worker terminal acknowledgement
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RebalanceCleanupWarningEntry {
|
||||
#[serde(rename = "bucket", default)]
|
||||
pub bucket: String,
|
||||
#[serde(rename = "object", default)]
|
||||
pub object: String,
|
||||
#[serde(rename = "message", default)]
|
||||
pub message: String,
|
||||
#[serde(rename = "timestamp", default)]
|
||||
pub timestamp: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RebalanceCleanupWarnings {
|
||||
#[serde(rename = "count", default)]
|
||||
pub count: u64,
|
||||
#[serde(rename = "lastMsg", default)]
|
||||
pub last_message: Option<String>,
|
||||
#[serde(rename = "lastBucket", default)]
|
||||
pub last_bucket: Option<String>,
|
||||
#[serde(rename = "lastObject", default)]
|
||||
pub last_object: Option<String>,
|
||||
#[serde(rename = "lastAt", default)]
|
||||
pub last_at: Option<OffsetDateTime>,
|
||||
#[serde(rename = "entries", default)]
|
||||
pub entries: Vec<RebalanceCleanupWarningEntry>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RebalanceStopPropagationRecord {
|
||||
#[serde(rename = "stopAttemptAt", default)]
|
||||
pub stop_attempt_at: Option<OffsetDateTime>,
|
||||
#[serde(rename = "stopFailures", default)]
|
||||
pub stop_failures: Vec<String>,
|
||||
#[serde(rename = "terminalReloadAttemptAt", default)]
|
||||
pub terminal_reload_attempt_at: Option<OffsetDateTime>,
|
||||
#[serde(rename = "terminalReloadFailures", default)]
|
||||
pub terminal_reload_failures: Vec<String>,
|
||||
}
|
||||
|
||||
impl RebalanceStopPropagationRecord {
|
||||
pub fn has_failures(&self) -> bool {
|
||||
!self.stop_failures.is_empty() || !self.terminal_reload_failures.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DiskStat {
|
||||
pub total_space: u64,
|
||||
pub available_space: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize, Clone)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RebalanceMeta {
|
||||
#[serde(skip)]
|
||||
pub cancel: Option<CancellationToken>, // To be invoked on rebalance-stop
|
||||
#[serde(skip)]
|
||||
pub last_refreshed_at: Option<OffsetDateTime>,
|
||||
#[serde(rename = "stopTs")]
|
||||
pub stopped_at: Option<OffsetDateTime>, // Time when rebalance-stop was issued
|
||||
#[serde(rename = "id")]
|
||||
pub id: String, // ID of the ongoing rebalance operation
|
||||
#[serde(rename = "pf")]
|
||||
pub percent_free_goal: f64, // Computed from total free space and capacity at the start of rebalance
|
||||
#[serde(rename = "rss")]
|
||||
pub pool_stats: Vec<RebalanceStats>, // Per-pool rebalance stats keyed by pool index
|
||||
}
|
||||
@@ -0,0 +1,547 @@
|
||||
use super::migration::MigrationVersionResult;
|
||||
use super::{
|
||||
DEFAULT_REBALANCE_MAX_ATTEMPTS, EVENT_REBALANCE_LISTING, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
|
||||
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_MAX_ATTEMPTS_ENV, REBALANCE_MIGRATION_LOCK_RETRY_CAP,
|
||||
REBALANCE_MIGRATION_RETRY_BASE_DELAY, RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome, Result,
|
||||
};
|
||||
use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
|
||||
use crate::core::pools::ListCallback;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::error::{
|
||||
Error, is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found, is_network_or_host_down,
|
||||
};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
|
||||
use rand::RngExt as _;
|
||||
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
|
||||
use std::sync::Arc;
|
||||
use tokio::time::Duration;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
pub(super) fn resolve_rebalance_worker_result<T>(
|
||||
set_idx: usize,
|
||||
worker_result: std::result::Result<Result<T>, tokio::task::JoinError>,
|
||||
) -> Result<T> {
|
||||
match worker_result {
|
||||
Ok(result) => result,
|
||||
Err(err) => Err(Error::other(format!("rebalance worker {set_idx} task join error: {err}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) type RebalanceEntryTask = tokio::task::JoinHandle<Result<RebalanceEntryOutcome>>;
|
||||
|
||||
pub(super) async fn wait_rebalance_entry_tasks(
|
||||
set_idx: usize,
|
||||
tasks: Arc<tokio::sync::Mutex<Vec<RebalanceEntryTask>>>,
|
||||
) -> Result<Option<String>> {
|
||||
let tasks = {
|
||||
let mut tasks = tasks.lock().await;
|
||||
std::mem::take(&mut *tasks)
|
||||
};
|
||||
|
||||
let mut first_error = None;
|
||||
let mut first_deferred = None;
|
||||
for task in tasks {
|
||||
match task.await {
|
||||
Ok(Ok(RebalanceEntryOutcome::Completed)) => {}
|
||||
Ok(Ok(RebalanceEntryOutcome::Deferred { last_error })) => {
|
||||
if first_deferred.is_none() {
|
||||
first_deferred = Some(last_error);
|
||||
}
|
||||
}
|
||||
Ok(Err(err)) => {
|
||||
error!("rebalance entry task failed for set {}: {}", set_idx, err);
|
||||
if first_error.is_none() {
|
||||
first_error = Some(err);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let err = Error::other(format!("rebalance entry task join error for set {set_idx}: {err}"));
|
||||
error!("{}", err);
|
||||
if first_error.is_none() {
|
||||
first_error = Some(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = first_error {
|
||||
Err(err)
|
||||
} else {
|
||||
Ok(first_deferred)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_save_task_result(
|
||||
pool_idx: usize,
|
||||
save_task_result: std::result::Result<Result<()>, tokio::task::JoinError>,
|
||||
) -> Result<()> {
|
||||
match save_task_result {
|
||||
Ok(result) => result.map_err(|err| Error::other(format!("rebalance save_task failed for pool {pool_idx}: {err}"))),
|
||||
Err(err) => Err(Error::other(format!("rebalance save_task for pool {pool_idx} join error: {err}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_meta_save_result(result: Result<()>, stage: &str) -> Result<()> {
|
||||
result.map_err(|err| Error::other(format!("rebalance meta save failed during {stage}: {err}")))
|
||||
}
|
||||
|
||||
pub(super) fn rebalance_meta_lock_error(err: rustfs_lock::LockError) -> Error {
|
||||
match err {
|
||||
rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
|
||||
object: REBAL_META_NAME.to_string(),
|
||||
required,
|
||||
achieved,
|
||||
},
|
||||
other => Error::other(format!(
|
||||
"failed to acquire rebalance metadata write lock on {}/{}: {other}",
|
||||
crate::disk::RUSTFS_META_BUCKET,
|
||||
REBAL_META_NAME
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_meta_load_result(result: Result<()>) -> Result<bool> {
|
||||
match result {
|
||||
Ok(()) => Ok(true),
|
||||
Err(Error::ConfigNotFound) => Ok(false),
|
||||
Err(err) => {
|
||||
error!("rebalanceMeta: load rebalance meta err {:?}", &err);
|
||||
Err(Error::other(format!("rebalance metadata load failed during load_rebalance_meta: {err}")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_stats_update_result(
|
||||
result: Result<()>,
|
||||
pool_idx: usize,
|
||||
bucket: &str,
|
||||
object_name: &str,
|
||||
) -> Result<()> {
|
||||
result.map_err(|err| {
|
||||
Error::other(format!(
|
||||
"rebalance stats update failed for pool {pool_idx} bucket {bucket} object {object_name}: {err}"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_file_info_versions_result<T, E>(
|
||||
result: std::result::Result<T, E>,
|
||||
bucket: &str,
|
||||
object_name: &str,
|
||||
) -> Result<T>
|
||||
where
|
||||
E: std::fmt::Display,
|
||||
{
|
||||
result.map_err(|err| Error::other(format!("rebalance file_info_versions failed for {bucket}/{object_name}: {err}")))
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_entry_cleanup_delete_result(
|
||||
result: Result<crate::object_api::ObjectInfo>,
|
||||
bucket: &str,
|
||||
object_name: &str,
|
||||
) -> Result<Option<String>> {
|
||||
match result {
|
||||
Ok(_) => Ok(None),
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
|
||||
Err(err) => Ok(Some(format!("rebalance cleanup delete failed for {bucket}/{object_name}: {err}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_migrate_result_error(
|
||||
err: Option<Error>,
|
||||
pool_idx: usize,
|
||||
bucket: &str,
|
||||
object_name: &str,
|
||||
version_id: Option<&str>,
|
||||
) -> Error {
|
||||
err.unwrap_or_else(|| {
|
||||
Error::other(format!(
|
||||
"rebalance migration reported failure without error for pool {pool_idx} entry {bucket}/{object_name} version {}",
|
||||
version_id.unwrap_or("none")
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn should_defer_rebalance_entry_failure(err: &Error) -> bool {
|
||||
is_transient_rebalance_error(err)
|
||||
}
|
||||
|
||||
pub(super) fn resolve_load_rebalance_stats_update_result(result: Result<()>) -> Result<()> {
|
||||
result.map_err(|err| Error::other(format!("rebalance metadata stats refresh failed after load: {err}")))
|
||||
}
|
||||
|
||||
pub(super) async fn send_rebalance_done_signal(
|
||||
done_tx: &tokio::sync::mpsc::Sender<Result<()>>,
|
||||
signal: Result<()>,
|
||||
pool_idx: usize,
|
||||
) -> Result<()> {
|
||||
done_tx
|
||||
.send(signal)
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("rebalance done signal send failed for pool {pool_idx}: {err}")))
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_terminal_error(primary_err: Error, signal_result: Result<()>) -> Error {
|
||||
match signal_result {
|
||||
Ok(()) => primary_err,
|
||||
Err(signal_err) => Error::other(format!("rebalance terminal signal failed after error {primary_err}: {signal_err}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_bucket_error(entry_error: Option<Error>, worker_error: Option<Error>) -> Result<()> {
|
||||
if let Some(err) = entry_error {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if let Some(err) = worker_error {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_bucket_result(
|
||||
result: Result<RebalanceBucketOutcome>,
|
||||
pool_idx: usize,
|
||||
bucket: &str,
|
||||
) -> Result<RebalanceBucketOutcome> {
|
||||
match result {
|
||||
Ok(outcome) => Ok(outcome),
|
||||
Err(err) if is_err_operation_canceled(&err) => Err(err),
|
||||
Err(err) => Err(Error::other(format!("rebalance bucket {bucket} failed for pool {pool_idx}: {err}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_transient_rebalance_error(err: &Error) -> bool {
|
||||
match err {
|
||||
Error::SlowDown
|
||||
| Error::ErasureReadQuorum
|
||||
| Error::ErasureWriteQuorum
|
||||
| Error::InsufficientReadQuorum(_, _)
|
||||
| Error::InsufficientWriteQuorum(_, _) => true,
|
||||
Error::Lock(lock_err) => is_rebalance_transient_lock_error(lock_err),
|
||||
Error::Io(io_err) => is_rebalance_transient_io_error(io_err) || is_rebalance_transient_message(&io_err.to_string()),
|
||||
_ => is_rebalance_transient_message(&err.to_string()) || is_network_or_host_down(&err.to_string(), true),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
|
||||
match err {
|
||||
rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::Network { .. } => true,
|
||||
rustfs_lock::LockError::Internal { message } => is_rebalance_transient_message(message),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
|
||||
if err.kind() == std::io::ErrorKind::TimedOut {
|
||||
return true;
|
||||
}
|
||||
|
||||
if let Some(disk_err) = err.get_ref().and_then(|err| err.downcast_ref::<DiskError>())
|
||||
&& *disk_err == DiskError::Timeout
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
let message = err.to_string();
|
||||
message.eq_ignore_ascii_case("timeout") || is_rebalance_transient_message(&message)
|
||||
}
|
||||
|
||||
fn is_rebalance_transient_message(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
message.contains("lock acquisition timed out")
|
||||
|| message.contains("remote lock rpc timed out")
|
||||
|| message.contains("keepalivetimedout")
|
||||
|| message.contains("i/o timeout")
|
||||
|| message.contains("operation timed out")
|
||||
}
|
||||
|
||||
pub(super) fn should_retry_rebalance_listing(err: &Error, attempt: usize, max_attempts: usize) -> bool {
|
||||
attempt + 1 < max_attempts && is_transient_rebalance_error(err)
|
||||
}
|
||||
|
||||
pub(super) fn parse_rebalance_max_attempts(value: Option<&str>) -> usize {
|
||||
value
|
||||
.and_then(|value| value.trim().parse::<usize>().ok())
|
||||
.filter(|attempts| *attempts > 0)
|
||||
.unwrap_or(DEFAULT_REBALANCE_MAX_ATTEMPTS)
|
||||
}
|
||||
|
||||
pub(super) fn rebalance_max_attempts() -> usize {
|
||||
parse_rebalance_max_attempts(std::env::var(REBALANCE_MAX_ATTEMPTS_ENV).ok().as_deref())
|
||||
}
|
||||
|
||||
pub(super) fn rebalance_listing_retry_delay(attempt: usize) -> Duration {
|
||||
let multiplier = u32::try_from(attempt.saturating_add(1)).unwrap_or(u32::MAX);
|
||||
REBALANCE_LISTING_RETRY_BASE_DELAY.saturating_mul(multiplier)
|
||||
}
|
||||
|
||||
fn is_rebalance_lock_or_rpc_timeout(err: &Error) -> bool {
|
||||
match err {
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { .. }) | Error::Lock(rustfs_lock::LockError::Network { .. }) => true,
|
||||
Error::Io(io_err) => is_rebalance_lock_or_rpc_timeout_message(&io_err.to_string()),
|
||||
_ => is_rebalance_lock_or_rpc_timeout_message(&err.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_rebalance_lock_or_rpc_timeout_message(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
message.contains("lock acquisition timed out")
|
||||
|| message.contains("remote lock rpc timed out")
|
||||
|| message.contains("keepalivetimedout")
|
||||
}
|
||||
|
||||
pub(super) fn rebalance_migration_retry_delay(attempt: usize, err: &Error) -> Duration {
|
||||
if is_rebalance_lock_or_rpc_timeout(err) {
|
||||
return rebalance_lock_retry_delay(attempt);
|
||||
}
|
||||
|
||||
let multiplier = u32::try_from(attempt.saturating_add(1)).unwrap_or(u32::MAX);
|
||||
REBALANCE_MIGRATION_RETRY_BASE_DELAY.saturating_mul(multiplier)
|
||||
}
|
||||
|
||||
fn rebalance_lock_retry_delay(attempt: usize) -> Duration {
|
||||
let lock_timeout = get_lock_acquire_timeout();
|
||||
let attempt_shift = u32::try_from(attempt.min(4)).unwrap_or(4);
|
||||
let multiplier = 1_u32.checked_shl(attempt_shift).unwrap_or(u32::MAX);
|
||||
let cap = lock_timeout
|
||||
.saturating_mul(multiplier)
|
||||
.min(REBALANCE_MIGRATION_LOCK_RETRY_CAP)
|
||||
.max(REBALANCE_MIGRATION_RETRY_BASE_DELAY);
|
||||
let max_millis = u64::try_from(cap.as_millis()).unwrap_or(u64::MAX).max(1);
|
||||
let jitter_millis = rand::rng().random_range(1..=max_millis);
|
||||
Duration::from_millis(jitter_millis)
|
||||
}
|
||||
|
||||
pub(super) async fn sleep_rebalance_migration_retry(delay: Duration) {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
|
||||
pub(super) async fn wait_rebalance_listing_retry(rx: &CancellationToken, delay: Duration) -> Result<()> {
|
||||
tokio::select! {
|
||||
_ = rx.cancelled() => Err(Error::OperationCanceled),
|
||||
_ = tokio::time::sleep(delay) => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn ensure_rebalance_listing_disks_available(has_disks: bool, bucket: &str) -> Result<()> {
|
||||
if !has_disks {
|
||||
return Err(Error::other(format!(
|
||||
"failed to list objects to rebalance for bucket {bucket}: no disks available"
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn with_rebalance_entry_context(stage: &str, bucket: &str, object_name: &str, err: Error) -> Error {
|
||||
Error::other(format!("rebalance entry {stage} failed for {bucket}/{object_name}: {err}"))
|
||||
}
|
||||
|
||||
pub(super) fn should_count_rebalance_version_complete(result: &MigrationVersionResult) -> bool {
|
||||
result.cleanup_ignored || (result.moved && !result.failed)
|
||||
}
|
||||
|
||||
pub(super) fn should_cleanup_rebalance_source_entry(rebalanced: usize, total_versions: usize) -> bool {
|
||||
rebalanced == total_versions
|
||||
}
|
||||
|
||||
pub(super) fn should_skip_rebalance_delete_marker(
|
||||
version: &rustfs_filemeta::FileInfo,
|
||||
remaining_versions: usize,
|
||||
replication_configured: bool,
|
||||
) -> bool {
|
||||
version.deleted && remaining_versions == 1 && !replication_configured
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_optional_bucket_config_result<T>(
|
||||
bucket: &str,
|
||||
stage: &str,
|
||||
result: Result<T>,
|
||||
) -> Result<Option<T>> {
|
||||
match result {
|
||||
Ok(config) => Ok(Some(config)),
|
||||
Err(Error::ConfigNotFound) => Ok(None),
|
||||
Err(err) => Err(Error::other(format!("rebalance {stage} config load failed for bucket {bucket}: {err}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn load_rebalance_bucket_configs(bucket: &str) -> Result<RebalanceBucketConfigs> {
|
||||
if bucket == crate::disk::RUSTFS_META_BUCKET {
|
||||
return Ok(RebalanceBucketConfigs::default());
|
||||
}
|
||||
|
||||
let _ = resolve_rebalance_optional_bucket_config_result(
|
||||
bucket,
|
||||
"versioning",
|
||||
crate::bucket::versioning_sys::BucketVersioningSys::get(bucket).await,
|
||||
)?;
|
||||
|
||||
Ok(RebalanceBucketConfigs {
|
||||
lifecycle_config: runtime_sources::bucket_lifecycle_config(bucket).await,
|
||||
lock_retention: crate::bucket::object_lock::objectlock_sys::BucketObjectLockSys::get(bucket).await,
|
||||
replication_config: resolve_rebalance_optional_bucket_config_result(
|
||||
bucket,
|
||||
"replication",
|
||||
crate::bucket::metadata_sys::get_replication_config(bucket).await,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn run_rebalance_listing_with_retry(
|
||||
set: Arc<SetDisks>,
|
||||
rx: CancellationToken,
|
||||
bucket: String,
|
||||
cb: ListCallback,
|
||||
set_idx: usize,
|
||||
max_attempts: usize,
|
||||
) -> Result<()> {
|
||||
let max_attempts = max_attempts.max(1);
|
||||
let mut last_error = None;
|
||||
|
||||
for attempt in 0..max_attempts {
|
||||
match set.list_objects_to_rebalance(rx.clone(), bucket.clone(), cb.clone()).await {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(err) if should_retry_rebalance_listing(&err, attempt, max_attempts) => {
|
||||
let next_attempt = attempt + 2;
|
||||
let delay = rebalance_listing_retry_delay(attempt);
|
||||
error!(
|
||||
"rebalance listing failed for bucket {} set {} attempt {}/{}: {}; retrying in {:?}",
|
||||
bucket,
|
||||
set_idx,
|
||||
attempt + 1,
|
||||
max_attempts,
|
||||
err,
|
||||
delay
|
||||
);
|
||||
last_error = Some(err);
|
||||
wait_rebalance_listing_retry(&rx, delay).await?;
|
||||
info!(
|
||||
"rebalance listing retrying bucket {} set {} attempt {}/{}",
|
||||
bucket, set_idx, next_attempt, max_attempts
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(Error::other(format!(
|
||||
"rebalance listing failed for bucket {bucket} set {set_idx} attempt {}/{}: {err}",
|
||||
attempt + 1,
|
||||
max_attempts
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::other(format!(
|
||||
"rebalance listing failed for bucket {bucket} set {set_idx} after {max_attempts} attempts: {}",
|
||||
last_error
|
||||
.map(|err| err.to_string())
|
||||
.unwrap_or_else(|| "unknown listing failure".to_string())
|
||||
)))
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
#[tracing::instrument(skip(self, rx, cb))]
|
||||
pub async fn list_objects_to_rebalance(
|
||||
self: &Arc<Self>,
|
||||
rx: CancellationToken,
|
||||
bucket: String,
|
||||
cb: ListCallback,
|
||||
) -> Result<()> {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
bucket = %bucket,
|
||||
state = "started",
|
||||
"Rebalance listing started"
|
||||
);
|
||||
let (disks, _) = self.get_online_disks_with_healing(false).await;
|
||||
ensure_rebalance_listing_disks_available(!disks.is_empty(), &bucket)?;
|
||||
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
bucket = %bucket,
|
||||
disk_count = disks.len(),
|
||||
state = "disks_resolved",
|
||||
"Rebalance listing disks resolved"
|
||||
);
|
||||
let listing_quorum = self.set_drive_count.div_ceil(2);
|
||||
|
||||
let resolver = MetadataResolutionParams {
|
||||
dir_quorum: listing_quorum,
|
||||
obj_quorum: listing_quorum,
|
||||
bucket: bucket.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let cb1 = cb.clone();
|
||||
list_path_raw(
|
||||
rx,
|
||||
ListPathRawOptions {
|
||||
disks: disks.iter().cloned().map(Some).collect(),
|
||||
bucket: bucket.clone(),
|
||||
recursive: true,
|
||||
min_disks: listing_quorum,
|
||||
skip_walkdir_total_timeout: true,
|
||||
agreed: Some(Box::new(move |entry: MetaCacheEntry| {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
entry = %entry.name,
|
||||
state = "agreed_entry",
|
||||
"Rebalance listing agreed entry"
|
||||
);
|
||||
Box::pin(cb1(entry))
|
||||
})),
|
||||
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
|
||||
let resolver = resolver.clone();
|
||||
let cb = cb.clone();
|
||||
|
||||
match entries.resolve(resolver) {
|
||||
Some(entry) => {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
entry = %entry.name,
|
||||
state = "resolved_partial_entry",
|
||||
"Rebalance listing resolved partial entry"
|
||||
);
|
||||
Box::pin(async move { cb(entry).await })
|
||||
}
|
||||
None => {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "partial_entry_missing",
|
||||
"Rebalance listing partial entry missing"
|
||||
);
|
||||
Box::pin(async {})
|
||||
}
|
||||
}
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
bucket = %bucket,
|
||||
state = "completed",
|
||||
"Rebalance listing completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user