gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 05:26:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): fence system metadata publication (#6444)

Browse Source 
* feat(scanner): fence usage publication during data movement

* fix(scanner): detect movement refresh state changes

* fix(scanner): fence publication during data movement

* fix(scanner): close movement epoch publication races

* fix(scanner): fence movement-sensitive publication paths

* fix(scanner): fence cache and heal recovery paths

* fix(scanner): carry publication epoch through scan cycle

* fix(scanner): recheck remote cache epoch after save

* fix(scanner): recheck local cache epoch before publish

* fix(scanner): fence data usage writers and baseline

* fix(scanner): expose decommission activity to publication fence

* fix(scanner): release publication gate before reads

* fix(scanner): complete publication fence integration

* fix(scanner): avoid empty usage baseline publication

* chore(scanner): gate test-only helpers

* fix: use decommission canceler in reload test

---------

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
2026-08-23 17:28:21 +08:00
committed by GitHub
parent 9cda615519
commit e196a134cc
26 changed files with 2465 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/core/pools.rs
+155 -33
View File
@@ -135,7 +135,7 @@ struct DecommissionOperation {
}
impl DecommissionCanceler {
fn new(token: CancellationToken) -> Self {
pub(crate) fn new(token: CancellationToken) -> Self {
Self {
operation: Arc::new(DecommissionOperation {
token,
@@ -153,7 +153,7 @@ impl DecommissionCanceler {
&self.operation.token
}
fn is_active(&self) -> bool {
pub(crate) fn is_active(&self) -> bool {
self.operation.active.load(Ordering::Acquire)
}
@@ -1444,6 +1444,30 @@ fn should_replace_pool_status_for_status_refresh(
!has_active_worker && persisted.last_update > current.last_update
}
fn pool_decommission_movement_snapshot(
info: Option<&PoolDecommissionInfo>,
) -> (bool, bool, bool, bool, bool, Option<OffsetDateTime>) {
info.map(|info| {
(
info.has_decommission_state(),
info.complete,
info.failed,
info.canceled,
info.queued,
info.start_time,
)
})
.unwrap_or_default()
}
pub(crate) fn pool_meta_movement_snapshot_changed(before: &PoolMeta, after: &PoolMeta) -> bool {
before.pools.len() != after.pools.len()
|| before.pools.iter().zip(after.pools.iter()).any(|(before, after)| {
pool_decommission_movement_snapshot(before.decommission.as_ref())
!= pool_decommission_movement_snapshot(after.decommission.as_ref())
})
}
/// Merges a persisted pool metadata snapshot into `current` monotonically:
/// a pool entry is replaced only when no active worker covers it and the
/// snapshot is strictly newer, so delayed snapshots never roll back local
@@ -3358,6 +3382,8 @@ impl ECStore {
.first()
.cloned()
.ok_or_else(|| Error::other("refresh_pool_status_meta: no pools available"))?;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let mut persisted = PoolMeta::default();
persisted.load(pool, self.pools.clone()).await?;
@@ -3370,7 +3396,9 @@ impl ECStore {
};
let mut pool_meta = self.pool_meta.write().await;
merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers);
if merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers) {
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
}
@@ -3489,11 +3517,19 @@ impl ECStore {
async fn decommission_cancel_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> {
ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?;
let _start_guard = self.start_gate.lock().await;
let operation_gate = self.ctx.decommission_operation_gate();
let operation_guard = operation_gate.write().await;
// Lock order: decommission_cancelers before pool_meta. Holding both makes
// owner validation and the terminal transition one atomic operation.
// Signal cancellation before waiting for the movement writer. A worker
// may still hold a publication read guard while it observes this
// signal; waiting for the writer first would deadlock that handoff.
if let Some(owner) = owner {
owner.cancel();
} else if let Some(canceler) = self.decommission_cancelers.read().await.get(idx).and_then(Option::as_ref) {
canceler.cancel();
}
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
// Lock order: movement gate, then decommission_cancelers, then pool_meta.
// Holding both state locks makes owner validation and the terminal
// transition one atomic operation.
let (should_save_pool_meta, should_reload_pool_meta, already_canceled, previous_pool_meta, terminal_canceler) = {
let cancelers = self.decommission_cancelers.read().await;
let mut lock = self.pool_meta.write().await;
@@ -3562,9 +3598,26 @@ impl ECStore {
self.release_decommission_canceler_slot(idx, canceler).await;
}
if should_save_pool_meta {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("decommission_cancel for pool {idx}");
resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?;
if let Err(err) =
resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
{
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "terminal_reload_failed",
error = %err,
"Decommission cancel saved locally but pool meta reload failed"
);
}
}
Ok(())
@@ -3589,7 +3642,11 @@ impl ECStore {
.unwrap_or((false, false, false, false));
ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled)?;
}
self.cancel_decommission_routines_and_wait(&[idx]).await;
// Cancel workers before waiting for the movement writer so active
// object operations can observe the signal and release read guards.
self.cancel_decommission_routines(&[idx]).await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let (should_reload_pool_meta, previous_pool_meta) = {
let mut pool_meta = self.pool_meta.write().await;
@@ -3606,17 +3663,37 @@ impl ECStore {
return Err(err);
}
if should_reload_pool_meta {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("clear_decommission for pool {idx}");
resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?;
if let Err(err) =
resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
{
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "terminal_reload_failed",
error = %err,
"Decommission clear saved locally but pool meta reload failed"
);
}
}
Ok(())
}
async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result<OffsetDateTime> {
// Serialize promotion and generation capture with clear/restart transitions.
let (changed, generation, save_error) = {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let mut pool_meta = self.pool_meta.write().await;
if pool_meta.pools.get(idx).is_none() {
return Err(Error::other("failed to start decommission: target pool was not found"));
@@ -3644,6 +3721,9 @@ impl ECStore {
return Err(err);
}
if changed {
self.ctx.advance_data_movement_operation_epoch();
}
if changed && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("promote_queued_decommission for pool {idx}");
if let Err(err) =
@@ -3668,6 +3748,8 @@ impl ECStore {
}
async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> {
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let changed = {
let mut pool_meta = self.pool_meta.write().await;
pool_meta.record_decommission_terminal_reload_failure(idx, stage, err.to_string())?
@@ -3708,19 +3790,20 @@ impl ECStore {
is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx))
}
#[cfg(test)]
async fn cancel_decommission_routines_and_wait(&self, indices: &[usize]) {
self.cancel_decommission_routines(indices).await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
}
async fn cancel_decommission_routines(&self, indices: &[usize]) {
{
let mut cancelers = self.decommission_cancelers.write().await;
for idx in indices {
take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), *idx);
}
}
self.wait_for_decommission_side_effects().await;
}
async fn wait_for_decommission_side_effects(&self) {
let operation_gate = self.ctx.decommission_operation_gate();
let _operation_guard = operation_gate.write().await;
}
async fn reserve_decommission_routines(
@@ -4218,7 +4301,7 @@ impl ECStore {
}
decommission_cancel_signal_result(rx.is_cancelled())?;
self.ensure_decommission_generation_current(idx, generation).await?;
let operation_gate = self.ctx.decommission_operation_gate();
let operation_gate = self.ctx.data_movement_operation_gate();
let bucket_incarnation_fence = match expected_bucket_incarnation_id {
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
@@ -5113,9 +5196,12 @@ impl ECStore {
{
ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?;
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
// Lock order: decommission_cancelers before pool_meta. Holding both makes
// owner validation and the terminal transition one atomic operation.
// Lock order: movement gate, then decommission_cancelers, then pool_meta.
// Holding both state locks makes owner validation and the terminal
// transition one atomic operation.
let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = {
let cancelers = self.decommission_cancelers.read().await;
let mut pool_meta = self.pool_meta.write().await;
@@ -5151,6 +5237,12 @@ impl ECStore {
if let Some(canceler) = terminal_canceler.as_ref() {
self.release_decommission_canceler_slot(idx, canceler).await;
}
if should_reload_pool_meta {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("decommission_failed for pool {idx}");
if let Some(err) = observe_decommission_terminal_reload_result(
@@ -5208,7 +5300,12 @@ impl ECStore {
}
self.verify_decommission_durable_ilm_receipts(idx).await?;
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
// Lock order: movement gate, then decommission_cancelers, then pool_meta.
// Holding both state locks makes owner validation and the terminal
// transition one atomic operation.
let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = {
let cancelers = self.decommission_cancelers.read().await;
let mut pool_meta = self.pool_meta.write().await;
@@ -5249,6 +5346,12 @@ impl ECStore {
if let Some(canceler) = terminal_canceler.as_ref() {
self.release_decommission_canceler_slot(idx, canceler).await;
}
if should_reload_pool_meta {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("complete_decommission for pool {idx}");
if let Some(err) = observe_decommission_terminal_reload_result(
@@ -5479,11 +5582,16 @@ impl ECStore {
self.ensure_decommission_rebalance_idle_after_refresh().await?;
let all_space_infos = self.get_decommission_all_pool_space_infos().await?;
self.cancel_decommission_routines_and_wait(&indices).await;
// Signal cancellation before waiting for the movement writer so active
// object operations can observe the signal and release read guards.
self.cancel_decommission_routines(&indices).await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let index_cancelers = if let Some((rx, local_indices)) = reservation {
// Lock order matches terminal transitions: decommission_cancelers
// before pool_meta while start_gate excludes another start.
// Lock order matches terminal transitions: movement gate, then
// decommission_cancelers, then pool_meta while start_gate excludes
// another start.
let mut cancelers = self.decommission_cancelers.write().await;
let pool_meta = self.pool_meta.read().await;
ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_space_infos)?;
@@ -5505,6 +5613,10 @@ impl ECStore {
let previous_pool_meta = self
.save_current_pool_meta_for_decommission_start(&indices, space_infos, decom_buckets)
.await?;
self.ctx.advance_data_movement_operation_epoch();
// The local durable transition is now fenced. Release the writer
// before any peer RPC; remote reload must not block scanner admission.
drop(_movement_guard);
if let Some(notification_sys) = runtime_sources::notification_sys()
&& let Err(err) = resolve_start_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await)
@@ -5519,11 +5631,20 @@ impl ECStore {
"Decommission start failed after pool metadata save"
);
{
let mut pool_meta = self.pool_meta.write().await;
rollback_start_decommission_pool_meta(&mut pool_meta, previous_pool_meta.clone());
}
if let Err(rollback_save_err) = self.save_current_pool_meta().await {
let rollback_result = {
let movement_guard = movement_gate.write().await;
{
let mut pool_meta = self.pool_meta.write().await;
rollback_start_decommission_pool_meta(&mut pool_meta, previous_pool_meta.clone());
}
let rollback_result = self.save_current_pool_meta().await;
if rollback_result.is_ok() {
self.ctx.advance_data_movement_operation_epoch();
}
drop(movement_guard);
rollback_result
};
if let Err(rollback_save_err) = rollback_result {
error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
@@ -6527,7 +6648,7 @@ impl ECStore {
generation: OffsetDateTime,
) -> Result<()> {
self.ensure_decommission_generation_current(idx, generation).await?;
let operation_gate = self.ctx.decommission_operation_gate();
let operation_gate = self.ctx.data_movement_operation_gate();
run_decommission_side_effect(rx, &operation_gate, || self.check_after_decommission_unfenced(idx)).await
}
@@ -8077,7 +8198,7 @@ mod pools_tests {
..Default::default()
};
merge_pool_status_refresh(&mut current, persisted, &[false]);
assert!(merge_pool_status_refresh(&mut current, persisted, &[false]));
let info = current.pools[0]
.decommission
@@ -8121,7 +8242,7 @@ mod pools_tests {
..Default::default()
};
merge_pool_status_refresh(&mut current, persisted, &[true]);
assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
let info = current.pools[0]
.decommission
@@ -8162,7 +8283,7 @@ mod pools_tests {
..Default::default()
};
merge_pool_status_refresh(&mut current, persisted, &[true]);
assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
let info = current.pools[0]
.decommission
@@ -8591,7 +8712,7 @@ mod pools_tests {
#[tokio::test]
async fn test_decommission_transition_waits_without_registered_canceler() {
let store = decommission_worker_test_store(PoolMeta::default(), vec![None]);
let operation_gate = store.ctx.decommission_operation_gate();
let operation_gate = store.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read().await;
let transition = tokio::spawn({
let store = store.clone();
@@ -11338,6 +11459,7 @@ mod pools_tests {
assert!(store.decommission_cancelers.read().await[0].is_none());
assert!(!canceler.is_active());
assert!(canceler.is_cancelled());
assert_eq!(store.ctx.data_movement_operation_epoch(), 1);
}
#[test]
crates/ecstore/src/data_usage/mod.rs
+260 -30
View File
@@ -388,8 +388,12 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
"nonconverged data usage observations cannot replace the quota-authoritative snapshot",
));
}
let Some(expected_publication_epoch) = store.scanner_data_usage_publication_epoch().await else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
// Prevent older data from overwriting newer persisted stats
if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
let existing_snapshot = load_data_usage_snapshot(store.clone()).await;
if let Ok((existing, source)) = existing_snapshot
&& source.is_authoritative()
&& let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now())
{
@@ -400,19 +404,31 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
return Ok(());
}
save_data_usage_in_backend(data_usage_info, store).await
save_data_usage_in_backend(data_usage_info, store, expected_publication_epoch).await
}
async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
async fn save_data_usage_in_backend(
data_usage_info: DataUsageInfo,
store: Arc<ECStore>,
expected_publication_epoch: u64,
) -> Result<(), Error> {
let data =
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
// Save to backend using the same mechanism as original code
let Some((publication_guard, publication_epoch)) = store.scanner_data_usage_publication_admission_guard().await else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
if publication_epoch != expected_publication_epoch {
return Err(Error::other("data usage publication epoch changed before save"));
}
crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
.await
.map_err(Error::other)?;
drop(publication_guard);
cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await;
cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref()))
.await;
// Invalidate the cached snapshot so readers observe the new save on their
// next request instead of waiting out the remaining TTL. The next cached
@@ -449,11 +465,24 @@ impl ObservedDataUsageSnapshotCleanup for ECStore {
}
}
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
where
async fn cleanup_observed_data_usage_after_authoritative_save_with_publication<S>(
store: &S,
authoritative: &DataUsageInfo,
publication_store: Option<&ECStore>,
) where
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
{
let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
let observed_read_epoch = match publication_store {
Some(publication_store) => {
let Some(epoch) = publication_store.scanner_data_usage_publication_epoch().await else {
return;
};
Some(epoch)
}
None => None,
};
let observed_snapshot = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await;
let (observed, revision) = match observed_snapshot {
Ok(Some(snapshot)) => snapshot,
Ok(None) => return,
Err(err) => {
@@ -469,6 +498,19 @@ where
return;
}
let publication_guard = match publication_store {
Some(publication_store) => {
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
else {
return;
};
if observed_read_epoch.is_some_and(|expected| expected != publication_epoch) {
return;
}
Some(guard)
}
None => None,
};
match store.delete_observed_data_usage_snapshot(&revision).await {
Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {}
Err(err) => {
@@ -479,6 +521,15 @@ where
);
}
}
drop(publication_guard);
}
#[cfg(test)]
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
where
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
{
cleanup_observed_data_usage_after_authoritative_save_with_publication(store, authoritative, None).await;
}
fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) {
@@ -519,7 +570,7 @@ pub async fn remove_bucket_usage_from_backend(store: Arc<ECStore>, bucket: &str)
pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> {
prepare_bucket_usage_for_namespace_change(bucket, None).await?;
remove_bucket_usage_from_backend_with_guard(store, bucket, None).await
remove_bucket_usage_from_backend_with_guard_fenced(store, bucket, None).await
}
pub(crate) async fn prepare_bucket_usage_for_namespace_change(
@@ -542,6 +593,7 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
Ok(())
}
#[cfg(test)]
pub(crate) async fn remove_bucket_usage_from_backend_with_guard<S>(
store: &S,
bucket: &str,
@@ -551,6 +603,24 @@ where
S: EcstoreObjectIO + ?Sized,
{
let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await;
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
result
}
pub(crate) async fn remove_bucket_usage_from_backend_with_guard_fenced(
store: &ECStore,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error> {
let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, Some(store)).await;
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
result
}
async fn invalidate_bucket_usage_snapshot_caches(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
) -> Result<(), Error> {
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
clear_data_usage_snapshot_cache(&mut snapshot_cache);
@@ -558,7 +628,7 @@ where
let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?;
clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
result
Ok(())
}
async fn load_data_usage_for_bucket_removal<S>(store: &S, object: &str) -> Result<Option<(DataUsageInfo, String)>, Error>
@@ -617,48 +687,83 @@ fn ensure_bucket_namespace_guard(
Ok(())
}
#[cfg(test)]
async fn remove_bucket_usage_from_backend_with_store_and_guard<S>(
store: &S,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, None).await
}
async fn remove_bucket_usage_from_backend_with_store_and_guard_and_publication<S>(
store: &S,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
publication_store: Option<&ECStore>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
ensure_bucket_namespace_guard(guard, bucket, "data usage primary cleanup")?;
let primary_seed_epoch = match publication_store {
Some(publication_store) => Some(
publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
),
None => None,
};
let primary_seed = load_data_usage_seed_for_missing_primary(store).await?;
remove_bucket_usage_from_object_with_retries(
remove_bucket_usage_from_object_with_retries_and_publication(
store,
DATA_USAGE_OBJ_NAME_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
Some(&primary_seed),
primary_seed.as_ref(),
guard,
publication_store.map(|store| (store, primary_seed_epoch)),
)
.await?;
ensure_bucket_namespace_guard(guard, bucket, "data usage backup cleanup")?;
let backup_seed_epoch = match publication_store {
Some(publication_store) => Some(
publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
),
None => None,
};
let backup_seed = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
.await?
.map_or(primary_seed, |(data_usage_info, _)| data_usage_info);
remove_bucket_usage_from_object_with_retries(
.map(|(data_usage_info, _)| data_usage_info)
.or_else(|| primary_seed.clone());
remove_bucket_usage_from_object_with_retries_and_publication(
store,
DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
Some(&backup_seed),
backup_seed.as_ref(),
guard,
publication_store.map(|store| (store, backup_seed_epoch)),
)
.await?;
ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?;
if let Err(err) = remove_bucket_usage_from_object_with_retries(
if let Err(err) = remove_bucket_usage_from_object_with_retries_and_publication(
store,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
None,
guard,
publication_store.map(|store| (store, None)),
)
.await
{
@@ -672,12 +777,21 @@ where
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
] {
remove_bucket_usage_from_object_with_retries(store, object, bucket, DATA_USAGE_REMOVE_CAS_RETRIES, None, guard).await?;
remove_bucket_usage_from_object_with_retries_and_publication(
store,
object,
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
None,
guard,
publication_store.map(|store| (store, None)),
)
.await?;
}
Ok(())
}
async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<DataUsageInfo, Error>
async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<Option<DataUsageInfo>, Error>
where
S: EcstoreObjectIO + ?Sized,
{
@@ -690,12 +804,13 @@ where
if !authoritative {
data_usage_info.usage_snapshot_complete = false;
}
return Ok(data_usage_info);
return Ok(Some(data_usage_info));
}
}
Ok(DataUsageInfo::default())
Ok(None)
}
#[cfg(test)]
async fn remove_bucket_usage_from_object_with_retries<S>(
store: &S,
object: &str,
@@ -704,12 +819,42 @@ async fn remove_bucket_usage_from_object_with_retries<S>(
missing_seed: Option<&DataUsageInfo>,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
remove_bucket_usage_from_object_with_retries_and_publication(store, object, bucket, cas_retries, missing_seed, guard, None)
.await
}
async fn remove_bucket_usage_from_object_with_retries_and_publication<S>(
store: &S,
object: &str,
bucket: &str,
cas_retries: usize,
missing_seed: Option<&DataUsageInfo>,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
publication: Option<(&ECStore, Option<u64>)>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
for attempt in 0..=cas_retries {
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cleanup")?;
let (mut data_usage_info, revision) = match load_data_usage_for_bucket_removal(store, object).await? {
let read_epoch = match publication {
Some((publication_store, expected_publication_epoch)) => {
let epoch = publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?;
if expected_publication_epoch.is_some_and(|expected| expected != epoch) {
return Err(Error::other("data usage publication epoch changed before snapshot read"));
}
Some(epoch)
}
None => None,
};
let loaded_snapshot = load_data_usage_for_bucket_removal(store, object).await?;
let (mut data_usage_info, revision) = match loaded_snapshot {
Some((data_usage_info, revision)) => (data_usage_info, Some(revision)),
None => match missing_seed {
Some(data_usage_info) => (data_usage_info.clone(), None),
@@ -733,6 +878,22 @@ where
},
};
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?;
let publication_guard = match publication {
Some((publication_store, expected_publication_epoch)) => {
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
if expected_publication_epoch
.or(read_epoch)
.is_some_and(|expected| expected != publication_epoch)
{
return Err(Error::other("data usage publication epoch changed before snapshot commit"));
}
Some(guard)
}
None => None,
};
let save_result = store
.put_object(
RUSTFS_META_BUCKET,
@@ -745,6 +906,7 @@ where
},
)
.await;
drop(publication_guard);
match save_result {
Ok(_) => return Ok(()),
Err(err) => {
@@ -2286,6 +2448,8 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
#[cfg(test)]
mod tests {
use super::*;
use crate::layout::endpoints::EndpointServerPools;
use crate::runtime::instance::InstanceContext;
use crate::storage_api_contracts::object::ObjectIO as _;
use rustfs_data_usage::BucketUsageInfo;
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
@@ -2308,6 +2472,7 @@ mod tests {
error_after_commit_put: Option<usize>,
advance_time_on_put: Option<Duration>,
advance_time_after_get: Option<(UsageObjectSlot, Duration)>,
advance_publication_epoch_after_get: Option<(UsageObjectSlot, Arc<InstanceContext>)>,
advance_time_before_put: Option<(usize, Duration)>,
advance_time_after_put: Option<(usize, Duration)>,
put_count: usize,
@@ -2385,10 +2550,21 @@ mod tests {
}
_ => None,
};
let advance_publication_epoch = match state.advance_publication_epoch_after_get {
Some((expected_slot, ref ctx)) if expected_slot == slot => {
let ctx = Arc::clone(ctx);
state.advance_publication_epoch_after_get = None;
Some(ctx)
}
_ => None,
};
drop(state);
if let Some(duration) = advance {
tokio::time::advance(duration).await;
}
if let Some(ctx) = advance_publication_epoch {
ctx.advance_data_movement_operation_epoch();
}
Ok(crate::object_api::GetObjectReader {
stream: Box::new(Cursor::new(data)),
object_info: ObjectInfo {
@@ -2589,6 +2765,23 @@ mod tests {
.to_string()
}
fn build_publication_store(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
let endpoint_pools = EndpointServerPools::default();
Arc::new(ECStore {
id: uuid::Uuid::new_v4(),
disk_map: HashMap::new(),
pools: Vec::new(),
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
pool_meta: RwLock::new(crate::core::pools::PoolMeta::default()),
rebalance_meta: RwLock::new(None),
decommission_cancelers: RwLock::new(Vec::new()),
start_gate: TokioMutex::new(()),
pool_meta_save_gate: TokioMutex::new(()),
ctx,
bucket_fence_registry: Arc::default(),
})
}
#[test]
fn data_usage_cache_absence_covers_the_variants_that_actually_arrive() {
// `to_object_err` rewrites the raw storage variants before they reach
@@ -4333,7 +4526,15 @@ mod tests {
.expect("namespace lock acquisition should not fail")
.expect("namespace lock should be acquired"),
);
let store = Arc::new(UsageCasStore::default());
let snapshot = data_usage_info_for_test(BUCKET, 2, 84, SystemTime::now());
let encoded = serde_json::to_vec(&snapshot).expect("usage snapshot should encode");
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
object: Some((encoded.clone(), 1)),
backup_object: Some((encoded, 1)),
..Default::default()
}),
});
let successor = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now());
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
*snapshot_cache = Some(CachedDataUsageSnapshot {
@@ -4432,21 +4633,17 @@ mod tests {
}
#[tokio::test]
async fn remove_bucket_usage_creates_primary_and_backup_fences_when_missing() {
async fn remove_bucket_usage_does_not_synthesize_authoritative_snapshot_when_all_missing() {
let store = Arc::new(UsageCasStore::default());
remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a")
.await
.expect("bucket removal should create both usage fences");
.expect("bucket removal should remain a no-op without a usage baseline");
let state = store.state.lock().await;
assert_eq!(state.put_count, 2);
for (data, revision) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() {
let saved = serde_json::from_slice::<DataUsageInfo>(data).expect("saved usage snapshot should decode");
assert_eq!(*revision, 1);
assert!(saved.last_update.is_some());
assert!(!data_usage_contains_bucket(&saved, "bucket-a"));
}
assert_eq!(state.put_count, 0);
assert!(state.object.is_none());
assert!(state.backup_object.is_none());
}
#[tokio::test]
@@ -4598,6 +4795,39 @@ mod tests {
assert_eq!(backup_err, Error::PreconditionFailed);
}
#[tokio::test]
async fn remove_bucket_usage_rejects_movement_epoch_flip_between_read_and_commit() {
let ctx = Arc::new(InstanceContext::new());
let publication_store = build_publication_store(ctx.clone());
let snapshot = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
object: Some((serde_json::to_vec(&snapshot).expect("usage snapshot should encode"), 1)),
advance_publication_epoch_after_get: Some((UsageObjectSlot::Primary, ctx)),
..Default::default()
}),
});
let expected_epoch = publication_store
.scanner_data_usage_publication_epoch()
.await
.expect("idle publication store should admit the initial read");
let err = remove_bucket_usage_from_object_with_retries_and_publication(
store.as_ref(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
"bucket-a",
0,
None,
None,
Some((publication_store.as_ref(), Some(expected_epoch))),
)
.await
.expect_err("a movement epoch flip during the read must fence the commit");
assert!(err.to_string().contains("epoch changed"));
assert_eq!(store.state.lock().await.put_count, 0);
}
#[tokio::test]
async fn remove_bucket_usage_confirms_ambiguous_committed_final_attempt() {
let initial = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
crates/ecstore/src/runtime/instance.rs
+76 -8
View File
@@ -52,11 +52,18 @@ use crate::services::tier::tier::TierConfigMgr;
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
use s3s::region::Region;
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, OnceLock};
use std::sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
};
use tokio::sync::{OnceCell, RwLock};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
/// Runtime state owned by a single `ECStore` instance.
///
/// This is intentionally minimal in the first migration slice; subsequent
@@ -160,10 +167,22 @@ pub struct InstanceContext {
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
/// Replaces the process-global cancel-token static.
background_cancel_token: OnceLock<CancellationToken>,
/// Serializes decommission data-movement operations with cancellation and
/// a subsequent restart. Readers are held across one object side effect;
/// the transition path takes the writer after cancelling the routine.
decommission_operation_gate: Arc<RwLock<()>>,
/// Serializes data-movement transitions with scanner publication commits.
/// Readers are held across one publication commit; movement transitions
/// take the writer at their durable state commit boundary.
data_movement_operation_gate: Arc<RwLock<()>>,
/// Monotonic admission epoch paired with the operation gate. A
/// publication admitted before a movement transition must never be
/// mistaken for one admitted after the transition.
data_movement_operation_epoch: AtomicU64,
/// Once the admission epoch reaches its reserved terminal value, no new
/// publication may be admitted. Keeping this state separate from the
/// saturating counter prevents an unchanged `u64::MAX` value from being
/// mistaken for a fresh epoch after overflow.
data_movement_operation_epoch_exhausted: AtomicBool,
/// Last storage-owned movement snapshot observed under the operation
/// gate. SetDisks cache writers fail closed until ECStore refreshes it.
scanner_publication_state: AtomicU8,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -204,7 +223,10 @@ impl InstanceContext {
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
decommission_operation_gate: Arc::new(RwLock::new(())),
data_movement_operation_gate: Arc::new(RwLock::new(())),
data_movement_operation_epoch: AtomicU64::new(0),
data_movement_operation_epoch_exhausted: AtomicBool::new(false),
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -223,8 +245,54 @@ impl InstanceContext {
self.lock_manager.clone()
}
pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.decommission_operation_gate)
pub(crate) fn data_movement_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.data_movement_operation_gate)
}
pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
self.data_movement_operation_epoch.load(Ordering::Acquire)
}
pub(crate) fn data_movement_operation_epoch_exhausted(&self) -> bool {
self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
}
pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
!self.data_movement_operation_epoch_exhausted()
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
}
pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
self.scanner_publication_state.store(
if blocked {
SCANNER_PUBLICATION_STATE_BLOCKED
} else {
SCANNER_PUBLICATION_STATE_ALLOWED
},
Ordering::Release,
);
}
pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
let _ = self
.data_movement_operation_epoch
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
let result = self.data_movement_operation_epoch.load(Ordering::Acquire);
if result == u64::MAX {
self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
}
result
}
#[cfg(test)]
pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
self.data_movement_operation_epoch.store(epoch, Ordering::Release);
self.data_movement_operation_epoch_exhausted
.store(epoch == u64::MAX, Ordering::Release);
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
}
/// Install the application-owned object-encryption resolver once.
crates/ecstore/src/services/rebalance/control.rs
+76 -11
View File
@@ -48,12 +48,12 @@ fn pool_rebalance_status_from_meta(meta: Option<&RebalanceMeta>, pool_index: usi
.unwrap_or_default()
}
fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) {
fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) -> bool {
if persisted.id.is_empty() && persisted.pool_stats.is_empty() {
clear_rebalance_status_refresh(current);
return;
return clear_rebalance_status_refresh(current);
}
let before = current.clone();
match current.as_mut() {
Some(current_meta) => {
if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict
@@ -66,14 +66,41 @@ fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted
*current = Some(persisted);
}
}
match (before.as_ref(), current.as_ref()) {
(None, None) => false,
(None, Some(_)) | (Some(_), None) => true,
(Some(before), Some(after)) => rebalance_movement_snapshot_changed(Some(before), after),
}
}
fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) {
fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) -> bool {
if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) {
*current = None;
current.take().is_some()
} else {
false
}
}
fn rebalance_movement_snapshot_changed(current: Option<&RebalanceMeta>, persisted: &RebalanceMeta) -> bool {
let Some(current) = current else {
return true;
};
current.id != persisted.id
|| current.stopped_at != persisted.stopped_at
|| current.pool_stats.len() != persisted.pool_stats.len()
|| current
.pool_stats
.iter()
.zip(persisted.pool_stats.iter())
.any(|(current, persisted)| {
current.participating != persisted.participating
|| current.info.status != persisted.info.status
|| current.info.stopping != persisted.info.stopping
})
}
impl ECStore {
pub(super) async fn save_rebalance_meta_with_merge<S>(
&self,
@@ -121,7 +148,10 @@ impl ECStore {
"Loading rebalance metadata"
);
let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
if resolve_rebalance_meta_load_result(meta.load(pool).await)? {
let movement_changed = rebalance_movement_snapshot_changed(self.rebalance_meta.read().await.as_ref(), &meta);
{
let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -130,6 +160,10 @@ impl ECStore {
drop(rebalance_meta);
}
if movement_changed {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?;
debug!(
event = EVENT_REBALANCE_STATE,
@@ -139,10 +173,15 @@ impl ECStore {
"Loaded rebalance metadata"
);
} else {
let movement_changed = self.rebalance_meta.read().await.is_some();
{
let mut rebalance_meta = self.rebalance_meta.write().await;
*rebalance_meta = None;
}
if movement_changed {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
@@ -160,14 +199,20 @@ impl ECStore {
pub async fn refresh_rebalance_status_meta(&self) -> Result<()> {
let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?;
let mut persisted = RebalanceMeta::new();
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
match persisted.load(pool).await {
Ok(()) => {
let mut rebalance_meta = self.rebalance_meta.write().await;
merge_rebalance_status_refresh(&mut rebalance_meta, persisted);
if merge_rebalance_status_refresh(&mut rebalance_meta, persisted) {
self.ctx.advance_data_movement_operation_epoch();
}
}
Err(Error::ConfigNotFound) => {
let mut rebalance_meta = self.rebalance_meta.write().await;
clear_rebalance_status_refresh(&mut rebalance_meta);
if clear_rebalance_status_refresh(&mut rebalance_meta) {
self.ctx.advance_data_movement_operation_epoch();
}
}
Err(err) => {
return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}")));
@@ -349,6 +394,8 @@ impl ECStore {
#[tracing::instrument(skip(self, bucktes))]
pub async fn init_rebalance_start(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let decommission_running = self.is_decommission_running().await;
{
@@ -356,12 +403,16 @@ impl ECStore {
validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
}
self.init_rebalance_meta(bucktes).await
let id = self.init_rebalance_meta(bucktes).await?;
self.ctx.advance_data_movement_operation_epoch();
Ok(id)
}
#[tracing::instrument(skip(self))]
pub async fn start_rebalance_for_id(self: &Arc<Self>, expected_id: &str) -> Result<()> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
{
let rebalance_meta = self.rebalance_meta.read().await;
@@ -379,7 +430,10 @@ impl ECStore {
}
}
self.start_rebalance().await
if self.start_rebalance_inner().await? {
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
}
pub async fn rollback_rebalance_start_for_id(self: &Arc<Self>, expected_id: Option<&str>, start_error: String) -> Result<()> {
@@ -548,6 +602,9 @@ impl ECStore {
#[tracing::instrument(skip(self))]
pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id)
@@ -560,6 +617,7 @@ impl ECStore {
.await,
"stop_rebalance",
)?;
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
@@ -570,6 +628,9 @@ impl ECStore {
expected_id: Option<&str>,
start_error: String,
) -> Result<()> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
rollback_rebalance_start_meta_snapshot_for_id(
@@ -587,6 +648,7 @@ impl ECStore {
.await,
"rollback_rebalance_start",
)?;
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
@@ -597,6 +659,8 @@ impl ECStore {
return Ok(());
}
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let encoded_error = encode_rebalance_stop_propagation_record(&record);
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -610,6 +674,7 @@ impl ECStore {
.await,
"record_rebalance_stop_propagation",
)?;
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
@@ -682,7 +747,7 @@ mod tests {
..Default::default()
};
merge_rebalance_status_refresh(&mut current, persisted);
assert!(merge_rebalance_status_refresh(&mut current, persisted));
let refreshed = current.as_ref().expect("refresh should keep rebalance metadata");
assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
@@ -721,7 +786,7 @@ mod tests {
..Default::default()
};
merge_rebalance_status_refresh(&mut current, persisted);
assert!(!merge_rebalance_status_refresh(&mut current, persisted));
assert!(
current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(),
crates/ecstore/src/services/rebalance/runtime.rs
+87 -47
View File
@@ -42,6 +42,15 @@ pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<Strin
impl ECStore {
#[tracing::instrument(skip_all)]
pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
if self.start_rebalance_inner().await? {
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
}
pub(super) async fn start_rebalance_inner(self: &Arc<Self>) -> Result<bool> {
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
@@ -55,6 +64,7 @@ impl ECStore {
let cancel_tx = CancellationToken::new();
let rx = cancel_tx.clone();
let mut meta_to_save = None;
let mut movement_changed = false;
{
let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -72,14 +82,16 @@ impl ECStore {
reason = "already_in_progress",
"Skipped duplicate rebalance start"
);
return Ok(());
return Ok(false);
}
let now = OffsetDateTime::now_utc();
if complete_rebalance_pools_at_goal(meta, now) {
meta_to_save = Some(meta.clone());
movement_changed = true;
}
if complete_rebalance_pools_with_empty_queue(meta, now) {
meta_to_save = Some(meta.clone());
movement_changed = true;
}
meta.cancel = Some(cancel_tx);
@@ -118,7 +130,7 @@ impl ECStore {
reason = "no_participants",
"Skipped rebalance start because no pools are participating"
);
return Ok(());
return Ok(movement_changed);
}
let mut workers_started = 0usize;
@@ -186,7 +198,7 @@ impl ECStore {
reason = "no_local_participants",
"Skipped rebalance start because no local pools are participating"
);
return Ok(());
return Ok(movement_changed);
}
info!(
@@ -197,7 +209,7 @@ impl ECStore {
worker_count = workers_started,
"Rebalance started"
);
Ok(())
Ok(true)
}
#[tracing::instrument(skip(self, rx))]
@@ -214,53 +226,77 @@ impl ECStore {
let mut quit = false;
loop {
let mut terminal_state_saved = false;
tokio::select! {
result = done_rx.recv() => {
quit = true;
let now = OffsetDateTime::now_utc();
let terminal_event = classify_rebalance_terminal_event(result, now);
msg = terminal_event.message().to_string();
let mut rebalance_meta = store.rebalance_meta.write().await;
if let Some(meta) = rebalance_meta.as_mut() {
let meta_stopped = meta.stopped_at.is_some();
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
&& has_rebalance_cleanup_warnings(pool_stat)
{
pool_stat.info.stopping = false;
pool_stat.info.status = RebalStatus::Failed;
pool_stat.info.end_time = Some(now);
pool_stat.info.last_error = Some(
pool_stat
.cleanup_warnings
.last_message
.clone()
.unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
);
} else if should_preserve_rebalance_stopped_state(
meta_stopped,
pool_stat.info.status,
&terminal_event,
) {
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "stopped_preserved",
"Preserved stopped rebalance status"
);
let movement_gate = store.ctx.data_movement_operation_gate();
let movement_guard = movement_gate.write().await;
let previous_meta = store.rebalance_meta.read().await.clone();
let terminal_state_present = {
let mut rebalance_meta = store.rebalance_meta.write().await;
if let Some(meta) = rebalance_meta.as_mut() {
let meta_stopped = meta.stopped_at.is_some();
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
&& has_rebalance_cleanup_warnings(pool_stat)
{
pool_stat.info.stopping = false;
pool_stat.info.status = RebalStatus::Failed;
pool_stat.info.end_time = Some(now);
pool_stat.info.last_error = Some(
pool_stat
.cleanup_warnings
.last_message
.clone()
.unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
);
} else if should_preserve_rebalance_stopped_state(
meta_stopped,
pool_stat.info.status,
&terminal_event,
) {
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "stopped_preserved",
"Preserved stopped rebalance status"
);
} else {
pool_stat.info.stopping = false;
apply_rebalance_terminal_event(
&mut pool_stat.info.status,
&mut pool_stat.info.end_time,
&mut pool_stat.info.last_error,
terminal_event,
now,
);
}
true
} else {
pool_stat.info.stopping = false;
apply_rebalance_terminal_event(
&mut pool_stat.info.status,
&mut pool_stat.info.end_time,
&mut pool_stat.info.last_error,
terminal_event,
now,
);
false
}
} else {
false
}
};
if terminal_state_present {
if let Err(err) = store.save_rebalance_stats_inner(pool_index, RebalSaveOpt::Stats).await {
let mut rebalance_meta = store.rebalance_meta.write().await;
*rebalance_meta = previous_meta;
drop(movement_guard);
return Err(Error::other(format!(
"rebalance terminal state save failed for pool {pool_index}: {err}"
)));
}
store.ctx.advance_data_movement_operation_epoch();
terminal_state_saved = true;
}
}
_ = timer.tick() => {
@@ -269,7 +305,7 @@ impl ECStore {
}
}
if let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await {
if !terminal_state_saved && let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await {
let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
error!("{} err: {:?}", msg, wrapped);
if quit {
@@ -590,16 +626,14 @@ impl ECStore {
meta.percent_free_goal,
)
{
pool_stat.info.status = RebalStatus::Completed;
pool_stat.info.end_time = Some(OffsetDateTime::now_utc());
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "completed",
state = "completion_ready",
percent_free = pfi,
"Marked rebalance pool completed"
"Rebalance pool reached completion goal"
);
return true;
}
@@ -612,6 +646,12 @@ impl ECStore {
impl ECStore {
#[tracing::instrument(skip(self))]
pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
self.save_rebalance_stats_inner(pool_idx, opt).await
}
pub(super) async fn save_rebalance_stats_inner(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
let Some(meta) = rebalance_meta.as_mut() else {
crates/ecstore/src/set_disk/mod.rs
+20
View File
@@ -3614,6 +3614,26 @@ impl SetDisks {
&self.ctx
}
/// Admit one short scanner cache publication under this set's instance
/// movement fence. The caller must hold the returned guard through its
/// final conditional cache write; no scan-round work belongs under it.
pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.scanner_publication_state_allowed() {
let epoch = self.ctx.data_movement_operation_epoch();
return Some((operation_guard, epoch));
}
// The owner deliberately marks the cached state UNKNOWN after every
// movement epoch advance. Do not strand remote scanner writers in that
// state: release this guard before asking the storage owner to refresh
// its durable movement snapshot, since the owner uses the same gate.
drop(operation_guard);
let owner = runtime_sources::object_store_handle().filter(|owner| Arc::ptr_eq(&owner.ctx, &self.ctx))?;
owner.scanner_data_usage_publication_admission_guard().await
}
/// Whether both sets' namespace-lock implementations cover the same object key.
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
crates/ecstore/src/store/bucket.rs
+1 -1
View File
@@ -327,7 +327,7 @@ impl ECStore {
async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
run_bucket_usage_cleanup(guard, bucket, async {
crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
crate::data_usage::remove_bucket_usage_from_backend_with_guard(self, bucket, guard).await
crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await
})
.await
}
crates/ecstore/src/store/init.rs
+5
View File
@@ -495,6 +495,11 @@ impl ECStore {
);
}
// Initialize the storage-owned scanner publication state only after
// both movement metadata sources have been loaded. SetDisks cache
// writers remain fail-closed until this snapshot is available.
let _ = self.scanner_data_usage_publication_blocked().await;
let pools = installed_pool_meta.return_resumable_pools();
let mut pool_indices = Vec::with_capacity(pools.len());
crates/ecstore/src/store/mod.rs
+149 -5
View File
@@ -44,7 +44,7 @@ use crate::error::{
use crate::runtime::global::DISK_RESERVE_FRACTION;
use crate::runtime::instance::InstanceContext;
use crate::runtime::sources as runtime_sources;
use crate::services::rebalance::RebalanceMeta;
use crate::services::rebalance::{RebalanceMeta, is_rebalance_conflicting_with_decommission};
use crate::storage_api_contracts::{
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
@@ -349,16 +349,81 @@ impl ECStore {
/// remain suspended until an operator clears or retries them, so they are
/// a publication barrier even after the worker has stopped.
pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
if self.scanner_data_movement_active().await {
let operation_gate = self.ctx.data_movement_operation_gate();
let _operation_guard = operation_gate.read_owned().await;
self.scanner_data_usage_publication_snapshot_blocked().await
}
async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
if self.ctx.data_movement_operation_epoch_exhausted() {
self.ctx.set_scanner_publication_state(true);
return true;
}
let decommission_cancelers = self.decommission_cancelers.read().await;
let decommission_active = decommission_cancelers
.iter()
.any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active));
let pool_meta = self.pool_meta.read().await;
pool_meta.pools.iter().any(|pool| {
let decommission_active = decommission_active
|| pool_meta.pools.iter().any(|pool| {
pool.decommission
.as_ref()
.is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
});
let decommission_terminal = pool_meta.pools.iter().any(|pool| {
pool.decommission
.as_ref()
.is_some_and(|info| !info.queued && (info.failed || info.canceled))
})
});
drop(pool_meta);
let rebalance_active = self
.rebalance_meta
.read()
.await
.as_ref()
.is_some_and(is_rebalance_conflicting_with_decommission);
let blocked = decommission_active || decommission_terminal || rebalance_active;
self.ctx.set_scanner_publication_state(blocked);
blocked
}
/// Admit one short data-usage publication commit under the same
/// per-instance gate used by decommission side effects and transitions.
/// The epoch is sampled while the read guard is held, so a transition
/// cannot cross this admission without waiting for the commit to finish.
pub async fn scanner_data_usage_publication_read_guard(&self) -> (tokio::sync::OwnedRwLockReadGuard<()>, u64) {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
let epoch = self.ctx.data_movement_operation_epoch();
(operation_guard, epoch)
}
/// Acquire the movement gate and inspect the movement owner once. The
/// state inspection is performed after acquiring the read guard so a
/// transition cannot update its durable state between the check and the
/// publication commit.
pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.data_movement_operation_epoch_exhausted() {
return None;
}
if self.scanner_data_usage_publication_snapshot_blocked().await {
return None;
}
Some((operation_guard, self.ctx.data_movement_operation_epoch()))
}
/// Capture the current publication epoch without holding the movement
/// gate across backend I/O. Callers must re-admit the same epoch before a
/// mutation commits.
pub(crate) async fn scanner_data_usage_publication_epoch(&self) -> Option<u64> {
let (operation_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
drop(operation_guard);
Some(epoch)
}
}
@@ -995,6 +1060,85 @@ mod tests {
}
}
#[tokio::test]
async fn scanner_data_usage_publication_admission_is_fenced_and_epoch_monotonic() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let operation_gate = store.ctx.data_movement_operation_gate();
let movement_guard = operation_gate.write().await;
let pending = {
let store = store.clone();
tokio::spawn(async move { store.scanner_data_usage_publication_admission_guard().await })
};
tokio::task::yield_now().await;
assert!(!pending.is_finished(), "publication admission must wait for a movement writer");
drop(movement_guard);
let (_, epoch) = pending
.await
.expect("publication admission task should not panic")
.expect("idle store should admit publication");
assert_eq!(epoch, 0);
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
let (_, next_epoch) = store
.scanner_data_usage_publication_admission_guard()
.await
.expect("idle store should admit the next publication");
assert_eq!(next_epoch, 1);
}
#[tokio::test]
async fn scanner_data_usage_publication_epoch_releases_gate_before_backend_io() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let epoch = store
.scanner_data_usage_publication_epoch()
.await
.expect("idle store should expose a publication epoch");
assert_eq!(epoch, 0);
let operation_gate = store.ctx.data_movement_operation_gate();
let _movement_guard = tokio::time::timeout(Duration::from_secs(1), operation_gate.write())
.await
.expect("epoch capture must not hold the movement gate across backend I/O");
}
#[tokio::test]
async fn scanner_data_usage_publication_admission_blocks_active_rebalance_snapshot() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
*store.rebalance_meta.write().await = Some(RebalanceMeta {
pool_stats: vec![crate::services::rebalance::RebalanceStats {
participating: true,
info: crate::services::rebalance::RebalanceInfo {
status: crate::services::rebalance::RebalStatus::Started,
..Default::default()
},
..Default::default()
}],
..Default::default()
});
assert!(
store.scanner_data_usage_publication_admission_guard().await.is_none(),
"active rebalance must fail closed at the storage-owned admission boundary"
);
}
#[tokio::test]
async fn scanner_publication_epoch_exhaustion_fails_closed_after_max() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
store.ctx.set_data_movement_operation_epoch_for_test(u64::MAX - 1);
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
assert!(store.ctx.data_movement_operation_epoch_exhausted());
assert!(
store.scanner_data_usage_publication_admission_guard().await.is_none(),
"publication must fail closed at the reserved terminal epoch"
);
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
assert!(store.ctx.data_movement_operation_epoch_exhausted());
assert!(store.scanner_data_usage_publication_blocked().await);
}
// The object graph is the isolation carrier: two ECStore instances holding
// distinct contexts report independent erasure state through their real
// `&self` accessors — no cross-contamination.
crates/ecstore/src/store/rebalance.rs
+14 -2
View File
@@ -694,6 +694,11 @@ impl ECStore {
/// callers only trigger missing-worker recovery after a real state change;
/// delayed snapshots are merged monotonically and never blind-assigned.
pub async fn reload_pool_meta(&self) -> Result<bool> {
// Serialize the durable reload with local movement transitions. Loading
// before acquiring this gate would allow a stale disk snapshot to
// overwrite a newer local transition after the writer commits.
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let mut reloaded = PoolMeta::default();
resolve_store_rebalance_pool_meta_reload_result(
reloaded.load(self.pools[0].clone(), self.pools.clone()).await,
@@ -701,15 +706,22 @@ impl ECStore {
)?;
// Lock order: release the decommission_cancelers guard before taking
// the pool_meta write guard; neither is held across the disk read.
// the pool_meta write guard; neither is held without the movement gate.
let active_workers = {
let cancelers = self.decommission_cancelers.read().await;
cancelers.iter().map(Option::is_some).collect::<Vec<_>>()
cancelers
.iter()
.map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
.collect::<Vec<_>>()
};
let incoming_has_pools = !reloaded.pools.is_empty();
let mut pool_meta = self.pool_meta.write().await;
let movement_before = pool_meta.clone();
let merged_newer = merge_pool_status_refresh(&mut pool_meta, reloaded, &active_workers);
if crate::core::pools::pool_meta_movement_snapshot_changed(&movement_before, &pool_meta) {
self.ctx.advance_data_movement_operation_epoch();
}
if !merged_newer && !incoming_has_pools {
warn!(