fix(ecstore): fence rebalance and decommission activation (#6400)

* fix(ecstore): fence rebalance and decommission activation * fix(ecstore): fence lost activation locks * fix(ecstore): bind rebalance workers to activation id * fix(ecstore): close rebalance activation races * test(ecstore): exercise lost rebalance commit fence * fix(ecstore): repair rebalance fence test wiring * test(ecstore): reuse rebalance metadata fixture * fix(ecstore): satisfy rebalance activation clippy checks * fix(ecstore): fence stale rebalance workers * fix(ecstore): commit rebalance activation after persistence * fix(ecstore): fence rebalance commits and unblock stop * test(ecstore): exercise real rebalance fences * fix(rebalance): cancel admin stop before activation wait * fix(ecstore): fence multipart staging on rebalance lock loss * fix(ecstore): adopt activations after durable commit * fix(rebalance): preserve committed activation recovery * fix(rebalance): make prepared stop terminal-safe * fix(ecstore): repair rebalance test imports * fix(ecstore): repair rebalance entry runtime failures * test(ecstore): fix activation fence synchronization * test(ecstore): scope rebalance disk trait import * test(ecstore): observe decommission lock attempt * fix(ecstore): align activation fence test imports * fix(ecstore): remove duplicate activation test import * fix(ecstore): resolve CI clippy failures * fix: satisfy activation merge lint gates --------- Co-authored-by: houseme <housemecn@gmail.com>
2026-08-29 16:37:07 +00:00 · 2026-08-24 14:03:21 +08:00
parent 40bf9f0425
commit 73cd1b5be2
27 changed files with 4270 additions and 328 deletions
@@ -21,8 +21,8 @@ use crate::bucket::{
        DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, ValidatedDurableIlmRecord,
        bucket_lifecycle_audit::LcEventSrc,
        bucket_lifecycle_ops::{
-            LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle,
-            lifecycle_delete_all_versions_blocked_by_replication,
+            LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_for_data_movement, apply_expiry_rule_in,
+            eval_action_from_lifecycle, lifecycle_delete_all_versions_blocked_by_replication,
        },
        classify_durable_ilm_record, get_expiry_configs,
        lifecycle::IlmAction,
@@ -34,6 +34,7 @@ use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
 use crate::config::com::{
    CONFIG_PREFIX, delete_config, read_config_limited_preserve_empty, read_config_limited_preserve_empty_with_metadata,
    read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty, save_config, save_config_with_opts,
+    save_config_with_opts_quiet,
 };
 use crate::data_movement;
 use crate::data_movement::backpressure::{self, DataMovementOperation};
@@ -55,7 +56,6 @@ use crate::storage_api_contracts::{
    bucket::{BucketOperations, BucketOptions, MakeBucketOptions},
    heal::HealOperations as _,
    list::ListOperations as _,
-    namespace::NamespaceLocking as _,
    object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _, ObjectOperations as _},
 };
 use crate::{core::sets::Sets, store::ECStore};
@@ -1517,7 +1517,7 @@ fn resolve_start_decommission_pool_meta_reload_result(result: Result<()>) -> Res
    resolve_decommission_pool_meta_reload_result(result, "start_decommission")
 }

-fn decommission_rebalance_meta_lock_error(err: rustfs_lock::LockError) -> Error {
+fn activation_rebalance_meta_lock_error(err: rustfs_lock::LockError) -> Error {
    match err {
        rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
            mode: "write",
@@ -1527,12 +1527,12 @@ fn decommission_rebalance_meta_lock_error(err: rustfs_lock::LockError) -> Error
            achieved,
        },
        other => Error::other(format!(
-            "failed to acquire rebalance metadata write lock before decommission start on {RUSTFS_META_BUCKET}/{REBAL_META_NAME}: {other}"
+            "failed to acquire rebalance activation lock on {RUSTFS_META_BUCKET}/{REBAL_META_NAME}: {other}"
        )),
    }
 }

-fn decommission_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error {
+fn activation_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error {
    match err {
        rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
            mode: "write",
@@ -1542,11 +1542,250 @@ fn decommission_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error {
            achieved,
        },
        other => Error::other(format!(
-            "failed to acquire pool metadata write lock before decommission start on {RUSTFS_META_BUCKET}/{POOL_META_NAME}: {other}"
+            "failed to acquire pool activation lock on {RUSTFS_META_BUCKET}/{POOL_META_NAME}: {other}"
        )),
    }
 }

+pub(crate) struct PoolRebalanceActivationFence {
+    pool_meta_guard: rustfs_lock::NamespaceLockGuard,
+    rebalance_meta_guard: rustfs_lock::NamespaceLockGuard,
+    fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
+    #[cfg(test)]
+    forced_lost: Arc<AtomicBool>,
+}
+
+impl PoolRebalanceActivationFence {
+    pub(crate) fn ensure_held(&self) -> Result<()> {
+        #[cfg(test)]
+        let forced_lost = self.forced_lost.load(Ordering::Acquire);
+        #[cfg(not(test))]
+        let forced_lost = false;
+        if forced_lost || self.pool_meta_guard.is_lock_lost() || self.rebalance_meta_guard.is_lock_lost() {
+            return Err(Error::other("activation lock lost before metadata commit or worker admission"));
+        }
+        if self
+            .fleet_proof
+            .as_ref()
+            .is_some_and(|proof| !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof))
+        {
+            return Err(Error::other("pool activation fleet capability proof expired before commit"));
+        }
+
+        Ok(())
+    }
+
+    pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
+        opts.add_namespace_lock_guard(&self.pool_meta_guard);
+        opts.add_namespace_lock_guard(&self.rebalance_meta_guard);
+    }
+
+    #[cfg(test)]
+    fn force_lost_for_test(&self) {
+        self.forced_lost.store(true, Ordering::Release);
+    }
+}
+
+pub(crate) async fn acquire_pool_rebalance_activation_locks<S>(
+    pool: Arc<S>,
+    fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
+) -> Result<PoolRebalanceActivationFence>
+where
+    S: crate::storage_api_contracts::namespace::NamespaceLocking<
+            Error = Error,
+            NamespaceLock = rustfs_lock::NamespaceLockWrapper,
+        >,
+{
+    // Activation lock order is always pool.bin -> rebalance.bin.
+    let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
+    let pool_meta_guard = pool_meta_lock
+        .get_write_lock(get_lock_acquire_timeout())
+        .await
+        .map_err(activation_pool_meta_lock_error)?;
+    let rebalance_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
+    let rebalance_meta_guard = rebalance_meta_lock
+        .get_write_lock(get_lock_acquire_timeout())
+        .await
+        .map_err(activation_rebalance_meta_lock_error)?;
+
+    Ok(PoolRebalanceActivationFence {
+        pool_meta_guard,
+        rebalance_meta_guard,
+        fleet_proof,
+        #[cfg(test)]
+        forced_lost: Arc::new(AtomicBool::new(false)),
+    })
+}
+
+pub(crate) async fn acquire_pool_activation_fleet_proof(
+    ctx: &crate::runtime::instance::InstanceContext,
+) -> Result<Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>> {
+    if !ctx.is_dist_erasure().await {
+        return Ok(None);
+    }
+    crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof()
+        .map(Some)
+        .ok_or_else(|| Error::other("pool activation requires a live fleet capability proof"))
+}
+
+#[cfg(test)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) enum PoolActivationStartKind {
+    Rebalance,
+    Decommission,
+}
+
+#[cfg(test)]
+struct PoolActivationDurableSaveBarrierState {
+    pool_key: usize,
+    arrived: tokio::sync::Notify,
+    release: tokio::sync::Notify,
+}
+
+#[cfg(test)]
+static POOL_ACTIVATION_DURABLE_SAVE_BARRIER: std::sync::OnceLock<
+    std::sync::Mutex<Option<Arc<PoolActivationDurableSaveBarrierState>>>,
+> = std::sync::OnceLock::new();
+
+#[cfg(test)]
+pub(crate) struct PoolActivationDurableSaveBarrier {
+    state: Arc<PoolActivationDurableSaveBarrierState>,
+}
+
+#[cfg(test)]
+fn pool_activation_test_pool_key<S>(pool: &Arc<S>) -> usize {
+    Arc::as_ptr(pool).cast::<()>() as usize
+}
+
+#[cfg(test)]
+impl PoolActivationDurableSaveBarrier {
+    pub(crate) fn install<S>(pool: &Arc<S>) -> Self {
+        let state = Arc::new(PoolActivationDurableSaveBarrierState {
+            pool_key: pool_activation_test_pool_key(pool),
+            arrived: tokio::sync::Notify::new(),
+            release: tokio::sync::Notify::new(),
+        });
+        let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("pool activation durable save barrier should not be poisoned");
+        assert!(barrier.is_none(), "pool activation durable save barrier must be unique");
+        *barrier = Some(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(crate) async fn wait_until_paused(&self) {
+        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified())
+            .await
+            .expect("activation should reach the post-durable-save barrier");
+    }
+
+    pub(crate) fn release_after_fence_loss(&self) {
+        self.state.release.notify_one();
+    }
+}
+
+#[cfg(test)]
+impl Drop for PoolActivationDurableSaveBarrier {
+    fn drop(&mut self) {
+        self.state.release.notify_one();
+        let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("pool activation durable save barrier should not be poisoned");
+        if barrier.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *barrier = None;
+        }
+    }
+}
+
+#[cfg(test)]
+pub(crate) async fn pause_pool_activation_after_durable_save<S>(pool: &Arc<S>, fence: &PoolRebalanceActivationFence) {
+    let pool_key = pool_activation_test_pool_key(pool);
+    let barrier = {
+        let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("pool activation durable save barrier should not be poisoned");
+        if barrier.as_ref().is_some_and(|state| state.pool_key == pool_key) {
+            barrier.take()
+        } else {
+            None
+        }
+    };
+    if let Some(barrier) = barrier {
+        barrier.arrived.notify_one();
+        barrier.release.notified().await;
+        fence.force_lost_for_test();
+    }
+}
+
+#[cfg(test)]
+struct PoolActivationStartProbeState {
+    kind: PoolActivationStartKind,
+    attempted: std::sync::atomic::AtomicBool,
+    notify: tokio::sync::Notify,
+}
+
+#[cfg(test)]
+static POOL_ACTIVATION_START_PROBES: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<PoolActivationStartProbeState>>>> =
+    std::sync::OnceLock::new();
+
+#[cfg(test)]
+pub(crate) struct PoolActivationStartProbe {
+    state: Arc<PoolActivationStartProbeState>,
+}
+
+#[cfg(test)]
+impl PoolActivationStartProbe {
+    pub(crate) fn install(kind: PoolActivationStartKind) -> Self {
+        let state = Arc::new(PoolActivationStartProbeState {
+            kind,
+            attempted: std::sync::atomic::AtomicBool::new(false),
+            notify: tokio::sync::Notify::new(),
+        });
+        POOL_ACTIVATION_START_PROBES
+            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
+            .lock()
+            .expect("pool activation start probe should not be poisoned")
+            .push(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(crate) async fn wait_until_attempted(&self) {
+        while !self.state.attempted.load(Ordering::Acquire) {
+            self.state.notify.notified().await;
+        }
+    }
+}
+
+#[cfg(test)]
+impl Drop for PoolActivationStartProbe {
+    fn drop(&mut self) {
+        let mut probes = POOL_ACTIVATION_START_PROBES
+            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
+            .lock()
+            .expect("pool activation start probe should not be poisoned");
+        probes.retain(|state| !Arc::ptr_eq(state, &self.state));
+    }
+}
+
+#[cfg(test)]
+pub(crate) fn observe_pool_activation_start_attempt(kind: PoolActivationStartKind) {
+    let probes = POOL_ACTIVATION_START_PROBES
+        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
+        .lock()
+        .expect("pool activation start probe should not be poisoned")
+        .iter()
+        .filter(|state| state.kind == kind)
+        .cloned()
+        .collect::<Vec<_>>();
+    for state in probes {
+        state.attempted.store(true, Ordering::Release);
+        state.notify.notify_one();
+    }
+}
+
 fn rollback_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool_meta: PoolMeta) {
    *pool_meta = previous_pool_meta;
 }
@@ -2369,9 +2608,10 @@ fn select_pool_meta_replica(replicas: Vec<PoolMetaReplica>) -> Result<PoolMetaSe
        return Err(Error::other("pool metadata recovery required: no storage pools available"));
    }

-    // V1 has no durable generation. Semantically equivalent legacy/current
-    // encodings can be normalized, but different canonical snapshots require
-    // an operator-selected recovery source instead of an inferred winner.
+    // V1 has no durable generation. Pool zero remains the canonical commit
+    // record because every writer saves it first and legacy startup read it
+    // alone. Divergent backups are ambiguous when pool zero is unavailable and
+    // require an operator-selected recovery source.
    let mut selected: Option<(usize, Vec<u8>, Vec<u8>, PoolMeta)> = None;
    let mut needs_repair = false;
    let mut repair_write_safe = true;
@@ -2401,11 +2641,19 @@ fn select_pool_meta_replica(replicas: Vec<PoolMetaReplica>) -> Result<PoolMetaSe
            PoolMetaReplica::Valid { raw, canonical, meta } => {
                if let Some((selected_idx, selected_raw, selected_canonical, _)) = selected.as_ref() {
                    if selected_canonical != &canonical {
-                        return Err(Error::other(format!(
-                            "pool metadata recovery required: valid replicas in pools {selected_idx} and {idx} diverge; restore one matching pool.bin snapshot before restart"
-                        )));
+                        if *selected_idx == 0 {
+                            // Pool zero is the durable commit record: every
+                            // pool metadata writer commits it before replicas,
+                            // and pre-replica-recovery startup read it alone.
+                            needs_repair = true;
+                        } else {
+                            return Err(Error::other(format!(
+                                "pool metadata recovery required: valid replicas in pools {selected_idx} and {idx} diverge; restore one matching pool.bin snapshot before restart"
+                            )));
+                        }
+                    } else {
+                        needs_repair |= selected_raw != &raw;
                    }
-                    needs_repair |= selected_raw != &raw;
                } else {
                    selected = Some((idx, raw, canonical, meta));
                }
@@ -2963,6 +3211,67 @@ impl PoolMeta {
        Ok(())
    }

+    async fn save_no_lock_with_activation_fence<S>(
+        &self,
+        pools: Vec<Arc<S>>,
+        activation_fence: PoolRebalanceActivationFence,
+    ) -> Result<()>
+    where
+        S: EcstoreObjectIO,
+    {
+        let data = self.encode_config_data()?;
+        if data.is_empty() {
+            return Ok(());
+        }
+        let mut pools = pools.into_iter();
+        let Some(canonical_pool) = pools.next() else {
+            return Ok(());
+        };
+        let mut opts = ObjectOptions {
+            max_parity: true,
+            no_lock: true,
+            ..Default::default()
+        };
+        activation_fence.add_namespace_lock_fence(&mut opts);
+        activation_fence.ensure_held()?;
+        #[cfg(test)]
+        let barrier_pool = canonical_pool.clone();
+        save_config_with_opts(canonical_pool, POOL_META_NAME, data.clone(), &opts).await?;
+        #[cfg(test)]
+        pause_pool_activation_after_durable_save(&barrier_pool, &activation_fence).await;
+
+        // Pool zero is canonical. Once its save succeeds, later writes only
+        // replicate committed state and must not reuse the admission fence.
+        // Failed replicas remain repairable by the next full pool metadata save.
+        drop(activation_fence);
+        for (pool_index, pool) in pools.enumerate() {
+            if let Err(err) = save_config_with_opts_quiet(
+                pool,
+                POOL_META_NAME,
+                data.clone(),
+                &ObjectOptions {
+                    max_parity: true,
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            {
+                warn!(
+                    event = EVENT_DECOMMISSION_STATE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_POOLS,
+                    pool_index = pool_index + 1,
+                    state = "activation_replica_repair_pending",
+                    error = %err,
+                    "Decommission activation replica repair pending"
+                );
+            }
+        }
+
+        Ok(())
+    }
+
    pub fn decommission_cancel(&mut self, idx: usize) -> bool {
        if let Some(stats) = self.pools.get_mut(idx) {
            if let Some(d) = &stats.decommission {
@@ -3434,6 +3743,30 @@ pub struct PoolSpaceInfo {
    pub used: usize,
 }

+#[cfg(test)]
+type DecommissionSpaceInfoOverrides = std::sync::Mutex<HashMap<uuid::Uuid, Vec<(usize, PoolSpaceInfo)>>>;
+
+#[cfg(test)]
+static DECOMMISSION_SPACE_INFO_OVERRIDES: std::sync::OnceLock<DecommissionSpaceInfoOverrides> = std::sync::OnceLock::new();
+
+#[cfg(test)]
+pub(crate) fn set_decommission_space_info_override_for_test(store_id: uuid::Uuid, space_infos: Vec<(usize, PoolSpaceInfo)>) {
+    DECOMMISSION_SPACE_INFO_OVERRIDES
+        .get_or_init(|| std::sync::Mutex::new(HashMap::new()))
+        .lock()
+        .expect("decommission space info override should not be poisoned")
+        .insert(store_id, space_infos);
+}
+
+#[cfg(test)]
+fn take_decommission_space_info_override_for_test(store_id: uuid::Uuid) -> Option<Vec<(usize, PoolSpaceInfo)>> {
+    DECOMMISSION_SPACE_INFO_OVERRIDES
+        .get_or_init(|| std::sync::Mutex::new(HashMap::new()))
+        .lock()
+        .expect("decommission space info override should not be poisoned")
+        .remove(&store_id)
+}
+
 #[derive(Debug, Default, Clone)]
 pub struct DecomBucketInfo {
    pub name: String,
@@ -3551,6 +3884,85 @@ fn lifecycle_action_skips_heal_version(action: IlmAction) -> bool {
    action.delete()
 }

+#[cfg(test)]
+struct LifecycleDataMovementMutationBarrierState {
+    bucket: String,
+    object: String,
+    arrived: tokio::sync::Notify,
+    release: tokio::sync::Notify,
+}
+
+#[cfg(test)]
+pub(crate) struct LifecycleDataMovementMutationBarrier {
+    state: Arc<LifecycleDataMovementMutationBarrierState>,
+}
+
+#[cfg(test)]
+static LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER: std::sync::OnceLock<
+    std::sync::Mutex<Option<Arc<LifecycleDataMovementMutationBarrierState>>>,
+> = std::sync::OnceLock::new();
+
+#[cfg(test)]
+impl LifecycleDataMovementMutationBarrier {
+    pub(crate) fn install(bucket: &str, object: &str) -> Self {
+        let state = Arc::new(LifecycleDataMovementMutationBarrierState {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            arrived: tokio::sync::Notify::new(),
+            release: tokio::sync::Notify::new(),
+        });
+        let mut slot = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("lifecycle data movement mutation barrier should not be poisoned");
+        assert!(slot.is_none(), "lifecycle data movement mutation barrier must be unique");
+        *slot = Some(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(crate) async fn wait_until_paused(&self) {
+        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified())
+            .await
+            .expect("lifecycle data movement should reach its mutation boundary");
+    }
+
+    pub(crate) fn release(&self) {
+        self.state.release.notify_one();
+    }
+}
+
+#[cfg(test)]
+impl Drop for LifecycleDataMovementMutationBarrier {
+    fn drop(&mut self) {
+        self.state.release.notify_one();
+        let mut slot = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("lifecycle data movement mutation barrier should not be poisoned");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(test)]
+async fn pause_lifecycle_data_movement_mutation(bucket: &str, object: &str, has_run_fence_signal: bool) {
+    if !has_run_fence_signal {
+        return;
+    }
+    let barrier = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("lifecycle data movement mutation barrier should not be poisoned")
+        .as_ref()
+        .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.arrived.notify_one();
+        barrier.release.notified().await;
+    }
+}
+
 fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result<bool> {
    if !apply_actions || applied {
        return Ok(true);
@@ -3570,6 +3982,7 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
    object_lock_config: Option<&ObjectLockConfiguration>,
    apply_actions: bool,
    event_source: &LcEventSrc,
+    lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
 ) -> Result<bool> {
    let Some(lifecycle_config) = lifecycle_config else {
        return Ok(false);
@@ -3585,16 +3998,31 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
                let Ok(bucket_incarnation_id) = store.bucket_incarnation_id_from_disk(bucket).await else {
                    return Ok(false);
                };
-                let _ =
-                    apply_expiry_on_transitioned_object(store, &object_info, &event, event_source, bucket_incarnation_id).await;
+                let _ = match lock_lost_signal {
+                    Some(signal) => {
+                        apply_expiry_rule_for_data_movement(store, &event, event_source, &object_info, Some(signal)).await
+                    }
+                    None => {
+                        apply_expiry_on_transitioned_object(store, &object_info, &event, event_source, bucket_incarnation_id)
+                            .await
+                    }
+                };
            }
            Ok(false)
        }
        action if lifecycle_action_removes_data_movement_version(action) => {
+            #[cfg(test)]
+            pause_lifecycle_data_movement_mutation(bucket, &version.name, lock_lost_signal.is_some()).await;
            if lifecycle_delete_all_versions_blocked_by_replication(store.clone(), bucket, &object_info.name, action).await? {
                return Ok(false);
            }
-            let applied = !apply_actions || apply_expiry_rule_in(store, &event, event_source, &object_info).await;
+            let applied = !apply_actions
+                || match lock_lost_signal {
+                    Some(signal) => {
+                        apply_expiry_rule_for_data_movement(store, &event, event_source, &object_info, Some(signal)).await
+                    }
+                    None => apply_expiry_rule_in(store, &event, event_source, &object_info).await,
+                };
            resolve_data_movement_lifecycle_expiry_result(action, apply_actions, applied)
        }
        _ => Ok(false),
@@ -3731,16 +4159,10 @@ impl ECStore {
            .first()
            .cloned()
            .ok_or_else(|| Error::other("decommission start rebalance metadata load failed: no storage pools available"))?;
-        let pool_meta_lock = rebalance_pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
-        let _pool_meta_guard = pool_meta_lock
-            .get_write_lock(get_lock_acquire_timeout())
-            .await
-            .map_err(decommission_pool_meta_lock_error)?;
-        let ns_lock = rebalance_pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
-        let _guard = ns_lock
-            .get_write_lock(get_lock_acquire_timeout())
-            .await
-            .map_err(decommission_rebalance_meta_lock_error)?;
+        #[cfg(test)]
+        observe_pool_activation_start_attempt(PoolActivationStartKind::Decommission);
+        let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
+        let activation_fence = acquire_pool_rebalance_activation_locks(rebalance_pool.clone(), fleet_proof).await?;

        let mut rebalance_meta = RebalanceMeta::new();
        match rebalance_meta
@@ -3786,7 +4208,10 @@ impl ECStore {
            latest_pool_meta.queue_buckets(idx, decom_buckets.clone());
        }

-        latest_pool_meta.save_no_lock(self.pools.clone()).await?;
+        activation_fence.ensure_held()?;
+        latest_pool_meta
+            .save_no_lock_with_activation_fence(self.pools.clone(), activation_fence)
+            .await?;
        {
            let mut pool_meta = self.pool_meta.write().await;
            *pool_meta = latest_pool_meta;
@@ -3800,6 +4225,11 @@ impl ECStore {
        ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)
    }

+    async fn ensure_decommission_rebalance_idle_after_refresh_under_start_gate(&self) -> Result<()> {
+        self.load_rebalance_meta_under_start_gate().await?;
+        ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)
+    }
+
    pub async fn status(&self, idx: usize) -> Result<PoolStatus> {
        let space_info = self.get_decommission_pool_space_info(idx).await?;

@@ -3855,6 +4285,11 @@ impl ECStore {
    }

    async fn get_decommission_all_pool_space_infos(&self) -> Result<Vec<(usize, PoolSpaceInfo)>> {
+        #[cfg(test)]
+        if let Some(space_infos) = take_decommission_space_info_override_for_test(self.id) {
+            return Ok(space_infos);
+        }
+
        let mut space_infos = Vec::with_capacity(self.pools.len());
        for idx in 0..self.pools.len() {
            space_infos.push((idx, self.get_decommission_pool_space_info(idx).await?));
@@ -5031,6 +5466,7 @@ impl ECStore {
                    object_lock_config.as_ref(),
                    true,
                    &LcEventSrc::Decom,
+                    None,
                )
                .await
            })
@@ -5570,6 +6006,7 @@ impl ECStore {
                        lifecycle_guard: bucket_incarnation_fence
                            .as_ref()
                            .and_then(|guard| guard.namespace_lock_guard()),
+                        namespace_lock_lost_signal: None,
                        object_mutation_fence: Some(&source_cleanup_mutation_fence),
                    },
                    "decommission",
@@ -6532,7 +6969,11 @@ impl ECStore {
        validate_start_decommission_request(&indices, self.single_pool())?;

        self.ensure_decommission_rebalance_idle_after_refresh().await?;
-        ensure_decommission_start_local_leader(&self.endpoints(), &indices)?;
+        #[cfg(test)]
+        let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints());
+        #[cfg(not(test))]
+        let endpoints = self.endpoints();
+        ensure_decommission_start_local_leader(&endpoints, &indices)?;

        for idx in indices.iter().copied() {
            ensure_valid_decommission_pool_index(self.pools.len(), idx)?;
@@ -6566,7 +7007,8 @@ impl ECStore {
        }

        let _start_guard = self.start_gate.lock().await;
-        self.ensure_decommission_rebalance_idle_after_refresh().await?;
+        self.ensure_decommission_rebalance_idle_after_refresh_under_start_gate()
+            .await?;

        let all_space_infos = self.get_decommission_all_pool_space_infos().await?;
        // Signal cancellation before waiting for the movement writer so active
@@ -7776,6 +8218,7 @@ impl ECStore {
                                object_lock_config.as_ref(),
                                false,
                                &LcEventSrc::Decom,
+                                None,
                            )
                            .await
                            {
@@ -7920,6 +8363,137 @@ mod tests {
    use crate::bucket::replication::{ReplicationState, ReplicationStatusType};
    use serde::Serialize;

+    #[tokio::test]
+    #[serial_test::serial]
+    async fn decommission_activation_replicates_commit_after_post_save_fence_loss() {
+        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
+        let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
+        let start_store = Arc::clone(&store);
+        let start_task = tokio::spawn(async move {
+            start_store
+                .save_current_pool_meta_for_decommission_start(
+                    &[0],
+                    vec![(
+                        0,
+                        PoolSpaceInfo {
+                            free: 50,
+                            total: 100,
+                            used: 50,
+                        },
+                    )],
+                    Vec::new(),
+                )
+                .await
+        });
+
+        barrier.wait_until_paused().await;
+        barrier.release_after_fence_loss();
+        tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
+            .await
+            .expect("decommission activation should finish after its canonical commit")
+            .expect("decommission activation task should not panic")
+            .expect("post-commit fence loss must not report the committed activation as failed");
+
+        let local = store.pool_meta.read().await;
+        assert!(pool_meta_has_active_decommission(&local));
+        drop(local);
+
+        for pool in &store.pools {
+            let mut persisted = PoolMeta::default();
+            persisted
+                .load_no_lock_from_replicas(vec![pool.clone()])
+                .await
+                .expect("every pool should retain readable committed decommission metadata");
+            assert!(
+                pool_meta_has_active_decommission(&persisted),
+                "every pool must adopt the canonical committed activation"
+            );
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn decommission_activation_adopts_canonical_commit_after_replica_failure() {
+        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
+        let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
+        let start_store = Arc::clone(&store);
+        let start_task = tokio::spawn(async move {
+            start_store
+                .save_current_pool_meta_for_decommission_start(
+                    &[0],
+                    vec![(
+                        0,
+                        PoolSpaceInfo {
+                            free: 50,
+                            total: 100,
+                            used: 50,
+                        },
+                    )],
+                    Vec::new(),
+                )
+                .await
+        });
+
+        barrier.wait_until_paused().await;
+        let mut replica_disks = Vec::new();
+        for set in &store.pools[1].disk_set {
+            let mut disks = set.disks.write().await;
+            let saved = std::mem::take(&mut *disks);
+            *disks = vec![None; saved.len()];
+            replica_disks.push(saved);
+        }
+        barrier.release_after_fence_loss();
+
+        tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
+            .await
+            .expect("decommission activation should finish after its canonical commit")
+            .expect("decommission activation task should not panic")
+            .expect("a replica save failure must not report the committed activation as failed");
+
+        for (set, disks) in store.pools[1].disk_set.iter().zip(replica_disks) {
+            *set.disks.write().await = disks;
+        }
+
+        let local = store.pool_meta.read().await;
+        assert!(pool_meta_has_active_decommission(&local));
+        drop(local);
+
+        let mut canonical = PoolMeta::default();
+        canonical
+            .load_no_lock_from_replicas(vec![store.pools[0].clone()])
+            .await
+            .expect("the canonical committed decommission metadata should remain readable");
+        assert!(pool_meta_has_active_decommission(&canonical));
+
+        let mut replica = PoolMeta::default();
+        replica
+            .load_no_lock_from_replicas(vec![store.pools[1].clone()])
+            .await
+            .expect("the stale replica metadata should remain readable after disks recover");
+        assert!(!pool_meta_has_active_decommission(&replica));
+
+        let mut reloaded = PoolMeta::default();
+        let replica_state = reloaded
+            .load_no_lock_from_replicas(store.pools.clone())
+            .await
+            .expect("the canonical commit should remain reloadable after a replica recovers stale");
+        assert!(pool_meta_has_active_decommission(&reloaded));
+        assert!(replica_state.needs_repair);
+        assert!(replica_state.repair_write_safe);
+
+        let worker_cancel = CancellationToken::new();
+        store
+            .spawn_decommission_routines(Arc::clone(&store), worker_cancel.clone(), vec![0])
+            .await
+            .expect("the committed activation should admit its decommission worker");
+        let admitted_cancel = store.decommission_cancelers.read().await[0]
+            .clone()
+            .expect("the admitted decommission worker should have a cancellation token");
+        assert!(!admitted_cancel.is_cancelled());
+        worker_cancel.cancel();
+        assert!(admitted_cancel.is_cancelled());
+    }
+
    fn pool_meta_replica_test_meta(cmd_line: &str) -> PoolMeta {
        PoolMeta {
            version: POOL_META_VERSION,
@@ -7979,14 +8553,28 @@ mod tests {
    }

    #[test]
-    fn pool_meta_replica_selection_rejects_partial_write_divergence() {
+    fn pool_meta_replica_selection_prefers_valid_canonical_first_copy() {
+        let selection = select_pool_meta_replica(vec![
+            decode_pool_meta_replica(pool_meta_replica_test_data("pool-canonical")),
+            decode_pool_meta_replica(pool_meta_replica_test_data("pool-stale")),
+        ])
+        .expect("pool zero is the durable commit record when a later replica is stale");
+
+        assert_eq!(selection.meta.pools[0].cmd_line, "pool-canonical");
+        assert!(selection.replica_state.needs_repair);
+        assert!(selection.replica_state.repair_write_safe);
+    }
+
+    #[test]
+    fn pool_meta_replica_selection_rejects_divergent_backups_without_canonical() {
        let err = select_pool_meta_replica(vec![
+            PoolMetaReplica::Corrupt("canonical unavailable".to_string()),
            decode_pool_meta_replica(pool_meta_replica_test_data("pool-old")),
            decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
        ])
-        .expect_err("different valid snapshots have no safe ordering without a generation protocol");
+        .expect_err("divergent backups have no safe ordering when the canonical copy is unavailable");

-        assert!(err.to_string().contains("valid replicas in pools 0 and 1 diverge"));
+        assert!(err.to_string().contains("valid replicas in pools 1 and 2 diverge"));
    }

    #[test]
@@ -9149,18 +9737,18 @@ mod pools_tests {
        DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP, DECOMMISSION_META_PREFIXES,
        DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
        DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT, DecomBucketInfo, DecommissionCanceler, DecommissionDurableIlmReceipt,
-        DecommissionEntryEnqueueResult, DecommissionStartPoolState, DecommissionTerminalState, ListCallback,
-        PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, QueuedDecommissionEntry, apply_decommission_status_space_info,
-        await_decommission_worker, bind_decommission_cancelers, bind_missing_decommission_cancelers,
-        cancel_decommission_canceler, clamp_decommission_entry_concurrency, classify_decommission_terminal_state,
-        count_decommission_item, decommission_cancel_signal_result, decommission_durable_ilm_receipt_path,
-        decommission_durable_ilm_receipt_run_prefix, decommission_durable_ilm_receipt_run_token,
-        decommission_entry_queue_capacity, decommission_item_size, decommission_meta_bucket_options,
-        decommission_retry_backoff_delay, decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency,
-        default_decommission_entry_concurrency, drain_decommission_entry_queue, enqueue_decommission_entry,
-        ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_generation,
-        ensure_decommission_listing_disks_available, ensure_decommission_not_rebalancing, ensure_decommission_start_allowed,
-        ensure_decommission_start_keeps_active_pool, ensure_decommission_start_local_leader,
+        DecommissionEntryEnqueueResult, DecommissionStartPoolState, DecommissionTerminalState, ListCallback, POOL_META_NAME,
+        PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, QueuedDecommissionEntry, REBAL_META_NAME,
+        acquire_pool_rebalance_activation_locks, apply_decommission_status_space_info, await_decommission_worker,
+        bind_decommission_cancelers, bind_missing_decommission_cancelers, cancel_decommission_canceler,
+        clamp_decommission_entry_concurrency, classify_decommission_terminal_state, count_decommission_item,
+        decommission_cancel_signal_result, decommission_durable_ilm_receipt_path, decommission_durable_ilm_receipt_run_prefix,
+        decommission_durable_ilm_receipt_run_token, decommission_entry_queue_capacity, decommission_item_size,
+        decommission_meta_bucket_options, decommission_retry_backoff_delay, decommission_start_pool_state, dedup_indices,
+        default_decommission_bucket_concurrency, default_decommission_entry_concurrency, drain_decommission_entry_queue,
+        enqueue_decommission_entry, ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed,
+        ensure_decommission_generation, ensure_decommission_listing_disks_available, ensure_decommission_not_rebalancing,
+        ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool, ensure_decommission_start_local_leader,
        ensure_decommission_start_pool_states, ensure_decommission_start_rebalance_meta_allowed,
        ensure_decommission_start_target_capacity, ensure_decommission_terminal_operation_supported,
        ensure_local_decommission_pool_leaders, ensure_valid_decommission_pool_index, get_by_index, guard_decommission_cancelers,
@@ -9203,15 +9791,45 @@ mod pools_tests {
    use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntry, ObjectPartInfo};
    use rustfs_filemeta::{MetaCacheEntries, MetadataResolutionParams};
    use rustfs_rio::Index;
+    use std::future::Future;
    use std::sync::{
        Arc, Mutex as StdMutex,
        atomic::{AtomicBool, AtomicUsize, Ordering},
    };
+    use std::task::{Context, Poll};
    use std::time::Duration as StdDuration;
    use time::{Duration, OffsetDateTime};
    use tokio::sync::Semaphore;
    use tokio_util::sync::CancellationToken;

+    #[derive(Debug)]
+    struct ActivationLockRecorder {
+        lock_manager: Arc<rustfs_lock::GlobalLockManager>,
+        owner: &'static str,
+        resources: StdMutex<Vec<String>>,
+    }
+
+    #[async_trait::async_trait]
+    impl crate::storage_api_contracts::namespace::NamespaceLocking for ActivationLockRecorder {
+        type Error = Error;
+        type NamespaceLock = rustfs_lock::NamespaceLockWrapper;
+
+        async fn new_ns_lock(&self, bucket: &str, object: &str) -> crate::error::Result<Self::NamespaceLock> {
+            self.resources
+                .lock()
+                .expect("activation lock recorder should not be poisoned")
+                .push(object.to_string());
+            Ok(rustfs_lock::NamespaceLockWrapper::new(
+                rustfs_lock::NamespaceLock::with_local_manager(
+                    "activation-lock-test".to_string(),
+                    Arc::clone(&self.lock_manager),
+                ),
+                rustfs_lock::ObjectKey::new(bucket, object),
+                self.owner.to_string(),
+            ))
+        }
+    }
+
    fn noop_decommission_list_callback() -> ListCallback {
        Arc::new(|_| Box::pin(async {}))
    }
@@ -9260,6 +9878,48 @@ mod pools_tests {
        }
    }

+    #[tokio::test]
+    async fn test_activation_fence_uses_one_lock_order_and_serializes_callers() {
+        let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
+        let first = Arc::new(ActivationLockRecorder {
+            lock_manager: Arc::clone(&manager),
+            owner: "first",
+            resources: StdMutex::new(Vec::new()),
+        });
+        let second = Arc::new(ActivationLockRecorder {
+            lock_manager: manager,
+            owner: "second",
+            resources: StdMutex::new(Vec::new()),
+        });
+
+        let first_guards = acquire_pool_rebalance_activation_locks(first.clone(), None)
+            .await
+            .expect("first activation should acquire both locks");
+        assert_eq!(
+            *first
+                .resources
+                .lock()
+                .expect("activation lock recorder should not be poisoned"),
+            vec![POOL_META_NAME.to_string(), REBAL_META_NAME.to_string()]
+        );
+
+        let mut second_acquire = Box::pin(acquire_pool_rebalance_activation_locks(second.clone(), None));
+        let mut context = Context::from_waker(futures::task::noop_waker_ref());
+        assert!(matches!(second_acquire.as_mut().poll(&mut context), Poll::Pending));
+
+        drop(first_guards);
+        second_acquire
+            .await
+            .expect("second activation should acquire both locks after the first releases them");
+        assert_eq!(
+            *second
+                .resources
+                .lock()
+                .expect("activation lock recorder should not be poisoned"),
+            vec![POOL_META_NAME.to_string(), REBAL_META_NAME.to_string()]
+        );
+    }
+
    #[test]
    fn decommission_receipt_run_token_changes_with_persisted_start_time() {
        let first = OffsetDateTime::from_unix_timestamp(1_000).expect("first run timestamp should be valid");
@@ -1241,8 +1241,16 @@ pub(crate) async fn make_local_two_set_sets() -> (Vec<tempfile::TempDir>, Arc<Se
    make_local_two_set_sets_with_ctx(bootstrap_ctx()).await
 }

-#[cfg(test)]
+#[cfg(any(test, feature = "test-util"))]
 pub(crate) async fn make_local_two_set_sets_with_ctx(ctx: Arc<InstanceContext>) -> (Vec<tempfile::TempDir>, Arc<Sets>) {
+    make_local_two_set_sets_for_pool_with_ctx(ctx, 0).await
+}
+
+#[cfg(any(test, feature = "test-util"))]
+pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
+    ctx: Arc<InstanceContext>,
+    pool_idx: usize,
+) -> (Vec<tempfile::TempDir>, Arc<Sets>) {
    use crate::layout::endpoint::Endpoint;
    use rustfs_lock::client::local::LocalClient;

@@ -1258,7 +1266,7 @@ pub(crate) async fn make_local_two_set_sets_with_ctx(ctx: Arc<InstanceContext>)
            let temp_dir = tempfile::tempdir().expect("tempdir should be created");
            let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("tempdir path should be utf8"))
                .expect("endpoint should parse");
-            endpoint.set_pool_index(0);
+            endpoint.set_pool_index(pool_idx);
            endpoint.set_set_index(set_index);
            endpoint.set_disk_index(disk_index);
            let disk = new_disk(
@@ -1294,7 +1302,7 @@ pub(crate) async fn make_local_two_set_sets_with_ctx(ctx: Arc<InstanceContext>)
                2,
                1,
                set_index,
-                0,
+                pool_idx,
                endpoints,
                format.clone(),
                lockers,
@@ -1307,7 +1315,7 @@ pub(crate) async fn make_local_two_set_sets_with_ctx(ctx: Arc<InstanceContext>)
    let sets = Arc::new(Sets {
        id: format.id,
        disk_set: disk_sets,
-        pool_idx: 0,
+        pool_idx,
        endpoints: PoolEndpoints {
            legacy: false,
            set_count: 2,