fix(ecstore): fence rebalance and decommission activation (#6400)

* fix(ecstore): fence rebalance and decommission activation

* fix(ecstore): fence lost activation locks

* fix(ecstore): bind rebalance workers to activation id

* fix(ecstore): close rebalance activation races

* test(ecstore): exercise lost rebalance commit fence

* fix(ecstore): repair rebalance fence test wiring

* test(ecstore): reuse rebalance metadata fixture

* fix(ecstore): satisfy rebalance activation clippy checks

* fix(ecstore): fence stale rebalance workers

* fix(ecstore): commit rebalance activation after persistence

* fix(ecstore): fence rebalance commits and unblock stop

* test(ecstore): exercise real rebalance fences

* fix(rebalance): cancel admin stop before activation wait

* fix(ecstore): fence multipart staging on rebalance lock loss

* fix(ecstore): adopt activations after durable commit

* fix(rebalance): preserve committed activation recovery

* fix(rebalance): make prepared stop terminal-safe

* fix(ecstore): repair rebalance test imports

* fix(ecstore): repair rebalance entry runtime failures

* test(ecstore): fix activation fence synchronization

* test(ecstore): scope rebalance disk trait import

* test(ecstore): observe decommission lock attempt

* fix(ecstore): align activation fence test imports

* fix(ecstore): remove duplicate activation test import

* fix(ecstore): resolve CI clippy failures

* fix: satisfy activation merge lint gates

---------

Co-authored-by: houseme <housemecn@gmail.com>

rustfs/src/admin/handlers/rebalance.rs

@@ -177,12 +177,15 @@ async fn rollback_cluster_rebalance_start(
                 terminal_reload_attempt_at: Some(terminal_reload_attempt_at),
                 terminal_reload_failures: terminal_reload_failures.clone(),
             };
-            store.record_rebalance_stop_propagation(record).await.map_err(|err| {
-                format!(
-                    "cluster rebalance rollback for {rebalance_id} partial; failed to persist stop propagation: {err}; {}",
-                    rebalance_rollback_failure_message(rebalance_id, &stop_failures, &terminal_reload_failures)
-                )
-            })?;
+            store
+                .record_rebalance_stop_propagation(rebalance_id, record)
+                .await
+                .map_err(|err| {
+                    format!(
+                        "cluster rebalance rollback for {rebalance_id} partial; failed to persist stop propagation: {err}; {}",
+                        rebalance_rollback_failure_message(rebalance_id, &stop_failures, &terminal_reload_failures)
+                    )
+                })?;
             return Err(rebalance_rollback_failure_message(
                 rebalance_id,
                 &stop_failures,
@@ -197,7 +200,7 @@ async fn rollback_cluster_rebalance_start(
         .await
         .map_err(|err| format!("local stop_rebalance rollback for {rebalance_id} failed: {err}"))?;
     store
-        .save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt)
+        .save_rebalance_stats_for_id(usize::MAX, RebalSaveOpt::StoppedAt, rebalance_id)
         .await
         .map_err(|err| format!("local rollback stop metadata save for {rebalance_id} failed: {err}"))?;
     Ok(())
@@ -679,7 +682,7 @@ impl Operation for RebalanceStart {
                                     terminal_reload_attempt_at: Some(terminal_reload_attempt_at),
                                     terminal_reload_failures: terminal_reload_failures.clone(),
                                 };
-                                store.record_rebalance_stop_propagation(record).await.map_err(|err| {
+                                store.record_rebalance_stop_propagation(&id, record).await.map_err(|err| {
                                     rebalance_internal_error(format!(
                                         "failed to persist rebalance local-start rollback propagation metadata: {err}"
                                     ))
@@ -869,6 +872,38 @@ impl Operation for RebalanceStatus {
     }
 }
 
+async fn rebalance_stop_target_id(store: &Arc<ECStore>) -> S3Result<Option<String>> {
+    store
+        .prepare_rebalance_stop()
+        .await
+        .map_err(|e| s3_error!(InternalError, "failed to prepare rebalance metadata for stop: {}", e))
+}
+
+async fn stop_rebalance_admission_first(
+    store: &Arc<ECStore>,
+    notification_sys: Option<&NotificationSys>,
+    expected_rebalance_id: &str,
+) -> S3Result<Vec<String>> {
+    // prepare_rebalance_stop already closed admission for this exact run.
+
+    if let Some(notification_sys) = notification_sys {
+        return notification_sys
+            .stop_rebalance_failures(Some(expected_rebalance_id))
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to stop rebalance via notification system: {}", e));
+    }
+
+    store
+        .stop_rebalance_for_id(Some(expected_rebalance_id))
+        .await
+        .map_err(|e| s3_error!(InternalError, "failed to stop rebalance: {}", e))?;
+    store
+        .save_rebalance_stats_for_id(usize::MAX, RebalSaveOpt::StoppedAt, expected_rebalance_id)
+        .await
+        .map_err(|e| s3_error!(InternalError, "failed to persist rebalance stop metadata: {}", e))?;
+    Ok(Vec::new())
+}
+
 // RebalanceStop
 pub struct RebalanceStop {}
 
@@ -916,36 +951,15 @@ impl Operation for RebalanceStop {
             return Err(s3_error!(InternalError, "object layer is not initialized"));
         };
 
-        store
-            .load_rebalance_meta()
-            .await
-            .map_err(|e| s3_error!(InternalError, "failed to load rebalance metadata before stop: {}", e))?;
-        let expected_rebalance_id = store.current_rebalance_id().await;
-
-        if !store.is_rebalance_conflicting_with_decommission().await {
+        let Some(expected_rebalance_id) = rebalance_stop_target_id(&store).await? else {
             log_rebalance_request_rejected("stop", "rebalance_not_started", &request_id, &actor, &remote_addr);
             return Err(s3_error!(NoSuchResource, "pool rebalance is not started"));
-        }
+        };
 
         let notification_sys = current_notification_system();
         let stop_attempt_at = OffsetDateTime::now_utc();
-        let mut stop_failures = Vec::new();
-        if let Some(notification_sys) = notification_sys.as_ref() {
-            stop_failures = notification_sys
-                .stop_rebalance_failures(expected_rebalance_id.as_deref())
-                .await
-                .map_err(|e| s3_error!(InternalError, "failed to stop rebalance via notification system: {}", e))?;
-        } else {
-            store
-                .stop_rebalance_for_id(expected_rebalance_id.as_deref())
-                .await
-                .map_err(|e| s3_error!(InternalError, "failed to stop rebalance: {}", e))?;
-
-            store
-                .save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt)
-                .await
-                .map_err(|e| s3_error!(InternalError, "failed to persist rebalance stop metadata: {}", e))?;
-        }
+        let stop_failures =
+            stop_rebalance_admission_first(&store, notification_sys.as_deref(), expected_rebalance_id.as_str()).await?;
 
         info!(
             event = EVENT_ADMIN_REBALANCE_STATE,
@@ -1007,7 +1021,7 @@ impl Operation for RebalanceStop {
                 terminal_reload_failures: terminal_reload_failures.clone(),
             };
             store
-                .record_rebalance_stop_propagation(record)
+                .record_rebalance_stop_propagation(expected_rebalance_id.as_str(), record)
                 .await
                 .map_err(|e| s3_error!(InternalError, "failed to persist rebalance stop propagation metadata: {}", e))?;
 
@@ -1081,15 +1095,218 @@ mod rebalance_handler_tests {
         RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, RebalanceStartStep, RebalanceStopPropagationStatus,
         build_rebalance_admin_status, build_rebalance_pool_statuses, build_rebalance_stop_propagation_status,
         rebalance_pool_used, rebalance_query_present, rebalance_remaining_buckets, rebalance_rollback_failure_message,
-        rebalance_rollback_stop_failure_message, rebalance_start_rollback_error, rebalance_start_steps, rebalance_used_pct,
-        rollback_result_label,
+        rebalance_rollback_stop_failure_message, rebalance_start_rollback_error, rebalance_start_steps, rebalance_stop_target_id,
+        rebalance_used_pct, rollback_result_label, stop_rebalance_admission_first,
     };
     use crate::admin::storage_api::rebalance::{
-        DiskStat, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta,
-        RebalanceStats, RebalanceStopPropagationRecord, encode_rebalance_stop_propagation_record,
+        DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo,
+        RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, encode_rebalance_stop_propagation_record,
     };
     use time::OffsetDateTime;
 
+    fn started_rebalance_meta(id: &str) -> RebalanceMeta {
+        RebalanceMeta {
+            id: id.to_string(),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    start_time: Some(OffsetDateTime::now_utc()),
+                    status: RebalStatus::Started,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn real_admin_stop_cancels_paused_entry_before_waiting_for_activation_gate() {
+        const REBALANCE_ID: &str = "admin-stop-paused-entry";
+        let mut fixture =
+            crate::admin::storage_api::ecstore_rebalance::test_util::PausedRebalanceEntryTestFixture::new(REBALANCE_ID).await;
+        fixture.wait_until_entry_paused().await;
+
+        let stop_store = fixture.store();
+        let mut stop_task = tokio::spawn(async move {
+            let expected_rebalance_id = rebalance_stop_target_id(&stop_store)
+                .await
+                .expect("admin stop target resolution should succeed")
+                .expect("the active rebalance should remain stoppable");
+            stop_rebalance_admission_first(&stop_store, None, expected_rebalance_id.as_str()).await
+        });
+        fixture.wait_until_admission_cancelled().await;
+        fixture.wait_until_stop_waiting_for_entry().await;
+        assert!(!stop_task.is_finished(), "admin stop must wait for the paused entry guard to drain");
+
+        fixture.release_entry();
+        fixture.assert_entry_cancelled().await;
+        let stop_failures = tokio::time::timeout(std::time::Duration::from_secs(5), &mut stop_task)
+            .await
+            .expect("admin stop should finish after the entry guard drains")
+            .expect("admin stop task should not panic")
+            .expect("admin stop should persist the terminal state");
+        assert!(stop_failures.is_empty());
+        assert!(!fixture.store().is_rebalance_conflicting_with_decommission().await);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn real_admin_stop_accepts_same_run_terminalization_after_prepare() {
+        const REBALANCE_ID: &str = "admin-stop-terminal-after-prepare";
+        const REPLACEMENT_ID: &str = "admin-stop-replacement";
+        let (_temp_dirs, store) =
+            crate::admin::storage_api::ecstore_rebalance::test_util::test_store_with_persisted_rebalance_meta(
+                started_rebalance_meta(REBALANCE_ID),
+            )
+            .await;
+        let terminal_barrier = std::sync::Arc::new(tokio::sync::Barrier::new(2));
+        let worker_barrier = std::sync::Arc::clone(&terminal_barrier);
+        let worker_store = std::sync::Arc::clone(&store);
+        let terminal_task = tokio::spawn(async move {
+            worker_barrier.wait().await;
+            {
+                let mut rebalance_meta = worker_store.rebalance_meta.write().await;
+                let meta = rebalance_meta
+                    .as_mut()
+                    .expect("the prepared rebalance metadata should remain installed");
+                assert_eq!(meta.id, REBALANCE_ID);
+                let pool = meta
+                    .pool_stats
+                    .first_mut()
+                    .expect("the prepared rebalance should have a pool");
+                pool.info.status = RebalStatus::Stopped;
+                pool.info.end_time = Some(OffsetDateTime::now_utc());
+            }
+            worker_store
+                .save_rebalance_stats_for_id(0, RebalSaveOpt::Stats, REBALANCE_ID)
+                .await
+        });
+
+        let expected_rebalance_id = rebalance_stop_target_id(&store)
+            .await
+            .expect("admin stop target resolution should succeed")
+            .expect("the active rebalance should remain stoppable");
+        assert_eq!(expected_rebalance_id, REBALANCE_ID);
+        let cancel = store
+            .rebalance_meta
+            .read()
+            .await
+            .as_ref()
+            .and_then(|meta| meta.cancel.clone())
+            .expect("prepare should install the admission cancellation token");
+        assert!(cancel.is_cancelled());
+
+        terminal_barrier.wait().await;
+        tokio::time::timeout(std::time::Duration::from_secs(30), terminal_task)
+            .await
+            .expect("worker terminalization should finish after the barrier opens")
+            .expect("worker terminalization task should not panic")
+            .expect("worker terminalization should persist");
+        assert!(!store.is_rebalance_conflicting_with_decommission().await);
+
+        *store.rebalance_meta.write().await = None;
+        store
+            .load_rebalance_meta()
+            .await
+            .expect("the worker terminal state should reload before the final stop");
+        {
+            let terminal = store.rebalance_meta.read().await;
+            let terminal = terminal.as_ref().expect("the worker terminal state should remain persisted");
+            assert_eq!(terminal.id, REBALANCE_ID);
+            assert_eq!(terminal.pool_stats[0].info.status, RebalStatus::Stopped);
+        }
+
+        let stop_failures = stop_rebalance_admission_first(&store, None, expected_rebalance_id.as_str())
+            .await
+            .expect("same-run terminalization after prepare should be a successful stop");
+        assert!(stop_failures.is_empty());
+
+        *store.rebalance_meta.write().await = Some(started_rebalance_meta(REPLACEMENT_ID));
+        let error = stop_rebalance_admission_first(&store, None, expected_rebalance_id.as_str())
+            .await
+            .expect_err("the prepared stop must not mutate a replacement run");
+        assert!(error.to_string().contains(REBALANCE_ID));
+        let replacement = store.rebalance_meta.read().await;
+        let replacement = replacement.as_ref().expect("the replacement run should remain installed");
+        assert_eq!(replacement.id, REPLACEMENT_ID);
+        assert_eq!(replacement.pool_stats[0].info.status, RebalStatus::Started);
+        assert!(replacement.cancel.is_none());
+        assert!(replacement.stopped_at.is_none());
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn real_admin_stop_loads_persisted_active_rebalance_from_cold_memory() {
+        const REBALANCE_ID: &str = "admin-stop-cold-memory";
+        let (_temp_dirs, store) =
+            crate::admin::storage_api::ecstore_rebalance::test_util::test_store_with_persisted_rebalance_meta(
+                started_rebalance_meta(REBALANCE_ID),
+            )
+            .await;
+        *store.rebalance_meta.write().await = None;
+        assert!(store.current_rebalance_id().await.is_none());
+
+        let expected_rebalance_id = rebalance_stop_target_id(&store)
+            .await
+            .expect("admin stop should load persisted rebalance metadata")
+            .expect("persisted active rebalance should be stoppable");
+        assert_eq!(expected_rebalance_id, REBALANCE_ID);
+
+        let stop_failures = stop_rebalance_admission_first(&store, None, expected_rebalance_id.as_str())
+            .await
+            .expect("admin stop should persist the terminal state after a cold load");
+        assert!(stop_failures.is_empty());
+        assert!(!store.is_rebalance_conflicting_with_decommission().await);
+
+        *store.rebalance_meta.write().await = None;
+        store
+            .load_rebalance_meta()
+            .await
+            .expect("the persisted terminal rebalance metadata should remain readable");
+        assert_eq!(store.current_rebalance_id().await.as_deref(), Some(REBALANCE_ID));
+        assert!(!store.is_rebalance_conflicting_with_decommission().await);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn real_admin_stop_refreshes_persisted_active_over_stale_inactive_memory() {
+        const PERSISTED_REBALANCE_ID: &str = "admin-stop-persisted-active";
+        const STALE_REBALANCE_ID: &str = "admin-stop-stale-terminal";
+        let (_temp_dirs, store) =
+            crate::admin::storage_api::ecstore_rebalance::test_util::test_store_with_persisted_rebalance_meta(
+                started_rebalance_meta(PERSISTED_REBALANCE_ID),
+            )
+            .await;
+        *store.rebalance_meta.write().await = Some(RebalanceMeta {
+            id: STALE_REBALANCE_ID.to_string(),
+            stopped_at: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        });
+        assert_eq!(store.current_rebalance_id().await.as_deref(), Some(STALE_REBALANCE_ID));
+        assert!(!store.is_rebalance_conflicting_with_decommission().await);
+
+        let expected_rebalance_id = rebalance_stop_target_id(&store)
+            .await
+            .expect("admin stop should refresh stale inactive local metadata")
+            .expect("persisted active rebalance should replace the stale local terminal state");
+        assert_eq!(expected_rebalance_id, PERSISTED_REBALANCE_ID);
+
+        let stop_failures = stop_rebalance_admission_first(&store, None, expected_rebalance_id.as_str())
+            .await
+            .expect("admin stop should persist the refreshed run's terminal state");
+        assert!(stop_failures.is_empty());
+
+        *store.rebalance_meta.write().await = None;
+        store
+            .load_rebalance_meta()
+            .await
+            .expect("the refreshed run's persisted terminal metadata should remain readable");
+        assert_eq!(store.current_rebalance_id().await.as_deref(), Some(PERSISTED_REBALANCE_ID));
+        assert!(!store.is_rebalance_conflicting_with_decommission().await);
+    }
+
     #[test]
     fn test_calculate_rebalance_progress_running() {
         let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap();

rustfs/src/admin/storage_api.rs

@@ -70,7 +70,9 @@ mod ecstore_notification {
 }
 
 #[allow(unused_imports)]
-mod ecstore_rebalance {
+pub(crate) mod ecstore_rebalance {
+    #[cfg(test)]
+    pub(crate) use crate::storage::storage_api::ecstore_rebalance::test_util;
     pub(crate) use crate::storage::storage_api::ecstore_rebalance::{
         DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo,
         RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, decode_rebalance_stop_propagation_record,

rustfs/src/app/object_usecase.rs

@@ -65,8 +65,10 @@ use super::storage_api::object_usecase::contract::http::HTTPPreconditions;
 use super::storage_api::object_usecase::contract::namespace::NamespaceLocking;
 use super::storage_api::object_usecase::contract::object::{ObjectIO as _, ObjectOperations as _};
 use super::storage_api::object_usecase::contract::range::HTTPRangeSpec;
+#[cfg(test)]
+use super::storage_api::object_usecase::data_usage::apply_bucket_usage_memory_overlay;
 use super::storage_api::object_usecase::data_usage::{
-    apply_bucket_usage_memory_overlay, quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
+    quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
     record_bucket_object_version_write_memory, record_bucket_object_write_memory,
     record_bucket_object_write_unknown_previous_memory,
 };

rustfs/src/storage/rpc/node_service.rs

@@ -153,7 +153,7 @@ fn remove_heal_control_replay(
 
 static HEAL_CONTROL_REPLAY_CACHE: OnceLock<tokio::sync::Mutex<HashMap<String, Arc<HealControlReplayEntry>>>> = OnceLock::new();
 static NODE_CAPABILITY_SERVER_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
-const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1;
+const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
 
 fn admit_heal_control_replay(
     replay_cache: &mut HashMap<String, Arc<HealControlReplayEntry>>,
@@ -3309,7 +3309,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn cross_pool_fence_probe_authenticates_supported_v1_state() {
+    async fn cross_pool_fence_probe_authenticates_supported_v2_state() {
         let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string());
         let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true);
         assert!(
@@ -3374,7 +3374,7 @@ mod tests {
 
         assert!(response.success);
         assert_eq!(response.error_info, None);
-        assert_eq!(&response.result[..4], &1_u32.to_be_bytes());
+        assert_eq!(&response.result[..4], &2_u32.to_be_bytes());
         let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(&response.result[4..])
             .expect("capability identity should decode");
         assert_eq!(topology_member, "node-a:9000");

rustfs/src/storage/storage_api.rs

@@ -502,6 +502,8 @@ pub(crate) mod ecstore_notification {
 
 #[allow(unused_imports)]
 pub(crate) mod ecstore_rebalance {
+    #[cfg(test)]
+    pub(crate) use rustfs_ecstore::api::rebalance::test_util;
     pub(crate) use rustfs_ecstore::api::rebalance::{
         DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo,
         RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, decode_rebalance_stop_propagation_record,