test(pool): cover rebalance retry and cold-start recovery (#6720)

* test(pool): fix warp log path and retry rebalance start

- warp writes now use a unique mktemp log file instead of a fixed
  /tmp/rustfs-warp.log: the runner user could not write the stale
  root-owned file, which made the background warp process die instantly
  (warp never ran). The workflow uploads /tmp/rustfs-warp.*.log.
- rebalance start is retried (6x, 20s apart): nightly builds gate
  rebalance activation on a live cross-pool fence fleet capability proof
  that takes ~10-20s to re-establish after a pool joins. Verified live:
  attempt 1 fails with 500 'pool activation requires a live fleet
  capability proof', attempt 2 succeeds.

* test(pool): annotate known server-side issues in failure output

When a node fails to start, grab the rustfs journal tail and match known
server-side error signatures (e.g. the fleet capability proof cold-start
regression, rustfs/backlog#2031), printing a hint with the tracking issue.
Also annotate the rebalance-start retry exhaustion and the rc.3 decommission
metacache-listing failure with actionable guidance.

* fix(ecstore): defer rebalance activation without fleet proof

---------

Co-authored-by: 马登山 <cxymds@qq.com>
Co-authored-by: cxymds <cxymds@gmail.com>

crates/ecstore/src/services/notification_sys.rs

@@ -234,6 +234,39 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
     });
 }
 
+#[cfg(test)]
+pub(crate) struct CrossPoolFenceFleetProofGuard {
+    previous_proof: Option<FleetCapabilityProof>,
+    previous_topology_conflict: bool,
+}
+
+#[cfg(test)]
+impl Drop for CrossPoolFenceFleetProofGuard {
+    fn drop(&mut self) {
+        let mut state = cross_pool_fence_fleet_proof_slot()
+            .write()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        state.proof = self.previous_proof.take();
+        state.topology_conflict = self.previous_topology_conflict;
+    }
+}
+
+/// Temporarily revoke the test proof so activation paths can exercise their
+/// fail-closed behavior without changing the process-wide topology binding.
+#[cfg(test)]
+pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceFleetProofGuard {
+    let mut state = cross_pool_fence_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    let guard = CrossPoolFenceFleetProofGuard {
+        previous_proof: state.proof.clone(),
+        previous_topology_conflict: state.topology_conflict,
+    };
+    state.proof = None;
+    state.topology_conflict = true;
+    guard
+}
+
 #[cfg(any(test, feature = "test-util"))]
 pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool {
     let mut state = cross_pool_fence_fleet_proof_slot()

crates/ecstore/src/services/rebalance/control.rs

@@ -570,10 +570,13 @@ impl ECStore {
     where
         S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
     {
-        let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
+        // Lock order: pool_meta_save_gate -> pool.bin -> rebalance.bin.
         let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
         pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
-        let activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), fleet_proof).await?;
+        // Classify the durable rebalance record while holding both namespace
+        // fences. A terminal record is a no-op and must not depend on the
+        // notification subsystem having published a fleet proof yet.
+        let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
         let pool_meta = self
             .load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, "rebalance worker activation")
             .await?;
@@ -597,10 +600,17 @@ impl ECStore {
         }
 
         activation_fence.ensure_held()?;
-        if !is_rebalance_conflicting_with_decommission(&persisted) {
+        if !crate::services::rebalance::rebalance_requires_worker_activation(&persisted) {
             return Ok(RebalanceWorkerActivationFence::NotStartedTerminal);
         }
 
+        // Active worker admission still requires the fail-closed fleet proof.
+        // Attach it immediately before the final fence validation so expiry or
+        // topology changes are checked again at every later commit boundary.
+        let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
+        activation_fence.set_fleet_proof(fleet_proof);
+        activation_fence.ensure_held()?;
+
         Ok(RebalanceWorkerActivationFence::Ready(Box::new(activation_fence)))
     }
 
@@ -1476,6 +1486,64 @@ mod tests {
         assert_activation_locks_released(&store).await;
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn rebalance_worker_skips_terminal_metadata_without_fleet_proof() {
+        let rebalance_id = "terminal-metadata-without-proof";
+        let completed = RebalanceMeta {
+            id: rebalance_id.to_string(),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Completed,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(completed)).await;
+        let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
+
+        let activation = store
+            .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
+            .await
+            .expect("terminal metadata should not require a fleet proof");
+        assert!(matches!(activation, RebalanceWorkerActivationFence::NotStartedTerminal));
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn rebalance_worker_still_requires_fleet_proof_for_active_metadata() {
+        let rebalance_id = "active-metadata-without-proof";
+        let active = RebalanceMeta {
+            id: rebalance_id.to_string(),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await;
+        let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
+
+        let err = match store
+            .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
+            .await
+        {
+            Ok(_) => panic!("active metadata must not be admitted without a fleet proof"),
+            Err(err) => err,
+        };
+        assert!(
+            err.to_string()
+                .contains("pool activation requires a live fleet capability proof")
+        );
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn rebalance_activation_adopts_commit_after_post_save_fence_loss() {

crates/ecstore/src/services/rebalance/meta.rs

@@ -214,6 +214,14 @@ pub(super) fn is_rebalance_in_progress(meta: &RebalanceMeta) -> bool {
     meta.pool_stats.iter().any(is_rebalance_pool_active)
 }
 
+/// Persisted rebalance metadata requires worker activation only while it has
+/// not reached a durable terminal marker and at least one pool is still marked
+/// active. Merely finding `rebalance.bin` is not evidence that admission is
+/// required: terminal metadata is retained for status reporting.
+pub(crate) fn rebalance_requires_worker_activation(meta: &RebalanceMeta) -> bool {
+    meta.stopped_at.is_none() && is_rebalance_in_progress(meta)
+}
+
 pub(crate) fn is_rebalance_conflicting_with_decommission(meta: &RebalanceMeta) -> bool {
     is_rebalance_in_progress(meta)
 }

crates/ecstore/src/services/rebalance/mod.rs

@@ -49,8 +49,8 @@ mod worker;
 
 #[cfg(feature = "test-util")]
 pub use entry::test_util::PausedRebalanceEntryTestFixture;
-pub(crate) use meta::is_rebalance_conflicting_with_decommission;
 pub use meta::{decode_rebalance_stop_propagation_record, encode_rebalance_stop_propagation_record};
+pub(crate) use meta::{is_rebalance_conflicting_with_decommission, rebalance_requires_worker_activation};
 pub use types::{
     DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta,
     RebalanceStats, RebalanceStopPropagationRecord,

crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs

@@ -22,11 +22,11 @@ use super::meta::{
     is_rebalance_in_progress, is_rebalance_meta_replaceable_for_new_id, is_rebalance_stopped_terminal_event,
     mark_rebalance_bucket_done, merge_rebalance_bucket_lists, merge_rebalance_meta, next_rebal_bucket_from_stat,
     percent_free_ratio, rebalance_goal_reached, rebalance_meta_load_no_data_error, rebalance_meta_load_unknown_format_error,
-    rebalance_meta_load_unknown_version_error, record_rebalance_cleanup_warning_in_meta, remove_rebalanced_buckets_from_queue,
-    resolve_next_rebalance_bucket, resolve_rebalance_participants, should_accept_rebalance_stats_update,
-    should_ignore_rebalance_data_usage_cache, should_pool_participate, should_preserve_rebalance_stopped_state,
-    should_skip_start_rebalance, stop_rebalance_meta_snapshot, stop_rebalance_state, take_bucket_from_rebalance_queue,
-    validate_init_rebalance_state, validate_start_rebalance_state,
+    rebalance_meta_load_unknown_version_error, rebalance_requires_worker_activation, record_rebalance_cleanup_warning_in_meta,
+    remove_rebalanced_buckets_from_queue, resolve_next_rebalance_bucket, resolve_rebalance_participants,
+    should_accept_rebalance_stats_update, should_ignore_rebalance_data_usage_cache, should_pool_participate,
+    should_preserve_rebalance_stopped_state, should_skip_start_rebalance, stop_rebalance_meta_snapshot, stop_rebalance_state,
+    take_bucket_from_rebalance_queue, validate_init_rebalance_state, validate_start_rebalance_state,
 };
 use super::migration::{
     MigrationBackend, MigrationVersionResult, migrate_entry_version, migrate_entry_version_with_retry_wait,
@@ -3386,6 +3386,52 @@ fn test_is_rebalance_in_progress_only_started_participants() {
     assert!(is_rebalance_in_progress(&meta));
 }
 
+#[test]
+fn test_rebalance_requires_worker_activation_only_for_active_non_stopped_metadata() {
+    let now = OffsetDateTime::now_utc();
+    let active = RebalanceMeta {
+        pool_stats: vec![RebalanceStats {
+            participating: true,
+            info: RebalanceInfo {
+                status: RebalStatus::Started,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
+    let stopped_active = RebalanceMeta {
+        stopped_at: Some(now),
+        pool_stats: active.pool_stats.clone(),
+        ..Default::default()
+    };
+
+    assert!(rebalance_requires_worker_activation(&active));
+    for status in [
+        RebalStatus::Completed,
+        RebalStatus::Stopped,
+        RebalStatus::Failed,
+        RebalStatus::None,
+    ] {
+        let terminal = RebalanceMeta {
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        assert!(
+            !rebalance_requires_worker_activation(&terminal),
+            "terminal status {status:?} must not resume"
+        );
+    }
+    assert!(!rebalance_requires_worker_activation(&stopped_active));
+}
+
 #[test]
 fn test_is_rebalance_conflicting_with_decommission_true_when_in_progress() {
     let now = OffsetDateTime::now_utc();