test(pool): cover rebalance retry and cold-start recovery (#6720)

* test(pool): fix warp log path and retry rebalance start

- warp writes now use a unique mktemp log file instead of a fixed
  /tmp/rustfs-warp.log: the runner user could not write the stale
  root-owned file, which made the background warp process die instantly
  (warp never ran). The workflow uploads /tmp/rustfs-warp.*.log.
- rebalance start is retried (6x, 20s apart): nightly builds gate
  rebalance activation on a live cross-pool fence fleet capability proof
  that takes ~10-20s to re-establish after a pool joins. Verified live:
  attempt 1 fails with 500 'pool activation requires a live fleet
  capability proof', attempt 2 succeeds.

* test(pool): annotate known server-side issues in failure output

When a node fails to start, grab the rustfs journal tail and match known
server-side error signatures (e.g. the fleet capability proof cold-start
regression, rustfs/backlog#2031), printing a hint with the tracking issue.
Also annotate the rebalance-start retry exhaustion and the rc.3 decommission
metacache-listing failure with actionable guidance.

* fix(ecstore): defer rebalance activation without fleet proof

---------

Co-authored-by: 马登山 <cxymds@qq.com>
Co-authored-by: cxymds <cxymds@gmail.com>

crates/ecstore/src/core/pools.rs

@@ -96,6 +96,8 @@ const LOG_SUBSYSTEM_POOLS: &str = "pools";
 const EVENT_DECOMMISSION_STATE: &str = "decommission_state";
 const EVENT_DECOMMISSION_BUCKET: &str = "decommission_bucket";
 const EVENT_DECOMMISSION_ENTRY: &str = "decommission_entry";
+const POOL_ACTIVATION_FLEET_PROOF_REQUIRED: &str = "pool activation requires a live fleet capability proof";
+const POOL_ACTIVATION_FLEET_PROOF_EXPIRED: &str = "pool activation fleet capability proof expired before commit";
 const DECOMMISSION_STAGE_MIGRATE_OBJECT: &str = "migrate_object";
 const DECOMMISSION_STAGE_CLEANUP_PREFLIGHT: &str = "cleanup_preflight";
 const DECOMMISSION_STAGE_SOURCE_CLEANUP: &str = "source_cleanup";
@@ -1832,6 +1834,13 @@ pub(crate) struct PoolRebalanceActivationFence {
 }
 
 impl PoolRebalanceActivationFence {
+    pub(crate) fn set_fleet_proof(
+        &mut self,
+        fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
+    ) {
+        self.fleet_proof = fleet_proof;
+    }
+
     pub(crate) fn ensure_held(&self) -> Result<()> {
         #[cfg(test)]
         let forced_lost = self.forced_lost.load(Ordering::Acquire);
@@ -1845,7 +1854,7 @@ impl PoolRebalanceActivationFence {
             .as_ref()
             .is_some_and(|proof| !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof))
         {
-            return Err(Error::other("pool activation fleet capability proof expired before commit"));
+            return Err(Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED));
         }
 
         Ok(())
@@ -1901,7 +1910,17 @@ pub(crate) async fn acquire_pool_activation_fleet_proof(
     }
     crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof()
         .map(Some)
-        .ok_or_else(|| Error::other("pool activation requires a live fleet capability proof"))
+        .ok_or_else(|| Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED))
+}
+
+pub(crate) fn is_pool_activation_fleet_proof_error(err: &Error) -> bool {
+    // Save-stage helpers add context by formatting the original error, so the
+    // marker may be nested in the display string. Restrict matching to the
+    // `Error::other` I/O shape used by this activation path.
+    matches!(err, Error::Io(io_error) if io_error.kind() == std::io::ErrorKind::Other && {
+        let message = io_error.to_string();
+        message.contains(POOL_ACTIVATION_FLEET_PROOF_REQUIRED) || message.contains(POOL_ACTIVATION_FLEET_PROOF_EXPIRED)
+    })
 }
 
 #[cfg(test)]
@@ -10828,6 +10847,15 @@ mod tests {
     use crate::bucket::replication::{ReplicationState, ReplicationStatusType};
     use serde::Serialize;
 
+    #[test]
+    fn pool_activation_fleet_proof_error_classifier_matches_only_retryable_proof_failures() {
+        assert!(is_pool_activation_fleet_proof_error(&Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED)));
+        assert!(is_pool_activation_fleet_proof_error(&Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED)));
+        let wrapped = format!("rebalance meta save failed during start_rebalance: {POOL_ACTIVATION_FLEET_PROOF_EXPIRED}");
+        assert!(is_pool_activation_fleet_proof_error(&Error::other(wrapped)));
+        assert!(!is_pool_activation_fleet_proof_error(&Error::ConfigNotFound));
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn decommission_activation_fence_loss_after_durable_save_blocks_publication() {