fix(ecstore): harden tier reader and restore cleanup races (#5035)

* fix(tier): hold generation lease through readers

Refs rustfs/backlog#1354

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(restore): fence failed cleanup by source identity

Refs rustfs/backlog#1356

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>

crates/ecstore/src/services/tier/test_util.rs

@@ -146,6 +146,7 @@ struct MockWarmBackendInner {
     put_versions: Mutex<Vec<(String, String)>>,
     remove_versions: Mutex<Vec<(String, String)>>,
     put_barrier: Mutex<Option<Arc<MockPutBarrierState>>>,
+    get_barrier: Mutex<Option<Arc<MockGetBarrierState>>>,
 }
 
 #[derive(Default)]
@@ -154,6 +155,13 @@ struct MockPutBarrierState {
     release: Notify,
 }
 
+#[derive(Default)]
+struct MockGetBarrierState {
+    arrived: Notify,
+    release: Notify,
+    fail_after_release: bool,
+}
+
 /// One-shot barrier that pauses a mock tier PUT after storing its remote body.
 pub struct MockPutBarrier {
     state: Arc<MockPutBarrierState>,
@@ -179,6 +187,31 @@ impl Drop for MockPutBarrier {
     }
 }
 
+/// One-shot barrier that pauses a mock tier GET before it reads remote bytes.
+pub struct MockGetBarrier {
+    state: Arc<MockGetBarrierState>,
+}
+
+impl MockGetBarrier {
+    /// Wait until the GET has reached the deterministic pause point.
+    pub async fn wait_until_paused(&self) {
+        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
+            .await
+            .expect("mock tier GET should reach the deterministic barrier");
+    }
+
+    /// Release the paused GET.
+    pub fn release(&self) {
+        self.state.release.notify_one();
+    }
+}
+
+impl Drop for MockGetBarrier {
+    fn drop(&mut self) {
+        self.state.release.notify_one();
+    }
+}
+
 /// In-memory [`WarmBackend`] for lifecycle / tiering integration tests.
 ///
 /// Cloning shares the same underlying storage, fault configuration, and
@@ -202,6 +235,17 @@ impl MockWarmBackend {
         MockPutBarrier { state }
     }
 
+    /// Arm a one-shot pause before the next tier GET, then return an error
+    /// after the test releases it.
+    pub async fn arm_failing_get_barrier(&self) -> MockGetBarrier {
+        let state = Arc::new(MockGetBarrierState {
+            fail_after_release: true,
+            ..Default::default()
+        });
+        *self.inner.get_barrier.lock().await = Some(Arc::clone(&state));
+        MockGetBarrier { state }
+    }
+
     // ---- fault injection -------------------------------------------------
 
     /// Replace the entire fault configuration.
@@ -500,6 +544,14 @@ impl WarmBackend for MockWarmBackend {
 
     async fn get(&self, object: &str, _rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
         self.precondition().await?;
+        let barrier = self.inner.get_barrier.lock().await.take();
+        if let Some(barrier) = barrier {
+            barrier.arrived.notify_one();
+            barrier.release.notified().await;
+            if barrier.fail_after_release {
+                return Err(std::io::Error::other("mock warm backend GET failed after barrier"));
+            }
+        }
         self.record(MockWarmOp::Get {
             object: object.to_string(),
         })

crates/ecstore/src/services/tier/tier.rs

@@ -2220,6 +2220,19 @@ impl TierConfigMgr {
         Ok(lease)
     }
 
+    #[cfg(test)]
+    pub(crate) async fn active_operation_lease_count(handle: &Arc<RwLock<Self>>, tier_name: &str) -> usize {
+        let manager = handle.read().await;
+        let Some(runtime) = registered_tier_driver_runtime(&manager) else {
+            return 0;
+        };
+        lock_unpoisoned(&runtime)
+            .generations
+            .get(tier_name)
+            .map(|generation| generation.active_leases.load(Ordering::Acquire))
+            .unwrap_or(0)
+    }
+
     fn replace_driver(&mut self, tier_name: &str, driver: WarmBackendImpl) -> std::result::Result<(), AdminError> {
         let Some(runtime) = registered_tier_driver_runtime(self) else {
             self.driver_cache.insert(tier_name.to_string(), driver);