fix(tiering): make rejected upload cleanup durable (#5059)

* fix(tiering): make rejected upload cleanup durable

* fix(tiering): close transition upload cancellation gap

* test(tiering): cover failed upload without candidate

* test(tiering): synchronize cancelled cleanup recovery

* test(tiering): stabilize cancelled cleanup recovery

Prefer cancellation when the tier delete journal recovery worker is racing an immediate tick, and build the cancelled-cleanup regression store with an already-cancelled token so production recovery cannot consume the test journal.

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>

crates/ecstore/src/services/tier/test_util.rs

@@ -57,7 +57,10 @@
 use std::collections::HashMap;
 use std::io::Cursor;
 use std::path::Path;
-use std::sync::Arc;
+use std::sync::{
+    Arc,
+    atomic::{AtomicBool, AtomicUsize, Ordering},
+};
 use std::time::Duration;
 
 use async_trait::async_trait;
@@ -75,6 +78,21 @@ use crate::services::tier::warm_backend::{WarmBackend, WarmBackendGetOpts, build
 use rustfs_filemeta::FileMeta;
 use rustfs_utils::path::path_join_buf;
 
+/// One-shot barrier before rejected transition cleanup resolves its ECStore.
+pub struct TransitionCleanupStoreBarrier(crate::set_disk::SetDiskTransitionCleanupStoreBarrier);
+
+impl TransitionCleanupStoreBarrier {
+    /// Install the barrier for the next rejected transition cleanup.
+    pub fn install() -> Self {
+        Self(crate::set_disk::SetDiskTransitionCleanupStoreBarrier::install())
+    }
+
+    /// Wait until the rejected transition reaches cleanup-store resolution.
+    pub async fn wait_until_paused(&self) {
+        self.0.wait_until_paused().await;
+    }
+}
+
 /// Default polling cadence used by the `wait_for_*` helpers.
 const POLL_INTERVAL: Duration = Duration::from_millis(50);
 
@@ -142,11 +160,15 @@ struct MockWarmBackendInner {
     faults: Mutex<FaultConfig>,
     put_read_limit: Mutex<Option<usize>>,
     put_remote_version: Mutex<Option<String>>,
+    reject_non_empty_remote_versions: AtomicBool,
+    fail_remove: AtomicBool,
+    exact_remove_count: AtomicUsize,
     op_log: Mutex<Vec<MockWarmOp>>,
     put_versions: Mutex<Vec<(String, String)>>,
     remove_versions: Mutex<Vec<(String, String)>>,
     put_barrier: Mutex<Option<Arc<MockPutBarrierState>>>,
     get_barrier: Mutex<Option<Arc<MockGetBarrierState>>>,
+    remove_barrier: Mutex<Option<Arc<MockRemoveBarrierState>>>,
 }
 
 #[derive(Default)]
@@ -162,6 +184,23 @@ struct MockGetBarrierState {
     fail_after_release: bool,
 }
 
+#[derive(Default)]
+struct MockRemoveBarrierState {
+    arrived: Notify,
+    release: Notify,
+    operation_dropped: Notify,
+}
+
+struct MockRemoveOperationGuard {
+    state: Arc<MockRemoveBarrierState>,
+}
+
+impl Drop for MockRemoveOperationGuard {
+    fn drop(&mut self) {
+        self.state.operation_dropped.notify_one();
+    }
+}
+
 /// One-shot barrier that pauses a mock tier PUT after storing its remote body.
 pub struct MockPutBarrier {
     state: Arc<MockPutBarrierState>,
@@ -212,6 +251,38 @@ impl Drop for MockGetBarrier {
     }
 }
 
+/// One-shot barrier that pauses and then fails a mock tier DELETE.
+pub struct MockRemoveBarrier {
+    state: Arc<MockRemoveBarrierState>,
+}
+
+impl MockRemoveBarrier {
+    /// Wait until DELETE reaches the deterministic failure point.
+    pub async fn wait_until_paused(&self) {
+        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
+            .await
+            .expect("mock tier DELETE should reach the deterministic barrier");
+    }
+
+    /// Release the paused DELETE, which then returns an injected error.
+    pub fn release(&self) {
+        self.state.release.notify_one();
+    }
+
+    /// Wait until the paused DELETE future completes or is cancelled.
+    pub async fn wait_until_operation_dropped(&self) {
+        tokio::time::timeout(Duration::from_secs(30), self.state.operation_dropped.notified())
+            .await
+            .expect("mock tier DELETE operation should be dropped");
+    }
+}
+
+impl Drop for MockRemoveBarrier {
+    fn drop(&mut self) {
+        self.state.release.notify_one();
+    }
+}
+
 /// In-memory [`WarmBackend`] for lifecycle / tiering integration tests.
 ///
 /// Cloning shares the same underlying storage, fault configuration, and
@@ -235,6 +306,15 @@ impl MockWarmBackend {
         MockPutBarrier { state }
     }
 
+    /// Pause and then fail the next DELETE after it reaches the backend.
+    pub async fn arm_failing_remove_barrier(&self) -> MockRemoveBarrier {
+        let state = Arc::new(MockRemoveBarrierState::default());
+        let mut barrier = self.inner.remove_barrier.lock().await;
+        assert!(barrier.is_none(), "mock tier DELETE barrier is already armed");
+        *barrier = Some(state.clone());
+        MockRemoveBarrier { state }
+    }
+
     /// Arm a one-shot pause before the next tier GET, then return an error
     /// after the test releases it.
     pub async fn arm_failing_get_barrier(&self) -> MockGetBarrier {
@@ -298,6 +378,16 @@ impl MockWarmBackend {
         *self.inner.put_remote_version.lock().await = remote_version;
     }
 
+    /// Reject non-empty remote versions before transition metadata is committed.
+    pub fn set_reject_non_empty_remote_versions(&self, reject: bool) {
+        self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release);
+    }
+
+    /// Enable or disable a persistent remove failure for durability tests.
+    pub fn set_remove_failure(&self, fail: bool) {
+        self.inner.fail_remove.store(fail, Ordering::Release);
+    }
+
     async fn precondition(&self) -> Result<(), std::io::Error> {
         let (latency, error) = {
             let faults = self.inner.faults.lock().await;
@@ -339,6 +429,11 @@ impl MockWarmBackend {
             .count()
     }
 
+    /// Number of exact-version trait remove calls, including failed attempts.
+    pub fn exact_remove_count(&self) -> usize {
+        self.inner.exact_remove_count.load(Ordering::Acquire)
+    }
+
     /// Return the exact object/version pairs produced by successful tier PUTs.
     pub async fn put_versions(&self) -> Vec<(String, String)> {
         self.inner.put_versions.lock().await.clone()
@@ -484,6 +579,13 @@ impl MockWarmBackend {
 
 #[async_trait]
 impl WarmBackend for MockWarmBackend {
+    fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
+        if self.inner.reject_non_empty_remote_versions.load(Ordering::Acquire) && !remote_version_id.is_empty() {
+            return Err(std::io::Error::other("mock warm backend requires an unversioned remote object"));
+        }
+        Ok(())
+    }
+
     async fn put(&self, object: &str, r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
         self.precondition().await?;
         let bytes = self.read_bytes(r).await?;
@@ -582,6 +684,15 @@ impl WarmBackend for MockWarmBackend {
 
     async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
         self.precondition().await?;
+        if let Some(barrier) = self.inner.remove_barrier.lock().await.take() {
+            let _operation = MockRemoveOperationGuard { state: barrier.clone() };
+            barrier.arrived.notify_one();
+            barrier.release.notified().await;
+            return Err(std::io::Error::other("mock warm backend remove failure after barrier"));
+        }
+        if self.inner.fail_remove.load(Ordering::Acquire) {
+            return Err(std::io::Error::other("mock warm backend remove failure"));
+        }
         let mut objects = self.inner.objects.lock().await;
         if let Some(stored) = objects.get(object)
             && !rv.is_empty()
@@ -603,6 +714,17 @@ impl WarmBackend for MockWarmBackend {
         Ok(())
     }
 
+    async fn remove_exact(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        if rv.is_empty() {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                "an exact mock tier delete requires a remote version ID",
+            ));
+        }
+        self.inner.exact_remove_count.fetch_add(1, Ordering::AcqRel);
+        self.remove(object, rv).await
+    }
+
     async fn in_use(&self) -> Result<bool, std::io::Error> {
         self.precondition().await?;
         self.record(MockWarmOp::InUse).await;

crates/ecstore/src/services/tier/tier.rs

@@ -672,6 +672,14 @@ struct SharedWarmBackendProxy(SharedWarmBackend);
 
 #[async_trait::async_trait]
 impl WarmBackend for SharedWarmBackendProxy {
+    async fn validate(&self) -> io::Result<()> {
+        self.0.validate().await
+    }
+
+    fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
+        self.0.validate_remote_version_id(remote_version_id)
+    }
+
     async fn put(&self, object: &str, r: crate::client::transition_api::ReaderImpl, length: i64) -> io::Result<String> {
         self.0.put(object, r, length).await
     }
@@ -699,6 +707,10 @@ impl WarmBackend for SharedWarmBackendProxy {
         self.0.remove(object, rv).await
     }
 
+    async fn remove_exact(&self, object: &str, rv: &str) -> io::Result<()> {
+        self.0.remove_exact(object, rv).await
+    }
+
     async fn in_use(&self) -> io::Result<bool> {
         self.0.in_use().await
     }
@@ -3046,6 +3058,22 @@ mod tests {
 
     #[async_trait::async_trait]
     impl WarmBackend for MockWarmBackend {
+        async fn validate(&self) -> std::result::Result<(), std::io::Error> {
+            if self.healthy {
+                Ok(())
+            } else {
+                Err(std::io::Error::other("mock validation failed"))
+            }
+        }
+
+        fn validate_remote_version_id(&self, remote_version_id: &str) -> std::result::Result<(), std::io::Error> {
+            if remote_version_id == "unsupported-version" {
+                Err(std::io::Error::other("mock remote version rejected"))
+            } else {
+                Ok(())
+            }
+        }
+
         async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> std::result::Result<String, std::io::Error> {
             if self.healthy {
                 Ok("mock-version".to_string())
@@ -3085,6 +3113,13 @@ mod tests {
             }
         }
 
+        async fn remove_exact(&self, object: &str, rv: &str) -> std::result::Result<(), std::io::Error> {
+            if rv == "exact-only" {
+                return Err(std::io::Error::other("mock exact remove forwarded"));
+            }
+            self.remove(object, rv).await
+        }
+
         async fn in_use(&self) -> std::result::Result<bool, std::io::Error> {
             match self.in_use_value {
                 Some(b) => Ok(b),
@@ -3442,6 +3477,32 @@ mod tests {
             .expect_err("an unhealthy backend must fail verification");
     }
 
+    #[tokio::test]
+    async fn shared_backend_proxy_forwards_validation_hooks() {
+        let unhealthy: SharedWarmBackend = Arc::new(MockWarmBackend {
+            in_use_value: Some(false),
+            healthy: false,
+        });
+        let proxy = SharedWarmBackendProxy(unhealthy);
+        let err = proxy.validate().await.expect_err("proxy must forward backend validation");
+        assert_eq!(err.to_string(), "mock validation failed");
+
+        let healthy: SharedWarmBackend = Arc::new(MockWarmBackend {
+            in_use_value: Some(false),
+            healthy: true,
+        });
+        let proxy = SharedWarmBackendProxy(healthy);
+        let err = proxy
+            .validate_remote_version_id("unsupported-version")
+            .expect_err("proxy must forward remote version validation");
+        assert_eq!(err.to_string(), "mock remote version rejected");
+        let err = proxy
+            .remove_exact("remote-object", "exact-only")
+            .await
+            .expect_err("proxy must forward exact-version cleanup");
+        assert_eq!(err.to_string(), "mock exact remove forwarded");
+    }
+
     // ---- pure query helpers --------------------------------------------
 
     #[test]

crates/ecstore/src/services/tier/warm_backend.rs

@@ -25,7 +25,7 @@ use crate::client::{
 };
 use crate::error::is_err_bucket_not_found;
 use crate::services::tier::{
-    tier::ERR_TIER_TYPE_UNSUPPORTED,
+    tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
     tier_config::{TierConfig, TierType},
     tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
     warm_backend_aliyun::WarmBackendAliyun,
@@ -65,6 +65,14 @@ pub struct WarmBackendGetOpts {
 
 #[async_trait::async_trait]
 pub trait WarmBackend {
+    async fn validate(&self) -> Result<(), std::io::Error> {
+        Ok(())
+    }
+
+    fn validate_remote_version_id(&self, _remote_version_id: &str) -> Result<(), std::io::Error> {
+        Ok(())
+    }
+
     /// Return `Ok` only after the backend has consumed the complete declared
     /// body and its storage service has acknowledged the PUT. The built-in S3
     /// family uses the transition client's declared-length request plus
@@ -83,6 +91,15 @@ pub trait WarmBackend {
     ) -> Result<String, std::io::Error>;
     async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error>;
     async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>;
+    async fn remove_exact(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        if rv.is_empty() {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                "an exact tier delete requires a remote version ID",
+            ));
+        }
+        self.remove(object, rv).await
+    }
     async fn in_use(&self) -> Result<bool, std::io::Error>;
 }
 
@@ -165,16 +182,23 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
 
 pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
     let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
+    w.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
     let remote_version_id = w
         .put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
-        .await;
-    if let Err(err) = remote_version_id {
-        return Err(ERR_TIER_PERM_ERR.clone());
+        .await
+        .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
+
+    if w.validate_remote_version_id(&remote_version_id).is_err() {
+        w.remove_exact(PROBE_OBJECT, &remote_version_id)
+            .await
+            .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
+        return Err(ERR_TIER_INVALID_CONFIG.clone());
     }
 
-    let r = w.get(PROBE_OBJECT, "", WarmBackendGetOpts::default()).await;
+    let read_result = w.get(PROBE_OBJECT, &remote_version_id, WarmBackendGetOpts::default()).await;
+    let remove_result = w.remove(PROBE_OBJECT, &remote_version_id).await;
     //xhttp.DrainBody(r);
-    if let Err(err) = r {
+    if read_result.is_err() || remove_result.is_err() {
         //if is_err_bucket_not_found(&err) {
         //    return Err(ERR_TIER_BUCKET_NOT_FOUND);
         //}
@@ -185,11 +209,6 @@ pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), Admin
         return Err(ERR_TIER_PERM_ERR.clone());
         //}
     }
-    if let Ok(version_id) = remote_version_id {
-        if let Err(err) = w.remove(PROBE_OBJECT, &version_id).await {
-            return Err(ERR_TIER_PERM_ERR.clone());
-        };
-    }
     Ok(())
 }
 
@@ -391,6 +410,237 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    };
+
+    const PROBE_VERSION: &str = "remote-v2";
+
+    struct RejectingValidationBackend {
+        validations: Arc<AtomicUsize>,
+        puts: Arc<AtomicUsize>,
+        removes: Arc<AtomicUsize>,
+    }
+
+    struct RejectingProbeVersionBackend {
+        gets: Arc<AtomicUsize>,
+        removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+    }
+
+    struct RecordingProbeBackend {
+        get_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+        removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+        fail_get: bool,
+    }
+
+    #[async_trait::async_trait]
+    impl WarmBackend for RejectingValidationBackend {
+        async fn validate(&self) -> Result<(), std::io::Error> {
+            self.validations.fetch_add(1, Ordering::SeqCst);
+            Err(std::io::Error::other("invalid backend configuration"))
+        }
+
+        async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
+            self.puts.fetch_add(1, Ordering::SeqCst);
+            Ok(String::new())
+        }
+
+        async fn put_with_meta(
+            &self,
+            object: &str,
+            r: ReaderImpl,
+            length: i64,
+            _meta: HashMap<String, String>,
+        ) -> Result<String, std::io::Error> {
+            self.put(object, r, length).await
+        }
+
+        async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+            Err(std::io::Error::other("get must not run after validation failure"))
+        }
+
+        async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
+            self.removes.fetch_add(1, Ordering::SeqCst);
+            Err(std::io::Error::other("remove must not run after validation failure"))
+        }
+
+        async fn in_use(&self) -> Result<bool, std::io::Error> {
+            Err(std::io::Error::other("in_use must not run after validation failure"))
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl WarmBackend for RejectingProbeVersionBackend {
+        fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
+            if remote_version_id.is_empty() {
+                Ok(())
+            } else {
+                Err(std::io::Error::other("probe returned a version ID"))
+            }
+        }
+
+        async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
+            Ok(uuid::Uuid::nil().to_string())
+        }
+
+        async fn put_with_meta(
+            &self,
+            object: &str,
+            r: ReaderImpl,
+            length: i64,
+            _meta: HashMap<String, String>,
+        ) -> Result<String, std::io::Error> {
+            self.put(object, r, length).await
+        }
+
+        async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+            self.gets.fetch_add(1, Ordering::SeqCst);
+            Err(std::io::Error::other("GET must not run for a rejected probe version"))
+        }
+
+        async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
+            Err(std::io::Error::other("generic remove must not run for a rejected fresh PUT response"))
+        }
+
+        async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            self.removed_versions.lock().await.push(rv.to_string());
+            Ok(())
+        }
+
+        async fn in_use(&self) -> Result<bool, std::io::Error> {
+            Ok(false)
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl WarmBackend for RecordingProbeBackend {
+        async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
+            Ok(PROBE_VERSION.to_string())
+        }
+
+        async fn put_with_meta(
+            &self,
+            object: &str,
+            r: ReaderImpl,
+            length: i64,
+            _meta: HashMap<String, String>,
+        ) -> Result<String, std::io::Error> {
+            self.put(object, r, length).await
+        }
+
+        async fn get(&self, _object: &str, rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+            self.get_versions.lock().await.push(rv.to_string());
+            if self.fail_get {
+                Err(std::io::Error::other("probe GET failed"))
+            } else {
+                Ok(ReadCloser::new(std::io::Cursor::new(Vec::new())))
+            }
+        }
+
+        async fn remove(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            self.removed_versions.lock().await.push(rv.to_string());
+            Ok(())
+        }
+
+        async fn in_use(&self) -> Result<bool, std::io::Error> {
+            Ok(false)
+        }
+    }
+
+    #[tokio::test]
+    async fn check_warm_backend_validates_before_probe_io() {
+        let validations = Arc::new(AtomicUsize::new(0));
+        let puts = Arc::new(AtomicUsize::new(0));
+        let removes = Arc::new(AtomicUsize::new(0));
+        let backend: WarmBackendImpl = Box::new(RejectingValidationBackend {
+            validations: validations.clone(),
+            puts: puts.clone(),
+            removes: removes.clone(),
+        });
+
+        let err = check_warm_backend(Some(&backend))
+            .await
+            .expect_err("invalid backend configuration should fail before probe I/O");
+
+        assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
+        assert_eq!(validations.load(Ordering::SeqCst), 1);
+        assert_eq!(puts.load(Ordering::SeqCst), 0);
+        assert_eq!(removes.load(Ordering::SeqCst), 0);
+    }
+
+    #[tokio::test]
+    async fn default_exact_remove_rejects_an_empty_version() {
+        let removes = Arc::new(AtomicUsize::new(0));
+        let backend = RejectingValidationBackend {
+            validations: Arc::new(AtomicUsize::new(0)),
+            puts: Arc::new(AtomicUsize::new(0)),
+            removes: removes.clone(),
+        };
+
+        let err = backend
+            .remove_exact("remote-object", "")
+            .await
+            .expect_err("an empty exact constraint must fail closed");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+        assert_eq!(removes.load(Ordering::SeqCst), 0);
+    }
+
+    #[tokio::test]
+    async fn check_warm_backend_removes_exact_probe_when_versioning_drifts() {
+        let gets = Arc::new(AtomicUsize::new(0));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
+            gets: gets.clone(),
+            removed_versions: removed_versions.clone(),
+        });
+
+        let err = check_warm_backend(Some(&backend))
+            .await
+            .expect_err("a probe version ID must fail an unversioned backend check");
+
+        assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
+        assert_eq!(gets.load(Ordering::SeqCst), 0);
+        assert_eq!(removed_versions.lock().await.as_slice(), [uuid::Uuid::nil().to_string()]);
+    }
+
+    #[tokio::test]
+    async fn check_warm_backend_forwards_probe_version_to_get_and_remove() {
+        let get_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
+            get_versions: get_versions.clone(),
+            removed_versions: removed_versions.clone(),
+            fail_get: false,
+        });
+
+        check_warm_backend(Some(&backend))
+            .await
+            .expect("a successful probe should validate, read, and remove its object");
+
+        assert_eq!(get_versions.lock().await.as_slice(), [PROBE_VERSION]);
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
+    #[tokio::test]
+    async fn check_warm_backend_removes_probe_after_get_failure() {
+        let get_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
+            get_versions: get_versions.clone(),
+            removed_versions: removed_versions.clone(),
+            fail_get: true,
+        });
+
+        let err = check_warm_backend(Some(&backend))
+            .await
+            .expect_err("a failed probe GET should return a permission error after cleanup");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert_eq!(get_versions.lock().await.as_slice(), [PROBE_VERSION]);
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
 
     #[test]
     fn build_transition_put_options_preserves_content_headers() {