test(ecstore): add shared MockWarmBackend test utility for lifecycle and tier tests (#4716)

* test(ecstore): extract shared MockWarmBackend into a test-util feature (backlog#1148 ilm-6)

The tier/lifecycle integration tests carried two byte-for-byte copies of an
in-memory WarmBackend mock — one in crates/scanner/tests and one in
rustfs/src/app — plus duplicated register_mock_tier and polling helpers. Both
implemented the same ecstore WarmBackend trait.

Consolidate them into ecstore behind a new `test-util` feature, exposed via the
`rustfs_ecstore::api::tier::test_util` facade:

- MockWarmBackend: in-memory WarmBackend with an operation log (for ordering
  assertions such as "local delete precedes remote remove") and fault injection
  (FaultConfig): unreachable, HTTP 5xx, credential rejection, injected latency,
  plus external_remove to simulate an out-of-band remote deletion.
- register_mock_tier / register_mock_tier_backend: register the mock into any
  TierConfigMgr handle (the global manager used by scanner tests or a
  per-instance one used by the app tests).
- xl.meta transition assertion helpers: read_transition_meta,
  assert_transition_meta_consistent (cross-shard consistency of the
  status/tier/remote-key/remote-version-id tuple plus free-version count), and
  free_version_count.
- polling helpers: wait_for_remote_absence, wait_for_object_count,
  wait_for_free_version_absence.

Both existing copies now consume this single definition; `rg 'struct
MockWarmBackend'` collapses to one. The feature is enabled only from
[dev-dependencies], so it never links into the production binary (resolver 3).

Designed for downstream ilm-8 (restore lifecycle) and ilm-11 (tier fault
injection matrix). Coordinates with #4706 (ilm-2), which adds op-logging to the
scanner mock — that op-logging is now part of this shared surface, so #4706
should rebase onto it.

Refs rustfs/backlog#1148 (ilm-6), rustfs/backlog#1155.

* test(ecstore): fix shared MockWarmBackend usage after main merge

- Access stored objects via MockWarmBackend::contains() instead of the now
  private inner objects map (fixes E0609 after the shared test-util refactor).
- Drop dead ReadCloser/ReaderImpl/DiskAPI imports and the unused
  transition_api test re-exports the mock extraction left behind.
- Reword the scanner/rustfs test-util dependency comments so they no longer
  embed the literal rustfs_ecstore:: path that trips the ECStore
  architecture-migration guard.

rustfs/Cargo.toml

@@ -215,6 +215,9 @@ opentelemetry_sdk = { workspace = true }
 rsa = { workspace = true }
 rcgen = { workspace = true }
 criterion = { workspace = true, features = ["html_reports"] }
+# Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via
+# the ecstore `api::tier::test_util` facade module (rustfs/backlog#1148 ilm-6).
+rustfs-ecstore = { workspace = true, features = ["test-util"] }
 
 [build-dependencies]
 http.workspace = true

rustfs/src/app/lifecycle_transition_api_test.rs

@@ -16,7 +16,6 @@ use super::storage_api::test::bucket::{
     lifecycle,
     metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG},
     metadata_sys,
-    transition_api::{ReadCloser, ReaderImpl},
 };
 use super::storage_api::test::contract::{
     bucket::{BucketOperations, BucketOptions, MakeBucketOptions},
@@ -26,7 +25,7 @@ use super::storage_api::test::contract::{
 };
 use super::storage_api::test::ecfs::FS;
 use super::storage_api::test::object_utils::to_s3s_etag;
-use super::storage_api::test::runtime::{AppWarmBackend, TierConfig, TierType, WarmBackendGetOpts};
+use super::storage_api::test::runtime::{MockWarmBackend, register_mock_tier as register_mock_tier_util};
 use super::storage_api::test::{
     ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, StorageObjectInfo as ObjectInfo,
     StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader,
@@ -44,17 +43,15 @@ use rustfs_utils::http::{SUFFIX_FORCE_DELETE, insert_header};
 use s3s::{S3Request, dto::*};
 use serial_test::serial;
 use std::{
-    collections::HashMap,
     convert::Infallible,
     env, fs as stdfs,
-    io::Cursor,
     path::PathBuf,
     sync::{Arc, Once, OnceLock},
     time::Duration,
 };
 use tokio::fs;
 use tokio::io::AsyncReadExt;
-use tokio::sync::{Barrier, Mutex};
+use tokio::sync::Barrier;
 use tokio_util::sync::CancellationToken;
 use uuid::Uuid;
 
@@ -285,90 +282,11 @@ fn del_marker_expiration_lifecycle_configuration(days: i32) -> BucketLifecycleCo
     }
 }
 
-#[derive(Clone, Default)]
-struct MockWarmBackend {
-    objects: Arc<Mutex<HashMap<String, Vec<u8>>>>,
-}
-
-impl MockWarmBackend {
-    async fn put_bytes(&self, object: &str, bytes: Vec<u8>) -> String {
-        self.objects.lock().await.insert(object.to_string(), bytes);
-        Uuid::new_v4().to_string()
-    }
-
-    async fn read_bytes(&self, reader: ReaderImpl) -> Result<Vec<u8>, std::io::Error> {
-        match reader {
-            ReaderImpl::Body(bytes) => Ok(bytes.to_vec()),
-            ReaderImpl::ObjectBody(mut reader) => {
-                let mut buf = Vec::new();
-                reader.stream.read_to_end(&mut buf).await?;
-                Ok(buf)
-            }
-        }
-    }
-}
-
-#[async_trait::async_trait]
-impl AppWarmBackend for MockWarmBackend {
-    async fn put(&self, object: &str, r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
-        let bytes = self.read_bytes(r).await?;
-        Ok(self.put_bytes(object, bytes).await)
-    }
-
-    async fn put_with_meta(
-        &self,
-        object: &str,
-        r: ReaderImpl,
-        _length: i64,
-        _meta: HashMap<String, String>,
-    ) -> Result<String, std::io::Error> {
-        let bytes = self.read_bytes(r).await?;
-        Ok(self.put_bytes(object, bytes).await)
-    }
-
-    async fn get(&self, object: &str, _rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
-        let objects = self.objects.lock().await;
-        let Some(bytes) = objects.get(object) else {
-            return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "mock object not found"));
-        };
-
-        let start = opts.start_offset.max(0) as usize;
-        let end = if opts.length > 0 {
-            start.saturating_add(opts.length as usize).min(bytes.len())
-        } else {
-            bytes.len()
-        };
-
-        Ok(tokio::io::BufReader::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
-    }
-
-    async fn remove(&self, object: &str, _rv: &str) -> Result<(), std::io::Error> {
-        self.objects.lock().await.remove(object);
-        Ok(())
-    }
-
-    async fn in_use(&self) -> Result<bool, std::io::Error> {
-        Ok(false)
-    }
-}
-
+/// Register the shared [`MockWarmBackend`] into this instance's tier config
+/// manager. Thin wrapper over the shared `register_mock_tier` helper
+/// (rustfs/backlog#1148 ilm-6).
 async fn register_mock_tier(tier_name: &str) -> MockWarmBackend {
-    let backend = MockWarmBackend::default();
-    let tier_config_mgr_handle = current_tier_config_handle();
-    let mut tier_config_mgr = tier_config_mgr_handle.write().await;
-    tier_config_mgr.tiers.insert(
-        tier_name.to_string(),
-        TierConfig {
-            version: "v1".to_string(),
-            tier_type: TierType::MinIO,
-            name: tier_name.to_string(),
-            ..Default::default()
-        },
-    );
-    tier_config_mgr
-        .driver_cache
-        .insert(tier_name.to_string(), Box::new(backend.clone()));
-    backend
+    register_mock_tier_util(&current_tier_config_handle(), tier_name).await
 }
 
 async fn wait_for_transition(ecstore: &Arc<ECStore>, bucket: &str, object: &str, timeout: Duration) -> Option<ObjectInfo> {
@@ -441,22 +359,6 @@ where
     }
 }
 
-async fn wait_for_remote_absence(backend: &MockWarmBackend, object: &str, timeout: Duration) -> bool {
-    let deadline = tokio::time::Instant::now() + timeout;
-
-    loop {
-        if !backend.objects.lock().await.contains_key(object) {
-            return true;
-        }
-
-        if tokio::time::Instant::now() >= deadline {
-            return false;
-        }
-
-        tokio::time::sleep(Duration::from_millis(50)).await;
-    }
-}
-
 async fn wait_for_object_absence(ecstore: &Arc<ECStore>, bucket: &str, object: &str, timeout: Duration) -> bool {
     let deadline = tokio::time::Instant::now() + timeout;
 
@@ -851,7 +753,7 @@ async fn put_and_copy_object_transition_immediately_via_usecases() {
 
     assert_eq!(put_info.transitioned_object.status, "complete");
     assert_eq!(put_info.transitioned_object.tier, tier_name);
-    assert!(backend.objects.lock().await.contains_key(&put_info.transitioned_object.name));
+    assert!(backend.contains(&put_info.transitioned_object.name).await);
 
     let src_bucket = format!("test-api-copy-src-{}", &Uuid::new_v4().simple().to_string()[..8]);
     let dst_bucket = format!("test-api-copy-dst-{}", &Uuid::new_v4().simple().to_string()[..8]);
@@ -887,7 +789,7 @@ async fn put_and_copy_object_transition_immediately_via_usecases() {
 
     assert_eq!(copy_info.transitioned_object.status, "complete");
     assert_eq!(copy_info.transitioned_object.tier, tier_name);
-    assert!(backend.objects.lock().await.contains_key(&copy_info.transitioned_object.name));
+    assert!(backend.contains(&copy_info.transitioned_object.name).await);
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
@@ -944,7 +846,7 @@ async fn complete_multipart_upload_transitions_immediately_via_usecase() {
 
     assert_eq!(info.transitioned_object.status, "complete");
     assert_eq!(info.transitioned_object.tier, tier_name);
-    assert!(backend.objects.lock().await.contains_key(&info.transitioned_object.name));
+    assert!(backend.contains(&info.transitioned_object.name).await);
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
@@ -993,13 +895,7 @@ async fn get_transitioned_object_uses_remote_codec_fallback_path() {
         assert_eq!(transitioned.transitioned_object.status, "complete");
         assert_eq!(transitioned.transitioned_object.tier, tier_name);
         assert!(!transitioned.transitioned_object.name.is_empty());
-        assert!(
-            backend
-                .objects
-                .lock()
-                .await
-                .contains_key(&transitioned.transitioned_object.name)
-        );
+        assert!(backend.contains(&transitioned.transitioned_object.name).await);
 
         let actual = read_object_bytes(&ecstore, bucket.as_str(), object).await;
         assert_eq!(actual, payload);
@@ -1036,7 +932,7 @@ async fn delete_transitioned_object_removes_remote_tier_copy_via_usecase() {
         .expect("object should transition before delete usecase runs");
     let remote_object = transitioned.transitioned_object.name.clone();
 
-    assert!(backend.objects.lock().await.contains_key(&remote_object));
+    assert!(backend.contains(&remote_object).await);
 
     let mut req = build_request(
         DeleteObjectInput::builder()
@@ -1058,7 +954,7 @@ async fn delete_transitioned_object_removes_remote_tier_copy_via_usecase() {
     );
 
     assert!(
-        wait_for_remote_absence(&backend, &remote_object, TRANSITION_WAIT_TIMEOUT).await,
+        backend.wait_for_remote_absence(&remote_object, TRANSITION_WAIT_TIMEOUT).await,
         "transitioned object should be removed from remote tier after delete usecase"
     );
 }
@@ -1134,7 +1030,7 @@ async fn immediate_transition_timeout_eventually_completes_via_compensation() {
 
     assert_eq!(info.transitioned_object.status, "complete");
     assert_eq!(info.transitioned_object.tier, tier_name);
-    assert!(backend.objects.lock().await.contains_key(&info.transitioned_object.name));
+    assert!(backend.contains(&info.transitioned_object.name).await);
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
@@ -1184,7 +1080,7 @@ async fn compensation_driven_copy_still_completes_transition() {
 
     assert_eq!(info.transitioned_object.status, "complete");
     assert_eq!(info.transitioned_object.tier, tier_name);
-    assert!(backend.objects.lock().await.contains_key(&info.transitioned_object.name));
+    assert!(backend.contains(&info.transitioned_object.name).await);
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
@@ -1244,7 +1140,7 @@ async fn compensation_driven_complete_multipart_upload_still_transitions() {
 
     assert_eq!(info.transitioned_object.status, "complete");
     assert_eq!(info.transitioned_object.tier, tier_name);
-    assert!(backend.objects.lock().await.contains_key(&info.transitioned_object.name));
+    assert!(backend.contains(&info.transitioned_object.name).await);
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
@@ -1276,7 +1172,7 @@ async fn compensation_driven_transition_still_cleans_remote_tier_on_delete() {
         .expect("object should eventually transition after compensation backfill");
     let remote_object = transitioned.transitioned_object.name.clone();
 
-    assert!(backend.objects.lock().await.contains_key(&remote_object));
+    assert!(backend.contains(&remote_object).await);
 
     let mut req = build_request(
         DeleteObjectInput::builder()
@@ -1298,7 +1194,7 @@ async fn compensation_driven_transition_still_cleans_remote_tier_on_delete() {
     );
 
     assert!(
-        wait_for_remote_absence(&backend, &remote_object, TRANSITION_WAIT_TIMEOUT).await,
+        backend.wait_for_remote_absence(&remote_object, TRANSITION_WAIT_TIMEOUT).await,
         "transitioned object should be removed from remote tier after delete usecase"
     );
 }
@@ -1332,7 +1228,7 @@ async fn compensation_driven_versioned_delete_still_creates_delete_marker() {
         .expect("object should eventually transition after compensation backfill");
     let remote_object = transitioned.transitioned_object.name.clone();
 
-    assert!(backend.objects.lock().await.contains_key(&remote_object));
+    assert!(backend.contains(&remote_object).await);
 
     let req = build_request(
         DeleteObjectInput::builder()
@@ -1352,7 +1248,7 @@ async fn compensation_driven_versioned_delete_still_creates_delete_marker() {
         "versioned delete should create a delete marker after compensation-driven transition"
     );
     assert!(
-        backend.objects.lock().await.contains_key(&remote_object),
+        backend.contains(&remote_object).await,
         "creating a delete marker should not remove the transitioned remote object version"
     );
 }
@@ -1387,7 +1283,7 @@ async fn compensation_driven_delete_marker_still_honors_lifecycle_cleanup() {
         .expect("object should eventually transition after compensation backfill");
     let remote_object = transitioned.transitioned_object.name.clone();
 
-    assert!(backend.objects.lock().await.contains_key(&remote_object));
+    assert!(backend.contains(&remote_object).await);
 
     let req = build_request(
         DeleteObjectInput::builder()
@@ -1407,7 +1303,7 @@ async fn compensation_driven_delete_marker_still_honors_lifecycle_cleanup() {
         "versioned delete should create a delete marker before lifecycle cleanup"
     );
     assert!(
-        backend.objects.lock().await.contains_key(&remote_object),
+        backend.contains(&remote_object).await,
         "delete marker creation should keep the transitioned remote object version"
     );
 
@@ -1429,7 +1325,7 @@ async fn compensation_driven_delete_marker_still_honors_lifecycle_cleanup() {
         "delete marker should remain visible after lifecycle update until cleanup completes"
     );
     assert!(
-        backend.objects.lock().await.contains_key(&remote_object),
+        backend.contains(&remote_object).await,
         "delete marker lifecycle cleanup should not remove the transitioned remote object version"
     );
 }

rustfs/src/app/storage_api.rs

@@ -145,14 +145,11 @@ pub(crate) mod runtime {
     pub(crate) type TierConfigMgr = crate::storage::storage_api::TierConfigMgr;
     pub(crate) type TransitionState = crate::storage::storage_api::TransitionState;
 
+    // Shared MockWarmBackend + register_mock_tier helper (rustfs/backlog#1148 ilm-6).
+    // The mock (and the tier types it used to need) now live in ecstore behind
+    // the `test-util` feature.
     #[cfg(test)]
-    pub(crate) type TierConfig = crate::storage::storage_api::ecstore_tier::tier_config::TierConfig;
-    #[cfg(test)]
-    pub(crate) type TierType = crate::storage::storage_api::ecstore_tier::tier_config::TierType;
-    #[cfg(test)]
-    pub(crate) use crate::storage::storage_api::ecstore_tier::warm_backend::WarmBackend as AppWarmBackend;
-    #[cfg(test)]
-    pub(crate) type WarmBackendGetOpts = crate::storage::storage_api::ecstore_tier::warm_backend::WarmBackendGetOpts;
+    pub(crate) use crate::storage::storage_api::ecstore_tier::test_util::{MockWarmBackend, register_mock_tier};
 
     pub(crate) fn set_global_storage_class(cfg: StorageClassConfig) {
         crate::storage::storage_api::ecstore_config::set_global_storage_class(cfg);
@@ -807,12 +804,6 @@ pub(crate) mod bucket {
         }
     }
 
-    #[cfg(test)]
-    pub(crate) mod transition_api {
-        pub(crate) type ReadCloser = crate::storage::storage_api::ecstore_client::transition_api::ReadCloser;
-        pub(crate) type ReaderImpl = crate::storage::storage_api::ecstore_client::transition_api::ReaderImpl;
-    }
-
     pub(crate) mod versioning_sys {
         pub(crate) type BucketVersioningSys = crate::storage::storage_api::ecstore_bucket::versioning_sys::BucketVersioningSys;
     }

rustfs/src/storage/storage_api.rs

@@ -335,8 +335,6 @@ pub(crate) mod ecstore_capacity {
 }
 
 pub(crate) mod ecstore_client {
-    #[cfg(test)]
-    pub(crate) use rustfs_ecstore::api::client::transition_api;
     pub(crate) use rustfs_ecstore::api::client::{admin_handler_utils, object_api_utils};
 }
 
@@ -464,9 +462,11 @@ pub(crate) mod ecstore_storage {
 
 pub(crate) mod ecstore_tier {
     pub(crate) use rustfs_ecstore::api::tier::tier::TierConfigMgr;
-    #[cfg(test)]
-    pub(crate) use rustfs_ecstore::api::tier::warm_backend;
     pub(crate) use rustfs_ecstore::api::tier::{tier, tier_admin, tier_config, tier_handlers};
+    // Shared lifecycle/tier test utilities behind ecstore's `test-util` feature
+    // (rustfs/backlog#1148 ilm-6). Only linked into test builds.
+    #[cfg(test)]
+    pub(crate) use rustfs_ecstore::api::tier::test_util;
 }
 
 pub(crate) const BUCKET_ACCELERATE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_ACCELERATE_CONFIG;