refactor: clean scanner heal runtime boundaries (#3571)

crates/scanner/src/data_usage_define.rs

@@ -28,20 +28,18 @@ use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS;
 pub use rustfs_data_usage::{
     BucketTargetUsageInfo, BucketUsageInfo, DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, hash_path,
 };
-use rustfs_ecstore::{
-    bucket::{lifecycle::lifecycle::TRANSITION_COMPLETE, replication::ReplicationConfig},
-    config::{com::save_config, storageclass},
-    disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
-    error::{Error, Result as StorageResult, StorageError},
-};
 use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
 use tokio::time::{Duration, Instant, sleep, timeout};
 use tracing::warn;
 
+use crate::storage_compat::{
+    BUCKET_META_PREFIX, EcstoreError as Error, EcstoreResult as StorageResult, RUSTFS_META_BUCKET, ReplicationConfig,
+    ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, StorageError, TRANSITION_COMPLETE, save_config,
+    storageclass,
+};
 pub use crate::storage_compat::{
     ScannerGetObjectReader, ScannerObjectIO, ScannerObjectInfo, ScannerObjectOptions, ScannerObjectToDelete, ScannerPutObjReader,
 };
-use crate::storage_compat::{ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions};
 
 // Data usage constants
 pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;

crates/scanner/src/runtime_config.rs

@@ -817,6 +817,7 @@ pub(crate) fn scanner_alert_excess_folders() -> u64 {
 #[cfg(test)]
 mod tests {
     use super::{ScannerRuntimeConfigSource, lookup_scanner_runtime_config, validate_scanner_runtime_config};
+    use crate::storage_compat::init_ecstore_config_for_scanner_tests;
     use rustfs_config::server_config::{Config as ServerConfig, KVS};
     use rustfs_config::{
         DEFAULT_DELIMITER, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE,
@@ -831,7 +832,7 @@ mod tests {
     use temp_env::{with_var, with_var_unset};
 
     fn server_config_with_scanner(entries: &[(&str, &str)]) -> ServerConfig {
-        rustfs_ecstore::config::init();
+        init_ecstore_config_for_scanner_tests();
         let mut config = ServerConfig::new();
         let mut kvs = KVS::new();
         for (key, value) in entries {

crates/scanner/src/scanner.rs

@@ -39,14 +39,6 @@ use rustfs_config::{
     ENV_SCANNER_CYCLE_MAX_OBJECTS,
 };
 use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
-use rustfs_ecstore::bucket::lifecycle::lifecycle::Lifecycle as _;
-use rustfs_ecstore::bucket::metadata_sys::{get_lifecycle_config, get_replication_config};
-use rustfs_ecstore::bucket::replication::ReplicationConfigurationExt as _;
-use rustfs_ecstore::config::com::{read_config, save_config};
-use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
-use rustfs_ecstore::error::Error as EcstoreError;
-use rustfs_ecstore::global::is_erasure_sd;
-use rustfs_ecstore::store::ECStore;
 use rustfs_storage_api::{BucketOperations, BucketOptions, NamespaceLocking as _};
 use serde::{Deserialize, Serialize};
 use tokio::sync::mpsc;
@@ -54,6 +46,11 @@ use tokio::time::{Duration, Instant};
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, instrument, warn};
 
+use crate::storage_compat::{
+    ECStore, EcstoreError, Lifecycle as _, RUSTFS_META_BUCKET, ReplicationConfigurationExt as _, get_lifecycle_config,
+    get_replication_config, is_erasure_sd, read_config, replace_bucket_usage_memory_from_info, save_config,
+};
+
 const LOG_COMPONENT_SCANNER: &str = "scanner";
 const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
 const LOG_SUBSYSTEM_BACKGROUND_HEAL: &str = "background_heal";
@@ -1059,7 +1056,7 @@ pub async fn store_data_usage_in_backend(
                 "Scanner data usage save failed"
             );
         } else {
-            rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await;
+            replace_bucket_usage_memory_from_info(&data_usage_info).await;
         }
         done_save();
 
@@ -1070,6 +1067,7 @@ pub async fn store_data_usage_in_backend(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::storage_compat::EcstoreResult;
     use crate::{
         ScannerGetObjectReader as GetObjectReader, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions,
         ScannerPutObjReader as PutObjReader,
@@ -1124,7 +1122,7 @@ mod tests {
 
     #[async_trait::async_trait]
     impl rustfs_storage_api::ObjectIO for MemoryConfigStore {
-        type Error = rustfs_ecstore::error::Error;
+        type Error = EcstoreError;
         type RangeSpec = rustfs_storage_api::HTTPRangeSpec;
         type HeaderMap = http::HeaderMap;
         type ObjectOptions = ObjectOptions;
@@ -1139,12 +1137,12 @@ mod tests {
             _range: Option<rustfs_storage_api::HTTPRangeSpec>,
             _h: http::HeaderMap,
             _opts: &ObjectOptions,
-        ) -> rustfs_ecstore::error::Result<GetObjectReader> {
+        ) -> EcstoreResult<GetObjectReader> {
             let objects = self.objects.lock().await;
             let data = objects
                 .get(&memory_config_key(bucket, object))
                 .cloned()
-                .ok_or(rustfs_ecstore::error::Error::FileNotFound)?;
+                .ok_or(EcstoreError::FileNotFound)?;
 
             Ok(GetObjectReader {
                 stream: Box::new(Cursor::new(data)),
@@ -1158,7 +1156,7 @@ mod tests {
             object: &str,
             data: &mut PutObjReader,
             _opts: &ObjectOptions,
-        ) -> rustfs_ecstore::error::Result<ObjectInfo> {
+        ) -> EcstoreResult<ObjectInfo> {
             let mut buf = Vec::new();
             data.stream.read_to_end(&mut buf).await?;
             self.objects.lock().await.insert(memory_config_key(bucket, object), buf);

crates/scanner/src/scanner_folder.rs

@@ -39,25 +39,6 @@ use rustfs_common::metrics::{
     IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource, UpdateCurrentPathFn,
     current_path_updater, global_metrics,
 };
-use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
-use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule};
-use rustfs_ecstore::bucket::lifecycle::evaluator::Evaluator;
-use rustfs_ecstore::bucket::lifecycle::{
-    bucket_lifecycle_ops::apply_transition_rule,
-    lifecycle::{Event, Lifecycle, ObjectOpts},
-};
-use rustfs_ecstore::bucket::replication::{
-    ReplicationConfig, ReplicationConfigurationExt as _, ReplicationQueueAdmission, queue_replication_heal_internal,
-};
-use rustfs_ecstore::bucket::versioning::VersioningApi;
-use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
-use rustfs_ecstore::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
-use rustfs_ecstore::disk::error::DiskError;
-use rustfs_ecstore::disk::{Disk, DiskAPI as _, DiskInfoOptions};
-use rustfs_ecstore::error::StorageError;
-use rustfs_ecstore::global::is_erasure;
-use rustfs_ecstore::pools::{path2_bucket_object, path2_bucket_object_with_base_path};
-use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket;
 use rustfs_filemeta::{
     MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ReplicateObjectInfo, ReplicationStatusType, ReplicationType,
 };
@@ -69,6 +50,12 @@ use tokio::sync::mpsc;
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, warn};
 
+use crate::storage_compat::{
+    BucketVersioningSys, Disk, DiskAPI as _, DiskError, DiskInfoOptions, Evaluator, Event, GLOBAL_ExpiryState, LcEventSrc,
+    Lifecycle, ListPathRawOptions, ObjectOpts, ReplicationConfig, ReplicationConfigurationExt as _, ReplicationQueueAdmission,
+    StorageError, VersioningApi, apply_expiry_rule, apply_transition_rule, is_erasure, is_reserved_or_invalid_bucket,
+    list_path_raw, path2_bucket_object, path2_bucket_object_with_base_path, queue_replication_heal_internal,
+};
 use crate::{ScannerObjectInfo as ObjectInfo, ScannerObjectToDelete as ObjectToDelete};
 
 const LOG_COMPONENT_SCANNER: &str = "scanner";
@@ -2436,7 +2423,7 @@ mod tests {
     use crate::SCANNER_SLEEPER;
 
     use super::*;
-    use rustfs_ecstore::disk::{DiskOption, endpoint::Endpoint, new_disk};
+    use crate::storage_compat::{DiskOption, Endpoint, new_disk};
     use rustfs_filemeta::{ReplicateObjectInfo, ReplicationType, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType};
     use serial_test::serial;
     #[cfg(unix)]

crates/scanner/src/scanner_io.rs

@@ -26,22 +26,6 @@ use rustfs_common::heal_channel::HealScanMode;
 use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics};
 #[cfg(test)]
 use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS};
-use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
-use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState;
-use rustfs_ecstore::bucket::lifecycle::lifecycle::Lifecycle;
-use rustfs_ecstore::bucket::metadata_sys::{get_lifecycle_config, get_object_lock_config, get_replication_config};
-use rustfs_ecstore::bucket::replication::{ReplicationConfig, ReplicationConfigurationExt};
-use rustfs_ecstore::bucket::versioning::VersioningApi as _;
-use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
-use rustfs_ecstore::config::storageclass;
-use rustfs_ecstore::disk::STORAGE_FORMAT_FILE;
-use rustfs_ecstore::disk::error::DiskError;
-use rustfs_ecstore::disk::{Disk, DiskAPI};
-use rustfs_ecstore::error::{Error, StorageError};
-use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
-use rustfs_ecstore::resolve_object_store_handle;
-use rustfs_ecstore::set_disk::SetDisks;
-use rustfs_ecstore::{error::Result, store::ECStore};
 use rustfs_filemeta::FileMeta;
 use rustfs_storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
 use rustfs_utils::path::path_join_buf;
@@ -59,6 +43,12 @@ use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, warn};
 
 use crate::ScannerObjectInfo as ObjectInfo;
+use crate::storage_compat::{
+    BucketTargetSys, BucketVersioningSys, Disk, DiskAPI, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result,
+    GLOBAL_ExpiryState, GLOBAL_TierConfigMgr, Lifecycle, ReplicationConfig, ReplicationConfigurationExt, STORAGE_FORMAT_FILE,
+    SetDisks, StorageError, VersioningApi as _, get_lifecycle_config, get_object_lock_config, get_replication_config,
+    resolve_scanner_object_store_handle, storageclass,
+};
 
 pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file";
 const LOG_COMPONENT_SCANNER: &str = "scanner";
@@ -1433,7 +1423,7 @@ impl ScannerIODisk for Disk {
 
         // TODO: object lock
 
-        let Some(ecstore) = resolve_object_store_handle() else {
+        let Some(ecstore) = resolve_scanner_object_store_handle() else {
             error!(
                 target: "rustfs::scanner::io",
                 event = EVENT_SCANNER_DISK_BUCKET_STATE,
@@ -1529,8 +1519,7 @@ impl ScannerIODisk for Disk {
 mod tests {
     use super::*;
     use crate::scanner_folder::ScannerItem;
-    use rustfs_ecstore::disk::{DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, new_disk};
-    use rustfs_ecstore::pools::path2_bucket_object_with_base_path;
+    use crate::storage_compat::{DiskOption, Endpoint, new_disk, path2_bucket_object_with_base_path};
     use serial_test::serial;
     use temp_env::with_var;
     use uuid::Uuid;

crates/scanner/src/storage_compat.rs

@@ -13,14 +13,48 @@
 // limitations under the License.
 
 use http::HeaderMap;
-use rustfs_ecstore::{
-    error::Error,
+pub(crate) use rustfs_ecstore::{
+    bucket::{
+        bucket_target_sys::BucketTargetSys,
+        lifecycle::{
+            bucket_lifecycle_audit::LcEventSrc,
+            bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule, apply_transition_rule},
+            evaluator::Evaluator,
+            lifecycle::{Event, Lifecycle, ObjectOpts, TRANSITION_COMPLETE},
+        },
+        metadata_sys::{get_lifecycle_config, get_object_lock_config, get_replication_config},
+        replication::{
+            ReplicationConfig, ReplicationConfigurationExt, ReplicationQueueAdmission, queue_replication_heal_internal,
+        },
+        versioning::VersioningApi,
+        versioning_sys::BucketVersioningSys,
+    },
+    cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
+    config::{
+        com::{read_config, save_config},
+        storageclass,
+    },
+    data_usage::replace_bucket_usage_memory_from_info,
+    disk::{BUCKET_META_PREFIX, Disk, DiskAPI, DiskInfoOptions, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, error::DiskError},
+    error::{Error as EcstoreError, Result as EcstoreResult, StorageError},
+    global::{GLOBAL_TierConfigMgr, is_erasure, is_erasure_sd},
+    pools::{path2_bucket_object, path2_bucket_object_with_base_path},
+    set_disk::SetDisks,
+    store::ECStore,
     store_api::{
         GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions,
         ObjectToDelete as EcstoreObjectToDelete, PutObjReader as EcstorePutObjReader,
     },
+    store_utils::is_reserved_or_invalid_bucket,
 };
 use rustfs_storage_api::{HTTPRangeSpec, ObjectIO};
+use std::sync::Arc;
+
+#[cfg(test)]
+pub(crate) use rustfs_ecstore::{
+    config::init as init_ecstore_config_for_scanner_tests,
+    disk::{DiskOption, endpoint::Endpoint, new_disk},
+};
 
 pub type ScannerGetObjectReader = EcstoreGetObjectReader;
 pub type ScannerObjectInfo = EcstoreObjectInfo;
@@ -28,9 +62,13 @@ pub type ScannerObjectOptions = EcstoreObjectOptions;
 pub type ScannerObjectToDelete = EcstoreObjectToDelete;
 pub type ScannerPutObjReader = EcstorePutObjReader;
 
+pub(crate) fn resolve_scanner_object_store_handle() -> Option<Arc<ECStore>> {
+    rustfs_ecstore::resolve_object_store_handle()
+}
+
 pub trait ScannerObjectIO:
     ObjectIO<
-        Error = Error,
+        Error = EcstoreError,
         RangeSpec = HTTPRangeSpec,
         HeaderMap = HeaderMap,
         ObjectOptions = ScannerObjectOptions,
@@ -43,7 +81,7 @@ pub trait ScannerObjectIO:
 
 impl<T> ScannerObjectIO for T where
     T: ObjectIO<
-            Error = Error,
+            Error = EcstoreError,
             RangeSpec = HTTPRangeSpec,
             HeaderMap = HeaderMap,
             ObjectOptions = ScannerObjectOptions,