refactor: clean runtime and test storage boundaries (#3573)

* refactor: clean runtime observability select boundaries

* refactor: clean test harness fuzz storage boundaries

crates/e2e_test/src/lib.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 mod reliant;
+mod storage_compat;
 
 // Common utilities for all E2E tests
 #[cfg(test)]

crates/e2e_test/src/reliant/grpc_lock_client.rs

@@ -15,8 +15,8 @@
 // Used by test_distributed_lock_4_nodes_grpc in lock.rs
 #![allow(dead_code)]
 
+use crate::storage_compat::ecstore::rpc::node_service_time_out_client_no_auth;
 use async_trait::async_trait;
-use rustfs_ecstore::rpc::node_service_time_out_client_no_auth;
 use rustfs_lock::{
     LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
     types::{LockMetadata, LockPriority},
@@ -41,7 +41,10 @@ impl GrpcLockClient {
         &self,
     ) -> Result<
         rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
-            tonic::service::interceptor::InterceptedService<tonic::transport::Channel, rustfs_ecstore::rpc::TonicInterceptor>,
+            tonic::service::interceptor::InterceptedService<
+                tonic::transport::Channel,
+                crate::storage_compat::ecstore::rpc::TonicInterceptor,
+            >,
         >,
     > {
         node_service_time_out_client_no_auth(&self.addr)

crates/e2e_test/src/reliant/node_interact_test.rs

@@ -14,10 +14,10 @@
 // limitations under the License.
 
 use crate::common::workspace_root;
+use crate::storage_compat::ecstore::disk::{VolumeInfo, WalkDirOptions};
+use crate::storage_compat::ecstore::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
 use futures::future::join_all;
 use rmp_serde::{Deserializer, Serializer};
-use rustfs_ecstore::disk::{VolumeInfo, WalkDirOptions};
-use rustfs_ecstore::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
 use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
 use rustfs_protos::proto_gen::node_service::WalkDirRequest;
 use rustfs_protos::{

crates/e2e_test/src/replication_extension_test.rs

@@ -15,13 +15,13 @@
 use crate::common::{
     RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
 };
+use crate::storage_compat::ecstore::bucket::bucket_target_sys::BucketTargetSys;
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
 use aws_sdk_s3::{Client, Config};
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
-use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
 use rustfs_madmin::{
     AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
     ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,

crates/e2e_test/src/storage_compat.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) use rustfs_ecstore as ecstore;

crates/heal/tests/common/mod.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) mod storage_compat;

crates/heal/tests/common/storage_compat.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) use rustfs_ecstore as ecstore;

crates/heal/tests/endpoint_index_test.rs

@@ -14,8 +14,10 @@
 
 //! test endpoint index settings
 
-use rustfs_ecstore::disk::endpoint::Endpoint;
-use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
+mod common;
+
+use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
+use crate::common::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
 use std::net::SocketAddr;
 use tempfile::TempDir;
 use tokio_util::sync::CancellationToken;
@@ -71,10 +73,12 @@ async fn test_endpoint_index_settings() -> anyhow::Result<()> {
     }
 
     // test ECStore initialization
-    rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await?;
+    crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone()).await?;
 
     let server_addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
-    let ecstore = rustfs_ecstore::store::ECStore::new(server_addr, endpoint_pools, CancellationToken::new()).await?;
+    let ecstore =
+        crate::common::storage_compat::ecstore::store::ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
+            .await?;
 
     println!("ECStore initialized successfully with {} pools", ecstore.pools.len());
 

crates/heal/tests/heal_bug_fixes_test.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+mod common;
+
 use rustfs_heal::heal::{
     event::{HealEvent, Severity},
     task::{HealPriority, HealType},
@@ -20,7 +22,7 @@ use rustfs_heal::heal::{
 
 #[test]
 fn test_heal_event_to_heal_request_no_panic() {
-    use rustfs_ecstore::disk::endpoint::Endpoint;
+    use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
 
     // Test that invalid pool/set indices don't cause panic
     // Create endpoint using try_from or similar method
@@ -45,7 +47,7 @@ fn test_heal_event_to_heal_request_no_panic() {
 
 #[test]
 fn test_heal_event_to_heal_request_valid_indices() {
-    use rustfs_ecstore::disk::endpoint::Endpoint;
+    use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
 
     // Test that valid indices work correctly
     let endpoint_result = Endpoint::try_from("http://localhost:9000");
@@ -190,11 +192,14 @@ fn test_heal_task_status_atomic_update() {
         }
         async fn get_disk_status(
             &self,
-            _endpoint: &rustfs_ecstore::disk::endpoint::Endpoint,
+            _endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
         ) -> rustfs_heal::Result<rustfs_heal::heal::storage::DiskStatus> {
             Ok(rustfs_heal::heal::storage::DiskStatus::Ok)
         }
-        async fn format_disk(&self, _endpoint: &rustfs_ecstore::disk::endpoint::Endpoint) -> rustfs_heal::Result<()> {
+        async fn format_disk(
+            &self,
+            _endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
+        ) -> rustfs_heal::Result<()> {
             Ok(())
         }
         async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_storage_api::BucketInfo>> {
@@ -248,7 +253,10 @@ fn test_heal_task_status_atomic_update() {
         ) -> rustfs_heal::Result<(Vec<String>, Option<String>, bool)> {
             Ok((vec![], None, false))
         }
-        async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<rustfs_ecstore::disk::DiskStore> {
+        async fn get_disk_for_resume(
+            &self,
+            _set_disk_id: &str,
+        ) -> rustfs_heal::Result<crate::common::storage_compat::ecstore::disk::DiskStore> {
             Err(rustfs_heal::Error::other("Not implemented in mock"))
         }
     }
@@ -318,11 +326,17 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
             Ok(Vec::new())
         }
 
-        async fn get_disk_status(&self, _endpoint: &rustfs_ecstore::disk::endpoint::Endpoint) -> rustfs_heal::Result<DiskStatus> {
+        async fn get_disk_status(
+            &self,
+            _endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
+        ) -> rustfs_heal::Result<DiskStatus> {
             Ok(DiskStatus::Ok)
         }
 
-        async fn format_disk(&self, _endpoint: &rustfs_ecstore::disk::endpoint::Endpoint) -> rustfs_heal::Result<()> {
+        async fn format_disk(
+            &self,
+            _endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
+        ) -> rustfs_heal::Result<()> {
             Ok(())
         }
 
@@ -392,7 +406,10 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
             Ok((Vec::new(), None, false))
         }
 
-        async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<rustfs_ecstore::disk::DiskStore> {
+        async fn get_disk_for_resume(
+            &self,
+            _set_disk_id: &str,
+        ) -> rustfs_heal::Result<crate::common::storage_compat::ecstore::disk::DiskStore> {
             Err(rustfs_heal::Error::other("not implemented"))
         }
     }

crates/heal/tests/heal_integration_test.rs

@@ -12,13 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use http::HeaderMap;
-use rustfs_common::heal_channel::{HealOpts, HealScanMode};
-use rustfs_ecstore::{
+mod common;
+
+use crate::common::storage_compat::ecstore::{
     disk::endpoint::Endpoint,
     endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
     store::ECStore,
 };
+use http::HeaderMap;
+use rustfs_common::heal_channel::{HealOpts, HealScanMode};
 use rustfs_heal::heal::{
     manager::{HealConfig, HealManager},
     storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI},
@@ -122,7 +124,9 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
     let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
 
     // format disks (only first time)
-    rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
+    crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
+        .await
+        .unwrap();
 
     // create ECStore with dynamic port 0 (let OS assign) or fixed 9001 if free
     let port = 9001; // for simplicity
@@ -140,7 +144,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
         .await
         .unwrap();
     let buckets = buckets_list.into_iter().map(|v| v.name).collect();
-    rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
+    crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
 
     // Create heal storage layer
     let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));

crates/iam/src/error.rs

@@ -179,20 +179,20 @@ impl Error {
     }
 }
 
-impl From<rustfs_ecstore::error::StorageError> for Error {
-    fn from(e: rustfs_ecstore::error::StorageError) -> Self {
+impl From<crate::storage_compat::ecstore::error::StorageError> for Error {
+    fn from(e: crate::storage_compat::ecstore::error::StorageError) -> Self {
         match e {
-            rustfs_ecstore::error::StorageError::ConfigNotFound => Error::ConfigNotFound,
+            crate::storage_compat::ecstore::error::StorageError::ConfigNotFound => Error::ConfigNotFound,
             _ => Error::other(e),
         }
     }
 }
 
-impl From<Error> for rustfs_ecstore::error::StorageError {
+impl From<Error> for crate::storage_compat::ecstore::error::StorageError {
     fn from(e: Error) -> Self {
         match e {
-            Error::ConfigNotFound => rustfs_ecstore::error::StorageError::ConfigNotFound,
-            _ => rustfs_ecstore::error::StorageError::other(e),
+            Error::ConfigNotFound => crate::storage_compat::ecstore::error::StorageError::ConfigNotFound,
+            _ => crate::storage_compat::ecstore::error::StorageError::other(e),
         }
     }
 }
@@ -330,13 +330,13 @@ mod tests {
     #[test]
     fn test_iam_error_from_storage_error() {
         // Test conversion from StorageError
-        let storage_error = rustfs_ecstore::error::StorageError::ConfigNotFound;
+        let storage_error = crate::storage_compat::ecstore::error::StorageError::ConfigNotFound;
         let iam_error: Error = storage_error.into();
         assert_eq!(iam_error, Error::ConfigNotFound);
 
         // Test reverse conversion
-        let back_to_storage: rustfs_ecstore::error::StorageError = iam_error.into();
-        assert_eq!(back_to_storage, rustfs_ecstore::error::StorageError::ConfigNotFound);
+        let back_to_storage: crate::storage_compat::ecstore::error::StorageError = iam_error.into();
+        assert_eq!(back_to_storage, crate::storage_compat::ecstore::error::StorageError::ConfigNotFound);
     }
 
     #[test]

crates/iam/src/lib.rs

@@ -13,9 +13,9 @@
 // limitations under the License.
 
 use crate::error::{Error, Result};
+use crate::storage_compat::ecstore::store::ECStore;
 use manager::IamCache;
 use oidc::OidcSys;
-use rustfs_ecstore::store::ECStore;
 use std::sync::{Arc, OnceLock};
 use store::object::ObjectStore;
 use sys::IamSys;
@@ -33,6 +33,7 @@ pub mod keyring;
 pub mod manager;
 pub mod oidc;
 pub mod oidc_state;
+mod storage_compat;
 pub mod store;
 pub mod sys;
 pub mod utils;

crates/iam/src/manager.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use crate::error::{Error, Result, is_err_config_not_found};
+use crate::storage_compat::ecstore::global::is_first_cluster_node_local;
 use crate::sys::{get_claims_from_token_with_secret, get_claims_from_token_with_secret_allow_missing_exp};
 use crate::{
     cache::{Cache, CacheEntity, LockedCache},
@@ -25,7 +26,6 @@ use crate::{
 };
 use futures::future::join_all;
 use rustfs_credentials::{Credentials, EMBEDDED_POLICY_TYPE, INHERITED_POLICY_TYPE, get_global_action_cred};
-use rustfs_ecstore::global::is_first_cluster_node_local;
 use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq, GroupDesc};
 use rustfs_policy::{
     arn::ARN,

crates/iam/src/storage_compat.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) use rustfs_ecstore as ecstore;

crates/iam/src/store/object.rs

@@ -14,6 +14,14 @@
 
 use super::{GroupInfo, MappedPolicy, Store, UserType};
 use crate::error::{Error, Result, is_err_config_not_found, is_err_no_such_group};
+use crate::storage_compat::ecstore::error::{Error as EcstoreError, StorageError, classify_system_path_failure_reason};
+use crate::storage_compat::ecstore::{
+    config::{
+        RUSTFS_CONFIG_PREFIX,
+        com::{delete_config, read_config_no_lock, read_config_with_metadata, save_config, save_config_with_opts},
+    },
+    store::ECStore,
+};
 use crate::{
     cache::{Cache, CacheEntity},
     error::{is_err_no_such_policy, is_err_no_such_user},
@@ -22,14 +30,6 @@ use crate::{
 };
 use futures::future::join_all;
 use rustfs_credentials::get_global_action_cred;
-use rustfs_ecstore::error::{Error as EcstoreError, StorageError, classify_system_path_failure_reason};
-use rustfs_ecstore::{
-    config::{
-        RUSTFS_CONFIG_PREFIX,
-        com::{delete_config, read_config_no_lock, read_config_with_metadata, save_config, save_config_with_opts},
-    },
-    store::ECStore,
-};
 use rustfs_io_metrics::record_system_path_failure;
 use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
 use rustfs_storage_api::{HTTPPreconditions, ListOperations as _, ObjectInfoOrErr as StorageObjectInfoOrErr};
@@ -589,7 +589,7 @@ impl ObjectStore {
 
         match read_config_no_lock(self.object_api.clone(), &probe_path).await {
             Ok(_) => Ok(()),
-            Err(rustfs_ecstore::error::StorageError::ConfigNotFound) => Err(Error::other(format!(
+            Err(crate::storage_compat::ecstore::error::StorageError::ConfigNotFound) => Err(Error::other(format!(
                 "Storage metadata not ready: probe object '{}' not found (expected IAM config to be initialized)",
                 probe_path
             ))),

crates/iam/src/sys.rs

@@ -19,13 +19,13 @@ use crate::error::{Error, Result};
 use crate::manager::extract_jwt_claims;
 use crate::manager::get_default_policyes;
 use crate::manager::{IamCache, IamSyncMetricsSnapshot};
+use crate::storage_compat::ecstore::notification_sys::get_global_notification_sys;
 use crate::store::GroupInfo;
 use crate::store::MappedPolicy;
 use crate::store::Store;
 use crate::store::UserType;
 use crate::utils::{extract_claims, extract_claims_allow_missing_exp};
 use rustfs_credentials::{Credentials, EMBEDDED_POLICY_TYPE, INHERITED_POLICY_TYPE, get_global_action_cred};
-use rustfs_ecstore::notification_sys::get_global_notification_sys;
 use rustfs_madmin::AddOrUpdateUserReq;
 use rustfs_madmin::GroupDesc;
 use rustfs_policy::arn::ARN;

crates/notify/src/config_manager.rs

@@ -316,13 +316,13 @@ impl NotifyConfigManager {
     where
         F: FnMut(&mut Config) -> bool,
     {
-        let Some(store) = rustfs_ecstore::global::resolve_object_store_handle() else {
+        let Some(store) = crate::storage_compat::ecstore::global::resolve_object_store_handle() else {
             return Err(NotificationError::StorageNotAvailable(
                 "Failed to save target configuration: server storage not initialized".to_string(),
             ));
         };
 
-        let mut new_config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone())
+        let mut new_config = crate::storage_compat::ecstore::config::com::read_config_without_migrate(store.clone())
             .await
             .map_err(|e| NotificationError::ReadConfig(e.to_string()))?;
 
@@ -338,7 +338,7 @@ impl NotifyConfigManager {
             return Ok(());
         }
 
-        rustfs_ecstore::config::com::save_server_config(store, &new_config)
+        crate::storage_compat::ecstore::config::com::save_server_config(store, &new_config)
             .await
             .map_err(|e| NotificationError::SaveConfig(e.to_string()))?;
 

crates/notify/src/storage_compat.rs

@@ -12,4 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub type NotifyObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
+pub(crate) use rustfs_ecstore as ecstore;
+
+pub type NotifyObjectInfo = ecstore::store_api::ObjectInfo;

crates/obs/src/lib.rs

@@ -64,6 +64,7 @@ mod error;
 mod global;
 mod logging;
 pub mod metrics;
+mod storage_compat;
 mod telemetry;
 
 pub use cleaner::*;

crates/obs/src/metrics/scheduler.rs

@@ -81,8 +81,8 @@ use crate::metrics::stats_collector::{
     collect_ilm_metric_stats, collect_internode_network_stats, collect_process_metric_bundle, collect_replication_stats,
     collect_scanner_metric_stats, collect_system_cpu_and_memory_stats_with,
 };
+use crate::storage_compat::ecstore::global::get_global_bucket_monitor;
 use rustfs_audit::audit_target_metrics;
-use rustfs_ecstore::global::get_global_bucket_monitor;
 use rustfs_notify::{notification_metrics_snapshot, notification_target_metrics};
 use rustfs_utils::get_env_opt_u64;
 use serde::Serialize;

crates/obs/src/metrics/stats_collector.rs

@@ -26,16 +26,16 @@ use crate::metrics::collectors::{
     DriveDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats,
     ProcessStatusType, ReplicationStats, ResourceStats, ScannerStats,
 };
+use crate::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, GLOBAL_TransitionState};
+use crate::storage_compat::ecstore::bucket::metadata_sys::get_quota_config;
+use crate::storage_compat::ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
+use crate::storage_compat::ecstore::data_usage::load_data_usage_from_backend;
+use crate::storage_compat::ecstore::global::get_global_bucket_monitor;
+use crate::storage_compat::ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
+use crate::storage_compat::ecstore::resolve_object_store_handle;
 use chrono::Utc;
 use rustfs_common::heal_channel::HealScanMode;
 use rustfs_common::metrics::global_metrics;
-use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, GLOBAL_TransitionState};
-use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
-use rustfs_ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
-use rustfs_ecstore::data_usage::load_data_usage_from_backend;
-use rustfs_ecstore::global::get_global_bucket_monitor;
-use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
-use rustfs_ecstore::resolve_object_store_handle;
 use rustfs_iam::{get_global_iam_sys, oidc::oidc_plugin_authn_metrics_snapshot};
 use rustfs_io_metrics::internode_metrics::global_internode_metrics;
 use rustfs_io_metrics::{ProcessStatusSnapshot, snapshot_process_resource_and_system};

crates/obs/src/storage_compat.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) use rustfs_ecstore as ecstore;

crates/s3select-api/src/object_store.rs

@@ -12,6 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::storage_compat::ecstore::error::{
+    StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
+};
+use crate::storage_compat::ecstore::resolve_object_store_handle;
+use crate::storage_compat::ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
+use crate::storage_compat::ecstore::store::ECStore;
 use async_trait::async_trait;
 use bytes::Bytes;
 use chrono::Utc;
@@ -25,10 +31,6 @@ use object_store::{
 };
 use pin_project_lite::pin_project;
 use rustfs_common::DEFAULT_DELIMITER;
-use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found};
-use rustfs_ecstore::resolve_object_store_handle;
-use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
-use rustfs_ecstore::store::ECStore;
 use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
 use s3s::S3Result;
 use s3s::dto::SelectObjectContentInput;

crates/s3select-api/src/storage_compat.rs

@@ -12,7 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use rustfs_ecstore::store_api::{
+pub(crate) use rustfs_ecstore as ecstore;
+
+use ecstore::store_api::{
     GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions,
 };
 

crates/scanner/tests/common/mod.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) mod storage_compat;

crates/scanner/tests/common/storage_compat.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) use rustfs_ecstore as ecstore;

crates/scanner/tests/lifecycle_integration_test.rs

@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use futures::FutureExt;
-use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
-use rustfs_ecstore::{
+mod common;
+
+use crate::common::storage_compat::ecstore::{
     bucket::lifecycle::lifecycle::TransitionOptions,
     bucket::metadata::BUCKET_LIFECYCLE_CONFIG,
     bucket::{
@@ -33,6 +33,8 @@ use rustfs_ecstore::{
         warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
     },
 };
+use futures::FutureExt;
+use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
 use rustfs_filemeta::FileMeta;
 use rustfs_scanner::scanner_folder::ScannerItem;
 use rustfs_scanner::scanner_io::ScannerIODisk;
@@ -123,7 +125,9 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
     let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
 
     // format disks (only first time)
-    rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
+    crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
+        .await
+        .unwrap();
 
     // create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
     let port = 9002; // for simplicity
@@ -141,10 +145,11 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
         .await
         .unwrap();
     let buckets = buckets_list.into_iter().map(|v| v.name).collect();
-    rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
+    crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
 
     // Initialize background expiry workers
-    rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
+    crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
+        .await;
 
     // Store in global once lock
     let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
@@ -192,7 +197,9 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
     };
 
     let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
-    rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
+    crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
+        .await
+        .unwrap();
 
     let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
     let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
@@ -207,10 +214,11 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
         .await
         .unwrap();
     let buckets = buckets_list.into_iter().map(|v| v.name).collect();
-    rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
+    crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
 
     if init_expiry {
-        rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
+        crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
+            .await;
     }
 
     (disk_paths, ecstore)
@@ -870,7 +878,7 @@ mod serial_tests {
         println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
 
         // Verify lifecycle configuration was set
-        match rustfs_ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
+        match crate::common::storage_compat::ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
             Ok(bucket_meta) => {
                 assert!(bucket_meta.lifecycle_config.is_some());
                 println!("✅ Bucket metadata retrieved successfully");
@@ -1276,7 +1284,8 @@ mod serial_tests {
             "stale transitioned remote object should still exist before scanner fallback runs"
         );
 
-        rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
+        crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
+            .await;
         scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
 
         assert!(
@@ -1337,7 +1346,8 @@ mod serial_tests {
             "stale transitioned remote object should still exist before scanner cleanup runs"
         );
 
-        rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
+        crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
+            .await;
         scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
 
         assert!(
@@ -1704,7 +1714,8 @@ mod serial_tests {
 
         assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
 
-        rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
+        crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
+            .await;
         scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
 
         assert!(
@@ -1808,7 +1819,8 @@ mod serial_tests {
             .await
             .expect("Failed to set noncurrent lifecycle configuration");
 
-        rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
+        crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
+            .await;
 
         scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
 

docs/architecture/migration-progress.md

@@ -5,20 +5,21 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
 ## Current Context
 
 - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
-- Branch: `overtrue/arch-app-storage-admin-runtime-boundaries`
-- Baseline: rebased onto `origin/main` at
-  `bdb2461b5594f736cbbcd3788f12d39966c2ec78` after
-  `rustfs/rustfs#3571` merged.
+- Branch: `overtrue/arch-test-harness-fuzz-storage-boundaries`
+- Baseline: `origin/main` after `rustfs/rustfs#3572` merged as
+  `8313767f75f694cb8cf1fb47e5c0f44dfc5bf2b8`.
 - PR type for this branch: `consumer-migration`
 - Runtime behavior changes: no migration behavior change expected; CI follow-up
   preserves empty-object erasure recovery by avoiding zero-byte SIMD decode.
-- Rust code changes: add app-, storage-, and admin-local ECStore compatibility
-  boundaries and route existing direct runtime imports through those deliberate
-  boundary modules. ECStore-owned definitions and runtime behavior stay in
-  ECStore.
+- Rust code changes: add RustFS root, observability, IAM, test harness, and
+  fuzz ECStore compatibility boundaries; extend notify and S3 Select
+  boundaries; route runtime, server, startup, capacity, table catalog, obs,
+  notify, S3 Select, IAM, scanner/heal integration tests, e2e helpers, and
+  fuzz direct ECStore imports through local boundaries. ECStore-owned
+  definitions and runtime behavior stay in ECStore.
 - CI/script changes: none.
-- Docs changes: record the larger DTO consumer-boundary cleanup slice and the
-  empty-object erasure recovery CI follow-up.
+- Docs changes: record the larger runtime/observability/select/IAM plus test
+  harness/fuzz boundary cleanup slice.
 
 ## Phase 0 Tasks
 
@@ -882,6 +883,47 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
     check, migration/layer guards, formatting, diff hygiene, Rust risk scan,
     full pre-commit, and required three-expert review passed.
 
+- [x] `API-031` Clean runtime, observability, S3 Select, notify, and IAM
+  ECStore runtime boundaries.
+  - Current branch: `overtrue/arch-runtime-observability-select-boundaries`.
+  - Completed slice: add RustFS root, obs, and IAM compatibility boundary
+    modules; extend notify and S3 Select compatibility modules; migrate direct
+    `rustfs_ecstore` imports in RustFS startup/server/runtime/table-catalog
+    code plus obs, notify, S3 Select, and IAM through those local boundaries.
+  - Acceptance: direct `rustfs_ecstore` references in those source areas are
+    limited to local compatibility boundary modules; runtime and crate business
+    modules consume local compatibility names.
+  - Must preserve: startup ordering, readiness/RPC behavior, capacity metrics,
+    table catalog object I/O behavior, notification config persistence, S3
+    Select object-store reads, IAM storage/error mapping, and observability
+    metrics collection behavior.
+  - Risk defense: this slice changes import ownership only; it does not move
+    ECStore definitions, alter runtime control flow, adjust readiness checks,
+    mutate table catalog metadata, change IAM policy behavior, or alter notify,
+    S3 Select, or obs runtime semantics.
+  - Verification: focused compile, direct import scan, migration/layer guards,
+    formatting, diff hygiene, Rust risk scan, full pre-commit, and required
+    three-expert review passed.
+
+- [x] `API-032` Clean test harness and fuzz ECStore compatibility boundaries.
+  - Current branch: `overtrue/arch-test-harness-fuzz-storage-boundaries`.
+  - Completed slice: add scanner/heal integration test, e2e test, and fuzz
+    target compatibility boundary modules; migrate direct `rustfs_ecstore`
+    imports in those test/fuzz harnesses through local boundaries.
+  - Acceptance: direct `rustfs_ecstore` references in scanner/heal integration
+    tests, e2e test helpers, and fuzz targets are limited to local
+    compatibility boundary modules; test and fuzz modules consume local
+    compatibility names.
+  - Must preserve: scanner lifecycle integration behavior, heal integration and
+    bug-fix test behavior, e2e node/grpc/replication helpers, fuzz target input
+    shape, and existing ECStore-owned concrete type ownership.
+  - Risk defense: this slice changes import ownership only; it does not move
+    ECStore definitions, alter test setup semantics, change fuzz inputs, adjust
+    runtime control flow, or mutate metadata formats.
+  - Verification: focused scanner/heal/e2e compile, fuzz target compile,
+    migration/layer guards, formatting check, diff hygiene, direct import scan,
+    risk scan, full pre-commit, and required three-expert review passed.
+
 ## Phase 8 Background Controller Tasks
 
 - [x] `BGC-001` Inventory background services.
@@ -1152,44 +1194,52 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
 
 1. `pure-move`/`consumer-migration`: continue larger cleanup slices with the
    loss-prevention guards active for remaining ECStore compatibility contracts
-   outside the app/storage/admin scanner/heal/Swift boundaries already cleaned.
+   outside the app/storage/admin scanner/heal/Swift/runtime/obs/notify/S3
+   Select/IAM boundaries already cleaned.
 
 ## Pre-Push Review Log
 
 | Expert | Status | Notes |
 |---|---|---|
-| Quality/architecture | passed | App, storage, and admin source modules now use local compatibility boundaries for ECStore runtime imports. |
-| Migration preservation | passed | ECStore remains the owner of backing types and behavior; app usecases, storage ECFS/SSE/RPC, and admin handlers/services keep existing semantics. |
-| Testing/verification | passed | Direct app/storage/admin import scan, RustFS test compile check, migration/layer guards, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` passed. |
+| Quality/architecture | passed | Runtime, server, startup, table catalog, obs, notify, S3 Select, IAM, scanner/heal integration tests, e2e helpers, and fuzz targets now use local compatibility boundaries for ECStore runtime imports. |
+| Migration preservation | passed | ECStore remains the owner of backing types and behavior; startup/readiness, table catalog, notify, S3 Select, IAM, obs, test harness, and fuzz semantics are unchanged. |
+| Testing/verification | passed | Focused scanner/heal/e2e and fuzz compile checks, guards, formatting check, diff hygiene, risk scan, and full `make pre-commit` passed. |
 
 ## Verification Notes
 
 Passed before push:
 
-- `cargo check --tests -p rustfs`: passed.
-- `rg -n 'rustfs_ecstore' rustfs/src/app rustfs/src/storage rustfs/src/admin --glob '*.rs'`:
-  remaining matches are deliberate app/storage/admin compatibility boundary
-  definitions.
+- `cargo check --tests -p rustfs -p rustfs-obs -p rustfs-notify -p rustfs-s3select-api -p rustfs-iam`:
+  passed.
+- `cargo check --tests -p rustfs-scanner -p rustfs-heal -p e2e_test`:
+  passed.
+- `cargo check --manifest-path fuzz/Cargo.toml --bins`: passed.
+- `rg -n 'rustfs_ecstore' rustfs/src crates/obs/src crates/notify/src crates/s3select-api/src crates/iam/src --glob '*.rs'`:
+  remaining matches are deliberate compatibility boundary definitions.
+- Direct import scan for target scanner/heal/e2e/fuzz paths: passed; remaining
+  matches are deliberate compatibility boundary definitions.
 - `./scripts/check_architecture_migration_rules.sh`: passed.
 - `./scripts/check_layer_dependencies.sh`: passed.
 - `cargo fmt --all --check`: passed.
 - `git diff --check`: passed.
-- Rust risk scan: reviewed mechanical path-only hits on existing
-  `unwrap`/`expect` and numeric-cast lines; no new risky logic was introduced.
+- Rust risk scan: reviewed added `.unwrap()` matches as preserved test setup
+  unwraps caused by path rewrite formatting; no new risky behavior added.
 - `make pre-commit`: passed.
 
 Notes:
 
-- This slice was prepared on the API-029 boundary and rebased onto main after
-  `rustfs/rustfs#3571` merged.
-- Direct app/storage/admin ECStore imports now remain only in their
-  compatibility boundary modules.
-- The slice does not alter app usecase behavior, storage behavior, admin
-  behavior, object I/O behavior, metadata serialization, or ECStore definitions.
+- This larger slice is based on `origin/main` after `rustfs/rustfs#3572`
+  merged.
+- Direct ECStore imports in the target runtime/obs/notify/S3 Select/IAM and
+  scanner/heal/e2e/fuzz areas now remain only in local compatibility boundary
+  modules.
+- The slice does not alter startup behavior, readiness behavior, table catalog
+  object I/O, notification persistence, S3 Select reads, IAM error mapping,
+  observability metrics, test/fuzz semantics, or ECStore definitions.
 
 ## Handoff Notes
 
 - Continue with larger consumer-migration batches outside the cleaned
-  app/storage/admin/scanner/heal/Swift runtime boundaries; keep ECStore-owned
-  behavior in ECStore until concrete behavior is isolated enough for a
-  pure-move slice.
+  app/storage/admin/scanner/heal/Swift/runtime/obs/notify/S3 Select/IAM/test
+  and fuzz boundaries; keep ECStore-owned behavior in ECStore until concrete
+  behavior is isolated enough for a pure-move slice.

fuzz/Cargo.lock

@@ -8760,6 +8760,7 @@ dependencies = [
  "async-trait",
  "serde",
  "time",
+ "uuid",
 ]
 
 [[package]]
@@ -8789,6 +8790,7 @@ dependencies = [
  "rustfs-kafka-async",
  "rustfs-s3-types",
  "rustfs-tls-runtime",
+ "rustfs-utils",
  "rustls",
  "rustls-native-certs",
  "s3s",

fuzz/fuzz_targets/bucket_validation.rs

@@ -1,7 +1,9 @@
 #![no_main]
 
+mod storage_compat;
+
 use libfuzzer_sys::fuzz_target;
-use rustfs_ecstore::bucket::utils::{check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict};
+use crate::storage_compat::ecstore::bucket::utils::{check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict};
 
 fn parse_case(data: &[u8]) -> (String, String) {
     let text = String::from_utf8_lossy(data);

fuzz/fuzz_targets/path_containment.rs

@@ -1,7 +1,9 @@
 #![no_main]
 
+mod storage_compat;
+
 use libfuzzer_sys::fuzz_target;
-use rustfs_ecstore::bucket::utils::{check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix};
+use crate::storage_compat::ecstore::bucket::utils::{check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix};
 use rustfs_utils::path::{clean, path_join};
 use std::path::{Path, PathBuf};
 

fuzz/fuzz_targets/storage_compat.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) use rustfs_ecstore as ecstore;

rustfs/src/capacity/service.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use rustfs_ecstore::disk::DiskAPI;
+use crate::storage_compat::ecstore::disk::DiskAPI;
 use rustfs_io_metrics::capacity_metrics::{
     record_capacity_cache_hit, record_capacity_cache_miss, record_capacity_cache_served, record_capacity_refresh_request,
     record_capacity_scan_mode,
@@ -263,7 +263,7 @@ pub async fn init_capacity_management_for_local_disks() {
         "Capacity manager state changed"
     );
 
-    let disks = rustfs_ecstore::store::all_local_disk().await;
+    let disks = crate::storage_compat::ecstore::store::all_local_disk().await;
     if disks.is_empty() {
         warn!(
             component = LOG_COMPONENT_CAPACITY,

rustfs/src/config/config_test.rs

@@ -16,9 +16,9 @@
 #[allow(unsafe_op_in_unsafe_fn)]
 mod tests {
     use crate::config::{CommandResult, Config, Opt, TlsCommands};
+    use crate::storage_compat::ecstore::disks_layout::DisksLayout;
     use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_OBS_ENDPOINT, RUSTFS_REGION};
     use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY};
-    use rustfs_ecstore::disks_layout::DisksLayout;
     use serial_test::serial;
     use std::env;
 
@@ -262,7 +262,7 @@ mod tests {
     #[test]
     #[serial]
     fn test_volumes_and_disk_layout_parsing() {
-        use rustfs_ecstore::disks_layout::DisksLayout;
+        use crate::storage_compat::ecstore::disks_layout::DisksLayout;
 
         // Test case 1: Single volume path
         let args = vec!["rustfs", "/data/vol1"];

rustfs/src/embedded.rs

@@ -51,10 +51,8 @@ use crate::init::{add_bucket_notification_configuration, init_buffer_profile_sys
 use crate::server::{ShutdownHandle, shutdown_event_notifier, start_http_server, stop_audit_system};
 use crate::startup_fs_guard::enforce_unsupported_fs_policy;
 use crate::startup_iam::{bootstrap_or_defer_iam_init, publish_ready_for_iam_bootstrap};
-use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
-use rustfs_credentials::init_global_action_credentials;
-use rustfs_ecstore::store::init_lock_clients;
-use rustfs_ecstore::{
+use crate::storage_compat::ecstore::store::init_lock_clients;
+use crate::storage_compat::ecstore::{
     bucket::replication::init_background_replication,
     bucket::{
         metadata_sys::init_bucket_metadata_sys,
@@ -68,6 +66,8 @@ use rustfs_ecstore::{
     store::init_local_disks,
     update_erasure_type,
 };
+use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
+use rustfs_credentials::init_global_action_credentials;
 use rustfs_obs::{init_obs, set_global_guard};
 use rustfs_storage_api::{BucketOperations, BucketOptions};
 use rustfs_utils::net::parse_and_resolve_address;
@@ -333,7 +333,7 @@ impl RustFSServerBuilder {
             let region = region_str
                 .parse()
                 .map_err(|e| ServerError::Init(format!("invalid region '{region_str}': {e}")))?;
-            rustfs_ecstore::global::set_global_region(region);
+            crate::storage_compat::ecstore::global::set_global_region(region);
         }
 
         let server_port = server_addr.port();

rustfs/src/error.rs

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use rustfs_ecstore::bucket::quota::QuotaError;
-use rustfs_ecstore::error::StorageError;
+use crate::storage_compat::ecstore::bucket::quota::QuotaError;
+use crate::storage_compat::ecstore::error::StorageError;
 use rustfs_storage_api::HTTPRangeError;
 use s3s::{S3Error, S3ErrorCode};
 

rustfs/src/init.rs

@@ -14,12 +14,12 @@
 
 use crate::server::ShutdownHandle;
 use crate::storage::{process_lambda_configurations, process_queue_configurations, process_topic_configurations};
+use crate::storage_compat::ecstore::bucket::metadata_sys;
 use crate::{admin, config, version};
 use rustfs_config::{
     DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK,
     ENV_RUSTFS_BUFFER_DEFAULT_SIZE, ENV_RUSTFS_BUFFER_MAX_SIZE, ENV_RUSTFS_BUFFER_MIN_SIZE, ENV_UPDATE_CHECK, RUSTFS_REGION,
 };
-use rustfs_ecstore::bucket::metadata_sys;
 use rustfs_notify::notifier_global;
 use rustfs_targets::arn::{ARN, TargetIDError};
 use rustfs_utils::get_env_usize;
@@ -157,7 +157,7 @@ fn arn_to_target_id(arn_str: &str) -> Result<rustfs_targets::arn::TargetID, Targ
 /// * `buckets` - A vector of bucket names to process
 #[instrument(skip_all)]
 pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
-    let global_region = rustfs_ecstore::global::get_global_region();
+    let global_region = crate::storage_compat::ecstore::global::get_global_region();
     let region = global_region
         .as_ref()
         .filter(|r| !r.as_str().is_empty())

rustfs/src/lib.rs

@@ -77,6 +77,7 @@ pub mod startup_server;
 pub mod startup_services;
 pub mod startup_storage;
 pub mod storage;
+pub(crate) mod storage_compat;
 pub(crate) mod table_catalog;
 pub mod tls;
 pub mod update;

rustfs/src/server/event.rs

@@ -14,7 +14,7 @@
 
 use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store};
 use crate::app::context::resolve_server_config;
-use rustfs_ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook};
+use crate::storage_compat::ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook};
 use rustfs_notify::EventArgs as NotifyEventArgs;
 use rustfs_s3_types::EventName;
 use std::net::SocketAddr;

rustfs/src/server/http.rs

@@ -33,6 +33,7 @@ use crate::server::{
 use crate::storage;
 use crate::storage::rpc::InternodeRpcService;
 use crate::storage::tonic_service::make_server;
+use crate::storage_compat::ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
 use bytes::Bytes;
 use http::{HeaderMap, Method, Request as HttpRequest, Response};
 use hyper_util::{
@@ -45,7 +46,6 @@ use metrics::{counter, gauge, histogram};
 use opentelemetry::global;
 use opentelemetry::trace::TraceContextExt;
 use rustfs_common::GlobalReadiness;
-use rustfs_ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
 use rustfs_keystone::KeystoneAuthLayer;
 #[cfg(feature = "swift")]
 use rustfs_protocols::SwiftService;

rustfs/src/server/module_switch.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use rustfs_ecstore::{
+use crate::storage_compat::ecstore::{
     config::com::{read_config, save_config},
     error::Error as StorageError,
     resolve_object_store_handle,

rustfs/src/server/readiness.rs

@@ -14,6 +14,9 @@
 
 use crate::server::{ServiceState, ServiceStateManager};
 use crate::server::{has_path_prefix, is_table_catalog_path};
+use crate::storage_compat::ecstore::global::is_dist_erasure;
+use crate::storage_compat::ecstore::global::{get_global_endpoints_opt, get_global_lock_clients};
+use crate::storage_compat::ecstore::resolve_object_store_handle;
 use bytes::Bytes;
 use http::{Request as HttpRequest, Response, StatusCode};
 use http_body::Body;
@@ -21,9 +24,6 @@ use http_body_util::{BodyExt, Full};
 use hyper::body::Incoming;
 use metrics::{counter, gauge};
 use rustfs_common::GlobalReadiness;
-use rustfs_ecstore::global::is_dist_erasure;
-use rustfs_ecstore::global::{get_global_endpoints_opt, get_global_lock_clients};
-use rustfs_ecstore::resolve_object_store_handle;
 use rustfs_iam::get_global_iam_sys;
 use rustfs_madmin::{Disk, StorageInfo};
 use rustfs_storage_api::StorageAdminApi;
@@ -498,11 +498,11 @@ async fn collect_storage_readiness_uncached() -> bool {
 
 fn set_lock_quorum_status(
     online_hosts: &HashSet<String>,
-    set_endpoints: &[rustfs_ecstore::disk::endpoint::Endpoint],
+    set_endpoints: &[crate::storage_compat::ecstore::disk::endpoint::Endpoint],
 ) -> LockQuorumStatus {
     let total_clients = set_endpoints
         .iter()
-        .map(rustfs_ecstore::disk::endpoint::Endpoint::host_port)
+        .map(crate::storage_compat::ecstore::disk::endpoint::Endpoint::host_port)
         .filter(|host| !host.is_empty())
         .collect::<HashSet<_>>();
     let total_clients_len = total_clients.len();
@@ -526,7 +526,7 @@ fn set_lock_quorum_status(
 }
 
 fn aggregate_lock_quorum_status(
-    pool_endpoints: &rustfs_ecstore::endpoints::EndpointServerPools,
+    pool_endpoints: &crate::storage_compat::ecstore::endpoints::EndpointServerPools,
     online_hosts: &HashSet<String>,
 ) -> LockQuorumStatus {
     let mut connected_clients = 0usize;
@@ -842,8 +842,8 @@ mod tests {
 
     #[test]
     fn aggregate_lock_quorum_status_requires_each_set_to_meet_quorum() {
-        use rustfs_ecstore::disk::endpoint::Endpoint;
-        use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
+        use crate::storage_compat::ecstore::disk::endpoint::Endpoint;
+        use crate::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
 
         let endpoints = vec![
             Endpoint {
@@ -900,8 +900,8 @@ mod tests {
 
     #[test]
     fn aggregate_lock_quorum_status_fails_when_any_set_loses_quorum() {
-        use rustfs_ecstore::disk::endpoint::Endpoint;
-        use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
+        use crate::storage_compat::ecstore::disk::endpoint::Endpoint;
+        use crate::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
 
         let endpoints = vec![
             Endpoint {

rustfs/src/startup_fs_guard.rs

@@ -12,11 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::storage_compat::ecstore::endpoints::EndpointServerPools;
 use rustfs_config::{
     DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, ENV_RUSTFS_UNSUPPORTED_FS_POLICY, RUSTFS_UNSUPPORTED_FS_POLICY_FAIL,
     RUSTFS_UNSUPPORTED_FS_POLICY_WARN,
 };
-use rustfs_ecstore::endpoints::EndpointServerPools;
 use std::collections::BTreeSet;
 use std::io::{Error, Result};
 use tracing::warn;

rustfs/src/startup_iam.rs

@@ -14,8 +14,8 @@
 
 use crate::app::context::{AppContext, get_global_app_context, init_global_app_context};
 use crate::server::{ServiceStateManager, publish_ready_when_runtime_ready};
+use crate::storage_compat::ecstore::store::ECStore;
 use rustfs_common::{GlobalReadiness, SystemStage};
-use rustfs_ecstore::store::ECStore;
 use rustfs_iam::init_iam_sys;
 use rustfs_kms::KmsServiceManager;
 use std::future::Future;

rustfs/src/startup_server.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::storage_compat::ecstore::global::set_global_rustfs_port;
 use crate::{
     capacity::capacity_integration::init_capacity_management,
     config::Config,
@@ -19,7 +20,6 @@ use crate::{
 };
 use rustfs_common::{GlobalReadiness, set_global_addr};
 use rustfs_credentials::init_global_action_credentials;
-use rustfs_ecstore::global::set_global_rustfs_port;
 use rustfs_utils::net::parse_and_resolve_address;
 use std::{
     io::{Error, Result},
@@ -57,7 +57,7 @@ pub async fn init_startup_listen_context(config: &Config) -> Result<StartupListe
     if let Some(region_str) = &config.region {
         region_str
             .parse::<s3s::region::Region>()
-            .map(rustfs_ecstore::global::set_global_region)
+            .map(crate::storage_compat::ecstore::global::set_global_region)
             .map_err(|err| Error::other(format!("invalid region '{}': {}", region_str, err)))?;
     }
 

rustfs/src/startup_services.rs

@@ -12,6 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::storage_compat::ecstore::{
+    bucket::{
+        metadata_sys::init_bucket_metadata_sys,
+        migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
+        replication::get_global_replication_pool,
+    },
+    endpoints::EndpointServerPools,
+    global::shutdown_background_services,
+    notification_sys::new_global_notification_sys,
+    store::ECStore,
+};
 use crate::{
     config::Config,
     init::{
@@ -28,17 +39,6 @@ use crate::{
 use futures_util::future::join_all;
 use rustfs_audit::AuditResult;
 use rustfs_common::GlobalReadiness;
-use rustfs_ecstore::{
-    bucket::{
-        metadata_sys::init_bucket_metadata_sys,
-        migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
-        replication::get_global_replication_pool,
-    },
-    endpoints::EndpointServerPools,
-    global::shutdown_background_services,
-    notification_sys::new_global_notification_sys,
-    store::ECStore,
-};
 use rustfs_heal::{
     create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager, shutdown_ahm_services,
 };
@@ -597,14 +597,16 @@ where
     start_audit().await
 }
 
-pub async fn init_notification_system(endpoint_pools: EndpointServerPools) -> rustfs_ecstore::error::Result<()> {
+pub async fn init_notification_system(endpoint_pools: EndpointServerPools) -> crate::storage_compat::ecstore::error::Result<()> {
     init_notification_system_with(|| new_global_notification_sys(endpoint_pools)).await
 }
 
-async fn init_notification_system_with<InitFn, InitFuture>(init_notification: InitFn) -> rustfs_ecstore::error::Result<()>
+async fn init_notification_system_with<InitFn, InitFuture>(
+    init_notification: InitFn,
+) -> crate::storage_compat::ecstore::error::Result<()>
 where
     InitFn: FnOnce() -> InitFuture,
-    InitFuture: Future<Output = rustfs_ecstore::error::Result<()>>,
+    InitFuture: Future<Output = crate::storage_compat::ecstore::error::Result<()>>,
 {
     init_notification().await
 }
@@ -662,7 +664,8 @@ mod tests {
 
     #[tokio::test]
     async fn notification_system_returns_source_error() {
-        let result = init_notification_system_with(|| async { Err(rustfs_ecstore::error::Error::FaultyDisk) }).await;
+        let result =
+            init_notification_system_with(|| async { Err(crate::storage_compat::ecstore::error::Error::FaultyDisk) }).await;
 
         assert!(result.is_err());
     }

rustfs/src/startup_storage.rs

@@ -13,8 +13,7 @@
 // limitations under the License.
 
 use crate::startup_fs_guard::enforce_unsupported_fs_policy;
-use rustfs_common::{GlobalReadiness, SystemStage};
-use rustfs_ecstore::{
+use crate::storage_compat::ecstore::{
     bucket::replication::init_background_replication,
     config as ecconfig,
     endpoints::EndpointServerPools,
@@ -22,6 +21,7 @@ use rustfs_ecstore::{
     store::{ECStore, init_local_disks, init_lock_clients, prewarm_local_disk_id_map},
     update_erasure_type,
 };
+use rustfs_common::{GlobalReadiness, SystemStage};
 use std::{
     io::{Error, Result},
     net::SocketAddr,

rustfs/src/storage_compat.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) use rustfs_ecstore as ecstore;

rustfs/src/table_catalog.rs

@@ -27,6 +27,16 @@ use std::{
     time::{Duration as StdDuration, Instant},
 };
 
+use crate::storage_compat::ecstore::bucket::{
+    metadata::{
+        BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG,
+        BUCKET_TABLE_RESERVED_PREFIX, table_catalog_path_hash,
+    },
+    metadata_sys,
+};
+use crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET;
+use crate::storage_compat::ecstore::error::{Error as EcstoreError, StorageError};
+use crate::storage_compat::ecstore::set_disk::get_lock_acquire_timeout;
 use bytes::Bytes;
 use datafusion::{
     arrow::datatypes::SchemaRef,
@@ -34,16 +44,6 @@ use datafusion::{
 };
 use http::HeaderMap;
 use metrics::{counter, histogram};
-use rustfs_ecstore::bucket::{
-    metadata::{
-        BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG,
-        BUCKET_TABLE_RESERVED_PREFIX, table_catalog_path_hash,
-    },
-    metadata_sys,
-};
-use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
-use rustfs_ecstore::error::{Error as EcstoreError, StorageError};
-use rustfs_ecstore::set_disk::get_lock_acquire_timeout;
 use rustfs_filemeta::FileInfo;
 use rustfs_storage_api::{
     HTTPPreconditions, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
@@ -7541,7 +7541,7 @@ mod tests {
             .map(|(bucket, _)| bucket.as_str())
             .collect::<BTreeSet<_>>();
 
-        assert_eq!(object_buckets, BTreeSet::from([rustfs_ecstore::disk::RUSTFS_META_BUCKET]));
+        assert_eq!(object_buckets, BTreeSet::from([crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET]));
     }
 
     #[tokio::test]
@@ -10409,7 +10409,7 @@ mod tests {
             .unwrap();
         backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
         backend
-            .fail_put_attempt(rustfs_ecstore::disk::RUSTFS_META_BUCKET, &idempotency_path, 1)
+            .fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &idempotency_path, 1)
             .await;
 
         let err = store
@@ -10459,7 +10459,7 @@ mod tests {
             .unwrap();
         backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
         backend
-            .fail_put_attempt(rustfs_ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
+            .fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
             .await;
 
         let request = TableCommitRequest {
@@ -10514,7 +10514,7 @@ mod tests {
             .unwrap();
         backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
         backend
-            .fail_put_attempt(rustfs_ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
+            .fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
             .await;
 
         let result = store
@@ -10568,7 +10568,7 @@ mod tests {
             .unwrap();
         backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
         backend
-            .fail_put_attempt(rustfs_ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
+            .fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
             .await;
 
         store
@@ -10622,7 +10622,7 @@ mod tests {
             .unwrap();
         backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
         backend
-            .fail_put_attempt(rustfs_ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
+            .fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
             .await;
 
         store
@@ -10673,7 +10673,7 @@ mod tests {
         backend.seed_object(bucket, &current_metadata, b"{}".to_vec()).await;
         backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
         backend
-            .fail_put_attempt(rustfs_ecstore::disk::RUSTFS_META_BUCKET, &table_path, 2)
+            .fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &table_path, 2)
             .await;
 
         let err = store
@@ -10730,7 +10730,7 @@ mod tests {
             .unwrap();
         backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
         backend
-            .fail_put_attempt(rustfs_ecstore::disk::RUSTFS_META_BUCKET, &idempotency_path, 2)
+            .fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &idempotency_path, 2)
             .await;
 
         let result = store