diff --git a/crates/e2e_test/src/reliant/grpc_lock_client.rs b/crates/e2e_test/src/reliant/grpc_lock_client.rs index 5c8e5c37e..400829ff3 100644 --- a/crates/e2e_test/src/reliant/grpc_lock_client.rs +++ b/crates/e2e_test/src/reliant/grpc_lock_client.rs @@ -16,7 +16,7 @@ #![allow(dead_code)] use async_trait::async_trait; -use rustfs_ecstore::api::rpc as ecstore_rpc; +use rustfs_ecstore::api::rpc::{TonicInterceptor, node_service_time_out_client_no_auth}; use rustfs_lock::{ LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result, types::{LockMetadata, LockPriority}, @@ -25,8 +25,6 @@ use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, Generall use tonic::Request; use tracing::{info, warn}; -type TonicInterceptor = ecstore_rpc::TonicInterceptor; - /// gRPC lock client without authentication for testing /// Similar to RemoteClient but uses no_auth client #[derive(Debug, Clone)] @@ -46,7 +44,7 @@ impl GrpcLockClient { tonic::service::interceptor::InterceptedService, >, > { - ecstore_rpc::node_service_time_out_client_no_auth(&self.addr) + node_service_time_out_client_no_auth(&self.addr) .await .map_err(|err| LockError::internal(format!("can not get client, err: {err}"))) } diff --git a/crates/e2e_test/src/reliant/node_interact_test.rs b/crates/e2e_test/src/reliant/node_interact_test.rs index 2e77e9df7..04256692b 100644 --- a/crates/e2e_test/src/reliant/node_interact_test.rs +++ b/crates/e2e_test/src/reliant/node_interact_test.rs @@ -16,8 +16,8 @@ use crate::common::workspace_root; use futures::future::join_all; use rmp_serde::{Deserializer, Serializer}; -use rustfs_ecstore::api::disk as ecstore_disk; -use rustfs_ecstore::api::rpc as ecstore_rpc; +use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions}; +use rustfs_ecstore::api::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter}; use rustfs_protos::proto_gen::node_service::WalkDirRequest; use rustfs_protos::{ @@ -36,12 +36,8 @@ use tonic::codegen::tokio_stream::StreamExt; const CLUSTER_ADDR: &str = "http://localhost:9000"; -type TonicInterceptor = ecstore_rpc::TonicInterceptor; -type VolumeInfo = ecstore_disk::VolumeInfo; -type WalkDirOptions = ecstore_disk::WalkDirOptions; - fn signature_interceptor() -> TonicInterceptor { - TonicInterceptor::Signature(ecstore_rpc::gen_tonic_signature_interceptor()) + TonicInterceptor::Signature(gen_tonic_signature_interceptor()) } #[tokio::test] @@ -61,7 +57,7 @@ async fn ping() -> Result<(), Box> { assert!(decoded_payload.is_ok()); // Create client - let mut client = ecstore_rpc::node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?; + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?; // Construct PingRequest let request = Request::new(PingRequest { @@ -86,7 +82,7 @@ async fn ping() -> Result<(), Box> { #[tokio::test] #[ignore = "requires running RustFS server at localhost:9000"] async fn make_volume() -> Result<(), Box> { - let mut client = ecstore_rpc::node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?; + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?; let request = Request::new(MakeVolumeRequest { disk: "data".to_string(), volume: "dandan".to_string(), @@ -104,7 +100,7 @@ async fn make_volume() -> Result<(), Box> { #[tokio::test] #[ignore = "requires running RustFS server at localhost:9000"] async fn list_volumes() -> Result<(), Box> { - let mut client = ecstore_rpc::node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?; + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?; let request = Request::new(ListVolumesRequest { disk: "data".to_string(), }); @@ -134,7 +130,7 @@ async fn walk_dir() -> Result<(), Box> { let (rd, mut wr) = tokio::io::duplex(1024); let mut buf = Vec::new(); opts.serialize(&mut Serializer::new(&mut buf))?; - let mut client = ecstore_rpc::node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?; + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?; let disk_path = std::env::var_os("RUSTFS_DISK_PATH").map(PathBuf::from).unwrap_or_else(|| { let mut path = workspace_root(); path.push("target"); @@ -187,7 +183,7 @@ async fn walk_dir() -> Result<(), Box> { #[tokio::test] #[ignore = "requires running RustFS server at localhost:9000"] async fn read_all() -> Result<(), Box> { - let mut client = ecstore_rpc::node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?; + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?; let request = Request::new(ReadAllRequest { disk: "data".to_string(), volume: "ff".to_string(), @@ -205,7 +201,7 @@ async fn read_all() -> Result<(), Box> { #[tokio::test] #[ignore = "requires running RustFS server at localhost:9000"] async fn storage_info() -> Result<(), Box> { - let mut client = ecstore_rpc::node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?; + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?; let request = Request::new(LocalStorageInfoRequest { metrics: true }); let response = client.local_storage_info(request).await?.into_inner(); diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index ca471085a..5a4d2d220 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -22,7 +22,7 @@ use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; use aws_sdk_s3::{Client, Config}; use http::header::{CONTENT_TYPE, HOST}; use reqwest::StatusCode; -use rustfs_ecstore::api::bucket as ecstore_bucket; +use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys; use rustfs_madmin::{ AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus, ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus, @@ -37,7 +37,6 @@ use time::Duration as TimeDuration; use tokio::time::{Duration, sleep}; type TestResult = Result<(), Box>; -type BucketTargetSys = ecstore_bucket::bucket_target_sys::BucketTargetSys; #[derive(Debug, Clone, serde::Deserialize)] struct ReplicationResetStatusResponse { diff --git a/crates/heal/tests/endpoint_index_test.rs b/crates/heal/tests/endpoint_index_test.rs index 687a610ec..ecec7fc11 100644 --- a/crates/heal/tests/endpoint_index_test.rs +++ b/crates/heal/tests/endpoint_index_test.rs @@ -14,19 +14,13 @@ //! test endpoint index settings -use rustfs_ecstore::api::disk as ecstore_disk; -use rustfs_ecstore::api::layout as ecstore_layout; -use rustfs_ecstore::api::storage as ecstore_storage; +use rustfs_ecstore::api::disk::endpoint::Endpoint; +use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints}; +use rustfs_ecstore::api::storage::{ECStore, init_local_disks}; use std::net::SocketAddr; use tempfile::TempDir; use tokio_util::sync::CancellationToken; -type ECStore = ecstore_storage::ECStore; -type Endpoint = ecstore_disk::endpoint::Endpoint; -type EndpointServerPools = ecstore_layout::EndpointServerPools; -type Endpoints = ecstore_layout::Endpoints; -type PoolEndpoints = ecstore_layout::PoolEndpoints; - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn test_endpoint_index_settings() -> anyhow::Result<()> { let temp_dir = TempDir::new()?; @@ -78,7 +72,7 @@ async fn test_endpoint_index_settings() -> anyhow::Result<()> { } // test ECStore initialization - ecstore_storage::init_local_disks(endpoint_pools.clone()).await?; + init_local_disks(endpoint_pools.clone()).await?; let server_addr: SocketAddr = "127.0.0.1:0".parse().unwrap(); let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()).await?; diff --git a/crates/heal/tests/heal_bug_fixes_test.rs b/crates/heal/tests/heal_bug_fixes_test.rs index c500747f5..25142623d 100644 --- a/crates/heal/tests/heal_bug_fixes_test.rs +++ b/crates/heal/tests/heal_bug_fixes_test.rs @@ -12,16 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use rustfs_ecstore::api::disk as ecstore_disk; +use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint}; use rustfs_heal::heal::{ event::{HealEvent, Severity}, task::{HealPriority, HealType}, utils, }; -type DiskStore = ecstore_disk::DiskStore; -type Endpoint = ecstore_disk::endpoint::Endpoint; - #[test] fn test_heal_event_to_heal_request_no_panic() { // Test that invalid pool/set indices don't cause panic diff --git a/crates/heal/tests/heal_integration_test.rs b/crates/heal/tests/heal_integration_test.rs index 19bec861b..11d9f8153 100644 --- a/crates/heal/tests/heal_integration_test.rs +++ b/crates/heal/tests/heal_integration_test.rs @@ -14,10 +14,10 @@ use http::HeaderMap; use rustfs_common::heal_channel::{HealOpts, HealScanMode}; -use rustfs_ecstore::api::bucket as ecstore_bucket; -use rustfs_ecstore::api::disk as ecstore_disk; -use rustfs_ecstore::api::layout as ecstore_layout; -use rustfs_ecstore::api::storage as ecstore_storage; +use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys; +use rustfs_ecstore::api::disk::endpoint::Endpoint; +use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints}; +use rustfs_ecstore::api::storage::{ECStore, init_local_disks}; use rustfs_heal::heal::{ manager::{HealConfig, HealManager}, storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI}, @@ -39,12 +39,6 @@ const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25); const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250); const NON_INLINE_TEST_DATA_SIZE: usize = 256 * 1024 + 137; -type ECStore = ecstore_storage::ECStore; -type Endpoint = ecstore_disk::endpoint::Endpoint; -type EndpointServerPools = ecstore_layout::EndpointServerPools; -type Endpoints = ecstore_layout::Endpoints; -type PoolEndpoints = ecstore_layout::PoolEndpoints; - fn non_inline_test_data() -> Vec { (0..NON_INLINE_TEST_DATA_SIZE).map(|idx| (idx % 251) as u8).collect() } @@ -127,7 +121,7 @@ async fn setup_test_env() -> (Vec, Arc, Arc (Vec, Arc, Arc, Arc)> = OnceLock::new(); static INIT: Once = Once::new(); const TRANSITION_WAIT_TIMEOUT: Duration = Duration::from_secs(15); -const BUCKET_LIFECYCLE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG; -const STORAGE_FORMAT_FILE: &str = ecstore_disk::STORAGE_FORMAT_FILE; - -type BucketVersioningSys = ecstore_bucket::versioning_sys::BucketVersioningSys; -type ECStore = ecstore_storage::ECStore; -type Endpoint = ecstore_disk::endpoint::Endpoint; -type EndpointServerPools = ecstore_layout::EndpointServerPools; -type Endpoints = ecstore_layout::Endpoints; -type PoolEndpoints = ecstore_layout::PoolEndpoints; -type DiskOption = ecstore_disk::DiskOption; -type ReadCloser = ecstore_client::transition_api::ReadCloser; -type ReaderImpl = ecstore_client::transition_api::ReaderImpl; -type TierConfig = ecstore_tier::tier_config::TierConfig; -type TierMinIO = ecstore_tier::tier_config::TierMinIO; -type TierType = ecstore_tier::tier_config::TierType; -type TransitionOptions = ecstore_bucket::lifecycle::lifecycle::TransitionOptions; -type WarmBackendGetOpts = ecstore_tier::warm_backend::WarmBackendGetOpts; fn init_tracing() { INIT.call_once(|| { @@ -133,7 +124,7 @@ async fn setup_test_env() -> (Vec, Arc) { let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]); // format disks (only first time) - ecstore_storage::init_local_disks(endpoint_pools.clone()).await.unwrap(); + init_local_disks(endpoint_pools.clone()).await.unwrap(); // create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free let port = 9002; // for simplicity @@ -151,10 +142,10 @@ async fn setup_test_env() -> (Vec, Arc) { .await .unwrap(); let buckets = buckets_list.into_iter().map(|v| v.name).collect(); - ecstore_bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await; + init_bucket_metadata_sys(ecstore.clone(), buckets).await; // Initialize background expiry workers - ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await; + init_background_expiry(ecstore.clone()).await; // Store in global once lock let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone())); @@ -202,7 +193,7 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec, Arc (Vec, Arc Result<(), Box "#; - metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?; + update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?; Ok(()) } @@ -313,7 +304,7 @@ async fn set_bucket_lifecycle_deletemarker(bucket_name: &str) -> Result<(), Box< "#; - metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?; + update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?; Ok(()) } @@ -336,7 +327,7 @@ async fn set_bucket_lifecycle_delmarker_expiration(bucket_name: &str, days: i64) "# ); - metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?; + update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?; Ok(()) } @@ -380,7 +371,7 @@ async fn set_bucket_lifecycle_transition_with_tier( "# ); - metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?; + update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?; Ok(()) } @@ -515,7 +506,7 @@ async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usi endpoint.set_pool_index(0); endpoint.set_set_index(0); endpoint.set_disk_index(0); - let disk = ecstore_disk::new_disk( + let disk = new_disk( &endpoint, &DiskOption { cleanup: false, @@ -599,7 +590,7 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str endpoint.set_pool_index(0); endpoint.set_set_index(0); endpoint.set_disk_index(0); - let disk = ecstore_disk::new_disk( + let disk = new_disk( &endpoint, &DiskOption { cleanup: false, @@ -610,13 +601,12 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str .expect("failed to open local disk"); let metadata_path = disk_path.join(bucket).join(object).join(STORAGE_FORMAT_FILE); let relative_path = metadata_path.to_string_lossy().to_string(); - let (_, scanner_path) = - ecstore_capacity::path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str()); + let (_, scanner_path) = path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str()); let file_type = fs::metadata(&metadata_path) .await .expect("failed to stat object metadata") .file_type(); - let lifecycle = metadata_sys::get(bucket) + let lifecycle = get_bucket_metadata(bucket) .await .expect("failed to load bucket metadata") .lifecycle_config @@ -642,7 +632,7 @@ async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) { endpoint.set_pool_index(0); endpoint.set_set_index(0); endpoint.set_disk_index(0); - let disk = ecstore_disk::new_disk( + let disk = new_disk( &endpoint, &DiskOption { cleanup: false, @@ -653,8 +643,7 @@ async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) { .expect("failed to open local disk"); let metadata_path = disk_path.join(bucket).join(object).join(STORAGE_FORMAT_FILE); let relative_path = metadata_path.to_string_lossy().to_string(); - let (_, scanner_path) = - ecstore_capacity::path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str()); + let (_, scanner_path) = path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str()); let file_type = fs::metadata(&metadata_path) .await .expect("failed to stat object metadata") @@ -882,7 +871,7 @@ mod serial_tests { println!("✅ Lifecycle configuration set for bucket: {bucket_name}"); // Verify lifecycle configuration was set - match metadata_sys::get(bucket_name.as_str()).await { + match get_bucket_metadata(bucket_name.as_str()).await { Ok(bucket_meta) => { assert!(bucket_meta.lifecycle_config.is_some()); println!("✅ Bucket metadata retrieved successfully"); @@ -971,12 +960,9 @@ mod serial_tests { .await .expect("Failed to upload transition metadata test object"); - ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( - ecstore.clone(), - put_bucket.as_str(), - ) - .await - .expect("Failed to enqueue transitioned put object"); + enqueue_transition_for_existing_objects(ecstore.clone(), put_bucket.as_str()) + .await + .expect("Failed to enqueue transitioned put object"); let put_info = wait_for_transition(&ecstore, put_bucket.as_str(), put_object, TRANSITION_WAIT_TIMEOUT) .await @@ -1044,12 +1030,9 @@ mod serial_tests { .await .expect("Failed to complete multipart upload"); - ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( - ecstore.clone(), - multipart_bucket.as_str(), - ) - .await - .expect("Failed to enqueue transitioned multipart object"); + enqueue_transition_for_existing_objects(ecstore.clone(), multipart_bucket.as_str()) + .await + .expect("Failed to enqueue transitioned multipart object"); let multipart_info = wait_for_transition(&ecstore, multipart_bucket.as_str(), multipart_object, TRANSITION_WAIT_TIMEOUT) .await @@ -1098,12 +1081,9 @@ mod serial_tests { .await .expect("Failed to copy object"); - ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( - ecstore.clone(), - dst_bucket.as_str(), - ) - .await - .expect("Failed to enqueue transitioned copied object"); + enqueue_transition_for_existing_objects(ecstore.clone(), dst_bucket.as_str()) + .await + .expect("Failed to enqueue transitioned copied object"); let copy_info = wait_for_transition(&ecstore, dst_bucket.as_str(), dst_object, TRANSITION_WAIT_TIMEOUT) .await @@ -1124,12 +1104,9 @@ mod serial_tests { .await .expect("Failed to set lifecycle configuration"); - ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( - ecstore.clone(), - bucket_name.as_str(), - ) - .await - .expect("Failed to enqueue transition for existing objects"); + enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str()) + .await + .expect("Failed to enqueue transition for existing objects"); let info = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT) .await @@ -1204,12 +1181,9 @@ mod serial_tests { .await .expect("Failed to complete multipart upload"); - ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( - ecstore.clone(), - bucket_name.as_str(), - ) - .await - .expect("Failed to enqueue transitioned restore object"); + enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str()) + .await + .expect("Failed to enqueue transitioned restore object"); let transitioned = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT) .await @@ -1279,12 +1253,9 @@ mod serial_tests { .expect("Failed to set lifecycle configuration"); upload_test_object(&ecstore, bucket_name.as_str(), object_name, initial_payload).await; - ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( - ecstore.clone(), - bucket_name.as_str(), - ) - .await - .expect("Failed to enqueue transitioned object"); + enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str()) + .await + .expect("Failed to enqueue transitioned object"); let transitioned = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT) .await @@ -1306,7 +1277,7 @@ mod serial_tests { "stale transitioned remote object should still exist before scanner fallback runs" ); - ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await; + init_background_expiry(ecstore.clone()).await; scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await; assert!( @@ -1367,7 +1338,7 @@ mod serial_tests { "stale transitioned remote object should still exist before scanner cleanup runs" ); - ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await; + init_background_expiry(ecstore.clone()).await; scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await; assert!( @@ -1410,12 +1381,9 @@ mod serial_tests { let remote_object = transitioned.transitioned_object.name.clone(); assert!(backend.objects.lock().await.contains_key(&remote_object)); - ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( - ecstore.clone(), - bucket_name.as_str(), - ) - .await - .expect("existing-object backfill should succeed after compensation transition"); + enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str()) + .await + .expect("existing-object backfill should succeed after compensation transition"); let info = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT) .await @@ -1460,7 +1428,7 @@ mod serial_tests { "# ); - metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) + update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) .await .expect("Failed to set lifecycle configuration"); @@ -1545,7 +1513,7 @@ mod serial_tests { "# ); - metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) + update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) .await .expect("Failed to set lifecycle configuration"); @@ -1737,7 +1705,7 @@ mod serial_tests { assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await); - ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await; + init_background_expiry(ecstore.clone()).await; scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await; assert!( @@ -1772,7 +1740,7 @@ mod serial_tests { "# ); - metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) + update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) .await .expect("Failed to set lifecycle configuration"); @@ -1837,11 +1805,11 @@ mod serial_tests { "#; - metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()) + update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()) .await .expect("Failed to set noncurrent lifecycle configuration"); - ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await; + init_background_expiry(ecstore.clone()).await; scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await; @@ -1875,7 +1843,7 @@ mod serial_tests { "#; - metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()) + update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()) .await .expect("Failed to set noncurrent lifecycle configuration"); @@ -1962,7 +1930,7 @@ mod serial_tests { "# ); - metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) + update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) .await .expect("Failed to set lifecycle configuration"); upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await; diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index 7311cfa00..5d5892eb8 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,17 +5,16 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-remaining-external-owner-symbols` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133`. -- Based on: API-133 slice. +- Branch: `overtrue/arch-test-fuzz-owner-symbols` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134`. +- Based on: API-134 slice. - PR type for this branch: `pure-move` - Runtime behavior changes: none. -- Rust code changes: replace the remaining heal, IAM, and observability owner - root ECStore module aliases with explicit local symbols, type aliases, - constants, and wrapper functions. -- CI/script changes: treat heal, IAM, and observability as completed external - owner roots and reject restored `ecstore_*` module aliases there. -- Docs changes: record the API-134 remaining external owner facade symbol cleanup. +- Rust code changes: replace e2e, heal/scanner integration-test, and fuzz-target + ECStore owner module aliases with explicit symbols. +- CI/script changes: treat the completed test/fuzz files as explicit-symbol + boundaries and reject restored `ecstore_*` module aliases there. +- Docs changes: record the API-135 test/fuzz owner facade symbol cleanup. ## Phase 0 Tasks @@ -3924,6 +3923,21 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block scan, branch freshness check, pre-commit quality gate, and three-expert review. +- [x] `API-135` Replace test and fuzz owner module aliases with symbols. + - Do: replace e2e, heal/scanner integration-test, and fuzz-target + `ecstore_*` module aliases with explicit ECStore symbols. + - Acceptance: the completed test/fuzz files no longer import broad + `ecstore_*` owner modules, direct symbols preserve the same ECStore facade + contracts, and the migration guard prevents reintroducing module aliases in + these files. + - Must preserve: e2e RPC client construction, replication target tests, heal + ECStore setup, scanner lifecycle/tier/transition behavior, and bucket/path + fuzz validation semantics. + - Verification: focused e2e/heal/scanner compile, fuzz manifest compile, + completed test/fuzz alias residual scan, migration/layer guards, formatting, + diff hygiene, Rust risk scan, branch freshness check, pre-commit quality + gate, and three-expert review. + ## Next PRs 1. `pure-move`: continue pruning remaining facade compatibility and owner boundaries. @@ -3932,14 +3946,28 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | pass | API-134 replaces the remaining heal, IAM, and observability owner-root ECStore module aliases with explicit local symbols and wrappers. | -| Migration preservation | pass | Heal disk lookup, IAM config/notification wrappers, and observability data-usage/quota/lifecycle/replication/capacity paths keep the same ECStore targets through owner-local symbols. | -| Testing/verification | pass | Focused heal/IAM/observability compile, completed-owner alias residual scan, migration/layer guards, formatting, diff hygiene, full pre-commit, and diff-only Rust risk scan passed. | +| Quality/architecture | pass | API-135 replaces e2e, heal/scanner integration-test, and fuzz-target ECStore owner module aliases with explicit imported symbols. | +| Migration preservation | pass | RPC client construction, replication target access, heal ECStore setup, scanner lifecycle/tier/transition paths, and fuzz validation call targets keep the same ECStore facade symbols. | +| Testing/verification | pass | Focused e2e/heal/scanner compile, fuzz manifest compile, completed test/fuzz alias residual scan, migration/layer guards, formatting, diff hygiene, full pre-commit, and diff-only Rust risk scan passed. | ## Verification Notes Passed before push: +- Issue #660 API-135 current slice: + - `cargo check --tests -p e2e_test -p rustfs-heal -p rustfs-scanner`: passed. + - `cargo check --manifest-path fuzz/Cargo.toml --bins`: passed. + - `cargo fmt --all`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - `make pre-commit`: passed. + - Completed test/fuzz module-alias residual scan: passed. + - Rust risk scan: diff-only scan found import and call-target rewrites only; + no new production unwrap/expect, panic/todo/unsafe, or risky behavior added. + - Issue #660 API-134 current slice: - `cargo check --tests -p rustfs-heal -p rustfs-iam -p rustfs-obs`: passed. - `cargo fmt --all`: passed. diff --git a/fuzz/fuzz_targets/bucket_validation.rs b/fuzz/fuzz_targets/bucket_validation.rs index 12dd240f2..46009b0a0 100644 --- a/fuzz/fuzz_targets/bucket_validation.rs +++ b/fuzz/fuzz_targets/bucket_validation.rs @@ -1,7 +1,9 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use rustfs_ecstore::api::bucket as ecstore_bucket; +use rustfs_ecstore::api::bucket::utils::{ + check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname, +}; fn parse_case(data: &[u8]) -> (String, String) { let text = String::from_utf8_lossy(data); @@ -22,10 +24,10 @@ fn looks_like_ipv4(text: &str) -> bool { fuzz_target!(|data: &[u8]| { let (bucket, object) = parse_case(data); - let strict_bucket_ok = ecstore_bucket::utils::check_valid_bucket_name_strict(&bucket).is_ok(); - let valid_bucket_for_object_ops = strict_bucket_ok || ecstore_bucket::utils::is_meta_bucketname(&bucket); - let pair_ok = ecstore_bucket::utils::check_bucket_and_object_names(&bucket, &object).is_ok(); - let list_ok = ecstore_bucket::utils::check_list_objs_args(&bucket, &object, &None).is_ok(); + let strict_bucket_ok = check_valid_bucket_name_strict(&bucket).is_ok(); + let valid_bucket_for_object_ops = strict_bucket_ok || is_meta_bucketname(&bucket); + let pair_ok = check_bucket_and_object_names(&bucket, &object).is_ok(); + let list_ok = check_list_objs_args(&bucket, &object, &None).is_ok(); if strict_bucket_ok { let trimmed = bucket.trim(); diff --git a/fuzz/fuzz_targets/path_containment.rs b/fuzz/fuzz_targets/path_containment.rs index aae37c32b..2bba67086 100644 --- a/fuzz/fuzz_targets/path_containment.rs +++ b/fuzz/fuzz_targets/path_containment.rs @@ -1,7 +1,9 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use rustfs_ecstore::api::bucket as ecstore_bucket; +use rustfs_ecstore::api::bucket::utils::{ + check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix, +}; use rustfs_utils::path::{clean, path_join}; use std::path::{Path, PathBuf}; @@ -92,9 +94,9 @@ fuzz_target!(|data: &[u8]| { let (bucket, base_object, extra, flags) = parse_case(data); let object = materialize_object(base_object, &extra, &flags); - let has_bad_component = ecstore_bucket::utils::has_bad_path_component(&object); - let is_valid_prefix = ecstore_bucket::utils::is_valid_object_prefix(&object); - let length_and_slash_ok = ecstore_bucket::utils::check_object_name_for_length_and_slash(&bucket, &object).is_ok(); + let has_bad_component = has_bad_path_component(&object); + let is_valid_prefix = is_valid_object_prefix(&object); + let length_and_slash_ok = check_object_name_for_length_and_slash(&bucket, &object).is_ok(); if object.is_empty() || !(is_valid_prefix && length_and_slash_ok) { return; diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 47434455d..8a5d9bbbb 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -1062,7 +1062,7 @@ fi rg -n --with-filename 'rustfs_ecstore::api::[a-z_]+::' rustfs/src crates fuzz \ --glob '*.rs' \ --glob '!crates/ecstore/**' | - rg -v '^(crates/heal/src/heal/mod\.rs|crates/iam/src/lib\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/lib\.rs):' || true + rg -v '^(fuzz/fuzz_targets/bucket_validation\.rs|fuzz/fuzz_targets/path_containment\.rs|crates/e2e_test/src/reliant/grpc_lock_client\.rs|crates/e2e_test/src/reliant/node_interact_test\.rs|crates/e2e_test/src/replication_extension_test\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/endpoint_index_test\.rs|crates/heal/tests/heal_bug_fixes_test\.rs|crates/heal/tests/heal_integration_test\.rs|crates/iam/src/lib\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs):' || true ) >"$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" if [[ -s "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" ]]; then @@ -1073,16 +1073,25 @@ fi cd "$ROOT_DIR" rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \ crates/heal/src/heal/mod.rs \ + crates/heal/tests/endpoint_index_test.rs \ + crates/heal/tests/heal_bug_fixes_test.rs \ + crates/heal/tests/heal_integration_test.rs \ crates/iam/src/lib.rs \ crates/notify/src/lib.rs \ crates/obs/src/metrics/mod.rs \ crates/protocols/src/swift/mod.rs \ crates/s3select-api/src/lib.rs \ - crates/scanner/src/lib.rs || true + crates/scanner/src/lib.rs \ + crates/scanner/tests/lifecycle_integration_test.rs \ + crates/e2e_test/src/reliant/grpc_lock_client.rs \ + crates/e2e_test/src/reliant/node_interact_test.rs \ + crates/e2e_test/src/replication_extension_test.rs \ + fuzz/fuzz_targets/bucket_validation.rs \ + fuzz/fuzz_targets/path_containment.rs || true ) >"$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE" if [[ -s "$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE" ]]; then - report_failure "completed external owner roots must expose explicit ECStore symbols instead of ecstore_* module aliases: $(paste -sd '; ' "$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE")" + report_failure "completed external owner and test/fuzz boundaries must expose explicit ECStore symbols instead of ecstore_* module aliases: $(paste -sd '; ' "$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE")" fi (