gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 13:16:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: centralize external ECStore facade aliases (#3746)

Browse Source
This commit is contained in:
Zhengchao An
2026-06-22 20:37:13 +08:00
committed by GitHub
parent 89805ffb4d
commit 88a87b3e8f
16 changed files with 222 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/tests/lifecycle_integration_test.rs
+88 -57
View File
@@ -12,29 +12,20 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use ecstore_bucket::metadata_sys;
use ecstore_disk::DiskAPI as _;
use ecstore_global::GLOBAL_TierConfigMgr;
use ecstore_tier::warm_backend::{WarmBackend as ScannerWarmBackend, build_transition_put_options};
use futures::FutureExt;
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
use rustfs_ecstore::api::{
bucket::{
lifecycle::{
bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, init_background_expiry},
lifecycle::TransitionOptions,
},
metadata::BUCKET_LIFECYCLE_CONFIG,
metadata_sys::{self, init_bucket_metadata_sys},
versioning_sys::BucketVersioningSys,
},
capacity::path2_bucket_object_with_base_path,
client::transition_api::{ReadCloser, ReaderImpl},
disk::{DiskAPI as _, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, new_disk},
global::GLOBAL_TierConfigMgr,
layout::{EndpointServerPools, Endpoints, PoolEndpoints},
storage::{ECStore, init_local_disks},
tier::{
tier_config::{TierConfig, TierMinIO, TierType},
warm_backend::{WarmBackend as ScannerWarmBackend, WarmBackendGetOpts, build_transition_put_options},
},
};
use rustfs_ecstore::api::bucket as ecstore_bucket;
use rustfs_ecstore::api::capacity as ecstore_capacity;
use rustfs_ecstore::api::client as ecstore_client;
use rustfs_ecstore::api::disk as ecstore_disk;
use rustfs_ecstore::api::global as ecstore_global;
use rustfs_ecstore::api::layout as ecstore_layout;
use rustfs_ecstore::api::storage as ecstore_storage;
use rustfs_ecstore::api::tier as ecstore_tier;
use rustfs_filemeta::FileMeta;
use rustfs_scanner::scanner_folder::ScannerItem;
use rustfs_scanner::scanner_io::ScannerIODisk;
@@ -66,6 +57,23 @@ use uuid::Uuid;
static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new();
static INIT: Once = Once::new();
const TRANSITION_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
const BUCKET_LIFECYCLE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
const STORAGE_FORMAT_FILE: &str = ecstore_disk::STORAGE_FORMAT_FILE;
type BucketVersioningSys = ecstore_bucket::versioning_sys::BucketVersioningSys;
type ECStore = ecstore_storage::ECStore;
type Endpoint = ecstore_disk::endpoint::Endpoint;
type EndpointServerPools = ecstore_layout::EndpointServerPools;
type Endpoints = ecstore_layout::Endpoints;
type PoolEndpoints = ecstore_layout::PoolEndpoints;
type DiskOption = ecstore_disk::DiskOption;
type ReadCloser = ecstore_client::transition_api::ReadCloser;
type ReaderImpl = ecstore_client::transition_api::ReaderImpl;
type TierConfig = ecstore_tier::tier_config::TierConfig;
type TierMinIO = ecstore_tier::tier_config::TierMinIO;
type TierType = ecstore_tier::tier_config::TierType;
type TransitionOptions = ecstore_bucket::lifecycle::lifecycle::TransitionOptions;
type WarmBackendGetOpts = ecstore_tier::warm_backend::WarmBackendGetOpts;
fn init_tracing() {
INIT.call_once(|| {
@@ -125,7 +133,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
// format disks (only first time)
init_local_disks(endpoint_pools.clone()).await.unwrap();
ecstore_storage::init_local_disks(endpoint_pools.clone()).await.unwrap();
// create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
let port = 9002; // for simplicity
@@ -143,10 +151,10 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
ecstore_bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
// Initialize background expiry workers
init_background_expiry(ecstore.clone()).await;
ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
// Store in global once lock
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
@@ -194,7 +202,7 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
};
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
init_local_disks(endpoint_pools.clone()).await.unwrap();
ecstore_storage::init_local_disks(endpoint_pools.clone()).await.unwrap();
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
@@ -209,10 +217,10 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
ecstore_bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
if init_expiry {
init_background_expiry(ecstore.clone()).await;
ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
}
(disk_paths, ecstore)
@@ -507,7 +515,7 @@ async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usi
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
let disk = ecstore_disk::new_disk(
&endpoint,
&DiskOption {
cleanup: false,
@@ -591,7 +599,7 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
let disk = ecstore_disk::new_disk(
&endpoint,
&DiskOption {
cleanup: false,
@@ -602,7 +610,8 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
.expect("failed to open local disk");
let metadata_path = disk_path.join(bucket).join(object).join(STORAGE_FORMAT_FILE);
let relative_path = metadata_path.to_string_lossy().to_string();
let (_, scanner_path) = path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str());
let (_, scanner_path) =
ecstore_capacity::path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str());
let file_type = fs::metadata(&metadata_path)
.await
.expect("failed to stat object metadata")
@@ -633,7 +642,7 @@ async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) {
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
let disk = ecstore_disk::new_disk(
&endpoint,
&DiskOption {
cleanup: false,
@@ -644,7 +653,8 @@ async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) {
.expect("failed to open local disk");
let metadata_path = disk_path.join(bucket).join(object).join(STORAGE_FORMAT_FILE);
let relative_path = metadata_path.to_string_lossy().to_string();
let (_, scanner_path) = path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str());
let (_, scanner_path) =
ecstore_capacity::path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str());
let file_type = fs::metadata(&metadata_path)
.await
.expect("failed to stat object metadata")
@@ -961,9 +971,12 @@ mod serial_tests {
.await
.expect("Failed to upload transition metadata test object");
enqueue_transition_for_existing_objects(ecstore.clone(), put_bucket.as_str())
.await
.expect("Failed to enqueue transitioned put object");
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
ecstore.clone(),
put_bucket.as_str(),
)
.await
.expect("Failed to enqueue transitioned put object");
let put_info = wait_for_transition(&ecstore, put_bucket.as_str(), put_object, TRANSITION_WAIT_TIMEOUT)
.await
@@ -1031,9 +1044,12 @@ mod serial_tests {
.await
.expect("Failed to complete multipart upload");
enqueue_transition_for_existing_objects(ecstore.clone(), multipart_bucket.as_str())
.await
.expect("Failed to enqueue transitioned multipart object");
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
ecstore.clone(),
multipart_bucket.as_str(),
)
.await
.expect("Failed to enqueue transitioned multipart object");
let multipart_info = wait_for_transition(&ecstore, multipart_bucket.as_str(), multipart_object, TRANSITION_WAIT_TIMEOUT)
.await
@@ -1082,9 +1098,12 @@ mod serial_tests {
.await
.expect("Failed to copy object");
enqueue_transition_for_existing_objects(ecstore.clone(), dst_bucket.as_str())
.await
.expect("Failed to enqueue transitioned copied object");
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
ecstore.clone(),
dst_bucket.as_str(),
)
.await
.expect("Failed to enqueue transitioned copied object");
let copy_info = wait_for_transition(&ecstore, dst_bucket.as_str(), dst_object, TRANSITION_WAIT_TIMEOUT)
.await
@@ -1105,9 +1124,12 @@ mod serial_tests {
.await
.expect("Failed to set lifecycle configuration");
enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str())
.await
.expect("Failed to enqueue transition for existing objects");
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
ecstore.clone(),
bucket_name.as_str(),
)
.await
.expect("Failed to enqueue transition for existing objects");
let info = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT)
.await
@@ -1182,9 +1204,12 @@ mod serial_tests {
.await
.expect("Failed to complete multipart upload");
enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str())
.await
.expect("Failed to enqueue transitioned restore object");
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
ecstore.clone(),
bucket_name.as_str(),
)
.await
.expect("Failed to enqueue transitioned restore object");
let transitioned = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT)
.await
@@ -1254,9 +1279,12 @@ mod serial_tests {
.expect("Failed to set lifecycle configuration");
upload_test_object(&ecstore, bucket_name.as_str(), object_name, initial_payload).await;
enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str())
.await
.expect("Failed to enqueue transitioned object");
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
ecstore.clone(),
bucket_name.as_str(),
)
.await
.expect("Failed to enqueue transitioned object");
let transitioned = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT)
.await
@@ -1278,7 +1306,7 @@ mod serial_tests {
"stale transitioned remote object should still exist before scanner fallback runs"
);
init_background_expiry(ecstore.clone()).await;
ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
@@ -1339,7 +1367,7 @@ mod serial_tests {
"stale transitioned remote object should still exist before scanner cleanup runs"
);
init_background_expiry(ecstore.clone()).await;
ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
@@ -1382,9 +1410,12 @@ mod serial_tests {
let remote_object = transitioned.transitioned_object.name.clone();
assert!(backend.objects.lock().await.contains_key(&remote_object));
enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str())
.await
.expect("existing-object backfill should succeed after compensation transition");
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
ecstore.clone(),
bucket_name.as_str(),
)
.await
.expect("existing-object backfill should succeed after compensation transition");
let info = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT)
.await
@@ -1706,7 +1737,7 @@ mod serial_tests {
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
init_background_expiry(ecstore.clone()).await;
ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
@@ -1810,7 +1841,7 @@ mod serial_tests {
.await
.expect("Failed to set noncurrent lifecycle configuration");
init_background_expiry(ecstore.clone()).await;
ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;