gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 15:37:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: flatten test harness storage compat aliases (#3596)

Browse Source 
* refactor: flatten test harness storage compat aliases

* refactor: flatten rustfs storage compat aliases (#3597)

* refactor: prune runtime storage compat surface (#3598)

* refactor: flatten runtime secondary storage compat (#3599)

* docs: add scheduler placement profiling baselines (#3600)

* feat: add observability topology capability contracts (#3601)
This commit is contained in:
安正超
2026-06-19 08:30:47 +08:00
committed by GitHub
parent b1c6578df1
commit ada6f7587e
87 changed files with 1551 additions and 990 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/tests/common/storage_compat.rs
+21 -64
View File
@@ -12,70 +12,27 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod ecstore {
#![allow(unused_imports)]
#![allow(dead_code, unused_imports)]
pub(crate) mod bucket {
pub(crate) mod lifecycle {
pub(crate) use rustfs_ecstore::bucket::lifecycle::lifecycle::TransitionOptions;
pub(crate) const BUCKET_LIFECYCLE_CONFIG: &str = rustfs_ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
pub(crate) const STORAGE_FORMAT_FILE: &str = rustfs_ecstore::disk::STORAGE_FORMAT_FILE;
pub(crate) mod bucket_lifecycle_ops {
pub(crate) use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{
enqueue_transition_for_existing_objects, init_background_expiry,
};
}
}
pub(crate) type TransitionOptions = rustfs_ecstore::bucket::lifecycle::lifecycle::TransitionOptions;
pub(crate) mod metadata {
pub(crate) use rustfs_ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
}
pub(crate) mod metadata_sys {
pub(crate) use rustfs_ecstore::bucket::metadata_sys::{get, init_bucket_metadata_sys, update};
}
pub(crate) mod versioning_sys {
pub(crate) use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
}
}
pub(crate) mod client {
pub(crate) mod transition_api {
pub(crate) use rustfs_ecstore::client::transition_api::{ReadCloser, ReaderImpl};
}
}
pub(crate) mod disk {
pub(crate) use rustfs_ecstore::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, new_disk};
pub(crate) mod endpoint {
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
}
}
pub(crate) mod endpoints {
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
}
pub(crate) mod global {
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
}
pub(crate) mod pools {
pub(crate) use rustfs_ecstore::pools::path2_bucket_object_with_base_path;
}
pub(crate) mod store {
pub(crate) use rustfs_ecstore::store::{ECStore, init_local_disks};
}
pub(crate) mod tier {
pub(crate) mod tier_config {
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierMinIO, TierType};
}
pub(crate) mod warm_backend {
pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options};
}
}
}
pub(crate) use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{
enqueue_transition_for_existing_objects, init_background_expiry,
};
pub(crate) use rustfs_ecstore::bucket::metadata_sys::{
get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata,
};
pub(crate) use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
pub(crate) use rustfs_ecstore::client::transition_api::{ReadCloser, ReaderImpl};
pub(crate) use rustfs_ecstore::disk::{DiskAPI, DiskOption, endpoint::Endpoint, new_disk};
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
pub(crate) use rustfs_ecstore::pools::path2_bucket_object_with_base_path;
pub(crate) use rustfs_ecstore::store::ECStore;
pub(crate) use rustfs_ecstore::store::init_local_disks;
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierMinIO, TierType};
pub(crate) use rustfs_ecstore::tier::warm_backend::WarmBackendGetOpts;
pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, build_transition_put_options};
crates/scanner/tests/lifecycle_integration_test.rs
+28 -50
View File
@@ -14,24 +14,12 @@
mod common;
use crate::common::storage_compat::ecstore::{
bucket::metadata::BUCKET_LIFECYCLE_CONFIG,
bucket::{
lifecycle::{TransitionOptions, bucket_lifecycle_ops::enqueue_transition_for_existing_objects},
metadata_sys,
versioning_sys::BucketVersioningSys,
},
client::transition_api::{ReadCloser, ReaderImpl},
disk::endpoint::Endpoint,
disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, new_disk},
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
global::GLOBAL_TierConfigMgr,
pools::path2_bucket_object_with_base_path,
store::ECStore,
tier::{
tier_config::{TierConfig, TierMinIO, TierType},
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
},
use crate::common::storage_compat::{
BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DiskAPI, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
GLOBAL_TierConfigMgr, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, TierConfig, TierMinIO, TierType,
TransitionOptions, WarmBackend, WarmBackendGetOpts, build_transition_put_options, enqueue_transition_for_existing_objects,
get_bucket_metadata, init_background_expiry, init_bucket_metadata_sys, init_local_disks, new_disk,
path2_bucket_object_with_base_path, update_bucket_metadata,
};
use futures::FutureExt;
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
@@ -125,9 +113,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
// format disks (only first time)
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
.await
.unwrap();
init_local_disks(endpoint_pools.clone()).await.unwrap();
// create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
let port = 9002; // for simplicity
@@ -145,11 +131,10 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
// Initialize background expiry workers
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
init_background_expiry(ecstore.clone()).await;
// Store in global once lock
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
@@ -197,9 +182,7 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
};
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
.await
.unwrap();
init_local_disks(endpoint_pools.clone()).await.unwrap();
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
@@ -214,11 +197,10 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
if init_expiry {
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
init_background_expiry(ecstore.clone()).await;
}
(disk_paths, ecstore)
@@ -287,7 +269,7 @@ async fn set_bucket_lifecycle(bucket_name: &str) -> Result<(), Box<dyn std::erro
</Rule>
</LifecycleConfiguration>"#;
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
Ok(())
}
@@ -311,7 +293,7 @@ async fn set_bucket_lifecycle_deletemarker(bucket_name: &str) -> Result<(), Box<
</Rule>
</LifecycleConfiguration>"#;
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
Ok(())
}
@@ -334,7 +316,7 @@ async fn set_bucket_lifecycle_delmarker_expiration(bucket_name: &str, days: i64)
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
Ok(())
}
@@ -378,7 +360,7 @@ async fn set_bucket_lifecycle_transition_with_tier(
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
Ok(())
}
@@ -613,7 +595,7 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
.await
.expect("failed to stat object metadata")
.file_type();
let lifecycle = metadata_sys::get(bucket)
let lifecycle = get_bucket_metadata(bucket)
.await
.expect("failed to load bucket metadata")
.lifecycle_config
@@ -878,7 +860,7 @@ mod serial_tests {
println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
// Verify lifecycle configuration was set
match crate::common::storage_compat::ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
match get_bucket_metadata(bucket_name.as_str()).await {
Ok(bucket_meta) => {
assert!(bucket_meta.lifecycle_config.is_some());
println!("✅ Bucket metadata retrieved successfully");
@@ -1284,8 +1266,7 @@ mod serial_tests {
"stale transitioned remote object should still exist before scanner fallback runs"
);
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
init_background_expiry(ecstore.clone()).await;
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
@@ -1346,8 +1327,7 @@ mod serial_tests {
"stale transitioned remote object should still exist before scanner cleanup runs"
);
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
init_background_expiry(ecstore.clone()).await;
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
@@ -1437,7 +1417,7 @@ mod serial_tests {
</Rule>
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
@@ -1522,7 +1502,7 @@ mod serial_tests {
</Rule>
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
@@ -1714,8 +1694,7 @@ mod serial_tests {
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
init_background_expiry(ecstore.clone()).await;
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
@@ -1750,7 +1729,7 @@ mod serial_tests {
</Rule>
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
@@ -1815,12 +1794,11 @@ mod serial_tests {
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>"#;
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
.await
.expect("Failed to set noncurrent lifecycle configuration");
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
init_background_expiry(ecstore.clone()).await;
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
@@ -1854,7 +1832,7 @@ mod serial_tests {
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>"#;
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
.await
.expect("Failed to set noncurrent lifecycle configuration");
@@ -1941,7 +1919,7 @@ mod serial_tests {
</Rule>
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;