gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-09 06:39:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(e2e): make scanner snapshot tests deterministic (#5812)

Browse Source 
* test(e2e): configure scanner snapshot timing

* style(e2e): format scanner snapshot tests
This commit is contained in:
cxymds
2026-08-08 05:43:56 +08:00
committed by GitHub
parent 8201a74f7f
commit a4712fae81
3 changed files with 49 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/bucket_stats_regression_test.rs
+19 -3
View File
@@ -31,7 +31,7 @@
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, awscurl_get, init_logging};
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, awscurl_get, init_logging};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
use rustfs_data_usage::DataUsageInfo;
@@ -65,7 +65,7 @@ mod tests {
info!("RT-09: bucket object count updates after PUT");
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV)
.await
.expect("start RustFS");
@@ -132,7 +132,7 @@ mod tests {
info!("RT-09b: bucket object count updates after DELETE");
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV)
.await
.expect("start RustFS");
@@ -153,6 +153,22 @@ mod tests {
.expect("put object");
}
let mut found_nonzero = false;
for attempt in 0..18 {
sleep(Duration::from_secs(5)).await;
if let Ok(usage) = get_data_usage(&env).await
&& let Some(bucket_usage) = usage.buckets_usage.get(bucket)
{
info!(" baseline attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
if bucket_usage.objects_count >= 5 {
found_nonzero = true;
break;
}
}
}
assert!(found_nonzero, "RT-09b setup failed: scanner did not observe the 5 uploaded objects");
// Delete all objects
for i in 0..5 {
client
crates/e2e_test/src/common.rs
+2
View File
@@ -47,6 +47,8 @@ use walkdir::WalkDir;
pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
pub const ENV_RUSTFS_BUILD_FEATURES: &str = "RUSTFS_BUILD_FEATURES";
pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
&[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
pub const TEST_BUCKET: &str = "e2e-test-bucket";
const RUSTFS_FULL_FEATURE: &str = "full";
crates/e2e_test/src/data_usage_test.rs
+28 -12
View File
@@ -18,7 +18,7 @@ use rustfs_data_usage::DataUsageInfo;
use serial_test::serial;
use tokio::time::{Duration, sleep};
use crate::common::{RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging};
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging};
async fn get_data_usage_info(env: &RustFSTestEnvironment) -> Result<DataUsageInfo, Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/datausageinfo", env.url);
@@ -35,16 +35,26 @@ where
F: FnMut(&DataUsageInfo) -> bool,
{
let mut last_usage = DataUsageInfo::default();
let mut last_query_error = None;
for _ in 0..45 {
let usage = get_data_usage_info(env).await?;
if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
return Ok(usage);
match get_data_usage_info(env).await {
Ok(usage) => {
last_query_error = None;
if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
return Ok(usage);
}
last_usage = usage;
}
Err(err) => last_query_error = Some(err.to_string()),
}
last_usage = usage;
sleep(Duration::from_secs(2)).await;
}
Err(format!("bucket usage did not converge for {bucket}; last usage: {last_usage:?}").into())
Err(format!(
"bucket usage did not converge for {bucket}; last usage: {last_usage:?}; last query error: {}",
last_query_error.as_deref().unwrap_or("none")
)
.into())
}
/// Regression test for data usage accuracy (issue #1012).
@@ -56,7 +66,7 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV).await?;
let client = env.create_s3_client();
@@ -74,8 +84,14 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
.await?;
}
// Query admin data usage API
let usage = get_data_usage_info(&env).await?;
let usage = wait_for_bucket_usage(&env, TEST_BUCKET, |usage| {
usage
.buckets_usage
.get(TEST_BUCKET)
.map(|bucket_usage| usage.objects_total_count >= 1000 && bucket_usage.objects_count >= 1000)
.unwrap_or(false)
})
.await?;
// Assert total object count and per-bucket count are not truncated
let bucket_usage = usage
@@ -108,7 +124,7 @@ async fn data_usage_reports_versioned_objects_and_delete_markers() -> Result<(),
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV).await?;
let client = env.create_s3_client();
let bucket = "data-usage-versioned";
@@ -184,8 +200,8 @@ async fn data_usage_reports_versioned_objects_and_delete_markers() -> Result<(),
assert_eq!(usage.versions_total_count, 3, "total version count should match bucket usage");
assert_eq!(usage.delete_markers_total_count, 1, "total delete marker count should match bucket usage");
env.stop_server();
env.start_rustfs_server(vec![]).await?;
env.restart_server_preserving_data(vec![], FAST_DATA_USAGE_SCANNER_ENV)
.await?;
let restarted_usage = wait_for_bucket_usage(&env, bucket, |usage| {
usage