gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(utils): add rustfs-test-utils crate and shared ECStore bootstrap (#4850)

Browse Source 
* test(utils): add rustfs-test-utils crate, absorb heal/iam ECStore bootstrap

backlog#1153 infra-1. The ~50-line "build a real temp-disk ECStore"
bootstrap was copy-pasted (and drifting) across the heal and iam
integration tests. This adds crates/test-utils (rustfs-test-utils, a
dev-dependency-only crate) owning that bootstrap and converts the four
copies into thin wrappers:

- TestECStoreEnvBuilder: disk_count (default 4), prefix (uuid-suffixed
  /tmp dir), base_dir (caller-owned dir, e.g. tempfile::TempDir),
  init_bucket_metadata (default true; the iam bootstrap test opts out
  to preserve its historical semantics). TestECStoreEnv exposes
  temp_root/disk_paths/ecstore plus a versioned-bucket helper, and
  init_tracing() replaces the per-file Once blocks.
- All rustfs_ecstore imports stay behind src/ecstore_test_compat.rs,
  the sanctioned test-compat boundary pattern (mirrors
  crates/iam/tests/ecstore_test_compat).
- heal: heal_integration_test / heal_b5_versioned_regression_test /
  heal_b920_subquorum_union_test drop their setup_test_env{,_n} copies
  for heal_env{,_n} wrappers; the tests/storage_api.rs integration
  surface shrinks to what test bodies still touch.
- iam: iam_bootstrap_no_lock_test drops build_local_ecstore; its
  ecstore_test_compat fixture shrinks to SetupType +
  update_erasure_type.

rg 'async fn setup_test_env' crates/heal crates/iam now returns 0.
Scanner's lifecycle tests are deliberately NOT absorbed (gated on
ilm-1; 14 of 15 are #[ignore]d today). Net -230 lines.

* fix(heal): drop tokio::fs import orphaned by the b920 bootstrap move

* fix(heal): drop tokio::fs import orphaned by the b5 bootstrap move
This commit is contained in:
Zhengchao An
2026-07-15 16:08:30 +08:00
committed by GitHub
parent 602a742a1b
commit f05a69d51b
14 changed files with 425 additions and 310 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+14
View File
@@ -9276,6 +9276,7 @@ dependencies = [
"rustfs-ecstore",
"rustfs-madmin",
"rustfs-storage-api",
"rustfs-test-utils",
"rustfs-utils",
"serde",
"serde_json",
@@ -9313,6 +9314,7 @@ dependencies = [
"rustfs-madmin",
"rustfs-policy",
"rustfs-storage-api",
"rustfs-test-utils",
"rustfs-utils",
"serde",
"serde_json",
@@ -9991,6 +9993,18 @@ dependencies = [
"uuid",
]
[[package]]
name = "rustfs-test-utils"
version = "1.0.0-beta.9"
dependencies = [
"rustfs-ecstore",
"rustfs-storage-api",
"tokio",
"tokio-util",
"tracing-subscriber",
"uuid",
]
[[package]]
name = "rustfs-tls-runtime"
version = "1.0.0-beta.9"
Cargo.toml
+2
View File
@@ -52,6 +52,7 @@ members = [
"crates/extension-schema", # Extension schema contracts
"crates/signer", # client signer
"crates/storage-api", # Storage API contracts
"crates/test-utils", # Shared test bootstrap helpers (dev-dependency only)
"crates/targets", # Target-specific configurations and utilities
"crates/trusted-proxies", # Trusted proxies management
"crates/tls-runtime", # Project-wide TLS runtime foundation
@@ -125,6 +126,7 @@ rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.9" }
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.9" }
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.9" }
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.9" }
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.9" }
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.9" }
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.9" }
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.9" }
crates/heal/Cargo.toml
+1
View File
@@ -48,6 +48,7 @@ base64 = { workspace = true }
[dev-dependencies]
serde_json = { workspace = true }
rustfs-test-utils = { workspace = true }
serial_test = { workspace = true }
tracing-subscriber = { workspace = true }
tempfile = { workspace = true }
crates/heal/tests/heal_b5_versioned_regression_test.rs
+19 -86
View File
@@ -18,7 +18,7 @@
//!
//! These drive the REAL `ECStoreHealStorage` (not a mock) against a real 4-disk
//! `ECStore`, mirroring `heal_integration_test.rs`. Every test is `#[serial]`
//! and re-runs the full `setup_test_env` init (which sets the process-global
//! and re-runs the full `heal_env` init (which sets the process-global
//! bucket-metadata-sys OnceCell) — under `cargo nextest` each test runs
//! in its own process so the OnceCell never collides.
@@ -34,20 +34,15 @@ use rustfs_heal::heal::{
use serial_test::serial;
use std::{
path::{Path, PathBuf},
sync::{Arc, Once},
sync::Arc,
time::Duration,
};
use tokio::fs;
use tokio_util::sync::CancellationToken;
use tracing::info;
use walkdir::WalkDir;
mod storage_api;
use storage_api::integration::{
BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _,
ObjectOperations as _, PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
};
use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _, ObjectOperations as _};
/// 256 KiB + change: large enough to be stored as non-inline erasure shards
/// (so each data version materializes as an on-disk `part.*` file we can assert
@@ -62,78 +57,16 @@ fn versioned_test_data(seed: u8) -> Vec<u8> {
.collect()
}
static INIT: Once = Once::new();
fn init_tracing() {
INIT.call_once(|| {
let _ = tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339())
.with_thread_names(true)
.try_init();
});
}
/// Build a real 4-disk `ECStore` + `ECStoreHealStorage`. Mirrors
/// `heal_integration_test::setup_test_env`.
async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
init_tracing();
let test_base_dir = format!("/tmp/rustfs_heal_b5_test_{}", uuid::Uuid::new_v4());
let temp_dir = PathBuf::from(&test_base_dir);
if temp_dir.exists() {
fs::remove_dir_all(&temp_dir).await.ok();
}
fs::create_dir_all(&temp_dir).await.unwrap();
let disk_paths = vec![
temp_dir.join("disk1"),
temp_dir.join("disk2"),
temp_dir.join("disk3"),
temp_dir.join("disk4"),
];
for disk_path in &disk_paths {
fs::create_dir_all(disk_path).await.unwrap();
}
let mut endpoints = Vec::new();
for (i, disk_path) in disk_paths.iter().enumerate() {
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(i);
endpoints.push(endpoint);
}
let pool_endpoints = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: "test".to_string(),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
};
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
init_local_disks(endpoint_pools.clone()).await.unwrap();
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
.await
.unwrap();
let buckets_list = ecstore
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
(disk_paths, ecstore, heal_storage)
/// Build a real 4-disk `ECStore` + `ECStoreHealStorage` via the shared
/// rustfs-test-utils environment (backlog#1153 infra-1). Mirrors
/// `heal_integration_test::heal_env`.
async fn heal_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
let env = rustfs_test_utils::TestECStoreEnv::builder()
.prefix("rustfs_heal_b5_test")
.build()
.await;
let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
(env.disk_paths, env.ecstore, heal_storage)
}
/// Create a bucket with S3 versioning ENABLED at creation time. Without this the
@@ -308,7 +241,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn test_enumeration_includes_all_versions_and_delete_marker_real_fixture() {
let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;
let (_disk_paths, ecstore, heal_storage) = heal_env().await;
let bucket = "b5-enum-versions";
let object = "obj.bin";
create_versioned_bucket(&ecstore, bucket).await;
@@ -345,7 +278,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn test_heal_old_nonlatest_version_after_disk_wipe() {
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
let (disk_paths, ecstore, heal_storage) = heal_env().await;
let bucket = "b5-old-version-heal";
let object = "obj.bin";
create_versioned_bucket(&ecstore, bucket).await;
@@ -397,7 +330,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn test_heal_delete_marker_latest_enumerated_and_healed() {
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
let (disk_paths, ecstore, heal_storage) = heal_env().await;
let bucket = "b5-dm-latest-heal";
let object = "obj.bin";
create_versioned_bucket(&ecstore, bucket).await;
@@ -463,7 +396,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn test_version_id_normalization_null_and_unversioned_real_fixture() {
let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;
let (_disk_paths, ecstore, heal_storage) = heal_env().await;
let bucket = "b5-unversioned-normalize";
create_unversioned_bucket(&ecstore, bucket).await;
@@ -499,7 +432,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn test_heal_unversioned_bucket_e2e() {
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
let (disk_paths, ecstore, heal_storage) = heal_env().await;
let bucket = "b5-unversioned-e2e";
create_unversioned_bucket(&ecstore, bucket).await;
@@ -584,7 +517,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn test_heal_resume_across_page_boundary_e2e() {
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
let (disk_paths, ecstore, heal_storage) = heal_env().await;
let bucket = "b5-resume-e2e";
create_versioned_bucket(&ecstore, bucket).await;
crates/heal/tests/heal_b920_subquorum_union_test.rs
+19 -78
View File
@@ -29,18 +29,13 @@ use rustfs_heal::heal::storage::{
use serial_test::serial;
use std::{
path::{Path, PathBuf},
sync::{Arc, Once},
sync::Arc,
};
use tokio::fs;
use tokio_util::sync::CancellationToken;
use walkdir::WalkDir;
mod storage_api;
use storage_api::integration::{
BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _,
PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
};
use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _};
/// 256 KiB + change: large enough to be stored as non-inline erasure shards, so
/// deleting the `xl.meta` file does NOT delete the data (the `part.*` shards live
@@ -57,16 +52,6 @@ fn versioned_test_data(seed: u8) -> Vec<u8> {
.collect()
}
static INIT: Once = Once::new();
fn init_tracing() {
INIT.call_once(|| {
let _ = tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.try_init();
});
}
/// Disable the dangling-delete grace window so the destructive path is genuinely
/// LIVE in these tests: without the decision-1 guard, a recoverable version WOULD
/// be dangling-deleted here. With grace at its 1h default the delete path would be
@@ -78,60 +63,16 @@ fn disable_dangling_grace() {
}
}
/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage`.
async fn setup_test_env_n(n_disks: usize) -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
init_tracing();
let test_base_dir = format!("/tmp/rustfs_heal_b920_test_{}", uuid::Uuid::new_v4());
let temp_dir = PathBuf::from(&test_base_dir);
if temp_dir.exists() {
fs::remove_dir_all(&temp_dir).await.ok();
}
fs::create_dir_all(&temp_dir).await.unwrap();
let disk_paths: Vec<PathBuf> = (0..n_disks).map(|i| temp_dir.join(format!("disk{}", i + 1))).collect();
for disk_path in &disk_paths {
fs::create_dir_all(disk_path).await.unwrap();
}
let mut endpoints = Vec::new();
for (i, disk_path) in disk_paths.iter().enumerate() {
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(i);
endpoints.push(endpoint);
}
let pool_endpoints = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: n_disks,
endpoints: Endpoints::from(endpoints),
cmd_line: "test".to_string(),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
};
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
init_local_disks(endpoint_pools.clone()).await.unwrap();
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
.await
.unwrap();
let buckets_list = ecstore
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
(disk_paths, ecstore, heal_storage)
/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage` via the
/// shared rustfs-test-utils environment (backlog#1153 infra-1).
async fn heal_env_n(n_disks: usize) -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
let env = rustfs_test_utils::TestECStoreEnv::builder()
.prefix("rustfs_heal_b920_test")
.disk_count(n_disks)
.build()
.await;
let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
(env.disk_paths, env.ecstore, heal_storage)
}
async fn create_versioned_bucket(ecstore: &Arc<ECStore>, bucket: &str) {
@@ -273,7 +214,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn disk_walk_page_enumerates_subquorum_version_omitted_by_list_object_versions() {
let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
let bucket = "b920-enum-gap";
let object = "obj.bin";
create_versioned_bucket(&ecstore, bucket).await;
@@ -310,7 +251,7 @@ mod serial_tests {
#[serial]
async fn union_meta_lost_data_present_is_repaired_not_destroyed() {
disable_dangling_grace();
let (disk_paths, ecstore, heal_storage) = setup_test_env_n(8).await;
let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
let bucket = "b920-meta-lost";
let object = "obj.bin";
create_versioned_bucket(&ecstore, bucket).await;
@@ -360,7 +301,7 @@ mod serial_tests {
#[serial]
async fn deep_heal_torn_minority_is_dangling_deleted_with_grace_zero() {
disable_dangling_grace();
let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
let bucket = "b920-torn";
let object = "obj.bin";
create_versioned_bucket(&ecstore, bucket).await;
@@ -409,7 +350,7 @@ mod serial_tests {
#[serial]
async fn deep_heal_restores_subquorum_but_reconstructable_version_wider_set() {
disable_dangling_grace();
let (disk_paths, ecstore, heal_storage) = setup_test_env_n(8).await;
let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
let bucket = "b920-reconstruct";
let object = "obj.bin";
create_versioned_bucket(&ecstore, bucket).await;
@@ -463,7 +404,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn disk_walk_multipage_resume_heals_each_version_once() {
let (_disk_paths, ecstore, _heal_storage) = setup_test_env_n(4).await;
let (_disk_paths, ecstore, _heal_storage) = heal_env_n(4).await;
let bucket = "b920-multipage";
create_versioned_bucket(&ecstore, bucket).await;
@@ -523,7 +464,7 @@ mod serial_tests {
#[serial]
async fn offline_disk_during_walk_does_not_dangling_delete() {
disable_dangling_grace();
let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
let bucket = "b920-offline";
let object = "obj.bin";
create_versioned_bucket(&ecstore, bucket).await;
@@ -570,7 +511,7 @@ mod serial_tests {
#[serial]
async fn deep_heal_keeps_present_ec2_plus_2_shards_healthy() {
disable_dangling_grace();
let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
let bucket = "b1044-deep-verify";
let object = "obj.bin";
create_versioned_bucket(&ecstore, bucket).await;
crates/heal/tests/heal_integration_test.rs
+18 -95
View File
@@ -22,20 +22,16 @@ use rustfs_heal::heal::{
use serial_test::serial;
use std::{
path::{Path, PathBuf},
sync::{Arc, Once},
sync::Arc,
time::Duration,
};
use tokio::fs;
use tokio_util::sync::CancellationToken;
use tracing::info;
use walkdir::WalkDir;
mod storage_api;
use storage_api::integration::{
BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, ObjectIO as _, ObjectOperations as _,
PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
};
use storage_api::integration::{BucketOperations, ECStore, ObjectIO as _, ObjectOperations as _};
const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25);
const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250);
@@ -58,89 +54,16 @@ async fn wait_for_path_exists(path: &Path, timeout: Duration, interval: Duration
}
}
static INIT: Once = Once::new();
pub fn init_tracing() {
INIT.call_once(|| {
let _ = tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339())
.with_thread_names(true)
.try_init();
});
}
/// Test helper: Create test environment with ECStore
async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
init_tracing();
// create temp dir as 4 disks with unique base dir
let test_base_dir = format!("/tmp/rustfs_heal_heal_test_{}", uuid::Uuid::new_v4());
let temp_dir = std::path::PathBuf::from(&test_base_dir);
if temp_dir.exists() {
fs::remove_dir_all(&temp_dir).await.ok();
}
fs::create_dir_all(&temp_dir).await.unwrap();
// create 4 disk dirs
let disk_paths = vec![
temp_dir.join("disk1"),
temp_dir.join("disk2"),
temp_dir.join("disk3"),
temp_dir.join("disk4"),
];
for disk_path in &disk_paths {
fs::create_dir_all(disk_path).await.unwrap();
}
// create EndpointServerPools
let mut endpoints = Vec::new();
for (i, disk_path) in disk_paths.iter().enumerate() {
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
// set correct index
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(i);
endpoints.push(endpoint);
}
let pool_endpoints = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: "test".to_string(),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
};
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
// format disks (only first time)
init_local_disks(endpoint_pools.clone()).await.unwrap();
// Use port 0 so nextest can run this integration binary in parallel
// with other ECStore-backed tests without sharing a fixed peer port.
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
.await
.unwrap();
// init bucket metadata system
let buckets_list = ecstore
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
// Create heal storage layer
let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
(disk_paths, ecstore, heal_storage)
/// Test helper: build the shared 4-disk temp-dir ECStore environment
/// (rustfs-test-utils, backlog#1153 infra-1) and wrap it in the heal storage
/// layer. Port 0 + uuid temp dirs keep this parallel-safe under nextest.
async fn heal_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
let env = rustfs_test_utils::TestECStoreEnv::builder()
.prefix("rustfs_heal_heal_test")
.build()
.await;
let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
(env.disk_paths, env.ecstore, heal_storage)
}
/// Test helper: Create a test bucket
@@ -169,7 +92,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn test_heal_object_basic() {
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
let (disk_paths, ecstore, heal_storage) = heal_env().await;
// Create test bucket and object
let bucket_name = "test-heal-object-basic";
@@ -238,7 +161,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn test_heal_object_reclaims_orphan_data_dir_when_healthy() {
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
let (disk_paths, ecstore, heal_storage) = heal_env().await;
let bucket_name = "test-heal-reclaim-orphan";
let object_name = "healthy-object.bin";
@@ -350,7 +273,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn test_heal_bucket_basic() {
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
let (disk_paths, ecstore, heal_storage) = heal_env().await;
// Create test bucket
let bucket_name = "test-heal-bucket-basic";
@@ -420,7 +343,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn test_heal_format_basic() {
let (disk_paths, _ecstore, heal_storage) = setup_test_env().await;
let (disk_paths, _ecstore, heal_storage) = heal_env().await;
// ─── 1️⃣ delete format.json on one disk ──────────────
let format_path = disk_paths[0].join(".rustfs.sys").join("format.json");
@@ -446,7 +369,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn test_heal_format_with_data() {
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
let (disk_paths, ecstore, heal_storage) = heal_env().await;
// Create test bucket and object
let bucket_name = "test-heal-format-with-data";
@@ -535,7 +458,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn test_heal_storage_api_direct() {
let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;
let (_disk_paths, ecstore, heal_storage) = heal_env().await;
// Test direct heal storage API calls
crates/heal/tests/storage_api.rs
+3 -7
View File
@@ -15,17 +15,13 @@ pub(crate) mod endpoint_index {
pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
}
// The temp-disk ECStore bootstrap itself lives in rustfs-test-utils
// (backlog#1153 infra-1); this surface keeps only what the heal test bodies
// still touch directly.
#[allow(unused_imports)]
pub(crate) mod integration {
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys;
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
pub(crate) use rustfs_ecstore::api::layout::EndpointServerPools;
pub(crate) use rustfs_ecstore::api::layout::Endpoints;
pub(crate) use rustfs_ecstore::api::layout::PoolEndpoints;
pub(crate) use rustfs_ecstore::api::storage::ECStore;
pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
pub(crate) use rustfs_storage_api::BucketOperations;
pub(crate) use rustfs_storage_api::BucketOptions;
pub(crate) use rustfs_storage_api::MakeBucketOptions;
pub(crate) use rustfs_storage_api::ObjectIO;
pub(crate) use rustfs_storage_api::ObjectOperations;
crates/iam/Cargo.toml
+1
View File
@@ -59,6 +59,7 @@ url = { workspace = true }
[dev-dependencies]
pollster.workspace = true
rustfs-test-utils = { workspace = true }
serial_test = { workspace = true }
temp-env = { workspace = true, features = ["async_closure"] }
tempfile = { workspace = true }
crates/iam/tests/ecstore_test_compat/mod.rs
+1 -3
View File
@@ -22,9 +22,7 @@
#[allow(unused_imports)]
pub(crate) mod fixture {
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints, SetupType};
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
pub(crate) use rustfs_ecstore::api::layout::SetupType;
// `update_erasure_type` is a write-side global facade entry. Its use is
// restricted to reviewed storage_api boundaries; this test-only module is
crates/iam/tests/iam_bootstrap_no_lock_test.rs
+10 -41
View File
@@ -31,55 +31,16 @@
mod ecstore_test_compat;
use ecstore_test_compat::fixture::{
ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, SetupType, init_local_disks, update_erasure_type,
};
use ecstore_test_compat::fixture::{SetupType, update_erasure_type};
use rustfs_iam::cache::Cache;
use rustfs_iam::store::object::ObjectStore;
use rustfs_iam::store::{GroupInfo, Store};
use serial_test::serial;
use std::collections::HashMap;
use std::sync::Arc;
use tokio_util::sync::CancellationToken;
const TEST_GROUP: &str = "seq-restart-group";
const TEST_MEMBERS: [&str; 2] = ["alice", "bob"];
async fn build_local_ecstore(temp_dir: &std::path::Path) -> Arc<ECStore> {
let disk_paths: Vec<_> = (1..=4).map(|i| temp_dir.join(format!("disk{i}"))).collect();
for disk_path in &disk_paths {
tokio::fs::create_dir_all(disk_path).await.unwrap();
}
let mut endpoints = Vec::new();
for (i, disk_path) in disk_paths.iter().enumerate() {
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(i);
endpoints.push(endpoint);
}
let pool_endpoints = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: "test".to_string(),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
};
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
init_local_disks(endpoint_pools.clone()).await.unwrap();
// Port 0 keeps this integration binary parallel-safe alongside other
// ECStore-backed tests.
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
.await
.unwrap()
}
/// Restores single-node erasure mode even when an assertion panics, so a
/// failing run cannot poison later `#[serial]` tests in this process.
struct ErasureModeGuard;
@@ -97,7 +58,15 @@ async fn load_all_bypasses_namespace_lock_quorum() {
// must be shortened before the first locked operation of this process.
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
let temp_dir = tempfile::TempDir::with_prefix("rustfs_iam_no_lock_test_").unwrap();
let ecstore = build_local_ecstore(temp_dir.path()).await;
// Shared temp-disk ECStore env (rustfs-test-utils, backlog#1153 infra-1).
// base_dir keeps cleanup ownership with this TempDir; the historical
// bootstrap never initialized the bucket-metadata system, so opt out.
let ecstore = rustfs_test_utils::TestECStoreEnv::builder()
.base_dir(temp_dir.path())
.init_bucket_metadata(false)
.build()
.await
.ecstore;
let store = ObjectStore::new(ecstore);
// Seed IAM data while namespace locks still work (single-node mode).
crates/test-utils/Cargo.toml
+40
View File
@@ -0,0 +1,40 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package]
name = "rustfs-test-utils"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
homepage.workspace = true
description = "Shared test bootstrap helpers for RustFS integration tests (dev-dependency only)."
keywords = ["testing", "storage", "rustfs", "Minio"]
categories = ["development-tools", "filesystem"]
documentation = "https://docs.rs/rustfs-test-utils/latest/rustfs_test_utils/"
[dependencies]
rustfs-ecstore = { workspace = true }
rustfs-storage-api = { workspace = true }
tokio = { workspace = true, features = ["fs", "rt"] }
tokio-util = { workspace = true }
tracing-subscriber = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
[lints]
workspace = true
[lib]
doctest = false
crates/test-utils/README.md
+56
View File
@@ -0,0 +1,56 @@
# rustfs-test-utils
Shared bootstrap helpers for RustFS integration tests (backlog#1153 infra-1).
Owns the "build a real temp-disk `ECStore`" setup that used to be copy-pasted
across the heal/iam/scanner integration tests and had already drifted.
## Usage
```rust
// [dev-dependencies] rustfs-test-utils = { workspace = true }
use rustfs_test_utils::TestECStoreEnv;
let env = TestECStoreEnv::builder()
.prefix("rustfs_myfeature_test") // /tmp/<prefix>_<uuid>
.disk_count(4) // single pool, single set
.build()
.await;
env.make_bucket("my-bucket", /* versioned = */ true).await;
// env.ecstore : Arc<ECStore> — bootstrapped store
// env.disk_paths : Vec<PathBuf> — disk1..diskN for fault injection
// env.temp_root : PathBuf — root dir (leaked by design, see below)
```
Builder knobs:
- `disk_count(n)` — disks in the single erasure set (default 4).
- `prefix(&str)` — temp-dir prefix; a uuid suffix keeps parallel nextest
processes isolated.
- `base_dir(path)` — use a caller-owned directory (e.g. `tempfile::TempDir`)
instead; the caller keeps cleanup ownership.
- `init_bucket_metadata(bool)` — run `init_bucket_metadata_sys` after boot
(default `true`; the IAM bootstrap test opts out).
`init_tracing()` is exposed separately and is called by `build()`.
The environment does **not** delete `temp_root` on drop: a failed test's
on-disk state stays inspectable, and heal tests keep manipulating
`disk_paths` after setup. Own the directory via `base_dir` if you want
automatic cleanup.
## Constraints
- **dev-dependency only.** No crate may list `rustfs-test-utils` under
`[dependencies]` — test scaffolding must never reach production binaries.
- **Single-process integration scope.** Multi-node / chaos harnesses are out
of scope (backlog#1100). Scanner's lifecycle tests are absorbed only after
ilm-1 activates them (they are `#[ignore]`d today).
- All `rustfs_ecstore` imports stay behind `src/ecstore_test_compat.rs` — the
sanctioned test-compat boundary pattern enforced by
`scripts/check_architecture_migration_rules.sh`. Extend that module, never
import the facade directly from other files.
- Tests using this env bind `127.0.0.1:0` (random port) and unique temp dirs,
so they stay parallel-safe under nextest's process-per-test model — do not
add fixed ports or shared paths here. See `docs/testing/README.md` for the
serial/nextest rules.
crates/test-utils/src/ecstore_test_compat.rs
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Test-only ECStore compatibility boundary for `rustfs-test-utils`.
//!
//! All direct `rustfs_ecstore` facade imports used by this crate must go
//! through this module (architecture migration rule:
//! `check_architecture_migration_rules.sh`; this file name is the sanctioned
//! test-compat boundary pattern, mirroring
//! `crates/iam/tests/ecstore_test_compat/mod.rs`). Keep the surface minimal —
//! only what the environment builder needs to assemble a temp-disk ECStore.
#[allow(unused_imports)]
pub(crate) mod fixture {
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys;
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
pub(crate) use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions};
}
crates/test-utils/src/lib.rs
+210
View File
@@ -0,0 +1,210 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Shared test bootstrap helpers for RustFS integration tests
//! (backlog#1153 infra-1).
//!
//! This crate is a **dev-dependency only**: it must never appear in any
//! crate's `[dependencies]`. It owns the ~50-line "build a real temp-disk
//! `ECStore`" bootstrap that used to be copy-pasted (and drift) across the
//! heal/iam/scanner integration tests.
//!
//! Single-process integration scope only — multi-node / chaos harnesses are
//! out of scope (backlog#1100).
mod ecstore_test_compat;
use std::path::PathBuf;
use std::sync::{Arc, Once};
use ecstore_test_compat::fixture::{
BucketOperations as _, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, PoolEndpoints,
init_bucket_metadata_sys, init_local_disks,
};
use tokio_util::sync::CancellationToken;
static INIT_TRACING: Once = Once::new();
/// Install the standard test tracing subscriber once per process
/// (`RUST_LOG`-driven). Safe to call from every test; later calls are no-ops.
pub fn init_tracing() {
INIT_TRACING.call_once(|| {
let _ = tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339())
.with_thread_names(true)
.try_init();
});
}
/// A real single-pool, single-set `ECStore` built over per-test temp-dir
/// "disks". Build one with [`TestECStoreEnv::builder`].
///
/// The environment intentionally does **not** delete `temp_root` on drop:
/// the historical bootstraps leaked their uuid-suffixed temp dirs so a failed
/// test's on-disk state stays inspectable, and several heal tests keep
/// manipulating `disk_paths` after setup. Callers that own the directory
/// lifetime (e.g. via `tempfile::TempDir`) should pass it through
/// [`TestECStoreEnvBuilder::base_dir`].
pub struct TestECStoreEnv {
/// Root directory holding the disk directories.
pub temp_root: PathBuf,
/// The per-disk directories (`disk1`..`diskN`) under `temp_root`.
pub disk_paths: Vec<PathBuf>,
/// The store, bootstrapped exactly like the historical test setups:
/// `init_local_disks` + `ECStore::new` on `127.0.0.1:0` (random port keeps
/// nextest's process-per-test parallelism safe).
pub ecstore: Arc<ECStore>,
}
impl TestECStoreEnv {
pub fn builder() -> TestECStoreEnvBuilder {
TestECStoreEnvBuilder::default()
}
/// Create a bucket, optionally with S3 versioning enabled at creation
/// time (without this a second PUT overwrites in place and DELETE removes
/// the object outright — no old versions or delete-marker-latest exist).
pub async fn make_bucket(&self, bucket: &str, versioned: bool) {
self.ecstore
.make_bucket(
bucket,
&MakeBucketOptions {
versioning_enabled: versioned,
..Default::default()
},
)
.await
.unwrap_or_else(|e| panic!("failed to create test bucket {bucket}: {e:?}"));
}
}
/// Builder for [`TestECStoreEnv`]. Defaults reproduce the historical heal
/// bootstrap: 4 disks, one pool, one set, bucket-metadata system initialized.
pub struct TestECStoreEnvBuilder {
disk_count: usize,
prefix: String,
base_dir: Option<PathBuf>,
init_bucket_metadata: bool,
}
impl Default for TestECStoreEnvBuilder {
fn default() -> Self {
Self {
disk_count: 4,
prefix: "rustfs_test_utils".to_string(),
base_dir: None,
init_bucket_metadata: true,
}
}
}
impl TestECStoreEnvBuilder {
/// Number of disk directories in the single erasure set (default 4).
pub fn disk_count(mut self, n: usize) -> Self {
self.disk_count = n;
self
}
/// Temp-dir name prefix, e.g. `rustfs_heal_b5_test` (a uuid suffix is
/// always appended). Ignored when [`base_dir`](Self::base_dir) is set.
pub fn prefix(mut self, prefix: &str) -> Self {
self.prefix = prefix.to_string();
self
}
/// Use a caller-owned directory (e.g. a `tempfile::TempDir` path) instead
/// of creating `/tmp/<prefix>_<uuid>`. The caller keeps cleanup ownership.
pub fn base_dir(mut self, dir: impl Into<PathBuf>) -> Self {
self.base_dir = Some(dir.into());
self
}
/// Whether to run `init_bucket_metadata_sys` after the store comes up
/// (default `true`, as the heal bootstraps did). The IAM bootstrap test
/// opts out to preserve its historical semantics.
pub fn init_bucket_metadata(mut self, yes: bool) -> Self {
self.init_bucket_metadata = yes;
self
}
/// Build the environment. Panics on any bootstrap failure — this is test
/// scaffolding, and a broken environment must fail the test loudly.
pub async fn build(self) -> TestECStoreEnv {
init_tracing();
let temp_root = match &self.base_dir {
Some(dir) => dir.clone(),
None => {
let root = PathBuf::from(format!("/tmp/{}_{}", self.prefix, uuid::Uuid::new_v4()));
if root.exists() {
tokio::fs::remove_dir_all(&root).await.ok();
}
root
}
};
tokio::fs::create_dir_all(&temp_root).await.expect("create test temp root");
let disk_paths: Vec<PathBuf> = (1..=self.disk_count).map(|i| temp_root.join(format!("disk{i}"))).collect();
for disk_path in &disk_paths {
tokio::fs::create_dir_all(disk_path).await.expect("create test disk dir");
}
let mut endpoints = Vec::new();
for (i, disk_path) in disk_paths.iter().enumerate() {
let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("utf-8 disk path")).expect("parse disk endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(i);
endpoints.push(endpoint);
}
let pool_endpoints = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: self.disk_count,
endpoints: Endpoints::from(endpoints),
cmd_line: "test".to_string(),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
};
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
// Port 0 keeps ECStore-backed integration binaries parallel-safe under
// nextest: no fixed peer port is ever shared between test processes.
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().expect("parse test addr");
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
.await
.expect("build test ECStore");
if self.init_bucket_metadata {
let buckets_list = ecstore
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("list buckets for metadata init");
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
}
TestECStoreEnv {
temp_root,
disk_paths,
ecstore,
}
}
}