From f05a69d51b3dceb7c0d1909d22ef53f3cb56e233 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 15 Jul 2026 16:08:30 +0800 Subject: [PATCH] test(utils): add rustfs-test-utils crate and shared ECStore bootstrap (#4850) * test(utils): add rustfs-test-utils crate, absorb heal/iam ECStore bootstrap backlog#1153 infra-1. The ~50-line "build a real temp-disk ECStore" bootstrap was copy-pasted (and drifting) across the heal and iam integration tests. This adds crates/test-utils (rustfs-test-utils, a dev-dependency-only crate) owning that bootstrap and converts the four copies into thin wrappers: - TestECStoreEnvBuilder: disk_count (default 4), prefix (uuid-suffixed /tmp dir), base_dir (caller-owned dir, e.g. tempfile::TempDir), init_bucket_metadata (default true; the iam bootstrap test opts out to preserve its historical semantics). TestECStoreEnv exposes temp_root/disk_paths/ecstore plus a versioned-bucket helper, and init_tracing() replaces the per-file Once blocks. - All rustfs_ecstore imports stay behind src/ecstore_test_compat.rs, the sanctioned test-compat boundary pattern (mirrors crates/iam/tests/ecstore_test_compat). - heal: heal_integration_test / heal_b5_versioned_regression_test / heal_b920_subquorum_union_test drop their setup_test_env{,_n} copies for heal_env{,_n} wrappers; the tests/storage_api.rs integration surface shrinks to what test bodies still touch. - iam: iam_bootstrap_no_lock_test drops build_local_ecstore; its ecstore_test_compat fixture shrinks to SetupType + update_erasure_type. rg 'async fn setup_test_env' crates/heal crates/iam now returns 0. Scanner's lifecycle tests are deliberately NOT absorbed (gated on ilm-1; 14 of 15 are #[ignore]d today). Net -230 lines. * fix(heal): drop tokio::fs import orphaned by the b920 bootstrap move * fix(heal): drop tokio::fs import orphaned by the b5 bootstrap move --- Cargo.lock | 14 ++ Cargo.toml | 2 + crates/heal/Cargo.toml | 1 + .../heal_b5_versioned_regression_test.rs | 105 ++------- .../tests/heal_b920_subquorum_union_test.rs | 97 ++------ crates/heal/tests/heal_integration_test.rs | 113 ++-------- crates/heal/tests/storage_api.rs | 10 +- crates/iam/Cargo.toml | 1 + crates/iam/tests/ecstore_test_compat/mod.rs | 4 +- .../iam/tests/iam_bootstrap_no_lock_test.rs | 51 +---- crates/test-utils/Cargo.toml | 40 ++++ crates/test-utils/README.md | 56 +++++ crates/test-utils/src/ecstore_test_compat.rs | 31 +++ crates/test-utils/src/lib.rs | 210 ++++++++++++++++++ 14 files changed, 425 insertions(+), 310 deletions(-) create mode 100644 crates/test-utils/Cargo.toml create mode 100644 crates/test-utils/README.md create mode 100644 crates/test-utils/src/ecstore_test_compat.rs create mode 100644 crates/test-utils/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index e23b2c31b..643b4acad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9276,6 +9276,7 @@ dependencies = [ "rustfs-ecstore", "rustfs-madmin", "rustfs-storage-api", + "rustfs-test-utils", "rustfs-utils", "serde", "serde_json", @@ -9313,6 +9314,7 @@ dependencies = [ "rustfs-madmin", "rustfs-policy", "rustfs-storage-api", + "rustfs-test-utils", "rustfs-utils", "serde", "serde_json", @@ -9991,6 +9993,18 @@ dependencies = [ "uuid", ] +[[package]] +name = "rustfs-test-utils" +version = "1.0.0-beta.9" +dependencies = [ + "rustfs-ecstore", + "rustfs-storage-api", + "tokio", + "tokio-util", + "tracing-subscriber", + "uuid", +] + [[package]] name = "rustfs-tls-runtime" version = "1.0.0-beta.9" diff --git a/Cargo.toml b/Cargo.toml index 762967b65..dc74e2f21 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ members = [ "crates/extension-schema", # Extension schema contracts "crates/signer", # client signer "crates/storage-api", # Storage API contracts + "crates/test-utils", # Shared test bootstrap helpers (dev-dependency only) "crates/targets", # Target-specific configurations and utilities "crates/trusted-proxies", # Trusted proxies management "crates/tls-runtime", # Project-wide TLS runtime foundation @@ -125,6 +126,7 @@ rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.9" } rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.9" } rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.9" } rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.9" } +rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.9" } rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.9" } rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.9" } rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.9" } diff --git a/crates/heal/Cargo.toml b/crates/heal/Cargo.toml index b60dfa118..1089a50ab 100644 --- a/crates/heal/Cargo.toml +++ b/crates/heal/Cargo.toml @@ -48,6 +48,7 @@ base64 = { workspace = true } [dev-dependencies] serde_json = { workspace = true } +rustfs-test-utils = { workspace = true } serial_test = { workspace = true } tracing-subscriber = { workspace = true } tempfile = { workspace = true } diff --git a/crates/heal/tests/heal_b5_versioned_regression_test.rs b/crates/heal/tests/heal_b5_versioned_regression_test.rs index 6d53a513b..b17d1315b 100644 --- a/crates/heal/tests/heal_b5_versioned_regression_test.rs +++ b/crates/heal/tests/heal_b5_versioned_regression_test.rs @@ -18,7 +18,7 @@ //! //! These drive the REAL `ECStoreHealStorage` (not a mock) against a real 4-disk //! `ECStore`, mirroring `heal_integration_test.rs`. Every test is `#[serial]` -//! and re-runs the full `setup_test_env` init (which sets the process-global +//! and re-runs the full `heal_env` init (which sets the process-global //! bucket-metadata-sys OnceCell) — under `cargo nextest` each test runs //! in its own process so the OnceCell never collides. @@ -34,20 +34,15 @@ use rustfs_heal::heal::{ use serial_test::serial; use std::{ path::{Path, PathBuf}, - sync::{Arc, Once}, + sync::Arc, time::Duration, }; -use tokio::fs; -use tokio_util::sync::CancellationToken; use tracing::info; use walkdir::WalkDir; mod storage_api; -use storage_api::integration::{ - BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _, - ObjectOperations as _, PoolEndpoints, init_bucket_metadata_sys, init_local_disks, -}; +use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _, ObjectOperations as _}; /// 256 KiB + change: large enough to be stored as non-inline erasure shards /// (so each data version materializes as an on-disk `part.*` file we can assert @@ -62,78 +57,16 @@ fn versioned_test_data(seed: u8) -> Vec { .collect() } -static INIT: Once = Once::new(); - -fn init_tracing() { - INIT.call_once(|| { - let _ = tracing_subscriber::fmt() - .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) - .with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339()) - .with_thread_names(true) - .try_init(); - }); -} - -/// Build a real 4-disk `ECStore` + `ECStoreHealStorage`. Mirrors -/// `heal_integration_test::setup_test_env`. -async fn setup_test_env() -> (Vec, Arc, Arc) { - init_tracing(); - - let test_base_dir = format!("/tmp/rustfs_heal_b5_test_{}", uuid::Uuid::new_v4()); - let temp_dir = PathBuf::from(&test_base_dir); - if temp_dir.exists() { - fs::remove_dir_all(&temp_dir).await.ok(); - } - fs::create_dir_all(&temp_dir).await.unwrap(); - - let disk_paths = vec![ - temp_dir.join("disk1"), - temp_dir.join("disk2"), - temp_dir.join("disk3"), - temp_dir.join("disk4"), - ]; - for disk_path in &disk_paths { - fs::create_dir_all(disk_path).await.unwrap(); - } - - let mut endpoints = Vec::new(); - for (i, disk_path) in disk_paths.iter().enumerate() { - let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap(); - endpoint.set_pool_index(0); - endpoint.set_set_index(0); - endpoint.set_disk_index(i); - endpoints.push(endpoint); - } - - let pool_endpoints = PoolEndpoints { - legacy: false, - set_count: 1, - drives_per_set: 4, - endpoints: Endpoints::from(endpoints), - cmd_line: "test".to_string(), - platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), - }; - let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]); - - init_local_disks(endpoint_pools.clone()).await.unwrap(); - - let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); - let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) - .await - .unwrap(); - - let buckets_list = ecstore - .list_bucket(&BucketOptions { - no_metadata: true, - ..Default::default() - }) - .await - .unwrap(); - let buckets = buckets_list.into_iter().map(|v| v.name).collect(); - init_bucket_metadata_sys(ecstore.clone(), buckets).await; - - let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone())); - (disk_paths, ecstore, heal_storage) +/// Build a real 4-disk `ECStore` + `ECStoreHealStorage` via the shared +/// rustfs-test-utils environment (backlog#1153 infra-1). Mirrors +/// `heal_integration_test::heal_env`. +async fn heal_env() -> (Vec, Arc, Arc) { + let env = rustfs_test_utils::TestECStoreEnv::builder() + .prefix("rustfs_heal_b5_test") + .build() + .await; + let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone())); + (env.disk_paths, env.ecstore, heal_storage) } /// Create a bucket with S3 versioning ENABLED at creation time. Without this the @@ -308,7 +241,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn test_enumeration_includes_all_versions_and_delete_marker_real_fixture() { - let (_disk_paths, ecstore, heal_storage) = setup_test_env().await; + let (_disk_paths, ecstore, heal_storage) = heal_env().await; let bucket = "b5-enum-versions"; let object = "obj.bin"; create_versioned_bucket(&ecstore, bucket).await; @@ -345,7 +278,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn test_heal_old_nonlatest_version_after_disk_wipe() { - let (disk_paths, ecstore, heal_storage) = setup_test_env().await; + let (disk_paths, ecstore, heal_storage) = heal_env().await; let bucket = "b5-old-version-heal"; let object = "obj.bin"; create_versioned_bucket(&ecstore, bucket).await; @@ -397,7 +330,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn test_heal_delete_marker_latest_enumerated_and_healed() { - let (disk_paths, ecstore, heal_storage) = setup_test_env().await; + let (disk_paths, ecstore, heal_storage) = heal_env().await; let bucket = "b5-dm-latest-heal"; let object = "obj.bin"; create_versioned_bucket(&ecstore, bucket).await; @@ -463,7 +396,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn test_version_id_normalization_null_and_unversioned_real_fixture() { - let (_disk_paths, ecstore, heal_storage) = setup_test_env().await; + let (_disk_paths, ecstore, heal_storage) = heal_env().await; let bucket = "b5-unversioned-normalize"; create_unversioned_bucket(&ecstore, bucket).await; @@ -499,7 +432,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn test_heal_unversioned_bucket_e2e() { - let (disk_paths, ecstore, heal_storage) = setup_test_env().await; + let (disk_paths, ecstore, heal_storage) = heal_env().await; let bucket = "b5-unversioned-e2e"; create_unversioned_bucket(&ecstore, bucket).await; @@ -584,7 +517,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn test_heal_resume_across_page_boundary_e2e() { - let (disk_paths, ecstore, heal_storage) = setup_test_env().await; + let (disk_paths, ecstore, heal_storage) = heal_env().await; let bucket = "b5-resume-e2e"; create_versioned_bucket(&ecstore, bucket).await; diff --git a/crates/heal/tests/heal_b920_subquorum_union_test.rs b/crates/heal/tests/heal_b920_subquorum_union_test.rs index e59eb9a78..9f6a4b863 100644 --- a/crates/heal/tests/heal_b920_subquorum_union_test.rs +++ b/crates/heal/tests/heal_b920_subquorum_union_test.rs @@ -29,18 +29,13 @@ use rustfs_heal::heal::storage::{ use serial_test::serial; use std::{ path::{Path, PathBuf}, - sync::{Arc, Once}, + sync::Arc, }; -use tokio::fs; -use tokio_util::sync::CancellationToken; use walkdir::WalkDir; mod storage_api; -use storage_api::integration::{ - BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _, - PoolEndpoints, init_bucket_metadata_sys, init_local_disks, -}; +use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _}; /// 256 KiB + change: large enough to be stored as non-inline erasure shards, so /// deleting the `xl.meta` file does NOT delete the data (the `part.*` shards live @@ -57,16 +52,6 @@ fn versioned_test_data(seed: u8) -> Vec { .collect() } -static INIT: Once = Once::new(); - -fn init_tracing() { - INIT.call_once(|| { - let _ = tracing_subscriber::fmt() - .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) - .try_init(); - }); -} - /// Disable the dangling-delete grace window so the destructive path is genuinely /// LIVE in these tests: without the decision-1 guard, a recoverable version WOULD /// be dangling-deleted here. With grace at its 1h default the delete path would be @@ -78,60 +63,16 @@ fn disable_dangling_grace() { } } -/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage`. -async fn setup_test_env_n(n_disks: usize) -> (Vec, Arc, Arc) { - init_tracing(); - - let test_base_dir = format!("/tmp/rustfs_heal_b920_test_{}", uuid::Uuid::new_v4()); - let temp_dir = PathBuf::from(&test_base_dir); - if temp_dir.exists() { - fs::remove_dir_all(&temp_dir).await.ok(); - } - fs::create_dir_all(&temp_dir).await.unwrap(); - - let disk_paths: Vec = (0..n_disks).map(|i| temp_dir.join(format!("disk{}", i + 1))).collect(); - for disk_path in &disk_paths { - fs::create_dir_all(disk_path).await.unwrap(); - } - - let mut endpoints = Vec::new(); - for (i, disk_path) in disk_paths.iter().enumerate() { - let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap(); - endpoint.set_pool_index(0); - endpoint.set_set_index(0); - endpoint.set_disk_index(i); - endpoints.push(endpoint); - } - - let pool_endpoints = PoolEndpoints { - legacy: false, - set_count: 1, - drives_per_set: n_disks, - endpoints: Endpoints::from(endpoints), - cmd_line: "test".to_string(), - platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), - }; - let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]); - - init_local_disks(endpoint_pools.clone()).await.unwrap(); - - let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); - let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) - .await - .unwrap(); - - let buckets_list = ecstore - .list_bucket(&BucketOptions { - no_metadata: true, - ..Default::default() - }) - .await - .unwrap(); - let buckets = buckets_list.into_iter().map(|v| v.name).collect(); - init_bucket_metadata_sys(ecstore.clone(), buckets).await; - - let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone())); - (disk_paths, ecstore, heal_storage) +/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage` via the +/// shared rustfs-test-utils environment (backlog#1153 infra-1). +async fn heal_env_n(n_disks: usize) -> (Vec, Arc, Arc) { + let env = rustfs_test_utils::TestECStoreEnv::builder() + .prefix("rustfs_heal_b920_test") + .disk_count(n_disks) + .build() + .await; + let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone())); + (env.disk_paths, env.ecstore, heal_storage) } async fn create_versioned_bucket(ecstore: &Arc, bucket: &str) { @@ -273,7 +214,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn disk_walk_page_enumerates_subquorum_version_omitted_by_list_object_versions() { - let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await; + let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await; let bucket = "b920-enum-gap"; let object = "obj.bin"; create_versioned_bucket(&ecstore, bucket).await; @@ -310,7 +251,7 @@ mod serial_tests { #[serial] async fn union_meta_lost_data_present_is_repaired_not_destroyed() { disable_dangling_grace(); - let (disk_paths, ecstore, heal_storage) = setup_test_env_n(8).await; + let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await; let bucket = "b920-meta-lost"; let object = "obj.bin"; create_versioned_bucket(&ecstore, bucket).await; @@ -360,7 +301,7 @@ mod serial_tests { #[serial] async fn deep_heal_torn_minority_is_dangling_deleted_with_grace_zero() { disable_dangling_grace(); - let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await; + let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await; let bucket = "b920-torn"; let object = "obj.bin"; create_versioned_bucket(&ecstore, bucket).await; @@ -409,7 +350,7 @@ mod serial_tests { #[serial] async fn deep_heal_restores_subquorum_but_reconstructable_version_wider_set() { disable_dangling_grace(); - let (disk_paths, ecstore, heal_storage) = setup_test_env_n(8).await; + let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await; let bucket = "b920-reconstruct"; let object = "obj.bin"; create_versioned_bucket(&ecstore, bucket).await; @@ -463,7 +404,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn disk_walk_multipage_resume_heals_each_version_once() { - let (_disk_paths, ecstore, _heal_storage) = setup_test_env_n(4).await; + let (_disk_paths, ecstore, _heal_storage) = heal_env_n(4).await; let bucket = "b920-multipage"; create_versioned_bucket(&ecstore, bucket).await; @@ -523,7 +464,7 @@ mod serial_tests { #[serial] async fn offline_disk_during_walk_does_not_dangling_delete() { disable_dangling_grace(); - let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await; + let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await; let bucket = "b920-offline"; let object = "obj.bin"; create_versioned_bucket(&ecstore, bucket).await; @@ -570,7 +511,7 @@ mod serial_tests { #[serial] async fn deep_heal_keeps_present_ec2_plus_2_shards_healthy() { disable_dangling_grace(); - let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await; + let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await; let bucket = "b1044-deep-verify"; let object = "obj.bin"; create_versioned_bucket(&ecstore, bucket).await; diff --git a/crates/heal/tests/heal_integration_test.rs b/crates/heal/tests/heal_integration_test.rs index 6000ecdda..55fe05570 100644 --- a/crates/heal/tests/heal_integration_test.rs +++ b/crates/heal/tests/heal_integration_test.rs @@ -22,20 +22,16 @@ use rustfs_heal::heal::{ use serial_test::serial; use std::{ path::{Path, PathBuf}, - sync::{Arc, Once}, + sync::Arc, time::Duration, }; use tokio::fs; -use tokio_util::sync::CancellationToken; use tracing::info; use walkdir::WalkDir; mod storage_api; -use storage_api::integration::{ - BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, ObjectIO as _, ObjectOperations as _, - PoolEndpoints, init_bucket_metadata_sys, init_local_disks, -}; +use storage_api::integration::{BucketOperations, ECStore, ObjectIO as _, ObjectOperations as _}; const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25); const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250); @@ -58,89 +54,16 @@ async fn wait_for_path_exists(path: &Path, timeout: Duration, interval: Duration } } -static INIT: Once = Once::new(); - -pub fn init_tracing() { - INIT.call_once(|| { - let _ = tracing_subscriber::fmt() - .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) - .with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339()) - .with_thread_names(true) - .try_init(); - }); -} - -/// Test helper: Create test environment with ECStore -async fn setup_test_env() -> (Vec, Arc, Arc) { - init_tracing(); - - // create temp dir as 4 disks with unique base dir - let test_base_dir = format!("/tmp/rustfs_heal_heal_test_{}", uuid::Uuid::new_v4()); - let temp_dir = std::path::PathBuf::from(&test_base_dir); - if temp_dir.exists() { - fs::remove_dir_all(&temp_dir).await.ok(); - } - fs::create_dir_all(&temp_dir).await.unwrap(); - - // create 4 disk dirs - let disk_paths = vec![ - temp_dir.join("disk1"), - temp_dir.join("disk2"), - temp_dir.join("disk3"), - temp_dir.join("disk4"), - ]; - - for disk_path in &disk_paths { - fs::create_dir_all(disk_path).await.unwrap(); - } - - // create EndpointServerPools - let mut endpoints = Vec::new(); - for (i, disk_path) in disk_paths.iter().enumerate() { - let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap(); - // set correct index - endpoint.set_pool_index(0); - endpoint.set_set_index(0); - endpoint.set_disk_index(i); - endpoints.push(endpoint); - } - - let pool_endpoints = PoolEndpoints { - legacy: false, - set_count: 1, - drives_per_set: 4, - endpoints: Endpoints::from(endpoints), - cmd_line: "test".to_string(), - platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), - }; - - let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]); - - // format disks (only first time) - init_local_disks(endpoint_pools.clone()).await.unwrap(); - - // Use port 0 so nextest can run this integration binary in parallel - // with other ECStore-backed tests without sharing a fixed peer port. - let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); - let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) - .await - .unwrap(); - - // init bucket metadata system - let buckets_list = ecstore - .list_bucket(&BucketOptions { - no_metadata: true, - ..Default::default() - }) - .await - .unwrap(); - let buckets = buckets_list.into_iter().map(|v| v.name).collect(); - init_bucket_metadata_sys(ecstore.clone(), buckets).await; - - // Create heal storage layer - let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone())); - - (disk_paths, ecstore, heal_storage) +/// Test helper: build the shared 4-disk temp-dir ECStore environment +/// (rustfs-test-utils, backlog#1153 infra-1) and wrap it in the heal storage +/// layer. Port 0 + uuid temp dirs keep this parallel-safe under nextest. +async fn heal_env() -> (Vec, Arc, Arc) { + let env = rustfs_test_utils::TestECStoreEnv::builder() + .prefix("rustfs_heal_heal_test") + .build() + .await; + let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone())); + (env.disk_paths, env.ecstore, heal_storage) } /// Test helper: Create a test bucket @@ -169,7 +92,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn test_heal_object_basic() { - let (disk_paths, ecstore, heal_storage) = setup_test_env().await; + let (disk_paths, ecstore, heal_storage) = heal_env().await; // Create test bucket and object let bucket_name = "test-heal-object-basic"; @@ -238,7 +161,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn test_heal_object_reclaims_orphan_data_dir_when_healthy() { - let (disk_paths, ecstore, heal_storage) = setup_test_env().await; + let (disk_paths, ecstore, heal_storage) = heal_env().await; let bucket_name = "test-heal-reclaim-orphan"; let object_name = "healthy-object.bin"; @@ -350,7 +273,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn test_heal_bucket_basic() { - let (disk_paths, ecstore, heal_storage) = setup_test_env().await; + let (disk_paths, ecstore, heal_storage) = heal_env().await; // Create test bucket let bucket_name = "test-heal-bucket-basic"; @@ -420,7 +343,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn test_heal_format_basic() { - let (disk_paths, _ecstore, heal_storage) = setup_test_env().await; + let (disk_paths, _ecstore, heal_storage) = heal_env().await; // ─── 1️⃣ delete format.json on one disk ────────────── let format_path = disk_paths[0].join(".rustfs.sys").join("format.json"); @@ -446,7 +369,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn test_heal_format_with_data() { - let (disk_paths, ecstore, heal_storage) = setup_test_env().await; + let (disk_paths, ecstore, heal_storage) = heal_env().await; // Create test bucket and object let bucket_name = "test-heal-format-with-data"; @@ -535,7 +458,7 @@ mod serial_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn test_heal_storage_api_direct() { - let (_disk_paths, ecstore, heal_storage) = setup_test_env().await; + let (_disk_paths, ecstore, heal_storage) = heal_env().await; // Test direct heal storage API calls diff --git a/crates/heal/tests/storage_api.rs b/crates/heal/tests/storage_api.rs index e0a01f801..d224f4dfc 100644 --- a/crates/heal/tests/storage_api.rs +++ b/crates/heal/tests/storage_api.rs @@ -15,17 +15,13 @@ pub(crate) mod endpoint_index { pub(crate) use rustfs_ecstore::api::storage::init_local_disks; } +// The temp-disk ECStore bootstrap itself lives in rustfs-test-utils +// (backlog#1153 infra-1); this surface keeps only what the heal test bodies +// still touch directly. #[allow(unused_imports)] pub(crate) mod integration { - pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys; - pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint; - pub(crate) use rustfs_ecstore::api::layout::EndpointServerPools; - pub(crate) use rustfs_ecstore::api::layout::Endpoints; - pub(crate) use rustfs_ecstore::api::layout::PoolEndpoints; pub(crate) use rustfs_ecstore::api::storage::ECStore; - pub(crate) use rustfs_ecstore::api::storage::init_local_disks; pub(crate) use rustfs_storage_api::BucketOperations; - pub(crate) use rustfs_storage_api::BucketOptions; pub(crate) use rustfs_storage_api::MakeBucketOptions; pub(crate) use rustfs_storage_api::ObjectIO; pub(crate) use rustfs_storage_api::ObjectOperations; diff --git a/crates/iam/Cargo.toml b/crates/iam/Cargo.toml index 1fb71610d..806a9a6b4 100644 --- a/crates/iam/Cargo.toml +++ b/crates/iam/Cargo.toml @@ -59,6 +59,7 @@ url = { workspace = true } [dev-dependencies] pollster.workspace = true +rustfs-test-utils = { workspace = true } serial_test = { workspace = true } temp-env = { workspace = true, features = ["async_closure"] } tempfile = { workspace = true } diff --git a/crates/iam/tests/ecstore_test_compat/mod.rs b/crates/iam/tests/ecstore_test_compat/mod.rs index 641d795ce..bccd89f45 100644 --- a/crates/iam/tests/ecstore_test_compat/mod.rs +++ b/crates/iam/tests/ecstore_test_compat/mod.rs @@ -22,9 +22,7 @@ #[allow(unused_imports)] pub(crate) mod fixture { - pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint; - pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints, SetupType}; - pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks}; + pub(crate) use rustfs_ecstore::api::layout::SetupType; // `update_erasure_type` is a write-side global facade entry. Its use is // restricted to reviewed storage_api boundaries; this test-only module is diff --git a/crates/iam/tests/iam_bootstrap_no_lock_test.rs b/crates/iam/tests/iam_bootstrap_no_lock_test.rs index f2e698404..9183a39dd 100644 --- a/crates/iam/tests/iam_bootstrap_no_lock_test.rs +++ b/crates/iam/tests/iam_bootstrap_no_lock_test.rs @@ -31,55 +31,16 @@ mod ecstore_test_compat; -use ecstore_test_compat::fixture::{ - ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, SetupType, init_local_disks, update_erasure_type, -}; +use ecstore_test_compat::fixture::{SetupType, update_erasure_type}; use rustfs_iam::cache::Cache; use rustfs_iam::store::object::ObjectStore; use rustfs_iam::store::{GroupInfo, Store}; use serial_test::serial; use std::collections::HashMap; -use std::sync::Arc; -use tokio_util::sync::CancellationToken; const TEST_GROUP: &str = "seq-restart-group"; const TEST_MEMBERS: [&str; 2] = ["alice", "bob"]; -async fn build_local_ecstore(temp_dir: &std::path::Path) -> Arc { - let disk_paths: Vec<_> = (1..=4).map(|i| temp_dir.join(format!("disk{i}"))).collect(); - for disk_path in &disk_paths { - tokio::fs::create_dir_all(disk_path).await.unwrap(); - } - - let mut endpoints = Vec::new(); - for (i, disk_path) in disk_paths.iter().enumerate() { - let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap(); - endpoint.set_pool_index(0); - endpoint.set_set_index(0); - endpoint.set_disk_index(i); - endpoints.push(endpoint); - } - - let pool_endpoints = PoolEndpoints { - legacy: false, - set_count: 1, - drives_per_set: 4, - endpoints: Endpoints::from(endpoints), - cmd_line: "test".to_string(), - platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), - }; - let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]); - - init_local_disks(endpoint_pools.clone()).await.unwrap(); - - // Port 0 keeps this integration binary parallel-safe alongside other - // ECStore-backed tests. - let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); - ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) - .await - .unwrap() -} - /// Restores single-node erasure mode even when an assertion panics, so a /// failing run cannot poison later `#[serial]` tests in this process. struct ErasureModeGuard; @@ -97,7 +58,15 @@ async fn load_all_bypasses_namespace_lock_quorum() { // must be shortened before the first locked operation of this process. temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async { let temp_dir = tempfile::TempDir::with_prefix("rustfs_iam_no_lock_test_").unwrap(); - let ecstore = build_local_ecstore(temp_dir.path()).await; + // Shared temp-disk ECStore env (rustfs-test-utils, backlog#1153 infra-1). + // base_dir keeps cleanup ownership with this TempDir; the historical + // bootstrap never initialized the bucket-metadata system, so opt out. + let ecstore = rustfs_test_utils::TestECStoreEnv::builder() + .base_dir(temp_dir.path()) + .init_bucket_metadata(false) + .build() + .await + .ecstore; let store = ObjectStore::new(ecstore); // Seed IAM data while namespace locks still work (single-node mode). diff --git a/crates/test-utils/Cargo.toml b/crates/test-utils/Cargo.toml new file mode 100644 index 000000000..afa127f70 --- /dev/null +++ b/crates/test-utils/Cargo.toml @@ -0,0 +1,40 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[package] +name = "rustfs-test-utils" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true +homepage.workspace = true +description = "Shared test bootstrap helpers for RustFS integration tests (dev-dependency only)." +keywords = ["testing", "storage", "rustfs", "Minio"] +categories = ["development-tools", "filesystem"] +documentation = "https://docs.rs/rustfs-test-utils/latest/rustfs_test_utils/" + +[dependencies] +rustfs-ecstore = { workspace = true } +rustfs-storage-api = { workspace = true } +tokio = { workspace = true, features = ["fs", "rt"] } +tokio-util = { workspace = true } +tracing-subscriber = { workspace = true } +uuid = { workspace = true, features = ["v4"] } + +[lints] +workspace = true + +[lib] +doctest = false diff --git a/crates/test-utils/README.md b/crates/test-utils/README.md new file mode 100644 index 000000000..b4b682678 --- /dev/null +++ b/crates/test-utils/README.md @@ -0,0 +1,56 @@ +# rustfs-test-utils + +Shared bootstrap helpers for RustFS integration tests (backlog#1153 infra-1). +Owns the "build a real temp-disk `ECStore`" setup that used to be copy-pasted +across the heal/iam/scanner integration tests and had already drifted. + +## Usage + +```rust +// [dev-dependencies] rustfs-test-utils = { workspace = true } +use rustfs_test_utils::TestECStoreEnv; + +let env = TestECStoreEnv::builder() + .prefix("rustfs_myfeature_test") // /tmp/_ + .disk_count(4) // single pool, single set + .build() + .await; + +env.make_bucket("my-bucket", /* versioned = */ true).await; +// env.ecstore : Arc — bootstrapped store +// env.disk_paths : Vec — disk1..diskN for fault injection +// env.temp_root : PathBuf — root dir (leaked by design, see below) +``` + +Builder knobs: + +- `disk_count(n)` — disks in the single erasure set (default 4). +- `prefix(&str)` — temp-dir prefix; a uuid suffix keeps parallel nextest + processes isolated. +- `base_dir(path)` — use a caller-owned directory (e.g. `tempfile::TempDir`) + instead; the caller keeps cleanup ownership. +- `init_bucket_metadata(bool)` — run `init_bucket_metadata_sys` after boot + (default `true`; the IAM bootstrap test opts out). + +`init_tracing()` is exposed separately and is called by `build()`. + +The environment does **not** delete `temp_root` on drop: a failed test's +on-disk state stays inspectable, and heal tests keep manipulating +`disk_paths` after setup. Own the directory via `base_dir` if you want +automatic cleanup. + +## Constraints + +- **dev-dependency only.** No crate may list `rustfs-test-utils` under + `[dependencies]` — test scaffolding must never reach production binaries. +- **Single-process integration scope.** Multi-node / chaos harnesses are out + of scope (backlog#1100). Scanner's lifecycle tests are absorbed only after + ilm-1 activates them (they are `#[ignore]`d today). +- All `rustfs_ecstore` imports stay behind `src/ecstore_test_compat.rs` — the + sanctioned test-compat boundary pattern enforced by + `scripts/check_architecture_migration_rules.sh`. Extend that module, never + import the facade directly from other files. +- Tests using this env bind `127.0.0.1:0` (random port) and unique temp dirs, + so they stay parallel-safe under nextest's process-per-test model — do not + add fixed ports or shared paths here. See `docs/testing/README.md` for the + serial/nextest rules. diff --git a/crates/test-utils/src/ecstore_test_compat.rs b/crates/test-utils/src/ecstore_test_compat.rs new file mode 100644 index 000000000..cfb208537 --- /dev/null +++ b/crates/test-utils/src/ecstore_test_compat.rs @@ -0,0 +1,31 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test-only ECStore compatibility boundary for `rustfs-test-utils`. +//! +//! All direct `rustfs_ecstore` facade imports used by this crate must go +//! through this module (architecture migration rule: +//! `check_architecture_migration_rules.sh`; this file name is the sanctioned +//! test-compat boundary pattern, mirroring +//! `crates/iam/tests/ecstore_test_compat/mod.rs`). Keep the surface minimal — +//! only what the environment builder needs to assemble a temp-disk ECStore. + +#[allow(unused_imports)] +pub(crate) mod fixture { + pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys; + pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint; + pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints}; + pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks}; + pub(crate) use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions}; +} diff --git a/crates/test-utils/src/lib.rs b/crates/test-utils/src/lib.rs new file mode 100644 index 000000000..977112225 --- /dev/null +++ b/crates/test-utils/src/lib.rs @@ -0,0 +1,210 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Shared test bootstrap helpers for RustFS integration tests +//! (backlog#1153 infra-1). +//! +//! This crate is a **dev-dependency only**: it must never appear in any +//! crate's `[dependencies]`. It owns the ~50-line "build a real temp-disk +//! `ECStore`" bootstrap that used to be copy-pasted (and drift) across the +//! heal/iam/scanner integration tests. +//! +//! Single-process integration scope only — multi-node / chaos harnesses are +//! out of scope (backlog#1100). + +mod ecstore_test_compat; + +use std::path::PathBuf; +use std::sync::{Arc, Once}; + +use ecstore_test_compat::fixture::{ + BucketOperations as _, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, PoolEndpoints, + init_bucket_metadata_sys, init_local_disks, +}; +use tokio_util::sync::CancellationToken; + +static INIT_TRACING: Once = Once::new(); + +/// Install the standard test tracing subscriber once per process +/// (`RUST_LOG`-driven). Safe to call from every test; later calls are no-ops. +pub fn init_tracing() { + INIT_TRACING.call_once(|| { + let _ = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339()) + .with_thread_names(true) + .try_init(); + }); +} + +/// A real single-pool, single-set `ECStore` built over per-test temp-dir +/// "disks". Build one with [`TestECStoreEnv::builder`]. +/// +/// The environment intentionally does **not** delete `temp_root` on drop: +/// the historical bootstraps leaked their uuid-suffixed temp dirs so a failed +/// test's on-disk state stays inspectable, and several heal tests keep +/// manipulating `disk_paths` after setup. Callers that own the directory +/// lifetime (e.g. via `tempfile::TempDir`) should pass it through +/// [`TestECStoreEnvBuilder::base_dir`]. +pub struct TestECStoreEnv { + /// Root directory holding the disk directories. + pub temp_root: PathBuf, + /// The per-disk directories (`disk1`..`diskN`) under `temp_root`. + pub disk_paths: Vec, + /// The store, bootstrapped exactly like the historical test setups: + /// `init_local_disks` + `ECStore::new` on `127.0.0.1:0` (random port keeps + /// nextest's process-per-test parallelism safe). + pub ecstore: Arc, +} + +impl TestECStoreEnv { + pub fn builder() -> TestECStoreEnvBuilder { + TestECStoreEnvBuilder::default() + } + + /// Create a bucket, optionally with S3 versioning enabled at creation + /// time (without this a second PUT overwrites in place and DELETE removes + /// the object outright — no old versions or delete-marker-latest exist). + pub async fn make_bucket(&self, bucket: &str, versioned: bool) { + self.ecstore + .make_bucket( + bucket, + &MakeBucketOptions { + versioning_enabled: versioned, + ..Default::default() + }, + ) + .await + .unwrap_or_else(|e| panic!("failed to create test bucket {bucket}: {e:?}")); + } +} + +/// Builder for [`TestECStoreEnv`]. Defaults reproduce the historical heal +/// bootstrap: 4 disks, one pool, one set, bucket-metadata system initialized. +pub struct TestECStoreEnvBuilder { + disk_count: usize, + prefix: String, + base_dir: Option, + init_bucket_metadata: bool, +} + +impl Default for TestECStoreEnvBuilder { + fn default() -> Self { + Self { + disk_count: 4, + prefix: "rustfs_test_utils".to_string(), + base_dir: None, + init_bucket_metadata: true, + } + } +} + +impl TestECStoreEnvBuilder { + /// Number of disk directories in the single erasure set (default 4). + pub fn disk_count(mut self, n: usize) -> Self { + self.disk_count = n; + self + } + + /// Temp-dir name prefix, e.g. `rustfs_heal_b5_test` (a uuid suffix is + /// always appended). Ignored when [`base_dir`](Self::base_dir) is set. + pub fn prefix(mut self, prefix: &str) -> Self { + self.prefix = prefix.to_string(); + self + } + + /// Use a caller-owned directory (e.g. a `tempfile::TempDir` path) instead + /// of creating `/tmp/_`. The caller keeps cleanup ownership. + pub fn base_dir(mut self, dir: impl Into) -> Self { + self.base_dir = Some(dir.into()); + self + } + + /// Whether to run `init_bucket_metadata_sys` after the store comes up + /// (default `true`, as the heal bootstraps did). The IAM bootstrap test + /// opts out to preserve its historical semantics. + pub fn init_bucket_metadata(mut self, yes: bool) -> Self { + self.init_bucket_metadata = yes; + self + } + + /// Build the environment. Panics on any bootstrap failure — this is test + /// scaffolding, and a broken environment must fail the test loudly. + pub async fn build(self) -> TestECStoreEnv { + init_tracing(); + + let temp_root = match &self.base_dir { + Some(dir) => dir.clone(), + None => { + let root = PathBuf::from(format!("/tmp/{}_{}", self.prefix, uuid::Uuid::new_v4())); + if root.exists() { + tokio::fs::remove_dir_all(&root).await.ok(); + } + root + } + }; + tokio::fs::create_dir_all(&temp_root).await.expect("create test temp root"); + + let disk_paths: Vec = (1..=self.disk_count).map(|i| temp_root.join(format!("disk{i}"))).collect(); + for disk_path in &disk_paths { + tokio::fs::create_dir_all(disk_path).await.expect("create test disk dir"); + } + + let mut endpoints = Vec::new(); + for (i, disk_path) in disk_paths.iter().enumerate() { + let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("utf-8 disk path")).expect("parse disk endpoint"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(i); + endpoints.push(endpoint); + } + + let pool_endpoints = PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: self.disk_count, + endpoints: Endpoints::from(endpoints), + cmd_line: "test".to_string(), + platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), + }; + let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]); + + init_local_disks(endpoint_pools.clone()).await.expect("init local disks"); + + // Port 0 keeps ECStore-backed integration binaries parallel-safe under + // nextest: no fixed peer port is ever shared between test processes. + let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().expect("parse test addr"); + let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) + .await + .expect("build test ECStore"); + + if self.init_bucket_metadata { + let buckets_list = ecstore + .list_bucket(&BucketOptions { + no_metadata: true, + ..Default::default() + }) + .await + .expect("list buckets for metadata init"); + let buckets = buckets_list.into_iter().map(|v| v.name).collect(); + init_bucket_metadata_sys(ecstore.clone(), buckets).await; + } + + TestECStoreEnv { + temp_root, + disk_paths, + ecstore, + } + } +}