mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 15:37:02 +00:00
test(utils): add rustfs-test-utils crate and shared ECStore bootstrap (#4850)
* test(utils): add rustfs-test-utils crate, absorb heal/iam ECStore bootstrap
backlog#1153 infra-1. The ~50-line "build a real temp-disk ECStore"
bootstrap was copy-pasted (and drifting) across the heal and iam
integration tests. This adds crates/test-utils (rustfs-test-utils, a
dev-dependency-only crate) owning that bootstrap and converts the four
copies into thin wrappers:
- TestECStoreEnvBuilder: disk_count (default 4), prefix (uuid-suffixed
/tmp dir), base_dir (caller-owned dir, e.g. tempfile::TempDir),
init_bucket_metadata (default true; the iam bootstrap test opts out
to preserve its historical semantics). TestECStoreEnv exposes
temp_root/disk_paths/ecstore plus a versioned-bucket helper, and
init_tracing() replaces the per-file Once blocks.
- All rustfs_ecstore imports stay behind src/ecstore_test_compat.rs,
the sanctioned test-compat boundary pattern (mirrors
crates/iam/tests/ecstore_test_compat).
- heal: heal_integration_test / heal_b5_versioned_regression_test /
heal_b920_subquorum_union_test drop their setup_test_env{,_n} copies
for heal_env{,_n} wrappers; the tests/storage_api.rs integration
surface shrinks to what test bodies still touch.
- iam: iam_bootstrap_no_lock_test drops build_local_ecstore; its
ecstore_test_compat fixture shrinks to SetupType +
update_erasure_type.
rg 'async fn setup_test_env' crates/heal crates/iam now returns 0.
Scanner's lifecycle tests are deliberately NOT absorbed (gated on
ilm-1; 14 of 15 are #[ignore]d today). Net -230 lines.
* fix(heal): drop tokio::fs import orphaned by the b920 bootstrap move
* fix(heal): drop tokio::fs import orphaned by the b5 bootstrap move
This commit is contained in:
Zhengchao An
@@ -48,6 +48,7 @@ base64 = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
rustfs-test-utils = { workspace = true }
|
||||
serial_test = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
//!
|
||||
//! These drive the REAL `ECStoreHealStorage` (not a mock) against a real 4-disk
|
||||
//! `ECStore`, mirroring `heal_integration_test.rs`. Every test is `#[serial]`
|
||||
//! and re-runs the full `setup_test_env` init (which sets the process-global
|
||||
//! and re-runs the full `heal_env` init (which sets the process-global
|
||||
//! bucket-metadata-sys OnceCell) — under `cargo nextest` each test runs
|
||||
//! in its own process so the OnceCell never collides.
|
||||
|
||||
@@ -34,20 +34,15 @@ use rustfs_heal::heal::{
|
||||
use serial_test::serial;
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Once},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::fs;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::info;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use storage_api::integration::{
|
||||
BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _,
|
||||
ObjectOperations as _, PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
|
||||
};
|
||||
use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _, ObjectOperations as _};
|
||||
|
||||
/// 256 KiB + change: large enough to be stored as non-inline erasure shards
|
||||
/// (so each data version materializes as an on-disk `part.*` file we can assert
|
||||
@@ -62,78 +57,16 @@ fn versioned_test_data(seed: u8) -> Vec<u8> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
fn init_tracing() {
|
||||
INIT.call_once(|| {
|
||||
let _ = tracing_subscriber::fmt()
|
||||
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
|
||||
.with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339())
|
||||
.with_thread_names(true)
|
||||
.try_init();
|
||||
});
|
||||
}
|
||||
|
||||
/// Build a real 4-disk `ECStore` + `ECStoreHealStorage`. Mirrors
|
||||
/// `heal_integration_test::setup_test_env`.
|
||||
async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
|
||||
init_tracing();
|
||||
|
||||
let test_base_dir = format!("/tmp/rustfs_heal_b5_test_{}", uuid::Uuid::new_v4());
|
||||
let temp_dir = PathBuf::from(&test_base_dir);
|
||||
if temp_dir.exists() {
|
||||
fs::remove_dir_all(&temp_dir).await.ok();
|
||||
}
|
||||
fs::create_dir_all(&temp_dir).await.unwrap();
|
||||
|
||||
let disk_paths = vec![
|
||||
temp_dir.join("disk1"),
|
||||
temp_dir.join("disk2"),
|
||||
temp_dir.join("disk3"),
|
||||
temp_dir.join("disk4"),
|
||||
];
|
||||
for disk_path in &disk_paths {
|
||||
fs::create_dir_all(disk_path).await.unwrap();
|
||||
}
|
||||
|
||||
let mut endpoints = Vec::new();
|
||||
for (i, disk_path) in disk_paths.iter().enumerate() {
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(i);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
|
||||
let pool_endpoints = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "test".to_string(),
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
};
|
||||
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
|
||||
|
||||
init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
|
||||
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let buckets_list = ecstore
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
|
||||
let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
|
||||
(disk_paths, ecstore, heal_storage)
|
||||
/// Build a real 4-disk `ECStore` + `ECStoreHealStorage` via the shared
|
||||
/// rustfs-test-utils environment (backlog#1153 infra-1). Mirrors
|
||||
/// `heal_integration_test::heal_env`.
|
||||
async fn heal_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("rustfs_heal_b5_test")
|
||||
.build()
|
||||
.await;
|
||||
let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
|
||||
(env.disk_paths, env.ecstore, heal_storage)
|
||||
}
|
||||
|
||||
/// Create a bucket with S3 versioning ENABLED at creation time. Without this the
|
||||
@@ -308,7 +241,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_enumeration_includes_all_versions_and_delete_marker_real_fixture() {
|
||||
let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;
|
||||
let (_disk_paths, ecstore, heal_storage) = heal_env().await;
|
||||
let bucket = "b5-enum-versions";
|
||||
let object = "obj.bin";
|
||||
create_versioned_bucket(&ecstore, bucket).await;
|
||||
@@ -345,7 +278,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_heal_old_nonlatest_version_after_disk_wipe() {
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env().await;
|
||||
let bucket = "b5-old-version-heal";
|
||||
let object = "obj.bin";
|
||||
create_versioned_bucket(&ecstore, bucket).await;
|
||||
@@ -397,7 +330,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_heal_delete_marker_latest_enumerated_and_healed() {
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env().await;
|
||||
let bucket = "b5-dm-latest-heal";
|
||||
let object = "obj.bin";
|
||||
create_versioned_bucket(&ecstore, bucket).await;
|
||||
@@ -463,7 +396,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_version_id_normalization_null_and_unversioned_real_fixture() {
|
||||
let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;
|
||||
let (_disk_paths, ecstore, heal_storage) = heal_env().await;
|
||||
let bucket = "b5-unversioned-normalize";
|
||||
create_unversioned_bucket(&ecstore, bucket).await;
|
||||
|
||||
@@ -499,7 +432,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_heal_unversioned_bucket_e2e() {
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env().await;
|
||||
let bucket = "b5-unversioned-e2e";
|
||||
create_unversioned_bucket(&ecstore, bucket).await;
|
||||
|
||||
@@ -584,7 +517,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_heal_resume_across_page_boundary_e2e() {
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env().await;
|
||||
let bucket = "b5-resume-e2e";
|
||||
create_versioned_bucket(&ecstore, bucket).await;
|
||||
|
||||
|
||||
@@ -29,18 +29,13 @@ use rustfs_heal::heal::storage::{
|
||||
use serial_test::serial;
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Once},
|
||||
sync::Arc,
|
||||
};
|
||||
use tokio::fs;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use storage_api::integration::{
|
||||
BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _,
|
||||
PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
|
||||
};
|
||||
use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _};
|
||||
|
||||
/// 256 KiB + change: large enough to be stored as non-inline erasure shards, so
|
||||
/// deleting the `xl.meta` file does NOT delete the data (the `part.*` shards live
|
||||
@@ -57,16 +52,6 @@ fn versioned_test_data(seed: u8) -> Vec<u8> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
fn init_tracing() {
|
||||
INIT.call_once(|| {
|
||||
let _ = tracing_subscriber::fmt()
|
||||
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
|
||||
.try_init();
|
||||
});
|
||||
}
|
||||
|
||||
/// Disable the dangling-delete grace window so the destructive path is genuinely
|
||||
/// LIVE in these tests: without the decision-1 guard, a recoverable version WOULD
|
||||
/// be dangling-deleted here. With grace at its 1h default the delete path would be
|
||||
@@ -78,60 +63,16 @@ fn disable_dangling_grace() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage`.
|
||||
async fn setup_test_env_n(n_disks: usize) -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
|
||||
init_tracing();
|
||||
|
||||
let test_base_dir = format!("/tmp/rustfs_heal_b920_test_{}", uuid::Uuid::new_v4());
|
||||
let temp_dir = PathBuf::from(&test_base_dir);
|
||||
if temp_dir.exists() {
|
||||
fs::remove_dir_all(&temp_dir).await.ok();
|
||||
}
|
||||
fs::create_dir_all(&temp_dir).await.unwrap();
|
||||
|
||||
let disk_paths: Vec<PathBuf> = (0..n_disks).map(|i| temp_dir.join(format!("disk{}", i + 1))).collect();
|
||||
for disk_path in &disk_paths {
|
||||
fs::create_dir_all(disk_path).await.unwrap();
|
||||
}
|
||||
|
||||
let mut endpoints = Vec::new();
|
||||
for (i, disk_path) in disk_paths.iter().enumerate() {
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(i);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
|
||||
let pool_endpoints = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: n_disks,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "test".to_string(),
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
};
|
||||
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
|
||||
|
||||
init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
|
||||
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let buckets_list = ecstore
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
|
||||
let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
|
||||
(disk_paths, ecstore, heal_storage)
|
||||
/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage` via the
|
||||
/// shared rustfs-test-utils environment (backlog#1153 infra-1).
|
||||
async fn heal_env_n(n_disks: usize) -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("rustfs_heal_b920_test")
|
||||
.disk_count(n_disks)
|
||||
.build()
|
||||
.await;
|
||||
let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
|
||||
(env.disk_paths, env.ecstore, heal_storage)
|
||||
}
|
||||
|
||||
async fn create_versioned_bucket(ecstore: &Arc<ECStore>, bucket: &str) {
|
||||
@@ -273,7 +214,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn disk_walk_page_enumerates_subquorum_version_omitted_by_list_object_versions() {
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b920-enum-gap";
|
||||
let object = "obj.bin";
|
||||
create_versioned_bucket(&ecstore, bucket).await;
|
||||
@@ -310,7 +251,7 @@ mod serial_tests {
|
||||
#[serial]
|
||||
async fn union_meta_lost_data_present_is_repaired_not_destroyed() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env_n(8).await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
|
||||
let bucket = "b920-meta-lost";
|
||||
let object = "obj.bin";
|
||||
create_versioned_bucket(&ecstore, bucket).await;
|
||||
@@ -360,7 +301,7 @@ mod serial_tests {
|
||||
#[serial]
|
||||
async fn deep_heal_torn_minority_is_dangling_deleted_with_grace_zero() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b920-torn";
|
||||
let object = "obj.bin";
|
||||
create_versioned_bucket(&ecstore, bucket).await;
|
||||
@@ -409,7 +350,7 @@ mod serial_tests {
|
||||
#[serial]
|
||||
async fn deep_heal_restores_subquorum_but_reconstructable_version_wider_set() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env_n(8).await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
|
||||
let bucket = "b920-reconstruct";
|
||||
let object = "obj.bin";
|
||||
create_versioned_bucket(&ecstore, bucket).await;
|
||||
@@ -463,7 +404,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn disk_walk_multipage_resume_heals_each_version_once() {
|
||||
let (_disk_paths, ecstore, _heal_storage) = setup_test_env_n(4).await;
|
||||
let (_disk_paths, ecstore, _heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b920-multipage";
|
||||
create_versioned_bucket(&ecstore, bucket).await;
|
||||
|
||||
@@ -523,7 +464,7 @@ mod serial_tests {
|
||||
#[serial]
|
||||
async fn offline_disk_during_walk_does_not_dangling_delete() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b920-offline";
|
||||
let object = "obj.bin";
|
||||
create_versioned_bucket(&ecstore, bucket).await;
|
||||
@@ -570,7 +511,7 @@ mod serial_tests {
|
||||
#[serial]
|
||||
async fn deep_heal_keeps_present_ec2_plus_2_shards_healthy() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b1044-deep-verify";
|
||||
let object = "obj.bin";
|
||||
create_versioned_bucket(&ecstore, bucket).await;
|
||||
|
||||
@@ -22,20 +22,16 @@ use rustfs_heal::heal::{
|
||||
use serial_test::serial;
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Once},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::fs;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::info;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use storage_api::integration::{
|
||||
BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, ObjectIO as _, ObjectOperations as _,
|
||||
PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
|
||||
};
|
||||
use storage_api::integration::{BucketOperations, ECStore, ObjectIO as _, ObjectOperations as _};
|
||||
|
||||
const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25);
|
||||
const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250);
|
||||
@@ -58,89 +54,16 @@ async fn wait_for_path_exists(path: &Path, timeout: Duration, interval: Duration
|
||||
}
|
||||
}
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
pub fn init_tracing() {
|
||||
INIT.call_once(|| {
|
||||
let _ = tracing_subscriber::fmt()
|
||||
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
|
||||
.with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339())
|
||||
.with_thread_names(true)
|
||||
.try_init();
|
||||
});
|
||||
}
|
||||
|
||||
/// Test helper: Create test environment with ECStore
|
||||
async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
|
||||
init_tracing();
|
||||
|
||||
// create temp dir as 4 disks with unique base dir
|
||||
let test_base_dir = format!("/tmp/rustfs_heal_heal_test_{}", uuid::Uuid::new_v4());
|
||||
let temp_dir = std::path::PathBuf::from(&test_base_dir);
|
||||
if temp_dir.exists() {
|
||||
fs::remove_dir_all(&temp_dir).await.ok();
|
||||
}
|
||||
fs::create_dir_all(&temp_dir).await.unwrap();
|
||||
|
||||
// create 4 disk dirs
|
||||
let disk_paths = vec![
|
||||
temp_dir.join("disk1"),
|
||||
temp_dir.join("disk2"),
|
||||
temp_dir.join("disk3"),
|
||||
temp_dir.join("disk4"),
|
||||
];
|
||||
|
||||
for disk_path in &disk_paths {
|
||||
fs::create_dir_all(disk_path).await.unwrap();
|
||||
}
|
||||
|
||||
// create EndpointServerPools
|
||||
let mut endpoints = Vec::new();
|
||||
for (i, disk_path) in disk_paths.iter().enumerate() {
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
|
||||
// set correct index
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(i);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
|
||||
let pool_endpoints = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "test".to_string(),
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
};
|
||||
|
||||
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
|
||||
|
||||
// format disks (only first time)
|
||||
init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
|
||||
// Use port 0 so nextest can run this integration binary in parallel
|
||||
// with other ECStore-backed tests without sharing a fixed peer port.
|
||||
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// init bucket metadata system
|
||||
let buckets_list = ecstore
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
|
||||
// Create heal storage layer
|
||||
let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
|
||||
|
||||
(disk_paths, ecstore, heal_storage)
|
||||
/// Test helper: build the shared 4-disk temp-dir ECStore environment
|
||||
/// (rustfs-test-utils, backlog#1153 infra-1) and wrap it in the heal storage
|
||||
/// layer. Port 0 + uuid temp dirs keep this parallel-safe under nextest.
|
||||
async fn heal_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("rustfs_heal_heal_test")
|
||||
.build()
|
||||
.await;
|
||||
let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
|
||||
(env.disk_paths, env.ecstore, heal_storage)
|
||||
}
|
||||
|
||||
/// Test helper: Create a test bucket
|
||||
@@ -169,7 +92,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_heal_object_basic() {
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env().await;
|
||||
|
||||
// Create test bucket and object
|
||||
let bucket_name = "test-heal-object-basic";
|
||||
@@ -238,7 +161,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_heal_object_reclaims_orphan_data_dir_when_healthy() {
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env().await;
|
||||
|
||||
let bucket_name = "test-heal-reclaim-orphan";
|
||||
let object_name = "healthy-object.bin";
|
||||
@@ -350,7 +273,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_heal_bucket_basic() {
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env().await;
|
||||
|
||||
// Create test bucket
|
||||
let bucket_name = "test-heal-bucket-basic";
|
||||
@@ -420,7 +343,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_heal_format_basic() {
|
||||
let (disk_paths, _ecstore, heal_storage) = setup_test_env().await;
|
||||
let (disk_paths, _ecstore, heal_storage) = heal_env().await;
|
||||
|
||||
// ─── 1️⃣ delete format.json on one disk ──────────────
|
||||
let format_path = disk_paths[0].join(".rustfs.sys").join("format.json");
|
||||
@@ -446,7 +369,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_heal_format_with_data() {
|
||||
let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env().await;
|
||||
|
||||
// Create test bucket and object
|
||||
let bucket_name = "test-heal-format-with-data";
|
||||
@@ -535,7 +458,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_heal_storage_api_direct() {
|
||||
let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;
|
||||
let (_disk_paths, ecstore, heal_storage) = heal_env().await;
|
||||
|
||||
// Test direct heal storage API calls
|
||||
|
||||
|
||||
@@ -15,17 +15,13 @@ pub(crate) mod endpoint_index {
|
||||
pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
|
||||
}
|
||||
|
||||
// The temp-disk ECStore bootstrap itself lives in rustfs-test-utils
|
||||
// (backlog#1153 infra-1); this surface keeps only what the heal test bodies
|
||||
// still touch directly.
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) mod integration {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys;
|
||||
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
|
||||
pub(crate) use rustfs_ecstore::api::layout::EndpointServerPools;
|
||||
pub(crate) use rustfs_ecstore::api::layout::Endpoints;
|
||||
pub(crate) use rustfs_ecstore::api::layout::PoolEndpoints;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore;
|
||||
pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
|
||||
pub(crate) use rustfs_storage_api::BucketOperations;
|
||||
pub(crate) use rustfs_storage_api::BucketOptions;
|
||||
pub(crate) use rustfs_storage_api::MakeBucketOptions;
|
||||
pub(crate) use rustfs_storage_api::ObjectIO;
|
||||
pub(crate) use rustfs_storage_api::ObjectOperations;
|
||||
|
||||
Reference in New Issue
Block a user