From b0b5ffb8e276ca4d5f37499b5274762a84289c31 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 17 Jul 2026 09:02:38 +0800 Subject: [PATCH] test(e2e): cluster harness drivesPerNode + 2-pool topology (backlog #1325) (#4937) test(e2e): add drivesPerNode and 2-pool cluster harness topology (backlog #1325) Extend the e2e cluster harness so tests can declare per-node multiple drives (drivesPerNode) and a two-pool topology, alongside a smoke suite that boots such a cluster and round-trips a PUT/GET. A new `ClusterTopology` describes node_count, drives_per_node, and pool membership, and `RustFSTestClusterEnvironment::with_topology` builds the matching on-disk drive directories and `RUSTFS_VOLUMES` string. `new(node_count)` now delegates to a single-pool single-drive topology, so existing single-drive cluster tests are byte-for-byte unchanged. `ClusterNode` gains `data_dirs` (all drives, `data_dirs[0] == data_dir`) and `pool_idx`; multi-drive/multi-pool clusters automatically add `RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true` because their drives share the same temp filesystem. The `RUSTFS_VOLUMES` assembly matches the two forms the server parser accepts on a single localhost machine (verified against ecstore `DisksLayout::from_volumes`): a single pool is the explicit enumeration of every `(node, drive)` endpoint (no ellipses, one legacy DistErasure pool), while multiple pools each contribute one ellipses argument `http:///drive{0...N-1}`. A pool argument is a single URL template, so it cannot enumerate multiple distinct-port hosts; a pool striped across several localhost nodes would need a host ellipses that forces a shared on-disk path and collides physically. The topology validator therefore requires every pool in a multi-pool layout to own exactly one node and requires drives_per_node >= 2 (the parser rejects a single-drive `drive{0...0}` ellipses pool). Genuine multi-node pools need real multi-host infrastructure and are deferred to the nightly cluster lane (backlog #1313/#1314). Unit tests assert the volumes-string layout for single-pool single-drive (backward compatible), single-pool multi-drive (8 explicit endpoints), and two-pool (one ellipses arg per pool), plus the validator rejections. The smoke suite `cluster_multidrive_pool_test` boots a 4-node x 2-drive single pool and a 2-node/2-pool cluster, asserts the volumes layout, and round-trips a PUT/GET using the harness readiness handshake (no fixed sleeps). It joins the six existing RustFSTestClusterEnvironment suites excluded from the merge-gate `e2e-full` profile so it runs in the nightly 4-node lane. Network fault injection (toxiproxy / socket proxy) and 5GiB large-object budgets remain out of scope for this block. --- .config/nextest.toml | 6 +- .../src/cluster_multidrive_pool_test.rs | 108 +++++ crates/e2e_test/src/common.rs | 393 +++++++++++++++++- crates/e2e_test/src/lib.rs | 4 + 4 files changed, 493 insertions(+), 18 deletions(-) create mode 100644 crates/e2e_test/src/cluster_multidrive_pool_test.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 2c59a6f17..42fa9a90c 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -263,8 +263,8 @@ path = "junit.xml" # The filter is "the whole e2e_test crate MINUS the sets owned by other lanes": # * protocols:: — FTPS/SFTP/WebDAV, still pinned to --test-threads=1 by fixed # ports; they join a scheduled lane once ci-6 randomises the ports (ci-7). -# * the 6 cluster suites that spin up a RustFSTestClusterEnvironment -# (cluster_concurrency, stale_multipart_cleanup_cluster, +# * the 7 cluster suites that spin up a RustFSTestClusterEnvironment +# (cluster_concurrency, cluster_multidrive_pool, stale_multipart_cleanup_cluster, # namespace_lock_quorum, heal_erasure_disk_rebuild, admin_timeout_regression, # object_lambda) — too heavy for the merge budget; they run in ci-7's # nightly 4-node lane. @@ -301,7 +301,7 @@ path = "junit.xml" default-filter = """ package(e2e_test) & !test(/^protocols::/) - & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/) + & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/) & !test(/^replication_extension_test::/) & !test(/^multipart_auth_test::test_signed_put_object_extract_(accepts_compat_header|expands_tar_entries_with_prefix_headers|expands_tar_gz_archive|expands_tbz2_archive|expands_tgz_archive|expands_txz_archive|expands_tzst_archive|normalizes_prefix_header_value|preserves_directory_markers_by_default|preserves_object_lock_legal_hold|preserves_object_lock_retention|preserves_pax_metadata_and_version_id|preserves_request_metadata_on_extracted_objects|preserves_sse_c|preserves_sse_s3_and_redirect|preserves_storage_class|uses_bucket_default_sse_s3)$/) & !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(prefers_exact_minio_prefix_over_suffix_fallback|supports_minio_prefix_and_directory_markers)$/) diff --git a/crates/e2e_test/src/cluster_multidrive_pool_test.rs b/crates/e2e_test/src/cluster_multidrive_pool_test.rs new file mode 100644 index 000000000..3f9193984 --- /dev/null +++ b/crates/e2e_test/src/cluster_multidrive_pool_test.rs @@ -0,0 +1,108 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Smoke tests for the multi-drive / multi-pool cluster harness. +//! +//! These belong to the nightly 4-node cluster lane (they spin up real RustFS +//! processes, so they are excluded from the merge-gate `e2e-full` profile in +//! `.config/nextest.toml`). They assert that: +//! +//! * `drivesPerNode > 1` produces a bootable single-pool cluster whose +//! `RUSTFS_VOLUMES` string enumerates every drive, and that a PUT/GET +//! round-trips through the multi-drive erasure layout. +//! * A two-pool topology (one node per pool, `drivesPerNode` drives each) boots +//! with the ellipses `RUSTFS_VOLUMES` form and round-trips a PUT/GET. +//! +//! Readiness is established by the harness's `start()` handshake (TCP reachability +//! plus an S3 `ListBuckets` poll) — there are no fixed sleeps. +//! +//! Out of scope for this block (tracked separately): network fault injection +//! (toxiproxy / socket proxy) and 5GiB large-object budgets. + +use crate::common::{ClusterTopology, RustFSTestClusterEnvironment}; +use serial_test::serial; + +type TestResult = Result<(), Box>; + +const BUCKET: &str = "cluster-multidrive-pool-bucket"; + +/// PUT an object then GET it back and assert the bytes round-trip. +async fn put_get_roundtrip(cluster: &RustFSTestClusterEnvironment, key: &str, payload: &[u8]) -> TestResult { + let writer = cluster.create_s3_client(0)?; + writer + .put_object() + .bucket(BUCKET) + .key(key) + .body(bytes::Bytes::copy_from_slice(payload).into()) + .send() + .await?; + + // Read back from the last node to exercise the cross-node/cross-pool path. + let reader = cluster.create_s3_client(cluster.nodes.len() - 1)?; + let got = reader.get_object().bucket(BUCKET).key(key).send().await?; + let body = got.body.collect().await?.into_bytes(); + assert_eq!(body.as_ref(), payload, "round-tripped object bytes must match"); + Ok(()) +} + +/// 4 nodes x 2 drives, single pool: the multi-drive layout boots and round-trips. +#[tokio::test] +#[serial] +async fn cluster_multidrive_single_pool_smoke() -> TestResult { + crate::common::init_logging(); + + let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, 2)).await?; + + // The single-pool multi-drive layout must list every (node, drive) endpoint + // explicitly (8 endpoints, no ellipses) so the server keeps one legacy pool. + let volumes = cluster.rustfs_volumes_arg(); + assert_eq!(volumes.split(' ').count(), 8, "expected 8 explicit endpoints, got: {volumes}"); + assert!(!volumes.contains('{'), "single-pool layout must not use ellipses: {volumes}"); + + cluster.start().await?; + cluster.create_test_bucket(BUCKET).await?; + + let payload = vec![0xA5u8; 512 * 1024]; + put_get_roundtrip(&cluster, "multidrive/object", &payload).await?; + Ok(()) +} + +/// Two single-node pools, 2 drives each: the multi-pool layout boots and +/// round-trips. Every pool is a distinct erasure pool (`pool_idx` 0 and 1). +#[tokio::test] +#[serial] +async fn cluster_two_pool_smoke() -> TestResult { + crate::common::init_logging(); + + let mut cluster = + RustFSTestClusterEnvironment::with_topology(ClusterTopology::per_node_pools(2, vec![vec![0], vec![1]])).await?; + + // The two-pool layout must emit one ellipses argument per pool. + let volumes = cluster.rustfs_volumes_arg(); + let args: Vec<&str> = volumes.split(' ').collect(); + assert_eq!(args.len(), 2, "expected two pool arguments, got: {volumes}"); + assert!( + args.iter().all(|a| a.contains("/drive{0...1}")), + "each pool arg must use the drive ellipses form: {volumes}" + ); + assert_eq!(cluster.nodes[0].pool_idx, 0); + assert_eq!(cluster.nodes[1].pool_idx, 1); + + cluster.start().await?; + cluster.create_test_bucket(BUCKET).await?; + + let payload = vec![0x5Au8; 256 * 1024]; + put_get_roundtrip(&cluster, "twopool/object", &payload).await?; + Ok(()) +} diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index 7689c87c4..c602227b1 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -717,15 +717,166 @@ pub async fn awscurl_delete( execute_awscurl(url, "DELETE", None, access_key, secret_key).await } +/// Cluster topology declaration for a `RustFSTestClusterEnvironment`. +/// +/// Describes how many nodes to launch, how many data drives each node exposes, +/// and how those nodes are grouped into erasure pools. The topology drives both +/// on-disk directory creation and the `RUSTFS_VOLUMES` string assembled for the +/// node processes. +/// +/// # Single-host expressibility constraints +/// +/// The harness runs every node on `127.0.0.1` with a distinct port, so the +/// `RUSTFS_VOLUMES` string can only use the two forms the server parser accepts +/// on a single machine (verified against `ecstore` `DisksLayout::from_volumes`): +/// +/// * **Single pool** — every `(node, drive)` endpoint listed explicitly, no +/// ellipses. This is the legacy form and yields exactly one pool spanning all +/// nodes and drives (`DistErasure`). Any `drives_per_node >= 1` works. +/// * **Multiple pools** — one ellipses arg per pool. A pool argument is a single +/// URL template, so it can enumerate several drives on *one* host but cannot +/// enumerate multiple distinct-port hosts. A pool that spanned two localhost +/// nodes would need a host ellipses (`127.0.0.1:900{0...1}`), which forces the +/// *same* on-disk path across those ports and collides physically. Therefore +/// every pool in a multi-pool topology owns exactly one node. In addition, the +/// parser rejects a single-drive ellipses pool (`drive{0...0}`), so a +/// multi-pool topology requires `drives_per_node >= 2`. +/// +/// Genuine multi-node pools (a pool striped across several hosts) require real +/// multi-host infrastructure and are deferred to the nightly cluster CI lane +/// (backlog #1313 / #1314); they are intentionally not expressible here. +#[derive(Clone, Debug)] +pub struct ClusterTopology { + /// Number of node processes to launch. + pub node_count: usize, + /// Number of independent data drives (directories) each node exposes. + pub drives_per_node: usize, + /// Pool membership as a list of node-index groups. An empty vector means a + /// single pool spanning every node. + pub pools: Vec>, +} + +impl ClusterTopology { + /// Single pool, one drive per node — identical to the historical + /// `RustFSTestClusterEnvironment::new` layout. + pub fn single_pool(node_count: usize) -> Self { + Self { + node_count, + drives_per_node: 1, + pools: Vec::new(), + } + } + + /// Single pool with `drives_per_node` drives on every node. The pool spans + /// all nodes and all drives (one `DistErasure` pool). + pub fn single_pool_multidrive(node_count: usize, drives_per_node: usize) -> Self { + Self { + node_count, + drives_per_node, + pools: Vec::new(), + } + } + + /// Multi-pool topology where every pool owns exactly one node. `pools` lists + /// the node index backing each pool (e.g. `vec![vec![0], vec![1]]` for a + /// two-pool cluster). Requires `drives_per_node >= 2` (see the type-level + /// note on single-host expressibility). + pub fn per_node_pools(drives_per_node: usize, pools: Vec>) -> Self { + let node_count = pools.iter().flatten().copied().max().map(|m| m + 1).unwrap_or(0); + Self { + node_count, + drives_per_node, + pools, + } + } + + /// Normalized pool membership: an empty `pools` becomes a single pool over + /// all node indices. + fn normalized_pools(&self) -> Vec> { + if self.pools.is_empty() { + vec![(0..self.node_count).collect()] + } else { + self.pools.clone() + } + } + + /// Validate that the topology is expressible on a single localhost machine. + fn validate(&self) -> Result<(), Box> { + if self.node_count == 0 { + return Err("Node count must be greater than zero".into()); + } + if self.drives_per_node == 0 { + return Err("drives_per_node must be greater than zero".into()); + } + + let pools = self.normalized_pools(); + + // Every referenced node index must be in range. + for (pool_idx, nodes) in pools.iter().enumerate() { + if nodes.is_empty() { + return Err(format!("pool {pool_idx} has no nodes").into()); + } + for &n in nodes { + if n >= self.node_count { + return Err(format!("pool {pool_idx} references node {n} but node_count is {}", self.node_count).into()); + } + } + } + + // Each node must belong to exactly one pool. + let mut seen = vec![0usize; self.node_count]; + for nodes in &pools { + for &n in nodes { + seen[n] += 1; + } + } + for (n, count) in seen.iter().enumerate() { + match count { + 0 => return Err(format!("node {n} is not assigned to any pool").into()), + 1 => {} + _ => return Err(format!("node {n} is assigned to more than one pool").into()), + } + } + + // Multi-pool constraints imposed by the single-host RUSTFS_VOLUMES syntax. + if pools.len() > 1 { + if self.drives_per_node < 2 { + return Err( + "multi-pool topology requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)" + .into(), + ); + } + for (pool_idx, nodes) in pools.iter().enumerate() { + if nodes.len() != 1 { + return Err(format!( + "pool {pool_idx} spans {} nodes; a pool striped across multiple localhost nodes is not expressible (needs host ellipses, which collides on shared paths). Use one node per pool, or the nightly multi-host lane (backlog #1313/#1314).", + nodes.len() + ) + .into()); + } + } + } + + Ok(()) + } +} + /// Represents a single RustFS server instance in a test cluster. /// /// Each `ClusterNode` tracks the node's network address, base URL for -/// S3-compatible requests, on-disk data directory, and the underlying +/// S3-compatible requests, on-disk data directories, and the underlying /// child process handle when the node is running. pub struct ClusterNode { pub address: String, pub url: String, + /// Primary data directory for the node. For single-drive nodes this is the + /// node's only drive; for multi-drive nodes it is the first drive. Kept as a + /// stable field so existing single-drive tests continue to compile. pub data_dir: String, + /// All data drives exposed by this node (`data_dirs[0] == data_dir`). + pub data_dirs: Vec, + /// Index of the pool this node belongs to. + pub pool_idx: usize, pub process: Option, } @@ -741,6 +892,7 @@ pub struct RustFSTestClusterEnvironment { pub access_key: String, pub secret_key: String, pub extra_env: Vec<(String, String)>, + pub topology: ClusterTopology, } impl RustFSTestClusterEnvironment { @@ -762,34 +914,82 @@ impl RustFSTestClusterEnvironment { /// * `Err(Box)` - An error if any step fails, such as temporary /// directory creation failure or available port lookup failure. pub async fn new(node_count: usize) -> Result> { - if node_count == 0 { - return Err("Node count must be greater than zero".into()); - } + Self::with_topology(ClusterTopology::single_pool(node_count)).await + } + + /// Create a RustFS test cluster environment from an explicit topology. + /// + /// Allocates a unique temporary root directory, an available TCP port per + /// node, and one data directory per drive. The topology is validated up front + /// (node/pool assignment, single-host multi-pool constraints); node processes + /// are not started at this stage. + /// + /// When the topology exposes more than one local drive on a node (multi-drive + /// or multi-pool layouts), `RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true` is added to + /// every node's environment: those drives live on the same temp filesystem, so + /// the server's distinct-physical-disk safety check would otherwise reject + /// startup. Single-drive single-pool clusters keep the historical environment + /// unchanged. + pub async fn with_topology(topology: ClusterTopology) -> Result> { + topology.validate()?; + let temp_dir = format!("/tmp/rustfs_cluster_test_{}", Uuid::new_v4()); fs::create_dir_all(&temp_dir).await?; - let mut nodes = Vec::with_capacity(node_count); - for i in 0..node_count { + // Map every node to its owning pool index. + let pools = topology.normalized_pools(); + let mut pool_of_node = vec![0usize; topology.node_count]; + for (pool_idx, nodes) in pools.iter().enumerate() { + for &n in nodes { + pool_of_node[n] = pool_idx; + } + } + + let multidrive = topology.drives_per_node > 1; + + let mut nodes = Vec::with_capacity(topology.node_count); + for (i, &pool_idx) in pool_of_node.iter().enumerate() { let port = RustFSTestEnvironment::find_available_port().await?; let address = format!("127.0.0.1:{}", port); let url = format!("http://{}", address); - let data_dir = format!("{}/node{}", temp_dir, i); - fs::create_dir_all(&data_dir).await?; + + // Single-drive nodes keep the historical `{temp}/node{i}` path so + // existing tests (and their on-disk assertions) are unaffected. + // Multi-drive nodes nest one `drive{d}` directory per drive so the + // ellipses form `drive{0...N-1}` can address them. + let data_dirs: Vec = if multidrive { + (0..topology.drives_per_node) + .map(|d| format!("{}/node{}/drive{}", temp_dir, i, d)) + .collect() + } else { + vec![format!("{}/node{}", temp_dir, i)] + }; + for dir in &data_dirs { + fs::create_dir_all(dir).await?; + } nodes.push(ClusterNode { address, url, - data_dir, + data_dir: data_dirs[0].clone(), + data_dirs, + pool_idx, process: None, }); } + let mut extra_env = Vec::new(); + if multidrive { + extra_env.push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string())); + } + Ok(Self { nodes, temp_dir, access_key: DEFAULT_ACCESS_KEY.to_string(), secret_key: DEFAULT_SECRET_KEY.to_string(), - extra_env: Vec::new(), + extra_env, + topology, }) } @@ -809,14 +1009,51 @@ impl RustFSTestClusterEnvironment { Ok(()) } - /// Build the volumes argument string for RustFS binary (internal helper method). + /// Build the `RUSTFS_VOLUMES` argument string for the cluster's topology. /// - /// Concatenates the address and data directory of all cluster nodes into a single string - /// used as the `RUSTFS_VOLUMES` environment variable for RustFS node processes. + /// * **Single pool** — every `(node, drive)` endpoint is listed explicitly and + /// space-joined. With no ellipses the server parser collapses them into one + /// legacy pool spanning all nodes and drives. + /// * **Multiple pools** — each pool (exactly one node) contributes one ellipses + /// argument `http:///drive{0...N-1}`; the space-separated + /// arguments become one pool each. + /// + /// The server splits `RUSTFS_VOLUMES` on spaces (`value_delimiter = ' '`), which + /// this assembly matches, and the resulting layout is verified against the + /// `ecstore` parser in the unit tests below. + /// Public view of the assembled `RUSTFS_VOLUMES` string, so tests can assert + /// the pool/drive layout without starting node processes. + pub fn rustfs_volumes_arg(&self) -> String { + self.build_volumes_arg() + } + fn build_volumes_arg(&self) -> String { - self.nodes + let pools = self.topology.normalized_pools(); + + if pools.len() <= 1 { + // Single pool: explicit enumeration of every drive on every node. + return self + .nodes + .iter() + .flat_map(|n| n.data_dirs.iter().map(move |dir| format!("http://{}{}", n.address, dir))) + .collect::>() + .join(" "); + } + + // Multi-pool: one ellipses argument per single-node pool. The drive + // directories are `/node{i}/drive{0..N-1}`, so the ellipses base is + // the shared parent of the node's drives. + pools .iter() - .map(|n| format!("http://{}{}", n.address, n.data_dir)) + .map(|nodes| { + let node = &self.nodes[nodes[0]]; + let base = node + .data_dirs + .first() + .and_then(|d| d.rsplit_once('/').map(|(parent, _)| parent)) + .unwrap_or(&node.data_dir); + format!("http://{}{}/drive{{0...{}}}", node.address, base, self.topology.drives_per_node - 1) + }) .collect::>() .join(" ") } @@ -1091,4 +1328,130 @@ mod tests { stdfs::remove_file(stamp_path).ok(); } + + /// Build a cluster environment struct in-memory (no ports, no processes) so + /// that `build_volumes_arg` can be exercised as a pure string builder. Node + /// directories mirror what `with_topology` would create for the topology. + fn fake_cluster(topology: ClusterTopology) -> RustFSTestClusterEnvironment { + let temp_dir = "/tmp/rustfs_cluster_test_FAKE".to_string(); + let pools = topology.normalized_pools(); + let mut pool_of_node = vec![0usize; topology.node_count]; + for (pool_idx, nodes) in pools.iter().enumerate() { + for &n in nodes { + pool_of_node[n] = pool_idx; + } + } + let multidrive = topology.drives_per_node > 1; + + let nodes = (0..topology.node_count) + .map(|i| { + let address = format!("127.0.0.1:{}", 9000 + i); + let data_dirs: Vec = if multidrive { + (0..topology.drives_per_node) + .map(|d| format!("{}/node{}/drive{}", temp_dir, i, d)) + .collect() + } else { + vec![format!("{}/node{}", temp_dir, i)] + }; + ClusterNode { + url: format!("http://{}", address), + address, + data_dir: data_dirs[0].clone(), + data_dirs, + pool_idx: pool_of_node[i], + process: None, + } + }) + .collect(); + + RustFSTestClusterEnvironment { + nodes, + temp_dir, + access_key: DEFAULT_ACCESS_KEY.to_string(), + secret_key: DEFAULT_SECRET_KEY.to_string(), + extra_env: Vec::new(), + topology, + } + } + + #[test] + fn volumes_single_pool_single_drive_is_backward_compatible() { + // The historical layout: one explicit endpoint per node, space-joined, + // no ellipses, no `drive` sub-directory. + let env = fake_cluster(ClusterTopology::single_pool(4)); + assert_eq!( + env.build_volumes_arg(), + "http://127.0.0.1:9000/tmp/rustfs_cluster_test_FAKE/node0 \ + http://127.0.0.1:9001/tmp/rustfs_cluster_test_FAKE/node1 \ + http://127.0.0.1:9002/tmp/rustfs_cluster_test_FAKE/node2 \ + http://127.0.0.1:9003/tmp/rustfs_cluster_test_FAKE/node3" + ); + } + + #[test] + fn volumes_single_pool_multidrive_enumerates_every_drive() { + // 4 nodes x 2 drives -> 8 explicit endpoints, one legacy pool. No + // ellipses, so the server parser keeps this as a single DistErasure pool. + let env = fake_cluster(ClusterTopology::single_pool_multidrive(4, 2)); + let expected = (0..4) + .flat_map(|i| { + (0..2).map(move |d| format!("http://127.0.0.1:{}/tmp/rustfs_cluster_test_FAKE/node{}/drive{}", 9000 + i, i, d)) + }) + .collect::>() + .join(" "); + assert_eq!(env.build_volumes_arg(), expected); + assert_eq!(env.build_volumes_arg().split(' ').count(), 8); + } + + #[test] + fn volumes_two_pool_uses_one_ellipses_arg_per_pool() { + // Two single-node pools, 2 drives each -> two ellipses arguments. The + // server parser treats each space-separated ellipses arg as its own pool. + let env = fake_cluster(ClusterTopology::per_node_pools(2, vec![vec![0], vec![1]])); + assert_eq!( + env.build_volumes_arg(), + "http://127.0.0.1:9000/tmp/rustfs_cluster_test_FAKE/node0/drive{0...1} \ + http://127.0.0.1:9001/tmp/rustfs_cluster_test_FAKE/node1/drive{0...1}" + ); + } + + #[test] + fn topology_rejects_multi_node_pool() { + // A pool striped across two localhost nodes is not expressible. + let err = ClusterTopology::per_node_pools(2, vec![vec![0, 1], vec![2, 3]]) + .validate() + .unwrap_err() + .to_string(); + assert!(err.contains("not expressible"), "unexpected error: {err}"); + } + + #[test] + fn topology_rejects_single_drive_multi_pool() { + // The server parser rejects a single-drive ellipses pool (`drive{0...0}`). + let err = ClusterTopology::per_node_pools(1, vec![vec![0], vec![1]]) + .validate() + .unwrap_err() + .to_string(); + assert!(err.contains("drives_per_node >= 2"), "unexpected error: {err}"); + } + + #[test] + fn topology_rejects_unassigned_and_duplicated_nodes() { + // Node 2 is never assigned to a pool. + let mut t = ClusterTopology::single_pool_multidrive(3, 2); + t.pools = vec![vec![0], vec![1]]; + assert!(t.validate().unwrap_err().to_string().contains("not assigned")); + + // Node 0 assigned twice. + let mut t = ClusterTopology::single_pool_multidrive(2, 2); + t.pools = vec![vec![0], vec![0]]; + assert!(t.validate().unwrap_err().to_string().contains("more than one pool")); + } + + #[test] + fn topology_single_pool_accepts_any_drive_count() { + assert!(ClusterTopology::single_pool(4).validate().is_ok()); + assert!(ClusterTopology::single_pool_multidrive(4, 4).validate().is_ok()); + assert!(ClusterTopology::single_pool_multidrive(1, 1).validate().is_ok()); + } } diff --git a/crates/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs index 0c1dc4cec..fef5d6b91 100644 --- a/crates/e2e_test/src/lib.rs +++ b/crates/e2e_test/src/lib.rs @@ -151,6 +151,10 @@ mod object_lock; #[cfg(test)] mod cluster_concurrency_test; +// Multi-drive (drivesPerNode) and 2-pool cluster harness smoke tests +#[cfg(test)] +mod cluster_multidrive_pool_test; + // PutObject / MultipartUpload with checksum (Content-MD5, x-amz-checksum-*) #[cfg(test)] mod checksum_upload_test;