From 37bda24e1c0b57723a0b2b6390b6a4e6b174d118 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 7 Sep 2026 19:07:42 +0800 Subject: [PATCH] fix: reject unsupported pool expansion with actionable errors (#7360) * fix: reject unsupported pool expansion with actionable errors Report singleton-pool and persisted-topology constraints before misleading startup retries. Preserve single-node multi-drive admission and existing parity policies, and cover format preservation plus operator recovery guidance for issue #6186. * fix: keep pool layout errors typed Preserve actionable pool layout diagnostics without adding generic formatted errors. Tighten the shrink-only baseline and assert that both typed payloads survive the I/O boundary. --------- Co-authored-by: houseme --- README.md | 9 + README_ZH.md | 9 + crates/ecstore/src/config/storageclass.rs | 29 ++- crates/ecstore/src/error/mod.rs | 55 +++++ crates/ecstore/src/layout/disks_layout.rs | 161 ++++++++++++- crates/ecstore/src/layout/endpoints.rs | 35 +++ crates/ecstore/src/store/init.rs | 32 ++- crates/ecstore/src/store/init_format.rs | 263 +++++++++++++++++++++- docs/testing/README.md | 2 + docs/testing/pool-layout-compatibility.md | 107 +++++++++ scripts/error-other-format-baseline.txt | 3 +- 11 files changed, 677 insertions(+), 28 deletions(-) create mode 100644 docs/testing/pool-layout-compatibility.md diff --git a/README.md b/README.md index bbb9d4957..11301393b 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,15 @@ Star RustFS on GitHub and be instantly notified of new releases. ## Quickstart +> [!IMPORTANT] +> **Pool expansion notice:** +> +> - A single-node single-drive (SNSD) deployment is supported only as a standalone local path. It cannot expand in place or be added as a Pool. To move to a multi-drive topology, create a new deployment and migrate data through S3. +> - Keep an existing multi-drive Pool's endpoints and Erasure Set width unchanged; expand by appending a new Pool. With ellipsis-based expansion, every Pool argument must contain an ellipsis expression and expand to at least two drive endpoints. +> - Single-node multi-drive Pools and multi-node Pools with one drive per node are allowed, subject to valid Erasure Set geometry and EC settings; acceptance does not guarantee host-failure tolerance. +> +> These topology rules follow MinIO, but automatic parity selection differs between the projects. See the [Pool layout compatibility and regression tests](docs/testing/pool-layout-compatibility.md) before expanding a deployment. + To get started with RustFS, follow these steps: ### 1. One-click Installation (Option 1) diff --git a/README_ZH.md b/README_ZH.md index dcf6fcfcc..baf6dab80 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -89,6 +89,15 @@ RustFS 是一个基于 Rust 构建的高性能分布式对象存储系统。Rust ## 快速开始 +> [!IMPORTANT] +> **Pool 扩容 Notice：** +> +> - 单节点单盘（SNSD）部署仅支持使用本地路径独立运行，不支持原地扩容，也不能作为 Pool 加入集群。如需改为多盘拓扑，请创建新部署并通过 S3 迁移数据。 +> - 已有多盘 Pool 的端点和 Erasure Set 宽度应保持不变，扩容应追加新的 Pool。使用省略号表达式扩容时，每个 Pool 参数都必须包含省略号表达式，并展开为至少两个磁盘端点。 +> - 允许单节点多盘 Pool，也允许多节点、每节点一盘的 Pool，但必须满足 Erasure Set 布局和 EC 配置要求；配置合法不代表能够容忍整台主机故障。 +> +> 这些拓扑规则与 MinIO 一致，但两者的默认 parity 选择方式存在差异。扩容前请阅读 [Pool 布局兼容性与回归测试说明](docs/testing/pool-layout-compatibility.md)。 + 请按照以下步骤快速上手 RustFS： ### 1. 一键安装脚本 (选项 1) diff --git a/crates/ecstore/src/config/storageclass.rs b/crates/ecstore/src/config/storageclass.rs index 751b88340..52e72c6b5 100644 --- a/crates/ecstore/src/config/storageclass.rs +++ b/crates/ecstore/src/config/storageclass.rs @@ -747,18 +747,27 @@ mod tests { let mut kvs = KVS::new(); kvs.insert(CLASS_STANDARD.to_string(), "EC:2".to_string()); - let err = lookup_config_for_pools_with_env(&kvs, &[4, 2], no_env_overrides()) - .expect_err("EC:2 must be rejected by the two-drive pool"); - assert!( - err.to_string().contains("pool 1") && err.to_string().contains("2 drives"), - "error must identify the rejecting pool: {err}" - ); + for drives in [2, 3] { + let err = lookup_config_for_pools_with_env(&kvs, &[4, drives], no_env_overrides()) + .expect_err("EC:2 must be rejected by a pool with fewer than four drives per set"); + assert!( + err.to_string().contains("pool 1") && err.to_string().contains(&format!("{drives} drives")), + "error must identify the rejecting pool: {err}" + ); + } + + let cfg = + lookup_config_for_pools_with_env(&kvs, &[4, 4], no_env_overrides()).expect("EC:2 is valid for both four-drive pools"); + assert_eq!(cfg.parities_for_sc(STANDARD), Some(vec![2, 2])); kvs.insert(CLASS_STANDARD.to_string(), "EC:1".to_string()); - let cfg = lookup_config_for_pools_with_env(&kvs, &[4, 2], no_env_overrides()).expect("EC:1 is valid for both pools"); - assert_eq!(cfg.parity_for_sc(STANDARD, 4), Some(1)); - assert_eq!(cfg.parity_for_sc(STANDARD, 2), Some(1)); - assert_eq!(cfg.get_parity_for_sc(STANDARD), Some(1)); + for drives in [2, 3, 4] { + let cfg = + lookup_config_for_pools_with_env(&kvs, &[4, drives], no_env_overrides()).expect("EC:1 is valid for both pools"); + assert_eq!(cfg.parity_for_sc(STANDARD, 4), Some(1)); + assert_eq!(cfg.parity_for_sc(STANDARD, drives), Some(1)); + assert_eq!(cfg.get_parity_for_sc(STANDARD), Some(1)); + } } #[test] diff --git a/crates/ecstore/src/error/mod.rs b/crates/ecstore/src/error/mod.rs index 918774c59..ec601df32 100644 --- a/crates/ecstore/src/error/mod.rs +++ b/crates/ecstore/src/error/mod.rs @@ -203,6 +203,19 @@ pub enum StorageError { NotFirstDisk, #[error("first disk wait")] FirstDiskWait, + #[error( + "unsupported pool expansion: an existing single-node single-drive (SNSD) deployment cannot be expanded in place (configured {configured_drives} drive endpoints); restart with the original single local path, or create a new multi-drive deployment and migrate data through S3" + )] + UnsupportedSnsdExpansion { configured_drives: usize }, + #[error( + "pool topology mismatch: stored {stored_drives} drives with {stored_set_drive_count} drives per erasure set, configured {configured_drives} drives with {configured_set_drive_count} drives per erasure set; an existing pool's drive count and erasure set width cannot be changed in place; restore its original endpoints and RUSTFS_ERASURE_SET_DRIVE_COUNT setting; to expand a multi-drive deployment, append a new pool with at least 2 drive endpoints" + )] + PoolTopologyMismatch { + stored_drives: usize, + stored_set_drive_count: usize, + configured_drives: usize, + configured_set_drive_count: usize, + }, // ── Operational ────────────────────────────────────────────────── #[error("Storage reached its minimum free drive threshold.")] @@ -629,6 +642,20 @@ impl Clone for StorageError { StorageError::ErasureWriteQuorum => StorageError::ErasureWriteQuorum, StorageError::NotFirstDisk => StorageError::NotFirstDisk, StorageError::FirstDiskWait => StorageError::FirstDiskWait, + StorageError::UnsupportedSnsdExpansion { configured_drives } => StorageError::UnsupportedSnsdExpansion { + configured_drives: *configured_drives, + }, + StorageError::PoolTopologyMismatch { + stored_drives, + stored_set_drive_count, + configured_drives, + configured_set_drive_count, + } => StorageError::PoolTopologyMismatch { + stored_drives: *stored_drives, + stored_set_drive_count: *stored_set_drive_count, + configured_drives: *configured_drives, + configured_set_drive_count: *configured_set_drive_count, + }, StorageError::TooManyOpenFiles => StorageError::TooManyOpenFiles, StorageError::NoHealRequired => StorageError::NoHealRequired, StorageError::Lock(e) => StorageError::Lock(e.clone()), @@ -735,6 +762,11 @@ impl StorageError { StorageError::ErasureWriteQuorum => StorageErrorCode::ErasureWriteQuorum, StorageError::NotFirstDisk => StorageErrorCode::NotFirstDisk, StorageError::FirstDiskWait => StorageErrorCode::FirstDiskWait, + // Topology diagnostics reuse the existing wire code; they are + // not disk errors and must retain their local identity for retry classification. + StorageError::UnsupportedSnsdExpansion { .. } | StorageError::PoolTopologyMismatch { .. } => { + StorageErrorCode::InvalidArgument + } StorageError::ConfigNotFound => StorageErrorCode::ConfigNotFound, StorageError::TooManyOpenFiles => StorageErrorCode::TooManyOpenFiles, StorageError::NoHealRequired => StorageErrorCode::NoHealRequired, @@ -1215,6 +1247,29 @@ mod tests { use super::*; use std::io::{Error as IoError, ErrorKind}; + #[test] + fn startup_topology_errors_preserve_identity_and_guidance() { + for error in [ + StorageError::UnsupportedSnsdExpansion { configured_drives: 4 }, + StorageError::PoolTopologyMismatch { + stored_drives: 4, + stored_set_drive_count: 4, + configured_drives: 8, + configured_set_drive_count: 8, + }, + ] { + let io_error: IoError = error.clone().into(); + let restored = StorageError::from(io_error); + assert_eq!(std::mem::discriminant(&restored), std::mem::discriminant(&error)); + assert_eq!(restored.to_string(), error.to_string()); + assert_eq!(restored.code(), StorageErrorCode::InvalidArgument); + assert!( + restored.narrow_to_disk().is_err(), + "startup diagnostics must not become disk/quorum errors" + ); + } + } + #[test] fn other_preserves_erasure_construction_source_chain() { use crate::erasure::coding::ErasureConstructionError; diff --git a/crates/ecstore/src/layout/disks_layout.rs b/crates/ecstore/src/layout/disks_layout.rs index 75ca09714..08c862f4e 100644 --- a/crates/ecstore/src/layout/disks_layout.rs +++ b/crates/ecstore/src/layout/disks_layout.rs @@ -25,6 +25,20 @@ pub(crate) const MAX_ERASURE_SET_DRIVE_COUNT: usize = 16; const SET_SIZES: [usize; 15] = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, MAX_ERASURE_SET_DRIVE_COUNT]; const ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT: &str = "RUSTFS_ERASURE_SET_DRIVE_COUNT"; +#[derive(Debug, thiserror::Error)] +enum PoolDriveCountError { + #[error( + "Incorrect number of endpoints provided, size {size}; an erasure pool requires at least {} drive endpoints on one or more nodes; for a standalone single-drive deployment, use a single local path without ellipses", + SET_SIZES[0] + )] + BelowMinimum { size: usize }, + #[error( + "Incorrect number of endpoints provided, size {size}; {}={set_drive_count} requires at least {set_drive_count} drive endpoints per pool", + ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT + )] + BelowSetWidth { size: usize, set_drive_count: usize }, +} + #[derive(Deserialize, Debug, Default)] pub struct PoolDisksLayout { cmd_line: String, @@ -132,7 +146,7 @@ impl DisksLayout { for arg in args.iter() { if !has_ellipses(&[arg]) && args.len() > 1 { return Err(Error::other( - "all args must have ellipses for pool expansion (Invalid arguments specified)", + "all args must have ellipses for pool expansion (Invalid arguments specified); each pool must expand to at least 2 drive endpoints on one or more nodes; a single-drive pool cannot be added to a multi-pool deployment", )); } @@ -396,9 +410,11 @@ fn get_set_indexes>( } for &size in total_sizes { - // Check if total_sizes has minimum range upto set_size - if size < SET_SIZES[0] || size < set_drive_count { - return Err(Error::other(format!("Incorrect number of endpoints provided, size {size}"))); + if size < SET_SIZES[0] { + return Err(Error::other(PoolDriveCountError::BelowMinimum { size })); + } + if size < set_drive_count { + return Err(Error::other(PoolDriveCountError::BelowSetWidth { size, set_drive_count })); } } @@ -707,7 +723,7 @@ mod test { arg: "http://rustfs{2...3}/export/set{1...0}", ..Default::default() }, - // Range cannot be smaller than 4 minimum. + // Ranges must use three dots. TestCase { num: 4, arg: "/export{1..2}", @@ -926,11 +942,146 @@ mod test { } } + #[test] + fn pool_expansion_accepts_single_node_multi_drive_pools() { + temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || { + for (volumes, drives) in [ + (["http://node1:9000/data{1...2}", "http://node2:9000/data{1...2}"], 2), + (["http://node1:9000/data{1...4}", "http://node2:9000/data{1...4}"], 4), + (["http://node{1...4}:9000/data", "http://node5:9000/data{1...4}"], 4), + (["http://node5:9000/data{1...4}", "http://node{1...4}:9000/data"], 4), + ] { + let layout = DisksLayout::from_volumes(&volumes).expect("single-node multi-drive pools are valid"); + + assert!(!layout.legacy); + assert_eq!(layout.pools.len(), 2); + for (index, volume) in volumes.iter().enumerate() { + assert_eq!(layout.get_set_count(index), 1); + assert_eq!(layout.get_drives_per_set(index), drives); + assert_eq!(layout.get_cmd_line(index), *volume); + } + } + }); + } + + #[test] + fn pool_expansion_accepts_multi_node_single_drive_pools() { + temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || { + for nodes in [2, 3, 4] { + let volumes = [ + format!("http://pool1-node{{1...{nodes}}}:9000/data"), + format!("http://pool2-node{{1...{nodes}}}:9000/data"), + ]; + let layout = DisksLayout::from_volumes(&volumes).expect("each node may contribute one drive to a pool"); + + assert_eq!(layout.pools.len(), 2); + for pool in 0..2 { + assert_eq!(layout.get_set_count(pool), 1); + assert_eq!(layout.get_drives_per_set(pool), nodes); + let expected = (1..=nodes) + .map(|node| format!("http://pool{}-node{node}:9000/data", pool + 1)) + .collect::>(); + assert_eq!(layout.pools[pool].layout, vec![expected]); + } + } + }); + } + + #[test] + fn explicit_endpoints_without_ellipses_form_one_pool() { + temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || { + let volumes = ["http://node1:9000/data", "http://node2:9000/data"]; + let layout = DisksLayout::from_volumes(&volumes).expect("explicit endpoints form one legacy pool"); + + assert!(layout.legacy); + assert_eq!(layout.pools.len(), 1); + assert_eq!(layout.pools[0].layout, vec![volumes.to_vec()]); + }); + } + + #[test] + fn standalone_single_drive_path_remains_supported() { + temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || { + let layout = DisksLayout::from_volumes(&["/data"]).expect("standalone single-drive deployment is valid"); + + assert!(layout.is_single_drive_layout()); + assert_eq!(layout.get_single_drive_layout(), "/data"); + }); + } + + #[test] + fn pool_expansion_rejects_plain_single_drive_pool_with_notice() { + temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || { + for volumes in [ + ["http://node{1...2}:9000/data", "http://node3:9000/data"], + ["http://node3:9000/data", "http://node{1...2}:9000/data"], + ] { + let err = DisksLayout::from_volumes(&volumes).expect_err("a plain endpoint cannot be an expansion pool"); + let message = err.to_string(); + + assert!(message.contains("all args must have ellipses for pool expansion"), "{message}"); + assert!(message.contains("at least 2 drive endpoints"), "{message}"); + } + }); + } + + #[test] + fn pool_expansion_rejects_singleton_ellipsis_pool_with_notice() { + temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || { + for singleton in ["http://node{3...3}:9000/data", "http://node3:9000/data{1...1}"] { + for volumes in [ + vec!["http://node{1...2}:9000/data", singleton], + vec![singleton, "http://node{1...2}:9000/data"], + vec![singleton], + ] { + let err = DisksLayout::from_volumes(&volumes).expect_err("a singleton range still contains one drive"); + let message = err.to_string(); + + assert_eq!(err.kind(), std::io::ErrorKind::Other); + assert!(matches!( + err.get_ref().and_then(|source| source.downcast_ref::()), + Some(PoolDriveCountError::BelowMinimum { size: 1 }) + )); + assert!(message.contains("at least 2 drive endpoints"), "{message}"); + assert!(message.contains("single local path without ellipses"), "{message}"); + } + } + }); + } + + #[test] + fn explicit_set_size_counts_drives_not_nodes() { + for volume in ["http://node1:9000/data{1...4}", "http://node{1...4}:9000/data"] { + let sets = get_all_sets(2, true, &[volume]).expect("four endpoints can form two two-drive sets"); + assert_eq!(sets.iter().map(Vec::len).collect::>(), vec![2, 2]); + } + } + + #[test] + fn undersized_pool_error_identifies_requested_set_size() { + let err = + get_all_sets(4, true, &["http://node{1...2}:9000/data"]).expect_err("two endpoints cannot fill a four-drive set"); + let message = err.to_string(); + + assert_eq!(err.kind(), std::io::ErrorKind::Other); + assert!(matches!( + err.get_ref().and_then(|source| source.downcast_ref::()), + Some(PoolDriveCountError::BelowSetWidth { + size: 2, + set_drive_count: 4 + }) + )); + assert!(message.contains("size 2"), "{message}"); + assert!(message.contains("RUSTFS_ERASURE_SET_DRIVE_COUNT=4"), "{message}"); + } + #[test] fn layout_errors_do_not_echo_url_credentials() { for volumes in [ vec!["http://:duplicate-secret@server/path", "http://:duplicate-secret@server/path"], vec!["http://:ellipsis...secret@server/path"], + vec!["http://server{1...2}/data", "http://:plain-secret@server3/data"], + vec!["http://server{1...2}/data", "http://:singleton-secret@server{3...3}/data"], ] { let err = DisksLayout::from_volumes(&volumes).unwrap_err(); assert!(!err.to_string().contains("secret"), "layout error leaked endpoint credentials: {err}"); diff --git a/crates/ecstore/src/layout/endpoints.rs b/crates/ecstore/src/layout/endpoints.rs index 3212e3822..1d9ad5faa 100644 --- a/crates/ecstore/src/layout/endpoints.rs +++ b/crates/ecstore/src/layout/endpoints.rs @@ -2432,6 +2432,41 @@ mod test { assert_eq!(local_endpoints[0].pool_idx, 1); } + #[tokio::test] + async fn pool_expansion_resolves_single_node_multi_drive_and_multi_node_single_drive_pools() { + for (additional_pool, expected_nodes) in [ + ("http://rustfs-5.example.invalid:9000/data{1...4}", 5), + ("http://rustfs-{5...8}.example.invalid:9000/data", 8), + ] { + let layout = temp_env::with_var("RUSTFS_ERASURE_SET_DRIVE_COUNT", Some("0"), || { + DisksLayout::from_volumes(&["http://rustfs-{1...4}.example.invalid:9000/data", additional_pool]) + }) + .expect("both single-node multi-drive and multi-node single-drive pools should parse"); + + let (pools, setup_type) = EndpointServerPools::create_server_endpoints_with( + "0.0.0.0:9000", + &layout, + Some(orchestrated_test_policy()), + Some("rustfs-1.example.invalid"), + ) + .await + .expect("pool admission must not impose a minimum node count or drives per node"); + + assert_eq!(setup_type, SetupType::DistErasure); + assert_eq!(pools.0.len(), 2); + assert_eq!(pools.get_nodes().len(), expected_nodes); + for (pool_index, pool) in (0_i32..).zip(&pools.0) { + assert_eq!((pool.set_count, pool.drives_per_set), (1, 4)); + assert_eq!(pool.endpoints.as_ref().len(), 4); + for (disk_index, endpoint) in (0_i32..).zip(pool.endpoints.as_ref()) { + assert_eq!(endpoint.pool_idx, pool_index); + assert_eq!(endpoint.set_idx, 0); + assert_eq!(endpoint.disk_idx, disk_index); + } + } + } + } + #[tokio::test] async fn explicit_local_endpoint_host_fails_closed_for_invalid_context_or_zero_match() { let args = vec![ diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index ef4089da2..6e472ce80 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -103,7 +103,10 @@ const REBALANCE_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(10); const REBALANCE_RESUME_RETRY_DELAY: Duration = Duration::from_secs(10); fn should_retry_format_load(err: &Error) -> bool { - !matches!(err, Error::CorruptedFormat) + !matches!( + err, + Error::CorruptedFormat | Error::UnsupportedSnsdExpansion { .. } | Error::PoolTopologyMismatch { .. } + ) } fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_resume_required: bool) -> bool { @@ -1784,6 +1787,33 @@ mod tests { assert!(should_retry_format_load(&StorageError::FirstDiskWait)); } + #[test] + fn test_should_retry_format_load_rejects_permanent_topology_errors() { + for error in [ + StorageError::UnsupportedSnsdExpansion { configured_drives: 4 }, + StorageError::PoolTopologyMismatch { + stored_drives: 4, + stored_set_drive_count: 4, + configured_drives: 8, + configured_set_drive_count: 8, + }, + ] { + assert!(!should_retry_format_load(&error), "topology errors require operator action: {error}"); + } + for error in [ + StorageError::DiskNotFound, + StorageError::Timeout, + StorageError::RemoteNotInitialized, + StorageError::NotFirstDisk, + StorageError::other(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)), + ] { + assert!( + should_retry_format_load(&error), + "transient failures retain their existing retry path: {error}" + ); + } + } + #[test] fn test_should_auto_start_rebalance_after_init_allows_active_rebalance_without_decommission() { assert!(should_auto_start_rebalance_after_init(false, true)); diff --git a/crates/ecstore/src/store/init_format.rs b/crates/ecstore/src/store/init_format.rs index c61059166..9bfb62c60 100644 --- a/crates/ecstore/src/store/init_format.rs +++ b/crates/ecstore/src/store/init_format.rs @@ -109,6 +109,21 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx( let fresh_bootstrap_proven = should_init_erasure_disks(&errs); let formats_present = formats.iter().flatten().count(); let mut format_quorum = (formats_present > 0).then(|| select_format_erasure_in_quorum(&formats, 0)); + // A resized pool may never reach quorum under its new endpoint count. + // Diagnose a valid, unambiguous stored layout before migration or waiting. + // A healthy quorum still takes precedence over foreign minority formats; + // conflicting or malformed observations retain their existing error path. + if format_quorum.as_ref().is_some_and(Result::is_err) + && let Some(reference) = formats.iter().flatten().next() + && formats.iter().flatten().all(|format| { + format.shared_identity() == reference.shared_identity() + && reference.erasure.sets.iter().flatten().any(|id| *id == format.erasure.this) + }) + && let Err(err @ (Error::UnsupportedSnsdExpansion { .. } | Error::PoolTopologyMismatch { .. })) = + check_format_erasure_value_for_topology(reference, formats.len(), set_drive_count) + { + return Err(err); + } if format_quorum.as_ref().is_none_or(Result::is_err) && errs.iter().any(|error| { matches!( @@ -661,15 +676,18 @@ fn check_format_erasure_value_for_topology(format: &FormatV3, format_count: usiz .len() .checked_mul(set_drive_count_in_format) .ok_or_else(|| Error::other("erasure set drive count overflow"))?; - if format_count != format_drive_count { - return Err(Error::other(format!( - "formats length for erasure.sets does not match: got {format_count}, expected {format_drive_count}" - ))); + if format_drive_count == 1 && format_count > 1 { + return Err(Error::UnsupportedSnsdExpansion { + configured_drives: format_count, + }); } - if set_drive_count_in_format != set_drive_count { - return Err(Error::other(format!( - "erasure set length for set_drive_count does not match: got {set_drive_count_in_format}, expected {set_drive_count}" - ))); + if format_count != format_drive_count || set_drive_count_in_format != set_drive_count { + return Err(Error::PoolTopologyMismatch { + stored_drives: format_drive_count, + stored_set_drive_count: set_drive_count_in_format, + configured_drives: format_count, + configured_set_drive_count: set_drive_count, + }); } Ok(()) } @@ -877,6 +895,10 @@ mod tests { use serial_test::serial; async fn local_disks(count: usize) -> (tempfile::TempDir, Vec>) { + local_disks_with_set_width(count, count).await + } + + async fn local_disks_with_set_width(count: usize, set_width: usize) -> (tempfile::TempDir, Vec>) { let temp_dir = tempfile::tempdir().expect("temporary disk root should be created"); let mut endpoints = Vec::with_capacity(count); for disk_index in 0..count { @@ -887,8 +909,8 @@ mod tests { let mut endpoint = Endpoint::try_from(path.to_str().expect("temporary disk path should be UTF-8")).expect("endpoint should parse"); endpoint.set_pool_index(0); - endpoint.set_set_index(0); - endpoint.set_disk_index(disk_index); + endpoint.set_set_index(disk_index / set_width); + endpoint.set_disk_index(disk_index % set_width); endpoints.push(endpoint); } @@ -912,6 +934,21 @@ mod tests { (temp_dir, disks) } + async fn format_bytes(disks: &[Option]) -> Vec>> { + let mut snapshots = Vec::with_capacity(disks.len()); + for disk in disks { + let disk = disk.as_ref().expect("snapshot disk should exist"); + // Inspect bytes even when the disk wrapper rejects a format whose + // stored slot differs from the attempted new endpoint geometry. + match tokio::fs::read(disk.path().join(RUSTFS_META_BUCKET).join(FORMAT_CONFIG_FILE)).await { + Ok(data) => snapshots.push(Some(data)), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => snapshots.push(None), + Err(err) => panic!("format snapshot failed: {err}"), + } + } + snapshots + } + async fn write_legacy_format(disk: &Option, format: &FormatV3) { write_legacy_bytes(disk, bytes::Bytes::from(format.to_json().expect("legacy format should serialize"))).await; } @@ -1116,6 +1153,212 @@ mod tests { ); } + #[tokio::test] + async fn single_drive_format_rejects_in_place_expansion_without_writes() { + for configured_drives in [2, 4] { + for first_disk in [false, true] { + let (_temp_dir, mut disks) = local_disks(configured_drives).await; + let mut original = FormatV3::new(1, 1); + original.erasure.this = original.erasure.sets[0][0]; + save_format_file(&disks[0], &Some(original)) + .await + .expect("SNSD format should be written"); + let before = format_bytes(&disks).await; + + let err = connect_load_init_formats(first_disk, &mut disks, 1, configured_drives, None) + .await + .expect_err("an existing SNSD deployment cannot expand in place"); + let message = err.to_string(); + assert!(message.contains("SNSD"), "expected a single-drive expansion error: {message}"); + assert!(message.contains("migrate data through S3"), "expected actionable guidance: {message}"); + assert_eq!(format_bytes(&disks).await, before, "neither old nor new formats may be written"); + } + } + } + + #[tokio::test] + async fn existing_pool_rejects_drive_count_or_set_width_changes_without_writes() { + for (stored_sets, stored_width, configured_sets, configured_width) in + [(1, 4, 1, 6), (1, 4, 1, 8), (1, 4, 1, 2), (1, 4, 2, 2), (2, 2, 1, 4)] + { + for first_disk in [false, true] { + let (_temp_dir, mut disks) = + local_disks_with_set_width(configured_sets * configured_width, configured_width).await; + let original = FormatV3::new(stored_sets, stored_width); + for (disk, disk_id) in disks.iter().zip(original.erasure.sets.iter().flatten()) { + let mut format = original.clone(); + format.erasure.this = *disk_id; + save_format_file(disk, &Some(format)) + .await + .expect("existing format should be written"); + } + let before = format_bytes(&disks).await; + + let err = connect_load_init_formats(first_disk, &mut disks, configured_sets, configured_width, None) + .await + .expect_err("an existing pool's geometry is immutable"); + let message = err.to_string(); + assert!(message.contains("pool topology mismatch"), "expected a topology error: {message}"); + assert!( + message.contains(&format!("stored 4 drives with {stored_width} drives per erasure set")), + "expected stored geometry: {message}" + ); + assert!(message.contains("append a new pool"), "expected expansion guidance: {message}"); + assert_eq!(format_bytes(&disks).await, before, "rejection must not rewrite any format"); + } + } + } + + #[tokio::test] + async fn subquorum_existing_layout_with_missing_drives_is_not_expansion() { + let (_temp_dir, mut disks) = local_disks(1).await; + let mut original = FormatV3::new(1, 4); + original.erasure.this = original.erasure.sets[0][0]; + save_format_file(&disks[0], &Some(original)) + .await + .expect("existing format should be written"); + disks.extend([None, None, None]); + + for first_disk in [false, true] { + assert!(matches!( + connect_load_init_formats(first_disk, &mut disks, 1, 4, None).await, + Err(Error::ErasureReadQuorum) + )); + } + } + + #[tokio::test] + async fn conflicting_layouts_without_quorum_are_not_expansion_proof() { + let (_temp_dir, mut disks) = local_disks(2).await; + for (index, (disk, width)) in disks.iter().zip([4, 2]).enumerate() { + let mut format = FormatV3::new(1, width); + format.erasure.this = format.erasure.sets[0][index]; + save_format_file(disk, &Some(format)) + .await + .expect("existing format should be written"); + } + disks.extend([None, None]); + + let result = connect_load_init_formats(true, &mut disks, 1, 4, None).await; + assert!(matches!(result, Err(Error::ErasureReadQuorum)), "conflicting layout result: {result:?}"); + } + + #[tokio::test] + async fn existing_format_quorum_ignores_single_drive_outlier() { + let (_temp_dir, mut disks) = local_disks(3).await; + let majority = FormatV3::new(1, 3); + for (index, disk) in disks.iter().enumerate() { + // Slot zero lets the SNSD outlier pass the disk wrapper's own + // slot check, so quorum selection must exclude the parsed format. + let mut format = if index == 0 { FormatV3::new(1, 1) } else { majority.clone() }; + format.erasure.this = format.erasure.sets[0][index]; + save_format_file(disk, &Some(format)) + .await + .expect("existing format should be written"); + } + + let loaded = connect_load_init_formats(true, &mut disks, 1, 3, None) + .await + .expect("a foreign SNSD outlier must not block a healthy majority"); + assert_eq!(loaded.shared_identity(), majority.shared_identity()); + assert!(disks[0].is_none(), "the foreign single-drive format must be quarantined"); + } + + #[tokio::test] + async fn multi_drive_pool_expansion_preserves_existing_format() { + let (_original_dir, mut disks) = local_disks(4).await; + let (_new_dir, mut new_disks) = local_disks(4).await; + let original = connect_load_init_formats(true, &mut disks, 1, 4, None) + .await + .expect("original multi-drive pool should initialize"); + let before = format_bytes(&disks).await; + + let added = connect_load_init_formats(true, &mut new_disks, 1, 4, Some(original.id)) + .await + .expect("a new multi-drive pool should initialize with the existing deployment ID"); + assert_eq!(added.id, original.id); + assert_ne!(added.erasure.sets, original.erasure.sets); + assert_eq!(format_bytes(&disks).await, before); + assert_eq!( + connect_load_init_formats(true, &mut disks, 1, 4, Some(original.id)) + .await + .expect("the original pool should restart with unchanged geometry"), + original + ); + assert_eq!( + connect_load_init_formats(true, &mut new_disks, 1, 4, Some(original.id)) + .await + .expect("the new pool should restart with its own format"), + added + ); + } + + #[tokio::test] + async fn store_startup_rejects_pool_resize_before_retry_loop() { + use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints}; + use tokio_util::sync::CancellationToken; + + for (stored_width, configured_width) in [(1, 4), (4, 8)] { + let (_temp_dir, disks) = local_disks(configured_width).await; + let original = FormatV3::new(1, stored_width); + for (disk, disk_id) in disks.iter().zip(&original.erasure.sets[0]) { + let mut format = original.clone(); + format.erasure.this = *disk_id; + save_format_file(disk, &Some(format)) + .await + .expect("old format should be written"); + } + let before = format_bytes(&disks).await; + let endpoints = disks.iter().flatten().map(|disk| disk.endpoint()).collect::>(); + let pools = EndpointServerPools::from(vec![PoolEndpoints { + legacy: true, + set_count: 1, + drives_per_set: configured_width, + endpoints: Endpoints::from(endpoints), + cmd_line: "test-pool".to_string(), + platform: String::new(), + }]); + let shutdown = CancellationToken::new(); + let result = temp_env::async_with_vars( + [ + (storageclass::STANDARD_ENV, None::<&str>), + (storageclass::RRS_ENV, None::<&str>), + (storageclass::OPTIMIZE_ENV, None::<&str>), + (storageclass::INLINE_BLOCK_ENV, None::<&str>), + ], + tokio::time::timeout( + std::time::Duration::from_secs(5), + crate::store::ECStore::new_with_instance_ctx( + "127.0.0.1:0".parse().expect("test address"), + pools, + shutdown.clone(), + Arc::new(InstanceContext::new()), + ), + ), + ) + .await; + shutdown.cancel(); + let err = result + .expect("invalid topology must abort without the format retry backoff") + .expect_err("resize must fail"); + match stored_width { + 1 => assert!(matches!(err, Error::UnsupportedSnsdExpansion { configured_drives: 4 }), "{err}"), + _ => assert!( + matches!( + err, + Error::PoolTopologyMismatch { + stored_drives: 4, + configured_drives: 8, + .. + } + ), + "{err}" + ), + } + assert_eq!(format_bytes(&disks).await, before, "failed store startup must not write formats"); + } + } + #[tokio::test] async fn existing_format_load_rejects_conflicting_formats_without_a_majority() { let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await; diff --git a/docs/testing/README.md b/docs/testing/README.md index 138f5b874..daa3f22bc 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -25,6 +25,8 @@ The [scanner checkpoint fixture](scanner-checkpoint-fixture.md) diagnoses retain The [scanner cache cost profile](scanner-cache-cost.md) separates clone, subtree copy, encoding, and counted save costs without changing production cache behavior. +The [Pool layout compatibility reference](pool-layout-compatibility.md) defines the topology and EC regression matrix for single-drive, single-node multi-drive, and multi-node expansion pools. + ## Naming conventions ### Reserved test-name substrings (migration gate) diff --git a/docs/testing/pool-layout-compatibility.md b/docs/testing/pool-layout-compatibility.md new file mode 100644 index 000000000..813933a5b --- /dev/null +++ b/docs/testing/pool-layout-compatibility.md @@ -0,0 +1,107 @@ +# Pool Layout Compatibility and Regression Tests + +**Use this when:** configuring `RUSTFS_VOLUMES` for expansion, investigating issue #6186, or changing pool admission and its regression tests. +**Source of truth:** `DisksLayout::from_volumes` and `get_set_indexes` in `crates/ecstore/src/layout/disks_layout.rs`, `EndpointServerPools::create_server_endpoints` in `crates/ecstore/src/layout/endpoints.rs`, startup format validation in `crates/ecstore/src/store/init_format.rs`, and `lookup_config_for_pools` in `crates/ecstore/src/config/storageclass.rs`. Geometry and parity invariants are owned by [erasure-coding.md](../architecture/erasure-coding.md). + +## Notice: count drives, not just nodes + +An erasure pool requires at least two drive endpoints. There is no additional admission rule requiring two nodes per pool or two drives per node. A single-node multi-drive pool and a multi-node pool with one drive per node may both be valid. + +For command-line / `RUSTFS_VOLUMES` expansion: + +- If any volume argument contains an ellipsis expression, each argument describes a separate pool and must contain an ellipsis expression. Each pool must expand to at least two distinct drive endpoints and form a valid set layout. +- A singleton range such as `http://node{3...3}:9000/data` still describes only one drive. It cannot bypass the minimum drive count. +- Without ellipses, all explicit endpoints describe one pool, not one pool per endpoint. +- A single local path such as `/data` remains a supported standalone single-drive deployment. A single URL endpoint is not a valid standalone single-drive endpoint, and a single-drive pool cannot be appended to a multi-pool deployment. +- An initialized single-node single-drive (SNSD) deployment cannot expand in place by adding endpoints or pools. Create a new multi-drive deployment and migrate data through S3 instead. Increasing the capacity of its underlying filesystem is not a pool-topology expansion and adds no redundancy. +- An existing multi-drive pool's drive count and set width are immutable. Preserve its original endpoints and `RUSTFS_ERASURE_SET_DRIVE_COUNT` setting, then append a new pool. Changing `/data{1...4}` to `/data{1...8}` resizes the old pool; appending `/other-data{1...4}` creates a new one. +- Multi-drive sets contain 2 through 16 drives. A pool may contain multiple sets; 16 is not a limit on total drives in a pool. Set divisibility, automatic layout symmetry, duplicate endpoints, endpoint locality, physical-disk validation, and storage-class validation still apply. +- An explicit storage-class parity must fit every pool's set width: `parity <= drives_per_set / 2`, with `STANDARD parity >= RRS parity`. Do not silently lower an explicit parity to admit a smaller pool. + +Topology acceptance is not a high-availability guarantee. Losing the only host of a single-node pool loses access to every shard in that pool. With a two-drive set at `EC:1`, losing one drive leaves read quorum but not write quorum. Plan failure domains and quorum separately from admission. + +These are valid four-drive-per-set topology examples, subject to the remaining startup checks: + +```text +# Two pools, each with four nodes and one drive per node. +RUSTFS_VOLUMES="http://node{1...4}:9000/data http://node{5...8}:9000/data" + +# A four-node pool plus a single-node, four-drive pool. +RUSTFS_VOLUMES="http://node{1...4}:9000/data http://node5:9000/data{1...4}" +``` + +## Rejection and recovery + +Invalid single-drive expansion arguments fail during layout parsing. When a syntactically valid layout tries to resize an initialized pool, startup compares the stored format with the configured drive count and set width before initializing or migrating formats for that pool: + +- `UnsupportedSnsdExpansion` explains that SNSD cannot expand in place and directs the operator to restore the single local path or migrate through S3 to a new deployment. +- `PoolTopologyMismatch` reports stored and configured drive counts and set widths, and directs the operator to restore the original pool and append a new pool instead. + +These are permanent startup errors, not retryable quorum failures. Rejection does not rewrite the affected pool's old format or initialize its new drives. Do not delete `format.json` to bypass it. This is a per-pool check, not an atomic, read-only preflight across every pool in the deployment. + +A healthy format quorum remains authoritative; a foreign or malformed minority is quarantined as before. Without a quorum, an unambiguous, valid observed layout can identify a topology mismatch before the wait/retry path. Conflicting observed layouts are not treated as proof of expansion. Missing disks and transient network failures alone do not establish a topology change and retain their existing handling. + +## MinIO comparison boundary + +The reference is MinIO Community source at commit `7aac2a2c5b7c882e68c1ce017d8256be2feea27f`, not an unversioned claim about all MinIO products or releases: + +- [Endpoint expansion](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/endpoint-ellipses.go): `mergeDisksLayoutFromArgs` requires ellipses on every expansion argument, and `getSetIndexes` rejects fewer than two endpoints. +- [Endpoint admission](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/endpoint.go): `CreatePoolEndpoints` does not require two nodes per pool; its standalone single-drive special case requires a local path. +- [Pool initialization](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/erasure-server-pool.go): `newErasureServerPools` checks a common parity against every pool. +- [Storage preparation](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/prepare-storage.go) and [format validation](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/format-erasure.go): persisted drive counts and set widths must match the configured pool; format-layout errors are not ordinary quorum-wait conditions. RustFS keeps its existing majority/minority handling rather than adopting MinIO's all-format validation order. + +The node/drive admission rules above match this baseline. This reference does not claim complete startup or storage-class equivalence: + +- RustFS resolves automatic parity independently for each pool's set width. For widths `[4, 2]`, automatic STANDARD parity resolves to `[2, 1]`. MinIO uses a common parity, initially selected from the first pool when no value is configured, and rejects a later pool that cannot accommodate it. RustFS's existing automatic policy is not changed by these regression tests. +- An explicit STANDARD `EC:2` rejects a two- or three-drive set in RustFS; `EC:1` fits both. Explicit configuration is shared, not a user-configurable per-pool override. +- RustFS also checks symmetry when `RUSTFS_ERASURE_SET_DRIVE_COUNT` is explicitly set. The MinIO baseline skips automatic symmetry selection for an explicit set width. The topology tests below do not establish equivalence for every explicit-width layout. + +## Regression matrix + +Layout tests use symbolic endpoints and fixed set-count inputs. Startup tests use temporary local drives and the production format-loading path, comparing format bytes before and after rejection. They do not require production disks, DNS records, or a running MinIO server. Storage-class tests inject configuration directly rather than mutating the process environment. + +| Scenario | Expected result | Regression guard | +|---|---|---| +| Standalone `/data` | One single-drive layout | `standalone_single_drive_path_remains_supported` | +| Standalone single URL endpoint | Reject; single-drive mode requires a local path | `test_create_pool_endpoints` | +| Two explicit URLs, no ellipses | One pool containing both drives | `explicit_endpoints_without_ellipses_form_one_pool` | +| Two single-node pools, each with 2 or 4 drives | Two valid pools | `pool_expansion_accepts_single_node_multi_drive_pools` | +| Four-node, one-drive-per-node pool mixed with a single-node, four-drive pool, in either order | Both pool boundaries and set widths preserved | `pool_expansion_accepts_single_node_multi_drive_pools` | +| Two pools with 2, 3, or 4 nodes per pool and one drive per node | One set per pool; every drive retained in its pool | `pool_expansion_accepts_multi_node_single_drive_pools` | +| Ellipsis pool mixed with a plain single-drive endpoint, in either order | Reject with the ellipsis requirement and minimum-drive notice | `pool_expansion_rejects_plain_single_drive_pool_with_notice` | +| Singleton host or drive range, alone or before/after another pool | Reject with the minimum-drive notice and standalone-path guidance | `pool_expansion_rejects_singleton_ellipsis_pool_with_notice` | +| Four drives on one node or four nodes, explicit set width 2 | Two two-drive sets | `explicit_set_size_counts_drives_not_nodes` | +| Two-drive pool, explicit set width 4 | Reject and identify the requested set width | `undersized_pool_error_identifies_requested_set_size` | +| Credentials in rejected plain or singleton pool endpoints | Errors do not echo secrets | `layout_errors_do_not_echo_url_credentials` | +| Mixed single-node multi-drive / multi-node single-drive pools through endpoint resolution | Distributed setup, correct node count and pool/set/disk indices | `pool_expansion_resolves_single_node_multi_drive_and_multi_node_single_drive_pools` | +| Additional set width 2 or 3, explicit STANDARD `EC:2` | Reject and identify the incompatible pool | `explicit_standard_parity_is_validated_against_every_pool` | +| Set widths `[4, 4]` with `EC:2`, or `[4, 2/3/4]` with `EC:1` | Shared explicit parity accepted | `explicit_standard_parity_is_validated_against_every_pool` | +| Explicit environment STANDARD `EC:2`, widths `[4, 2]` | Reject; do not clamp parity | `explicit_environment_standard_parity_is_not_clamped` | +| Automatic parity, widths `[4, 2]` | Preserve RustFS's existing per-pool `[2, 1]` policy | `automatic_parity_is_resolved_per_pool` | +| Existing SNSD plus new drives, on first/non-first server | Reject with SNSD migration guidance; old format unchanged and new drives unformatted | `single_drive_format_rejects_in_place_expansion_without_writes` | +| Existing four-drive pool resized to 2, 6, or 8 drives, or regrouped between one four-drive set and two two-drive sets | Reject with stored/configured geometry and append-pool guidance; no format writes | `existing_pool_rejects_drive_count_or_set_width_changes_without_writes` | +| Existing four-drive pool with only one drive reachable | Retain quorum failure, not an expansion error | `subquorum_existing_layout_with_missing_drives_is_not_expansion` | +| Conflicting four-drive and two-drive formats without a quorum | Retain quorum failure; do not infer the original topology | `conflicting_layouts_without_quorum_are_not_expansion_proof` | +| Healthy three-drive majority with a foreign SNSD minority | Start with the majority and quarantine the outlier | `existing_format_quorum_ignores_single_drive_outlier` | +| New four-drive pool alongside an initialized four-drive pool | Preserve the deployment ID and original format; original pool restarts | `multi_drive_pool_expansion_preserves_existing_format` | +| Typed SNSD/topology errors versus missing-disk, network, and quorum errors | Only permanent topology/corruption errors bypass the format retry loop | `test_should_retry_format_load_rejects_permanent_topology_errors` | +| Full store startup, SNSD to four drives or four-drive pool to eight | Return the typed topology error before retry backoff; no format writes | `store_startup_rejects_pool_resize_before_retry_loop` | +| Startup topology error cloning and I/O wrapping | Retain error type and guidance; do not narrow into a disk/quorum error | `startup_topology_errors_preserve_identity_and_guidance` | + +Layout and endpoint guards live in the layout source files above; parity guards live in the storage-class module. The existing `test_get_set_indexes` and `test_into_endpoint_set` tables cover larger, multi-set layouts and malformed ranges. + +Run the focused crate tests: + +```bash +cargo nextest run -p rustfs-ecstore --lib \ + -E 'test(layout::disks_layout::) | test(layout::endpoints::) | test(config::storageclass::) | test(store::init_format::) | test(test_should_retry_format_load) | test(error::)' +``` + +## Runtime coverage + +Keep the existing single-node multi-drive pool scenarios. They are valid topologies, not exceptions that need a node-count bypass: + +- `cluster_two_pool_smoke` in `crates/e2e_test/src/cluster_multidrive_pool_test.rs` exercises real S3 traffic against two pools. +- `four_node_pool_expand_preserves_objects_then_rebalance` in `crates/e2e_test/src/distributed/expand_decommission_rebalance_test.rs` appends pools, verifies existing objects, restarts, and exercises rebalance. + +The localhost harness uses separate processes and ports; it does not prove independent physical-host failure tolerance. See [distributed-e2e.md](distributed-e2e.md) for the binary, filesystem, and execution requirements before running expansion tests. Parser and endpoint unit tests establish admission, not persistent-data migration safety or production availability. diff --git a/scripts/error-other-format-baseline.txt b/scripts/error-other-format-baseline.txt index 4a45bab07..613b4098d 100644 --- a/scripts/error-other-format-baseline.txt +++ b/scripts/error-other-format-baseline.txt @@ -35,7 +35,7 @@ 1|crates/ecstore/src/erasure/coding/decode_reader.rs 10|crates/ecstore/src/erasure/coding/encode.rs 25|crates/ecstore/src/erasure/coding/erasure.rs -4|crates/ecstore/src/layout/disks_layout.rs +3|crates/ecstore/src/layout/disks_layout.rs 2|crates/ecstore/src/layout/endpoint.rs 17|crates/ecstore/src/layout/endpoints.rs 1|crates/ecstore/src/layout/format.rs @@ -67,7 +67,6 @@ 5|crates/ecstore/src/store/bucket.rs 1|crates/ecstore/src/store/heal_walk.rs 12|crates/ecstore/src/store/init.rs -2|crates/ecstore/src/store/init_format.rs 3|crates/ecstore/src/store/multipart.rs 6|crates/ecstore/src/store/object.rs 5|crates/ecstore/src/store/rebalance/support.rs