From 16d667da0aa4edec88a1c1f3d25ea3dfdbbfbae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Sat, 20 Jun 2026 09:58:50 +0800 Subject: [PATCH] refactor: move ecstore format layout owners (#3646) --- crates/ecstore/src/disk/format.rs | 533 +---------- crates/ecstore/src/disks_layout.rs | 914 +----------------- crates/ecstore/src/layout/disks_layout.rs | 927 +++++++++++++++++++ crates/ecstore/src/layout/format.rs | 546 +++++++++++ crates/ecstore/src/layout/mod.rs | 2 + crates/ecstore/src/layout/set_layout.rs | 2 +- docs/architecture/ecstore-layout-boundary.md | 8 + docs/architecture/migration-progress.md | 53 +- 8 files changed, 1527 insertions(+), 1458 deletions(-) create mode 100644 crates/ecstore/src/layout/disks_layout.rs create mode 100644 crates/ecstore/src/layout/format.rs diff --git a/crates/ecstore/src/disk/format.rs b/crates/ecstore/src/disk/format.rs index 592f6f93b..47622a7d0 100644 --- a/crates/ecstore/src/disk/format.rs +++ b/crates/ecstore/src/disk/format.rs @@ -12,535 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::disk::error::{Error, Result}; -use crate::disk::{DiskInfo, error::DiskError}; -use serde::{Deserialize, Serialize}; -use serde_json::Error as JsonError; -use uuid::Uuid; - -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub enum FormatMetaVersion { - #[serde(rename = "1")] - V1, - - #[serde(other)] - Unknown, -} - -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub enum FormatBackend { - #[serde(rename = "xl")] - Erasure, - #[serde(rename = "xl-single")] - ErasureSingle, - - #[serde(other)] - Unknown, -} - -/// Represents the V3 backend disk structure version -/// under `.rustfs.sys` and actual data namespace. -/// -/// FormatErasureV3 - structure holds format config version '3'. -/// -/// The V3 format to support "large bucket" support where a bucket -/// can span multiple erasure sets. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct FormatErasureV3 { - /// Version of 'xl' format. - pub version: FormatErasureVersion, - - /// This field carries assigned disk uuid. - pub this: Uuid, - - /// Sets field carries the input disk order generated the first - /// time when fresh disks were supplied, it is a two-dimensional - /// array second dimension represents list of disks used per set. - pub sets: Vec>, - - /// Distribution algorithm represents the hashing algorithm - /// to pick the right set index for an object. - #[serde(rename = "distributionAlgo")] - pub distribution_algo: DistributionAlgoVersion, -} - -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub enum FormatErasureVersion { - #[serde(rename = "1")] - V1, - #[serde(rename = "2")] - V2, - #[serde(rename = "3")] - V3, - - #[serde(other)] - Unknown, -} - -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub enum DistributionAlgoVersion { - #[serde(rename = "CRCMOD")] - V1, - #[serde(rename = "SIPMOD")] - V2, - #[serde(rename = "SIPMOD+PARITY")] - V3, -} - -/// format.json currently has the format: -/// -/// ```json -/// { -/// "version": "1", -/// "format": "XXXXX", -/// "id": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX", -/// "XXXXX": { -// -/// } -/// } -/// ``` -/// -/// Ideally we will never have a situation where we will have to change the -/// fields of this struct and deal with related migration. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct FormatV3 { - /// Version of the format config. - pub version: FormatMetaVersion, - - /// Format indicates the backend format type, supports two values 'xl' and 'xl-single'. - pub format: FormatBackend, - - /// ID is the identifier for the rustfs deployment - pub id: Uuid, - - #[serde(rename = "xl")] - pub erasure: FormatErasureV3, - // /// DiskInfo is an extended type which returns current - // /// disk usage per path. - #[serde(skip)] - pub disk_info: Option, -} - -impl TryFrom<&[u8]> for FormatV3 { - type Error = JsonError; - - fn try_from(data: &[u8]) -> std::result::Result { - serde_json::from_slice(data) - } -} - -impl TryFrom<&str> for FormatV3 { - type Error = JsonError; - - fn try_from(data: &str) -> std::result::Result { - serde_json::from_str(data) - } -} - -impl FormatV3 { - /// Create a new format config with the given number of sets and set length. - pub fn new(num_sets: usize, set_len: usize) -> Self { - let format = if set_len == 1 { - FormatBackend::ErasureSingle - } else { - FormatBackend::Erasure - }; - - let erasure = FormatErasureV3 { - version: FormatErasureVersion::V3, - this: Uuid::nil(), - sets: (0..num_sets) - .map(|_| (0..set_len).map(|_| Uuid::new_v4()).collect()) - .collect(), - distribution_algo: DistributionAlgoVersion::V3, - }; - - Self { - version: FormatMetaVersion::V1, - format, - id: Uuid::new_v4(), - erasure, - disk_info: None, - } - } - - /// Returns the number of drives in the erasure set. - pub fn drives(&self) -> usize { - self.erasure.sets.iter().map(|v| v.len()).sum() - } - - pub fn to_json(&self) -> std::result::Result { - serde_json::to_string(self) - } - - /// returns the i,j'th position of the input `diskID` against the reference - /// - /// format, after successful validation. - /// - i'th position is the set index - /// - j'th position is the disk index in the current set - pub fn find_disk_index_by_disk_id(&self, disk_id: Uuid) -> Result<(usize, usize)> { - if disk_id == Uuid::nil() { - return Err(Error::from(DiskError::DiskNotFound)); - } - if disk_id == Uuid::max() { - return Err(Error::other("disk offline")); - } - - for (i, set) in self.erasure.sets.iter().enumerate() { - for (j, d) in set.iter().enumerate() { - if disk_id.eq(d) { - return Ok((i, j)); - } - } - } - - Err(Error::other(format!("disk id not found {disk_id}"))) - } - - pub fn check_other(&self, other: &FormatV3) -> Result<()> { - let mut tmp = other.clone(); - let this = tmp.erasure.this; - tmp.erasure.this = Uuid::nil(); - - if self.erasure.sets.len() != other.erasure.sets.len() { - return Err(Error::other(format!( - "Expected number of sets {}, got {}", - self.erasure.sets.len(), - other.erasure.sets.len() - ))); - } - - for i in 0..self.erasure.sets.len() { - if self.erasure.sets[i].len() != other.erasure.sets[i].len() { - return Err(Error::other(format!( - "Each set should be of same size, expected {}, got {}", - self.erasure.sets[i].len(), - other.erasure.sets[i].len() - ))); - } - - for j in 0..self.erasure.sets[i].len() { - if self.erasure.sets[i][j] != other.erasure.sets[i][j] { - return Err(Error::other(format!( - "UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)", - i, - j, - self.erasure.sets[i][j].to_string(), - other.erasure.sets[i][j].to_string(), - ))); - } - } - } - - for i in 0..tmp.erasure.sets.len() { - for j in 0..tmp.erasure.sets[i].len() { - if this == tmp.erasure.sets[i][j] { - return Ok(()); - } - } - } - - Err(Error::other(format!( - "DriveID {:?} not found in any drive sets {:?}", - this, other.erasure.sets - ))) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_format_v1() { - let format = FormatV3::new(1, 4); - - let str = serde_json::to_string(&format); - println!("{str:?}"); - - let data = r#" - { - "version": "1", - "format": "xl", - "id": "321b3874-987d-4c15-8fa5-757c956b1243", - "xl": { - "version": "1", - "this": null, - "sets": [ - [ - "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5", - "c26315da-05cf-4778-a9ea-b44ea09f58c5", - "fb87a891-18d3-44cf-a46f-bcc15093a038", - "356a925c-57b9-4313-88b3-053edf1104dc" - ] - ], - "distributionAlgo": "CRCMOD" - } - }"#; - - let p = FormatV3::try_from(data); - - println!("{p:?}"); - } - - #[test] - fn test_format_v3_new_single_disk() { - let format = FormatV3::new(1, 1); - - assert_eq!(format.version, FormatMetaVersion::V1); - assert_eq!(format.format, FormatBackend::ErasureSingle); - assert_eq!(format.erasure.version, FormatErasureVersion::V3); - assert_eq!(format.erasure.sets.len(), 1); - assert_eq!(format.erasure.sets[0].len(), 1); - assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3); - assert_eq!(format.erasure.this, Uuid::nil()); - } - - #[test] - fn test_format_v3_new_multiple_sets() { - let format = FormatV3::new(2, 4); - - assert_eq!(format.version, FormatMetaVersion::V1); - assert_eq!(format.format, FormatBackend::Erasure); - assert_eq!(format.erasure.version, FormatErasureVersion::V3); - assert_eq!(format.erasure.sets.len(), 2); - assert_eq!(format.erasure.sets[0].len(), 4); - assert_eq!(format.erasure.sets[1].len(), 4); - assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3); - } - - #[test] - fn test_format_v3_drives() { - let format = FormatV3::new(2, 4); - assert_eq!(format.drives(), 8); // 2 sets * 4 drives each - - let format_single = FormatV3::new(1, 1); - assert_eq!(format_single.drives(), 1); // 1 set * 1 drive - } - - #[test] - fn test_format_v3_to_json() { - let format = FormatV3::new(1, 2); - let json_result = format.to_json(); - - assert!(json_result.is_ok()); - let json_str = json_result.unwrap(); - assert!(json_str.contains("\"version\":\"1\"")); - assert!(json_str.contains("\"format\":\"xl\"")); - } - - #[test] - fn test_format_v3_from_json() { - let json_data = r#"{ - "version": "1", - "format": "xl-single", - "id": "321b3874-987d-4c15-8fa5-757c956b1243", - "xl": { - "version": "3", - "this": "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5", - "sets": [ - [ - "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5" - ] - ], - "distributionAlgo": "SIPMOD+PARITY" - } - }"#; - - let format = FormatV3::try_from(json_data); - assert!(format.is_ok()); - - let format = format.unwrap(); - assert_eq!(format.format, FormatBackend::ErasureSingle); - assert_eq!(format.erasure.version, FormatErasureVersion::V3); - assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3); - assert_eq!(format.erasure.sets.len(), 1); - assert_eq!(format.erasure.sets[0].len(), 1); - } - - #[test] - fn test_format_v3_from_bytes() { - let json_data = r#"{ - "version": "1", - "format": "xl", - "id": "321b3874-987d-4c15-8fa5-757c956b1243", - "xl": { - "version": "2", - "this": "00000000-0000-0000-0000-000000000000", - "sets": [ - [ - "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5", - "c26315da-05cf-4778-a9ea-b44ea09f58c5" - ] - ], - "distributionAlgo": "SIPMOD" - } - }"#; - - let format = FormatV3::try_from(json_data.as_bytes()); - assert!(format.is_ok()); - - let format = format.unwrap(); - assert_eq!(format.erasure.version, FormatErasureVersion::V2); - assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V2); - assert_eq!(format.erasure.sets[0].len(), 2); - } - - #[test] - fn test_format_v3_invalid_json() { - let invalid_json = r#"{"invalid": "json"}"#; - let format = FormatV3::try_from(invalid_json); - assert!(format.is_err()); - } - - #[test] - fn test_find_disk_index_by_disk_id() { - let mut format = FormatV3::new(2, 2); - let target_disk_id = Uuid::new_v4(); - format.erasure.sets[1][0] = target_disk_id; - - let result = format.find_disk_index_by_disk_id(target_disk_id); - assert!(result.is_ok()); - assert_eq!(result.unwrap(), (1, 0)); - } - - #[test] - fn test_find_disk_index_nil_uuid() { - let format = FormatV3::new(1, 2); - let result = format.find_disk_index_by_disk_id(Uuid::nil()); - assert!(result.is_err()); - assert!(matches!(result.unwrap_err(), Error::DiskNotFound)); - } - - #[test] - fn test_find_disk_index_max_uuid() { - let format = FormatV3::new(1, 2); - let result = format.find_disk_index_by_disk_id(Uuid::max()); - assert!(result.is_err()); - } - - #[test] - fn test_find_disk_index_not_found() { - let format = FormatV3::new(1, 2); - let non_existent_id = Uuid::new_v4(); - let result = format.find_disk_index_by_disk_id(non_existent_id); - assert!(result.is_err()); - } - - #[test] - fn test_check_other_identical() { - let format1 = FormatV3::new(2, 4); - let mut format2 = format1.clone(); - format2.erasure.this = format1.erasure.sets[0][0]; - - let result = format1.check_other(&format2); - assert!(result.is_ok()); - } - - #[test] - fn test_check_other_different_set_count() { - let format1 = FormatV3::new(2, 4); - let format2 = FormatV3::new(3, 4); - - let result = format1.check_other(&format2); - assert!(result.is_err()); - } - - #[test] - fn test_check_other_different_set_size() { - let format1 = FormatV3::new(2, 4); - let format2 = FormatV3::new(2, 6); - - let result = format1.check_other(&format2); - assert!(result.is_err()); - } - - #[test] - fn test_check_other_different_disk_id() { - let format1 = FormatV3::new(1, 2); - let mut format2 = format1.clone(); - format2.erasure.sets[0][0] = Uuid::new_v4(); - - let result = format1.check_other(&format2); - assert!(result.is_err()); - } - - #[test] - fn test_check_other_disk_not_in_sets() { - let format1 = FormatV3::new(1, 2); - let mut format2 = format1.clone(); - format2.erasure.this = Uuid::new_v4(); // Set to a UUID not in any set - - let result = format1.check_other(&format2); - assert!(result.is_err()); - } - - #[test] - fn test_format_meta_version_serialization() { - let v1 = FormatMetaVersion::V1; - let json = serde_json::to_string(&v1).unwrap(); - assert_eq!(json, "\"1\""); - - let unknown = FormatMetaVersion::Unknown; - let deserialized: FormatMetaVersion = serde_json::from_str("\"unknown\"").unwrap(); - assert_eq!(deserialized, unknown); - } - - #[test] - fn test_format_backend_serialization() { - let erasure = FormatBackend::Erasure; - let json = serde_json::to_string(&erasure).unwrap(); - assert_eq!(json, "\"xl\""); - - let single = FormatBackend::ErasureSingle; - let json = serde_json::to_string(&single).unwrap(); - assert_eq!(json, "\"xl-single\""); - - let unknown = FormatBackend::Unknown; - let deserialized: FormatBackend = serde_json::from_str("\"unknown\"").unwrap(); - assert_eq!(deserialized, unknown); - } - - #[test] - fn test_format_erasure_version_serialization() { - let v1 = FormatErasureVersion::V1; - let json = serde_json::to_string(&v1).unwrap(); - assert_eq!(json, "\"1\""); - - let v2 = FormatErasureVersion::V2; - let json = serde_json::to_string(&v2).unwrap(); - assert_eq!(json, "\"2\""); - - let v3 = FormatErasureVersion::V3; - let json = serde_json::to_string(&v3).unwrap(); - assert_eq!(json, "\"3\""); - } - - #[test] - fn test_distribution_algo_version_serialization() { - let v1 = DistributionAlgoVersion::V1; - let json = serde_json::to_string(&v1).unwrap(); - assert_eq!(json, "\"CRCMOD\""); - - let v2 = DistributionAlgoVersion::V2; - let json = serde_json::to_string(&v2).unwrap(); - assert_eq!(json, "\"SIPMOD\""); - - let v3 = DistributionAlgoVersion::V3; - let json = serde_json::to_string(&v3).unwrap(); - assert_eq!(json, "\"SIPMOD+PARITY\""); - } - - #[test] - fn test_format_v3_round_trip_serialization() { - let original = FormatV3::new(2, 3); - let json = original.to_json().unwrap(); - let deserialized = FormatV3::try_from(json.as_str()).unwrap(); - - assert_eq!(original.version, deserialized.version); - assert_eq!(original.format, deserialized.format); - assert_eq!(original.erasure.version, deserialized.erasure.version); - assert_eq!(original.erasure.sets.len(), deserialized.erasure.sets.len()); - assert_eq!(original.erasure.distribution_algo, deserialized.erasure.distribution_algo); - } -} +pub use crate::layout::format::*; diff --git a/crates/ecstore/src/disks_layout.rs b/crates/ecstore/src/disks_layout.rs index 2eba85c70..34fdd3729 100644 --- a/crates/ecstore/src/disks_layout.rs +++ b/crates/ecstore/src/disks_layout.rs @@ -12,916 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -use rustfs_utils::string::{ArgPattern, find_ellipses_patterns, has_ellipses}; -use serde::Deserialize; -use std::collections::HashSet; -use std::env; -use std::io::{Error, Result}; -use tracing::debug; - -/// Supported set sizes this is used to find the optimal -/// single set size. -const SET_SIZES: [usize; 15] = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; -const ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT: &str = "RUSTFS_ERASURE_SET_DRIVE_COUNT"; - -#[derive(Deserialize, Debug, Default)] -pub struct PoolDisksLayout { - cmd_line: String, - layout: Vec>, -} - -impl PoolDisksLayout { - fn new(args: impl Into, layout: Vec>) -> Self { - PoolDisksLayout { - cmd_line: args.into(), - layout, - } - } - - fn count(&self) -> usize { - self.layout.len() - } - - fn get_cmd_line(&self) -> &str { - &self.cmd_line - } - - pub fn iter(&self) -> impl Iterator> { - self.layout.iter() - } -} - -#[derive(Deserialize, Debug, Default)] -pub struct DisksLayout { - pub legacy: bool, - pub pools: Vec, -} - -// impl> TryFrom<&[T]> for DisksLayout { -// type Error = Error; - -// fn try_from(args: &[T]) -> Result { -// if args.is_empty() { -// return Err(Error::from_string("Invalid argument")); -// } - -// let is_ellipses = args.iter().any(|v| has_ellipses(&[v])); - -// // None of the args have ellipses use the old style. -// if !is_ellipses { -// let set_args = get_all_sets(is_ellipses, args)?; - -// return Ok(DisksLayout { -// legacy: true, -// pools: vec![PoolDisksLayout::new( -// args.iter().map(AsRef::as_ref).collect::>().join(" "), -// set_args, -// )], -// }); -// } - -// let mut layout = Vec::with_capacity(args.len()); -// for arg in args.iter() { -// if !has_ellipses(&[arg]) && args.len() > 1 { -// return Err(Error::from_string( -// "all args must have ellipses for pool expansion (Invalid arguments specified)", -// )); -// } - -// let set_args = get_all_sets(is_ellipses, &[arg])?; - -// layout.push(PoolDisksLayout::new(arg.as_ref(), set_args)); -// } - -// Ok(DisksLayout { -// legacy: false, -// pools: layout, -// }) -// } -// } - -impl DisksLayout { - pub fn from_volumes>(args: &[T]) -> Result { - if args.is_empty() { - return Err(Error::other("Invalid argument")); - } - - let is_ellipses = args.iter().any(|v| has_ellipses(&[v])); - - let set_drive_count_env = env::var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT).unwrap_or_else(|err| { - debug!("{} not set use default:0, {:?}", ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, err); - "0".to_string() - }); - let set_drive_count: usize = set_drive_count_env.parse().map_err(Error::other)?; - - // None of the args have ellipses use the old style. - if !is_ellipses { - let set_args = get_all_sets(set_drive_count, is_ellipses, args)?; - - return Ok(DisksLayout { - legacy: true, - pools: vec![PoolDisksLayout::new( - args.iter().map(AsRef::as_ref).collect::>().join(" "), - set_args, - )], - }); - } - - let mut layout = Vec::with_capacity(args.len()); - for arg in args.iter() { - if !has_ellipses(&[arg]) && args.len() > 1 { - return Err(Error::other( - "all args must have ellipses for pool expansion (Invalid arguments specified)", - )); - } - - let set_args = get_all_sets(set_drive_count, is_ellipses, &[arg])?; - - layout.push(PoolDisksLayout::new(arg.as_ref(), set_args)); - } - - Ok(DisksLayout { - legacy: false, - pools: layout, - }) - } - - pub fn is_empty_layout(&self) -> bool { - self.pools.is_empty() - || self.pools[0].layout.is_empty() - || self.pools[0].layout[0].is_empty() - || self.pools[0].layout[0][0].is_empty() - } - - pub fn is_single_drive_layout(&self) -> bool { - self.pools.len() == 1 && self.pools[0].layout.len() == 1 && self.pools[0].layout[0].len() == 1 - } - - pub fn get_single_drive_layout(&self) -> &str { - &self.pools[0].layout[0][0] - } - - /// returns the total number of sets in the layout. - pub fn get_set_count(&self, i: usize) -> usize { - self.pools.get(i).map_or(0, |v| v.count()) - } - - /// returns the total number of drives in the layout. - pub fn get_drives_per_set(&self, i: usize) -> usize { - self.pools.get(i).map_or(0, |v| v.layout.first().map_or(0, |v| v.len())) - } - - /// returns the command line for the given index. - pub fn get_cmd_line(&self, i: usize) -> String { - self.pools.get(i).map_or(String::new(), |v| v.get_cmd_line().to_owned()) - } -} - -/// parses all ellipses input arguments, expands them into -/// corresponding list of endpoints chunked evenly in accordance with a -/// specific set size. -/// -/// For example: {1...64} is divided into 4 sets each of size 16. -/// This applies to even distributed setup syntax as well. -fn get_all_sets>(set_drive_count: usize, is_ellipses: bool, args: &[T]) -> Result>> { - let endpoint_set = if is_ellipses { - EndpointSet::from_volumes(args, set_drive_count)? - } else { - let set_indexes = if args.len() > 1 { - get_set_indexes(args, &[args.len()], set_drive_count, &[])? - } else { - vec![vec![args.len()]] - }; - let endpoints = args.iter().map(|v| v.as_ref().to_string()).collect(); - - EndpointSet::new(endpoints, set_indexes) - }; - - let set_args = endpoint_set.get(); - - let mut unique_args = HashSet::with_capacity(set_args.len()); - for args in set_args.iter() { - for arg in args { - if unique_args.contains(arg) { - return Err(Error::other(format!("Input args {arg} has duplicate ellipses"))); - } - unique_args.insert(arg); - } - } - - Ok(set_args) -} - -/// represents parsed ellipses values, also provides -/// methods to get the sets of endpoints. -#[derive(Debug, Default)] -struct EndpointSet { - _arg_patterns: Vec, - endpoints: Vec, - set_indexes: Vec>, -} - -// impl> TryFrom<&[T]> for EndpointSet { -// type Error = Error; - -// fn try_from(args: &[T]) -> Result { -// let mut arg_patterns = Vec::with_capacity(args.len()); -// for arg in args { -// arg_patterns.push(find_ellipses_patterns(arg.as_ref())?); -// } - -// let total_sizes = get_total_sizes(&arg_patterns); -// let set_indexes = get_set_indexes(args, &total_sizes, &arg_patterns)?; - -// let mut endpoints = Vec::new(); -// for ap in arg_patterns.iter() { -// let aps = ap.expand(); -// for bs in aps { -// endpoints.push(bs.join("")); -// } -// } - -// Ok(EndpointSet { -// set_indexes, -// _arg_patterns: arg_patterns, -// endpoints, -// }) -// } -// } - -impl EndpointSet { - /// Create a new EndpointSet with the given endpoints and set indexes. - pub fn new(endpoints: Vec, set_indexes: Vec>) -> Self { - Self { - endpoints, - set_indexes, - ..Default::default() - } - } - - pub fn from_volumes>(args: &[T], set_drive_count: usize) -> Result { - let mut arg_patterns = Vec::with_capacity(args.len()); - for arg in args { - arg_patterns.push(find_ellipses_patterns(arg.as_ref())?); - } - - let total_sizes = get_total_sizes(&arg_patterns); - let set_indexes = get_set_indexes(args, &total_sizes, set_drive_count, &arg_patterns)?; - - let mut endpoints = Vec::new(); - for ap in arg_patterns.iter() { - let aps = ap.expand(); - for bs in aps { - endpoints.push(bs.join("")); - } - } - - Ok(EndpointSet { - set_indexes, - _arg_patterns: arg_patterns, - endpoints, - }) - } - - /// returns the sets representation of the endpoints - /// this function also intelligently decides on what will - /// be the right set size etc. - pub fn get(&self) -> Vec> { - let mut sets: Vec> = Vec::new(); - - let mut start = 0; - for set_idx in self.set_indexes.iter() { - for idx in set_idx { - let end = idx + start; - sets.push(self.endpoints[start..end].to_vec()); - start = end; - } - } - sets - } -} - -/// returns the greatest common divisor of all the ellipses sizes. -fn get_divisible_size(total_sizes: &[usize]) -> usize { - fn gcd(mut x: usize, mut y: usize) -> usize { - while y != 0 { - // be equivalent to: x, y = y, x%y - std::mem::swap(&mut x, &mut y); - y %= x; - } - x - } - - total_sizes.iter().skip(1).fold(total_sizes[0], |acc, &y| gcd(acc, y)) -} - -fn possible_set_counts(set_size: usize) -> Vec { - let mut ss = Vec::new(); - for s in SET_SIZES { - if set_size.is_multiple_of(s) { - ss.push(s); - } - } - ss -} - -/// checks whether given count is a valid set size for erasure coding. -fn is_valid_set_size(count: usize) -> bool { - count >= SET_SIZES[0] && count <= SET_SIZES[SET_SIZES.len() - 1] -} - -/// Final set size with all the symmetry accounted for. -fn common_set_drive_count(divisible_size: usize, set_counts: &[usize]) -> usize { - // prefers set_counts to be sorted for optimal behavior. - if divisible_size < set_counts[set_counts.len() - 1] { - return divisible_size; - } - - let mut prev_d = divisible_size / set_counts[0]; - let mut set_size = 0; - for &cnt in set_counts { - if divisible_size.is_multiple_of(cnt) { - let d = divisible_size / cnt; - if d <= prev_d { - prev_d = d; - set_size = cnt; - } - } - } - set_size -} - -/// returns symmetrical setCounts based on the input argument patterns, -/// the symmetry calculation is to ensure that we also use uniform number -/// of drives common across all ellipses patterns. -fn possible_set_counts_with_symmetry(set_counts: &[usize], arg_patterns: &[ArgPattern]) -> Vec { - let mut new_set_counts: HashSet = HashSet::new(); - - for &ss in set_counts { - let mut symmetry = false; - for arg_pattern in arg_patterns { - for p in arg_pattern.as_ref().iter() { - if p.len() > ss { - symmetry = (p.len() % ss) == 0; - } else { - symmetry = (ss % p.len()) == 0; - } - } - } - - if !new_set_counts.contains(&ss) && (symmetry || arg_patterns.is_empty()) { - new_set_counts.insert(ss); - } - } - - let mut set_counts: Vec = new_set_counts.into_iter().collect(); - set_counts.sort_unstable(); - - set_counts -} - -/// returns list of indexes which provides the set size -/// on each index, this function also determines the final set size -/// The final set size has the affinity towards choosing smaller -/// indexes (total sets) -fn get_set_indexes>( - args: &[T], - total_sizes: &[usize], - set_drive_count: usize, - arg_patterns: &[ArgPattern], -) -> Result>> { - if args.is_empty() || total_sizes.is_empty() { - return Err(Error::other("Invalid argument")); - } - - for &size in total_sizes { - // Check if total_sizes has minimum range upto set_size - if size < SET_SIZES[0] || size < set_drive_count { - return Err(Error::other(format!("Incorrect number of endpoints provided, size {size}"))); - } - } - - let common_size = get_divisible_size(total_sizes); - let mut set_counts = possible_set_counts(common_size); - if set_counts.is_empty() { - return Err(Error::other(format!( - "Incorrect number of endpoints provided, number of drives {} is not divisible by any supported erasure set sizes {}", - common_size, 0 - ))); - } - - // Returns possible set counts with symmetry. - set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns); - if set_counts.is_empty() { - return Err(Error::other("No symmetric distribution detected with input endpoints provided")); - } - - let set_size = { - if set_drive_count > 0 { - let has_set_drive_count = set_counts.contains(&set_drive_count); - - if !has_set_drive_count { - return Err(Error::other(format!( - "Invalid set drive count {}. Acceptable values for {:?} number drives are {:?}", - set_drive_count, common_size, &set_counts - ))); - } - set_drive_count - } else { - set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns); - if set_counts.is_empty() { - return Err(Error::other(format!( - "No symmetric distribution detected with input endpoints , drives {} cannot be spread symmetrically by any supported erasure set sizes {:?}", - common_size, &set_counts - ))); - } - // Final set size with all the symmetry accounted for. - common_set_drive_count(common_size, &set_counts) - } - }; - - if !is_valid_set_size(set_size) { - return Err(Error::other("Incorrect number of endpoints provided3")); - } - - Ok(total_sizes - .iter() - .map(|&size| (0..(size / set_size)).map(|_| set_size).collect()) - .collect()) -} - -/// Return the total size for each argument patterns. -fn get_total_sizes(arg_patterns: &[ArgPattern]) -> Vec { - arg_patterns.iter().map(|v| v.total_sizes()).collect() -} - -#[cfg(test)] -mod test { - use rustfs_utils::string::Pattern; - - use super::*; - - impl PartialEq for EndpointSet { - fn eq(&self, other: &Self) -> bool { - self._arg_patterns == other._arg_patterns && self.set_indexes == other.set_indexes - } - } - - #[test] - fn test_get_divisible_size() { - struct TestCase { - total_sizes: Vec, - result: usize, - } - - let test_cases = [ - TestCase { - total_sizes: vec![24, 32, 16], - result: 8, - }, - TestCase { - total_sizes: vec![32, 8, 4], - result: 4, - }, - TestCase { - total_sizes: vec![8, 8, 8], - result: 8, - }, - TestCase { - total_sizes: vec![24], - result: 24, - }, - ]; - - for (i, test_case) in test_cases.iter().enumerate() { - let ret = get_divisible_size(&test_case.total_sizes); - assert_eq!(ret, test_case.result, "Test{}: Expected {}, got {}", i + 1, test_case.result, ret); - } - } - - #[test] - fn test_get_set_indexes() { - #[derive(Default)] - struct TestCase<'a> { - num: usize, - args: Vec<&'a str>, - total_sizes: Vec, - indexes: Vec>, - success: bool, - } - - let test_cases = [ - TestCase { - num: 1, - args: vec!["data{1...17}/export{1...52}"], - total_sizes: vec![14144], - ..Default::default() - }, - TestCase { - num: 2, - args: vec!["data{1...3}"], - total_sizes: vec![3], - indexes: vec![vec![3]], - success: true, - }, - TestCase { - num: 3, - args: vec!["data/controller1/export{1...2}, data/controller2/export{1...4}, data/controller3/export{1...8}"], - total_sizes: vec![2, 4, 8], - indexes: vec![vec![2], vec![2, 2], vec![2, 2, 2, 2]], - success: true, - }, - TestCase { - num: 4, - args: vec!["data{1...27}"], - total_sizes: vec![27], - indexes: vec![vec![9, 9, 9]], - success: true, - }, - TestCase { - num: 5, - args: vec!["http://host{1...3}/data{1...180}"], - total_sizes: vec![540], - indexes: vec![vec![ - 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, - 15, 15, 15, 15, 15, 15, 15, 15, 15, - ]], - success: true, - }, - TestCase { - num: 6, - args: vec!["http://host{1...2}.rack{1...4}/data{1...180}"], - total_sizes: vec![1440], - indexes: vec![vec![ - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, - ]], - success: true, - }, - TestCase { - num: 7, - args: vec!["http://host{1...2}/data{1...180}"], - total_sizes: vec![360], - indexes: vec![vec![ - 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, - 12, 12, 12, - ]], - success: true, - }, - TestCase { - num: 8, - args: vec!["data/controller1/export{1...4}, data/controller2/export{1...8}, data/controller3/export{1...12}"], - total_sizes: vec![4, 8, 12], - indexes: vec![vec![4], vec![4, 4], vec![4, 4, 4]], - success: true, - }, - TestCase { - num: 9, - args: vec!["data{1...64}"], - total_sizes: vec![64], - indexes: vec![vec![16, 16, 16, 16]], - success: true, - }, - TestCase { - num: 10, - args: vec!["data{1...24}"], - total_sizes: vec![24], - indexes: vec![vec![12, 12]], - success: true, - }, - TestCase { - num: 11, - args: vec!["data/controller{1...11}/export{1...8}"], - total_sizes: vec![88], - indexes: vec![vec![11, 11, 11, 11, 11, 11, 11, 11]], - success: true, - }, - TestCase { - num: 12, - args: vec!["data{1...4}"], - total_sizes: vec![4], - indexes: vec![vec![4]], - success: true, - }, - TestCase { - num: 13, - args: vec!["data/controller1/export{1...10}, data/controller2/export{1...10}, data/controller3/export{1...10}"], - total_sizes: vec![10, 10, 10], - indexes: vec![vec![10], vec![10], vec![10]], - success: true, - }, - TestCase { - num: 14, - args: vec!["data{1...16}/export{1...52}"], - total_sizes: vec![832], - indexes: vec![vec![ - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - ]], - success: true, - }, - TestCase { - num: 15, - args: vec!["https://node{1...3}.example.net/mnt/drive{1...8}"], - total_sizes: vec![24], - indexes: vec![vec![12, 12]], - success: true, - }, - ]; - - for test_case in test_cases { - let mut arg_patterns = Vec::new(); - for v in test_case.args.iter() { - match find_ellipses_patterns(v) { - Ok(patterns) => { - arg_patterns.push(patterns); - } - Err(err) => { - panic!("Test{}: Unexpected failure {:?}", test_case.num, err); - } - } - } - - match get_set_indexes(test_case.args.as_slice(), test_case.total_sizes.as_slice(), 0, arg_patterns.as_slice()) { - Ok(got_indexes) => { - if !test_case.success { - panic!("Test{}: Expected failure but passed instead", test_case.num); - } - - assert_eq!( - test_case.indexes, got_indexes, - "Test{}: Expected {:?}, got {:?}", - test_case.num, test_case.indexes, got_indexes - ) - } - Err(err) => { - if test_case.success { - panic!("Test{}: Expected success but failed instead {:?}", test_case.num, err); - } - } - } - } - } - - fn get_sequences(start: usize, number: usize, padding_len: usize) -> Vec { - let mut seq = Vec::new(); - for i in start..=number { - if padding_len == 0 { - seq.push(format!("{i}")); - } else { - seq.push(format!("{i:0padding_len$}")); - } - } - seq - } - - #[test] - fn test_into_endpoint_set() { - #[derive(Default)] - struct TestCase<'a> { - num: usize, - arg: &'a str, - es: EndpointSet, - success: bool, - } - - let test_cases = [ - // Tests invalid inputs. - TestCase { - num: 1, - arg: "...", - ..Default::default() - }, - // No range specified. - TestCase { - num: 2, - arg: "{...}", - ..Default::default() - }, - // Invalid range. - TestCase { - num: 3, - arg: "http://rustfs{2...3}/export/set{1...0}", - ..Default::default() - }, - // Range cannot be smaller than 4 minimum. - TestCase { - num: 4, - arg: "/export{1..2}", - ..Default::default() - }, - // Unsupported characters. - TestCase { - num: 5, - arg: "/export/test{1...2O}", - ..Default::default() - }, - // Tests valid inputs. - TestCase { - num: 6, - arg: "{1...27}", - es: EndpointSet { - _arg_patterns: vec![ArgPattern::new(vec![Pattern { - seq: get_sequences(1, 27, 0), - ..Default::default() - }])], - set_indexes: vec![vec![9, 9, 9]], - ..Default::default() - }, - success: true, - }, - TestCase { - num: 7, - arg: "/export/set{1...64}", - es: EndpointSet { - _arg_patterns: vec![ArgPattern::new(vec![Pattern { - seq: get_sequences(1, 64, 0), - prefix: "/export/set".to_owned(), - ..Default::default() - }])], - set_indexes: vec![vec![16, 16, 16, 16]], - ..Default::default() - }, - success: true, - }, - // Valid input for distributed setup. - TestCase { - num: 8, - arg: "http://rustfs{2...3}/export/set{1...64}", - es: EndpointSet { - _arg_patterns: vec![ArgPattern::new(vec![ - Pattern { - seq: get_sequences(1, 64, 0), - ..Default::default() - }, - Pattern { - seq: get_sequences(2, 3, 0), - prefix: "http://rustfs".to_owned(), - suffix: "/export/set".to_owned(), - }, - ])], - set_indexes: vec![vec![16, 16, 16, 16, 16, 16, 16, 16]], - ..Default::default() - }, - success: true, - }, - // Supporting some advanced cases. - TestCase { - num: 9, - arg: "http://rustfs{1...64}.mydomain.net/data", - es: EndpointSet { - _arg_patterns: vec![ArgPattern::new(vec![Pattern { - seq: get_sequences(1, 64, 0), - prefix: "http://rustfs".to_owned(), - suffix: ".mydomain.net/data".to_owned(), - }])], - set_indexes: vec![vec![16, 16, 16, 16]], - ..Default::default() - }, - success: true, - }, - TestCase { - num: 10, - arg: "http://rack{1...4}.mydomain.rustfs{1...16}/data", - es: EndpointSet { - _arg_patterns: vec![ArgPattern::new(vec![ - Pattern { - seq: get_sequences(1, 16, 0), - suffix: "/data".to_owned(), - ..Default::default() - }, - Pattern { - seq: get_sequences(1, 4, 0), - prefix: "http://rack".to_owned(), - suffix: ".mydomain.rustfs".to_owned(), - }, - ])], - set_indexes: vec![vec![16, 16, 16, 16]], - ..Default::default() - }, - success: true, - }, - // Supporting kubernetes cases. - TestCase { - num: 11, - arg: "http://rustfs{0...15}.mydomain.net/data{0...1}", - es: EndpointSet { - _arg_patterns: vec![ArgPattern::new(vec![ - Pattern { - seq: get_sequences(0, 1, 0), - ..Default::default() - }, - Pattern { - seq: get_sequences(0, 15, 0), - prefix: "http://rustfs".to_owned(), - suffix: ".mydomain.net/data".to_owned(), - }, - ])], - set_indexes: vec![vec![16, 16]], - ..Default::default() - }, - success: true, - }, - // No host regex, just disks. - TestCase { - num: 12, - arg: "http://server1/data{1...32}", - es: EndpointSet { - _arg_patterns: vec![ArgPattern::new(vec![Pattern { - seq: get_sequences(1, 32, 0), - prefix: "http://server1/data".to_owned(), - ..Default::default() - }])], - set_indexes: vec![vec![16, 16]], - ..Default::default() - }, - success: true, - }, - // No host regex, just disks with two position numerics. - TestCase { - num: 13, - arg: "http://server1/data{01...32}", - es: EndpointSet { - _arg_patterns: vec![ArgPattern::new(vec![Pattern { - seq: get_sequences(1, 32, 2), - prefix: "http://server1/data".to_owned(), - ..Default::default() - }])], - set_indexes: vec![vec![16, 16]], - ..Default::default() - }, - success: true, - }, - // More than 2 ellipses are supported as well. - TestCase { - num: 14, - arg: "http://rustfs{2...3}/export/set{1...64}/test{1...2}", - es: EndpointSet { - _arg_patterns: vec![ArgPattern::new(vec![ - Pattern { - seq: get_sequences(1, 2, 0), - ..Default::default() - }, - Pattern { - seq: get_sequences(1, 64, 0), - suffix: "/test".to_owned(), - ..Default::default() - }, - Pattern { - seq: get_sequences(2, 3, 0), - prefix: "http://rustfs".to_owned(), - suffix: "/export/set".to_owned(), - }, - ])], - set_indexes: vec![vec![16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16]], - ..Default::default() - }, - success: true, - }, - // More than an ellipse per argument for standalone setup. - TestCase { - num: 15, - arg: "/export{1...10}/disk{1...10}", - es: EndpointSet { - _arg_patterns: vec![ArgPattern::new(vec![ - Pattern { - seq: get_sequences(1, 10, 0), - ..Default::default() - }, - Pattern { - seq: get_sequences(1, 10, 0), - prefix: "/export".to_owned(), - suffix: "/disk".to_owned(), - }, - ])], - set_indexes: vec![vec![10, 10, 10, 10, 10, 10, 10, 10, 10, 10]], - ..Default::default() - }, - success: true, - }, - ]; - - for test_case in test_cases { - match EndpointSet::from_volumes([test_case.arg].as_slice(), 0) { - Ok(got_es) => { - if !test_case.success { - panic!("Test{}: Expected failure but passed instead", test_case.num); - } - - assert_eq!( - test_case.es, got_es, - "Test{}: Expected {:?}, got {:?}", - test_case.num, test_case.es, got_es - ) - } - Err(err) => { - if test_case.success { - panic!("Test{}: Expected success but failed instead {:?}", test_case.num, err); - } - } - } - } - } -} +pub use crate::layout::disks_layout::*; diff --git a/crates/ecstore/src/layout/disks_layout.rs b/crates/ecstore/src/layout/disks_layout.rs new file mode 100644 index 000000000..2eba85c70 --- /dev/null +++ b/crates/ecstore/src/layout/disks_layout.rs @@ -0,0 +1,927 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_utils::string::{ArgPattern, find_ellipses_patterns, has_ellipses}; +use serde::Deserialize; +use std::collections::HashSet; +use std::env; +use std::io::{Error, Result}; +use tracing::debug; + +/// Supported set sizes this is used to find the optimal +/// single set size. +const SET_SIZES: [usize; 15] = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; +const ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT: &str = "RUSTFS_ERASURE_SET_DRIVE_COUNT"; + +#[derive(Deserialize, Debug, Default)] +pub struct PoolDisksLayout { + cmd_line: String, + layout: Vec>, +} + +impl PoolDisksLayout { + fn new(args: impl Into, layout: Vec>) -> Self { + PoolDisksLayout { + cmd_line: args.into(), + layout, + } + } + + fn count(&self) -> usize { + self.layout.len() + } + + fn get_cmd_line(&self) -> &str { + &self.cmd_line + } + + pub fn iter(&self) -> impl Iterator> { + self.layout.iter() + } +} + +#[derive(Deserialize, Debug, Default)] +pub struct DisksLayout { + pub legacy: bool, + pub pools: Vec, +} + +// impl> TryFrom<&[T]> for DisksLayout { +// type Error = Error; + +// fn try_from(args: &[T]) -> Result { +// if args.is_empty() { +// return Err(Error::from_string("Invalid argument")); +// } + +// let is_ellipses = args.iter().any(|v| has_ellipses(&[v])); + +// // None of the args have ellipses use the old style. +// if !is_ellipses { +// let set_args = get_all_sets(is_ellipses, args)?; + +// return Ok(DisksLayout { +// legacy: true, +// pools: vec![PoolDisksLayout::new( +// args.iter().map(AsRef::as_ref).collect::>().join(" "), +// set_args, +// )], +// }); +// } + +// let mut layout = Vec::with_capacity(args.len()); +// for arg in args.iter() { +// if !has_ellipses(&[arg]) && args.len() > 1 { +// return Err(Error::from_string( +// "all args must have ellipses for pool expansion (Invalid arguments specified)", +// )); +// } + +// let set_args = get_all_sets(is_ellipses, &[arg])?; + +// layout.push(PoolDisksLayout::new(arg.as_ref(), set_args)); +// } + +// Ok(DisksLayout { +// legacy: false, +// pools: layout, +// }) +// } +// } + +impl DisksLayout { + pub fn from_volumes>(args: &[T]) -> Result { + if args.is_empty() { + return Err(Error::other("Invalid argument")); + } + + let is_ellipses = args.iter().any(|v| has_ellipses(&[v])); + + let set_drive_count_env = env::var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT).unwrap_or_else(|err| { + debug!("{} not set use default:0, {:?}", ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, err); + "0".to_string() + }); + let set_drive_count: usize = set_drive_count_env.parse().map_err(Error::other)?; + + // None of the args have ellipses use the old style. + if !is_ellipses { + let set_args = get_all_sets(set_drive_count, is_ellipses, args)?; + + return Ok(DisksLayout { + legacy: true, + pools: vec![PoolDisksLayout::new( + args.iter().map(AsRef::as_ref).collect::>().join(" "), + set_args, + )], + }); + } + + let mut layout = Vec::with_capacity(args.len()); + for arg in args.iter() { + if !has_ellipses(&[arg]) && args.len() > 1 { + return Err(Error::other( + "all args must have ellipses for pool expansion (Invalid arguments specified)", + )); + } + + let set_args = get_all_sets(set_drive_count, is_ellipses, &[arg])?; + + layout.push(PoolDisksLayout::new(arg.as_ref(), set_args)); + } + + Ok(DisksLayout { + legacy: false, + pools: layout, + }) + } + + pub fn is_empty_layout(&self) -> bool { + self.pools.is_empty() + || self.pools[0].layout.is_empty() + || self.pools[0].layout[0].is_empty() + || self.pools[0].layout[0][0].is_empty() + } + + pub fn is_single_drive_layout(&self) -> bool { + self.pools.len() == 1 && self.pools[0].layout.len() == 1 && self.pools[0].layout[0].len() == 1 + } + + pub fn get_single_drive_layout(&self) -> &str { + &self.pools[0].layout[0][0] + } + + /// returns the total number of sets in the layout. + pub fn get_set_count(&self, i: usize) -> usize { + self.pools.get(i).map_or(0, |v| v.count()) + } + + /// returns the total number of drives in the layout. + pub fn get_drives_per_set(&self, i: usize) -> usize { + self.pools.get(i).map_or(0, |v| v.layout.first().map_or(0, |v| v.len())) + } + + /// returns the command line for the given index. + pub fn get_cmd_line(&self, i: usize) -> String { + self.pools.get(i).map_or(String::new(), |v| v.get_cmd_line().to_owned()) + } +} + +/// parses all ellipses input arguments, expands them into +/// corresponding list of endpoints chunked evenly in accordance with a +/// specific set size. +/// +/// For example: {1...64} is divided into 4 sets each of size 16. +/// This applies to even distributed setup syntax as well. +fn get_all_sets>(set_drive_count: usize, is_ellipses: bool, args: &[T]) -> Result>> { + let endpoint_set = if is_ellipses { + EndpointSet::from_volumes(args, set_drive_count)? + } else { + let set_indexes = if args.len() > 1 { + get_set_indexes(args, &[args.len()], set_drive_count, &[])? + } else { + vec![vec![args.len()]] + }; + let endpoints = args.iter().map(|v| v.as_ref().to_string()).collect(); + + EndpointSet::new(endpoints, set_indexes) + }; + + let set_args = endpoint_set.get(); + + let mut unique_args = HashSet::with_capacity(set_args.len()); + for args in set_args.iter() { + for arg in args { + if unique_args.contains(arg) { + return Err(Error::other(format!("Input args {arg} has duplicate ellipses"))); + } + unique_args.insert(arg); + } + } + + Ok(set_args) +} + +/// represents parsed ellipses values, also provides +/// methods to get the sets of endpoints. +#[derive(Debug, Default)] +struct EndpointSet { + _arg_patterns: Vec, + endpoints: Vec, + set_indexes: Vec>, +} + +// impl> TryFrom<&[T]> for EndpointSet { +// type Error = Error; + +// fn try_from(args: &[T]) -> Result { +// let mut arg_patterns = Vec::with_capacity(args.len()); +// for arg in args { +// arg_patterns.push(find_ellipses_patterns(arg.as_ref())?); +// } + +// let total_sizes = get_total_sizes(&arg_patterns); +// let set_indexes = get_set_indexes(args, &total_sizes, &arg_patterns)?; + +// let mut endpoints = Vec::new(); +// for ap in arg_patterns.iter() { +// let aps = ap.expand(); +// for bs in aps { +// endpoints.push(bs.join("")); +// } +// } + +// Ok(EndpointSet { +// set_indexes, +// _arg_patterns: arg_patterns, +// endpoints, +// }) +// } +// } + +impl EndpointSet { + /// Create a new EndpointSet with the given endpoints and set indexes. + pub fn new(endpoints: Vec, set_indexes: Vec>) -> Self { + Self { + endpoints, + set_indexes, + ..Default::default() + } + } + + pub fn from_volumes>(args: &[T], set_drive_count: usize) -> Result { + let mut arg_patterns = Vec::with_capacity(args.len()); + for arg in args { + arg_patterns.push(find_ellipses_patterns(arg.as_ref())?); + } + + let total_sizes = get_total_sizes(&arg_patterns); + let set_indexes = get_set_indexes(args, &total_sizes, set_drive_count, &arg_patterns)?; + + let mut endpoints = Vec::new(); + for ap in arg_patterns.iter() { + let aps = ap.expand(); + for bs in aps { + endpoints.push(bs.join("")); + } + } + + Ok(EndpointSet { + set_indexes, + _arg_patterns: arg_patterns, + endpoints, + }) + } + + /// returns the sets representation of the endpoints + /// this function also intelligently decides on what will + /// be the right set size etc. + pub fn get(&self) -> Vec> { + let mut sets: Vec> = Vec::new(); + + let mut start = 0; + for set_idx in self.set_indexes.iter() { + for idx in set_idx { + let end = idx + start; + sets.push(self.endpoints[start..end].to_vec()); + start = end; + } + } + sets + } +} + +/// returns the greatest common divisor of all the ellipses sizes. +fn get_divisible_size(total_sizes: &[usize]) -> usize { + fn gcd(mut x: usize, mut y: usize) -> usize { + while y != 0 { + // be equivalent to: x, y = y, x%y + std::mem::swap(&mut x, &mut y); + y %= x; + } + x + } + + total_sizes.iter().skip(1).fold(total_sizes[0], |acc, &y| gcd(acc, y)) +} + +fn possible_set_counts(set_size: usize) -> Vec { + let mut ss = Vec::new(); + for s in SET_SIZES { + if set_size.is_multiple_of(s) { + ss.push(s); + } + } + ss +} + +/// checks whether given count is a valid set size for erasure coding. +fn is_valid_set_size(count: usize) -> bool { + count >= SET_SIZES[0] && count <= SET_SIZES[SET_SIZES.len() - 1] +} + +/// Final set size with all the symmetry accounted for. +fn common_set_drive_count(divisible_size: usize, set_counts: &[usize]) -> usize { + // prefers set_counts to be sorted for optimal behavior. + if divisible_size < set_counts[set_counts.len() - 1] { + return divisible_size; + } + + let mut prev_d = divisible_size / set_counts[0]; + let mut set_size = 0; + for &cnt in set_counts { + if divisible_size.is_multiple_of(cnt) { + let d = divisible_size / cnt; + if d <= prev_d { + prev_d = d; + set_size = cnt; + } + } + } + set_size +} + +/// returns symmetrical setCounts based on the input argument patterns, +/// the symmetry calculation is to ensure that we also use uniform number +/// of drives common across all ellipses patterns. +fn possible_set_counts_with_symmetry(set_counts: &[usize], arg_patterns: &[ArgPattern]) -> Vec { + let mut new_set_counts: HashSet = HashSet::new(); + + for &ss in set_counts { + let mut symmetry = false; + for arg_pattern in arg_patterns { + for p in arg_pattern.as_ref().iter() { + if p.len() > ss { + symmetry = (p.len() % ss) == 0; + } else { + symmetry = (ss % p.len()) == 0; + } + } + } + + if !new_set_counts.contains(&ss) && (symmetry || arg_patterns.is_empty()) { + new_set_counts.insert(ss); + } + } + + let mut set_counts: Vec = new_set_counts.into_iter().collect(); + set_counts.sort_unstable(); + + set_counts +} + +/// returns list of indexes which provides the set size +/// on each index, this function also determines the final set size +/// The final set size has the affinity towards choosing smaller +/// indexes (total sets) +fn get_set_indexes>( + args: &[T], + total_sizes: &[usize], + set_drive_count: usize, + arg_patterns: &[ArgPattern], +) -> Result>> { + if args.is_empty() || total_sizes.is_empty() { + return Err(Error::other("Invalid argument")); + } + + for &size in total_sizes { + // Check if total_sizes has minimum range upto set_size + if size < SET_SIZES[0] || size < set_drive_count { + return Err(Error::other(format!("Incorrect number of endpoints provided, size {size}"))); + } + } + + let common_size = get_divisible_size(total_sizes); + let mut set_counts = possible_set_counts(common_size); + if set_counts.is_empty() { + return Err(Error::other(format!( + "Incorrect number of endpoints provided, number of drives {} is not divisible by any supported erasure set sizes {}", + common_size, 0 + ))); + } + + // Returns possible set counts with symmetry. + set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns); + if set_counts.is_empty() { + return Err(Error::other("No symmetric distribution detected with input endpoints provided")); + } + + let set_size = { + if set_drive_count > 0 { + let has_set_drive_count = set_counts.contains(&set_drive_count); + + if !has_set_drive_count { + return Err(Error::other(format!( + "Invalid set drive count {}. Acceptable values for {:?} number drives are {:?}", + set_drive_count, common_size, &set_counts + ))); + } + set_drive_count + } else { + set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns); + if set_counts.is_empty() { + return Err(Error::other(format!( + "No symmetric distribution detected with input endpoints , drives {} cannot be spread symmetrically by any supported erasure set sizes {:?}", + common_size, &set_counts + ))); + } + // Final set size with all the symmetry accounted for. + common_set_drive_count(common_size, &set_counts) + } + }; + + if !is_valid_set_size(set_size) { + return Err(Error::other("Incorrect number of endpoints provided3")); + } + + Ok(total_sizes + .iter() + .map(|&size| (0..(size / set_size)).map(|_| set_size).collect()) + .collect()) +} + +/// Return the total size for each argument patterns. +fn get_total_sizes(arg_patterns: &[ArgPattern]) -> Vec { + arg_patterns.iter().map(|v| v.total_sizes()).collect() +} + +#[cfg(test)] +mod test { + use rustfs_utils::string::Pattern; + + use super::*; + + impl PartialEq for EndpointSet { + fn eq(&self, other: &Self) -> bool { + self._arg_patterns == other._arg_patterns && self.set_indexes == other.set_indexes + } + } + + #[test] + fn test_get_divisible_size() { + struct TestCase { + total_sizes: Vec, + result: usize, + } + + let test_cases = [ + TestCase { + total_sizes: vec![24, 32, 16], + result: 8, + }, + TestCase { + total_sizes: vec![32, 8, 4], + result: 4, + }, + TestCase { + total_sizes: vec![8, 8, 8], + result: 8, + }, + TestCase { + total_sizes: vec![24], + result: 24, + }, + ]; + + for (i, test_case) in test_cases.iter().enumerate() { + let ret = get_divisible_size(&test_case.total_sizes); + assert_eq!(ret, test_case.result, "Test{}: Expected {}, got {}", i + 1, test_case.result, ret); + } + } + + #[test] + fn test_get_set_indexes() { + #[derive(Default)] + struct TestCase<'a> { + num: usize, + args: Vec<&'a str>, + total_sizes: Vec, + indexes: Vec>, + success: bool, + } + + let test_cases = [ + TestCase { + num: 1, + args: vec!["data{1...17}/export{1...52}"], + total_sizes: vec![14144], + ..Default::default() + }, + TestCase { + num: 2, + args: vec!["data{1...3}"], + total_sizes: vec![3], + indexes: vec![vec![3]], + success: true, + }, + TestCase { + num: 3, + args: vec!["data/controller1/export{1...2}, data/controller2/export{1...4}, data/controller3/export{1...8}"], + total_sizes: vec![2, 4, 8], + indexes: vec![vec![2], vec![2, 2], vec![2, 2, 2, 2]], + success: true, + }, + TestCase { + num: 4, + args: vec!["data{1...27}"], + total_sizes: vec![27], + indexes: vec![vec![9, 9, 9]], + success: true, + }, + TestCase { + num: 5, + args: vec!["http://host{1...3}/data{1...180}"], + total_sizes: vec![540], + indexes: vec![vec![ + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, + ]], + success: true, + }, + TestCase { + num: 6, + args: vec!["http://host{1...2}.rack{1...4}/data{1...180}"], + total_sizes: vec![1440], + indexes: vec![vec![ + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, + ]], + success: true, + }, + TestCase { + num: 7, + args: vec!["http://host{1...2}/data{1...180}"], + total_sizes: vec![360], + indexes: vec![vec![ + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 12, 12, 12, + ]], + success: true, + }, + TestCase { + num: 8, + args: vec!["data/controller1/export{1...4}, data/controller2/export{1...8}, data/controller3/export{1...12}"], + total_sizes: vec![4, 8, 12], + indexes: vec![vec![4], vec![4, 4], vec![4, 4, 4]], + success: true, + }, + TestCase { + num: 9, + args: vec!["data{1...64}"], + total_sizes: vec![64], + indexes: vec![vec![16, 16, 16, 16]], + success: true, + }, + TestCase { + num: 10, + args: vec!["data{1...24}"], + total_sizes: vec![24], + indexes: vec![vec![12, 12]], + success: true, + }, + TestCase { + num: 11, + args: vec!["data/controller{1...11}/export{1...8}"], + total_sizes: vec![88], + indexes: vec![vec![11, 11, 11, 11, 11, 11, 11, 11]], + success: true, + }, + TestCase { + num: 12, + args: vec!["data{1...4}"], + total_sizes: vec![4], + indexes: vec![vec![4]], + success: true, + }, + TestCase { + num: 13, + args: vec!["data/controller1/export{1...10}, data/controller2/export{1...10}, data/controller3/export{1...10}"], + total_sizes: vec![10, 10, 10], + indexes: vec![vec![10], vec![10], vec![10]], + success: true, + }, + TestCase { + num: 14, + args: vec!["data{1...16}/export{1...52}"], + total_sizes: vec![832], + indexes: vec![vec![ + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + ]], + success: true, + }, + TestCase { + num: 15, + args: vec!["https://node{1...3}.example.net/mnt/drive{1...8}"], + total_sizes: vec![24], + indexes: vec![vec![12, 12]], + success: true, + }, + ]; + + for test_case in test_cases { + let mut arg_patterns = Vec::new(); + for v in test_case.args.iter() { + match find_ellipses_patterns(v) { + Ok(patterns) => { + arg_patterns.push(patterns); + } + Err(err) => { + panic!("Test{}: Unexpected failure {:?}", test_case.num, err); + } + } + } + + match get_set_indexes(test_case.args.as_slice(), test_case.total_sizes.as_slice(), 0, arg_patterns.as_slice()) { + Ok(got_indexes) => { + if !test_case.success { + panic!("Test{}: Expected failure but passed instead", test_case.num); + } + + assert_eq!( + test_case.indexes, got_indexes, + "Test{}: Expected {:?}, got {:?}", + test_case.num, test_case.indexes, got_indexes + ) + } + Err(err) => { + if test_case.success { + panic!("Test{}: Expected success but failed instead {:?}", test_case.num, err); + } + } + } + } + } + + fn get_sequences(start: usize, number: usize, padding_len: usize) -> Vec { + let mut seq = Vec::new(); + for i in start..=number { + if padding_len == 0 { + seq.push(format!("{i}")); + } else { + seq.push(format!("{i:0padding_len$}")); + } + } + seq + } + + #[test] + fn test_into_endpoint_set() { + #[derive(Default)] + struct TestCase<'a> { + num: usize, + arg: &'a str, + es: EndpointSet, + success: bool, + } + + let test_cases = [ + // Tests invalid inputs. + TestCase { + num: 1, + arg: "...", + ..Default::default() + }, + // No range specified. + TestCase { + num: 2, + arg: "{...}", + ..Default::default() + }, + // Invalid range. + TestCase { + num: 3, + arg: "http://rustfs{2...3}/export/set{1...0}", + ..Default::default() + }, + // Range cannot be smaller than 4 minimum. + TestCase { + num: 4, + arg: "/export{1..2}", + ..Default::default() + }, + // Unsupported characters. + TestCase { + num: 5, + arg: "/export/test{1...2O}", + ..Default::default() + }, + // Tests valid inputs. + TestCase { + num: 6, + arg: "{1...27}", + es: EndpointSet { + _arg_patterns: vec![ArgPattern::new(vec![Pattern { + seq: get_sequences(1, 27, 0), + ..Default::default() + }])], + set_indexes: vec![vec![9, 9, 9]], + ..Default::default() + }, + success: true, + }, + TestCase { + num: 7, + arg: "/export/set{1...64}", + es: EndpointSet { + _arg_patterns: vec![ArgPattern::new(vec![Pattern { + seq: get_sequences(1, 64, 0), + prefix: "/export/set".to_owned(), + ..Default::default() + }])], + set_indexes: vec![vec![16, 16, 16, 16]], + ..Default::default() + }, + success: true, + }, + // Valid input for distributed setup. + TestCase { + num: 8, + arg: "http://rustfs{2...3}/export/set{1...64}", + es: EndpointSet { + _arg_patterns: vec![ArgPattern::new(vec![ + Pattern { + seq: get_sequences(1, 64, 0), + ..Default::default() + }, + Pattern { + seq: get_sequences(2, 3, 0), + prefix: "http://rustfs".to_owned(), + suffix: "/export/set".to_owned(), + }, + ])], + set_indexes: vec![vec![16, 16, 16, 16, 16, 16, 16, 16]], + ..Default::default() + }, + success: true, + }, + // Supporting some advanced cases. + TestCase { + num: 9, + arg: "http://rustfs{1...64}.mydomain.net/data", + es: EndpointSet { + _arg_patterns: vec![ArgPattern::new(vec![Pattern { + seq: get_sequences(1, 64, 0), + prefix: "http://rustfs".to_owned(), + suffix: ".mydomain.net/data".to_owned(), + }])], + set_indexes: vec![vec![16, 16, 16, 16]], + ..Default::default() + }, + success: true, + }, + TestCase { + num: 10, + arg: "http://rack{1...4}.mydomain.rustfs{1...16}/data", + es: EndpointSet { + _arg_patterns: vec![ArgPattern::new(vec![ + Pattern { + seq: get_sequences(1, 16, 0), + suffix: "/data".to_owned(), + ..Default::default() + }, + Pattern { + seq: get_sequences(1, 4, 0), + prefix: "http://rack".to_owned(), + suffix: ".mydomain.rustfs".to_owned(), + }, + ])], + set_indexes: vec![vec![16, 16, 16, 16]], + ..Default::default() + }, + success: true, + }, + // Supporting kubernetes cases. + TestCase { + num: 11, + arg: "http://rustfs{0...15}.mydomain.net/data{0...1}", + es: EndpointSet { + _arg_patterns: vec![ArgPattern::new(vec![ + Pattern { + seq: get_sequences(0, 1, 0), + ..Default::default() + }, + Pattern { + seq: get_sequences(0, 15, 0), + prefix: "http://rustfs".to_owned(), + suffix: ".mydomain.net/data".to_owned(), + }, + ])], + set_indexes: vec![vec![16, 16]], + ..Default::default() + }, + success: true, + }, + // No host regex, just disks. + TestCase { + num: 12, + arg: "http://server1/data{1...32}", + es: EndpointSet { + _arg_patterns: vec![ArgPattern::new(vec![Pattern { + seq: get_sequences(1, 32, 0), + prefix: "http://server1/data".to_owned(), + ..Default::default() + }])], + set_indexes: vec![vec![16, 16]], + ..Default::default() + }, + success: true, + }, + // No host regex, just disks with two position numerics. + TestCase { + num: 13, + arg: "http://server1/data{01...32}", + es: EndpointSet { + _arg_patterns: vec![ArgPattern::new(vec![Pattern { + seq: get_sequences(1, 32, 2), + prefix: "http://server1/data".to_owned(), + ..Default::default() + }])], + set_indexes: vec![vec![16, 16]], + ..Default::default() + }, + success: true, + }, + // More than 2 ellipses are supported as well. + TestCase { + num: 14, + arg: "http://rustfs{2...3}/export/set{1...64}/test{1...2}", + es: EndpointSet { + _arg_patterns: vec![ArgPattern::new(vec![ + Pattern { + seq: get_sequences(1, 2, 0), + ..Default::default() + }, + Pattern { + seq: get_sequences(1, 64, 0), + suffix: "/test".to_owned(), + ..Default::default() + }, + Pattern { + seq: get_sequences(2, 3, 0), + prefix: "http://rustfs".to_owned(), + suffix: "/export/set".to_owned(), + }, + ])], + set_indexes: vec![vec![16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16]], + ..Default::default() + }, + success: true, + }, + // More than an ellipse per argument for standalone setup. + TestCase { + num: 15, + arg: "/export{1...10}/disk{1...10}", + es: EndpointSet { + _arg_patterns: vec![ArgPattern::new(vec![ + Pattern { + seq: get_sequences(1, 10, 0), + ..Default::default() + }, + Pattern { + seq: get_sequences(1, 10, 0), + prefix: "/export".to_owned(), + suffix: "/disk".to_owned(), + }, + ])], + set_indexes: vec![vec![10, 10, 10, 10, 10, 10, 10, 10, 10, 10]], + ..Default::default() + }, + success: true, + }, + ]; + + for test_case in test_cases { + match EndpointSet::from_volumes([test_case.arg].as_slice(), 0) { + Ok(got_es) => { + if !test_case.success { + panic!("Test{}: Expected failure but passed instead", test_case.num); + } + + assert_eq!( + test_case.es, got_es, + "Test{}: Expected {:?}, got {:?}", + test_case.num, test_case.es, got_es + ) + } + Err(err) => { + if test_case.success { + panic!("Test{}: Expected success but failed instead {:?}", test_case.num, err); + } + } + } + } + } +} diff --git a/crates/ecstore/src/layout/format.rs b/crates/ecstore/src/layout/format.rs new file mode 100644 index 000000000..592f6f93b --- /dev/null +++ b/crates/ecstore/src/layout/format.rs @@ -0,0 +1,546 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::disk::error::{Error, Result}; +use crate::disk::{DiskInfo, error::DiskError}; +use serde::{Deserialize, Serialize}; +use serde_json::Error as JsonError; +use uuid::Uuid; + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub enum FormatMetaVersion { + #[serde(rename = "1")] + V1, + + #[serde(other)] + Unknown, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub enum FormatBackend { + #[serde(rename = "xl")] + Erasure, + #[serde(rename = "xl-single")] + ErasureSingle, + + #[serde(other)] + Unknown, +} + +/// Represents the V3 backend disk structure version +/// under `.rustfs.sys` and actual data namespace. +/// +/// FormatErasureV3 - structure holds format config version '3'. +/// +/// The V3 format to support "large bucket" support where a bucket +/// can span multiple erasure sets. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct FormatErasureV3 { + /// Version of 'xl' format. + pub version: FormatErasureVersion, + + /// This field carries assigned disk uuid. + pub this: Uuid, + + /// Sets field carries the input disk order generated the first + /// time when fresh disks were supplied, it is a two-dimensional + /// array second dimension represents list of disks used per set. + pub sets: Vec>, + + /// Distribution algorithm represents the hashing algorithm + /// to pick the right set index for an object. + #[serde(rename = "distributionAlgo")] + pub distribution_algo: DistributionAlgoVersion, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub enum FormatErasureVersion { + #[serde(rename = "1")] + V1, + #[serde(rename = "2")] + V2, + #[serde(rename = "3")] + V3, + + #[serde(other)] + Unknown, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub enum DistributionAlgoVersion { + #[serde(rename = "CRCMOD")] + V1, + #[serde(rename = "SIPMOD")] + V2, + #[serde(rename = "SIPMOD+PARITY")] + V3, +} + +/// format.json currently has the format: +/// +/// ```json +/// { +/// "version": "1", +/// "format": "XXXXX", +/// "id": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX", +/// "XXXXX": { +// +/// } +/// } +/// ``` +/// +/// Ideally we will never have a situation where we will have to change the +/// fields of this struct and deal with related migration. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct FormatV3 { + /// Version of the format config. + pub version: FormatMetaVersion, + + /// Format indicates the backend format type, supports two values 'xl' and 'xl-single'. + pub format: FormatBackend, + + /// ID is the identifier for the rustfs deployment + pub id: Uuid, + + #[serde(rename = "xl")] + pub erasure: FormatErasureV3, + // /// DiskInfo is an extended type which returns current + // /// disk usage per path. + #[serde(skip)] + pub disk_info: Option, +} + +impl TryFrom<&[u8]> for FormatV3 { + type Error = JsonError; + + fn try_from(data: &[u8]) -> std::result::Result { + serde_json::from_slice(data) + } +} + +impl TryFrom<&str> for FormatV3 { + type Error = JsonError; + + fn try_from(data: &str) -> std::result::Result { + serde_json::from_str(data) + } +} + +impl FormatV3 { + /// Create a new format config with the given number of sets and set length. + pub fn new(num_sets: usize, set_len: usize) -> Self { + let format = if set_len == 1 { + FormatBackend::ErasureSingle + } else { + FormatBackend::Erasure + }; + + let erasure = FormatErasureV3 { + version: FormatErasureVersion::V3, + this: Uuid::nil(), + sets: (0..num_sets) + .map(|_| (0..set_len).map(|_| Uuid::new_v4()).collect()) + .collect(), + distribution_algo: DistributionAlgoVersion::V3, + }; + + Self { + version: FormatMetaVersion::V1, + format, + id: Uuid::new_v4(), + erasure, + disk_info: None, + } + } + + /// Returns the number of drives in the erasure set. + pub fn drives(&self) -> usize { + self.erasure.sets.iter().map(|v| v.len()).sum() + } + + pub fn to_json(&self) -> std::result::Result { + serde_json::to_string(self) + } + + /// returns the i,j'th position of the input `diskID` against the reference + /// + /// format, after successful validation. + /// - i'th position is the set index + /// - j'th position is the disk index in the current set + pub fn find_disk_index_by_disk_id(&self, disk_id: Uuid) -> Result<(usize, usize)> { + if disk_id == Uuid::nil() { + return Err(Error::from(DiskError::DiskNotFound)); + } + if disk_id == Uuid::max() { + return Err(Error::other("disk offline")); + } + + for (i, set) in self.erasure.sets.iter().enumerate() { + for (j, d) in set.iter().enumerate() { + if disk_id.eq(d) { + return Ok((i, j)); + } + } + } + + Err(Error::other(format!("disk id not found {disk_id}"))) + } + + pub fn check_other(&self, other: &FormatV3) -> Result<()> { + let mut tmp = other.clone(); + let this = tmp.erasure.this; + tmp.erasure.this = Uuid::nil(); + + if self.erasure.sets.len() != other.erasure.sets.len() { + return Err(Error::other(format!( + "Expected number of sets {}, got {}", + self.erasure.sets.len(), + other.erasure.sets.len() + ))); + } + + for i in 0..self.erasure.sets.len() { + if self.erasure.sets[i].len() != other.erasure.sets[i].len() { + return Err(Error::other(format!( + "Each set should be of same size, expected {}, got {}", + self.erasure.sets[i].len(), + other.erasure.sets[i].len() + ))); + } + + for j in 0..self.erasure.sets[i].len() { + if self.erasure.sets[i][j] != other.erasure.sets[i][j] { + return Err(Error::other(format!( + "UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)", + i, + j, + self.erasure.sets[i][j].to_string(), + other.erasure.sets[i][j].to_string(), + ))); + } + } + } + + for i in 0..tmp.erasure.sets.len() { + for j in 0..tmp.erasure.sets[i].len() { + if this == tmp.erasure.sets[i][j] { + return Ok(()); + } + } + } + + Err(Error::other(format!( + "DriveID {:?} not found in any drive sets {:?}", + this, other.erasure.sets + ))) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_format_v1() { + let format = FormatV3::new(1, 4); + + let str = serde_json::to_string(&format); + println!("{str:?}"); + + let data = r#" + { + "version": "1", + "format": "xl", + "id": "321b3874-987d-4c15-8fa5-757c956b1243", + "xl": { + "version": "1", + "this": null, + "sets": [ + [ + "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5", + "c26315da-05cf-4778-a9ea-b44ea09f58c5", + "fb87a891-18d3-44cf-a46f-bcc15093a038", + "356a925c-57b9-4313-88b3-053edf1104dc" + ] + ], + "distributionAlgo": "CRCMOD" + } + }"#; + + let p = FormatV3::try_from(data); + + println!("{p:?}"); + } + + #[test] + fn test_format_v3_new_single_disk() { + let format = FormatV3::new(1, 1); + + assert_eq!(format.version, FormatMetaVersion::V1); + assert_eq!(format.format, FormatBackend::ErasureSingle); + assert_eq!(format.erasure.version, FormatErasureVersion::V3); + assert_eq!(format.erasure.sets.len(), 1); + assert_eq!(format.erasure.sets[0].len(), 1); + assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3); + assert_eq!(format.erasure.this, Uuid::nil()); + } + + #[test] + fn test_format_v3_new_multiple_sets() { + let format = FormatV3::new(2, 4); + + assert_eq!(format.version, FormatMetaVersion::V1); + assert_eq!(format.format, FormatBackend::Erasure); + assert_eq!(format.erasure.version, FormatErasureVersion::V3); + assert_eq!(format.erasure.sets.len(), 2); + assert_eq!(format.erasure.sets[0].len(), 4); + assert_eq!(format.erasure.sets[1].len(), 4); + assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3); + } + + #[test] + fn test_format_v3_drives() { + let format = FormatV3::new(2, 4); + assert_eq!(format.drives(), 8); // 2 sets * 4 drives each + + let format_single = FormatV3::new(1, 1); + assert_eq!(format_single.drives(), 1); // 1 set * 1 drive + } + + #[test] + fn test_format_v3_to_json() { + let format = FormatV3::new(1, 2); + let json_result = format.to_json(); + + assert!(json_result.is_ok()); + let json_str = json_result.unwrap(); + assert!(json_str.contains("\"version\":\"1\"")); + assert!(json_str.contains("\"format\":\"xl\"")); + } + + #[test] + fn test_format_v3_from_json() { + let json_data = r#"{ + "version": "1", + "format": "xl-single", + "id": "321b3874-987d-4c15-8fa5-757c956b1243", + "xl": { + "version": "3", + "this": "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5", + "sets": [ + [ + "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5" + ] + ], + "distributionAlgo": "SIPMOD+PARITY" + } + }"#; + + let format = FormatV3::try_from(json_data); + assert!(format.is_ok()); + + let format = format.unwrap(); + assert_eq!(format.format, FormatBackend::ErasureSingle); + assert_eq!(format.erasure.version, FormatErasureVersion::V3); + assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3); + assert_eq!(format.erasure.sets.len(), 1); + assert_eq!(format.erasure.sets[0].len(), 1); + } + + #[test] + fn test_format_v3_from_bytes() { + let json_data = r#"{ + "version": "1", + "format": "xl", + "id": "321b3874-987d-4c15-8fa5-757c956b1243", + "xl": { + "version": "2", + "this": "00000000-0000-0000-0000-000000000000", + "sets": [ + [ + "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5", + "c26315da-05cf-4778-a9ea-b44ea09f58c5" + ] + ], + "distributionAlgo": "SIPMOD" + } + }"#; + + let format = FormatV3::try_from(json_data.as_bytes()); + assert!(format.is_ok()); + + let format = format.unwrap(); + assert_eq!(format.erasure.version, FormatErasureVersion::V2); + assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V2); + assert_eq!(format.erasure.sets[0].len(), 2); + } + + #[test] + fn test_format_v3_invalid_json() { + let invalid_json = r#"{"invalid": "json"}"#; + let format = FormatV3::try_from(invalid_json); + assert!(format.is_err()); + } + + #[test] + fn test_find_disk_index_by_disk_id() { + let mut format = FormatV3::new(2, 2); + let target_disk_id = Uuid::new_v4(); + format.erasure.sets[1][0] = target_disk_id; + + let result = format.find_disk_index_by_disk_id(target_disk_id); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), (1, 0)); + } + + #[test] + fn test_find_disk_index_nil_uuid() { + let format = FormatV3::new(1, 2); + let result = format.find_disk_index_by_disk_id(Uuid::nil()); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), Error::DiskNotFound)); + } + + #[test] + fn test_find_disk_index_max_uuid() { + let format = FormatV3::new(1, 2); + let result = format.find_disk_index_by_disk_id(Uuid::max()); + assert!(result.is_err()); + } + + #[test] + fn test_find_disk_index_not_found() { + let format = FormatV3::new(1, 2); + let non_existent_id = Uuid::new_v4(); + let result = format.find_disk_index_by_disk_id(non_existent_id); + assert!(result.is_err()); + } + + #[test] + fn test_check_other_identical() { + let format1 = FormatV3::new(2, 4); + let mut format2 = format1.clone(); + format2.erasure.this = format1.erasure.sets[0][0]; + + let result = format1.check_other(&format2); + assert!(result.is_ok()); + } + + #[test] + fn test_check_other_different_set_count() { + let format1 = FormatV3::new(2, 4); + let format2 = FormatV3::new(3, 4); + + let result = format1.check_other(&format2); + assert!(result.is_err()); + } + + #[test] + fn test_check_other_different_set_size() { + let format1 = FormatV3::new(2, 4); + let format2 = FormatV3::new(2, 6); + + let result = format1.check_other(&format2); + assert!(result.is_err()); + } + + #[test] + fn test_check_other_different_disk_id() { + let format1 = FormatV3::new(1, 2); + let mut format2 = format1.clone(); + format2.erasure.sets[0][0] = Uuid::new_v4(); + + let result = format1.check_other(&format2); + assert!(result.is_err()); + } + + #[test] + fn test_check_other_disk_not_in_sets() { + let format1 = FormatV3::new(1, 2); + let mut format2 = format1.clone(); + format2.erasure.this = Uuid::new_v4(); // Set to a UUID not in any set + + let result = format1.check_other(&format2); + assert!(result.is_err()); + } + + #[test] + fn test_format_meta_version_serialization() { + let v1 = FormatMetaVersion::V1; + let json = serde_json::to_string(&v1).unwrap(); + assert_eq!(json, "\"1\""); + + let unknown = FormatMetaVersion::Unknown; + let deserialized: FormatMetaVersion = serde_json::from_str("\"unknown\"").unwrap(); + assert_eq!(deserialized, unknown); + } + + #[test] + fn test_format_backend_serialization() { + let erasure = FormatBackend::Erasure; + let json = serde_json::to_string(&erasure).unwrap(); + assert_eq!(json, "\"xl\""); + + let single = FormatBackend::ErasureSingle; + let json = serde_json::to_string(&single).unwrap(); + assert_eq!(json, "\"xl-single\""); + + let unknown = FormatBackend::Unknown; + let deserialized: FormatBackend = serde_json::from_str("\"unknown\"").unwrap(); + assert_eq!(deserialized, unknown); + } + + #[test] + fn test_format_erasure_version_serialization() { + let v1 = FormatErasureVersion::V1; + let json = serde_json::to_string(&v1).unwrap(); + assert_eq!(json, "\"1\""); + + let v2 = FormatErasureVersion::V2; + let json = serde_json::to_string(&v2).unwrap(); + assert_eq!(json, "\"2\""); + + let v3 = FormatErasureVersion::V3; + let json = serde_json::to_string(&v3).unwrap(); + assert_eq!(json, "\"3\""); + } + + #[test] + fn test_distribution_algo_version_serialization() { + let v1 = DistributionAlgoVersion::V1; + let json = serde_json::to_string(&v1).unwrap(); + assert_eq!(json, "\"CRCMOD\""); + + let v2 = DistributionAlgoVersion::V2; + let json = serde_json::to_string(&v2).unwrap(); + assert_eq!(json, "\"SIPMOD\""); + + let v3 = DistributionAlgoVersion::V3; + let json = serde_json::to_string(&v3).unwrap(); + assert_eq!(json, "\"SIPMOD+PARITY\""); + } + + #[test] + fn test_format_v3_round_trip_serialization() { + let original = FormatV3::new(2, 3); + let json = original.to_json().unwrap(); + let deserialized = FormatV3::try_from(json.as_str()).unwrap(); + + assert_eq!(original.version, deserialized.version); + assert_eq!(original.format, deserialized.format); + assert_eq!(original.erasure.version, deserialized.erasure.version); + assert_eq!(original.erasure.sets.len(), deserialized.erasure.sets.len()); + assert_eq!(original.erasure.distribution_algo, deserialized.erasure.distribution_algo); + } +} diff --git a/crates/ecstore/src/layout/mod.rs b/crates/ecstore/src/layout/mod.rs index 4b1934524..0fd12aefc 100644 --- a/crates/ecstore/src/layout/mod.rs +++ b/crates/ecstore/src/layout/mod.rs @@ -4,4 +4,6 @@ //! topology separate from runtime `Sets`/`SetDisks` orchestration before any //! file moves happen. +pub(crate) mod disks_layout; +pub(crate) mod format; pub(crate) mod set_layout; diff --git a/crates/ecstore/src/layout/set_layout.rs b/crates/ecstore/src/layout/set_layout.rs index 63f8d4a31..cc73252a4 100644 --- a/crates/ecstore/src/layout/set_layout.rs +++ b/crates/ecstore/src/layout/set_layout.rs @@ -1,4 +1,4 @@ -use crate::disk::format::{DistributionAlgoVersion, FormatV3}; +use crate::layout::format::{DistributionAlgoVersion, FormatV3}; use std::collections::HashSet; use std::io::{Error, Result}; use uuid::Uuid; diff --git a/docs/architecture/ecstore-layout-boundary.md b/docs/architecture/ecstore-layout-boundary.md index d3a769ebf..1d616a4c1 100644 --- a/docs/architecture/ecstore-layout-boundary.md +++ b/docs/architecture/ecstore-layout-boundary.md @@ -33,6 +33,14 @@ expansion. It may describe: Static layout must not own disk handles, lock clients, reconnect loops, repair state, or shutdown signaling. +## Format And Disk Layout Ownership + +`layout::format` owns persisted format structures and disk UUID position lookup. +`layout::disks_layout` owns command-line volume expansion into pool/set layout. + +Compatibility paths remain available through `disk::format` and `disks_layout` +until downstream callers are moved or compatibility coverage allows removal. + ## Runtime Set Orchestration Runtime orchestration remains owned by `Sets` and `SetDisks` until a later pure diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index 9fe0c2d77..70d9e4f75 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,16 +5,17 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-ecstore-layout-foundation` -- Baseline: `overtrue/arch-extension-runtime-snapshots` - (`6ee557f63c4f8eb45334af23a6b3a32e0ee009e9`). -- Stacked on: local R-033, which is stacked on local R-032 and rustfs/rustfs#3642. +- Branch: `overtrue/arch-ecstore-layout-format-move` +- Baseline: `overtrue/arch-ecstore-layout-foundation` + (`2e10661d088a665c789fdb092871c6b52675aa1e`). +- Stacked on: local E-001/E-SET-001, which is stacked on local R-033 and + local R-032. - PR type for this branch: `pure-move` - Runtime behavior changes: none. -- Rust code changes: add the ECStore internal layout skeleton and read-only set - layout boundary tests. +- Rust code changes: pure-move ECStore format and disk-layout expansion modules + into the internal layout bucket while preserving old public paths. - CI/script changes: none. -- Docs changes: record the ECStore layout boundary and E-001/E-SET-001 slice. +- Docs changes: record the ECStore format/disk-layout pure move slice. ## Phase 0 Tasks @@ -2225,10 +2226,25 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block checks, migration/layer guards, formatting, diff hygiene, Rust risk scan, branch freshness check, pre-commit quality gate, and three-expert review. +- [x] `E-002/E-LAYOUT-001` Move ECStore format and disk-layout owners. + - Do: pure-move persisted format ownership and disk-layout expansion into + the ECStore layout bucket while keeping compatibility stubs at the old + public paths. + - Acceptance: `crate::disk::format::*` and `crate::disks_layout::*` remain + usable, `layout::format` owns `FormatV3`, and `layout::disks_layout` owns + CLI volume expansion. + - Must preserve: format JSON wire shape, disk UUID lookup, distribution + algorithm, `RUSTFS_ERASURE_SET_DRIVE_COUNT` handling, endpoint expansion, + and old public module paths. + - Verification: focused ECStore format and disks-layout tests, + ECStore/RustFS/Heal compile checks, migration/layer guards, formatting, + diff hygiene, Rust risk scan, branch freshness check, pre-commit quality + gate, and three-expert review. + ## Next PRs -1. `pure-move`: move ECStore layout files into the new layout bucket with - compatibility coverage after E-001/E-SET-001 lands. +1. `pure-move`: continue moving endpoint grouping and runtime-neutral layout + helpers once E-002/E-LAYOUT-001 lands. 2. `pure-move`: continue pruning residual embedded startup-only orchestration once the lifecycle helpers are merged. @@ -2236,9 +2252,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | E-001/E-SET-001 adds only ECStore layout skeleton boundaries and read-only set-layout contracts before file moves. | -| Migration preservation | passed | Format distribution, local disk replacement, lock client mapping, public module paths, and runtime set behavior remain unchanged. | -| Testing/verification | passed | Focused ECStore set-layout checks, compile checks, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. | +| Quality/architecture | passed | E-002/E-LAYOUT-001 is a pure move into ECStore layout with compatibility stubs at old public paths; no new runtime owner or dependency boundary is introduced. | +| Migration preservation | passed | Format JSON, disk UUID lookup, distribution algorithm, disk-layout expansion, and old public module paths remain preserved. | +| Testing/verification | passed | Focused format/layout checks, compile checks, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. | ## Verification Notes @@ -2300,6 +2316,19 @@ Passed before push: - `make pre-commit`: passed. - Three-expert review: passed. +- Issue #660 E-002/E-LAYOUT-001 current slice: + - `cargo test -p rustfs-ecstore format::test -- --nocapture`: passed. + - `cargo test -p rustfs-ecstore disks_layout -- --nocapture`: passed. + - `cargo check -p rustfs-ecstore -p rustfs -p rustfs-heal`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - Rust risk scan on changed Rust files: passed; only existing test-only + unwrap/println/panic/expect paths were present. + - `make pre-commit`: passed. + - Three-expert review: passed. + - Issue #660 X-012 current slice: - `cargo test -p rustfs-extension-schema`: passed. - `cargo check -p rustfs-extension-schema`: passed.