gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: move ecstore format layout owners (#3646)

Browse Source
This commit is contained in:
安正超
2026-06-20 09:58:50 +08:00
committed by GitHub
parent 88dfdda4a0
commit 16d667da0a
8 changed files with 1527 additions and 1458 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/disk/format.rs
+1 -532
View File
@@ -12,535 +12,4 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::disk::error::{Error, Result};
use crate::disk::{DiskInfo, error::DiskError};
use serde::{Deserialize, Serialize};
use serde_json::Error as JsonError;
use uuid::Uuid;
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum FormatMetaVersion {
#[serde(rename = "1")]
V1,
#[serde(other)]
Unknown,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum FormatBackend {
#[serde(rename = "xl")]
Erasure,
#[serde(rename = "xl-single")]
ErasureSingle,
#[serde(other)]
Unknown,
}
/// Represents the V3 backend disk structure version
/// under `.rustfs.sys` and actual data namespace.
///
/// FormatErasureV3 - structure holds format config version '3'.
///
/// The V3 format to support "large bucket" support where a bucket
/// can span multiple erasure sets.
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct FormatErasureV3 {
/// Version of 'xl' format.
pub version: FormatErasureVersion,
/// This field carries assigned disk uuid.
pub this: Uuid,
/// Sets field carries the input disk order generated the first
/// time when fresh disks were supplied, it is a two-dimensional
/// array second dimension represents list of disks used per set.
pub sets: Vec<Vec<Uuid>>,
/// Distribution algorithm represents the hashing algorithm
/// to pick the right set index for an object.
#[serde(rename = "distributionAlgo")]
pub distribution_algo: DistributionAlgoVersion,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum FormatErasureVersion {
#[serde(rename = "1")]
V1,
#[serde(rename = "2")]
V2,
#[serde(rename = "3")]
V3,
#[serde(other)]
Unknown,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum DistributionAlgoVersion {
#[serde(rename = "CRCMOD")]
V1,
#[serde(rename = "SIPMOD")]
V2,
#[serde(rename = "SIPMOD+PARITY")]
V3,
}
/// format.json currently has the format:
///
/// ```json
/// {
/// "version": "1",
/// "format": "XXXXX",
/// "id": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
/// "XXXXX": {
//
/// }
/// }
/// ```
///
/// Ideally we will never have a situation where we will have to change the
/// fields of this struct and deal with related migration.
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct FormatV3 {
/// Version of the format config.
pub version: FormatMetaVersion,
/// Format indicates the backend format type, supports two values 'xl' and 'xl-single'.
pub format: FormatBackend,
/// ID is the identifier for the rustfs deployment
pub id: Uuid,
#[serde(rename = "xl")]
pub erasure: FormatErasureV3,
// /// DiskInfo is an extended type which returns current
// /// disk usage per path.
#[serde(skip)]
pub disk_info: Option<DiskInfo>,
}
impl TryFrom<&[u8]> for FormatV3 {
type Error = JsonError;
fn try_from(data: &[u8]) -> std::result::Result<Self, Self::Error> {
serde_json::from_slice(data)
}
}
impl TryFrom<&str> for FormatV3 {
type Error = JsonError;
fn try_from(data: &str) -> std::result::Result<Self, Self::Error> {
serde_json::from_str(data)
}
}
impl FormatV3 {
/// Create a new format config with the given number of sets and set length.
pub fn new(num_sets: usize, set_len: usize) -> Self {
let format = if set_len == 1 {
FormatBackend::ErasureSingle
} else {
FormatBackend::Erasure
};
let erasure = FormatErasureV3 {
version: FormatErasureVersion::V3,
this: Uuid::nil(),
sets: (0..num_sets)
.map(|_| (0..set_len).map(|_| Uuid::new_v4()).collect())
.collect(),
distribution_algo: DistributionAlgoVersion::V3,
};
Self {
version: FormatMetaVersion::V1,
format,
id: Uuid::new_v4(),
erasure,
disk_info: None,
}
}
/// Returns the number of drives in the erasure set.
pub fn drives(&self) -> usize {
self.erasure.sets.iter().map(|v| v.len()).sum()
}
pub fn to_json(&self) -> std::result::Result<String, JsonError> {
serde_json::to_string(self)
}
/// returns the i,j'th position of the input `diskID` against the reference
///
/// format, after successful validation.
/// - i'th position is the set index
/// - j'th position is the disk index in the current set
pub fn find_disk_index_by_disk_id(&self, disk_id: Uuid) -> Result<(usize, usize)> {
if disk_id == Uuid::nil() {
return Err(Error::from(DiskError::DiskNotFound));
}
if disk_id == Uuid::max() {
return Err(Error::other("disk offline"));
}
for (i, set) in self.erasure.sets.iter().enumerate() {
for (j, d) in set.iter().enumerate() {
if disk_id.eq(d) {
return Ok((i, j));
}
}
}
Err(Error::other(format!("disk id not found {disk_id}")))
}
pub fn check_other(&self, other: &FormatV3) -> Result<()> {
let mut tmp = other.clone();
let this = tmp.erasure.this;
tmp.erasure.this = Uuid::nil();
if self.erasure.sets.len() != other.erasure.sets.len() {
return Err(Error::other(format!(
"Expected number of sets {}, got {}",
self.erasure.sets.len(),
other.erasure.sets.len()
)));
}
for i in 0..self.erasure.sets.len() {
if self.erasure.sets[i].len() != other.erasure.sets[i].len() {
return Err(Error::other(format!(
"Each set should be of same size, expected {}, got {}",
self.erasure.sets[i].len(),
other.erasure.sets[i].len()
)));
}
for j in 0..self.erasure.sets[i].len() {
if self.erasure.sets[i][j] != other.erasure.sets[i][j] {
return Err(Error::other(format!(
"UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)",
i,
j,
self.erasure.sets[i][j].to_string(),
other.erasure.sets[i][j].to_string(),
)));
}
}
}
for i in 0..tmp.erasure.sets.len() {
for j in 0..tmp.erasure.sets[i].len() {
if this == tmp.erasure.sets[i][j] {
return Ok(());
}
}
}
Err(Error::other(format!(
"DriveID {:?} not found in any drive sets {:?}",
this, other.erasure.sets
)))
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_format_v1() {
let format = FormatV3::new(1, 4);
let str = serde_json::to_string(&format);
println!("{str:?}");
let data = r#"
{
"version": "1",
"format": "xl",
"id": "321b3874-987d-4c15-8fa5-757c956b1243",
"xl": {
"version": "1",
"this": null,
"sets": [
[
"8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
"c26315da-05cf-4778-a9ea-b44ea09f58c5",
"fb87a891-18d3-44cf-a46f-bcc15093a038",
"356a925c-57b9-4313-88b3-053edf1104dc"
]
],
"distributionAlgo": "CRCMOD"
}
}"#;
let p = FormatV3::try_from(data);
println!("{p:?}");
}
#[test]
fn test_format_v3_new_single_disk() {
let format = FormatV3::new(1, 1);
assert_eq!(format.version, FormatMetaVersion::V1);
assert_eq!(format.format, FormatBackend::ErasureSingle);
assert_eq!(format.erasure.version, FormatErasureVersion::V3);
assert_eq!(format.erasure.sets.len(), 1);
assert_eq!(format.erasure.sets[0].len(), 1);
assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3);
assert_eq!(format.erasure.this, Uuid::nil());
}
#[test]
fn test_format_v3_new_multiple_sets() {
let format = FormatV3::new(2, 4);
assert_eq!(format.version, FormatMetaVersion::V1);
assert_eq!(format.format, FormatBackend::Erasure);
assert_eq!(format.erasure.version, FormatErasureVersion::V3);
assert_eq!(format.erasure.sets.len(), 2);
assert_eq!(format.erasure.sets[0].len(), 4);
assert_eq!(format.erasure.sets[1].len(), 4);
assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3);
}
#[test]
fn test_format_v3_drives() {
let format = FormatV3::new(2, 4);
assert_eq!(format.drives(), 8); // 2 sets * 4 drives each
let format_single = FormatV3::new(1, 1);
assert_eq!(format_single.drives(), 1); // 1 set * 1 drive
}
#[test]
fn test_format_v3_to_json() {
let format = FormatV3::new(1, 2);
let json_result = format.to_json();
assert!(json_result.is_ok());
let json_str = json_result.unwrap();
assert!(json_str.contains("\"version\":\"1\""));
assert!(json_str.contains("\"format\":\"xl\""));
}
#[test]
fn test_format_v3_from_json() {
let json_data = r#"{
"version": "1",
"format": "xl-single",
"id": "321b3874-987d-4c15-8fa5-757c956b1243",
"xl": {
"version": "3",
"this": "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
"sets": [
[
"8ab9a908-f869-4f1f-8e42-eb067ffa7eb5"
]
],
"distributionAlgo": "SIPMOD+PARITY"
}
}"#;
let format = FormatV3::try_from(json_data);
assert!(format.is_ok());
let format = format.unwrap();
assert_eq!(format.format, FormatBackend::ErasureSingle);
assert_eq!(format.erasure.version, FormatErasureVersion::V3);
assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3);
assert_eq!(format.erasure.sets.len(), 1);
assert_eq!(format.erasure.sets[0].len(), 1);
}
#[test]
fn test_format_v3_from_bytes() {
let json_data = r#"{
"version": "1",
"format": "xl",
"id": "321b3874-987d-4c15-8fa5-757c956b1243",
"xl": {
"version": "2",
"this": "00000000-0000-0000-0000-000000000000",
"sets": [
[
"8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
"c26315da-05cf-4778-a9ea-b44ea09f58c5"
]
],
"distributionAlgo": "SIPMOD"
}
}"#;
let format = FormatV3::try_from(json_data.as_bytes());
assert!(format.is_ok());
let format = format.unwrap();
assert_eq!(format.erasure.version, FormatErasureVersion::V2);
assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V2);
assert_eq!(format.erasure.sets[0].len(), 2);
}
#[test]
fn test_format_v3_invalid_json() {
let invalid_json = r#"{"invalid": "json"}"#;
let format = FormatV3::try_from(invalid_json);
assert!(format.is_err());
}
#[test]
fn test_find_disk_index_by_disk_id() {
let mut format = FormatV3::new(2, 2);
let target_disk_id = Uuid::new_v4();
format.erasure.sets[1][0] = target_disk_id;
let result = format.find_disk_index_by_disk_id(target_disk_id);
assert!(result.is_ok());
assert_eq!(result.unwrap(), (1, 0));
}
#[test]
fn test_find_disk_index_nil_uuid() {
let format = FormatV3::new(1, 2);
let result = format.find_disk_index_by_disk_id(Uuid::nil());
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::DiskNotFound));
}
#[test]
fn test_find_disk_index_max_uuid() {
let format = FormatV3::new(1, 2);
let result = format.find_disk_index_by_disk_id(Uuid::max());
assert!(result.is_err());
}
#[test]
fn test_find_disk_index_not_found() {
let format = FormatV3::new(1, 2);
let non_existent_id = Uuid::new_v4();
let result = format.find_disk_index_by_disk_id(non_existent_id);
assert!(result.is_err());
}
#[test]
fn test_check_other_identical() {
let format1 = FormatV3::new(2, 4);
let mut format2 = format1.clone();
format2.erasure.this = format1.erasure.sets[0][0];
let result = format1.check_other(&format2);
assert!(result.is_ok());
}
#[test]
fn test_check_other_different_set_count() {
let format1 = FormatV3::new(2, 4);
let format2 = FormatV3::new(3, 4);
let result = format1.check_other(&format2);
assert!(result.is_err());
}
#[test]
fn test_check_other_different_set_size() {
let format1 = FormatV3::new(2, 4);
let format2 = FormatV3::new(2, 6);
let result = format1.check_other(&format2);
assert!(result.is_err());
}
#[test]
fn test_check_other_different_disk_id() {
let format1 = FormatV3::new(1, 2);
let mut format2 = format1.clone();
format2.erasure.sets[0][0] = Uuid::new_v4();
let result = format1.check_other(&format2);
assert!(result.is_err());
}
#[test]
fn test_check_other_disk_not_in_sets() {
let format1 = FormatV3::new(1, 2);
let mut format2 = format1.clone();
format2.erasure.this = Uuid::new_v4(); // Set to a UUID not in any set
let result = format1.check_other(&format2);
assert!(result.is_err());
}
#[test]
fn test_format_meta_version_serialization() {
let v1 = FormatMetaVersion::V1;
let json = serde_json::to_string(&v1).unwrap();
assert_eq!(json, "\"1\"");
let unknown = FormatMetaVersion::Unknown;
let deserialized: FormatMetaVersion = serde_json::from_str("\"unknown\"").unwrap();
assert_eq!(deserialized, unknown);
}
#[test]
fn test_format_backend_serialization() {
let erasure = FormatBackend::Erasure;
let json = serde_json::to_string(&erasure).unwrap();
assert_eq!(json, "\"xl\"");
let single = FormatBackend::ErasureSingle;
let json = serde_json::to_string(&single).unwrap();
assert_eq!(json, "\"xl-single\"");
let unknown = FormatBackend::Unknown;
let deserialized: FormatBackend = serde_json::from_str("\"unknown\"").unwrap();
assert_eq!(deserialized, unknown);
}
#[test]
fn test_format_erasure_version_serialization() {
let v1 = FormatErasureVersion::V1;
let json = serde_json::to_string(&v1).unwrap();
assert_eq!(json, "\"1\"");
let v2 = FormatErasureVersion::V2;
let json = serde_json::to_string(&v2).unwrap();
assert_eq!(json, "\"2\"");
let v3 = FormatErasureVersion::V3;
let json = serde_json::to_string(&v3).unwrap();
assert_eq!(json, "\"3\"");
}
#[test]
fn test_distribution_algo_version_serialization() {
let v1 = DistributionAlgoVersion::V1;
let json = serde_json::to_string(&v1).unwrap();
assert_eq!(json, "\"CRCMOD\"");
let v2 = DistributionAlgoVersion::V2;
let json = serde_json::to_string(&v2).unwrap();
assert_eq!(json, "\"SIPMOD\"");
let v3 = DistributionAlgoVersion::V3;
let json = serde_json::to_string(&v3).unwrap();
assert_eq!(json, "\"SIPMOD+PARITY\"");
}
#[test]
fn test_format_v3_round_trip_serialization() {
let original = FormatV3::new(2, 3);
let json = original.to_json().unwrap();
let deserialized = FormatV3::try_from(json.as_str()).unwrap();
assert_eq!(original.version, deserialized.version);
assert_eq!(original.format, deserialized.format);
assert_eq!(original.erasure.version, deserialized.erasure.version);
assert_eq!(original.erasure.sets.len(), deserialized.erasure.sets.len());
assert_eq!(original.erasure.distribution_algo, deserialized.erasure.distribution_algo);
}
}
pub use crate::layout::format::*;
crates/ecstore/src/disks_layout.rs
+1 -913
View File
@@ -12,916 +12,4 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_utils::string::{ArgPattern, find_ellipses_patterns, has_ellipses};
use serde::Deserialize;
use std::collections::HashSet;
use std::env;
use std::io::{Error, Result};
use tracing::debug;
/// Supported set sizes this is used to find the optimal
/// single set size.
const SET_SIZES: [usize; 15] = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16];
const ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT: &str = "RUSTFS_ERASURE_SET_DRIVE_COUNT";
#[derive(Deserialize, Debug, Default)]
pub struct PoolDisksLayout {
cmd_line: String,
layout: Vec<Vec<String>>,
}
impl PoolDisksLayout {
fn new(args: impl Into<String>, layout: Vec<Vec<String>>) -> Self {
PoolDisksLayout {
cmd_line: args.into(),
layout,
}
}
fn count(&self) -> usize {
self.layout.len()
}
fn get_cmd_line(&self) -> &str {
&self.cmd_line
}
pub fn iter(&self) -> impl Iterator<Item = &Vec<String>> {
self.layout.iter()
}
}
#[derive(Deserialize, Debug, Default)]
pub struct DisksLayout {
pub legacy: bool,
pub pools: Vec<PoolDisksLayout>,
}
// impl<T: AsRef<str>> TryFrom<&[T]> for DisksLayout {
// type Error = Error;
// fn try_from(args: &[T]) -> Result<Self, Self::Error> {
// if args.is_empty() {
// return Err(Error::from_string("Invalid argument"));
// }
// let is_ellipses = args.iter().any(|v| has_ellipses(&[v]));
// // None of the args have ellipses use the old style.
// if !is_ellipses {
// let set_args = get_all_sets(is_ellipses, args)?;
// return Ok(DisksLayout {
// legacy: true,
// pools: vec![PoolDisksLayout::new(
// args.iter().map(AsRef::as_ref).collect::<Vec<&str>>().join(" "),
// set_args,
// )],
// });
// }
// let mut layout = Vec::with_capacity(args.len());
// for arg in args.iter() {
// if !has_ellipses(&[arg]) && args.len() > 1 {
// return Err(Error::from_string(
// "all args must have ellipses for pool expansion (Invalid arguments specified)",
// ));
// }
// let set_args = get_all_sets(is_ellipses, &[arg])?;
// layout.push(PoolDisksLayout::new(arg.as_ref(), set_args));
// }
// Ok(DisksLayout {
// legacy: false,
// pools: layout,
// })
// }
// }
impl DisksLayout {
pub fn from_volumes<T: AsRef<str>>(args: &[T]) -> Result<Self> {
if args.is_empty() {
return Err(Error::other("Invalid argument"));
}
let is_ellipses = args.iter().any(|v| has_ellipses(&[v]));
let set_drive_count_env = env::var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT).unwrap_or_else(|err| {
debug!("{} not set use default:0, {:?}", ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, err);
"0".to_string()
});
let set_drive_count: usize = set_drive_count_env.parse().map_err(Error::other)?;
// None of the args have ellipses use the old style.
if !is_ellipses {
let set_args = get_all_sets(set_drive_count, is_ellipses, args)?;
return Ok(DisksLayout {
legacy: true,
pools: vec![PoolDisksLayout::new(
args.iter().map(AsRef::as_ref).collect::<Vec<&str>>().join(" "),
set_args,
)],
});
}
let mut layout = Vec::with_capacity(args.len());
for arg in args.iter() {
if !has_ellipses(&[arg]) && args.len() > 1 {
return Err(Error::other(
"all args must have ellipses for pool expansion (Invalid arguments specified)",
));
}
let set_args = get_all_sets(set_drive_count, is_ellipses, &[arg])?;
layout.push(PoolDisksLayout::new(arg.as_ref(), set_args));
}
Ok(DisksLayout {
legacy: false,
pools: layout,
})
}
pub fn is_empty_layout(&self) -> bool {
self.pools.is_empty()
|| self.pools[0].layout.is_empty()
|| self.pools[0].layout[0].is_empty()
|| self.pools[0].layout[0][0].is_empty()
}
pub fn is_single_drive_layout(&self) -> bool {
self.pools.len() == 1 && self.pools[0].layout.len() == 1 && self.pools[0].layout[0].len() == 1
}
pub fn get_single_drive_layout(&self) -> &str {
&self.pools[0].layout[0][0]
}
/// returns the total number of sets in the layout.
pub fn get_set_count(&self, i: usize) -> usize {
self.pools.get(i).map_or(0, |v| v.count())
}
/// returns the total number of drives in the layout.
pub fn get_drives_per_set(&self, i: usize) -> usize {
self.pools.get(i).map_or(0, |v| v.layout.first().map_or(0, |v| v.len()))
}
/// returns the command line for the given index.
pub fn get_cmd_line(&self, i: usize) -> String {
self.pools.get(i).map_or(String::new(), |v| v.get_cmd_line().to_owned())
}
}
/// parses all ellipses input arguments, expands them into
/// corresponding list of endpoints chunked evenly in accordance with a
/// specific set size.
///
/// For example: {1...64} is divided into 4 sets each of size 16.
/// This applies to even distributed setup syntax as well.
fn get_all_sets<T: AsRef<str>>(set_drive_count: usize, is_ellipses: bool, args: &[T]) -> Result<Vec<Vec<String>>> {
let endpoint_set = if is_ellipses {
EndpointSet::from_volumes(args, set_drive_count)?
} else {
let set_indexes = if args.len() > 1 {
get_set_indexes(args, &[args.len()], set_drive_count, &[])?
} else {
vec![vec![args.len()]]
};
let endpoints = args.iter().map(|v| v.as_ref().to_string()).collect();
EndpointSet::new(endpoints, set_indexes)
};
let set_args = endpoint_set.get();
let mut unique_args = HashSet::with_capacity(set_args.len());
for args in set_args.iter() {
for arg in args {
if unique_args.contains(arg) {
return Err(Error::other(format!("Input args {arg} has duplicate ellipses")));
}
unique_args.insert(arg);
}
}
Ok(set_args)
}
/// represents parsed ellipses values, also provides
/// methods to get the sets of endpoints.
#[derive(Debug, Default)]
struct EndpointSet {
_arg_patterns: Vec<ArgPattern>,
endpoints: Vec<String>,
set_indexes: Vec<Vec<usize>>,
}
// impl<T: AsRef<str>> TryFrom<&[T]> for EndpointSet {
// type Error = Error;
// fn try_from(args: &[T]) -> Result<Self, Self::Error> {
// let mut arg_patterns = Vec::with_capacity(args.len());
// for arg in args {
// arg_patterns.push(find_ellipses_patterns(arg.as_ref())?);
// }
// let total_sizes = get_total_sizes(&arg_patterns);
// let set_indexes = get_set_indexes(args, &total_sizes, &arg_patterns)?;
// let mut endpoints = Vec::new();
// for ap in arg_patterns.iter() {
// let aps = ap.expand();
// for bs in aps {
// endpoints.push(bs.join(""));
// }
// }
// Ok(EndpointSet {
// set_indexes,
// _arg_patterns: arg_patterns,
// endpoints,
// })
// }
// }
impl EndpointSet {
/// Create a new EndpointSet with the given endpoints and set indexes.
pub fn new(endpoints: Vec<String>, set_indexes: Vec<Vec<usize>>) -> Self {
Self {
endpoints,
set_indexes,
..Default::default()
}
}
pub fn from_volumes<T: AsRef<str>>(args: &[T], set_drive_count: usize) -> Result<Self> {
let mut arg_patterns = Vec::with_capacity(args.len());
for arg in args {
arg_patterns.push(find_ellipses_patterns(arg.as_ref())?);
}
let total_sizes = get_total_sizes(&arg_patterns);
let set_indexes = get_set_indexes(args, &total_sizes, set_drive_count, &arg_patterns)?;
let mut endpoints = Vec::new();
for ap in arg_patterns.iter() {
let aps = ap.expand();
for bs in aps {
endpoints.push(bs.join(""));
}
}
Ok(EndpointSet {
set_indexes,
_arg_patterns: arg_patterns,
endpoints,
})
}
/// returns the sets representation of the endpoints
/// this function also intelligently decides on what will
/// be the right set size etc.
pub fn get(&self) -> Vec<Vec<String>> {
let mut sets: Vec<Vec<String>> = Vec::new();
let mut start = 0;
for set_idx in self.set_indexes.iter() {
for idx in set_idx {
let end = idx + start;
sets.push(self.endpoints[start..end].to_vec());
start = end;
}
}
sets
}
}
/// returns the greatest common divisor of all the ellipses sizes.
fn get_divisible_size(total_sizes: &[usize]) -> usize {
fn gcd(mut x: usize, mut y: usize) -> usize {
while y != 0 {
// be equivalent to: x, y = y, x%y
std::mem::swap(&mut x, &mut y);
y %= x;
}
x
}
total_sizes.iter().skip(1).fold(total_sizes[0], |acc, &y| gcd(acc, y))
}
fn possible_set_counts(set_size: usize) -> Vec<usize> {
let mut ss = Vec::new();
for s in SET_SIZES {
if set_size.is_multiple_of(s) {
ss.push(s);
}
}
ss
}
/// checks whether given count is a valid set size for erasure coding.
fn is_valid_set_size(count: usize) -> bool {
count >= SET_SIZES[0] && count <= SET_SIZES[SET_SIZES.len() - 1]
}
/// Final set size with all the symmetry accounted for.
fn common_set_drive_count(divisible_size: usize, set_counts: &[usize]) -> usize {
// prefers set_counts to be sorted for optimal behavior.
if divisible_size < set_counts[set_counts.len() - 1] {
return divisible_size;
}
let mut prev_d = divisible_size / set_counts[0];
let mut set_size = 0;
for &cnt in set_counts {
if divisible_size.is_multiple_of(cnt) {
let d = divisible_size / cnt;
if d <= prev_d {
prev_d = d;
set_size = cnt;
}
}
}
set_size
}
/// returns symmetrical setCounts based on the input argument patterns,
/// the symmetry calculation is to ensure that we also use uniform number
/// of drives common across all ellipses patterns.
fn possible_set_counts_with_symmetry(set_counts: &[usize], arg_patterns: &[ArgPattern]) -> Vec<usize> {
let mut new_set_counts: HashSet<usize> = HashSet::new();
for &ss in set_counts {
let mut symmetry = false;
for arg_pattern in arg_patterns {
for p in arg_pattern.as_ref().iter() {
if p.len() > ss {
symmetry = (p.len() % ss) == 0;
} else {
symmetry = (ss % p.len()) == 0;
}
}
}
if !new_set_counts.contains(&ss) && (symmetry || arg_patterns.is_empty()) {
new_set_counts.insert(ss);
}
}
let mut set_counts: Vec<usize> = new_set_counts.into_iter().collect();
set_counts.sort_unstable();
set_counts
}
/// returns list of indexes which provides the set size
/// on each index, this function also determines the final set size
/// The final set size has the affinity towards choosing smaller
/// indexes (total sets)
fn get_set_indexes<T: AsRef<str>>(
args: &[T],
total_sizes: &[usize],
set_drive_count: usize,
arg_patterns: &[ArgPattern],
) -> Result<Vec<Vec<usize>>> {
if args.is_empty() || total_sizes.is_empty() {
return Err(Error::other("Invalid argument"));
}
for &size in total_sizes {
// Check if total_sizes has minimum range upto set_size
if size < SET_SIZES[0] || size < set_drive_count {
return Err(Error::other(format!("Incorrect number of endpoints provided, size {size}")));
}
}
let common_size = get_divisible_size(total_sizes);
let mut set_counts = possible_set_counts(common_size);
if set_counts.is_empty() {
return Err(Error::other(format!(
"Incorrect number of endpoints provided, number of drives {} is not divisible by any supported erasure set sizes {}",
common_size, 0
)));
}
// Returns possible set counts with symmetry.
set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns);
if set_counts.is_empty() {
return Err(Error::other("No symmetric distribution detected with input endpoints provided"));
}
let set_size = {
if set_drive_count > 0 {
let has_set_drive_count = set_counts.contains(&set_drive_count);
if !has_set_drive_count {
return Err(Error::other(format!(
"Invalid set drive count {}. Acceptable values for {:?} number drives are {:?}",
set_drive_count, common_size, &set_counts
)));
}
set_drive_count
} else {
set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns);
if set_counts.is_empty() {
return Err(Error::other(format!(
"No symmetric distribution detected with input endpoints , drives {} cannot be spread symmetrically by any supported erasure set sizes {:?}",
common_size, &set_counts
)));
}
// Final set size with all the symmetry accounted for.
common_set_drive_count(common_size, &set_counts)
}
};
if !is_valid_set_size(set_size) {
return Err(Error::other("Incorrect number of endpoints provided3"));
}
Ok(total_sizes
.iter()
.map(|&size| (0..(size / set_size)).map(|_| set_size).collect())
.collect())
}
/// Return the total size for each argument patterns.
fn get_total_sizes(arg_patterns: &[ArgPattern]) -> Vec<usize> {
arg_patterns.iter().map(|v| v.total_sizes()).collect()
}
#[cfg(test)]
mod test {
use rustfs_utils::string::Pattern;
use super::*;
impl PartialEq for EndpointSet {
fn eq(&self, other: &Self) -> bool {
self._arg_patterns == other._arg_patterns && self.set_indexes == other.set_indexes
}
}
#[test]
fn test_get_divisible_size() {
struct TestCase {
total_sizes: Vec<usize>,
result: usize,
}
let test_cases = [
TestCase {
total_sizes: vec![24, 32, 16],
result: 8,
},
TestCase {
total_sizes: vec![32, 8, 4],
result: 4,
},
TestCase {
total_sizes: vec![8, 8, 8],
result: 8,
},
TestCase {
total_sizes: vec![24],
result: 24,
},
];
for (i, test_case) in test_cases.iter().enumerate() {
let ret = get_divisible_size(&test_case.total_sizes);
assert_eq!(ret, test_case.result, "Test{}: Expected {}, got {}", i + 1, test_case.result, ret);
}
}
#[test]
fn test_get_set_indexes() {
#[derive(Default)]
struct TestCase<'a> {
num: usize,
args: Vec<&'a str>,
total_sizes: Vec<usize>,
indexes: Vec<Vec<usize>>,
success: bool,
}
let test_cases = [
TestCase {
num: 1,
args: vec!["data{1...17}/export{1...52}"],
total_sizes: vec![14144],
..Default::default()
},
TestCase {
num: 2,
args: vec!["data{1...3}"],
total_sizes: vec![3],
indexes: vec![vec![3]],
success: true,
},
TestCase {
num: 3,
args: vec!["data/controller1/export{1...2}, data/controller2/export{1...4}, data/controller3/export{1...8}"],
total_sizes: vec![2, 4, 8],
indexes: vec![vec![2], vec![2, 2], vec![2, 2, 2, 2]],
success: true,
},
TestCase {
num: 4,
args: vec!["data{1...27}"],
total_sizes: vec![27],
indexes: vec![vec![9, 9, 9]],
success: true,
},
TestCase {
num: 5,
args: vec!["http://host{1...3}/data{1...180}"],
total_sizes: vec![540],
indexes: vec![vec![
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15,
]],
success: true,
},
TestCase {
num: 6,
args: vec!["http://host{1...2}.rack{1...4}/data{1...180}"],
total_sizes: vec![1440],
indexes: vec![vec![
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16,
]],
success: true,
},
TestCase {
num: 7,
args: vec!["http://host{1...2}/data{1...180}"],
total_sizes: vec![360],
indexes: vec![vec![
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
12, 12, 12,
]],
success: true,
},
TestCase {
num: 8,
args: vec!["data/controller1/export{1...4}, data/controller2/export{1...8}, data/controller3/export{1...12}"],
total_sizes: vec![4, 8, 12],
indexes: vec![vec![4], vec![4, 4], vec![4, 4, 4]],
success: true,
},
TestCase {
num: 9,
args: vec!["data{1...64}"],
total_sizes: vec![64],
indexes: vec![vec![16, 16, 16, 16]],
success: true,
},
TestCase {
num: 10,
args: vec!["data{1...24}"],
total_sizes: vec![24],
indexes: vec![vec![12, 12]],
success: true,
},
TestCase {
num: 11,
args: vec!["data/controller{1...11}/export{1...8}"],
total_sizes: vec![88],
indexes: vec![vec![11, 11, 11, 11, 11, 11, 11, 11]],
success: true,
},
TestCase {
num: 12,
args: vec!["data{1...4}"],
total_sizes: vec![4],
indexes: vec![vec![4]],
success: true,
},
TestCase {
num: 13,
args: vec!["data/controller1/export{1...10}, data/controller2/export{1...10}, data/controller3/export{1...10}"],
total_sizes: vec![10, 10, 10],
indexes: vec![vec![10], vec![10], vec![10]],
success: true,
},
TestCase {
num: 14,
args: vec!["data{1...16}/export{1...52}"],
total_sizes: vec![832],
indexes: vec![vec![
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
]],
success: true,
},
TestCase {
num: 15,
args: vec!["https://node{1...3}.example.net/mnt/drive{1...8}"],
total_sizes: vec![24],
indexes: vec![vec![12, 12]],
success: true,
},
];
for test_case in test_cases {
let mut arg_patterns = Vec::new();
for v in test_case.args.iter() {
match find_ellipses_patterns(v) {
Ok(patterns) => {
arg_patterns.push(patterns);
}
Err(err) => {
panic!("Test{}: Unexpected failure {:?}", test_case.num, err);
}
}
}
match get_set_indexes(test_case.args.as_slice(), test_case.total_sizes.as_slice(), 0, arg_patterns.as_slice()) {
Ok(got_indexes) => {
if !test_case.success {
panic!("Test{}: Expected failure but passed instead", test_case.num);
}
assert_eq!(
test_case.indexes, got_indexes,
"Test{}: Expected {:?}, got {:?}",
test_case.num, test_case.indexes, got_indexes
)
}
Err(err) => {
if test_case.success {
panic!("Test{}: Expected success but failed instead {:?}", test_case.num, err);
}
}
}
}
}
fn get_sequences(start: usize, number: usize, padding_len: usize) -> Vec<String> {
let mut seq = Vec::new();
for i in start..=number {
if padding_len == 0 {
seq.push(format!("{i}"));
} else {
seq.push(format!("{i:0padding_len$}"));
}
}
seq
}
#[test]
fn test_into_endpoint_set() {
#[derive(Default)]
struct TestCase<'a> {
num: usize,
arg: &'a str,
es: EndpointSet,
success: bool,
}
let test_cases = [
// Tests invalid inputs.
TestCase {
num: 1,
arg: "...",
..Default::default()
},
// No range specified.
TestCase {
num: 2,
arg: "{...}",
..Default::default()
},
// Invalid range.
TestCase {
num: 3,
arg: "http://rustfs{2...3}/export/set{1...0}",
..Default::default()
},
// Range cannot be smaller than 4 minimum.
TestCase {
num: 4,
arg: "/export{1..2}",
..Default::default()
},
// Unsupported characters.
TestCase {
num: 5,
arg: "/export/test{1...2O}",
..Default::default()
},
// Tests valid inputs.
TestCase {
num: 6,
arg: "{1...27}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![Pattern {
seq: get_sequences(1, 27, 0),
..Default::default()
}])],
set_indexes: vec![vec![9, 9, 9]],
..Default::default()
},
success: true,
},
TestCase {
num: 7,
arg: "/export/set{1...64}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![Pattern {
seq: get_sequences(1, 64, 0),
prefix: "/export/set".to_owned(),
..Default::default()
}])],
set_indexes: vec![vec![16, 16, 16, 16]],
..Default::default()
},
success: true,
},
// Valid input for distributed setup.
TestCase {
num: 8,
arg: "http://rustfs{2...3}/export/set{1...64}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![
Pattern {
seq: get_sequences(1, 64, 0),
..Default::default()
},
Pattern {
seq: get_sequences(2, 3, 0),
prefix: "http://rustfs".to_owned(),
suffix: "/export/set".to_owned(),
},
])],
set_indexes: vec![vec![16, 16, 16, 16, 16, 16, 16, 16]],
..Default::default()
},
success: true,
},
// Supporting some advanced cases.
TestCase {
num: 9,
arg: "http://rustfs{1...64}.mydomain.net/data",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![Pattern {
seq: get_sequences(1, 64, 0),
prefix: "http://rustfs".to_owned(),
suffix: ".mydomain.net/data".to_owned(),
}])],
set_indexes: vec![vec![16, 16, 16, 16]],
..Default::default()
},
success: true,
},
TestCase {
num: 10,
arg: "http://rack{1...4}.mydomain.rustfs{1...16}/data",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![
Pattern {
seq: get_sequences(1, 16, 0),
suffix: "/data".to_owned(),
..Default::default()
},
Pattern {
seq: get_sequences(1, 4, 0),
prefix: "http://rack".to_owned(),
suffix: ".mydomain.rustfs".to_owned(),
},
])],
set_indexes: vec![vec![16, 16, 16, 16]],
..Default::default()
},
success: true,
},
// Supporting kubernetes cases.
TestCase {
num: 11,
arg: "http://rustfs{0...15}.mydomain.net/data{0...1}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![
Pattern {
seq: get_sequences(0, 1, 0),
..Default::default()
},
Pattern {
seq: get_sequences(0, 15, 0),
prefix: "http://rustfs".to_owned(),
suffix: ".mydomain.net/data".to_owned(),
},
])],
set_indexes: vec![vec![16, 16]],
..Default::default()
},
success: true,
},
// No host regex, just disks.
TestCase {
num: 12,
arg: "http://server1/data{1...32}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![Pattern {
seq: get_sequences(1, 32, 0),
prefix: "http://server1/data".to_owned(),
..Default::default()
}])],
set_indexes: vec![vec![16, 16]],
..Default::default()
},
success: true,
},
// No host regex, just disks with two position numerics.
TestCase {
num: 13,
arg: "http://server1/data{01...32}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![Pattern {
seq: get_sequences(1, 32, 2),
prefix: "http://server1/data".to_owned(),
..Default::default()
}])],
set_indexes: vec![vec![16, 16]],
..Default::default()
},
success: true,
},
// More than 2 ellipses are supported as well.
TestCase {
num: 14,
arg: "http://rustfs{2...3}/export/set{1...64}/test{1...2}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![
Pattern {
seq: get_sequences(1, 2, 0),
..Default::default()
},
Pattern {
seq: get_sequences(1, 64, 0),
suffix: "/test".to_owned(),
..Default::default()
},
Pattern {
seq: get_sequences(2, 3, 0),
prefix: "http://rustfs".to_owned(),
suffix: "/export/set".to_owned(),
},
])],
set_indexes: vec![vec![16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16]],
..Default::default()
},
success: true,
},
// More than an ellipse per argument for standalone setup.
TestCase {
num: 15,
arg: "/export{1...10}/disk{1...10}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![
Pattern {
seq: get_sequences(1, 10, 0),
..Default::default()
},
Pattern {
seq: get_sequences(1, 10, 0),
prefix: "/export".to_owned(),
suffix: "/disk".to_owned(),
},
])],
set_indexes: vec![vec![10, 10, 10, 10, 10, 10, 10, 10, 10, 10]],
..Default::default()
},
success: true,
},
];
for test_case in test_cases {
match EndpointSet::from_volumes([test_case.arg].as_slice(), 0) {
Ok(got_es) => {
if !test_case.success {
panic!("Test{}: Expected failure but passed instead", test_case.num);
}
assert_eq!(
test_case.es, got_es,
"Test{}: Expected {:?}, got {:?}",
test_case.num, test_case.es, got_es
)
}
Err(err) => {
if test_case.success {
panic!("Test{}: Expected success but failed instead {:?}", test_case.num, err);
}
}
}
}
}
}
pub use crate::layout::disks_layout::*;
crates/ecstore/src/layout/disks_layout.rs
+927
View File
@@ -0,0 +1,927 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_utils::string::{ArgPattern, find_ellipses_patterns, has_ellipses};
use serde::Deserialize;
use std::collections::HashSet;
use std::env;
use std::io::{Error, Result};
use tracing::debug;
/// Supported set sizes this is used to find the optimal
/// single set size.
const SET_SIZES: [usize; 15] = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16];
const ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT: &str = "RUSTFS_ERASURE_SET_DRIVE_COUNT";
#[derive(Deserialize, Debug, Default)]
pub struct PoolDisksLayout {
cmd_line: String,
layout: Vec<Vec<String>>,
}
impl PoolDisksLayout {
fn new(args: impl Into<String>, layout: Vec<Vec<String>>) -> Self {
PoolDisksLayout {
cmd_line: args.into(),
layout,
}
}
fn count(&self) -> usize {
self.layout.len()
}
fn get_cmd_line(&self) -> &str {
&self.cmd_line
}
pub fn iter(&self) -> impl Iterator<Item = &Vec<String>> {
self.layout.iter()
}
}
#[derive(Deserialize, Debug, Default)]
pub struct DisksLayout {
pub legacy: bool,
pub pools: Vec<PoolDisksLayout>,
}
// impl<T: AsRef<str>> TryFrom<&[T]> for DisksLayout {
// type Error = Error;
// fn try_from(args: &[T]) -> Result<Self, Self::Error> {
// if args.is_empty() {
// return Err(Error::from_string("Invalid argument"));
// }
// let is_ellipses = args.iter().any(|v| has_ellipses(&[v]));
// // None of the args have ellipses use the old style.
// if !is_ellipses {
// let set_args = get_all_sets(is_ellipses, args)?;
// return Ok(DisksLayout {
// legacy: true,
// pools: vec![PoolDisksLayout::new(
// args.iter().map(AsRef::as_ref).collect::<Vec<&str>>().join(" "),
// set_args,
// )],
// });
// }
// let mut layout = Vec::with_capacity(args.len());
// for arg in args.iter() {
// if !has_ellipses(&[arg]) && args.len() > 1 {
// return Err(Error::from_string(
// "all args must have ellipses for pool expansion (Invalid arguments specified)",
// ));
// }
// let set_args = get_all_sets(is_ellipses, &[arg])?;
// layout.push(PoolDisksLayout::new(arg.as_ref(), set_args));
// }
// Ok(DisksLayout {
// legacy: false,
// pools: layout,
// })
// }
// }
impl DisksLayout {
pub fn from_volumes<T: AsRef<str>>(args: &[T]) -> Result<Self> {
if args.is_empty() {
return Err(Error::other("Invalid argument"));
}
let is_ellipses = args.iter().any(|v| has_ellipses(&[v]));
let set_drive_count_env = env::var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT).unwrap_or_else(|err| {
debug!("{} not set use default:0, {:?}", ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, err);
"0".to_string()
});
let set_drive_count: usize = set_drive_count_env.parse().map_err(Error::other)?;
// None of the args have ellipses use the old style.
if !is_ellipses {
let set_args = get_all_sets(set_drive_count, is_ellipses, args)?;
return Ok(DisksLayout {
legacy: true,
pools: vec![PoolDisksLayout::new(
args.iter().map(AsRef::as_ref).collect::<Vec<&str>>().join(" "),
set_args,
)],
});
}
let mut layout = Vec::with_capacity(args.len());
for arg in args.iter() {
if !has_ellipses(&[arg]) && args.len() > 1 {
return Err(Error::other(
"all args must have ellipses for pool expansion (Invalid arguments specified)",
));
}
let set_args = get_all_sets(set_drive_count, is_ellipses, &[arg])?;
layout.push(PoolDisksLayout::new(arg.as_ref(), set_args));
}
Ok(DisksLayout {
legacy: false,
pools: layout,
})
}
pub fn is_empty_layout(&self) -> bool {
self.pools.is_empty()
|| self.pools[0].layout.is_empty()
|| self.pools[0].layout[0].is_empty()
|| self.pools[0].layout[0][0].is_empty()
}
pub fn is_single_drive_layout(&self) -> bool {
self.pools.len() == 1 && self.pools[0].layout.len() == 1 && self.pools[0].layout[0].len() == 1
}
pub fn get_single_drive_layout(&self) -> &str {
&self.pools[0].layout[0][0]
}
/// returns the total number of sets in the layout.
pub fn get_set_count(&self, i: usize) -> usize {
self.pools.get(i).map_or(0, |v| v.count())
}
/// returns the total number of drives in the layout.
pub fn get_drives_per_set(&self, i: usize) -> usize {
self.pools.get(i).map_or(0, |v| v.layout.first().map_or(0, |v| v.len()))
}
/// returns the command line for the given index.
pub fn get_cmd_line(&self, i: usize) -> String {
self.pools.get(i).map_or(String::new(), |v| v.get_cmd_line().to_owned())
}
}
/// parses all ellipses input arguments, expands them into
/// corresponding list of endpoints chunked evenly in accordance with a
/// specific set size.
///
/// For example: {1...64} is divided into 4 sets each of size 16.
/// This applies to even distributed setup syntax as well.
fn get_all_sets<T: AsRef<str>>(set_drive_count: usize, is_ellipses: bool, args: &[T]) -> Result<Vec<Vec<String>>> {
let endpoint_set = if is_ellipses {
EndpointSet::from_volumes(args, set_drive_count)?
} else {
let set_indexes = if args.len() > 1 {
get_set_indexes(args, &[args.len()], set_drive_count, &[])?
} else {
vec![vec![args.len()]]
};
let endpoints = args.iter().map(|v| v.as_ref().to_string()).collect();
EndpointSet::new(endpoints, set_indexes)
};
let set_args = endpoint_set.get();
let mut unique_args = HashSet::with_capacity(set_args.len());
for args in set_args.iter() {
for arg in args {
if unique_args.contains(arg) {
return Err(Error::other(format!("Input args {arg} has duplicate ellipses")));
}
unique_args.insert(arg);
}
}
Ok(set_args)
}
/// represents parsed ellipses values, also provides
/// methods to get the sets of endpoints.
#[derive(Debug, Default)]
struct EndpointSet {
_arg_patterns: Vec<ArgPattern>,
endpoints: Vec<String>,
set_indexes: Vec<Vec<usize>>,
}
// impl<T: AsRef<str>> TryFrom<&[T]> for EndpointSet {
// type Error = Error;
// fn try_from(args: &[T]) -> Result<Self, Self::Error> {
// let mut arg_patterns = Vec::with_capacity(args.len());
// for arg in args {
// arg_patterns.push(find_ellipses_patterns(arg.as_ref())?);
// }
// let total_sizes = get_total_sizes(&arg_patterns);
// let set_indexes = get_set_indexes(args, &total_sizes, &arg_patterns)?;
// let mut endpoints = Vec::new();
// for ap in arg_patterns.iter() {
// let aps = ap.expand();
// for bs in aps {
// endpoints.push(bs.join(""));
// }
// }
// Ok(EndpointSet {
// set_indexes,
// _arg_patterns: arg_patterns,
// endpoints,
// })
// }
// }
impl EndpointSet {
/// Create a new EndpointSet with the given endpoints and set indexes.
pub fn new(endpoints: Vec<String>, set_indexes: Vec<Vec<usize>>) -> Self {
Self {
endpoints,
set_indexes,
..Default::default()
}
}
pub fn from_volumes<T: AsRef<str>>(args: &[T], set_drive_count: usize) -> Result<Self> {
let mut arg_patterns = Vec::with_capacity(args.len());
for arg in args {
arg_patterns.push(find_ellipses_patterns(arg.as_ref())?);
}
let total_sizes = get_total_sizes(&arg_patterns);
let set_indexes = get_set_indexes(args, &total_sizes, set_drive_count, &arg_patterns)?;
let mut endpoints = Vec::new();
for ap in arg_patterns.iter() {
let aps = ap.expand();
for bs in aps {
endpoints.push(bs.join(""));
}
}
Ok(EndpointSet {
set_indexes,
_arg_patterns: arg_patterns,
endpoints,
})
}
/// returns the sets representation of the endpoints
/// this function also intelligently decides on what will
/// be the right set size etc.
pub fn get(&self) -> Vec<Vec<String>> {
let mut sets: Vec<Vec<String>> = Vec::new();
let mut start = 0;
for set_idx in self.set_indexes.iter() {
for idx in set_idx {
let end = idx + start;
sets.push(self.endpoints[start..end].to_vec());
start = end;
}
}
sets
}
}
/// returns the greatest common divisor of all the ellipses sizes.
fn get_divisible_size(total_sizes: &[usize]) -> usize {
fn gcd(mut x: usize, mut y: usize) -> usize {
while y != 0 {
// be equivalent to: x, y = y, x%y
std::mem::swap(&mut x, &mut y);
y %= x;
}
x
}
total_sizes.iter().skip(1).fold(total_sizes[0], |acc, &y| gcd(acc, y))
}
fn possible_set_counts(set_size: usize) -> Vec<usize> {
let mut ss = Vec::new();
for s in SET_SIZES {
if set_size.is_multiple_of(s) {
ss.push(s);
}
}
ss
}
/// checks whether given count is a valid set size for erasure coding.
fn is_valid_set_size(count: usize) -> bool {
count >= SET_SIZES[0] && count <= SET_SIZES[SET_SIZES.len() - 1]
}
/// Final set size with all the symmetry accounted for.
fn common_set_drive_count(divisible_size: usize, set_counts: &[usize]) -> usize {
// prefers set_counts to be sorted for optimal behavior.
if divisible_size < set_counts[set_counts.len() - 1] {
return divisible_size;
}
let mut prev_d = divisible_size / set_counts[0];
let mut set_size = 0;
for &cnt in set_counts {
if divisible_size.is_multiple_of(cnt) {
let d = divisible_size / cnt;
if d <= prev_d {
prev_d = d;
set_size = cnt;
}
}
}
set_size
}
/// returns symmetrical setCounts based on the input argument patterns,
/// the symmetry calculation is to ensure that we also use uniform number
/// of drives common across all ellipses patterns.
fn possible_set_counts_with_symmetry(set_counts: &[usize], arg_patterns: &[ArgPattern]) -> Vec<usize> {
let mut new_set_counts: HashSet<usize> = HashSet::new();
for &ss in set_counts {
let mut symmetry = false;
for arg_pattern in arg_patterns {
for p in arg_pattern.as_ref().iter() {
if p.len() > ss {
symmetry = (p.len() % ss) == 0;
} else {
symmetry = (ss % p.len()) == 0;
}
}
}
if !new_set_counts.contains(&ss) && (symmetry || arg_patterns.is_empty()) {
new_set_counts.insert(ss);
}
}
let mut set_counts: Vec<usize> = new_set_counts.into_iter().collect();
set_counts.sort_unstable();
set_counts
}
/// returns list of indexes which provides the set size
/// on each index, this function also determines the final set size
/// The final set size has the affinity towards choosing smaller
/// indexes (total sets)
fn get_set_indexes<T: AsRef<str>>(
args: &[T],
total_sizes: &[usize],
set_drive_count: usize,
arg_patterns: &[ArgPattern],
) -> Result<Vec<Vec<usize>>> {
if args.is_empty() || total_sizes.is_empty() {
return Err(Error::other("Invalid argument"));
}
for &size in total_sizes {
// Check if total_sizes has minimum range upto set_size
if size < SET_SIZES[0] || size < set_drive_count {
return Err(Error::other(format!("Incorrect number of endpoints provided, size {size}")));
}
}
let common_size = get_divisible_size(total_sizes);
let mut set_counts = possible_set_counts(common_size);
if set_counts.is_empty() {
return Err(Error::other(format!(
"Incorrect number of endpoints provided, number of drives {} is not divisible by any supported erasure set sizes {}",
common_size, 0
)));
}
// Returns possible set counts with symmetry.
set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns);
if set_counts.is_empty() {
return Err(Error::other("No symmetric distribution detected with input endpoints provided"));
}
let set_size = {
if set_drive_count > 0 {
let has_set_drive_count = set_counts.contains(&set_drive_count);
if !has_set_drive_count {
return Err(Error::other(format!(
"Invalid set drive count {}. Acceptable values for {:?} number drives are {:?}",
set_drive_count, common_size, &set_counts
)));
}
set_drive_count
} else {
set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns);
if set_counts.is_empty() {
return Err(Error::other(format!(
"No symmetric distribution detected with input endpoints , drives {} cannot be spread symmetrically by any supported erasure set sizes {:?}",
common_size, &set_counts
)));
}
// Final set size with all the symmetry accounted for.
common_set_drive_count(common_size, &set_counts)
}
};
if !is_valid_set_size(set_size) {
return Err(Error::other("Incorrect number of endpoints provided3"));
}
Ok(total_sizes
.iter()
.map(|&size| (0..(size / set_size)).map(|_| set_size).collect())
.collect())
}
/// Return the total size for each argument patterns.
fn get_total_sizes(arg_patterns: &[ArgPattern]) -> Vec<usize> {
arg_patterns.iter().map(|v| v.total_sizes()).collect()
}
#[cfg(test)]
mod test {
use rustfs_utils::string::Pattern;
use super::*;
impl PartialEq for EndpointSet {
fn eq(&self, other: &Self) -> bool {
self._arg_patterns == other._arg_patterns && self.set_indexes == other.set_indexes
}
}
#[test]
fn test_get_divisible_size() {
struct TestCase {
total_sizes: Vec<usize>,
result: usize,
}
let test_cases = [
TestCase {
total_sizes: vec![24, 32, 16],
result: 8,
},
TestCase {
total_sizes: vec![32, 8, 4],
result: 4,
},
TestCase {
total_sizes: vec![8, 8, 8],
result: 8,
},
TestCase {
total_sizes: vec![24],
result: 24,
},
];
for (i, test_case) in test_cases.iter().enumerate() {
let ret = get_divisible_size(&test_case.total_sizes);
assert_eq!(ret, test_case.result, "Test{}: Expected {}, got {}", i + 1, test_case.result, ret);
}
}
#[test]
fn test_get_set_indexes() {
#[derive(Default)]
struct TestCase<'a> {
num: usize,
args: Vec<&'a str>,
total_sizes: Vec<usize>,
indexes: Vec<Vec<usize>>,
success: bool,
}
let test_cases = [
TestCase {
num: 1,
args: vec!["data{1...17}/export{1...52}"],
total_sizes: vec![14144],
..Default::default()
},
TestCase {
num: 2,
args: vec!["data{1...3}"],
total_sizes: vec![3],
indexes: vec![vec![3]],
success: true,
},
TestCase {
num: 3,
args: vec!["data/controller1/export{1...2}, data/controller2/export{1...4}, data/controller3/export{1...8}"],
total_sizes: vec![2, 4, 8],
indexes: vec![vec![2], vec![2, 2], vec![2, 2, 2, 2]],
success: true,
},
TestCase {
num: 4,
args: vec!["data{1...27}"],
total_sizes: vec![27],
indexes: vec![vec![9, 9, 9]],
success: true,
},
TestCase {
num: 5,
args: vec!["http://host{1...3}/data{1...180}"],
total_sizes: vec![540],
indexes: vec![vec![
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15,
]],
success: true,
},
TestCase {
num: 6,
args: vec!["http://host{1...2}.rack{1...4}/data{1...180}"],
total_sizes: vec![1440],
indexes: vec![vec![
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16,
]],
success: true,
},
TestCase {
num: 7,
args: vec!["http://host{1...2}/data{1...180}"],
total_sizes: vec![360],
indexes: vec![vec![
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
12, 12, 12,
]],
success: true,
},
TestCase {
num: 8,
args: vec!["data/controller1/export{1...4}, data/controller2/export{1...8}, data/controller3/export{1...12}"],
total_sizes: vec![4, 8, 12],
indexes: vec![vec![4], vec![4, 4], vec![4, 4, 4]],
success: true,
},
TestCase {
num: 9,
args: vec!["data{1...64}"],
total_sizes: vec![64],
indexes: vec![vec![16, 16, 16, 16]],
success: true,
},
TestCase {
num: 10,
args: vec!["data{1...24}"],
total_sizes: vec![24],
indexes: vec![vec![12, 12]],
success: true,
},
TestCase {
num: 11,
args: vec!["data/controller{1...11}/export{1...8}"],
total_sizes: vec![88],
indexes: vec![vec![11, 11, 11, 11, 11, 11, 11, 11]],
success: true,
},
TestCase {
num: 12,
args: vec!["data{1...4}"],
total_sizes: vec![4],
indexes: vec![vec![4]],
success: true,
},
TestCase {
num: 13,
args: vec!["data/controller1/export{1...10}, data/controller2/export{1...10}, data/controller3/export{1...10}"],
total_sizes: vec![10, 10, 10],
indexes: vec![vec![10], vec![10], vec![10]],
success: true,
},
TestCase {
num: 14,
args: vec!["data{1...16}/export{1...52}"],
total_sizes: vec![832],
indexes: vec![vec![
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
]],
success: true,
},
TestCase {
num: 15,
args: vec!["https://node{1...3}.example.net/mnt/drive{1...8}"],
total_sizes: vec![24],
indexes: vec![vec![12, 12]],
success: true,
},
];
for test_case in test_cases {
let mut arg_patterns = Vec::new();
for v in test_case.args.iter() {
match find_ellipses_patterns(v) {
Ok(patterns) => {
arg_patterns.push(patterns);
}
Err(err) => {
panic!("Test{}: Unexpected failure {:?}", test_case.num, err);
}
}
}
match get_set_indexes(test_case.args.as_slice(), test_case.total_sizes.as_slice(), 0, arg_patterns.as_slice()) {
Ok(got_indexes) => {
if !test_case.success {
panic!("Test{}: Expected failure but passed instead", test_case.num);
}
assert_eq!(
test_case.indexes, got_indexes,
"Test{}: Expected {:?}, got {:?}",
test_case.num, test_case.indexes, got_indexes
)
}
Err(err) => {
if test_case.success {
panic!("Test{}: Expected success but failed instead {:?}", test_case.num, err);
}
}
}
}
}
fn get_sequences(start: usize, number: usize, padding_len: usize) -> Vec<String> {
let mut seq = Vec::new();
for i in start..=number {
if padding_len == 0 {
seq.push(format!("{i}"));
} else {
seq.push(format!("{i:0padding_len$}"));
}
}
seq
}
#[test]
fn test_into_endpoint_set() {
#[derive(Default)]
struct TestCase<'a> {
num: usize,
arg: &'a str,
es: EndpointSet,
success: bool,
}
let test_cases = [
// Tests invalid inputs.
TestCase {
num: 1,
arg: "...",
..Default::default()
},
// No range specified.
TestCase {
num: 2,
arg: "{...}",
..Default::default()
},
// Invalid range.
TestCase {
num: 3,
arg: "http://rustfs{2...3}/export/set{1...0}",
..Default::default()
},
// Range cannot be smaller than 4 minimum.
TestCase {
num: 4,
arg: "/export{1..2}",
..Default::default()
},
// Unsupported characters.
TestCase {
num: 5,
arg: "/export/test{1...2O}",
..Default::default()
},
// Tests valid inputs.
TestCase {
num: 6,
arg: "{1...27}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![Pattern {
seq: get_sequences(1, 27, 0),
..Default::default()
}])],
set_indexes: vec![vec![9, 9, 9]],
..Default::default()
},
success: true,
},
TestCase {
num: 7,
arg: "/export/set{1...64}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![Pattern {
seq: get_sequences(1, 64, 0),
prefix: "/export/set".to_owned(),
..Default::default()
}])],
set_indexes: vec![vec![16, 16, 16, 16]],
..Default::default()
},
success: true,
},
// Valid input for distributed setup.
TestCase {
num: 8,
arg: "http://rustfs{2...3}/export/set{1...64}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![
Pattern {
seq: get_sequences(1, 64, 0),
..Default::default()
},
Pattern {
seq: get_sequences(2, 3, 0),
prefix: "http://rustfs".to_owned(),
suffix: "/export/set".to_owned(),
},
])],
set_indexes: vec![vec![16, 16, 16, 16, 16, 16, 16, 16]],
..Default::default()
},
success: true,
},
// Supporting some advanced cases.
TestCase {
num: 9,
arg: "http://rustfs{1...64}.mydomain.net/data",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![Pattern {
seq: get_sequences(1, 64, 0),
prefix: "http://rustfs".to_owned(),
suffix: ".mydomain.net/data".to_owned(),
}])],
set_indexes: vec![vec![16, 16, 16, 16]],
..Default::default()
},
success: true,
},
TestCase {
num: 10,
arg: "http://rack{1...4}.mydomain.rustfs{1...16}/data",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![
Pattern {
seq: get_sequences(1, 16, 0),
suffix: "/data".to_owned(),
..Default::default()
},
Pattern {
seq: get_sequences(1, 4, 0),
prefix: "http://rack".to_owned(),
suffix: ".mydomain.rustfs".to_owned(),
},
])],
set_indexes: vec![vec![16, 16, 16, 16]],
..Default::default()
},
success: true,
},
// Supporting kubernetes cases.
TestCase {
num: 11,
arg: "http://rustfs{0...15}.mydomain.net/data{0...1}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![
Pattern {
seq: get_sequences(0, 1, 0),
..Default::default()
},
Pattern {
seq: get_sequences(0, 15, 0),
prefix: "http://rustfs".to_owned(),
suffix: ".mydomain.net/data".to_owned(),
},
])],
set_indexes: vec![vec![16, 16]],
..Default::default()
},
success: true,
},
// No host regex, just disks.
TestCase {
num: 12,
arg: "http://server1/data{1...32}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![Pattern {
seq: get_sequences(1, 32, 0),
prefix: "http://server1/data".to_owned(),
..Default::default()
}])],
set_indexes: vec![vec![16, 16]],
..Default::default()
},
success: true,
},
// No host regex, just disks with two position numerics.
TestCase {
num: 13,
arg: "http://server1/data{01...32}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![Pattern {
seq: get_sequences(1, 32, 2),
prefix: "http://server1/data".to_owned(),
..Default::default()
}])],
set_indexes: vec![vec![16, 16]],
..Default::default()
},
success: true,
},
// More than 2 ellipses are supported as well.
TestCase {
num: 14,
arg: "http://rustfs{2...3}/export/set{1...64}/test{1...2}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![
Pattern {
seq: get_sequences(1, 2, 0),
..Default::default()
},
Pattern {
seq: get_sequences(1, 64, 0),
suffix: "/test".to_owned(),
..Default::default()
},
Pattern {
seq: get_sequences(2, 3, 0),
prefix: "http://rustfs".to_owned(),
suffix: "/export/set".to_owned(),
},
])],
set_indexes: vec![vec![16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16]],
..Default::default()
},
success: true,
},
// More than an ellipse per argument for standalone setup.
TestCase {
num: 15,
arg: "/export{1...10}/disk{1...10}",
es: EndpointSet {
_arg_patterns: vec![ArgPattern::new(vec![
Pattern {
seq: get_sequences(1, 10, 0),
..Default::default()
},
Pattern {
seq: get_sequences(1, 10, 0),
prefix: "/export".to_owned(),
suffix: "/disk".to_owned(),
},
])],
set_indexes: vec![vec![10, 10, 10, 10, 10, 10, 10, 10, 10, 10]],
..Default::default()
},
success: true,
},
];
for test_case in test_cases {
match EndpointSet::from_volumes([test_case.arg].as_slice(), 0) {
Ok(got_es) => {
if !test_case.success {
panic!("Test{}: Expected failure but passed instead", test_case.num);
}
assert_eq!(
test_case.es, got_es,
"Test{}: Expected {:?}, got {:?}",
test_case.num, test_case.es, got_es
)
}
Err(err) => {
if test_case.success {
panic!("Test{}: Expected success but failed instead {:?}", test_case.num, err);
}
}
}
}
}
}
crates/ecstore/src/layout/format.rs
+546
View File
@@ -0,0 +1,546 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::disk::error::{Error, Result};
use crate::disk::{DiskInfo, error::DiskError};
use serde::{Deserialize, Serialize};
use serde_json::Error as JsonError;
use uuid::Uuid;
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum FormatMetaVersion {
#[serde(rename = "1")]
V1,
#[serde(other)]
Unknown,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum FormatBackend {
#[serde(rename = "xl")]
Erasure,
#[serde(rename = "xl-single")]
ErasureSingle,
#[serde(other)]
Unknown,
}
/// Represents the V3 backend disk structure version
/// under `.rustfs.sys` and actual data namespace.
///
/// FormatErasureV3 - structure holds format config version '3'.
///
/// The V3 format to support "large bucket" support where a bucket
/// can span multiple erasure sets.
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct FormatErasureV3 {
/// Version of 'xl' format.
pub version: FormatErasureVersion,
/// This field carries assigned disk uuid.
pub this: Uuid,
/// Sets field carries the input disk order generated the first
/// time when fresh disks were supplied, it is a two-dimensional
/// array second dimension represents list of disks used per set.
pub sets: Vec<Vec<Uuid>>,
/// Distribution algorithm represents the hashing algorithm
/// to pick the right set index for an object.
#[serde(rename = "distributionAlgo")]
pub distribution_algo: DistributionAlgoVersion,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum FormatErasureVersion {
#[serde(rename = "1")]
V1,
#[serde(rename = "2")]
V2,
#[serde(rename = "3")]
V3,
#[serde(other)]
Unknown,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum DistributionAlgoVersion {
#[serde(rename = "CRCMOD")]
V1,
#[serde(rename = "SIPMOD")]
V2,
#[serde(rename = "SIPMOD+PARITY")]
V3,
}
/// format.json currently has the format:
///
/// ```json
/// {
/// "version": "1",
/// "format": "XXXXX",
/// "id": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
/// "XXXXX": {
//
/// }
/// }
/// ```
///
/// Ideally we will never have a situation where we will have to change the
/// fields of this struct and deal with related migration.
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct FormatV3 {
/// Version of the format config.
pub version: FormatMetaVersion,
/// Format indicates the backend format type, supports two values 'xl' and 'xl-single'.
pub format: FormatBackend,
/// ID is the identifier for the rustfs deployment
pub id: Uuid,
#[serde(rename = "xl")]
pub erasure: FormatErasureV3,
// /// DiskInfo is an extended type which returns current
// /// disk usage per path.
#[serde(skip)]
pub disk_info: Option<DiskInfo>,
}
impl TryFrom<&[u8]> for FormatV3 {
type Error = JsonError;
fn try_from(data: &[u8]) -> std::result::Result<Self, Self::Error> {
serde_json::from_slice(data)
}
}
impl TryFrom<&str> for FormatV3 {
type Error = JsonError;
fn try_from(data: &str) -> std::result::Result<Self, Self::Error> {
serde_json::from_str(data)
}
}
impl FormatV3 {
/// Create a new format config with the given number of sets and set length.
pub fn new(num_sets: usize, set_len: usize) -> Self {
let format = if set_len == 1 {
FormatBackend::ErasureSingle
} else {
FormatBackend::Erasure
};
let erasure = FormatErasureV3 {
version: FormatErasureVersion::V3,
this: Uuid::nil(),
sets: (0..num_sets)
.map(|_| (0..set_len).map(|_| Uuid::new_v4()).collect())
.collect(),
distribution_algo: DistributionAlgoVersion::V3,
};
Self {
version: FormatMetaVersion::V1,
format,
id: Uuid::new_v4(),
erasure,
disk_info: None,
}
}
/// Returns the number of drives in the erasure set.
pub fn drives(&self) -> usize {
self.erasure.sets.iter().map(|v| v.len()).sum()
}
pub fn to_json(&self) -> std::result::Result<String, JsonError> {
serde_json::to_string(self)
}
/// returns the i,j'th position of the input `diskID` against the reference
///
/// format, after successful validation.
/// - i'th position is the set index
/// - j'th position is the disk index in the current set
pub fn find_disk_index_by_disk_id(&self, disk_id: Uuid) -> Result<(usize, usize)> {
if disk_id == Uuid::nil() {
return Err(Error::from(DiskError::DiskNotFound));
}
if disk_id == Uuid::max() {
return Err(Error::other("disk offline"));
}
for (i, set) in self.erasure.sets.iter().enumerate() {
for (j, d) in set.iter().enumerate() {
if disk_id.eq(d) {
return Ok((i, j));
}
}
}
Err(Error::other(format!("disk id not found {disk_id}")))
}
pub fn check_other(&self, other: &FormatV3) -> Result<()> {
let mut tmp = other.clone();
let this = tmp.erasure.this;
tmp.erasure.this = Uuid::nil();
if self.erasure.sets.len() != other.erasure.sets.len() {
return Err(Error::other(format!(
"Expected number of sets {}, got {}",
self.erasure.sets.len(),
other.erasure.sets.len()
)));
}
for i in 0..self.erasure.sets.len() {
if self.erasure.sets[i].len() != other.erasure.sets[i].len() {
return Err(Error::other(format!(
"Each set should be of same size, expected {}, got {}",
self.erasure.sets[i].len(),
other.erasure.sets[i].len()
)));
}
for j in 0..self.erasure.sets[i].len() {
if self.erasure.sets[i][j] != other.erasure.sets[i][j] {
return Err(Error::other(format!(
"UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)",
i,
j,
self.erasure.sets[i][j].to_string(),
other.erasure.sets[i][j].to_string(),
)));
}
}
}
for i in 0..tmp.erasure.sets.len() {
for j in 0..tmp.erasure.sets[i].len() {
if this == tmp.erasure.sets[i][j] {
return Ok(());
}
}
}
Err(Error::other(format!(
"DriveID {:?} not found in any drive sets {:?}",
this, other.erasure.sets
)))
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_format_v1() {
let format = FormatV3::new(1, 4);
let str = serde_json::to_string(&format);
println!("{str:?}");
let data = r#"
{
"version": "1",
"format": "xl",
"id": "321b3874-987d-4c15-8fa5-757c956b1243",
"xl": {
"version": "1",
"this": null,
"sets": [
[
"8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
"c26315da-05cf-4778-a9ea-b44ea09f58c5",
"fb87a891-18d3-44cf-a46f-bcc15093a038",
"356a925c-57b9-4313-88b3-053edf1104dc"
]
],
"distributionAlgo": "CRCMOD"
}
}"#;
let p = FormatV3::try_from(data);
println!("{p:?}");
}
#[test]
fn test_format_v3_new_single_disk() {
let format = FormatV3::new(1, 1);
assert_eq!(format.version, FormatMetaVersion::V1);
assert_eq!(format.format, FormatBackend::ErasureSingle);
assert_eq!(format.erasure.version, FormatErasureVersion::V3);
assert_eq!(format.erasure.sets.len(), 1);
assert_eq!(format.erasure.sets[0].len(), 1);
assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3);
assert_eq!(format.erasure.this, Uuid::nil());
}
#[test]
fn test_format_v3_new_multiple_sets() {
let format = FormatV3::new(2, 4);
assert_eq!(format.version, FormatMetaVersion::V1);
assert_eq!(format.format, FormatBackend::Erasure);
assert_eq!(format.erasure.version, FormatErasureVersion::V3);
assert_eq!(format.erasure.sets.len(), 2);
assert_eq!(format.erasure.sets[0].len(), 4);
assert_eq!(format.erasure.sets[1].len(), 4);
assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3);
}
#[test]
fn test_format_v3_drives() {
let format = FormatV3::new(2, 4);
assert_eq!(format.drives(), 8); // 2 sets * 4 drives each
let format_single = FormatV3::new(1, 1);
assert_eq!(format_single.drives(), 1); // 1 set * 1 drive
}
#[test]
fn test_format_v3_to_json() {
let format = FormatV3::new(1, 2);
let json_result = format.to_json();
assert!(json_result.is_ok());
let json_str = json_result.unwrap();
assert!(json_str.contains("\"version\":\"1\""));
assert!(json_str.contains("\"format\":\"xl\""));
}
#[test]
fn test_format_v3_from_json() {
let json_data = r#"{
"version": "1",
"format": "xl-single",
"id": "321b3874-987d-4c15-8fa5-757c956b1243",
"xl": {
"version": "3",
"this": "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
"sets": [
[
"8ab9a908-f869-4f1f-8e42-eb067ffa7eb5"
]
],
"distributionAlgo": "SIPMOD+PARITY"
}
}"#;
let format = FormatV3::try_from(json_data);
assert!(format.is_ok());
let format = format.unwrap();
assert_eq!(format.format, FormatBackend::ErasureSingle);
assert_eq!(format.erasure.version, FormatErasureVersion::V3);
assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3);
assert_eq!(format.erasure.sets.len(), 1);
assert_eq!(format.erasure.sets[0].len(), 1);
}
#[test]
fn test_format_v3_from_bytes() {
let json_data = r#"{
"version": "1",
"format": "xl",
"id": "321b3874-987d-4c15-8fa5-757c956b1243",
"xl": {
"version": "2",
"this": "00000000-0000-0000-0000-000000000000",
"sets": [
[
"8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
"c26315da-05cf-4778-a9ea-b44ea09f58c5"
]
],
"distributionAlgo": "SIPMOD"
}
}"#;
let format = FormatV3::try_from(json_data.as_bytes());
assert!(format.is_ok());
let format = format.unwrap();
assert_eq!(format.erasure.version, FormatErasureVersion::V2);
assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V2);
assert_eq!(format.erasure.sets[0].len(), 2);
}
#[test]
fn test_format_v3_invalid_json() {
let invalid_json = r#"{"invalid": "json"}"#;
let format = FormatV3::try_from(invalid_json);
assert!(format.is_err());
}
#[test]
fn test_find_disk_index_by_disk_id() {
let mut format = FormatV3::new(2, 2);
let target_disk_id = Uuid::new_v4();
format.erasure.sets[1][0] = target_disk_id;
let result = format.find_disk_index_by_disk_id(target_disk_id);
assert!(result.is_ok());
assert_eq!(result.unwrap(), (1, 0));
}
#[test]
fn test_find_disk_index_nil_uuid() {
let format = FormatV3::new(1, 2);
let result = format.find_disk_index_by_disk_id(Uuid::nil());
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::DiskNotFound));
}
#[test]
fn test_find_disk_index_max_uuid() {
let format = FormatV3::new(1, 2);
let result = format.find_disk_index_by_disk_id(Uuid::max());
assert!(result.is_err());
}
#[test]
fn test_find_disk_index_not_found() {
let format = FormatV3::new(1, 2);
let non_existent_id = Uuid::new_v4();
let result = format.find_disk_index_by_disk_id(non_existent_id);
assert!(result.is_err());
}
#[test]
fn test_check_other_identical() {
let format1 = FormatV3::new(2, 4);
let mut format2 = format1.clone();
format2.erasure.this = format1.erasure.sets[0][0];
let result = format1.check_other(&format2);
assert!(result.is_ok());
}
#[test]
fn test_check_other_different_set_count() {
let format1 = FormatV3::new(2, 4);
let format2 = FormatV3::new(3, 4);
let result = format1.check_other(&format2);
assert!(result.is_err());
}
#[test]
fn test_check_other_different_set_size() {
let format1 = FormatV3::new(2, 4);
let format2 = FormatV3::new(2, 6);
let result = format1.check_other(&format2);
assert!(result.is_err());
}
#[test]
fn test_check_other_different_disk_id() {
let format1 = FormatV3::new(1, 2);
let mut format2 = format1.clone();
format2.erasure.sets[0][0] = Uuid::new_v4();
let result = format1.check_other(&format2);
assert!(result.is_err());
}
#[test]
fn test_check_other_disk_not_in_sets() {
let format1 = FormatV3::new(1, 2);
let mut format2 = format1.clone();
format2.erasure.this = Uuid::new_v4(); // Set to a UUID not in any set
let result = format1.check_other(&format2);
assert!(result.is_err());
}
#[test]
fn test_format_meta_version_serialization() {
let v1 = FormatMetaVersion::V1;
let json = serde_json::to_string(&v1).unwrap();
assert_eq!(json, "\"1\"");
let unknown = FormatMetaVersion::Unknown;
let deserialized: FormatMetaVersion = serde_json::from_str("\"unknown\"").unwrap();
assert_eq!(deserialized, unknown);
}
#[test]
fn test_format_backend_serialization() {
let erasure = FormatBackend::Erasure;
let json = serde_json::to_string(&erasure).unwrap();
assert_eq!(json, "\"xl\"");
let single = FormatBackend::ErasureSingle;
let json = serde_json::to_string(&single).unwrap();
assert_eq!(json, "\"xl-single\"");
let unknown = FormatBackend::Unknown;
let deserialized: FormatBackend = serde_json::from_str("\"unknown\"").unwrap();
assert_eq!(deserialized, unknown);
}
#[test]
fn test_format_erasure_version_serialization() {
let v1 = FormatErasureVersion::V1;
let json = serde_json::to_string(&v1).unwrap();
assert_eq!(json, "\"1\"");
let v2 = FormatErasureVersion::V2;
let json = serde_json::to_string(&v2).unwrap();
assert_eq!(json, "\"2\"");
let v3 = FormatErasureVersion::V3;
let json = serde_json::to_string(&v3).unwrap();
assert_eq!(json, "\"3\"");
}
#[test]
fn test_distribution_algo_version_serialization() {
let v1 = DistributionAlgoVersion::V1;
let json = serde_json::to_string(&v1).unwrap();
assert_eq!(json, "\"CRCMOD\"");
let v2 = DistributionAlgoVersion::V2;
let json = serde_json::to_string(&v2).unwrap();
assert_eq!(json, "\"SIPMOD\"");
let v3 = DistributionAlgoVersion::V3;
let json = serde_json::to_string(&v3).unwrap();
assert_eq!(json, "\"SIPMOD+PARITY\"");
}
#[test]
fn test_format_v3_round_trip_serialization() {
let original = FormatV3::new(2, 3);
let json = original.to_json().unwrap();
let deserialized = FormatV3::try_from(json.as_str()).unwrap();
assert_eq!(original.version, deserialized.version);
assert_eq!(original.format, deserialized.format);
assert_eq!(original.erasure.version, deserialized.erasure.version);
assert_eq!(original.erasure.sets.len(), deserialized.erasure.sets.len());
assert_eq!(original.erasure.distribution_algo, deserialized.erasure.distribution_algo);
}
}
crates/ecstore/src/layout/mod.rs
+2
View File
@@ -4,4 +4,6 @@
//! topology separate from runtime `Sets`/`SetDisks` orchestration before any
//! file moves happen.
pub(crate) mod disks_layout;
pub(crate) mod format;
pub(crate) mod set_layout;
crates/ecstore/src/layout/set_layout.rs
+1 -1
View File
@@ -1,4 +1,4 @@
use crate::disk::format::{DistributionAlgoVersion, FormatV3};
use crate::layout::format::{DistributionAlgoVersion, FormatV3};
use std::collections::HashSet;
use std::io::{Error, Result};
use uuid::Uuid;
docs/architecture/ecstore-layout-boundary.md
+8
View File
@@ -33,6 +33,14 @@ expansion. It may describe:
Static layout must not own disk handles, lock clients, reconnect loops, repair
state, or shutdown signaling.
## Format And Disk Layout Ownership
`layout::format` owns persisted format structures and disk UUID position lookup.
`layout::disks_layout` owns command-line volume expansion into pool/set layout.
Compatibility paths remain available through `disk::format` and `disks_layout`
until downstream callers are moved or compatibility coverage allows removal.
## Runtime Set Orchestration
Runtime orchestration remains owned by `Sets` and `SetDisks` until a later pure
docs/architecture/migration-progress.md
+41 -12
View File
@@ -5,16 +5,17 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-ecstore-layout-foundation`
- Baseline: `overtrue/arch-extension-runtime-snapshots`
(`6ee557f63c4f8eb45334af23a6b3a32e0ee009e9`).
- Stacked on: local R-033, which is stacked on local R-032 and rustfs/rustfs#3642.
- Branch: `overtrue/arch-ecstore-layout-format-move`
- Baseline: `overtrue/arch-ecstore-layout-foundation`
(`2e10661d088a665c789fdb092871c6b52675aa1e`).
- Stacked on: local E-001/E-SET-001, which is stacked on local R-033 and
local R-032.
- PR type for this branch: `pure-move`
- Runtime behavior changes: none.
- Rust code changes: add the ECStore internal layout skeleton and read-only set
layout boundary tests.
- Rust code changes: pure-move ECStore format and disk-layout expansion modules
into the internal layout bucket while preserving old public paths.
- CI/script changes: none.
- Docs changes: record the ECStore layout boundary and E-001/E-SET-001 slice.
- Docs changes: record the ECStore format/disk-layout pure move slice.
## Phase 0 Tasks
@@ -2225,10 +2226,25 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
checks, migration/layer guards, formatting, diff hygiene, Rust risk scan,
branch freshness check, pre-commit quality gate, and three-expert review.
- [x] `E-002/E-LAYOUT-001` Move ECStore format and disk-layout owners.
- Do: pure-move persisted format ownership and disk-layout expansion into
the ECStore layout bucket while keeping compatibility stubs at the old
public paths.
- Acceptance: `crate::disk::format::*` and `crate::disks_layout::*` remain
usable, `layout::format` owns `FormatV3`, and `layout::disks_layout` owns
CLI volume expansion.
- Must preserve: format JSON wire shape, disk UUID lookup, distribution
algorithm, `RUSTFS_ERASURE_SET_DRIVE_COUNT` handling, endpoint expansion,
and old public module paths.
- Verification: focused ECStore format and disks-layout tests,
ECStore/RustFS/Heal compile checks, migration/layer guards, formatting,
diff hygiene, Rust risk scan, branch freshness check, pre-commit quality
gate, and three-expert review.
## Next PRs
1. `pure-move`: move ECStore layout files into the new layout bucket with
compatibility coverage after E-001/E-SET-001 lands.
1. `pure-move`: continue moving endpoint grouping and runtime-neutral layout
helpers once E-002/E-LAYOUT-001 lands.
2. `pure-move`: continue pruning residual embedded startup-only orchestration
once the lifecycle helpers are merged.
@@ -2236,9 +2252,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
| Expert | Status | Notes |
|---|---|---|
| Quality/architecture | passed | E-001/E-SET-001 adds only ECStore layout skeleton boundaries and read-only set-layout contracts before file moves. |
| Migration preservation | passed | Format distribution, local disk replacement, lock client mapping, public module paths, and runtime set behavior remain unchanged. |
| Testing/verification | passed | Focused ECStore set-layout checks, compile checks, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. |
| Quality/architecture | passed | E-002/E-LAYOUT-001 is a pure move into ECStore layout with compatibility stubs at old public paths; no new runtime owner or dependency boundary is introduced. |
| Migration preservation | passed | Format JSON, disk UUID lookup, distribution algorithm, disk-layout expansion, and old public module paths remain preserved. |
| Testing/verification | passed | Focused format/layout checks, compile checks, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. |
## Verification Notes
@@ -2300,6 +2316,19 @@ Passed before push:
- `make pre-commit`: passed.
- Three-expert review: passed.
- Issue #660 E-002/E-LAYOUT-001 current slice:
- `cargo test -p rustfs-ecstore format::test -- --nocapture`: passed.
- `cargo test -p rustfs-ecstore disks_layout -- --nocapture`: passed.
- `cargo check -p rustfs-ecstore -p rustfs -p rustfs-heal`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- Rust risk scan on changed Rust files: passed; only existing test-only
unwrap/println/panic/expect paths were present.
- `make pre-commit`: passed.
- Three-expert review: passed.
- Issue #660 X-012 current slice:
- `cargo test -p rustfs-extension-schema`: passed.
- `cargo check -p rustfs-extension-schema`: passed.