gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-06 05:17:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:rustfs/s3-rustfs into feature/observability-metrics

Browse Source 
# Conflicts:
#	.github/workflows/build.yml
#	.github/workflows/ci.yml
#	Cargo.lock
#	Cargo.toml
#	appauth/src/token.rs
#	crates/config/src/config.rs
#	crates/event-notifier/examples/simple.rs
#	crates/event-notifier/src/global.rs
#	crates/event-notifier/src/lib.rs
#	crates/event-notifier/src/notifier.rs
#	crates/event-notifier/src/store.rs
#	crates/filemeta/src/filemeta.rs
#	crates/notify/examples/webhook.rs
#	crates/utils/Cargo.toml
#	ecstore/Cargo.toml
#	ecstore/src/cmd/bucket_replication.rs
#	ecstore/src/config/com.rs
#	ecstore/src/disk/error.rs
#	ecstore/src/disk/mod.rs
#	ecstore/src/set_disk.rs
#	ecstore/src/store_api.rs
#	ecstore/src/store_list_objects.rs
#	iam/Cargo.toml
#	iam/src/manager.rs
#	policy/Cargo.toml
#	rustfs/src/admin/rpc.rs
#	rustfs/src/main.rs
#	rustfs/src/storage/mod.rs
This commit is contained in:
houseme
2025-06-19 13:16:48 +08:00
parent 09e8bc8f02 450db14305
commit e6b019c29d
249 changed files with 25137 additions and 11731 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/disk/endpoint.rs
+160 -27
View File
@@ -1,6 +1,6 @@
use crate::utils::net;
use common::error::{Error, Result};
use super::error::{Error, Result};
use path_absolutize::Absolutize;
use rustfs_utils::{is_local_host, is_socket_addr};
use std::{fmt::Display, path::Path};
use url::{ParseError, Url};
@@ -40,10 +40,10 @@ impl TryFrom<&str> for Endpoint {
type Error = Error;
/// Performs the conversion.
fn try_from(value: &str) -> Result<Self, Self::Error> {
fn try_from(value: &str) -> std::result::Result<Self, Self::Error> {
// check whether given path is not empty.
if ["", "/", "\\"].iter().any(|&v| v.eq(value)) {
return Err(Error::from_string("empty or root endpoint is not supported"));
return Err(Error::other("empty or root endpoint is not supported"));
}
let mut is_local = false;
@@ -59,7 +59,7 @@ impl TryFrom<&str> for Endpoint {
&& url.fragment().is_none()
&& url.query().is_none())
{
return Err(Error::from_string("invalid URL endpoint format"));
return Err(Error::other("invalid URL endpoint format"));
}
let path = url.path().to_string();
@@ -76,12 +76,12 @@ impl TryFrom<&str> for Endpoint {
let path = Path::new(&path[1..]).absolutize()?;
if path.parent().is_none() || Path::new("").eq(&path) {
return Err(Error::from_string("empty or root path is not supported in URL endpoint"));
return Err(Error::other("empty or root path is not supported in URL endpoint"));
}
match path.to_str() {
Some(v) => url.set_path(v),
None => return Err(Error::from_string("invalid path")),
None => return Err(Error::other("invalid path")),
}
url
@@ -93,15 +93,15 @@ impl TryFrom<&str> for Endpoint {
}
Err(e) => match e {
ParseError::InvalidPort => {
return Err(Error::from_string("invalid URL endpoint format: port number must be between 1 to 65535"))
return Err(Error::other("invalid URL endpoint format: port number must be between 1 to 65535"));
}
ParseError::EmptyHost => return Err(Error::from_string("invalid URL endpoint format: empty host name")),
ParseError::EmptyHost => return Err(Error::other("invalid URL endpoint format: empty host name")),
ParseError::RelativeUrlWithoutBase => {
// like /foo
is_local = true;
url_parse_from_file_path(value)?
}
_ => return Err(Error::from_string(format!("invalid URL endpoint format: {}", e))),
_ => return Err(Error::other(format!("invalid URL endpoint format: {}", e))),
},
};
@@ -144,7 +144,7 @@ impl Endpoint {
pub fn update_is_local(&mut self, local_port: u16) -> Result<()> {
match (self.url.scheme(), self.url.host()) {
(v, Some(host)) if v != "file" => {
self.is_local = net::is_local_host(host, self.url.port().unwrap_or_default(), local_port)?;
self.is_local = is_local_host(host, self.url.port().unwrap_or_default(), local_port)?;
}
_ => {}
}
@@ -185,18 +185,18 @@ fn url_parse_from_file_path(value: &str) -> Result<Url> {
// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
// /mnt/export1. So we go ahead and start the rustfs server in FS modes in these cases.
let addr: Vec<&str> = value.splitn(2, '/').collect();
if net::is_socket_addr(addr[0]) {
return Err(Error::from_string("invalid URL endpoint format: missing scheme http or https"));
if is_socket_addr(addr[0]) {
return Err(Error::other("invalid URL endpoint format: missing scheme http or https"));
}
let file_path = match Path::new(value).absolutize() {
Ok(path) => path,
Err(err) => return Err(Error::from_string(format!("absolute path failed: {}", err))),
Err(err) => return Err(Error::other(format!("absolute path failed: {}", err))),
};
match Url::from_file_path(file_path) {
Ok(url) => Ok(url),
Err(_) => Err(Error::from_string("Convert a file path into an URL failed")),
Err(_) => Err(Error::other("Convert a file path into an URL failed")),
}
}
@@ -260,49 +260,49 @@ mod test {
arg: "",
expected_endpoint: None,
expected_type: None,
expected_err: Some(Error::from_string("empty or root endpoint is not supported")),
expected_err: Some(Error::other("empty or root endpoint is not supported")),
},
TestCase {
arg: "/",
expected_endpoint: None,
expected_type: None,
expected_err: Some(Error::from_string("empty or root endpoint is not supported")),
expected_err: Some(Error::other("empty or root endpoint is not supported")),
},
TestCase {
arg: "\\",
expected_endpoint: None,
expected_type: None,
expected_err: Some(Error::from_string("empty or root endpoint is not supported")),
expected_err: Some(Error::other("empty or root endpoint is not supported")),
},
TestCase {
arg: "c://foo",
expected_endpoint: None,
expected_type: None,
expected_err: Some(Error::from_string("invalid URL endpoint format")),
expected_err: Some(Error::other("invalid URL endpoint format")),
},
TestCase {
arg: "ftp://foo",
expected_endpoint: None,
expected_type: None,
expected_err: Some(Error::from_string("invalid URL endpoint format")),
expected_err: Some(Error::other("invalid URL endpoint format")),
},
TestCase {
arg: "http://server/path?location",
expected_endpoint: None,
expected_type: None,
expected_err: Some(Error::from_string("invalid URL endpoint format")),
expected_err: Some(Error::other("invalid URL endpoint format")),
},
TestCase {
arg: "http://:/path",
expected_endpoint: None,
expected_type: None,
expected_err: Some(Error::from_string("invalid URL endpoint format: empty host name")),
expected_err: Some(Error::other("invalid URL endpoint format: empty host name")),
},
TestCase {
arg: "http://:8080/path",
expected_endpoint: None,
expected_type: None,
expected_err: Some(Error::from_string("invalid URL endpoint format: empty host name")),
expected_err: Some(Error::other("invalid URL endpoint format: empty host name")),
},
TestCase {
arg: "http://server:/path",
@@ -320,25 +320,25 @@ mod test {
arg: "https://93.184.216.34:808080/path",
expected_endpoint: None,
expected_type: None,
expected_err: Some(Error::from_string("invalid URL endpoint format: port number must be between 1 to 65535")),
expected_err: Some(Error::other("invalid URL endpoint format: port number must be between 1 to 65535")),
},
TestCase {
arg: "http://server:8080//",
expected_endpoint: None,
expected_type: None,
expected_err: Some(Error::from_string("empty or root path is not supported in URL endpoint")),
expected_err: Some(Error::other("empty or root path is not supported in URL endpoint")),
},
TestCase {
arg: "http://server:8080/",
expected_endpoint: None,
expected_type: None,
expected_err: Some(Error::from_string("empty or root path is not supported in URL endpoint")),
expected_err: Some(Error::other("empty or root path is not supported in URL endpoint")),
},
TestCase {
arg: "192.168.1.210:9000",
expected_endpoint: None,
expected_type: None,
expected_err: Some(Error::from_string("invalid URL endpoint format: missing scheme http or https")),
expected_err: Some(Error::other("invalid URL endpoint format: missing scheme http or https")),
},
];
@@ -372,4 +372,137 @@ mod test {
}
}
}
#[test]
fn test_endpoint_display() {
// Test file path display
let file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
let display_str = format!("{}", file_endpoint);
assert_eq!(display_str, "/tmp/data");
// Test URL display
let url_endpoint = Endpoint::try_from("http://example.com:9000/path").unwrap();
let display_str = format!("{}", url_endpoint);
assert_eq!(display_str, "http://example.com:9000/path");
}
#[test]
fn test_endpoint_type() {
let file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
assert_eq!(file_endpoint.get_type(), EndpointType::Path);
let url_endpoint = Endpoint::try_from("http://example.com:9000/path").unwrap();
assert_eq!(url_endpoint.get_type(), EndpointType::Url);
}
#[test]
fn test_endpoint_indexes() {
let mut endpoint = Endpoint::try_from("/tmp/data").unwrap();
// Test initial values
assert_eq!(endpoint.pool_idx, -1);
assert_eq!(endpoint.set_idx, -1);
assert_eq!(endpoint.disk_idx, -1);
// Test setting indexes
endpoint.set_pool_index(2);
endpoint.set_set_index(3);
endpoint.set_disk_index(4);
assert_eq!(endpoint.pool_idx, 2);
assert_eq!(endpoint.set_idx, 3);
assert_eq!(endpoint.disk_idx, 4);
}
#[test]
fn test_endpoint_grid_host() {
let endpoint = Endpoint::try_from("http://example.com:9000/path").unwrap();
assert_eq!(endpoint.grid_host(), "http://example.com:9000");
let endpoint_no_port = Endpoint::try_from("https://example.com/path").unwrap();
assert_eq!(endpoint_no_port.grid_host(), "https://example.com");
let file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
assert_eq!(file_endpoint.grid_host(), "");
}
#[test]
fn test_endpoint_host_port() {
let endpoint = Endpoint::try_from("http://example.com:9000/path").unwrap();
assert_eq!(endpoint.host_port(), "example.com:9000");
let endpoint_no_port = Endpoint::try_from("https://example.com/path").unwrap();
assert_eq!(endpoint_no_port.host_port(), "example.com");
let file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
assert_eq!(file_endpoint.host_port(), "");
}
#[test]
fn test_endpoint_get_file_path() {
let file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
assert_eq!(file_endpoint.get_file_path(), "/tmp/data");
let url_endpoint = Endpoint::try_from("http://example.com:9000/path/to/data").unwrap();
assert_eq!(url_endpoint.get_file_path(), "/path/to/data");
}
#[test]
fn test_endpoint_clone_and_equality() {
let endpoint1 = Endpoint::try_from("/tmp/data").unwrap();
let endpoint2 = endpoint1.clone();
assert_eq!(endpoint1, endpoint2);
assert_eq!(endpoint1.url, endpoint2.url);
assert_eq!(endpoint1.is_local, endpoint2.is_local);
assert_eq!(endpoint1.pool_idx, endpoint2.pool_idx);
assert_eq!(endpoint1.set_idx, endpoint2.set_idx);
assert_eq!(endpoint1.disk_idx, endpoint2.disk_idx);
}
#[test]
fn test_endpoint_with_special_paths() {
// Test with complex paths
let complex_path = "/var/lib/rustfs/data/bucket1";
let endpoint = Endpoint::try_from(complex_path).unwrap();
assert_eq!(endpoint.get_file_path(), complex_path);
assert!(endpoint.is_local);
assert_eq!(endpoint.get_type(), EndpointType::Path);
}
#[test]
fn test_endpoint_update_is_local() {
let mut endpoint = Endpoint::try_from("http://localhost:9000/path").unwrap();
let result = endpoint.update_is_local(9000);
assert!(result.is_ok());
let mut file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
let result = file_endpoint.update_is_local(9000);
assert!(result.is_ok());
}
#[test]
fn test_url_parse_from_file_path() {
let result = url_parse_from_file_path("/tmp/test");
assert!(result.is_ok());
let url = result.unwrap();
assert_eq!(url.scheme(), "file");
}
#[test]
fn test_endpoint_hash() {
use std::collections::HashSet;
let endpoint1 = Endpoint::try_from("/tmp/data1").unwrap();
let endpoint2 = Endpoint::try_from("/tmp/data2").unwrap();
let endpoint3 = endpoint1.clone();
let mut set = HashSet::new();
set.insert(endpoint1);
set.insert(endpoint2);
set.insert(endpoint3); // Should not be added as it's equal to endpoint1
assert_eq!(set.len(), 2);
}
}
ecstore/src/disk/error.rs
+617 -340
View File
File diff suppressed because it is too large Load Diff
ecstore/src/disk/error_conv.rs
+439
View File
@@ -0,0 +1,439 @@
use super::error::DiskError;
pub fn to_file_error(io_err: std::io::Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::NotFound => DiskError::FileNotFound.into(),
std::io::ErrorKind::PermissionDenied => DiskError::FileAccessDenied.into(),
std::io::ErrorKind::IsADirectory => DiskError::IsNotRegular.into(),
std::io::ErrorKind::NotADirectory => DiskError::FileAccessDenied.into(),
std::io::ErrorKind::DirectoryNotEmpty => DiskError::FileAccessDenied.into(),
std::io::ErrorKind::UnexpectedEof => DiskError::FaultyDisk.into(),
std::io::ErrorKind::TooManyLinks => DiskError::TooManyOpenFiles.into(),
std::io::ErrorKind::InvalidInput => DiskError::FileNotFound.into(),
std::io::ErrorKind::InvalidData => DiskError::FileCorrupt.into(),
std::io::ErrorKind::StorageFull => DiskError::DiskFull.into(),
_ => io_err,
}
}
pub fn to_volume_error(io_err: std::io::Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::NotFound => DiskError::VolumeNotFound.into(),
std::io::ErrorKind::PermissionDenied => DiskError::DiskAccessDenied.into(),
std::io::ErrorKind::DirectoryNotEmpty => DiskError::VolumeNotEmpty.into(),
std::io::ErrorKind::NotADirectory => DiskError::IsNotRegular.into(),
std::io::ErrorKind::Other => match io_err.downcast::<DiskError>() {
Ok(err) => match err {
DiskError::FileNotFound => DiskError::VolumeNotFound.into(),
DiskError::FileAccessDenied => DiskError::DiskAccessDenied.into(),
err => err.into(),
},
Err(err) => to_file_error(err),
},
_ => to_file_error(io_err),
}
}
pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::NotFound => DiskError::DiskNotFound.into(),
std::io::ErrorKind::PermissionDenied => DiskError::DiskAccessDenied.into(),
std::io::ErrorKind::Other => match io_err.downcast::<DiskError>() {
Ok(err) => match err {
DiskError::FileNotFound => DiskError::DiskNotFound.into(),
DiskError::VolumeNotFound => DiskError::DiskNotFound.into(),
DiskError::FileAccessDenied => DiskError::DiskAccessDenied.into(),
DiskError::VolumeAccessDenied => DiskError::DiskAccessDenied.into(),
err => err.into(),
},
Err(err) => to_volume_error(err),
},
_ => to_volume_error(io_err),
}
}
// only errors from FileSystem operations
pub fn to_access_error(io_err: std::io::Error, per_err: DiskError) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::PermissionDenied => per_err.into(),
std::io::ErrorKind::NotADirectory => per_err.into(),
std::io::ErrorKind::NotFound => DiskError::VolumeNotFound.into(),
std::io::ErrorKind::UnexpectedEof => DiskError::FaultyDisk.into(),
std::io::ErrorKind::Other => match io_err.downcast::<DiskError>() {
Ok(err) => match err {
DiskError::DiskAccessDenied => per_err.into(),
DiskError::FileAccessDenied => per_err.into(),
DiskError::FileNotFound => DiskError::VolumeNotFound.into(),
err => err.into(),
},
Err(err) => to_volume_error(err),
},
_ => to_volume_error(io_err),
}
}
pub fn to_unformatted_disk_error(io_err: std::io::Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::NotFound => DiskError::UnformattedDisk.into(),
std::io::ErrorKind::PermissionDenied => DiskError::DiskAccessDenied.into(),
std::io::ErrorKind::Other => match io_err.downcast::<DiskError>() {
Ok(err) => match err {
DiskError::FileNotFound => DiskError::UnformattedDisk.into(),
DiskError::DiskNotFound => DiskError::UnformattedDisk.into(),
DiskError::VolumeNotFound => DiskError::UnformattedDisk.into(),
DiskError::FileAccessDenied => DiskError::DiskAccessDenied.into(),
DiskError::DiskAccessDenied => DiskError::DiskAccessDenied.into(),
_ => DiskError::CorruptedBackend.into(),
},
Err(_err) => DiskError::CorruptedBackend.into(),
},
_ => DiskError::CorruptedBackend.into(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::{Error as IoError, ErrorKind};
// Helper function to create IO errors with specific kinds
fn create_io_error(kind: ErrorKind) -> IoError {
IoError::new(kind, "test error")
}
// Helper function to create IO errors with DiskError as the source
fn create_io_error_with_disk_error(disk_error: DiskError) -> IoError {
IoError::other(disk_error)
}
// Helper function to check if an IoError contains a specific DiskError
fn contains_disk_error(io_error: IoError, expected: DiskError) -> bool {
if let Ok(disk_error) = io_error.downcast::<DiskError>() {
std::mem::discriminant(&disk_error) == std::mem::discriminant(&expected)
} else {
false
}
}
#[test]
fn test_to_file_error_basic_conversions() {
// Test NotFound -> FileNotFound
let result = to_file_error(create_io_error(ErrorKind::NotFound));
assert!(contains_disk_error(result, DiskError::FileNotFound));
// Test PermissionDenied -> FileAccessDenied
let result = to_file_error(create_io_error(ErrorKind::PermissionDenied));
assert!(contains_disk_error(result, DiskError::FileAccessDenied));
// Test IsADirectory -> IsNotRegular
let result = to_file_error(create_io_error(ErrorKind::IsADirectory));
assert!(contains_disk_error(result, DiskError::IsNotRegular));
// Test NotADirectory -> FileAccessDenied
let result = to_file_error(create_io_error(ErrorKind::NotADirectory));
assert!(contains_disk_error(result, DiskError::FileAccessDenied));
// Test DirectoryNotEmpty -> FileAccessDenied
let result = to_file_error(create_io_error(ErrorKind::DirectoryNotEmpty));
assert!(contains_disk_error(result, DiskError::FileAccessDenied));
// Test UnexpectedEof -> FaultyDisk
let result = to_file_error(create_io_error(ErrorKind::UnexpectedEof));
assert!(contains_disk_error(result, DiskError::FaultyDisk));
// Test TooManyLinks -> TooManyOpenFiles
#[cfg(unix)]
{
let result = to_file_error(create_io_error(ErrorKind::TooManyLinks));
assert!(contains_disk_error(result, DiskError::TooManyOpenFiles));
}
// Test InvalidInput -> FileNotFound
let result = to_file_error(create_io_error(ErrorKind::InvalidInput));
assert!(contains_disk_error(result, DiskError::FileNotFound));
// Test InvalidData -> FileCorrupt
let result = to_file_error(create_io_error(ErrorKind::InvalidData));
assert!(contains_disk_error(result, DiskError::FileCorrupt));
// Test StorageFull -> DiskFull
#[cfg(unix)]
{
let result = to_file_error(create_io_error(ErrorKind::StorageFull));
assert!(contains_disk_error(result, DiskError::DiskFull));
}
}
#[test]
fn test_to_file_error_passthrough_unknown() {
// Test that unknown error kinds are passed through unchanged
let original = create_io_error(ErrorKind::Interrupted);
let result = to_file_error(original);
assert_eq!(result.kind(), ErrorKind::Interrupted);
}
#[test]
fn test_to_volume_error_basic_conversions() {
// Test NotFound -> VolumeNotFound
let result = to_volume_error(create_io_error(ErrorKind::NotFound));
assert!(contains_disk_error(result, DiskError::VolumeNotFound));
// Test PermissionDenied -> DiskAccessDenied
let result = to_volume_error(create_io_error(ErrorKind::PermissionDenied));
assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
// Test DirectoryNotEmpty -> VolumeNotEmpty
let result = to_volume_error(create_io_error(ErrorKind::DirectoryNotEmpty));
assert!(contains_disk_error(result, DiskError::VolumeNotEmpty));
// Test NotADirectory -> IsNotRegular
let result = to_volume_error(create_io_error(ErrorKind::NotADirectory));
assert!(contains_disk_error(result, DiskError::IsNotRegular));
}
#[test]
fn test_to_volume_error_other_with_disk_error() {
// Test Other error kind with FileNotFound DiskError -> VolumeNotFound
let io_error = create_io_error_with_disk_error(DiskError::FileNotFound);
let result = to_volume_error(io_error);
assert!(contains_disk_error(result, DiskError::VolumeNotFound));
// Test Other error kind with FileAccessDenied DiskError -> DiskAccessDenied
let io_error = create_io_error_with_disk_error(DiskError::FileAccessDenied);
let result = to_volume_error(io_error);
assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
// Test Other error kind with other DiskError -> passthrough
let io_error = create_io_error_with_disk_error(DiskError::DiskFull);
let result = to_volume_error(io_error);
assert!(contains_disk_error(result, DiskError::DiskFull));
}
#[test]
fn test_to_volume_error_fallback_to_file_error() {
// Test fallback to to_file_error for unknown error kinds
let result = to_volume_error(create_io_error(ErrorKind::Interrupted));
assert_eq!(result.kind(), ErrorKind::Interrupted);
}
#[test]
fn test_to_disk_error_basic_conversions() {
// Test NotFound -> DiskNotFound
let result = to_disk_error(create_io_error(ErrorKind::NotFound));
assert!(contains_disk_error(result, DiskError::DiskNotFound));
// Test PermissionDenied -> DiskAccessDenied
let result = to_disk_error(create_io_error(ErrorKind::PermissionDenied));
assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
}
#[test]
fn test_to_disk_error_other_with_disk_error() {
// Test Other error kind with FileNotFound DiskError -> DiskNotFound
let io_error = create_io_error_with_disk_error(DiskError::FileNotFound);
let result = to_disk_error(io_error);
assert!(contains_disk_error(result, DiskError::DiskNotFound));
// Test Other error kind with VolumeNotFound DiskError -> DiskNotFound
let io_error = create_io_error_with_disk_error(DiskError::VolumeNotFound);
let result = to_disk_error(io_error);
assert!(contains_disk_error(result, DiskError::DiskNotFound));
// Test Other error kind with FileAccessDenied DiskError -> DiskAccessDenied
let io_error = create_io_error_with_disk_error(DiskError::FileAccessDenied);
let result = to_disk_error(io_error);
assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
// Test Other error kind with VolumeAccessDenied DiskError -> DiskAccessDenied
let io_error = create_io_error_with_disk_error(DiskError::VolumeAccessDenied);
let result = to_disk_error(io_error);
assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
// Test Other error kind with other DiskError -> passthrough
let io_error = create_io_error_with_disk_error(DiskError::DiskFull);
let result = to_disk_error(io_error);
assert!(contains_disk_error(result, DiskError::DiskFull));
}
#[test]
fn test_to_disk_error_fallback_to_volume_error() {
// Test fallback to to_volume_error for unknown error kinds
let result = to_disk_error(create_io_error(ErrorKind::Interrupted));
assert_eq!(result.kind(), ErrorKind::Interrupted);
}
#[test]
fn test_to_access_error_basic_conversions() {
let permission_error = DiskError::FileAccessDenied;
// Test PermissionDenied -> specified permission error
let result = to_access_error(create_io_error(ErrorKind::PermissionDenied), permission_error);
assert!(contains_disk_error(result, DiskError::FileAccessDenied));
// Test NotADirectory -> specified permission error
let result = to_access_error(create_io_error(ErrorKind::NotADirectory), DiskError::FileAccessDenied);
assert!(contains_disk_error(result, DiskError::FileAccessDenied));
// Test NotFound -> VolumeNotFound
let result = to_access_error(create_io_error(ErrorKind::NotFound), DiskError::FileAccessDenied);
assert!(contains_disk_error(result, DiskError::VolumeNotFound));
// Test UnexpectedEof -> FaultyDisk
let result = to_access_error(create_io_error(ErrorKind::UnexpectedEof), DiskError::FileAccessDenied);
assert!(contains_disk_error(result, DiskError::FaultyDisk));
}
#[test]
fn test_to_access_error_other_with_disk_error() {
let permission_error = DiskError::VolumeAccessDenied;
// Test Other error kind with DiskAccessDenied -> specified permission error
let io_error = create_io_error_with_disk_error(DiskError::DiskAccessDenied);
let result = to_access_error(io_error, permission_error);
assert!(contains_disk_error(result, DiskError::VolumeAccessDenied));
// Test Other error kind with FileAccessDenied -> specified permission error
let io_error = create_io_error_with_disk_error(DiskError::FileAccessDenied);
let result = to_access_error(io_error, DiskError::VolumeAccessDenied);
assert!(contains_disk_error(result, DiskError::VolumeAccessDenied));
// Test Other error kind with FileNotFound -> VolumeNotFound
let io_error = create_io_error_with_disk_error(DiskError::FileNotFound);
let result = to_access_error(io_error, DiskError::VolumeAccessDenied);
assert!(contains_disk_error(result, DiskError::VolumeNotFound));
// Test Other error kind with other DiskError -> passthrough
let io_error = create_io_error_with_disk_error(DiskError::DiskFull);
let result = to_access_error(io_error, DiskError::VolumeAccessDenied);
assert!(contains_disk_error(result, DiskError::DiskFull));
}
#[test]
fn test_to_access_error_fallback_to_volume_error() {
let permission_error = DiskError::FileAccessDenied;
// Test fallback to to_volume_error for unknown error kinds
let result = to_access_error(create_io_error(ErrorKind::Interrupted), permission_error);
assert_eq!(result.kind(), ErrorKind::Interrupted);
}
#[test]
fn test_to_unformatted_disk_error_basic_conversions() {
// Test NotFound -> UnformattedDisk
let result = to_unformatted_disk_error(create_io_error(ErrorKind::NotFound));
assert!(contains_disk_error(result, DiskError::UnformattedDisk));
// Test PermissionDenied -> DiskAccessDenied
let result = to_unformatted_disk_error(create_io_error(ErrorKind::PermissionDenied));
assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
}
#[test]
fn test_to_unformatted_disk_error_other_with_disk_error() {
// Test Other error kind with FileNotFound -> UnformattedDisk
let io_error = create_io_error_with_disk_error(DiskError::FileNotFound);
let result = to_unformatted_disk_error(io_error);
assert!(contains_disk_error(result, DiskError::UnformattedDisk));
// Test Other error kind with DiskNotFound -> UnformattedDisk
let io_error = create_io_error_with_disk_error(DiskError::DiskNotFound);
let result = to_unformatted_disk_error(io_error);
assert!(contains_disk_error(result, DiskError::UnformattedDisk));
// Test Other error kind with VolumeNotFound -> UnformattedDisk
let io_error = create_io_error_with_disk_error(DiskError::VolumeNotFound);
let result = to_unformatted_disk_error(io_error);
assert!(contains_disk_error(result, DiskError::UnformattedDisk));
// Test Other error kind with FileAccessDenied -> DiskAccessDenied
let io_error = create_io_error_with_disk_error(DiskError::FileAccessDenied);
let result = to_unformatted_disk_error(io_error);
assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
// Test Other error kind with DiskAccessDenied -> DiskAccessDenied
let io_error = create_io_error_with_disk_error(DiskError::DiskAccessDenied);
let result = to_unformatted_disk_error(io_error);
assert!(contains_disk_error(result, DiskError::DiskAccessDenied));
// Test Other error kind with other DiskError -> CorruptedBackend
let io_error = create_io_error_with_disk_error(DiskError::DiskFull);
let result = to_unformatted_disk_error(io_error);
assert!(contains_disk_error(result, DiskError::CorruptedBackend));
}
#[test]
fn test_to_unformatted_disk_error_recursive_behavior() {
// Test with non-Other error kind that should be handled without infinite recursion
let result = to_unformatted_disk_error(create_io_error(ErrorKind::Interrupted));
// This should not cause infinite recursion and should produce CorruptedBackend
assert!(contains_disk_error(result, DiskError::CorruptedBackend));
}
#[test]
fn test_error_chain_conversions() {
// Test complex error conversion chains
let original_error = create_io_error(ErrorKind::NotFound);
// Chain: NotFound -> FileNotFound (via to_file_error) -> VolumeNotFound (via to_volume_error)
let file_error = to_file_error(original_error);
let volume_error = to_volume_error(file_error);
assert!(contains_disk_error(volume_error, DiskError::VolumeNotFound));
}
#[test]
fn test_cross_platform_error_kinds() {
// Test error kinds that may not be available on all platforms
#[cfg(unix)]
{
let result = to_file_error(create_io_error(ErrorKind::TooManyLinks));
assert!(contains_disk_error(result, DiskError::TooManyOpenFiles));
}
#[cfg(unix)]
{
let result = to_file_error(create_io_error(ErrorKind::StorageFull));
assert!(contains_disk_error(result, DiskError::DiskFull));
}
}
#[test]
fn test_error_conversion_with_different_kinds() {
// Test multiple error kinds to ensure comprehensive coverage
let test_cases = vec![
(ErrorKind::NotFound, DiskError::FileNotFound),
(ErrorKind::PermissionDenied, DiskError::FileAccessDenied),
(ErrorKind::IsADirectory, DiskError::IsNotRegular),
(ErrorKind::InvalidData, DiskError::FileCorrupt),
];
for (kind, expected_disk_error) in test_cases {
let result = to_file_error(create_io_error(kind));
assert!(
contains_disk_error(result, expected_disk_error.clone()),
"Failed for ErrorKind::{:?} -> DiskError::{:?}",
kind,
expected_disk_error
);
}
}
#[test]
fn test_volume_error_conversion_chain() {
// Test volume error conversion with different input types
let test_cases = vec![
(ErrorKind::NotFound, DiskError::VolumeNotFound),
(ErrorKind::PermissionDenied, DiskError::DiskAccessDenied),
(ErrorKind::DirectoryNotEmpty, DiskError::VolumeNotEmpty),
];
for (kind, expected_disk_error) in test_cases {
let result = to_volume_error(create_io_error(kind));
assert!(
contains_disk_error(result, expected_disk_error.clone()),
"Failed for ErrorKind::{:?} -> DiskError::{:?}",
kind,
expected_disk_error
);
}
}
}
ecstore/src/disk/error_reduce.rs
+162
View File
@@ -0,0 +1,162 @@
use super::error::Error;
pub static OBJECT_OP_IGNORED_ERRS: &[Error] = &[
Error::DiskNotFound,
Error::FaultyDisk,
Error::FaultyRemoteDisk,
Error::DiskAccessDenied,
Error::DiskOngoingReq,
Error::UnformattedDisk,
];
pub static BUCKET_OP_IGNORED_ERRS: &[Error] = &[
Error::DiskNotFound,
Error::FaultyDisk,
Error::FaultyRemoteDisk,
Error::DiskAccessDenied,
Error::UnformattedDisk,
];
pub static BASE_IGNORED_ERRS: &[Error] = &[Error::DiskNotFound, Error::FaultyDisk, Error::FaultyRemoteDisk];
pub fn reduce_write_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error], quorun: usize) -> Option<Error> {
reduce_quorum_errs(errors, ignored_errs, quorun, Error::ErasureWriteQuorum)
}
pub fn reduce_read_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error], quorun: usize) -> Option<Error> {
reduce_quorum_errs(errors, ignored_errs, quorun, Error::ErasureReadQuorum)
}
pub fn reduce_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error], quorun: usize, quorun_err: Error) -> Option<Error> {
let (max_count, err) = reduce_errs(errors, ignored_errs);
if max_count >= quorun { err } else { Some(quorun_err) }
}
pub fn reduce_errs(errors: &[Option<Error>], ignored_errs: &[Error]) -> (usize, Option<Error>) {
let nil_error = Error::other("nil".to_string());
// 首先统计 None 的数量（作为 nil 错误）
let nil_count = errors.iter().filter(|e| e.is_none()).count();
let err_counts = errors
.iter()
.filter_map(|e| e.as_ref()) // 只处理 Some 的错误
.fold(std::collections::HashMap::new(), |mut acc, e| {
if is_ignored_err(ignored_errs, e) {
return acc;
}
*acc.entry(e.clone()).or_insert(0) += 1;
acc
});
// 找到最高频率的非 nil 错误
let (best_err, best_count) = err_counts
.into_iter()
.max_by(|(_, c1), (_, c2)| c1.cmp(c2))
.unwrap_or((nil_error.clone(), 0));
// 比较 nil 错误和最高频率的非 nil 错误, 优先选择 nil 错误
if nil_count > best_count || (nil_count == best_count && nil_count > 0) {
(nil_count, None)
} else {
(best_count, Some(best_err))
}
}
pub fn is_ignored_err(ignored_errs: &[Error], err: &Error) -> bool {
ignored_errs.iter().any(|e| e == err)
}
pub fn count_errs(errors: &[Option<Error>], err: &Error) -> usize {
errors.iter().filter(|&e| e.as_ref() == Some(err)).count()
}
pub fn is_all_buckets_not_found(errs: &[Option<Error>]) -> bool {
for err in errs.iter() {
if let Some(err) = err {
if err == &Error::DiskNotFound || err == &Error::VolumeNotFound {
continue;
}
return false;
}
return false;
}
!errs.is_empty()
}
#[cfg(test)]
mod tests {
use super::*;
fn err_io(msg: &str) -> Error {
Error::Io(std::io::Error::other(msg))
}
#[test]
fn test_reduce_errs_basic() {
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None];
let ignored = vec![];
let (count, err) = reduce_errs(&errors, &ignored);
assert_eq!(count, 2);
assert_eq!(err, Some(e1));
}
#[test]
fn test_reduce_errs_ignored() {
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e2.clone()), Some(e1.clone()), Some(e2.clone()), None];
let ignored = vec![e2.clone()];
let (count, err) = reduce_errs(&errors, &ignored);
assert_eq!(count, 2);
assert_eq!(err, Some(e1));
}
#[test]
fn test_reduce_quorum_errs() {
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None];
let ignored = vec![];
let quorum_err = Error::FaultyDisk;
// quorum = 2, should return e1
let res = reduce_quorum_errs(&errors, &ignored, 2, quorum_err.clone());
assert_eq!(res, Some(e1));
// quorum = 3, should return quorum error
let res = reduce_quorum_errs(&errors, &ignored, 3, quorum_err.clone());
assert_eq!(res, Some(quorum_err));
}
#[test]
fn test_count_errs() {
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e2.clone()), Some(e1.clone()), None];
assert_eq!(count_errs(&errors, &e1), 2);
assert_eq!(count_errs(&errors, &e2), 1);
}
#[test]
fn test_is_ignored_err() {
let e1 = err_io("a");
let e2 = err_io("b");
let ignored = vec![e1.clone()];
assert!(is_ignored_err(&ignored, &e1));
assert!(!is_ignored_err(&ignored, &e2));
}
#[test]
fn test_reduce_errs_nil_tiebreak() {
// Error::Nil and another error have the same count, should prefer Nil
let e1 = err_io("a");
let errors = vec![Some(e1.clone()), None, Some(e1.clone()), None]; // e1:2, Nil:2
let ignored = vec![];
let (count, err) = reduce_errs(&errors, &ignored);
assert_eq!(count, 2);
assert_eq!(err, None); // None means Error::Nil is preferred
}
}
ecstore/src/disk/format.rs
+272 -11
View File
@@ -1,5 +1,5 @@
use super::{error::DiskError, DiskInfo};
use common::error::{Error, Result};
use super::error::{Error, Result};
use super::{DiskInfo, error::DiskError};
use serde::{Deserialize, Serialize};
use serde_json::Error as JsonError;
use uuid::Uuid;
@@ -110,7 +110,7 @@ pub struct FormatV3 {
impl TryFrom<&[u8]> for FormatV3 {
type Error = JsonError;
fn try_from(data: &[u8]) -> Result<Self, JsonError> {
fn try_from(data: &[u8]) -> std::result::Result<Self, Self::Error> {
serde_json::from_slice(data)
}
}
@@ -118,7 +118,7 @@ impl TryFrom<&[u8]> for FormatV3 {
impl TryFrom<&str> for FormatV3 {
type Error = JsonError;
fn try_from(data: &str) -> Result<Self, JsonError> {
fn try_from(data: &str) -> std::result::Result<Self, Self::Error> {
serde_json::from_str(data)
}
}
@@ -155,7 +155,7 @@ impl FormatV3 {
self.erasure.sets.iter().map(|v| v.len()).sum()
}
pub fn to_json(&self) -> Result<String, JsonError> {
pub fn to_json(&self) -> std::result::Result<String, JsonError> {
serde_json::to_string(self)
}
@@ -169,7 +169,7 @@ impl FormatV3 {
return Err(Error::from(DiskError::DiskNotFound));
}
if disk_id == Uuid::max() {
return Err(Error::msg("disk offline"));
return Err(Error::other("disk offline"));
}
for (i, set) in self.erasure.sets.iter().enumerate() {
@@ -180,7 +180,7 @@ impl FormatV3 {
}
}
Err(Error::msg(format!("disk id not found {}", disk_id)))
Err(Error::other(format!("disk id not found {}", disk_id)))
}
pub fn check_other(&self, other: &FormatV3) -> Result<()> {
@@ -189,7 +189,7 @@ impl FormatV3 {
tmp.erasure.this = Uuid::nil();
if self.erasure.sets.len() != other.erasure.sets.len() {
return Err(Error::from_string(format!(
return Err(Error::other(format!(
"Expected number of sets {}, got {}",
self.erasure.sets.len(),
other.erasure.sets.len()
@@ -198,7 +198,7 @@ impl FormatV3 {
for i in 0..self.erasure.sets.len() {
if self.erasure.sets[i].len() != other.erasure.sets[i].len() {
return Err(Error::from_string(format!(
return Err(Error::other(format!(
"Each set should be of same size, expected {}, got {}",
self.erasure.sets[i].len(),
other.erasure.sets[i].len()
@@ -207,7 +207,7 @@ impl FormatV3 {
for j in 0..self.erasure.sets[i].len() {
if self.erasure.sets[i][j] != other.erasure.sets[i][j] {
return Err(Error::from_string(format!(
return Err(Error::other(format!(
"UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)",
i,
j,
@@ -226,7 +226,7 @@ impl FormatV3 {
}
}
Err(Error::msg(format!(
Err(Error::other(format!(
"DriveID {:?} not found in any drive sets {:?}",
this, other.erasure.sets
)))
@@ -268,4 +268,265 @@ mod test {
println!("{:?}", p);
}
#[test]
fn test_format_v3_new_single_disk() {
let format = FormatV3::new(1, 1);
assert_eq!(format.version, FormatMetaVersion::V1);
assert_eq!(format.format, FormatBackend::ErasureSingle);
assert_eq!(format.erasure.version, FormatErasureVersion::V3);
assert_eq!(format.erasure.sets.len(), 1);
assert_eq!(format.erasure.sets[0].len(), 1);
assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3);
assert_eq!(format.erasure.this, Uuid::nil());
}
#[test]
fn test_format_v3_new_multiple_sets() {
let format = FormatV3::new(2, 4);
assert_eq!(format.version, FormatMetaVersion::V1);
assert_eq!(format.format, FormatBackend::Erasure);
assert_eq!(format.erasure.version, FormatErasureVersion::V3);
assert_eq!(format.erasure.sets.len(), 2);
assert_eq!(format.erasure.sets[0].len(), 4);
assert_eq!(format.erasure.sets[1].len(), 4);
assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3);
}
#[test]
fn test_format_v3_drives() {
let format = FormatV3::new(2, 4);
assert_eq!(format.drives(), 8); // 2 sets * 4 drives each
let format_single = FormatV3::new(1, 1);
assert_eq!(format_single.drives(), 1); // 1 set * 1 drive
}
#[test]
fn test_format_v3_to_json() {
let format = FormatV3::new(1, 2);
let json_result = format.to_json();
assert!(json_result.is_ok());
let json_str = json_result.unwrap();
assert!(json_str.contains("\"version\":\"1\""));
assert!(json_str.contains("\"format\":\"xl\""));
}
#[test]
fn test_format_v3_from_json() {
let json_data = r#"{
"version": "1",
"format": "xl-single",
"id": "321b3874-987d-4c15-8fa5-757c956b1243",
"xl": {
"version": "3",
"this": "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
"sets": [
[
"8ab9a908-f869-4f1f-8e42-eb067ffa7eb5"
]
],
"distributionAlgo": "SIPMOD+PARITY"
}
}"#;
let format = FormatV3::try_from(json_data);
assert!(format.is_ok());
let format = format.unwrap();
assert_eq!(format.format, FormatBackend::ErasureSingle);
assert_eq!(format.erasure.version, FormatErasureVersion::V3);
assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V3);
assert_eq!(format.erasure.sets.len(), 1);
assert_eq!(format.erasure.sets[0].len(), 1);
}
#[test]
fn test_format_v3_from_bytes() {
let json_data = r#"{
"version": "1",
"format": "xl",
"id": "321b3874-987d-4c15-8fa5-757c956b1243",
"xl": {
"version": "2",
"this": "00000000-0000-0000-0000-000000000000",
"sets": [
[
"8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
"c26315da-05cf-4778-a9ea-b44ea09f58c5"
]
],
"distributionAlgo": "SIPMOD"
}
}"#;
let format = FormatV3::try_from(json_data.as_bytes());
assert!(format.is_ok());
let format = format.unwrap();
assert_eq!(format.erasure.version, FormatErasureVersion::V2);
assert_eq!(format.erasure.distribution_algo, DistributionAlgoVersion::V2);
assert_eq!(format.erasure.sets[0].len(), 2);
}
#[test]
fn test_format_v3_invalid_json() {
let invalid_json = r#"{"invalid": "json"}"#;
let format = FormatV3::try_from(invalid_json);
assert!(format.is_err());
}
#[test]
fn test_find_disk_index_by_disk_id() {
let mut format = FormatV3::new(2, 2);
let target_disk_id = Uuid::new_v4();
format.erasure.sets[1][0] = target_disk_id;
let result = format.find_disk_index_by_disk_id(target_disk_id);
assert!(result.is_ok());
assert_eq!(result.unwrap(), (1, 0));
}
#[test]
fn test_find_disk_index_nil_uuid() {
let format = FormatV3::new(1, 2);
let result = format.find_disk_index_by_disk_id(Uuid::nil());
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::DiskNotFound));
}
#[test]
fn test_find_disk_index_max_uuid() {
let format = FormatV3::new(1, 2);
let result = format.find_disk_index_by_disk_id(Uuid::max());
assert!(result.is_err());
}
#[test]
fn test_find_disk_index_not_found() {
let format = FormatV3::new(1, 2);
let non_existent_id = Uuid::new_v4();
let result = format.find_disk_index_by_disk_id(non_existent_id);
assert!(result.is_err());
}
#[test]
fn test_check_other_identical() {
let format1 = FormatV3::new(2, 4);
let mut format2 = format1.clone();
format2.erasure.this = format1.erasure.sets[0][0];
let result = format1.check_other(&format2);
assert!(result.is_ok());
}
#[test]
fn test_check_other_different_set_count() {
let format1 = FormatV3::new(2, 4);
let format2 = FormatV3::new(3, 4);
let result = format1.check_other(&format2);
assert!(result.is_err());
}
#[test]
fn test_check_other_different_set_size() {
let format1 = FormatV3::new(2, 4);
let format2 = FormatV3::new(2, 6);
let result = format1.check_other(&format2);
assert!(result.is_err());
}
#[test]
fn test_check_other_different_disk_id() {
let format1 = FormatV3::new(1, 2);
let mut format2 = format1.clone();
format2.erasure.sets[0][0] = Uuid::new_v4();
let result = format1.check_other(&format2);
assert!(result.is_err());
}
#[test]
fn test_check_other_disk_not_in_sets() {
let format1 = FormatV3::new(1, 2);
let mut format2 = format1.clone();
format2.erasure.this = Uuid::new_v4(); // Set to a UUID not in any set
let result = format1.check_other(&format2);
assert!(result.is_err());
}
#[test]
fn test_format_meta_version_serialization() {
let v1 = FormatMetaVersion::V1;
let json = serde_json::to_string(&v1).unwrap();
assert_eq!(json, "\"1\"");
let unknown = FormatMetaVersion::Unknown;
let deserialized: FormatMetaVersion = serde_json::from_str("\"unknown\"").unwrap();
assert_eq!(deserialized, unknown);
}
#[test]
fn test_format_backend_serialization() {
let erasure = FormatBackend::Erasure;
let json = serde_json::to_string(&erasure).unwrap();
assert_eq!(json, "\"xl\"");
let single = FormatBackend::ErasureSingle;
let json = serde_json::to_string(&single).unwrap();
assert_eq!(json, "\"xl-single\"");
let unknown = FormatBackend::Unknown;
let deserialized: FormatBackend = serde_json::from_str("\"unknown\"").unwrap();
assert_eq!(deserialized, unknown);
}
#[test]
fn test_format_erasure_version_serialization() {
let v1 = FormatErasureVersion::V1;
let json = serde_json::to_string(&v1).unwrap();
assert_eq!(json, "\"1\"");
let v2 = FormatErasureVersion::V2;
let json = serde_json::to_string(&v2).unwrap();
assert_eq!(json, "\"2\"");
let v3 = FormatErasureVersion::V3;
let json = serde_json::to_string(&v3).unwrap();
assert_eq!(json, "\"3\"");
}
#[test]
fn test_distribution_algo_version_serialization() {
let v1 = DistributionAlgoVersion::V1;
let json = serde_json::to_string(&v1).unwrap();
assert_eq!(json, "\"CRCMOD\"");
let v2 = DistributionAlgoVersion::V2;
let json = serde_json::to_string(&v2).unwrap();
assert_eq!(json, "\"SIPMOD\"");
let v3 = DistributionAlgoVersion::V3;
let json = serde_json::to_string(&v3).unwrap();
assert_eq!(json, "\"SIPMOD+PARITY\"");
}
#[test]
fn test_format_v3_round_trip_serialization() {
let original = FormatV3::new(2, 3);
let json = original.to_json().unwrap();
let deserialized = FormatV3::try_from(json.as_str()).unwrap();
assert_eq!(original.version, deserialized.version);
assert_eq!(original.format, deserialized.format);
assert_eq!(original.erasure.version, deserialized.erasure.version);
assert_eq!(original.erasure.sets.len(), deserialized.erasure.sets.len());
assert_eq!(original.erasure.distribution_algo, deserialized.erasure.distribution_algo);
}
}
ecstore/src/disk/fs.rs
+530
View File
@@ -0,0 +1,530 @@
use std::{fs::Metadata, path::Path};
use tokio::{
fs::{self, File},
io,
};
pub const SLASH_SEPARATOR: &str = "/";
#[cfg(not(windows))]
pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool {
use std::os::unix::fs::MetadataExt;
if f1.dev() != f2.dev() {
return false;
}
if f1.ino() != f2.ino() {
return false;
}
if f1.size() != f2.size() {
return false;
}
if f1.permissions() != f2.permissions() {
return false;
}
if f1.mtime() != f2.mtime() {
return false;
}
true
}
#[cfg(windows)]
pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool {
if f1.permissions() != f2.permissions() {
return false;
}
if f1.file_type() != f2.file_type() {
return false;
}
if f1.len() != f2.len() {
return false;
}
true
}
type FileMode = usize;
pub const O_RDONLY: FileMode = 0x00000;
pub const O_WRONLY: FileMode = 0x00001;
pub const O_RDWR: FileMode = 0x00002;
pub const O_CREATE: FileMode = 0x00040;
// pub const O_EXCL: FileMode = 0x00080;
// pub const O_NOCTTY: FileMode = 0x00100;
pub const O_TRUNC: FileMode = 0x00200;
// pub const O_NONBLOCK: FileMode = 0x00800;
pub const O_APPEND: FileMode = 0x00400;
// pub const O_SYNC: FileMode = 0x01000;
// pub const O_ASYNC: FileMode = 0x02000;
// pub const O_CLOEXEC: FileMode = 0x80000;
// read: bool,
// write: bool,
// append: bool,
// truncate: bool,
// create: bool,
// create_new: bool,
pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
let mut opts = fs::OpenOptions::new();
match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
O_RDONLY => {
opts.read(true);
}
O_WRONLY => {
opts.write(true);
}
O_RDWR => {
opts.read(true);
opts.write(true);
}
_ => (),
};
if mode & O_CREATE != 0 {
opts.create(true);
}
if mode & O_APPEND != 0 {
opts.append(true);
}
if mode & O_TRUNC != 0 {
opts.truncate(true);
}
opts.open(path.as_ref()).await
}
pub async fn access(path: impl AsRef<Path>) -> io::Result<()> {
fs::metadata(path).await?;
Ok(())
}
pub fn access_std(path: impl AsRef<Path>) -> io::Result<()> {
tokio::task::block_in_place(|| std::fs::metadata(path))?;
Ok(())
}
pub async fn lstat(path: impl AsRef<Path>) -> io::Result<Metadata> {
fs::metadata(path).await
}
pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
tokio::task::block_in_place(|| std::fs::metadata(path))
}
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
fs::create_dir_all(path.as_ref()).await
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
let meta = fs::metadata(path.as_ref()).await?;
if meta.is_dir() {
fs::remove_dir(path.as_ref()).await
} else {
fs::remove_file(path.as_ref()).await
}
}
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
let meta = fs::metadata(path.as_ref()).await?;
if meta.is_dir() {
fs::remove_dir_all(path.as_ref()).await
} else {
fs::remove_file(path.as_ref()).await
}
}
#[tracing::instrument(level = "debug", skip_all)]
pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
let path = path.as_ref();
tokio::task::block_in_place(|| {
let meta = std::fs::metadata(path)?;
if meta.is_dir() {
std::fs::remove_dir(path)
} else {
std::fs::remove_file(path)
}
})
}
pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
let path = path.as_ref();
tokio::task::block_in_place(|| {
let meta = std::fs::metadata(path)?;
if meta.is_dir() {
std::fs::remove_dir_all(path)
} else {
std::fs::remove_file(path)
}
})
}
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
fs::create_dir(path.as_ref()).await
}
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
fs::rename(from, to).await
}
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
tokio::task::block_in_place(|| std::fs::rename(from, to))
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
fs::read(path.as_ref()).await
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
use tokio::io::AsyncWriteExt;
#[tokio::test]
async fn test_file_mode_constants() {
assert_eq!(O_RDONLY, 0x00000);
assert_eq!(O_WRONLY, 0x00001);
assert_eq!(O_RDWR, 0x00002);
assert_eq!(O_CREATE, 0x00040);
assert_eq!(O_TRUNC, 0x00200);
assert_eq!(O_APPEND, 0x00400);
}
#[tokio::test]
async fn test_open_file_read_only() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test_readonly.txt");
// Create a test file
tokio::fs::write(&file_path, b"test content").await.unwrap();
// Test opening in read-only mode
let file = open_file(&file_path, O_RDONLY).await;
assert!(file.is_ok());
}
#[tokio::test]
async fn test_open_file_write_only() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test_writeonly.txt");
// Test opening in write-only mode with create flag
let mut file = open_file(&file_path, O_WRONLY | O_CREATE).await.unwrap();
// Should be able to write
file.write_all(b"write test").await.unwrap();
file.flush().await.unwrap();
}
#[tokio::test]
async fn test_open_file_read_write() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test_readwrite.txt");
// Test opening in read-write mode with create flag
let mut file = open_file(&file_path, O_RDWR | O_CREATE).await.unwrap();
// Should be able to write and read
file.write_all(b"read-write test").await.unwrap();
file.flush().await.unwrap();
}
#[tokio::test]
async fn test_open_file_append() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test_append.txt");
// Create initial content
tokio::fs::write(&file_path, b"initial").await.unwrap();
// Open in append mode
let mut file = open_file(&file_path, O_WRONLY | O_APPEND).await.unwrap();
file.write_all(b" appended").await.unwrap();
file.flush().await.unwrap();
// Verify content
let content = tokio::fs::read_to_string(&file_path).await.unwrap();
assert_eq!(content, "initial appended");
}
#[tokio::test]
async fn test_open_file_truncate() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test_truncate.txt");
// Create initial content
tokio::fs::write(&file_path, b"initial content").await.unwrap();
// Open with truncate flag
let mut file = open_file(&file_path, O_WRONLY | O_TRUNC).await.unwrap();
file.write_all(b"new").await.unwrap();
file.flush().await.unwrap();
// Verify content was truncated
let content = tokio::fs::read_to_string(&file_path).await.unwrap();
assert_eq!(content, "new");
}
#[tokio::test]
async fn test_access() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test_access.txt");
// Should fail for non-existent file
assert!(access(&file_path).await.is_err());
// Create file and test again
tokio::fs::write(&file_path, b"test").await.unwrap();
assert!(access(&file_path).await.is_ok());
}
#[test]
fn test_access_std() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test_access_std.txt");
// Should fail for non-existent file
assert!(access_std(&file_path).is_err());
// Create file and test again
std::fs::write(&file_path, b"test").unwrap();
assert!(access_std(&file_path).is_ok());
}
#[tokio::test]
async fn test_lstat() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test_lstat.txt");
// Create test file
tokio::fs::write(&file_path, b"test content").await.unwrap();
// Test lstat
let metadata = lstat(&file_path).await.unwrap();
assert!(metadata.is_file());
assert_eq!(metadata.len(), 12); // "test content" is 12 bytes
}
#[test]
fn test_lstat_std() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test_lstat_std.txt");
// Create test file
std::fs::write(&file_path, b"test content").unwrap();
// Test lstat_std
let metadata = lstat_std(&file_path).unwrap();
assert!(metadata.is_file());
assert_eq!(metadata.len(), 12); // "test content" is 12 bytes
}
#[tokio::test]
async fn test_make_dir_all() {
let temp_dir = TempDir::new().unwrap();
let nested_path = temp_dir.path().join("level1").join("level2").join("level3");
// Should create nested directories
assert!(make_dir_all(&nested_path).await.is_ok());
assert!(nested_path.exists());
assert!(nested_path.is_dir());
}
#[tokio::test]
async fn test_remove_file() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test_remove.txt");
// Create test file
tokio::fs::write(&file_path, b"test").await.unwrap();
assert!(file_path.exists());
// Remove file
assert!(remove(&file_path).await.is_ok());
assert!(!file_path.exists());
}
#[tokio::test]
async fn test_remove_directory() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path().join("test_remove_dir");
// Create test directory
tokio::fs::create_dir(&dir_path).await.unwrap();
assert!(dir_path.exists());
// Remove directory
assert!(remove(&dir_path).await.is_ok());
assert!(!dir_path.exists());
}
#[tokio::test]
async fn test_remove_all() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path().join("test_remove_all");
let file_path = dir_path.join("nested_file.txt");
// Create nested structure
tokio::fs::create_dir(&dir_path).await.unwrap();
tokio::fs::write(&file_path, b"nested content").await.unwrap();
// Remove all
assert!(remove_all(&dir_path).await.is_ok());
assert!(!dir_path.exists());
}
#[test]
fn test_remove_std() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test_remove_std.txt");
// Create test file
std::fs::write(&file_path, b"test").unwrap();
assert!(file_path.exists());
// Remove file
assert!(remove_std(&file_path).is_ok());
assert!(!file_path.exists());
}
#[test]
fn test_remove_all_std() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path().join("test_remove_all_std");
let file_path = dir_path.join("nested_file.txt");
// Create nested structure
std::fs::create_dir(&dir_path).unwrap();
std::fs::write(&file_path, b"nested content").unwrap();
// Remove all
assert!(remove_all_std(&dir_path).is_ok());
assert!(!dir_path.exists());
}
#[tokio::test]
async fn test_mkdir() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path().join("test_mkdir");
// Create directory
assert!(mkdir(&dir_path).await.is_ok());
assert!(dir_path.exists());
assert!(dir_path.is_dir());
}
#[tokio::test]
async fn test_rename() {
let temp_dir = TempDir::new().unwrap();
let old_path = temp_dir.path().join("old_name.txt");
let new_path = temp_dir.path().join("new_name.txt");
// Create test file
tokio::fs::write(&old_path, b"test content").await.unwrap();
assert!(old_path.exists());
assert!(!new_path.exists());
// Rename file
assert!(rename(&old_path, &new_path).await.is_ok());
assert!(!old_path.exists());
assert!(new_path.exists());
// Verify content preserved
let content = tokio::fs::read_to_string(&new_path).await.unwrap();
assert_eq!(content, "test content");
}
#[test]
fn test_rename_std() {
let temp_dir = TempDir::new().unwrap();
let old_path = temp_dir.path().join("old_name_std.txt");
let new_path = temp_dir.path().join("new_name_std.txt");
// Create test file
std::fs::write(&old_path, b"test content").unwrap();
assert!(old_path.exists());
assert!(!new_path.exists());
// Rename file
assert!(rename_std(&old_path, &new_path).is_ok());
assert!(!old_path.exists());
assert!(new_path.exists());
// Verify content preserved
let content = std::fs::read_to_string(&new_path).unwrap();
assert_eq!(content, "test content");
}
#[tokio::test]
async fn test_read_file() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test_read.txt");
let test_content = b"This is test content for reading";
tokio::fs::write(&file_path, test_content).await.unwrap();
// Read file
let read_content = read_file(&file_path).await.unwrap();
assert_eq!(read_content, test_content);
}
#[tokio::test]
async fn test_read_file_nonexistent() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("nonexistent.txt");
// Should fail for non-existent file
assert!(read_file(&file_path).await.is_err());
}
#[tokio::test]
async fn test_same_file() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test_same.txt");
// Create test file
tokio::fs::write(&file_path, b"test content").await.unwrap();
// Get metadata twice
let metadata1 = tokio::fs::metadata(&file_path).await.unwrap();
let metadata2 = tokio::fs::metadata(&file_path).await.unwrap();
// Should be the same file
assert!(same_file(&metadata1, &metadata2));
}
#[tokio::test]
async fn test_different_files() {
let temp_dir = TempDir::new().unwrap();
let file1_path = temp_dir.path().join("file1.txt");
let file2_path = temp_dir.path().join("file2.txt");
// Create two different files
tokio::fs::write(&file1_path, b"content1").await.unwrap();
tokio::fs::write(&file2_path, b"content2").await.unwrap();
// Get metadata
let metadata1 = tokio::fs::metadata(&file1_path).await.unwrap();
let metadata2 = tokio::fs::metadata(&file2_path).await.unwrap();
// Should be different files
assert!(!same_file(&metadata1, &metadata2));
}
#[test]
fn test_slash_separator() {
assert_eq!(SLASH_SEPARATOR, "/");
}
}
ecstore/src/disk/local.rs
+560 -486
View File
File diff suppressed because it is too large Load Diff
ecstore/src/disk/mod.rs
+397 -616
View File
File diff suppressed because it is too large Load Diff
ecstore/src/disk/os.rs
+21 -38
View File
@@ -3,31 +3,28 @@ use std::{
path::{Component, Path},
};
use crate::{
disk::error::{is_sys_err_not_dir, is_sys_err_path_not_found, os_is_not_exist},
utils::{self, os::same_disk},
};
use common::error::{Error, Result};
use super::error::Result;
use crate::disk::error_conv::to_file_error;
use tokio::fs;
use super::error::{os_err_to_file_err, os_is_exist, DiskError};
use super::error::DiskError;
pub fn check_path_length(path_name: &str) -> Result<()> {
// Apple OS X path length is limited to 1016
if cfg!(target_os = "macos") && path_name.len() > 1016 {
return Err(Error::new(DiskError::FileNameTooLong));
return Err(DiskError::FileNameTooLong);
}
// Disallow more than 1024 characters on windows, there
// are no known name_max limits on Windows.
if cfg!(target_os = "windows") && path_name.len() > 1024 {
return Err(Error::new(DiskError::FileNameTooLong));
return Err(DiskError::FileNameTooLong);
}
// On Unix we reject paths if they are just '.', '..' or '/'
let invalid_paths = [".", "..", "/"];
if invalid_paths.contains(&path_name) {
return Err(Error::new(DiskError::FileAccessDenied));
return Err(DiskError::FileAccessDenied);
}
// Check each path segment length is > 255 on all Unix
@@ -40,7 +37,7 @@ pub fn check_path_length(path_name: &str) -> Result<()> {
_ => {
count += 1;
if count > 255 {
return Err(Error::new(DiskError::FileNameTooLong));
return Err(DiskError::FileNameTooLong);
}
}
}
@@ -55,19 +52,15 @@ pub fn is_root_disk(disk_path: &str, root_disk: &str) -> Result<bool> {
return Ok(false);
}
same_disk(disk_path, root_disk)
rustfs_utils::os::same_disk(disk_path, root_disk).map_err(|e| to_file_error(e).into())
}
pub async fn make_dir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> Result<()> {
check_path_length(path.as_ref().to_string_lossy().to_string().as_str())?;
if let Err(e) = reliable_mkdir_all(path.as_ref(), base_dir.as_ref()).await {
if is_sys_err_not_dir(&e) || is_sys_err_path_not_found(&e) {
return Err(Error::new(DiskError::FileAccessDenied));
}
return Err(os_err_to_file_err(e));
}
reliable_mkdir_all(path.as_ref(), base_dir.as_ref())
.await
.map_err(to_file_error)?;
Ok(())
}
@@ -77,7 +70,7 @@ pub async fn is_empty_dir(path: impl AsRef<Path>) -> bool {
}
// read_dir count read limit. when count == 0 unlimit.
pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> Result<Vec<String>> {
pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> std::io::Result<Vec<String>> {
let mut entries = fs::read_dir(path.as_ref()).await?;
let mut volumes = Vec::new();
@@ -96,7 +89,7 @@ pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> Result<Vec<String>>
if file_type.is_file() {
volumes.push(name);
} else if file_type.is_dir() {
volumes.push(format!("{}{}", name, utils::path::SLASH_SEPARATOR));
volumes.push(format!("{}{}", name, super::fs::SLASH_SEPARATOR));
}
count -= 1;
if count == 0 {
@@ -115,17 +108,7 @@ pub async fn rename_all(
) -> Result<()> {
reliable_rename(src_file_path, dst_file_path.as_ref(), base_dir)
.await
.map_err(|e| {
if is_sys_err_not_dir(&e) || !os_is_not_exist(&e) || is_sys_err_path_not_found(&e) {
Error::new(DiskError::FileAccessDenied)
} else if os_is_not_exist(&e) {
Error::new(DiskError::FileNotFound)
} else if os_is_exist(&e) {
Error::new(DiskError::IsNotRegular)
} else {
Error::new(e)
}
})?;
.map_err(to_file_error)?;
Ok(())
}
@@ -144,8 +127,8 @@ pub async fn reliable_rename(
let mut i = 0;
loop {
if let Err(e) = utils::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
if os_is_not_exist(&e) && i == 0 {
if let Err(e) = super::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
if e.kind() == io::ErrorKind::NotFound && i == 0 {
i += 1;
continue;
}
@@ -171,7 +154,7 @@ pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Pat
let mut base_dir = base_dir.as_ref();
loop {
if let Err(e) = os_mkdir_all(path.as_ref(), base_dir).await {
if os_is_not_exist(&e) && i == 0 {
if e.kind() == io::ErrorKind::NotFound && i == 0 {
i += 1;
if let Some(base_parent) = base_dir.parent() {
@@ -200,8 +183,8 @@ pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>
if let Some(parent) = dir_path.as_ref().parent() {
// 不支持递归，直接 create_dir_all 了
if let Err(e) = utils::fs::make_dir_all(&parent).await {
if os_is_exist(&e) {
if let Err(e) = super::fs::make_dir_all(&parent).await {
if e.kind() == io::ErrorKind::AlreadyExists {
return Ok(());
}
@@ -210,8 +193,8 @@ pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>
// Box::pin(os_mkdir_all(&parent, &base_dir)).await?;
}
if let Err(e) = utils::fs::mkdir(dir_path.as_ref()).await {
if os_is_exist(&e) {
if let Err(e) = super::fs::mkdir(dir_path.as_ref()).await {
if e.kind() == io::ErrorKind::AlreadyExists {
return Ok(());
}
ecstore/src/disk/remote.rs
+491 -233
View File
File diff suppressed because it is too large Load Diff