move ecsotre/error to common

This commit is contained in:
weisd
2025-03-25 17:14:40 +08:00
parent c3b47adf39
commit 545ae79e44
81 changed files with 444 additions and 359 deletions
Generated
+2
View File
@@ -1872,6 +1872,7 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
name = "e2e_test" name = "e2e_test"
version = "0.0.1" version = "0.0.1"
dependencies = [ dependencies = [
"common",
"ecstore", "ecstore",
"flatbuffers", "flatbuffers",
"futures", "futures",
@@ -2933,6 +2934,7 @@ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
"base64-simd", "base64-simd",
"common",
"crypto", "crypto",
"ecstore", "ecstore",
"futures", "futures",
+5
View File
@@ -61,6 +61,11 @@ impl Error {
pub fn downcast_mut<T: std::error::Error + 'static>(&mut self) -> Option<&mut T> { pub fn downcast_mut<T: std::error::Error + 'static>(&mut self) -> Option<&mut T> {
self.inner.downcast_mut() self.inner.downcast_mut()
} }
pub fn to_io_err(&self) -> Option<std::io::Error> {
self.downcast_ref::<std::io::Error>()
.map(|e| std::io::Error::new(e.kind(), e.to_string()))
}
} }
impl<T: std::error::Error + Send + Sync + 'static> From<T> for Error { impl<T: std::error::Error + Send + Sync + 'static> From<T> for Error {
+2
View File
@@ -6,6 +6,7 @@ license.workspace = true
repository.workspace = true repository.workspace = true
rust-version.workspace = true rust-version.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lints] [lints]
@@ -26,3 +27,4 @@ tokio = { workspace = true }
tower.workspace = true tower.workspace = true
url.workspace = true url.workspace = true
madmin.workspace =true madmin.workspace =true
common.workspace = true
+1 -1
View File
@@ -125,7 +125,7 @@ async fn walk_dir() -> Result<(), Box<dyn Error>> {
println!("{}", resp.error_info.unwrap_or("".to_string())); println!("{}", resp.error_info.unwrap_or("".to_string()));
} }
let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry) let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
.map_err(|e| ecstore::error::Error::from_string(format!("Unexpected response: {:?}", response))) .map_err(|e| common::error::Error::from_string(format!("Unexpected response: {:?}", response)))
.unwrap(); .unwrap();
out.write_obj(&entry).await.unwrap(); out.write_obj(&entry).await.unwrap();
} }
+3 -7
View File
@@ -1,12 +1,12 @@
use crate::{ use crate::{
disk::{error::DiskError, Disk, DiskAPI}, disk::{error::DiskError, Disk, DiskAPI},
erasure::{ReadAt, Writer}, erasure::{ReadAt, Writer},
error::{Error, Result},
io::{FileReader, FileWriter}, io::{FileReader, FileWriter},
store_api::BitrotAlgorithm, store_api::BitrotAlgorithm,
}; };
use blake2::Blake2b512; use blake2::Blake2b512;
use blake2::Digest as _; use blake2::Digest as _;
use common::error::{Error, Result};
use highway::{HighwayHash, HighwayHasher, Key}; use highway::{HighwayHash, HighwayHasher, Key};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use sha2::{digest::core_api::BlockSizeUser, Digest, Sha256}; use sha2::{digest::core_api::BlockSizeUser, Digest, Sha256};
@@ -731,14 +731,10 @@ pub fn new_bitrot_filereader(
mod test { mod test {
use std::collections::HashMap; use std::collections::HashMap;
use crate::{disk::error::DiskError, store_api::BitrotAlgorithm};
use common::error::{Error, Result};
use hex_simd::decode_to_vec; use hex_simd::decode_to_vec;
use crate::{
disk::error::DiskError,
error::{Error, Result},
store_api::BitrotAlgorithm,
};
// use super::{bitrot_writer_sum, new_bitrot_reader}; // use super::{bitrot_writer_sum, new_bitrot_reader};
#[test] #[test]
+1 -1
View File
@@ -1,4 +1,4 @@
use crate::error::Error; use common::error::Error;
#[derive(Debug, thiserror::Error, PartialEq, Eq)] #[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum BucketMetadataError { pub enum BucketMetadataError {
+2 -2
View File
@@ -16,9 +16,9 @@ use std::sync::Arc;
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::error; use tracing::error;
use crate::config::common::{read_config, save_config}; use crate::config::com::{read_config, save_config};
use crate::error::{Error, Result};
use crate::{config, new_object_layer_fn}; use crate::{config, new_object_layer_fn};
use common::error::{Error, Result};
use crate::disk::BUCKET_META_PREFIX; use crate::disk::BUCKET_META_PREFIX;
use crate::store::ECStore; use crate::store::ECStore;
+1 -1
View File
@@ -8,10 +8,10 @@ use crate::bucket::utils::is_meta_bucketname;
use crate::config; use crate::config;
use crate::config::error::ConfigError; use crate::config::error::ConfigError;
use crate::disk::error::DiskError; use crate::disk::error::DiskError;
use crate::error::{Error, Result};
use crate::global::{is_dist_erasure, is_erasure, new_object_layer_fn, GLOBAL_Endpoints}; use crate::global::{is_dist_erasure, is_erasure, new_object_layer_fn, GLOBAL_Endpoints};
use crate::store::ECStore; use crate::store::ECStore;
use crate::utils::xml::deserialize; use crate::utils::xml::deserialize;
use common::error::{Error, Result};
use futures::future::join_all; use futures::future::join_all;
use s3s::dto::{ use s3s::dto::{
BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration, BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration,
+1 -1
View File
@@ -1,4 +1,4 @@
use crate::error::{Error, Result}; use common::error::{Error, Result};
// use rmp_serde::Serializer as rmpSerializer; // use rmp_serde::Serializer as rmpSerializer;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap; use std::collections::HashMap;
+1 -1
View File
@@ -1,5 +1,5 @@
use super::keyname::{KeyName, ALL_SUPPORT_KEYS}; use super::keyname::{KeyName, ALL_SUPPORT_KEYS};
use crate::error::Error; use common::error::Error;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::{collections::HashSet, fmt, str::FromStr}; use std::{collections::HashSet, fmt, str::FromStr};
@@ -3,7 +3,7 @@ use lazy_static::lazy_static;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::str::FromStr; use std::str::FromStr;
use crate::error::Error; use common::error::Error;
use super::key::Key; use super::key::Key;
+1 -1
View File
@@ -1,8 +1,8 @@
use crate::error::{Error, Result};
use crate::{ use crate::{
bucket::policy::condition::keyname::COMMOM_KEYS, bucket::policy::condition::keyname::COMMOM_KEYS,
utils::{self, wildcard}, utils::{self, wildcard},
}; };
use common::error::{Error, Result};
use core::fmt; use core::fmt;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::{ use std::{
+1 -1
View File
@@ -3,7 +3,7 @@ use super::{
metadata_sys::get_bucket_metadata_sys, metadata_sys::get_bucket_metadata_sys,
policy::bucket_policy::{BucketPolicy, BucketPolicyArgs}, policy::bucket_policy::{BucketPolicy, BucketPolicyArgs},
}; };
use crate::error::Result; use common::error::Result;
use tracing::warn; use tracing::warn;
pub struct PolicySys {} pub struct PolicySys {}
+1 -1
View File
@@ -1,4 +1,4 @@
use crate::error::Result; use common::error::Result;
use rmp_serde::Serializer as rmpSerializer; use rmp_serde::Serializer as rmpSerializer;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
+1 -1
View File
@@ -1,4 +1,4 @@
use crate::error::Result; use common::error::Result;
use rmp_serde::Serializer as rmpSerializer; use rmp_serde::Serializer as rmpSerializer;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::time::Duration; use std::time::Duration;
+2 -1
View File
@@ -1,4 +1,5 @@
use crate::{disk::RUSTFS_META_BUCKET, error::Error}; use crate::disk::RUSTFS_META_BUCKET;
use common::error::{Error, Result};
pub fn is_meta_bucketname(name: &str) -> bool { pub fn is_meta_bucketname(name: &str) -> bool {
name.starts_with(RUSTFS_META_BUCKET) name.starts_with(RUSTFS_META_BUCKET)
+1 -1
View File
@@ -1,6 +1,6 @@
use super::{metadata_sys::get_bucket_metadata_sys, versioning::VersioningApi}; use super::{metadata_sys::get_bucket_metadata_sys, versioning::VersioningApi};
use crate::disk::RUSTFS_META_BUCKET; use crate::disk::RUSTFS_META_BUCKET;
use crate::error::Result; use common::error::Result;
use s3s::dto::VersioningConfiguration; use s3s::dto::VersioningConfiguration;
use tracing::warn; use tracing::warn;
+1 -1
View File
@@ -14,7 +14,7 @@ use std::{
use tokio::{spawn, sync::Mutex}; use tokio::{spawn, sync::Mutex};
use crate::error::Result; use common::error::Result;
pub type UpdateFn<T> = Box<dyn Fn() -> Pin<Box<dyn Future<Output = Result<T>> + Send>> + Send + Sync + 'static>; pub type UpdateFn<T> = Box<dyn Fn() -> Pin<Box<dyn Future<Output = Result<T>> + Send>> + Send + Sync + 'static>;
+7 -5
View File
@@ -1,11 +1,9 @@
use crate::disk::{DiskAPI, DiskStore, MetaCacheEntries, MetaCacheEntry, WalkDirOptions};
use crate::{ use crate::{
disk::error::{is_err_eof, is_err_file_not_found, is_err_volume_not_found, DiskError}, disk::error::{is_err_eof, is_err_file_not_found, is_err_volume_not_found, DiskError},
metacache::writer::MetacacheReader, metacache::writer::MetacacheReader,
}; };
use crate::{ use common::error::{Error, Result};
disk::{DiskAPI, DiskStore, MetaCacheEntries, MetaCacheEntry, WalkDirOptions},
error::{Error, Result},
};
use futures::future::join_all; use futures::future::join_all;
use std::{future::Future, pin::Pin, sync::Arc}; use std::{future::Future, pin::Pin, sync::Arc};
use tokio::{spawn, sync::broadcast::Receiver as B_Receiver}; use tokio::{spawn, sync::broadcast::Receiver as B_Receiver};
@@ -140,7 +138,11 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
} }
let revjob = spawn(async move { let revjob = spawn(async move {
let mut errs: Vec<Option<Error>> = vec![None; readers.len()]; let mut errs: Vec<Option<Error>> = Vec::with_capacity(readers.len());
for _ in 0..readers.len() {
errs.push(None);
}
loop { loop {
let mut current = MetaCacheEntry::default(); let mut current = MetaCacheEntry::default();
@@ -1,10 +1,10 @@
use super::error::{is_err_config_not_found, ConfigError}; use super::error::{is_err_config_not_found, ConfigError};
use super::{storageclass, Config, GLOBAL_StorageClass, KVS}; use super::{storageclass, Config, GLOBAL_StorageClass, KVS};
use crate::disk::RUSTFS_META_BUCKET; use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result};
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI}; use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
use crate::store_err::is_err_object_not_found; use crate::store_err::is_err_object_not_found;
use crate::utils::path::SLASH_SEPARATOR; use crate::utils::path::SLASH_SEPARATOR;
use common::error::{Error, Result};
use http::HeaderMap; use http::HeaderMap;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use std::collections::HashSet; use std::collections::HashSet;
+2 -1
View File
@@ -1,4 +1,5 @@
use crate::{disk, error::Error, store_err::is_err_object_not_found}; use crate::{disk, store_err::is_err_object_not_found};
use common::error::Error;
#[derive(Debug, PartialEq, thiserror::Error)] #[derive(Debug, PartialEq, thiserror::Error)]
pub enum ConfigError { pub enum ConfigError {
+2 -4
View File
@@ -1,9 +1,7 @@
use std::time::Duration; use std::time::Duration;
use crate::{ use crate::utils::bool_flag::parse_bool;
error::{Error, Result}, use common::error::{Error, Result};
utils::bool_flag::parse_bool,
};
#[derive(Debug, Default)] #[derive(Debug, Default)]
pub struct Config { pub struct Config {
+3 -3
View File
@@ -1,12 +1,12 @@
pub mod common; pub mod com;
pub mod error; pub mod error;
#[allow(dead_code)] #[allow(dead_code)]
pub mod heal; pub mod heal;
pub mod storageclass; pub mod storageclass;
use crate::error::Result;
use crate::store::ECStore; use crate::store::ECStore;
use common::{lookup_configs, read_config_without_migrate, STORAGE_CLASS_SUB_SYS}; use com::{lookup_configs, read_config_without_migrate, STORAGE_CLASS_SUB_SYS};
use common::error::Result;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap; use std::collections::HashMap;
+2 -4
View File
@@ -1,9 +1,7 @@
use std::env; use std::env;
use crate::{ use crate::config::KV;
config::KV, use common::error::{Error, Result};
error::{Error, Result},
};
use super::KVS; use super::KVS;
use lazy_static::lazy_static; use lazy_static::lazy_static;
+1 -1
View File
@@ -1,5 +1,5 @@
use crate::error::{Error, Result};
use crate::utils::net; use crate::utils::net;
use common::error::{Error, Result};
use path_absolutize::Absolutize; use path_absolutize::Absolutize;
use std::{fmt::Display, path::Path}; use std::{fmt::Display, path::Path};
use url::{ParseError, Url}; use url::{ParseError, Url};
+2 -4
View File
@@ -2,11 +2,9 @@ use std::io::{self, ErrorKind};
use tracing::error; use tracing::error;
use crate::quorum::CheckErrorFn;
use crate::utils::ERROR_TYPE_MASK; use crate::utils::ERROR_TYPE_MASK;
use crate::{ use common::error::{Error, Result};
error::{Error, Result},
quorum::CheckErrorFn,
};
// DiskError == StorageErr // DiskError == StorageErr
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
+1 -1
View File
@@ -1,5 +1,5 @@
use super::{error::DiskError, DiskInfo}; use super::{error::DiskError, DiskInfo};
use crate::error::{Error, Result}; use common::error::{Error, Result};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Error as JsonError; use serde_json::Error as JsonError;
use uuid::Uuid; use uuid::Uuid;
+1 -1
View File
@@ -18,7 +18,6 @@ use crate::disk::error::{
}; };
use crate::disk::os::{check_path_length, is_empty_dir}; use crate::disk::os::{check_path_length, is_empty_dir};
use crate::disk::STORAGE_FORMAT_FILE; use crate::disk::STORAGE_FORMAT_FILE;
use crate::error::{Error, Result};
use crate::file_meta::{get_file_info, read_xl_meta_no_data, FileInfoOpts}; use crate::file_meta::{get_file_info, read_xl_meta_no_data, FileInfoOpts};
use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
use crate::heal::data_scanner::{has_active_rules, scan_data_folder, ScannerItem, ShouldSleepFn, SizeSummary}; use crate::heal::data_scanner::{has_active_rules, scan_data_folder, ScannerItem, ShouldSleepFn, SizeSummary};
@@ -47,6 +46,7 @@ use crate::{
utils, utils,
}; };
use common::defer; use common::defer;
use common::error::{Error, Result};
use path_absolutize::Absolutize; use path_absolutize::Absolutize;
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::fmt::Debug; use std::fmt::Debug;
+1 -1
View File
@@ -16,7 +16,6 @@ pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
use crate::{ use crate::{
bucket::{metadata_sys::get_versioning_config, versioning::VersioningApi}, bucket::{metadata_sys::get_versioning_config, versioning::VersioningApi},
error::{Error, Result},
file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion, VersionType}, file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion, VersionType},
heal::{ heal::{
data_scanner::ShouldSleepFn, data_scanner::ShouldSleepFn,
@@ -27,6 +26,7 @@ use crate::{
store_api::{FileInfo, ObjectInfo, RawFileInfo}, store_api::{FileInfo, ObjectInfo, RawFileInfo},
utils::path::SLASH_SEPARATOR, utils::path::SLASH_SEPARATOR,
}; };
use common::error::{Error, Result};
use endpoint::Endpoint; use endpoint::Endpoint;
use error::DiskError; use error::DiskError;
use local::LocalDisk; use local::LocalDisk;
+3 -4
View File
@@ -3,14 +3,13 @@ use std::{
path::{Component, Path}, path::{Component, Path},
}; };
use tokio::fs;
use tracing::info;
use crate::{ use crate::{
disk::error::{is_sys_err_not_dir, is_sys_err_path_not_found, os_is_not_exist}, disk::error::{is_sys_err_not_dir, is_sys_err_path_not_found, os_is_not_exist},
error::{Error, Result},
utils::{self, os::same_disk}, utils::{self, os::same_disk},
}; };
use common::error::{Error, Result};
use tokio::fs;
use tracing::info;
use super::error::{os_err_to_file_err, os_is_exist, DiskError}; use super::error::{os_err_to_file_err, os_is_exist, DiskError};
+1 -1
View File
@@ -28,7 +28,6 @@ use super::{
}; };
use crate::{ use crate::{
disk::error::DiskError, disk::error::DiskError,
error::{Error, Result},
heal::{ heal::{
data_scanner::ShouldSleepFn, data_scanner::ShouldSleepFn,
data_usage_cache::{DataUsageCache, DataUsageEntry}, data_usage_cache::{DataUsageCache, DataUsageEntry},
@@ -41,6 +40,7 @@ use crate::{
io::{FileReader, FileWriter, HttpFileReader, HttpFileWriter}, io::{FileReader, FileWriter, HttpFileReader, HttpFileWriter},
utils::proto_err_to_err, utils::proto_err_to_err,
}; };
use common::error::{Error, Result};
use protos::proto_gen::node_service::RenamePartRequst; use protos::proto_gen::node_service::RenamePartRequst;
#[derive(Debug)] #[derive(Debug)]
+1 -1
View File
@@ -1,5 +1,5 @@
use crate::error::{Error, Result};
use crate::utils::ellipses::*; use crate::utils::ellipses::*;
use common::error::{Error, Result};
use serde::Deserialize; use serde::Deserialize;
use std::collections::HashSet; use std::collections::HashSet;
use std::env; use std::env;
+1 -1
View File
@@ -3,10 +3,10 @@ use tracing::warn;
use crate::{ use crate::{
disk::endpoint::{Endpoint, EndpointType}, disk::endpoint::{Endpoint, EndpointType},
disks_layout::DisksLayout, disks_layout::DisksLayout,
error::{Error, Result},
global::global_rustfs_port, global::global_rustfs_port,
utils::net::{self, XHost}, utils::net::{self, XHost},
}; };
use common::error::{Error, Result};
use std::{ use std::{
collections::{hash_map::Entry, HashMap, HashSet}, collections::{hash_map::Entry, HashMap, HashSet},
net::IpAddr, net::IpAddr,
+3 -2
View File
@@ -1,6 +1,7 @@
use crate::bitrot::{BitrotReader, BitrotWriter}; use crate::bitrot::{BitrotReader, BitrotWriter};
use crate::error::{Error, Result}; use crate::error::clone_err;
use crate::quorum::{object_op_ignored_errs, reduce_write_quorum_errs}; use crate::quorum::{object_op_ignored_errs, reduce_write_quorum_errs};
use common::error::{Error, Result};
use futures::future::join_all; use futures::future::join_all;
use reed_solomon_erasure::galois_8::ReedSolomon; use reed_solomon_erasure::galois_8::ReedSolomon;
use std::any::Any; use std::any::Any;
@@ -487,7 +488,7 @@ impl Erasure {
} }
} }
if !errs.is_empty() { if !errs.is_empty() {
return Err(errs[0].clone()); return Err(clone_err(&errs[0]));
} }
Ok(()) Ok(())
+100 -84
View File
@@ -1,106 +1,122 @@
use crate::disk::error::{clone_disk_err, DiskError}; use crate::disk::error::{clone_disk_err, DiskError};
use common::error::Error;
use std::io; use std::io;
use tracing_error::{SpanTrace, SpanTraceStatus}; // use tracing_error::{SpanTrace, SpanTraceStatus};
pub type StdError = Box<dyn std::error::Error + Send + Sync + 'static>; // pub type StdError = Box<dyn std::error::Error + Send + Sync + 'static>;
pub type Result<T = (), E = Error> = std::result::Result<T, E>; // pub type Result<T = (), E = Error> = std::result::Result<T, E>;
#[derive(Debug)] // #[derive(Debug)]
pub struct Error { // pub struct Error {
inner: Box<dyn std::error::Error + Send + Sync + 'static>, // inner: Box<dyn std::error::Error + Send + Sync + 'static>,
span_trace: SpanTrace, // span_trace: SpanTrace,
} // }
impl Error { // impl Error {
/// Create a new error from a `std::error::Error`. // /// Create a new error from a `std::error::Error`.
#[must_use] // #[must_use]
#[track_caller] // #[track_caller]
pub fn new<T: std::error::Error + Send + Sync + 'static>(source: T) -> Self { // pub fn new<T: std::error::Error + Send + Sync + 'static>(source: T) -> Self {
Self::from_std_error(source.into()) // Self::from_std_error(source.into())
} // }
/// Create a new error from a `std::error::Error`. // /// Create a new error from a `std::error::Error`.
#[must_use] // #[must_use]
#[track_caller] // #[track_caller]
pub fn from_std_error(inner: StdError) -> Self { // pub fn from_std_error(inner: StdError) -> Self {
Self { // Self {
inner, // inner,
span_trace: SpanTrace::capture(), // span_trace: SpanTrace::capture(),
} // }
} // }
/// Create a new error from a string. // /// Create a new error from a string.
#[must_use] // #[must_use]
#[track_caller] // #[track_caller]
pub fn from_string(s: impl Into<String>) -> Self { // pub fn from_string(s: impl Into<String>) -> Self {
Self::msg(s) // Self::msg(s)
} // }
/// Create a new error from a string. // /// Create a new error from a string.
#[must_use] // #[must_use]
#[track_caller] // #[track_caller]
pub fn msg(s: impl Into<String>) -> Self { // pub fn msg(s: impl Into<String>) -> Self {
Self::from_std_error(s.into().into()) // Self::from_std_error(s.into().into())
} // }
/// Returns `true` if the inner type is the same as `T`. // /// Returns `true` if the inner type is the same as `T`.
#[inline] // #[inline]
pub fn is<T: std::error::Error + 'static>(&self) -> bool { // pub fn is<T: std::error::Error + 'static>(&self) -> bool {
self.inner.is::<T>() // self.inner.is::<T>()
} // }
/// Returns some reference to the inner value if it is of type `T`, or // /// Returns some reference to the inner value if it is of type `T`, or
/// `None` if it isn't. // /// `None` if it isn't.
#[inline] // #[inline]
pub fn downcast_ref<T: std::error::Error + 'static>(&self) -> Option<&T> { // pub fn downcast_ref<T: std::error::Error + 'static>(&self) -> Option<&T> {
self.inner.downcast_ref() // self.inner.downcast_ref()
} // }
/// Returns some mutable reference to the inner value if it is of type `T`, or // /// Returns some mutable reference to the inner value if it is of type `T`, or
/// `None` if it isn't. // /// `None` if it isn't.
#[inline] // #[inline]
pub fn downcast_mut<T: std::error::Error + 'static>(&mut self) -> Option<&mut T> { // pub fn downcast_mut<T: std::error::Error + 'static>(&mut self) -> Option<&mut T> {
self.inner.downcast_mut() // self.inner.downcast_mut()
} // }
pub fn to_io_err(&self) -> Option<io::Error> { // pub fn to_io_err(&self) -> Option<io::Error> {
self.downcast_ref::<io::Error>() // self.downcast_ref::<io::Error>()
.map(|e| io::Error::new(e.kind(), e.to_string())) // .map(|e| io::Error::new(e.kind(), e.to_string()))
} // }
} // }
impl<T: std::error::Error + Send + Sync + 'static> From<T> for Error { // impl<T: std::error::Error + Send + Sync + 'static> From<T> for Error {
fn from(e: T) -> Self { // fn from(e: T) -> Self {
Self::new(e) // Self::new(e)
} // }
} // }
impl std::fmt::Display for Error { // impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.inner)?; // write!(f, "{}", self.inner)?;
if self.span_trace.status() != SpanTraceStatus::EMPTY { // if self.span_trace.status() != SpanTraceStatus::EMPTY {
write!(f, "\nspan_trace:\n{}", self.span_trace)?; // write!(f, "\nspan_trace:\n{}", self.span_trace)?;
} // }
Ok(()) // Ok(())
} // }
} // }
impl Clone for Error { // impl Clone for Error {
fn clone(&self) -> Self { // fn clone(&self) -> Self {
if let Some(e) = self.downcast_ref::<DiskError>() { // if let Some(e) = self.downcast_ref::<DiskError>() {
clone_disk_err(e) // clone_disk_err(e)
} else if let Some(e) = self.downcast_ref::<io::Error>() { // } else if let Some(e) = self.downcast_ref::<io::Error>() {
if let Some(code) = e.raw_os_error() { // if let Some(code) = e.raw_os_error() {
Error::new(io::Error::from_raw_os_error(code)) // Error::new(io::Error::from_raw_os_error(code))
} else { // } else {
Error::new(io::Error::new(e.kind(), e.to_string())) // Error::new(io::Error::new(e.kind(), e.to_string()))
} // }
// } else {
// // TODO: 优化其他类型
// Error::msg(self.to_string())
// }
// }
// }
pub fn clone_err(e: &Error) -> Error {
if let Some(e) = e.downcast_ref::<DiskError>() {
clone_disk_err(e)
} else if let Some(e) = e.downcast_ref::<io::Error>() {
if let Some(code) = e.raw_os_error() {
Error::new(io::Error::from_raw_os_error(code))
} else { } else {
// TODO: 优化其他类型 Error::new(io::Error::new(e.kind(), e.to_string()))
Error::msg(self.to_string())
} }
} else {
//TODO: 优化其他类型
Error::msg(e.to_string())
} }
} }
+9 -10
View File
@@ -1,4 +1,13 @@
use crate::disk::FileInfoVersions;
use crate::file_meta_inline::InlineData;
use crate::store_api::RawFileInfo;
use crate::store_err::StorageError;
use crate::{
disk::error::DiskError,
store_api::{ErasureInfo, FileInfo, ObjectPartInfo, ERASURE_ALGORITHM},
};
use byteorder::ByteOrder; use byteorder::ByteOrder;
use common::error::{Error, Result};
use rmp::Marker; use rmp::Marker;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::cmp::Ordering; use std::cmp::Ordering;
@@ -11,16 +20,6 @@ use tracing::{error, warn};
use uuid::Uuid; use uuid::Uuid;
use xxhash_rust::xxh64; use xxhash_rust::xxh64;
use crate::disk::FileInfoVersions;
use crate::file_meta_inline::InlineData;
use crate::store_api::RawFileInfo;
use crate::store_err::StorageError;
use crate::{
disk::error::DiskError,
error::{Error, Result},
store_api::{ErasureInfo, FileInfo, ObjectPartInfo, ERASURE_ALGORITHM},
};
// XL header specifies the format // XL header specifies the format
pub static XL_FILE_HEADER: [u8; 4] = [b'X', b'L', b'2', b' ']; pub static XL_FILE_HEADER: [u8; 4] = [b'X', b'L', b'2', b' '];
// pub static XL_FILE_VERSION_CURRENT: [u8; 4] = [0; 4]; // pub static XL_FILE_VERSION_CURRENT: [u8; 4] = [0; 4];
+2 -3
View File
@@ -1,8 +1,7 @@
use common::error::{Error, Result};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::error::{Error, Result};
use std::io::{Cursor, Read}; use std::io::{Cursor, Read};
use uuid::Uuid;
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct InlineData(Vec<u8>); pub struct InlineData(Vec<u8>);
+1 -1
View File
@@ -23,7 +23,6 @@ use crate::heal::heal_ops::{HealSource, BG_HEALING_UUID};
use crate::{ use crate::{
config::RUSTFS_CONFIG_PREFIX, config::RUSTFS_CONFIG_PREFIX,
disk::{endpoint::Endpoint, error::DiskError, DiskAPI, DiskInfoOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, disk::{endpoint::Endpoint, error::DiskError, DiskAPI, DiskInfoOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
error::{Error, Result},
global::{GLOBAL_BackgroundHealRoutine, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP}, global::{GLOBAL_BackgroundHealRoutine, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP},
heal::{ heal::{
data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT}, data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT},
@@ -36,6 +35,7 @@ use crate::{
store_api::{BucketInfo, BucketOptions, StorageAPI}, store_api::{BucketInfo, BucketOptions, StorageAPI},
utils::path::{path_join, SLASH_SEPARATOR}, utils::path::{path_join, SLASH_SEPARATOR},
}; };
use common::error::{Error, Result};
pub static DEFAULT_MONITOR_NEW_DISK_INTERVAL: Duration = Duration::from_secs(10); pub static DEFAULT_MONITOR_NEW_DISK_INTERVAL: Duration = Duration::from_secs(10);
+17 -18
View File
@@ -12,22 +12,6 @@ use std::{
time::{Duration, SystemTime}, time::{Duration, SystemTime},
}; };
use chrono::{DateTime, Utc};
use lazy_static::lazy_static;
use rand::Rng;
use rmp_serde::{Deserializer, Serializer};
use s3s::dto::{ReplicationConfiguration, ReplicationRuleStatus};
use serde::{Deserialize, Serialize};
use tokio::{
sync::{
broadcast,
mpsc::{self, Sender},
RwLock,
},
time::sleep,
};
use tracing::{error, info};
use super::{ use super::{
data_scanner_metric::{globalScannerMetrics, ScannerMetric, ScannerMetrics}, data_scanner_metric::{globalScannerMetrics, ScannerMetric, ScannerMetrics},
data_usage::{store_data_usage_in_backend, DATA_USAGE_BLOOM_NAME_PATH}, data_usage::{store_data_usage_in_backend, DATA_USAGE_BLOOM_NAME_PATH},
@@ -38,11 +22,10 @@ use crate::heal::data_usage::DATA_USAGE_ROOT;
use crate::{ use crate::{
cache_value::metacache_set::{list_path_raw, ListPathRawOptions}, cache_value::metacache_set::{list_path_raw, ListPathRawOptions},
config::{ config::{
common::{read_config, save_config}, com::{read_config, save_config},
heal::Config, heal::Config,
}, },
disk::{error::DiskError, DiskInfoOptions, DiskStore, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}, disk::{error::DiskError, DiskInfoOptions, DiskStore, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams},
error::{Error, Result},
global::{GLOBAL_BackgroundHealState, GLOBAL_IsErasure, GLOBAL_IsErasureSD}, global::{GLOBAL_BackgroundHealState, GLOBAL_IsErasure, GLOBAL_IsErasureSD},
heal::{ heal::{
data_usage::BACKGROUND_HEAL_INFO_PATH, data_usage::BACKGROUND_HEAL_INFO_PATH,
@@ -61,6 +44,22 @@ use crate::{
disk::DiskAPI, disk::DiskAPI,
store_api::{FileInfo, ObjectInfo}, store_api::{FileInfo, ObjectInfo},
}; };
use chrono::{DateTime, Utc};
use common::error::{Error, Result};
use lazy_static::lazy_static;
use rand::Rng;
use rmp_serde::{Deserializer, Serializer};
use s3s::dto::{ReplicationConfiguration, ReplicationRuleStatus};
use serde::{Deserialize, Serialize};
use tokio::{
sync::{
broadcast,
mpsc::{self, Sender},
RwLock,
},
time::sleep,
};
use tracing::{error, info};
const DATA_SCANNER_SLEEP_PER_FOLDER: Duration = Duration::from_millis(1); // Time to wait between folders. const DATA_SCANNER_SLEEP_PER_FOLDER: Duration = Duration::from_millis(1); // Time to wait between folders.
const DATA_USAGE_UPDATE_DIR_CYCLES: u32 = 16; // Visit all folders every n cycles. const DATA_USAGE_UPDATE_DIR_CYCLES: u32 = 16; // Visit all folders every n cycles.
+7 -8
View File
@@ -1,22 +1,21 @@
use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use std::{collections::HashMap, sync::Arc, time::SystemTime};
use tokio::sync::mpsc::Receiver;
use tracing::{error, warn};
use crate::{ use crate::{
bucket::metadata_sys::get_replication_config, bucket::metadata_sys::get_replication_config,
config::{ config::{
common::{read_config, save_config}, com::{read_config, save_config},
error::is_err_config_not_found, error::is_err_config_not_found,
}, },
disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
error::Result,
new_object_layer_fn, new_object_layer_fn,
store::ECStore, store::ECStore,
store_err::to_object_err, store_err::to_object_err,
utils::path::SLASH_SEPARATOR, utils::path::SLASH_SEPARATOR,
}; };
use common::error::Result;
use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use std::{collections::HashMap, sync::Arc, time::SystemTime};
use tokio::sync::mpsc::Receiver;
use tracing::{error, warn};
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
const DATA_USAGE_OBJ_NAME: &str = ".usage.json"; const DATA_USAGE_OBJ_NAME: &str = ".usage.json";
+2 -2
View File
@@ -1,11 +1,11 @@
use crate::config::common::save_config; use crate::config::com::save_config;
use crate::disk::error::DiskError; use crate::disk::error::DiskError;
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::error::{Error, Result};
use crate::new_object_layer_fn; use crate::new_object_layer_fn;
use crate::set_disk::SetDisks; use crate::set_disk::SetDisks;
use crate::store_api::{BucketInfo, ObjectIO, ObjectOptions}; use crate::store_api::{BucketInfo, ObjectIO, ObjectOptions};
use bytesize::ByteSize; use bytesize::ByteSize;
use common::error::{Error, Result};
use http::HeaderMap; use http::HeaderMap;
use path_clean::PathClean; use path_clean::PathClean;
use rand::Rng; use rand::Rng;
+6 -7
View File
@@ -4,22 +4,21 @@ use std::{
time::SystemTime, time::SystemTime,
}; };
use chrono::{DateTime, Utc};
use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
use tokio::sync::RwLock;
use crate::{ use crate::{
config::storageclass::{RRS, STANDARD}, config::storageclass::{RRS, STANDARD},
disk::{DeleteOptions, DiskAPI, DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, disk::{DeleteOptions, DiskAPI, DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
error::{Error, Result},
global::GLOBAL_BackgroundHealState, global::GLOBAL_BackgroundHealState,
heal::heal_ops::HEALING_TRACKER_FILENAME, heal::heal_ops::HEALING_TRACKER_FILENAME,
new_object_layer_fn, new_object_layer_fn,
store_api::{BucketInfo, StorageAPI}, store_api::{BucketInfo, StorageAPI},
utils::fs::read_file, utils::fs::read_file,
}; };
use chrono::{DateTime, Utc};
use common::error::{Error, Result};
use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
use tokio::sync::RwLock;
use super::{background_heal_ops::get_local_disks_to_heal, heal_ops::BG_HEALING_UUID}; use super::{background_heal_ops::get_local_disks_to_heal, heal_ops::BG_HEALING_UUID};
+2 -2
View File
@@ -6,7 +6,7 @@ use super::{
}; };
use crate::store_api::StorageAPI; use crate::store_api::StorageAPI;
use crate::{ use crate::{
config::common::CONFIG_PREFIX, config::com::CONFIG_PREFIX,
disk::RUSTFS_META_BUCKET, disk::RUSTFS_META_BUCKET,
global::GLOBAL_BackgroundHealRoutine, global::GLOBAL_BackgroundHealRoutine,
heal::{error::ERR_HEAL_STOP_SIGNALLED, heal_commands::DRIVE_STATE_OK}, heal::{error::ERR_HEAL_STOP_SIGNALLED, heal_commands::DRIVE_STATE_OK},
@@ -14,7 +14,6 @@ use crate::{
use crate::{ use crate::{
disk::{endpoint::Endpoint, MetaCacheEntry}, disk::{endpoint::Endpoint, MetaCacheEntry},
endpoints::Endpoints, endpoints::Endpoints,
error::{Error, Result},
global::GLOBAL_IsDistErasure, global::GLOBAL_IsDistErasure,
heal::heal_commands::{HealStartSuccess, HEAL_UNKNOWN_SCAN}, heal::heal_commands::{HealStartSuccess, HEAL_UNKNOWN_SCAN},
new_object_layer_fn, new_object_layer_fn,
@@ -25,6 +24,7 @@ use crate::{
utils::path::path_join, utils::path::path_join,
}; };
use chrono::Utc; use chrono::Utc;
use common::error::{Error, Result};
use futures::join; use futures::join;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use madmin::heal_commands::{HealDriveInfo, HealItemType, HealResultItem}; use madmin::heal_commands::{HealDriveInfo, HealItemType, HealResultItem};
+4 -4
View File
@@ -1,6 +1,6 @@
use crate::disk::MetaCacheEntry; use crate::disk::MetaCacheEntry;
use crate::error::Error; use crate::error::clone_err;
use crate::error::Result; use common::error::{Error, Result};
use rmp::Marker; use rmp::Marker;
use std::str::from_utf8; use std::str::from_utf8;
use tokio::io::AsyncRead; use tokio::io::AsyncRead;
@@ -246,7 +246,7 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
self.check_init().await?; self.check_init().await?;
if let Some(err) = &self.err { if let Some(err) = &self.err {
return Err(err.clone()); return Err(clone_err(err));
} }
let mut n = size; let mut n = size;
@@ -285,7 +285,7 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
self.check_init().await?; self.check_init().await?;
if let Some(err) = &self.err { if let Some(err) = &self.err {
return Err(err.clone()); return Err(clone_err(err));
} }
match rmp::decode::read_bool(&mut self.read_more(1).await?) { match rmp::decode::read_bool(&mut self.read_more(1).await?) {
+1 -1
View File
@@ -1,8 +1,8 @@
use crate::endpoints::EndpointServerPools; use crate::endpoints::EndpointServerPools;
use crate::error::{Error, Result};
use crate::global::get_global_endpoints; use crate::global::get_global_endpoints;
use crate::peer_rest_client::PeerRestClient; use crate::peer_rest_client::PeerRestClient;
use crate::StorageAPI; use crate::StorageAPI;
use common::error::{Error, Result};
use futures::future::join_all; use futures::future::join_all;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use madmin::{ItemState, ServerProperties}; use madmin::{ItemState, ServerProperties};
+18 -18
View File
@@ -1,18 +1,6 @@
use async_trait::async_trait;
use futures::future::join_all;
use madmin::heal_commands::{HealDriveInfo, HealResultItem};
use protos::node_service_time_out_client;
use protos::proto_gen::node_service::{
DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest,
};
use regex::Regex;
use std::{collections::HashMap, fmt::Debug, sync::Arc};
use tokio::sync::RwLock;
use tonic::Request;
use tracing::info;
use crate::disk::error::is_all_buckets_not_found; use crate::disk::error::is_all_buckets_not_found;
use crate::disk::{DiskAPI, DiskStore}; use crate::disk::{DiskAPI, DiskStore};
use crate::error::clone_err;
use crate::global::GLOBAL_LOCAL_DISK_MAP; use crate::global::GLOBAL_LOCAL_DISK_MAP;
use crate::heal::heal_commands::{ use crate::heal::heal_commands::{
HealOpts, DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, HEAL_ITEM_BUCKET, HealOpts, DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, HEAL_ITEM_BUCKET,
@@ -25,9 +13,21 @@ use crate::utils::wildcard::is_rustfs_meta_bucket_name;
use crate::{ use crate::{
disk::{self, error::DiskError, VolumeInfo}, disk::{self, error::DiskError, VolumeInfo},
endpoints::{EndpointServerPools, Node}, endpoints::{EndpointServerPools, Node},
error::{Error, Result},
store_api::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, store_api::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
}; };
use async_trait::async_trait;
use common::error::{Error, Result};
use futures::future::join_all;
use madmin::heal_commands::{HealDriveInfo, HealResultItem};
use protos::node_service_time_out_client;
use protos::proto_gen::node_service::{
DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest,
};
use regex::Regex;
use std::{collections::HashMap, fmt::Debug, sync::Arc};
use tokio::sync::RwLock;
use tonic::Request;
use tracing::info;
type Client = Arc<Box<dyn PeerS3Client>>; type Client = Arc<Box<dyn PeerS3Client>>;
@@ -95,7 +95,7 @@ impl S3PeerSys {
for (i, client) in self.clients.iter().enumerate() { for (i, client) in self.clients.iter().enumerate() {
if let Some(v) = client.get_pools() { if let Some(v) = client.get_pools() {
if v.contains(&pool_idx) { if v.contains(&pool_idx) {
per_pool_errs.push(errs[i].clone()); per_pool_errs.push(errs[i].as_ref().map(|e| clone_err(e)));
} }
} }
} }
@@ -130,7 +130,7 @@ impl S3PeerSys {
for (i, client) in self.clients.iter().enumerate() { for (i, client) in self.clients.iter().enumerate() {
if let Some(v) = client.get_pools() { if let Some(v) = client.get_pools() {
if v.contains(&pool_idx) { if v.contains(&pool_idx) {
per_pool_errs.push(errs[i].clone()); per_pool_errs.push(errs[i].as_ref().map(|e| clone_err(e)));
} }
} }
} }
@@ -781,7 +781,7 @@ pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result<HealResu
let bucket = bucket.to_string(); let bucket = bucket.to_string();
let bs_clone = before_state.clone(); let bs_clone = before_state.clone();
let as_clone = after_state.clone(); let as_clone = after_state.clone();
let errs_clone = errs.clone(); let errs_clone = errs.iter().map(|e| e.as_ref().map(|e| clone_err(e))).collect::<Vec<_>>();
futures.push(async move { futures.push(async move {
if bs_clone.read().await[idx] == DRIVE_STATE_MISSING { if bs_clone.read().await[idx] == DRIVE_STATE_MISSING {
info!("bucket not find, will recreate"); info!("bucket not find, will recreate");
@@ -795,7 +795,7 @@ pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result<HealResu
} }
} }
} }
errs_clone[idx].clone() errs_clone[idx].as_ref().map(|e| clone_err(e))
}); });
} }
+2 -2
View File
@@ -1,8 +1,7 @@
use crate::bucket::versioning_sys::BucketVersioningSys; use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::config::common::{read_config, save_config, CONFIG_PREFIX}; use crate::config::com::{read_config, save_config, CONFIG_PREFIX};
use crate::config::error::ConfigError; use crate::config::error::ConfigError;
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::error::{Error, Result};
use crate::heal::heal_commands::HealOpts; use crate::heal::heal_commands::HealOpts;
use crate::new_object_layer_fn; use crate::new_object_layer_fn;
use crate::notification_sys::get_global_notification_sys; use crate::notification_sys::get_global_notification_sys;
@@ -11,6 +10,7 @@ use crate::store_err::{is_err_bucket_exists, StorageError};
use crate::utils::path::{path_join, SLASH_SEPARATOR}; use crate::utils::path::{path_join, SLASH_SEPARATOR};
use crate::{sets::Sets, store::ECStore}; use crate::{sets::Sets, store::ECStore};
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use common::error::{Error, Result};
use rmp_serde::{Deserializer, Serializer}; use rmp_serde::{Deserializer, Serializer};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::fmt::Display; use std::fmt::Display;
+3 -2
View File
@@ -1,4 +1,5 @@
use crate::{disk::error::DiskError, error::Error}; use crate::{disk::error::DiskError, error::clone_err};
use common::error::Error;
use std::{collections::HashMap, fmt::Debug}; use std::{collections::HashMap, fmt::Debug};
// pub type CheckErrorFn = fn(e: &Error) -> bool; // pub type CheckErrorFn = fn(e: &Error) -> bool;
@@ -106,7 +107,7 @@ fn reduce_errs(errs: &[Option<Error>], ignored_errs: &[Box<dyn CheckErrorFn>]) -
if let Some(&c) = error_counts.get(&max_err) { if let Some(&c) = error_counts.get(&max_err) {
if let Some(&err_idx) = error_map.get(&max_err) { if let Some(&err_idx) = error_map.get(&max_err) {
let err = errs[err_idx].clone(); let err = errs[err_idx].as_ref().map(|e| clone_err(e));
return (c, err); return (c, err);
} }
+33 -21
View File
@@ -1,12 +1,3 @@
use std::{
collections::{HashMap, HashSet},
io::{Cursor, Write},
mem::replace,
path::Path,
sync::Arc,
time::Duration,
};
use crate::{ use crate::{
bitrot::{bitrot_verify, close_bitrot_writers, new_bitrot_filereader, new_bitrot_filewriter, BitrotFileWriter}, bitrot::{bitrot_verify, close_bitrot_writers, new_bitrot_filereader, new_bitrot_filewriter, BitrotFileWriter},
cache_value::metacache_set::{list_path_raw, ListPathRawOptions}, cache_value::metacache_set::{list_path_raw, ListPathRawOptions},
@@ -20,7 +11,7 @@ use crate::{
UpdateMetadataOpts, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, UpdateMetadataOpts, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
}, },
erasure::Erasure, erasure::Erasure,
error::{Error, Result}, error::clone_err,
file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion}, file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion},
global::{ global::{
get_global_deployment_id, is_dist_erasure, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, get_global_deployment_id, is_dist_erasure, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP,
@@ -64,6 +55,7 @@ use crate::{
}; };
use bytesize::ByteSize; use bytesize::ByteSize;
use chrono::Utc; use chrono::Utc;
use common::error::{Error, Result};
use futures::future::join_all; use futures::future::join_all;
use glob::Pattern; use glob::Pattern;
use http::HeaderMap; use http::HeaderMap;
@@ -82,6 +74,14 @@ use rand::{
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use std::hash::Hash; use std::hash::Hash;
use std::time::SystemTime; use std::time::SystemTime;
use std::{
collections::{HashMap, HashSet},
io::{Cursor, Write},
mem::replace,
path::Path,
sync::Arc,
time::Duration,
};
use time::OffsetDateTime; use time::OffsetDateTime;
use tokio::{ use tokio::{
io::{empty, AsyncWrite}, io::{empty, AsyncWrite},
@@ -320,7 +320,7 @@ impl SetDisks {
} }
Err(e) => { Err(e) => {
// ress.push(None); // ress.push(None);
errs.push(Some(e.clone())); errs.push(Some(clone_err(e)));
} }
} }
} }
@@ -1242,7 +1242,7 @@ impl SetDisks {
Err(err) => { Err(err) => {
for item in errs.iter_mut() { for item in errs.iter_mut() {
if item.is_none() { if item.is_none() {
*item = Some(err.clone()) *item = Some(clone_err(&err));
} }
} }
@@ -2292,7 +2292,7 @@ impl SetDisks {
"file({} : {}) part corrupt too much, can not to fix, disks_to_heal_count: {}, parity_blocks: {}", "file({} : {}) part corrupt too much, can not to fix, disks_to_heal_count: {}, parity_blocks: {}",
bucket, object, disks_to_heal_count, lastest_meta.erasure.parity_blocks bucket, object, disks_to_heal_count, lastest_meta.erasure.parity_blocks
); );
let mut t_errs = vec![None; errs.len()];
// Allow for dangling deletes, on versions that have DataDir missing etc. // Allow for dangling deletes, on versions that have DataDir missing etc.
// this would end up restoring the correct readable versions. // this would end up restoring the correct readable versions.
match self match self
@@ -2315,13 +2315,22 @@ impl SetDisks {
} else { } else {
Error::new(DiskError::FileNotFound) Error::new(DiskError::FileNotFound)
}; };
let mut t_errs = Vec::with_capacity(errs.len());
for _ in 0..errs.len() {
t_errs.push(None);
}
return Ok(( return Ok((
self.default_heal_result(m, &t_errs, bucket, object, version_id).await, self.default_heal_result(m, &t_errs, bucket, object, version_id).await,
Some(derr), Some(derr),
)); ));
} }
Err(err) => { Err(err) => {
t_errs = vec![Some(err.clone()); errs.len()]; // t_errs = vec![Some(err.clone()); errs.len()];
let mut t_errs = Vec::with_capacity(errs.len());
for _ in 0..errs.len() {
t_errs.push(Some(clone_err(&err)));
}
return Ok(( return Ok((
self.default_heal_result(FileInfo::default(), &t_errs, bucket, object, version_id) self.default_heal_result(FileInfo::default(), &t_errs, bucket, object, version_id)
.await, .await,
@@ -3521,7 +3530,7 @@ impl SetDisks {
} }
if let Some(err) = ret_err.as_ref() { if let Some(err) = ret_err.as_ref() {
return Err(err.clone()); return Err(clone_err(err));
} }
if !tracker.read().await.queue_buckets.is_empty() { if !tracker.read().await.queue_buckets.is_empty() {
return Err(Error::from_string(format!( return Err(Error::from_string(format!(
@@ -4438,7 +4447,7 @@ impl StorageAPI for SetDisks {
} }
}; };
let mut upload_ids = Vec::new(); let mut upload_ids: Vec<String> = Vec::new();
for disk in disks.iter().flatten() { for disk in disks.iter().flatten() {
if !disk.is_online().await { if !disk.is_online().await {
@@ -5227,7 +5236,10 @@ async fn disks_with_all_parts(
let erasure_distribution_reliable = inconsistent <= parts_metadata.len() / 2; let erasure_distribution_reliable = inconsistent <= parts_metadata.len() / 2;
let mut meta_errs = vec![None; errs.len()]; let mut meta_errs = Vec::with_capacity(errs.len());
for _ in 0..errs.len() {
meta_errs.push(None);
}
for (index, disk) in online_disks.iter().enumerate() { for (index, disk) in online_disks.iter().enumerate() {
let disk = if let Some(disk) = disk { let disk = if let Some(disk) = disk {
@@ -5237,8 +5249,8 @@ async fn disks_with_all_parts(
continue; continue;
}; };
if let Some(err) = errs[index].clone() { if let Some(err) = &errs[index] {
meta_errs[index] = Some(err); meta_errs[index] = Some(clone_err(err));
continue; continue;
} }
if !disk.is_online().await { if !disk.is_online().await {
@@ -5394,7 +5406,7 @@ pub fn should_heal_object_on_disk(
match err { match err {
Some(err) => match err.downcast_ref::<DiskError>() { Some(err) => match err.downcast_ref::<DiskError>() {
Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) | Some(DiskError::FileCorrupt) => { Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) | Some(DiskError::FileCorrupt) => {
return (true, Some(err.clone())); return (true, Some(clone_err(err)));
} }
_ => {} _ => {}
}, },
@@ -5417,7 +5429,7 @@ pub fn should_heal_object_on_disk(
} }
} }
} }
(false, err.clone()) (false, err.as_ref().map(|e| clone_err(e)))
} }
async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<madmin::Disk> { async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<madmin::Disk> {
+50 -31
View File
@@ -1,14 +1,6 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
use common::globals::GLOBAL_Local_Node_Name;
use futures::future::join_all;
use http::HeaderMap;
use lock::{namespace_lock::NsLockMap, new_lock_api, LockApi};
use madmin::heal_commands::{HealDriveInfo, HealResultItem};
use tokio::sync::RwLock;
use uuid::Uuid;
use crate::{ use crate::{
disk::{ disk::{
error::{is_unformatted_disk, DiskError}, error::{is_unformatted_disk, DiskError},
@@ -16,7 +8,6 @@ use crate::{
new_disk, DiskAPI, DiskInfo, DiskOption, DiskStore, new_disk, DiskAPI, DiskInfo, DiskOption, DiskStore,
}, },
endpoints::{Endpoints, PoolEndpoints}, endpoints::{Endpoints, PoolEndpoints},
error::{Error, Result},
global::{is_dist_erasure, GLOBAL_LOCAL_DISK_SET_DRIVES}, global::{is_dist_erasure, GLOBAL_LOCAL_DISK_SET_DRIVES},
heal::heal_commands::{ heal::heal_commands::{
HealOpts, DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, HEAL_ITEM_METADATA, HealOpts, DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, HEAL_ITEM_METADATA,
@@ -31,6 +22,14 @@ use crate::{
store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file}, store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
utils::{hash, path::path_join_buf}, utils::{hash, path::path_join_buf},
}; };
use common::error::{Error, Result};
use common::globals::GLOBAL_Local_Node_Name;
use futures::future::join_all;
use http::HeaderMap;
use lock::{namespace_lock::NsLockMap, new_lock_api, LockApi};
use madmin::heal_commands::{HealDriveInfo, HealResultItem};
use tokio::sync::RwLock;
use uuid::Uuid;
use crate::heal::heal_ops::HealSequence; use crate::heal::heal_ops::HealSequence;
use tokio::time::Duration; use tokio::time::Duration;
@@ -767,30 +766,50 @@ async fn init_storage_disks_with_errors(
opts: &DiskOption, opts: &DiskOption,
) -> (Vec<Option<DiskStore>>, Vec<Option<Error>>) { ) -> (Vec<Option<DiskStore>>, Vec<Option<Error>>) {
// Bootstrap disks. // Bootstrap disks.
let disks = Arc::new(RwLock::new(vec![None; endpoints.as_ref().len()])); // let disks = Arc::new(RwLock::new(vec![None; endpoints.as_ref().len()]));
let errs = Arc::new(RwLock::new(vec![None; endpoints.as_ref().len()])); // let errs = Arc::new(RwLock::new(vec![None; endpoints.as_ref().len()]));
let mut futures = Vec::with_capacity(endpoints.as_ref().len()); let mut futures = Vec::with_capacity(endpoints.as_ref().len());
for (index, endpoint) in endpoints.as_ref().iter().enumerate() { for endpoint in endpoints.as_ref().iter() {
let ep = endpoint.clone(); futures.push(new_disk(endpoint, opts));
let opt = opts.clone();
let disks_clone = disks.clone(); // let ep = endpoint.clone();
let errs_clone = errs.clone(); // let opt = opts.clone();
futures.push(tokio::spawn(async move { // let disks_clone = disks.clone();
match new_disk(&ep, &opt).await { // let errs_clone = errs.clone();
Ok(disk) => { // futures.push(tokio::spawn(async move {
disks_clone.write().await[index] = Some(disk); // match new_disk(&ep, &opt).await {
errs_clone.write().await[index] = None; // Ok(disk) => {
} // disks_clone.write().await[index] = Some(disk);
Err(err) => { // errs_clone.write().await[index] = None;
disks_clone.write().await[index] = None; // }
errs_clone.write().await[index] = Some(err); // Err(err) => {
} // disks_clone.write().await[index] = None;
} // errs_clone.write().await[index] = Some(err);
})); // }
// }
// }));
} }
let _ = join_all(futures).await; // let _ = join_all(futures).await;
let disks = disks.read().await.clone(); // let disks = disks.read().await.clone();
let errs = errs.read().await.clone(); // let errs = errs.read().await.clone();
let mut disks = Vec::with_capacity(endpoints.as_ref().len());
let mut errs = Vec::with_capacity(endpoints.as_ref().len());
let results = join_all(futures).await;
for result in results {
match result {
Ok(disk) => {
disks.push(Some(disk));
errs.push(None);
}
Err(err) => {
disks.push(None);
errs.push(Some(err));
}
}
}
(disks, errs) (disks, errs)
} }
+60 -35
View File
@@ -6,6 +6,7 @@ use crate::config::GLOBAL_StorageClass;
use crate::config::{self, storageclass, GLOBAL_ConfigSys}; use crate::config::{self, storageclass, GLOBAL_ConfigSys};
use crate::disk::endpoint::{Endpoint, EndpointType}; use crate::disk::endpoint::{Endpoint, EndpointType};
use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions, MetaCacheEntry}; use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions, MetaCacheEntry};
use crate::error::clone_err;
use crate::global::{ use crate::global::{
is_dist_erasure, is_erasure_sd, set_global_deployment_id, set_object_layer, DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, is_dist_erasure, is_erasure_sd, set_global_deployment_id, set_object_layer, DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION,
DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES,
@@ -30,7 +31,6 @@ use crate::{
bucket::metadata::BucketMetadata, bucket::metadata::BucketMetadata,
disk::{error::DiskError, new_disk, DiskOption, DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, disk::{error::DiskError, new_disk, DiskOption, DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
endpoints::EndpointServerPools, endpoints::EndpointServerPools,
error::{Error, Result},
peer::S3PeerSys, peer::S3PeerSys,
sets::Sets, sets::Sets,
store_api::{ store_api::{
@@ -40,6 +40,7 @@ use crate::{
}, },
store_init, utils, store_init, utils,
}; };
use common::error::{Error, Result};
use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port}; use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port};
use futures::future::join_all; use futures::future::join_all;
use glob::Pattern; use glob::Pattern;
@@ -664,7 +665,7 @@ impl ECStore {
has_def_pool = true; has_def_pool = true;
if !is_err_object_not_found(err) && !is_err_version_not_found(err) { if !is_err_object_not_found(err) && !is_err_version_not_found(err) {
return Err(err.clone()); return Err(clone_err(&err));
} }
if pinfo.object_info.delete_marker && !pinfo.object_info.name.is_empty() { if pinfo.object_info.delete_marker && !pinfo.object_info.name.is_empty() {
@@ -802,7 +803,7 @@ impl ECStore {
} }
let _ = task.await; let _ = task.await;
if let Some(err) = first_err.read().await.as_ref() { if let Some(err) = first_err.read().await.as_ref() {
return Err(err.clone()); return Err(clone_err(&err));
} }
Ok(()) Ok(())
} }
@@ -932,8 +933,8 @@ impl ECStore {
} }
} }
if derrs[0].is_some() { if let Some(e) = &derrs[0] {
return Err(derrs[0].as_ref().unwrap().clone()); return Err(clone_err(e));
} }
Ok(objs[0].as_ref().unwrap().clone()) Ok(objs[0].as_ref().unwrap().clone())
@@ -1056,13 +1057,23 @@ struct PoolErr {
err: Option<Error>, err: Option<Error>,
} }
#[derive(Debug, Default, Clone)] #[derive(Debug, Default)]
pub struct PoolObjInfo { pub struct PoolObjInfo {
pub index: usize, pub index: usize,
pub object_info: ObjectInfo, pub object_info: ObjectInfo,
pub err: Option<Error>, pub err: Option<Error>,
} }
impl Clone for PoolObjInfo {
fn clone(&self) -> Self {
Self {
index: self.index,
object_info: self.object_info.clone(),
err: self.err.as_ref().map(|e| clone_err(e)),
}
}
}
// #[derive(Debug, Default, Clone)] // #[derive(Debug, Default, Clone)]
// pub struct ListPathOptions { // pub struct ListPathOptions {
// pub id: String, // pub id: String,
@@ -2037,45 +2048,59 @@ impl StorageAPI for ECStore {
) -> Result<(HealResultItem, Option<Error>)> { ) -> Result<(HealResultItem, Option<Error>)> {
info!("ECStore heal_object"); info!("ECStore heal_object");
let object = utils::path::encode_dir_object(object); let object = utils::path::encode_dir_object(object);
let errs = Arc::new(RwLock::new(vec![None; self.pools.len()]));
let results = Arc::new(RwLock::new(vec![HealResultItem::default(); self.pools.len()]));
let mut futures = Vec::with_capacity(self.pools.len());
for (idx, pool) in self.pools.iter().enumerate() {
//TODO: IsSuspended
let object = object.clone();
let results = results.clone();
let errs = errs.clone();
futures.push(async move {
match pool.heal_object(bucket, &object, version_id, opts).await {
Ok((mut result, err)) => {
result.object = utils::path::decode_dir_object(&result.object);
results.write().await.insert(idx, result);
errs.write().await[idx] = err;
}
Err(err) => {
errs.write().await[idx] = Some(err);
}
}
});
}
let _ = join_all(futures).await;
// Return the first nil error let mut futures = Vec::with_capacity(self.pools.len());
for (index, err) in errs.read().await.iter().enumerate() { for pool in self.pools.iter() {
//TODO: IsSuspended
futures.push(pool.heal_object(bucket, &object, version_id, opts));
// futures.push(async move {
// match pool.heal_object(bucket, &object, version_id, opts).await {
// Ok((mut result, err)) => {
// result.object = utils::path::decode_dir_object(&result.object);
// results.write().await.insert(idx, result);
// errs.write().await[idx] = err;
// }
// Err(err) => {
// errs.write().await[idx] = Some(err);
// }
// }
// });
}
let results = join_all(futures).await;
let mut errs = Vec::with_capacity(self.pools.len());
let mut ress = Vec::with_capacity(self.pools.len());
for res in results.into_iter() {
match res {
Ok((result, err)) => {
let mut result = result;
result.object = utils::path::decode_dir_object(&result.object);
ress.push(result);
errs.push(err);
}
Err(err) => {
errs.push(Some(err));
ress.push(HealResultItem::default());
}
}
}
for (idx, err) in errs.iter().enumerate() {
if err.is_none() { if err.is_none() {
return Ok((results.write().await.remove(index), None)); return Ok((ress.remove(idx), None));
} }
} }
// No pool returned a nil error, return the first non 'not found' error // No pool returned a nil error, return the first non 'not found' error
for (index, err) in errs.read().await.iter().enumerate() { for (index, err) in errs.iter().enumerate() {
match err { match err {
Some(err) => match err.downcast_ref::<DiskError>() { Some(err) => match err.downcast_ref::<DiskError>() {
Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {} Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {}
_ => return Ok((results.write().await.remove(index), Some(err.clone()))), _ => return Ok((ress.remove(index), Some(clone_err(err)))),
}, },
None => { None => {
return Ok((results.write().await.remove(index), None)); return Ok((ress.remove(index), None));
} }
} }
} }
@@ -2227,7 +2252,7 @@ impl StorageAPI for ECStore {
} }
if !errs.is_empty() { if !errs.is_empty() {
return Err(errs[0].clone()); return Err(clone_err(&errs[0]));
} }
Ok(()) Ok(())
+2 -7
View File
@@ -1,13 +1,8 @@
use crate::heal::heal_ops::HealSequence; use crate::heal::heal_ops::HealSequence;
use crate::io::FileReader; use crate::io::FileReader;
use crate::store_utils::clean_metadata; use crate::store_utils::clean_metadata;
use crate::{ use crate::{disk::DiskStore, heal::heal_commands::HealOpts, utils::path::decode_dir_object, xhttp};
disk::DiskStore, use common::error::{Error, Result};
error::{Error, Result},
heal::heal_commands::HealOpts,
utils::path::decode_dir_object,
xhttp,
};
use http::{HeaderMap, HeaderValue}; use http::{HeaderMap, HeaderValue};
use madmin::heal_commands::HealResultItem; use madmin::heal_commands::HealResultItem;
use rmp_serde::Serializer; use rmp_serde::Serializer;
+1 -1
View File
@@ -1,8 +1,8 @@
use crate::{ use crate::{
disk::error::{is_err_file_not_found, DiskError}, disk::error::{is_err_file_not_found, DiskError},
error::Error,
utils::path::decode_dir_object, utils::path::decode_dir_object,
}; };
use common::error::Error;
#[derive(Debug, thiserror::Error, PartialEq, Eq)] #[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum StorageError { pub enum StorageError {
+1 -1
View File
@@ -7,9 +7,9 @@ use crate::{
new_disk, DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, new_disk, DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET,
}, },
endpoints::Endpoints, endpoints::Endpoints,
error::{Error, Result},
heal::heal_commands::init_healing_tracker, heal::heal_commands::init_healing_tracker,
}; };
use common::error::{Error, Result};
use futures::future::join_all; use futures::future::join_all;
use std::{ use std::{
collections::{hash_map::Entry, HashMap}, collections::{hash_map::Entry, HashMap},
+7 -6
View File
@@ -6,7 +6,7 @@ use crate::disk::{
DiskInfo, DiskStore, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, DiskInfo, DiskStore, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry,
MetadataResolutionParams, MetadataResolutionParams,
}; };
use crate::error::{Error, Result}; use crate::error::clone_err;
use crate::file_meta::merge_file_meta_versions; use crate::file_meta::merge_file_meta_versions;
use crate::peer::is_reserved_or_invalid_bucket; use crate::peer::is_reserved_or_invalid_bucket;
use crate::set_disk::SetDisks; use crate::set_disk::SetDisks;
@@ -16,6 +16,7 @@ use crate::store_err::{is_err_bucket_not_found, to_object_err, StorageError};
use crate::utils::path::{self, base_dir_from_prefix, SLASH_SEPARATOR}; use crate::utils::path::{self, base_dir_from_prefix, SLASH_SEPARATOR};
use crate::StorageAPI; use crate::StorageAPI;
use crate::{store::ECStore, store_api::ListObjectsV2Info}; use crate::{store::ECStore, store_api::ListObjectsV2Info};
use common::error::{Error, Result};
use futures::future::join_all; use futures::future::join_all;
use rand::seq::SliceRandom; use rand::seq::SliceRandom;
use rand::thread_rng; use rand::thread_rng;
@@ -508,7 +509,7 @@ impl ECStore {
// cancel channel // cancel channel
let (cancel_tx, cancel_rx) = broadcast::channel(1); let (cancel_tx, cancel_rx) = broadcast::channel(1);
let (err_tx, mut err_rx) = broadcast::channel::<Error>(1); let (err_tx, mut err_rx) = broadcast::channel::<Arc<Error>>(1);
let (sender, recv) = mpsc::channel(o.limit as usize); let (sender, recv) = mpsc::channel(o.limit as usize);
@@ -521,7 +522,7 @@ impl ECStore {
opts.stop_disk_at_limit = true; opts.stop_disk_at_limit = true;
if let Err(err) = store.list_merged(cancel_rx1, opts, sender).await { if let Err(err) = store.list_merged(cancel_rx1, opts, sender).await {
error!("list_merged err {:?}", err); error!("list_merged err {:?}", err);
let _ = err_tx1.send(err); let _ = err_tx1.send(Arc::new(err));
} }
}); });
@@ -533,7 +534,7 @@ impl ECStore {
let job2 = tokio::spawn(async move { let job2 = tokio::spawn(async move {
if let Err(err) = gather_results(cancel_rx2, opts, recv, result_tx).await { if let Err(err) = gather_results(cancel_rx2, opts, recv, result_tx).await {
error!("gather_results err {:?}", err); error!("gather_results err {:?}", err);
let _ = err_tx2.send(err); let _ = err_tx2.send(Arc::new(err));
} }
}); });
@@ -545,7 +546,7 @@ impl ECStore {
match res{ match res{
Ok(o) => { Ok(o) => {
error!("list_path err_rx.recv() ok {:?}", &o); error!("list_path err_rx.recv() ok {:?}", &o);
MetaCacheEntriesSortedResult{ entries: None, err: Some(o) } MetaCacheEntriesSortedResult{ entries: None, err: Some(clone_err(o.as_ref())) }
}, },
Err(err) => { Err(err) => {
error!("list_path err_rx.recv() err {:?}", &err); error!("list_path err_rx.recv() err {:?}", &err);
@@ -659,7 +660,7 @@ impl ECStore {
continue; continue;
} }
return Err(err.clone()); return Err(clone_err(err));
} else { } else {
all_at_eof = false; all_at_eof = false;
continue; continue;
+1 -1
View File
@@ -1,4 +1,4 @@
use crate::error::{Error, Result}; use common::error::{Error, Result};
pub fn parse_bool(str: &str) -> Result<bool> { pub fn parse_bool(str: &str) -> Result<bool> {
match str { match str {
+1 -1
View File
@@ -1,4 +1,4 @@
use crate::error::{Error, Result}; use common::error::{Error, Result};
use lazy_static::*; use lazy_static::*;
use regex::Regex; use regex::Regex;
+1 -1
View File
@@ -1,10 +1,10 @@
use crate::bucket::error::BucketMetadataError; use crate::bucket::error::BucketMetadataError;
use crate::config::error::ConfigError; use crate::config::error::ConfigError;
use crate::disk::error::DiskError; use crate::disk::error::DiskError;
use crate::error::Error;
use crate::quorum::QuorumError; use crate::quorum::QuorumError;
use crate::store_err::StorageError; use crate::store_err::StorageError;
use crate::store_init::ErasureError; use crate::store_init::ErasureError;
use common::error::Error;
use protos::proto_gen::node_service::Error as Proto_Error; use protos::proto_gen::node_service::Error as Proto_Error;
pub mod bool_flag; pub mod bool_flag;
+1 -1
View File
@@ -1,4 +1,4 @@
use crate::error::{Error, Result}; use common::error::{Error, Result};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use std::{ use std::{
collections::HashSet, collections::HashSet,
+2 -1
View File
@@ -1,5 +1,6 @@
use super::IOStats; use super::IOStats;
use crate::{disk::Info, error::Result}; use crate::disk::Info;
use common::error::Result;
use nix::sys::{stat::stat, statfs::statfs}; use nix::sys::{stat::stat, statfs::statfs};
use std::io::{Error, ErrorKind}; use std::io::{Error, ErrorKind};
use std::path::Path; use std::path::Path;
+1
View File
@@ -31,6 +31,7 @@ tracing.workspace = true
madmin.workspace = true madmin.workspace = true
lazy_static.workspace = true lazy_static.workspace = true
regex = "1.11.1" regex = "1.11.1"
common.workspace = true
[dev-dependencies] [dev-dependencies]
test-case.workspace = true test-case.workspace = true
+1 -1
View File
@@ -1,4 +1,4 @@
use ecstore::error::{Error, Result}; use common::error::{Error, Result};
use regex::Regex; use regex::Regex;
const ARN_PREFIX_ARN: &str = "arn"; const ARN_PREFIX_ARN: &str = "arn";
+1 -1
View File
@@ -3,7 +3,7 @@ use crate::policy::Policy;
use crate::sys::{iam_policy_claim_name_sa, Validator, INHERITED_POLICY_TYPE}; use crate::sys::{iam_policy_claim_name_sa, Validator, INHERITED_POLICY_TYPE};
use crate::utils; use crate::utils;
use crate::utils::extract_claims; use crate::utils::extract_claims;
use ecstore::error::{Error, Result}; use common::error::{Error, Result};
use serde::de::DeserializeOwned; use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::{json, Value}; use serde_json::{json, Value};
+25 -7
View File
@@ -1,3 +1,6 @@
use ecstore::disk::error::clone_disk_err;
use ecstore::disk::error::DiskError;
use crate::policy; use crate::policy;
#[derive(thiserror::Error, Debug)] #[derive(thiserror::Error, Debug)]
@@ -6,7 +9,7 @@ pub enum Error {
PolicyError(#[from] policy::Error), PolicyError(#[from] policy::Error),
#[error("ecsotre error: {0}")] #[error("ecsotre error: {0}")]
EcstoreError(ecstore::error::Error), EcstoreError(common::error::Error),
#[error("{0}")] #[error("{0}")]
StringError(String), StringError(String),
@@ -96,7 +99,7 @@ pub enum Error {
// matches!(e, Error::NoSuchUser(_)) // matches!(e, Error::NoSuchUser(_))
// } // }
pub fn is_err_no_such_policy(err: &ecstore::error::Error) -> bool { pub fn is_err_no_such_policy(err: &common::error::Error) -> bool {
if let Some(e) = err.downcast_ref::<Error>() { if let Some(e) = err.downcast_ref::<Error>() {
matches!(e, Error::NoSuchPolicy) matches!(e, Error::NoSuchPolicy)
} else { } else {
@@ -104,7 +107,7 @@ pub fn is_err_no_such_policy(err: &ecstore::error::Error) -> bool {
} }
} }
pub fn is_err_no_such_user(err: &ecstore::error::Error) -> bool { pub fn is_err_no_such_user(err: &common::error::Error) -> bool {
if let Some(e) = err.downcast_ref::<Error>() { if let Some(e) = err.downcast_ref::<Error>() {
matches!(e, Error::NoSuchUser(_)) matches!(e, Error::NoSuchUser(_))
} else { } else {
@@ -112,7 +115,7 @@ pub fn is_err_no_such_user(err: &ecstore::error::Error) -> bool {
} }
} }
pub fn is_err_no_such_account(err: &ecstore::error::Error) -> bool { pub fn is_err_no_such_account(err: &common::error::Error) -> bool {
if let Some(e) = err.downcast_ref::<Error>() { if let Some(e) = err.downcast_ref::<Error>() {
matches!(e, Error::NoSuchAccount(_)) matches!(e, Error::NoSuchAccount(_))
} else { } else {
@@ -120,7 +123,7 @@ pub fn is_err_no_such_account(err: &ecstore::error::Error) -> bool {
} }
} }
pub fn is_err_no_such_temp_account(err: &ecstore::error::Error) -> bool { pub fn is_err_no_such_temp_account(err: &common::error::Error) -> bool {
if let Some(e) = err.downcast_ref::<Error>() { if let Some(e) = err.downcast_ref::<Error>() {
matches!(e, Error::NoSuchTempAccount(_)) matches!(e, Error::NoSuchTempAccount(_))
} else { } else {
@@ -128,7 +131,7 @@ pub fn is_err_no_such_temp_account(err: &ecstore::error::Error) -> bool {
} }
} }
pub fn is_err_no_such_group(err: &ecstore::error::Error) -> bool { pub fn is_err_no_such_group(err: &common::error::Error) -> bool {
if let Some(e) = err.downcast_ref::<Error>() { if let Some(e) = err.downcast_ref::<Error>() {
matches!(e, Error::NoSuchGroup(_)) matches!(e, Error::NoSuchGroup(_))
} else { } else {
@@ -136,10 +139,25 @@ pub fn is_err_no_such_group(err: &ecstore::error::Error) -> bool {
} }
} }
pub fn is_err_no_such_service_account(err: &ecstore::error::Error) -> bool { pub fn is_err_no_such_service_account(err: &common::error::Error) -> bool {
if let Some(e) = err.downcast_ref::<Error>() { if let Some(e) = err.downcast_ref::<Error>() {
matches!(e, Error::NoSuchServiceAccount(_)) matches!(e, Error::NoSuchServiceAccount(_))
} else { } else {
false false
} }
} }
pub fn clone_err(e: &common::error::Error) -> common::error::Error {
if let Some(e) = e.downcast_ref::<DiskError>() {
clone_disk_err(e)
} else if let Some(e) = e.downcast_ref::<std::io::Error>() {
if let Some(code) = e.raw_os_error() {
common::error::Error::new(std::io::Error::from_raw_os_error(code))
} else {
common::error::Error::new(std::io::Error::new(e.kind(), e.to_string()))
}
} else {
//TODO: 优化其他类型
common::error::Error::msg(e.to_string())
}
}
+1 -1
View File
@@ -1,5 +1,5 @@
use auth::Credentials; use auth::Credentials;
use ecstore::error::{Error, Result}; use common::error::{Error, Result};
use ecstore::store::ECStore; use ecstore::store::ECStore;
use error::Error as IamError; use error::Error as IamError;
use log::debug; use log::debug;
+2 -4
View File
@@ -12,11 +12,9 @@ use crate::{
MAX_SVCSESSION_POLICY_SIZE, SESSION_POLICY_NAME, SESSION_POLICY_NAME_EXTRACTED, STATUS_DISABLED, STATUS_ENABLED, MAX_SVCSESSION_POLICY_SIZE, SESSION_POLICY_NAME, SESSION_POLICY_NAME_EXTRACTED, STATUS_DISABLED, STATUS_ENABLED,
}, },
}; };
use common::error::{Error, Result};
use ecstore::config::error::is_err_config_not_found;
use ecstore::utils::{crypto::base64_encode, path::path_join_buf}; use ecstore::utils::{crypto::base64_encode, path::path_join_buf};
use ecstore::{
config::error::is_err_config_not_found,
error::{Error, Result},
};
use log::{debug, warn}; use log::{debug, warn};
use madmin::{AccountStatus, AddOrUpdateUserReq, GroupDesc}; use madmin::{AccountStatus, AddOrUpdateUserReq, GroupDesc};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
+1 -1
View File
@@ -1,4 +1,4 @@
use ecstore::error::{Error, Result}; use common::error::{Error, Result};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::{collections::HashSet, ops::Deref}; use std::{collections::HashSet, ops::Deref};
use strum::{EnumString, IntoStaticStr}; use strum::{EnumString, IntoStaticStr};
+1 -1
View File
@@ -1,4 +1,4 @@
use ecstore::error::{Error, Result}; use common::error::{Error, Result};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use strum::{EnumString, IntoStaticStr}; use strum::{EnumString, IntoStaticStr};
+1 -1
View File
@@ -1,6 +1,6 @@
use super::key_name::KeyName; use super::key_name::KeyName;
use crate::{policy::Error as PolicyError, sys::Validator}; use crate::{policy::Error as PolicyError, sys::Validator};
use ecstore::error::Error; use common::error::Error;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+1 -1
View File
@@ -1,4 +1,4 @@
use ecstore::error::{Error, Result}; use common::error::{Error, Result};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::ops::Deref; use std::ops::Deref;
+2 -2
View File
@@ -1,6 +1,6 @@
use super::{Effect, Error as IamError, Statement, ID}; use super::{Effect, Error as IamError, Statement, ID};
use crate::sys::{Args, Validator, DEFAULT_VERSION}; use crate::sys::{Args, Validator, DEFAULT_VERSION};
use ecstore::error::{Error, Result}; use common::error::{Error, Result};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashSet; use std::collections::HashSet;
@@ -323,7 +323,7 @@ pub mod default {
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*; use super::*;
use ecstore::error::Result; use common::error::{Error, Result};
#[tokio::test] #[tokio::test]
async fn test_parse_policy() -> Result<()> { async fn test_parse_policy() -> Result<()> {
+1 -1
View File
@@ -1,4 +1,4 @@
use ecstore::error::{Error, Result}; use common::error::{Error, Result};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::{ use std::{
collections::{HashMap, HashSet}, collections::{HashMap, HashSet},
+1 -1
View File
@@ -1,7 +1,7 @@
use crate::sys::{Args, Validator}; use crate::sys::{Args, Validator};
use super::{action::Action, ActionSet, Effect, Error as IamError, Functions, ResourceSet, ID}; use super::{action::Action, ActionSet, Effect, Error as IamError, Functions, ResourceSet, ID};
use ecstore::error::{Error, Result}; use common::error::{Error, Result};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Clone, Default, Debug)] #[derive(Serialize, Deserialize, Clone, Default, Debug)]
+1 -1
View File
@@ -1,7 +1,7 @@
pub mod object; pub mod object;
use crate::{auth::UserIdentity, cache::Cache, policy::PolicyDoc}; use crate::{auth::UserIdentity, cache::Cache, policy::PolicyDoc};
use ecstore::error::Result; use common::error::Result;
use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde::{de::DeserializeOwned, Deserialize, Serialize};
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use time::OffsetDateTime; use time::OffsetDateTime;
+2 -2
View File
@@ -7,13 +7,13 @@ use crate::{
manager::{extract_jwt_claims, get_default_policyes}, manager::{extract_jwt_claims, get_default_policyes},
policy::PolicyDoc, policy::PolicyDoc,
}; };
use common::error::{Error, Result};
use ecstore::{ use ecstore::{
config::{ config::{
common::{delete_config, read_config, read_config_with_metadata, save_config}, com::{delete_config, read_config, read_config_with_metadata, save_config},
error::is_err_config_not_found, error::is_err_config_not_found,
RUSTFS_CONFIG_PREFIX, RUSTFS_CONFIG_PREFIX,
}, },
error::{Error, Result},
store::ECStore, store::ECStore,
store_api::{ObjectInfo, ObjectOptions}, store_api::{ObjectInfo, ObjectOptions},
store_list_objects::{ObjectInfoOrErr, WalkOptions}, store_list_objects::{ObjectInfoOrErr, WalkOptions},
+1 -1
View File
@@ -24,7 +24,7 @@ use crate::policy::PolicyDoc;
use crate::store::MappedPolicy; use crate::store::MappedPolicy;
use crate::store::Store; use crate::store::Store;
use crate::store::UserType; use crate::store::UserType;
use ecstore::error::{Error, Result}; use common::error::{Error, Result};
use ecstore::utils::crypto::base64_decode; use ecstore::utils::crypto::base64_decode;
use ecstore::utils::crypto::base64_encode; use ecstore::utils::crypto::base64_encode;
use madmin::AddOrUpdateUserReq; use madmin::AddOrUpdateUserReq;
+1 -1
View File
@@ -1,4 +1,4 @@
use ecstore::error::{Error, Result}; use common::error::{Error, Result};
use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header}; use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header};
use rand::{Rng, RngCore}; use rand::{Rng, RngCore};
use serde::{de::DeserializeOwned, Serialize}; use serde::{de::DeserializeOwned, Serialize};
+1 -1
View File
@@ -1,12 +1,12 @@
use super::router::Operation; use super::router::Operation;
use crate::storage::error::to_s3_error; use crate::storage::error::to_s3_error;
use bytes::Bytes; use bytes::Bytes;
use common::error::Error as ec_Error;
use ecstore::admin_server_info::get_server_info; use ecstore::admin_server_info::get_server_info;
use ecstore::bucket::policy::action::{Action, ActionSet}; use ecstore::bucket::policy::action::{Action, ActionSet};
use ecstore::bucket::policy::bucket_policy::{BPStatement, BucketPolicy}; use ecstore::bucket::policy::bucket_policy::{BPStatement, BucketPolicy};
use ecstore::bucket::policy::effect::Effect; use ecstore::bucket::policy::effect::Effect;
use ecstore::bucket::policy::resource::{Resource, ResourceSet}; use ecstore::bucket::policy::resource::{Resource, ResourceSet};
use ecstore::error::Error as ec_Error;
use ecstore::global::GLOBAL_ALlHealState; use ecstore::global::GLOBAL_ALlHealState;
use ecstore::heal::data_usage::load_data_usage_from_backend; use ecstore::heal::data_usage::load_data_usage_from_backend;
use ecstore::heal::heal_commands::HealOpts; use ecstore::heal::heal_commands::HealOpts;
+1 -1
View File
@@ -1,12 +1,12 @@
use std::{collections::HashMap, io::Cursor, pin::Pin}; use std::{collections::HashMap, io::Cursor, pin::Pin};
use common::error::Error as EcsError;
use ecstore::{ use ecstore::{
admin_server_info::get_local_server_property, admin_server_info::get_local_server_property,
bucket::{metadata::load_bucket_metadata, metadata_sys}, bucket::{metadata::load_bucket_metadata, metadata_sys},
disk::{ disk::{
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadOptions, UpdateMetadataOpts, DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadOptions, UpdateMetadataOpts,
}, },
error::Error as EcsError,
heal::{ heal::{
data_usage_cache::DataUsageCache, data_usage_cache::DataUsageCache,
heal_commands::{get_local_background_heal_status, HealOpts}, heal_commands::{get_local_background_heal_status, HealOpts},
+2 -2
View File
@@ -1,6 +1,6 @@
use ecstore::{disk::error::is_err_file_not_found, error::Error, store_err::StorageError}; use common::error::Error;
use ecstore::{disk::error::is_err_file_not_found, store_err::StorageError};
use s3s::{s3_error, S3Error, S3ErrorCode}; use s3s::{s3_error, S3Error, S3ErrorCode};
pub fn to_s3_error(err: Error) -> S3Error { pub fn to_s3_error(err: Error) -> S3Error {
if let Some(storage_err) = err.downcast_ref::<StorageError>() { if let Some(storage_err) = err.downcast_ref::<StorageError>() {
return match storage_err { return match storage_err {
+1 -1
View File
@@ -1,5 +1,5 @@
use common::error::{Error, Result};
use ecstore::bucket::versioning_sys::BucketVersioningSys; use ecstore::bucket::versioning_sys::BucketVersioningSys;
use ecstore::error::{Error, Result};
use ecstore::store_api::ObjectOptions; use ecstore::store_api::ObjectOptions;
use ecstore::store_err::StorageError; use ecstore::store_err::StorageError;
use ecstore::utils::path::is_dir_object; use ecstore::utils::path::is_dir_object;