gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-10 07:06:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge main

Browse Source
This commit is contained in:
weisd
2024-09-27 20:12:54 +08:00
parent bd02676dbc cef6a583b7
commit 91e65867bb
43 changed files with 3798 additions and 737 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/bucket_meta.rs
+9
View File
@@ -1,3 +1,5 @@
use std::collections::HashMap;
use rmp_serde::Serializer;
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
@@ -15,6 +17,9 @@ pub struct BucketMetadata {
format: u16,
version: u16,
pub name: String,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub tagging: Option<HashMap<String, String>>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub created: Option<OffsetDateTime>,
}
@@ -54,4 +59,8 @@ impl BucketMetadata {
Ok(buf)
}
pub fn unmarshal_from(buffer: &[u8]) -> Result<Self> {
Ok(rmp_serde::from_slice(buffer)?)
}
}
ecstore/src/disk/error.rs
+325 -15
View File
@@ -1,36 +1,105 @@
use crate::error::{Error, Result};
use std::io::{self, ErrorKind};
use crate::{
error::{Error, Result},
quorum::CheckErrorFn,
};
// DiskError == StorageErr
#[derive(Debug, thiserror::Error)]
pub enum DiskError {
#[error("maximum versions exceeded, please delete few versions to proceed")]
MaxVersionsExceeded,
#[error("unexpected error")]
Unexpected,
#[error("corrupted format")]
CorruptedFormat,
#[error("corrupted backend")]
CorruptedBackend,
#[error("unformatted disk error")]
UnformattedDisk,
#[error("inconsistent drive found")]
InconsistentDisk,
#[error("drive does not support O_DIRECT")]
UnsupportedDisk,
#[error("drive path full")]
DiskFull,
#[error("disk not a dir")]
DiskNotDir,
#[error("disk not found")]
DiskNotFound,
#[error("drive still did not complete the request")]
DiskOngoingReq,
#[error("drive is part of root drive, will not be used")]
DriveIsRoot,
#[error("remote drive is faulty")]
FaultyRemoteDisk,
#[error("drive is faulty")]
FaultyDisk,
#[error("drive access denied")]
DiskAccessDenied,
#[error("file not found")]
FileNotFound,
#[error("file version not found")]
FileVersionNotFound,
#[error("disk not found")]
DiskNotFound,
#[error("too many open files, please increase 'ulimit -n'")]
TooManyOpenFiles,
#[error("disk access denied")]
FileAccessDenied,
#[error("InconsistentDisk")]
InconsistentDisk,
#[error("file name too long")]
FileNameTooLong,
#[error("volume already exists")]
VolumeExists,
#[error("unformatted disk error")]
UnformattedDisk,
#[error("not of regular file type")]
IsNotRegular,
#[error("unsupport disk")]
UnsupportedDisk,
#[error("disk not a dir")]
DiskNotDir,
#[error("path not found")]
PathNotFound,
#[error("volume not found")]
VolumeNotFound,
#[error("volume is not empty")]
VolumeNotEmpty,
#[error("volume access denied")]
VolumeAccessDenied,
#[error("disk access denied")]
FileAccessDenied,
#[error("file is corrupted")]
FileCorrupt,
#[error("bit-rot hash algorithm is invalid")]
BitrotHashAlgoInvalid,
#[error("Rename across devices not allowed, please fix your backend configuration")]
CrossDeviceLink,
#[error("less data available than what was requested")]
LessData,
#[error("more data was sent than what was advertised")]
MoreData,
}
impl DiskError {
@@ -93,3 +162,244 @@ impl PartialEq for DiskError {
core::mem::discriminant(self) == core::mem::discriminant(other)
}
}
impl CheckErrorFn for DiskError {
fn is(&self, e: &Error) -> bool {
self.is(e)
}
}
pub fn clone_disk_err(e: &DiskError) -> Error {
match e {
DiskError::MaxVersionsExceeded => Error::new(DiskError::MaxVersionsExceeded),
DiskError::Unexpected => Error::new(DiskError::Unexpected),
DiskError::CorruptedFormat => Error::new(DiskError::CorruptedFormat),
DiskError::CorruptedBackend => Error::new(DiskError::CorruptedBackend),
DiskError::UnformattedDisk => Error::new(DiskError::UnformattedDisk),
DiskError::InconsistentDisk => Error::new(DiskError::InconsistentDisk),
DiskError::UnsupportedDisk => Error::new(DiskError::UnsupportedDisk),
DiskError::DiskFull => Error::new(DiskError::DiskFull),
DiskError::DiskNotDir => Error::new(DiskError::DiskNotDir),
DiskError::DiskNotFound => Error::new(DiskError::DiskNotFound),
DiskError::DiskOngoingReq => Error::new(DiskError::DiskOngoingReq),
DiskError::DriveIsRoot => Error::new(DiskError::DriveIsRoot),
DiskError::FaultyRemoteDisk => Error::new(DiskError::FaultyRemoteDisk),
DiskError::FaultyDisk => Error::new(DiskError::FaultyDisk),
DiskError::DiskAccessDenied => Error::new(DiskError::DiskAccessDenied),
DiskError::FileNotFound => Error::new(DiskError::FileNotFound),
DiskError::FileVersionNotFound => Error::new(DiskError::FileVersionNotFound),
DiskError::TooManyOpenFiles => Error::new(DiskError::TooManyOpenFiles),
DiskError::FileNameTooLong => Error::new(DiskError::FileNameTooLong),
DiskError::VolumeExists => Error::new(DiskError::VolumeExists),
DiskError::IsNotRegular => Error::new(DiskError::IsNotRegular),
DiskError::PathNotFound => Error::new(DiskError::PathNotFound),
DiskError::VolumeNotFound => Error::new(DiskError::VolumeNotFound),
DiskError::VolumeNotEmpty => Error::new(DiskError::VolumeNotEmpty),
DiskError::VolumeAccessDenied => Error::new(DiskError::VolumeAccessDenied),
DiskError::FileAccessDenied => Error::new(DiskError::FileAccessDenied),
DiskError::FileCorrupt => Error::new(DiskError::FileCorrupt),
DiskError::BitrotHashAlgoInvalid => Error::new(DiskError::BitrotHashAlgoInvalid),
DiskError::CrossDeviceLink => Error::new(DiskError::CrossDeviceLink),
DiskError::LessData => Error::new(DiskError::LessData),
DiskError::MoreData => Error::new(DiskError::MoreData),
}
}
pub fn os_err_to_file_err(e: io::Error) -> Error {
match e.kind() {
io::ErrorKind::NotFound => Error::new(DiskError::FileNotFound),
io::ErrorKind::PermissionDenied => Error::new(DiskError::FileAccessDenied),
// io::ErrorKind::ConnectionRefused => todo!(),
// io::ErrorKind::ConnectionReset => todo!(),
// io::ErrorKind::HostUnreachable => todo!(),
// io::ErrorKind::NetworkUnreachable => todo!(),
// io::ErrorKind::ConnectionAborted => todo!(),
// io::ErrorKind::NotConnected => todo!(),
// io::ErrorKind::AddrInUse => todo!(),
// io::ErrorKind::AddrNotAvailable => todo!(),
// io::ErrorKind::NetworkDown => todo!(),
// io::ErrorKind::BrokenPipe => todo!(),
// io::ErrorKind::AlreadyExists => todo!(),
// io::ErrorKind::WouldBlock => todo!(),
// io::ErrorKind::NotADirectory => DiskError::FileNotFound,
// io::ErrorKind::IsADirectory => DiskError::FileNotFound,
// io::ErrorKind::DirectoryNotEmpty => DiskError::VolumeNotEmpty,
// io::ErrorKind::ReadOnlyFilesystem => todo!(),
// io::ErrorKind::FilesystemLoop => todo!(),
// io::ErrorKind::StaleNetworkFileHandle => todo!(),
// io::ErrorKind::InvalidInput => todo!(),
// io::ErrorKind::InvalidData => todo!(),
// io::ErrorKind::TimedOut => todo!(),
// io::ErrorKind::WriteZero => todo!(),
// io::ErrorKind::StorageFull => DiskError::DiskFull,
// io::ErrorKind::NotSeekable => todo!(),
// io::ErrorKind::FilesystemQuotaExceeded => todo!(),
// io::ErrorKind::FileTooLarge => todo!(),
// io::ErrorKind::ResourceBusy => todo!(),
// io::ErrorKind::ExecutableFileBusy => todo!(),
// io::ErrorKind::Deadlock => todo!(),
// io::ErrorKind::CrossesDevices => todo!(),
// io::ErrorKind::TooManyLinks =>DiskError::TooManyOpenFiles,
// io::ErrorKind::InvalidFilename => todo!(),
// io::ErrorKind::ArgumentListTooLong => todo!(),
// io::ErrorKind::Interrupted => todo!(),
// io::ErrorKind::Unsupported => todo!(),
// io::ErrorKind::UnexpectedEof => todo!(),
// io::ErrorKind::OutOfMemory => todo!(),
// io::ErrorKind::Other => todo!(),
// TODO: 把不支持的king用字符串处理
_ => Error::new(e),
}
}
pub fn is_sys_err_no_space(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 28;
}
false
}
pub fn is_sys_err_invalid_arg(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 22;
}
false
}
pub fn is_sys_err_io(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 5;
}
false
}
pub fn is_sys_err_is_dir(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 21;
}
false
}
pub fn is_sys_err_not_dir(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 20;
}
false
}
pub fn is_sys_err_too_long(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 63;
}
false
}
pub fn is_sys_err_too_many_symlinks(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 62;
}
false
}
pub fn is_sys_err_not_empty(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
if no == 66 {
return true;
}
if cfg!(target_os = "solaris") && no == 17 {
return true;
}
if cfg!(target_os = "windows") && no == 145 {
return true;
}
}
false
}
pub fn is_sys_err_path_not_found(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
if cfg!(target_os = "windows") {
if no == 3 {
return true;
}
} else {
if no == 2 {
return true;
}
}
}
false
}
pub fn is_sys_err_handle_invalid(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
if cfg!(target_os = "windows") {
if no == 6 {
return true;
}
} else {
return false;
}
}
false
}
pub fn is_sys_err_cross_device(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 18;
}
false
}
pub fn is_sys_err_too_many_files(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 23 || no == 24;
}
false
}
pub fn os_is_not_exist(e: &io::Error) -> bool {
e.kind() == ErrorKind::NotFound
}
pub fn os_is_permission(e: &io::Error) -> bool {
if e.kind() == ErrorKind::PermissionDenied {
return true;
}
if let Some(no) = e.raw_os_error() {
if no == 30 {
return true;
}
}
false
}
pub fn os_is_exist(e: &io::Error) -> bool {
e.kind() == ErrorKind::AlreadyExists
}
// map_err_not_exists
pub fn map_err_not_exists(e: io::Error) -> Error {
if os_is_not_exist(&e) {
return Error::new(DiskError::VolumeNotEmpty);
} else if is_sys_err_io(&e) {
return Error::new(DiskError::FaultyDisk);
}
Error::new(e)
}
pub fn convert_access_error(e: io::Error, per_err: DiskError) -> Error {
if os_is_not_exist(&e) {
return Error::new(DiskError::VolumeNotEmpty);
} else if is_sys_err_io(&e) {
return Error::new(DiskError::FaultyDisk);
} else if os_is_permission(&e) {
return Error::new(per_err);
}
Error::new(e)
}
ecstore/src/disk/format.rs
+49
View File
@@ -182,6 +182,55 @@ impl FormatV3 {
Err(Error::msg(format!("disk id not found {}", disk_id)))
}
pub fn check_other(&self, other: &FormatV3) -> Result<()> {
let mut tmp = other.clone();
let this = tmp.erasure.this;
tmp.erasure.this = Uuid::nil();
if self.erasure.sets.len() != other.erasure.sets.len() {
return Err(Error::from_string(format!(
"Expected number of sets {}, got {}",
self.erasure.sets.len(),
other.erasure.sets.len()
)));
}
for i in 0..self.erasure.sets.len() {
if self.erasure.sets[i].len() != other.erasure.sets[i].len() {
return Err(Error::from_string(format!(
"Each set should be of same size, expected {}, got {}",
self.erasure.sets[i].len(),
other.erasure.sets[i].len()
)));
}
for j in 0..self.erasure.sets[i].len() {
if self.erasure.sets[i][j] != self.erasure.sets[i][j] {
return Err(Error::from_string(format!(
"UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)",
i,
j,
self.erasure.sets[i][j].to_string(),
other.erasure.sets[i][j].to_string(),
)));
}
}
}
for i in 0..tmp.erasure.sets.len() {
for j in 0..tmp.erasure.sets[i].len() {
if this == tmp.erasure.sets[i][j] {
return Ok(());
}
}
}
Err(Error::msg(format!(
"DriveID {:?} not found in any drive sets {:?}",
this, other.erasure.sets
)))
}
}
#[cfg(test)]
ecstore/src/disk/local.rs
+952 -346
View File
File diff suppressed because it is too large Load Diff
ecstore/src/disk/mod.rs
+87 -32
View File
@@ -2,6 +2,7 @@ pub mod endpoint;
pub mod error;
pub mod format;
mod local;
pub mod os;
pub mod remote;
pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
@@ -18,7 +19,8 @@ use crate::{
file_meta::FileMeta,
store_api::{FileInfo, RawFileInfo},
};
use bytes::Bytes;
use endpoint::Endpoint;
use futures::StreamExt;
use protos::proto_gen::node_service::{
node_service_client::NodeServiceClient, ReadAtRequest, ReadAtResponse, WriteRequest, WriteResponse,
@@ -50,39 +52,51 @@ pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result<DiskS
#[async_trait::async_trait]
pub trait DiskAPI: Debug + Send + Sync + 'static {
fn to_string(&self) -> String;
async fn is_online(&self) -> bool;
fn is_local(&self) -> bool;
fn path(&self) -> PathBuf;
// LastConn
fn host_name(&self) -> String;
fn endpoint(&self) -> Endpoint;
async fn close(&self) -> Result<()>;
async fn get_disk_id(&self) -> Option<Uuid>;
async fn get_disk_id(&self) -> Result<Option<Uuid>>;
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes>;
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()>;
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>;
async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<FileWriter>;
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter>;
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader>;
// 读目录下的所有文件、目录
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>>;
fn path(&self) -> PathBuf;
fn get_disk_location(&self) -> DiskLocation;
// Healing
// DiskInfo
// NSScanner
// Volume operations.
async fn make_volume(&self, volume: &str) -> Result<()>;
async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>;
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>>;
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo>;
async fn delete_volume(&self, volume: &str) -> Result<()>;
// 并发边读边写 TODO: wr io.Writer
async fn walk_dir(&self, opts: WalkDirOptions) -> Result<Vec<MetaCacheEntry>>;
async fn rename_data(
// Metadata operations
async fn delete_version(
&self,
src_volume: &str,
src_path: &str,
file_info: FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp>;
async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>;
async fn delete_volume(&self, volume: &str) -> Result<()>;
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>>;
async fn make_volume(&self, volume: &str) -> Result<()>;
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo>;
volume: &str,
path: &str,
fi: FileInfo,
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<RawFileInfo>;
async fn delete_versions(
&self,
volume: &str,
versions: Vec<FileInfoVersions>,
opts: DeleteOptions,
) -> Result<Vec<Option<Error>>>;
async fn delete_paths(&self, volume: &str, paths: &[&str]) -> Result<()>;
async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>;
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts) -> Result<()>;
async fn read_version(
&self,
org_volume: &str,
@@ -92,13 +106,50 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
opts: &ReadOptions,
) -> Result<FileInfo>;
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo>;
async fn delete_versions(
async fn rename_data(
&self,
volume: &str,
versions: Vec<FileInfoVersions>,
opts: DeleteOptions,
) -> Result<Vec<Option<Error>>>;
src_volume: &str,
src_path: &str,
file_info: FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp>;
// File operations.
// 读目录下的所有文件、目录
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>>;
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader>;
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter>;
async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<FileWriter>;
// ReadFileStream
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>;
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()>;
// CheckParts
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
// VerifyFile
// StatInfoFile
// ReadParts
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>>;
// CleanAbandonedData
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()>;
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>>;
}
#[derive(Debug, Serialize, Deserialize)]
pub struct UpdateMetadataOpts {
pub no_persistence: bool,
}
pub struct DiskLocation {
pub pool_idx: Option<usize>,
pub set_idx: Option<usize>,
pub disk_idx: Option<usize>,
}
impl DiskLocation {
pub fn valid(&self) -> bool {
self.pool_idx.is_some() && self.set_idx.is_some() && self.disk_idx.is_some()
}
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
@@ -214,20 +265,24 @@ impl MetaCacheEntry {
}
}
#[derive(Debug, Default)]
pub struct DiskOption {
pub cleanup: bool,
pub health_check: bool,
}
#[derive(Serialize, Deserialize)]
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct RenameDataResp {
pub old_data_dir: Option<Uuid>,
pub sign: Option<Vec<u8>>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct DeleteOptions {
pub recursive: bool,
pub immediate: bool,
pub undo_write: bool,
pub old_data_dir: Option<Uuid>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
ecstore/src/disk/os.rs
+204
View File
@@ -0,0 +1,204 @@
use std::{
io,
path::{Component, Path},
};
use tokio::fs;
use crate::{
disk::error::{is_sys_err_not_dir, is_sys_err_path_not_found, os_is_not_exist},
error::{Error, Result},
utils,
};
use super::error::{os_err_to_file_err, os_is_exist, DiskError};
pub fn check_path_length(path_name: &str) -> Result<()> {
// Apple OS X path length is limited to 1016
if cfg!(target_os = "macos") && path_name.len() > 1016 {
return Err(Error::new(DiskError::FileNameTooLong));
}
// Disallow more than 1024 characters on windows, there
// are no known name_max limits on Windows.
if cfg!(target_os = "windows") && path_name.len() > 1024 {
return Err(Error::new(DiskError::FileNameTooLong));
}
// On Unix we reject paths if they are just '.', '..' or '/'
let invalid_paths = [".", "..", "/"];
if invalid_paths.contains(&path_name) {
return Err(Error::new(DiskError::FileAccessDenied));
}
// Check each path segment length is > 255 on all Unix
// platforms, look for this value as NAME_MAX in
// /usr/include/linux/limits.h
let mut count = 0usize;
for c in path_name.chars() {
match c {
'/' | '\\' if cfg!(target_os = "windows") => count = 0, // Reset
_ => {
count += 1;
if count > 255 {
return Err(Error::new(DiskError::FileNameTooLong));
}
}
}
}
// Success.
Ok(())
}
pub async fn make_dir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> Result<()> {
check_path_length(path.as_ref().to_string_lossy().to_string().as_str())?;
if let Err(e) = reliable_mkdir_all(path.as_ref(), base_dir.as_ref()).await {
if is_sys_err_not_dir(&e) {
return Err(Error::new(DiskError::FileAccessDenied));
} else if is_sys_err_path_not_found(&e) {
return Err(Error::new(DiskError::FileAccessDenied));
}
return Err(os_err_to_file_err(e));
}
Ok(())
}
// read_dir count read limit. when count == 0 unlimit.
pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> Result<Vec<String>> {
let mut entries = fs::read_dir(path.as_ref()).await?;
let mut volumes = Vec::new();
let mut count = count;
while let Some(entry) = entries.next_entry().await? {
let name = entry.file_name().to_string_lossy().to_string();
if name == "" || name == "." || name == ".." {
continue;
}
let file_type = entry.file_type().await?;
if file_type.is_dir() {
count -= 1;
volumes.push(format!("{}{}", name, utils::path::SLASH_SEPARATOR));
if count == 0 {
break;
}
}
}
Ok(volumes)
}
pub async fn rename_all(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
base_dir: impl AsRef<Path>,
) -> Result<()> {
reliable_rename(src_file_path, dst_file_path, base_dir).await.map_err(|e| {
if is_sys_err_not_dir(&e) || !os_is_not_exist(&e) {
Error::new(DiskError::FileAccessDenied)
} else if is_sys_err_path_not_found(&e) {
Error::new(DiskError::FileAccessDenied)
} else if os_is_not_exist(&e) {
Error::new(DiskError::FileNotFound)
} else if os_is_exist(&e) {
Error::new(DiskError::IsNotRegular)
} else {
Error::new(e)
}
})?;
Ok(())
}
pub async fn reliable_rename(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
base_dir: impl AsRef<Path>,
) -> io::Result<()> {
if let Some(parent) = dst_file_path.as_ref().parent() {
reliable_mkdir_all(parent, base_dir.as_ref()).await?;
}
let mut i = 0;
loop {
if let Err(e) = utils::fs::rename(src_file_path.as_ref(), dst_file_path.as_ref()).await {
if os_is_not_exist(&e) && i == 0 {
i += 1;
continue;
}
return Err(e);
}
break;
}
Ok(())
}
pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
let mut i = 0;
let mut base_dir = base_dir.as_ref();
loop {
if let Err(e) = os_mkdir_all(path.as_ref(), base_dir).await {
if os_is_not_exist(&e) && i == 0 {
i += 1;
if let Some(base_parent) = base_dir.parent() {
if let Some(c) = base_parent.components().next() {
if c != Component::RootDir {
base_dir = base_parent
}
}
}
continue;
}
return Err(e);
}
break;
}
Ok(())
}
pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
if !base_dir.as_ref().to_string_lossy().is_empty() {
if base_dir.as_ref().starts_with(dir_path.as_ref()) {
return Ok(());
}
}
if let Some(parent) = dir_path.as_ref().parent() {
// 不支持递归，直接create_dir_all了
if let Err(e) = utils::fs::make_dir_all(&parent).await {
if os_is_exist(&e) {
return Ok(());
}
return Err(e);
}
// Box::pin(os_mkdir_all(&parent, &base_dir)).await?;
}
if let Err(e) = utils::fs::mkdir(dir_path.as_ref()).await {
if os_is_exist(&e) {
return Ok(());
}
return Err(e);
}
Ok(())
}
ecstore/src/disk/remote.rs
+141 -14
View File
@@ -1,30 +1,30 @@
use std::path::PathBuf;
use bytes::Bytes;
use futures::lock::Mutex;
use protos::{
node_service_time_out_client,
proto_gen::node_service::{
DeleteRequest, DeleteVersionsRequest, DeleteVolumeRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest,
MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
RenameFileRequst, StatVolumeRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest,
DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, ListDirRequest,
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, ReadVersionRequest,
ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest, UpdateMetadataRequest, WalkDirRequest,
WriteAllRequest, WriteMetadataRequest,
},
};
use tonic::Request;
use tracing::info;
use uuid::Uuid;
use super::{
endpoint::Endpoint, DeleteOptions, DiskAPI, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter,
MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp,
UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
};
use crate::{
disk::error::DiskError,
error::{Error, Result},
store_api::{FileInfo, RawFileInfo},
};
use super::{
endpoint::Endpoint, DeleteOptions, DiskAPI, DiskOption, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp, VolumeInfo,
WalkDirOptions,
};
use protos::proto_gen::node_service::RenamePartRequst;
#[derive(Debug)]
pub struct RemoteDisk {
@@ -32,6 +32,7 @@ pub struct RemoteDisk {
pub addr: String,
pub url: url::Url,
pub root: PathBuf,
endpoint: Endpoint,
}
impl RemoteDisk {
@@ -44,6 +45,7 @@ impl RemoteDisk {
addr,
url: ep.url.clone(),
root,
endpoint: ep.clone(),
})
}
}
@@ -51,9 +53,27 @@ impl RemoteDisk {
// TODO: all api need to handle errors
#[async_trait::async_trait]
impl DiskAPI for RemoteDisk {
fn to_string(&self) -> String {
self.endpoint.to_string()
}
fn is_local(&self) -> bool {
false
}
fn host_name(&self) -> String {
self.endpoint.host_port()
}
async fn is_online(&self) -> bool {
// TODO: 连接状态
if let Ok(_) = node_service_time_out_client(&self.addr).await {
return true;
}
false
}
fn endpoint(&self) -> Endpoint {
self.endpoint.clone()
}
async fn close(&self) -> Result<()> {
Ok(())
}
@@ -61,8 +81,16 @@ impl DiskAPI for RemoteDisk {
self.root.clone()
}
async fn get_disk_id(&self) -> Option<Uuid> {
self.id.lock().await.clone()
fn get_disk_location(&self) -> DiskLocation {
DiskLocation {
pool_idx: self.endpoint.pool_idx,
set_idx: self.endpoint.set_idx,
disk_idx: self.endpoint.pool_idx,
}
}
async fn get_disk_id(&self) -> Result<Option<Uuid>> {
Ok(self.id.lock().await.clone())
}
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
let mut lock = self.id.lock().await;
@@ -71,7 +99,7 @@ impl DiskAPI for RemoteDisk {
Ok(())
}
async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes> {
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
info!("read_all");
let mut client = node_service_time_out_client(&self.addr)
.await
@@ -90,7 +118,7 @@ impl DiskAPI for RemoteDisk {
return Err(DiskError::FileNotFound.into());
}
Ok(Bytes::from(response.data))
Ok(response.data)
}
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
@@ -135,7 +163,28 @@ impl DiskAPI for RemoteDisk {
Ok(())
}
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()> {
info!("rename_part");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::from_string(format!("can not get client, err: {}", err.to_string())))?;
let request = Request::new(RenamePartRequst {
disk: self.root.to_string_lossy().to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
meta,
});
let response = client.rename_part(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
Ok(())
}
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
info!("rename_file");
let mut client = node_service_time_out_client(&self.addr)
@@ -366,6 +415,51 @@ impl DiskAPI for RemoteDisk {
Ok(volume_info)
}
async fn delete_paths(&self, volume: &str, paths: &[&str]) -> Result<()> {
info!("delete_paths");
let paths = paths.iter().map(|s| s.to_string()).collect::<Vec<String>>();
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::from_string(format!("can not get client, err: {}", err.to_string())))?;
let request = Request::new(DeletePathsRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
paths,
});
let response = client.delete_paths(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
Ok(())
}
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts) -> Result<()> {
info!("update_metadata");
let file_info = serde_json::to_string(&fi)?;
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::from_string(format!("can not get client, err: {}", err.to_string())))?;
let request = Request::new(UpdateMetadataRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
opts,
});
let response = client.update_metadata(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
Ok(())
}
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
info!("write_metadata");
let file_info = serde_json::to_string(&fi)?;
@@ -442,7 +536,40 @@ impl DiskAPI for RemoteDisk {
Ok(raw_file_info)
}
async fn delete_version(
&self,
volume: &str,
path: &str,
fi: FileInfo,
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<RawFileInfo> {
info!("delete_version");
let file_info = serde_json::to_string(&fi)?;
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::from_string(format!("can not get client, err: {}", err.to_string())))?;
let request = Request::new(DeleteVersionRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
force_del_marker,
opts,
});
let response = client.delete_version(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
Ok(raw_file_info)
}
async fn delete_versions(
&self,
volume: &str,
ecstore/src/erasure.rs
+30 -24
View File
@@ -1,4 +1,5 @@
use crate::error::{Error, Result, StdError};
use crate::quorum::{object_op_ignored_errs, reduce_write_quorum_errs};
use bytes::Bytes;
use futures::future::join_all;
use futures::{Stream, StreamExt};
@@ -25,7 +26,7 @@ pub struct Erasure {
impl Erasure {
pub fn new(data_shards: usize, parity_shards: usize, block_size: usize) -> Self {
warn!(
debug!(
"Erasure new data_shards {},parity_shards {} block_size {} ",
data_shards, parity_shards, block_size
);
@@ -46,17 +47,17 @@ impl Erasure {
pub async fn encode<S>(
&self,
body: S,
writers: &mut [FileWriter],
writers: &mut [Option<FileWriter>],
// block_size: usize,
total_size: usize,
_write_quorum: usize,
write_quorum: usize,
) -> Result<usize>
where
S: Stream<Item = Result<Bytes, StdError>> + Send + Sync + 'static,
{
let mut stream = ChunkedStream::new(body, total_size, self.block_size, false);
let mut total: usize = 0;
let mut idx = 0;
// let mut idx = 0;
while let Some(result) = stream.next().await {
match result {
Ok(data) => {
@@ -67,36 +68,41 @@ impl Erasure {
break;
}
idx += 1;
debug!("encode {} get data {}", idx, data.len());
// idx += 1;
// debug!("encode {} get data {}", idx, data.len());
let blocks = self.encode_data(data.as_ref())?;
debug!(
"encode shard {} size: {}/{} from block_size {}, total_size {} ",
idx,
blocks[0].len(),
blocks.len(),
data.len(),
total_size
);
// debug!(
// "encode shard {} size: {}/{} from block_size {}, total_size {} ",
// idx,
// blocks[0].len(),
// blocks.len(),
// data.len(),
// total_size
// );
let mut errs = Vec::new();
for (i, w) in writers.iter_mut().enumerate() {
match w.write(blocks[i].as_ref()).await {
if w.is_none() {
continue;
}
match w.as_mut().unwrap().write(blocks[i].as_ref()).await {
Ok(_) => errs.push(None),
Err(e) => errs.push(Some(e)),
}
}
// debug!("{} encode_data write errs:{:?}", self.id, errs);
// // TODO: reduceWriteQuorumErrs
// for err in errs.iter() {
// if err.is_some() {
// return Err(Error::msg("message"));
// }
// }
let none_count = errs.iter().filter(|&x| x.is_none()).count();
if none_count >= write_quorum {
continue;
}
if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) {
warn!("Erasure encode errs {:?}", &errs);
return Err(err);
}
}
Err(e) => return Err(Error::from_std_error(e)),
}
@@ -154,7 +160,7 @@ impl Erasure {
break;
}
debug!("decode {} block_offset {},block_length {} ", block_idx, block_offset, block_length);
// debug!("decode {} block_offset {},block_length {} ", block_idx, block_offset, block_length);
let mut bufs = reader.read().await?;
@@ -168,7 +174,7 @@ impl Erasure {
bytes_writed += writed_n;
debug!("decode {} writed_n {}, total_writed: {} ", block_idx, writed_n, bytes_writed);
// debug!("decode {} writed_n {}, total_writed: {} ", block_idx, writed_n, bytes_writed);
}
if bytes_writed != length {
ecstore/src/error.rs
+29
View File
@@ -1,5 +1,9 @@
use std::io;
use tracing_error::{SpanTrace, SpanTraceStatus};
use crate::disk::error::{clone_disk_err, DiskError};
pub type StdError = Box<dyn std::error::Error + Send + Sync + 'static>;
pub type Result<T = (), E = Error> = std::result::Result<T, E>;
@@ -61,6 +65,14 @@ impl Error {
pub fn downcast_mut<T: std::error::Error + 'static>(&mut self) -> Option<&mut T> {
self.inner.downcast_mut()
}
pub fn to_io_err(&self) -> Option<io::Error> {
if let Some(e) = self.downcast_ref::<io::Error>() {
Some(io::Error::new(e.kind(), e.to_string()))
} else {
None
}
}
}
impl<T: std::error::Error + Send + Sync + 'static> From<T> for Error {
@@ -80,3 +92,20 @@ impl std::fmt::Display for Error {
Ok(())
}
}
impl Clone for Error {
fn clone(&self) -> Self {
if let Some(e) = self.downcast_ref::<DiskError>() {
clone_disk_err(e)
} else if let Some(e) = self.downcast_ref::<io::Error>() {
if let Some(code) = e.raw_os_error() {
Error::new(io::Error::from_raw_os_error(code))
} else {
Error::new(io::Error::new(e.kind(), e.to_string()))
}
} else {
// TODO: 优化其他类型
Error::msg(self.to_string())
}
}
}
ecstore/src/lib.rs
+3 -1
View File
@@ -1,4 +1,4 @@
mod bucket_meta;
pub mod bucket_meta;
mod chunk_stream;
pub mod disk;
pub mod disks_layout;
@@ -7,8 +7,10 @@ pub mod erasure;
pub mod error;
mod file_meta;
pub mod peer;
mod quorum;
pub mod set_disk;
mod sets;
mod storage_class;
pub mod store;
pub mod store_api;
mod store_init;
ecstore/src/peer.rs
+30 -8
View File
@@ -12,7 +12,7 @@ use crate::{
disk::{self, error::DiskError, VolumeInfo},
endpoints::{EndpointServerPools, Node},
error::{Error, Result},
store_api::{BucketInfo, BucketOptions, MakeBucketOptions},
store_api::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
};
type Client = Arc<Box<dyn PeerS3Client>>;
@@ -21,7 +21,7 @@ type Client = Arc<Box<dyn PeerS3Client>>;
pub trait PeerS3Client: Debug + Sync + Send + 'static {
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()>;
async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>>;
async fn delete_bucket(&self, bucket: &str) -> Result<()>;
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()>;
async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo>;
fn get_pools(&self) -> Option<Vec<usize>>;
}
@@ -142,10 +142,10 @@ impl S3PeerSys {
Ok(buckets)
}
pub async fn delete_bucket(&self, bucket: &str) -> Result<()> {
pub async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
let mut futures = Vec::with_capacity(self.clients.len());
for cli in self.clients.iter() {
futures.push(cli.delete_bucket(bucket));
futures.push(cli.delete_bucket(bucket, &opts));
}
let mut errors = Vec::with_capacity(self.clients.len());
@@ -258,6 +258,7 @@ impl PeerS3Client for LocalPeerS3Client {
}
}
warn!("list_bucket ress {:?}", &ress);
warn!("list_bucket errs {:?}", &errs);
let mut uniq_map: HashMap<&String, &VolumeInfo> = HashMap::new();
@@ -356,7 +357,7 @@ impl PeerS3Client for LocalPeerS3Client {
.ok_or(Error::new(DiskError::VolumeNotFound))
}
async fn delete_bucket(&self, bucket: &str) -> Result<()> {
async fn delete_bucket(&self, bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> {
let local_disks = all_local_disk().await;
let mut futures = Vec::with_capacity(local_disks.len());
@@ -368,14 +369,34 @@ impl PeerS3Client for LocalPeerS3Client {
let mut errs = Vec::new();
let mut recreate = false;
for res in results {
match res {
Ok(_) => errs.push(None),
Err(e) => errs.push(Some(e)),
Err(e) => {
if DiskError::VolumeNotEmpty.is(&e) {
recreate = true;
}
errs.push(Some(e))
}
}
}
// TODO: errVolumeNotEmpty 不删除，把已经删除的重新创建
// errVolumeNotEmpty 不删除，把已经删除的重新创建
let mut idx = 0;
for err in errs {
if err.is_none() && recreate {
let _ = local_disks[idx].make_volume(bucket).await;
}
idx += 1;
}
if recreate {
return Err(Error::new(DiskError::VolumeNotEmpty));
}
// TODO: reduceWriteQuorumErrs
@@ -450,10 +471,11 @@ impl PeerS3Client for RemotePeerS3Client {
Ok(bucket_info)
}
async fn delete_bucket(&self, bucket: &str) -> Result<()> {
async fn delete_bucket(&self, bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> {
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::from_string(format!("can not get client, err: {}", err.to_string())))?;
let request = Request::new(DeleteBucketRequest {
bucket: bucket.to_string(),
});
ecstore/src/quorum.rs
+127
View File
@@ -0,0 +1,127 @@
use crate::{disk::error::DiskError, error::Error};
use std::{collections::HashMap, fmt::Debug};
// pub type CheckErrorFn = fn(e: &Error) -> bool;
pub trait CheckErrorFn: Debug + Send + Sync + 'static {
fn is(&self, e: &Error) -> bool;
}
#[derive(Debug, thiserror::Error)]
pub enum QuorumError {
#[error("Read quorum not met")]
Read,
#[error("disk not found")]
Write,
}
pub fn base_ignored_errs() -> Vec<Box<dyn CheckErrorFn>> {
vec![
Box::new(DiskError::DiskNotFound),
Box::new(DiskError::FaultyDisk),
Box::new(DiskError::FaultyRemoteDisk),
]
}
// object_op_ignored_errs
pub fn object_op_ignored_errs() -> Vec<Box<dyn CheckErrorFn>> {
let mut base = base_ignored_errs();
let ext: Vec<Box<dyn CheckErrorFn>> = vec![
// Box::new(DiskError::DiskNotFound),
// Box::new(DiskError::FaultyDisk),
// Box::new(DiskError::FaultyRemoteDisk),
Box::new(DiskError::DiskAccessDenied),
Box::new(DiskError::UnformattedDisk),
Box::new(DiskError::DiskOngoingReq),
];
base.extend(ext);
base
}
// 用于检查错误是否被忽略的函数
fn is_err_ignored(err: &Error, ignored_errs: &Vec<Box<dyn CheckErrorFn>>) -> bool {
ignored_errs.iter().any(|ignored_err| ignored_err.is(err))
}
// 减少错误数量并返回出现次数最多的错误
fn reduce_errs(errs: &Vec<Option<Error>>, ignored_errs: &Vec<Box<dyn CheckErrorFn>>) -> (usize, Option<Error>) {
let mut error_counts: HashMap<String, usize> = HashMap::new();
let mut error_map: HashMap<String, usize> = HashMap::new(); // 存err位置
let nil = "nil".to_string();
for (i, operr) in errs.iter().enumerate() {
if operr.is_none() {
*error_counts.entry(nil.clone()).or_insert(0) += 1;
let _ = *error_map.entry(nil.clone()).or_insert(i);
continue;
}
let err = operr.as_ref().unwrap();
if is_err_ignored(err, &ignored_errs) {
continue;
}
let errstr = err.to_string();
let _ = *error_map.entry(errstr.clone()).or_insert(i);
*error_counts.entry(errstr.clone()).or_insert(0) += 1;
}
let mut max = 0;
let mut max_err = nil.clone();
for (&ref err, &count) in error_counts.iter() {
if count > max || (count == max && *err == nil) {
max = count;
max_err = err.clone();
}
}
if let Some(&c) = error_counts.get(&max_err) {
if let Some(&err_idx) = error_map.get(&max_err) {
let err = errs[err_idx].clone();
return (c, err);
}
return (c, None);
}
(0, None)
}
// 根据quorum验证错误数量
fn reduce_quorum_errs(
errs: &Vec<Option<Error>>,
ignored_errs: &Vec<Box<dyn CheckErrorFn>>,
quorum: usize,
quorum_err: QuorumError,
) -> Option<Error> {
let (max_count, max_err) = reduce_errs(errs, ignored_errs);
if max_count >= quorum {
max_err
} else {
Some(Error::new(quorum_err))
}
}
// 根据读quorum验证错误数量
// 返回最大错误数量的下标，或QuorumError
pub fn reduce_read_quorum_errs(
errs: &Vec<Option<Error>>,
ignored_errs: &Vec<Box<dyn CheckErrorFn>>,
read_quorum: usize,
) -> Option<Error> {
reduce_quorum_errs(errs, ignored_errs, read_quorum, QuorumError::Read)
}
// 根据写quorum验证错误数量
// 返回最大错误数量的下标，或QuorumError
pub fn reduce_write_quorum_errs(
errs: &Vec<Option<Error>>,
ignored_errs: &Vec<Box<dyn CheckErrorFn>>,
write_quorum: usize,
) -> Option<Error> {
reduce_quorum_errs(errs, ignored_errs, write_quorum, QuorumError::Write)
}
ecstore/src/sets.rs
+120 -31
View File
@@ -1,3 +1,4 @@
#![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use common::globals::GLOBAL_Local_Node_Name;
@@ -17,19 +18,25 @@ use crate::{
set_disk::SetDisks,
store::{GLOBAL_IsDistErasure, GLOBAL_LOCAL_DISK_SET_DRIVES},
store_api::{
BucketInfo, BucketOptions, CompletePart, DeletedObject, GetObjectReader, HTTPRangeSpec, ListObjectsV2Info,
MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI,
BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec,
ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo,
PutObjReader, StorageAPI,
},
utils::hash,
};
#[derive(Debug)]
use tokio::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::info;
use tracing::warn;
#[derive(Debug, Clone)]
pub struct Sets {
pub id: Uuid,
// pub sets: Vec<Objects>,
// pub disk_set: Vec<Vec<Option<DiskStore>>>, // [set_count_idx][set_drive_count_idx] = disk_idx
pub disk_set: Vec<SetDisks>, // [set_count_idx][set_drive_count_idx] = disk_idx
pub lockers: Vec<Vec<LockApi>>,
pub disk_set: Vec<Arc<SetDisks>>, // [set_count_idx][set_drive_count_idx] = disk_idx
pub pool_idx: usize,
pub endpoints: PoolEndpoints,
pub format: FormatV3,
@@ -37,6 +44,7 @@ pub struct Sets {
pub set_count: usize,
pub set_drive_count: usize,
pub distribution_algo: DistributionAlgoVersion,
ctx: CancellationToken,
}
impl Sets {
@@ -46,7 +54,7 @@ impl Sets {
fm: &FormatV3,
pool_idx: usize,
partiy_count: usize,
) -> Result<Self> {
) -> Result<Arc<Self>> {
let set_count = fm.erasure.sets.len();
let set_drive_count = fm.erasure.sets[0].len();
@@ -72,10 +80,16 @@ impl Sets {
for i in 0..set_count {
let mut set_drive = Vec::with_capacity(set_drive_count);
let mut set_endpoints = Vec::with_capacity(set_drive_count);
for j in 0..set_drive_count {
let idx = i * set_drive_count + j;
let mut disk = disks[idx].clone();
let endpoint = endpoints.endpoints.as_ref().get(idx).cloned();
set_endpoints.push(endpoint);
if disk.is_none() {
warn!("sets new set_drive {}-{} is none", i, j);
set_drive.push(None);
continue;
}
@@ -87,6 +101,7 @@ impl Sets {
};
if local_disk.is_none() {
warn!("sets new set_drive {}-{} local_disk is none", i, j);
set_drive.push(None);
continue;
}
@@ -96,28 +111,33 @@ impl Sets {
disk = local_disk;
}
if let Some(_disk_id) = disk.as_ref().unwrap().get_disk_id().await {
if let Some(_disk_id) = disk.as_ref().unwrap().get_disk_id().await? {
set_drive.push(disk);
} else {
warn!("sets new set_drive {}-{} get_disk_id is none", i, j);
set_drive.push(None);
}
}
// warn!("sets new set_drive {:?}", &set_drive);
let set_disks = SetDisks {
disks: set_drive,
lockers: lockers[i].clone(),
locker_owner: GLOBAL_Local_Node_Name.read().await.to_string(),
ns_mutex: Arc::new(RwLock::new(NsLockMap::new(*GLOBAL_IsDistErasure.read().await))),
disks: RwLock::new(set_drive),
set_drive_count,
parity_count: partiy_count,
default_parity_count: partiy_count,
set_index: i,
pool_index: pool_idx,
set_endpoints,
format: fm.clone(),
};
disk_set.push(set_disks);
disk_set.push(Arc::new(set_disks));
}
let sets = Self {
let sets = Arc::new(Self {
id: fm.id,
// sets: todo!(),
disk_set,
@@ -129,15 +149,58 @@ impl Sets {
set_count,
set_drive_count,
distribution_algo: fm.erasure.distribution_algo.clone(),
};
ctx: CancellationToken::new(),
});
let asets = sets.clone();
tokio::spawn(async move { asets.monitor_and_connect_endpoints().await });
Ok(sets)
}
pub fn get_disks(&self, set_idx: usize) -> SetDisks {
pub async fn monitor_and_connect_endpoints(&self) {
tokio::time::sleep(Duration::from_secs(5)).await;
info!("start monitor_and_connect_endpoints");
self.connect_disks().await;
// TODO: config interval
let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(15 * 3));
let cloned_token = self.ctx.clone();
loop {
tokio::select! {
_= interval.tick()=>{
// debug!("tick...");
self.connect_disks().await;
interval.reset();
},
_ = cloned_token.cancelled() => {
warn!("ctx cancelled");
break;
}
}
}
warn!("monitor_and_connect_endpoints exit");
}
async fn connect_disks(&self) {
// debug!("start connect_disks ...");
for set in self.disk_set.iter() {
set.connect_disks().await;
}
// debug!("done connect_disks ...");
}
pub fn get_disks(&self, set_idx: usize) -> Arc<SetDisks> {
self.disk_set[set_idx].clone()
}
pub fn get_disks_by_key(&self, key: &str) -> SetDisks {
pub fn get_disks_by_key(&self, key: &str) -> Arc<SetDisks> {
self.get_disks(self.get_hashed_set_index(key))
}
@@ -224,8 +287,7 @@ impl StorageAPI for Sets {
let mut set_obj_map = HashMap::new();
// hash key
let mut i = 0;
for obj in objects.iter() {
for (i, obj) in objects.iter().enumerate() {
let idx = self.get_hashed_set_index(obj.object_name.as_str());
if !set_obj_map.contains_key(&idx) {
@@ -237,34 +299,54 @@ impl StorageAPI for Sets {
obj: obj.clone(),
}],
);
} else {
if let Some(val) = set_obj_map.get_mut(&idx) {
val.push(DelObj {
// set_idx: idx,
orig_idx: i,
obj: obj.clone(),
});
}
} else if let Some(val) = set_obj_map.get_mut(&idx) {
val.push(DelObj {
// set_idx: idx,
orig_idx: i,
obj: obj.clone(),
});
}
i += 1;
}
// let semaphore = Arc::new(Semaphore::new(num_cpus::get()));
// let mut jhs = Vec::with_capacity(semaphore.available_permits());
// for (k, v) in set_obj_map {
// let disks = self.get_disks(k);
// let semaphore = semaphore.clone();
// let opts = opts.clone();
// let bucket = bucket.to_string();
// let jh = tokio::spawn(async move {
// let _permit = semaphore.acquire().await.unwrap();
// let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
// disks.delete_objects(&bucket, objs, opts).await
// });
// jhs.push(jh);
// }
// let mut results = Vec::with_capacity(jhs.len());
// for jh in jhs {
// results.push(jh.await?.unwrap());
// }
// for (dobjects, errs) in results {
// del_objects.extend(dobjects);
// del_errs.extend(errs);
// }
// TODO: 并发
for (k, v) in set_obj_map {
let disks = self.get_disks(k);
let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
let (dobjects, errs) = disks.delete_objects(bucket, objs, opts.clone()).await?;
let mut i = 0;
for err in errs {
for (i, err) in errs.into_iter().enumerate() {
let obj = v.get(i).unwrap();
del_errs[obj.orig_idx] = err;
del_objects[obj.orig_idx] = dobjects.get(i).unwrap().clone();
i += 1;
}
}
@@ -294,6 +376,13 @@ impl StorageAPI for Sets {
async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object).get_object_info(bucket, object, opts).await
}
async fn put_object_info(&self, bucket: &str, object: &str, info: ObjectInfo, opts: &ObjectOptions) -> Result<()> {
self.get_disks_by_key(object)
.put_object_info(bucket, object, info, opts)
.await
}
async fn get_object_reader(
&self,
bucket: &str,
@@ -306,7 +395,7 @@ impl StorageAPI for Sets {
.get_object_reader(bucket, object, range, h, opts)
.await
}
async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result<()> {
async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object).put_object(bucket, object, data, opts).await
}
@@ -345,7 +434,7 @@ impl StorageAPI for Sets {
.await
}
async fn delete_bucket(&self, _bucket: &str) -> Result<()> {
async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> {
unimplemented!()
}
}
ecstore/src/storage_class.rs
+28
View File
@@ -0,0 +1,28 @@
// use crate::error::{Error, Result};
// default_partiy_count 默认配置，根据磁盘总数分配校验磁盘数量
pub fn default_partiy_count(drive: usize) -> usize {
match drive {
1 => 0,
2 | 3 => 1,
4 | 5 => 2,
6 | 7 => 3,
_ => 4,
}
}
// Define the minimum number of parity drives required.
// const MIN_PARITY_DRIVES: usize = 0;
// // ValidateParity validates standard storage class parity.
// pub fn validate_parity(ss_parity: usize, set_drive_count: usize) -> Result<()> {
// // if ss_parity > 0 && ss_parity < MIN_PARITY_DRIVES {
// // return Err(Error::msg(format!("parity {} 应该大于等于 {}", ss_parity, MIN_PARITY_DRIVES)));
// // }
// if ss_parity > set_drive_count / 2 {
// return Err(Error::msg(format!("parity {} 应该小于等于 {}", ss_parity, set_drive_count / 2)));
// }
// Ok(())
// }
ecstore/src/store.rs
+173 -159
View File
@@ -1,17 +1,17 @@
#![allow(clippy::map_entry)]
use crate::disk::endpoint::EndpointType;
use crate::{
bucket_meta::BucketMetadata,
disk::{
endpoint::EndpointType, error::DiskError, new_disk, DeleteOptions, DiskOption, DiskStore, WalkDirOptions,
BUCKET_META_PREFIX, RUSTFS_META_BUCKET,
},
disk::{error::DiskError, new_disk, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
endpoints::{EndpointServerPools, SetupType},
error::{Error, Result},
peer::S3PeerSys,
sets::Sets,
storage_class::default_partiy_count,
store_api::{
BucketInfo, BucketOptions, CompletePart, DeletedObject, GetObjectReader, HTTPRangeSpec, ListObjectsInfo,
ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo,
PutObjReader, StorageAPI,
BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec,
ListObjectsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete,
PartInfo, PutObjReader, StorageAPI,
},
store_init, utils,
};
@@ -26,8 +26,9 @@ use std::{
time::Duration,
};
use time::OffsetDateTime;
use tokio::sync::Semaphore;
use tokio::{fs, sync::RwLock};
use tracing::{debug, info, warn};
use tracing::{debug, info};
use uuid::Uuid;
use lazy_static::lazy_static;
@@ -53,10 +54,11 @@ pub async fn update_erasure_type(setup_type: SetupType) {
*is_erasure_sd = setup_type == SetupType::ErasureSD;
}
type TypeLocalDiskSetDrives = Vec<Vec<Vec<Option<DiskStore>>>>;
lazy_static! {
pub static ref GLOBAL_LOCAL_DISK_MAP: Arc<RwLock<HashMap<String, Option<DiskStore>>>> = Arc::new(RwLock::new(HashMap::new()));
pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<Vec<Vec<Vec<Option<DiskStore>>>>>> =
Arc::new(RwLock::new(Vec::new()));
pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<TypeLocalDiskSetDrives>> = Arc::new(RwLock::new(Vec::new()));
}
pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
@@ -78,7 +80,7 @@ pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
pub async fn all_local_disk_path() -> Vec<String> {
let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await;
disk_map.keys().map(|v| v.clone()).collect()
disk_map.keys().cloned().collect()
}
pub async fn all_local_disk() -> Vec<DiskStore> {
@@ -152,12 +154,13 @@ pub struct ECStore {
pub id: uuid::Uuid,
// pub disks: Vec<DiskStore>,
pub disk_map: HashMap<usize, Vec<Option<DiskStore>>>,
pub pools: Vec<Sets>,
pub pools: Vec<Arc<Sets>>,
pub peer_sys: S3PeerSys,
// pub local_disks: Vec<DiskStore>,
}
impl ECStore {
#[allow(clippy::new_ret_no_self)]
pub async fn new(_address: String, endpoint_pools: EndpointServerPools) -> Result<()> {
// let layouts = DisksLayout::try_from(endpoints.as_slice())?;
@@ -179,11 +182,13 @@ impl ECStore {
)
.await;
info!("endpoint_pools: {:?}", endpoint_pools);
debug!("endpoint_pools: {:?}", endpoint_pools);
for (i, pool_eps) in endpoint_pools.as_ref().iter().enumerate() {
// TODO: read from config parseStorageClass
let partiy_count = store_init::default_partiy_count(pool_eps.drives_per_set);
let partiy_count = default_partiy_count(pool_eps.drives_per_set);
// validate_parity(partiy_count, pool_eps.drives_per_set)?;
let (disks, errs) = crate::store_init::init_disks(
&pool_eps.endpoints,
@@ -232,7 +237,6 @@ impl ECStore {
}
let sets = Sets::new(disks.clone(), pool_eps, &fm, i, partiy_count).await?;
pools.push(sets);
disk_map.insert(i, disks);
@@ -294,62 +298,54 @@ impl ECStore {
for sets in self.pools.iter() {
for set in sets.disk_set.iter() {
for disk in set.disks.iter() {
if disk.is_none() {
continue;
}
let disk = disk.as_ref().unwrap();
let opts = opts.clone();
// let mut wr = &mut wr;
futures.push(disk.walk_dir(opts));
// tokio::spawn(async move { disk.walk_dir(opts, wr).await });
}
futures.push(set.walk_dir(&opts));
}
}
let results = join_all(futures).await;
let mut errs = Vec::new();
// let mut errs = Vec::new();
let mut ress = Vec::new();
let mut uniq = HashSet::new();
for res in results {
match res {
Ok(entrys) => {
for entry in entrys {
if !uniq.contains(&entry.name) {
uniq.insert(entry.name.clone());
// TODO: 过滤
if opts.limit > 0 && ress.len() as i32 >= opts.limit {
return Ok(ress);
}
for (disks_ress, _disks_errs) in results {
for (_i, disks_res) in disks_ress.iter().enumerate() {
if disks_res.is_none() {
// TODO handle errs
continue;
}
let entrys = disks_res.as_ref().unwrap();
if entry.is_object() {
let fi = entry.to_fileinfo(&opts.bucket)?;
if fi.is_some() {
ress.push(fi.unwrap().into_object_info(&opts.bucket, &entry.name, false));
}
continue;
}
for entry in entrys {
if !uniq.contains(&entry.name) {
uniq.insert(entry.name.clone());
// TODO: 过滤
if opts.limit > 0 && ress.len() as i32 >= opts.limit {
return Ok(ress);
}
if entry.is_dir() {
ress.push(ObjectInfo {
is_dir: true,
bucket: opts.bucket.clone(),
name: entry.name,
..Default::default()
});
if entry.is_object() {
let fi = entry.to_fileinfo(&opts.bucket)?;
if let Some(f) = fi {
ress.push(f.to_object_info(&opts.bucket, &entry.name, false));
}
continue;
}
if entry.is_dir() {
ress.push(ObjectInfo {
is_dir: true,
bucket: opts.bucket.clone(),
name: entry.name.clone(),
..Default::default()
});
}
}
errs.push(None);
}
Err(e) => errs.push(Some(e)),
}
}
warn!("list_merged errs {:?}", errs);
// warn!("list_merged errs {:?}", errs);
Ok(ress)
}
@@ -358,21 +354,23 @@ impl ECStore {
let mut futures = Vec::new();
for sets in self.pools.iter() {
for set in sets.disk_set.iter() {
for disk in set.disks.iter() {
if disk.is_none() {
continue;
}
let disk = disk.as_ref().unwrap();
futures.push(disk.delete(
bucket,
prefix,
DeleteOptions {
recursive: true,
immediate: false,
},
));
}
futures.push(set.delete_all(bucket, prefix));
// let disks = set.disks.read().await;
// let dd = disks.clone();
// for disk in dd {
// if disk.is_none() {
// continue;
// }
// // let disk = disk.as_ref().unwrap().clone();
// // futures.push(disk.delete(
// // bucket,
// // prefix,
// // DeleteOptions {
// // recursive: true,
// // immediate: false,
// // },
// // ));
// }
}
}
let results = join_all(futures).await;
@@ -400,62 +398,68 @@ impl ECStore {
object: &str,
opts: &ObjectOptions,
) -> Result<(PoolObjInfo, Vec<Error>)> {
let mut futures = Vec::new();
for pool in self.pools.iter() {
futures.push(pool.get_object_info(bucket, object, opts));
}
let results = join_all(futures).await;
let mut ress = Vec::new();
let mut i = 0;
// join_all结果跟输入顺序一致
for res in results {
let index = i;
match res {
Ok(r) => {
ress.push(PoolObjInfo {
index,
object_info: r,
err: None,
});
}
Err(e) => {
ress.push(PoolObjInfo {
index,
err: Some(e),
..Default::default()
});
}
}
i += 1;
}
ress.sort_by(|a, b| {
let at = a.object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
let bt = b.object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
at.cmp(&bt)
});
for res in ress {
// check
if res.err.is_none() {
// TODO: let errs = self.poolsWithObject()
return Ok((res, Vec::new()));
}
}
let ret = PoolObjInfo::default();
Ok((ret, Vec::new()))
internal_get_pool_info_existing_with_opts(&self.pools, bucket, object, opts).await
}
}
async fn internal_get_pool_info_existing_with_opts(
pools: &[Arc<Sets>],
bucket: &str,
object: &str,
opts: &ObjectOptions,
) -> Result<(PoolObjInfo, Vec<Error>)> {
let mut futures = Vec::new();
for pool in pools.iter() {
futures.push(pool.get_object_info(bucket, object, opts));
}
let results = join_all(futures).await;
let mut ress = Vec::new();
// join_all结果跟输入顺序一致
for (i, res) in results.into_iter().enumerate() {
let index = i;
match res {
Ok(r) => {
ress.push(PoolObjInfo {
index,
object_info: r,
err: None,
});
}
Err(e) => {
ress.push(PoolObjInfo {
index,
err: Some(e),
..Default::default()
});
}
}
}
ress.sort_by(|a, b| {
let at = a.object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
let bt = b.object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
at.cmp(&bt)
});
for res in ress {
// check
if res.err.is_none() {
// TODO: let errs = self.poolsWithObject()
return Ok((res, Vec::new()));
}
}
let ret = PoolObjInfo::default();
Ok((ret, Vec::new()))
}
#[derive(Debug, Default)]
pub struct PoolObjInfo {
pub index: usize,
@@ -497,6 +501,15 @@ impl StorageAPI for ECStore {
Ok(buckets)
}
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
self.peer_sys.delete_bucket(bucket, opts).await?;
// 删除meta
self.delete_all(RUSTFS_META_BUCKET, format!("{}/{}", BUCKET_META_PREFIX, bucket).as_str())
.await?;
Ok(())
}
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
// TODO: check valid bucket name
@@ -559,21 +572,34 @@ impl StorageAPI for ECStore {
del_errs.push(None)
}
// TODO: limte 限制并发数量
let opt = ObjectOptions::default();
// 取所有poolObjInfo
let mut futures = Vec::new();
for obj in objects.iter() {
futures.push(self.get_pool_info_existing_with_opts(bucket, &obj.object_name, &opt));
}
let mut jhs = Vec::new();
let semaphore = Arc::new(Semaphore::new(num_cpus::get()));
let pools = Arc::new(self.pools.clone());
let results = join_all(futures).await;
for obj in objects.iter() {
let (semaphore, pools, bucket, object_name, opt) = (
semaphore.clone(),
pools.clone(),
bucket.to_string(),
obj.object_name.to_string(),
ObjectOptions::default(),
);
let jh = tokio::spawn(async move {
let _permit = semaphore.acquire().await.unwrap();
internal_get_pool_info_existing_with_opts(pools.as_ref(), &bucket, &object_name, &opt).await
});
jhs.push(jh);
}
let mut results = Vec::new();
for jh in jhs {
results.push(jh.await.unwrap());
}
// 记录pool Index 对应的objects pool_idx -> objects idx
let mut pool_index_objects = HashMap::new();
let mut i = 0;
for res in results {
for (i, res) in results.into_iter().enumerate() {
match res {
Ok((pinfo, _)) => {
if pinfo.object_info.delete_marker && opts.version_id.is_empty() {
@@ -601,8 +627,6 @@ impl StorageAPI for ECStore {
del_errs[i] = Some(e)
}
}
i += 1;
}
if !pool_index_objects.is_empty() {
@@ -615,16 +639,7 @@ impl StorageAPI for ECStore {
let obj_idxs = vals.unwrap();
// 取对应obj,理论上不会none
let objs: Vec<ObjectToDelete> = obj_idxs
.iter()
.filter_map(|&idx| {
if let Some(obj) = objects.get(idx) {
Some(obj.clone())
} else {
None
}
})
.collect();
let objs: Vec<ObjectToDelete> = obj_idxs.iter().filter_map(|&idx| objects.get(idx).cloned()).collect();
if objs.is_empty() {
continue;
@@ -633,8 +648,7 @@ impl StorageAPI for ECStore {
let (pdel_objs, perrs) = sets.delete_objects(bucket, objs, opts.clone()).await?;
// perrs的顺序理论上跟obj_idxs顺序一致
let mut i = 0;
for err in perrs {
for (i, err) in perrs.into_iter().enumerate() {
let obj_idx = obj_idxs[i];
if err.is_some() {
@@ -645,8 +659,6 @@ impl StorageAPI for ECStore {
dobj.object_name = utils::path::decode_dir_object(&dobj.object_name);
del_objects[obj_idx] = dobj;
i += 1;
}
}
}
@@ -655,7 +667,7 @@ impl StorageAPI for ECStore {
}
async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
if opts.delete_prefix {
self.delete_prefix(bucket, &object).await?;
self.delete_prefix(bucket, object).await?;
return Ok(ObjectInfo::default());
}
@@ -717,6 +729,17 @@ impl StorageAPI for ECStore {
unimplemented!()
}
async fn put_object_info(&self, bucket: &str, object: &str, info: ObjectInfo, opts: &ObjectOptions) -> Result<()> {
let object = utils::path::encode_dir_object(object);
if self.single_pool() {
return self.pools[0].put_object_info(bucket, object.as_str(), info, opts).await;
}
unimplemented!()
}
async fn get_object_reader(
&self,
bucket: &str,
@@ -733,7 +756,7 @@ impl StorageAPI for ECStore {
unimplemented!()
}
async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result<()> {
async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
// checkPutObjectArgs
let object = utils::path::encode_dir_object(object);
@@ -790,15 +813,6 @@ impl StorageAPI for ECStore {
}
unimplemented!()
}
async fn delete_bucket(&self, bucket: &str) -> Result<()> {
self.peer_sys.delete_bucket(bucket).await?;
// 删除meta
self.delete_all(RUSTFS_META_BUCKET, format!("{}/{}", BUCKET_META_PREFIX, bucket).as_str())
.await?;
Ok(())
}
}
async fn init_local_peer(endpoint_pools: &EndpointServerPools, host: &String, port: &String) {
ecstore/src/store_api.rs
+26 -3
View File
@@ -1,3 +1,5 @@
use std::collections::HashMap;
use crate::error::{Error, Result};
use http::HeaderMap;
use rmp_serde::Serializer;
@@ -25,6 +27,10 @@ pub struct FileInfo {
pub fresh: bool, // indicates this is a first time call to write FileInfo.
pub parts: Vec<ObjectPartInfo>,
pub is_latest: bool,
// #[serde(skip_serializing_if = "Option::is_none", default)]
pub tags: Option<HashMap<String, String>>,
pub metadata: Option<HashMap<String, String>>,
pub num_versions: usize,
}
// impl Default for FileInfo {
@@ -92,6 +98,14 @@ impl FileInfo {
false
}
pub fn get_etag(&self) -> Option<String> {
if let Some(meta) = &self.metadata {
meta.get("etag").cloned()
} else {
None
}
}
pub fn write_quorum(&self, quorum: usize) -> usize {
if self.deleted {
return quorum;
@@ -137,7 +151,7 @@ impl FileInfo {
self.parts.sort_by(|a, b| a.number.cmp(&b.number));
}
pub fn into_object_info(&self, bucket: &str, object: &str, _versioned: bool) -> ObjectInfo {
pub fn to_object_info(&self, bucket: &str, object: &str, _versioned: bool) -> ObjectInfo {
ObjectInfo {
bucket: bucket.to_string(),
name: object.to_string(),
@@ -150,6 +164,7 @@ impl FileInfo {
size: self.size,
parts: self.parts.clone(),
is_latest: self.is_latest,
tags: self.tags.clone(),
}
}
// to_part_offset 取offset 所在的part index, 返回part index, offset
@@ -245,6 +260,10 @@ pub struct MakeBucketOptions {
pub force_create: bool,
}
pub struct DeleteBucketOptions {
pub force: bool, // Force deletion
}
#[derive(Debug)]
pub struct PutObjReader {
pub stream: StreamingBlob,
@@ -426,6 +445,7 @@ pub struct ObjectInfo {
pub delete_marker: bool,
pub parts: Vec<ObjectPartInfo>,
pub is_latest: bool,
pub tags: Option<HashMap<String, String>>,
}
#[derive(Debug, Default)]
@@ -492,7 +512,7 @@ pub struct DeletedObject {
#[async_trait::async_trait]
pub trait StorageAPI {
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()>;
async fn delete_bucket(&self, bucket: &str) -> Result<()>;
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()>;
async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>>;
async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo>;
async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo>;
@@ -513,6 +533,9 @@ pub trait StorageAPI {
start_after: &str,
) -> Result<ListObjectsV2Info>;
async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
async fn put_object_info(&self, bucket: &str, object: &str, info: ObjectInfo, opts: &ObjectOptions) -> Result<()>;
async fn get_object_reader(
&self,
bucket: &str,
@@ -521,7 +544,7 @@ pub trait StorageAPI {
h: HeaderMap,
opts: &ObjectOptions,
) -> Result<GetObjectReader>;
async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result<()>;
async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo>;
async fn put_object_part(
&self,
bucket: &str,
ecstore/src/store_init.rs
+18 -35
View File
@@ -13,7 +13,7 @@ use std::{
fmt::Debug,
};
use tracing::warn;
use tracing::{debug, warn};
use uuid::Uuid;
pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec<Option<DiskStore>>, Vec<Option<Error>>) {
@@ -50,13 +50,11 @@ pub async fn connect_load_init_formats(
set_drive_count: usize,
deployment_id: Option<Uuid>,
) -> Result<FormatV3, Error> {
warn!("connect_load_init_formats id: {:?}, first_disk: {}", deployment_id, first_disk);
let (formats, errs) = load_format_erasure_all(disks, false).await;
DiskError::check_disk_fatal_errs(&errs)?;
debug!("load_format_erasure_all errs {:?}", &errs);
warn!("load_format_erasure_all errs {:?}", &errs);
DiskError::check_disk_fatal_errs(&errs)?;
check_format_erasure_values(&formats, set_drive_count)?;
@@ -65,8 +63,9 @@ pub async fn connect_load_init_formats(
// new format and save
let fms = init_format_erasure(disks, set_count, set_drive_count, deployment_id);
let _errs = save_format_file_all(disks, &fms).await;
let errs = save_format_file_all(disks, &fms).await;
debug!("save_format_file_all errs {:?}", &errs);
// TODO: check quorum
// reduceWriteQuorumErrs(&errs)?;
@@ -131,15 +130,8 @@ fn get_format_erasure_in_quorum(formats: &[Option<FormatV3>]) -> Result<FormatV3
let (max_drives, max_count) = countmap.iter().max_by_key(|&(_, c)| c).unwrap_or((&0, &0));
warn!("get_format_erasure_in_quorum fi: {:?}", &formats);
if *max_drives == 0 || *max_count <= formats.len() / 2 {
warn!(
"*max_drives == 0 || *max_count < formats.len() / 2, {} || {}<{}",
max_drives,
max_count,
formats.len() / 2
);
warn!("get_format_erasure_in_quorum fi: {:?}", &formats);
return Err(Error::new(ErasureError::ErasureReadQuorum));
}
@@ -189,26 +181,22 @@ fn check_format_erasure_value(format: &FormatV3) -> Result<()> {
Ok(())
}
pub fn default_partiy_count(drive: usize) -> usize {
match drive {
1 => 0,
2 | 3 => 1,
4 | 5 => 2,
6 | 7 => 3,
_ => 4,
}
}
// load_format_erasure_all 读取所有foramt.json
async fn load_format_erasure_all(disks: &[Option<DiskStore>], heal: bool) -> (Vec<Option<FormatV3>>, Vec<Option<Error>>) {
let mut futures = Vec::with_capacity(disks.len());
for disk in disks.iter() {
futures.push(read_format_file(disk, heal));
}
let mut datas = Vec::with_capacity(disks.len());
let mut errors = Vec::with_capacity(disks.len());
for disk in disks.iter() {
if disk.is_none() {
datas.push(None);
errors.push(Some(Error::new(DiskError::DiskNotFound)));
}
let disk = disk.as_ref().unwrap();
futures.push(load_format_erasure(disk, heal));
}
let results = join_all(futures).await;
let mut i = 0;
for result in results {
@@ -233,12 +221,7 @@ async fn load_format_erasure_all(disks: &[Option<DiskStore>], heal: bool) -> (Ve
(datas, errors)
}
async fn read_format_file(disk: &Option<DiskStore>, _heal: bool) -> Result<FormatV3, Error> {
if disk.is_none() {
return Err(Error::new(DiskError::DiskNotFound));
}
let disk = disk.as_ref().unwrap();
pub async fn load_format_erasure(disk: &DiskStore, _heal: bool) -> Result<FormatV3, Error> {
let data = disk
.read_all(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE)
.await
@@ -249,7 +232,7 @@ async fn read_format_file(disk: &Option<DiskStore>, _heal: bool) -> Result<Forma
None => e,
})?;
let fm = FormatV3::try_from(data.as_ref())?;
let fm = FormatV3::try_from(data.as_slice())?;
// TODO: heal
ecstore/src/utils/fs.rs
+136
View File
@@ -0,0 +1,136 @@
use std::{fs::Metadata, path::Path};
use tokio::{
fs::{self, File},
io,
};
#[cfg(not(windows))]
pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool {
use std::os::unix::fs::MetadataExt;
if f1.dev() != f2.dev() {
return false;
}
if f1.ino() != f2.ino() {
return false;
}
if f1.size() != f2.size() {
return false;
}
if f1.permissions() != f2.permissions() {
return false;
}
if f1.mtime() != f2.mtime() {
return false;
}
true
}
#[cfg(windows)]
pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool {
if f1.permissions() != f2.permissions() {
return false;
}
if f1.file_type() != f2.file_type() {
return false;
}
if f1.len() != f2.len() {
return false;
}
true
}
type FileMode = usize;
pub const O_RDONLY: FileMode = 0x00000;
pub const O_WRONLY: FileMode = 0x00001;
pub const O_RDWR: FileMode = 0x00002;
pub const O_CREATE: FileMode = 0x00040;
// pub const O_EXCL: FileMode = 0x00080;
// pub const O_NOCTTY: FileMode = 0x00100;
pub const O_TRUNC: FileMode = 0x00200;
// pub const O_NONBLOCK: FileMode = 0x00800;
pub const O_APPEND: FileMode = 0x00400;
// pub const O_SYNC: FileMode = 0x01000;
// pub const O_ASYNC: FileMode = 0x02000;
// pub const O_CLOEXEC: FileMode = 0x80000;
// read: bool,
// write: bool,
// append: bool,
// truncate: bool,
// create: bool,
// create_new: bool,
pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
let mut opts = fs::OpenOptions::new();
match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
O_RDONLY => {
opts.read(true);
}
O_WRONLY => {
opts.write(true);
}
O_RDWR => {
opts.read(true);
opts.write(true);
}
_ => (),
};
if mode & O_CREATE != 0 {
opts.create(true);
}
if mode & O_APPEND != 0 {
opts.append(true);
}
if mode & O_TRUNC != 0 {
opts.truncate(true);
}
opts.open(path.as_ref()).await
}
pub async fn access(path: impl AsRef<Path>) -> io::Result<()> {
fs::metadata(path).await?;
Ok(())
}
pub async fn lstat(path: impl AsRef<Path>) -> io::Result<Metadata> {
fs::metadata(path).await
}
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
fs::create_dir_all(path.as_ref()).await
}
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
let meta = fs::metadata(path.as_ref()).await?;
if meta.is_dir() {
fs::remove_dir(path.as_ref()).await
} else {
fs::remove_file(path.as_ref()).await
}
}
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
fs::create_dir(path.as_ref()).await
}
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
fs::rename(from, to).await
}
pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
fs::read(path.as_ref()).await
}
ecstore/src/utils/mod.rs
+1
View File
@@ -1,5 +1,6 @@
pub mod crypto;
pub mod ellipses;
pub mod fs;
pub mod hash;
pub mod net;
pub mod path;
ecstore/src/utils/path.rs
+1 -1
View File
@@ -1,6 +1,6 @@
const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__";
const SLASH_SEPARATOR: &str = "/";
pub const SLASH_SEPARATOR: &str = "/";
pub fn has_suffix(s: &str, suffix: &str) -> bool {
if cfg!(target_os = "windows") {
ecstore/src/writer.rs
+5 -32
View File
@@ -39,38 +39,11 @@ impl<'a> AsyncWrite for AppendWriter<'a> {
let mut fut = Box::pin(self.async_write(buf));
debug!("AsyncWrite poll_write {}, buf:{}", self.disk.id(), buf.len());
// while let Poll::Ready(e) = fut.as_mut().poll(cx) {
// let a = match e {
// Ok(_) => {
// debug!("Ready ok {}", self.disk.id());
// Poll::Ready(Ok(buf.len()))
// }
// Err(e) => {
// debug!("Ready err {}", self.disk.id());
// Poll::Ready(Err(e))
// }
// };
// return a;
// }
// Poll::Pending
match fut.as_mut().poll(cx) {
Poll::Pending => {
debug!("Pending {}", self.disk.id());
Poll::Pending
}
Poll::Ready(e) => match e {
Ok(_) => {
debug!("Ready ok {}", self.disk.id());
Poll::Ready(Ok(buf.len()))
}
Err(e) => {
debug!("Ready err {}", self.disk.id());
Poll::Ready(Err(e))
}
},
let mut fut = self.get_mut().async_write(buf);
match futures::future::poll_fn(|cx| fut.as_mut().poll(cx)).start(cx) {
Ready(Ok(n)) => Ready(Ok(n)),
Ready(Err(e)) => Ready(Err(e)),
Pending => Pending,
}
}