gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

mc test ok

Browse Source
This commit is contained in:
weisd
2025-06-06 16:15:26 +08:00
parent 3dacde092c
commit c589972fa7
55 changed files with 5053 additions and 17559 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1
View File
@@ -7710,6 +7710,7 @@ dependencies = [
"rustfs-event-notifier",
"rustfs-filemeta",
"rustfs-obs",
"rustfs-rio",
"rustfs-utils",
"rustfs-zip",
"rustls 0.23.27",
common/common/src/error.rs
+7
View File
@@ -11,6 +11,13 @@ pub struct Error {
}
impl Error {
pub fn other<E>(error: E) -> Self
where
E: std::fmt::Display + Into<Box<dyn std::error::Error + Send + Sync>>,
{
Self::from_std_error(error.into())
}
/// Create a new error from a `std::error::Error`.
#[must_use]
#[track_caller]
crates/disk/Cargo.toml
-33
View File
@@ -1,33 +0,0 @@
[package]
name = "rustfs-disk"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
[dependencies]
url.workspace = true
rustfs-filemeta.workspace = true
rustfs-error.workspace = true
rustfs-rio.workspace = true
serde.workspace = true
serde_json.workspace = true
uuid.workspace = true
tracing.workspace = true
tokio.workspace = true
path-absolutize = "3.1.1"
rustfs-utils = {workspace = true, features =["net"]}
async-trait.workspace = true
time.workspace = true
rustfs-metacache.workspace = true
futures.workspace = true
madmin.workspace = true
protos.workspace = true
tonic.workspace = true
urlencoding = "2.1.3"
rmp-serde.workspace = true
http.workspace = true
[lints]
workspace = true
crates/disk/src/api.rs
-667
View File
@@ -1,667 +0,0 @@
use crate::{endpoint::Endpoint, local::LocalDisk, remote::RemoteDisk};
use madmin::DiskMetrics;
use rustfs_error::{Error, Result};
use rustfs_filemeta::{FileInfo, FileInfoVersions, RawFileInfo};
use serde::{Deserialize, Serialize};
use std::{fmt::Debug, path::PathBuf, sync::Arc};
use time::OffsetDateTime;
use tokio::io::{AsyncRead, AsyncWrite};
use uuid::Uuid;
pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp";
pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash";
pub const BUCKET_META_PREFIX: &str = "buckets";
pub const FORMAT_CONFIG_FILE: &str = "format.json";
pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
pub type DiskStore = Arc<Disk>;
#[derive(Debug)]
pub enum Disk {
Local(LocalDisk),
Remote(RemoteDisk),
}
#[async_trait::async_trait]
impl DiskAPI for Disk {
#[tracing::instrument(skip(self))]
fn to_string(&self) -> String {
match self {
Disk::Local(local_disk) => local_disk.to_string(),
Disk::Remote(remote_disk) => remote_disk.to_string(),
}
}
#[tracing::instrument(skip(self))]
fn is_local(&self) -> bool {
match self {
Disk::Local(local_disk) => local_disk.is_local(),
Disk::Remote(remote_disk) => remote_disk.is_local(),
}
}
#[tracing::instrument(skip(self))]
fn host_name(&self) -> String {
match self {
Disk::Local(local_disk) => local_disk.host_name(),
Disk::Remote(remote_disk) => remote_disk.host_name(),
}
}
#[tracing::instrument(skip(self))]
async fn is_online(&self) -> bool {
match self {
Disk::Local(local_disk) => local_disk.is_online().await,
Disk::Remote(remote_disk) => remote_disk.is_online().await,
}
}
#[tracing::instrument(skip(self))]
fn endpoint(&self) -> Endpoint {
match self {
Disk::Local(local_disk) => local_disk.endpoint(),
Disk::Remote(remote_disk) => remote_disk.endpoint(),
}
}
#[tracing::instrument(skip(self))]
async fn close(&self) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.close().await,
Disk::Remote(remote_disk) => remote_disk.close().await,
}
}
#[tracing::instrument(skip(self))]
fn path(&self) -> PathBuf {
match self {
Disk::Local(local_disk) => local_disk.path(),
Disk::Remote(remote_disk) => remote_disk.path(),
}
}
#[tracing::instrument(skip(self))]
fn get_disk_location(&self) -> DiskLocation {
match self {
Disk::Local(local_disk) => local_disk.get_disk_location(),
Disk::Remote(remote_disk) => remote_disk.get_disk_location(),
}
}
#[tracing::instrument(skip(self))]
async fn get_disk_id(&self) -> Result<Option<Uuid>> {
match self {
Disk::Local(local_disk) => local_disk.get_disk_id().await,
Disk::Remote(remote_disk) => remote_disk.get_disk_id().await,
}
}
#[tracing::instrument(skip(self))]
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.set_disk_id(id).await,
Disk::Remote(remote_disk) => remote_disk.set_disk_id(id).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
match self {
Disk::Local(local_disk) => local_disk.read_all(volume, path).await,
Disk::Remote(remote_disk) => remote_disk.read_all(volume, path).await,
}
}
#[tracing::instrument(skip(self))]
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.write_all(volume, path, data).await,
Disk::Remote(remote_disk) => remote_disk.write_all(volume, path, data).await,
}
}
#[tracing::instrument(skip(self))]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete(volume, path, opt).await,
Disk::Remote(remote_disk) => remote_disk.delete(volume, path, opt).await,
}
}
#[tracing::instrument(skip(self))]
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
match self {
Disk::Local(local_disk) => local_disk.verify_file(volume, path, fi).await,
Disk::Remote(remote_disk) => remote_disk.verify_file(volume, path, fi).await,
}
}
#[tracing::instrument(skip(self))]
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
match self {
Disk::Local(local_disk) => local_disk.check_parts(volume, path, fi).await,
Disk::Remote(remote_disk) => remote_disk.check_parts(volume, path, fi).await,
}
}
#[tracing::instrument(skip(self))]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.rename_part(src_volume, src_path, dst_volume, dst_path, meta).await,
Disk::Remote(remote_disk) => {
remote_disk
.rename_part(src_volume, src_path, dst_volume, dst_path, meta)
.await
}
}
}
#[tracing::instrument(skip(self))]
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.rename_file(src_volume, src_path, dst_volume, dst_path).await,
Disk::Remote(remote_disk) => remote_disk.rename_file(src_volume, src_path, dst_volume, dst_path).await,
}
}
#[tracing::instrument(skip(self))]
async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result<Box<dyn AsyncWrite>> {
match self {
Disk::Local(local_disk) => local_disk.create_file(_origvolume, volume, path, _file_size).await,
Disk::Remote(remote_disk) => remote_disk.create_file(_origvolume, volume, path, _file_size).await,
}
}
#[tracing::instrument(skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<Box<dyn AsyncWrite>> {
match self {
Disk::Local(local_disk) => local_disk.append_file(volume, path).await,
Disk::Remote(remote_disk) => remote_disk.append_file(volume, path).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_file(&self, volume: &str, path: &str) -> Result<Box<dyn AsyncRead>> {
match self {
Disk::Local(local_disk) => local_disk.read_file(volume, path).await,
Disk::Remote(remote_disk) => remote_disk.read_file(volume, path).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Box<dyn AsyncRead>> {
match self {
Disk::Local(local_disk) => local_disk.read_file_stream(volume, path, offset, length).await,
Disk::Remote(remote_disk) => remote_disk.read_file_stream(volume, path, offset, length).await,
}
}
#[tracing::instrument(skip(self))]
async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
match self {
Disk::Local(local_disk) => local_disk.list_dir(_origvolume, volume, _dir_path, _count).await,
Disk::Remote(remote_disk) => remote_disk.list_dir(_origvolume, volume, _dir_path, _count).await,
}
}
#[tracing::instrument(skip(self, wr))]
async fn walk_dir<W: AsyncWrite + Unpin + Send + Sync + 'static>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.walk_dir(opts, wr).await,
Disk::Remote(remote_disk) => remote_disk.walk_dir(opts, wr).await,
}
}
#[tracing::instrument(skip(self))]
async fn rename_data(
&self,
src_volume: &str,
src_path: &str,
fi: FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
match self {
Disk::Local(local_disk) => local_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await,
Disk::Remote(remote_disk) => remote_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await,
}
}
#[tracing::instrument(skip(self))]
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.make_volumes(volumes).await,
Disk::Remote(remote_disk) => remote_disk.make_volumes(volumes).await,
}
}
#[tracing::instrument(skip(self))]
async fn make_volume(&self, volume: &str) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.make_volume(volume).await,
Disk::Remote(remote_disk) => remote_disk.make_volume(volume).await,
}
}
#[tracing::instrument(skip(self))]
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
match self {
Disk::Local(local_disk) => local_disk.list_volumes().await,
Disk::Remote(remote_disk) => remote_disk.list_volumes().await,
}
}
#[tracing::instrument(skip(self))]
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
match self {
Disk::Local(local_disk) => local_disk.stat_volume(volume).await,
Disk::Remote(remote_disk) => remote_disk.stat_volume(volume).await,
}
}
#[tracing::instrument(skip(self))]
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete_paths(volume, paths).await,
Disk::Remote(remote_disk) => remote_disk.delete_paths(volume, paths).await,
}
}
#[tracing::instrument(skip(self))]
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.update_metadata(volume, path, fi, opts).await,
Disk::Remote(remote_disk) => remote_disk.update_metadata(volume, path, fi, opts).await,
}
}
#[tracing::instrument(skip(self))]
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.write_metadata(_org_volume, volume, path, fi).await,
Disk::Remote(remote_disk) => remote_disk.write_metadata(_org_volume, volume, path, fi).await,
}
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_version(
&self,
_org_volume: &str,
volume: &str,
path: &str,
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo> {
match self {
Disk::Local(local_disk) => local_disk.read_version(_org_volume, volume, path, version_id, opts).await,
Disk::Remote(remote_disk) => remote_disk.read_version(_org_volume, volume, path, version_id, opts).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
match self {
Disk::Local(local_disk) => local_disk.read_xl(volume, path, read_data).await,
Disk::Remote(remote_disk) => remote_disk.read_xl(volume, path, read_data).await,
}
}
#[tracing::instrument(skip(self))]
async fn delete_version(
&self,
volume: &str,
path: &str,
fi: FileInfo,
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete_version(volume, path, fi, force_del_marker, opts).await,
Disk::Remote(remote_disk) => remote_disk.delete_version(volume, path, fi, force_del_marker, opts).await,
}
}
#[tracing::instrument(skip(self))]
async fn delete_versions(
&self,
volume: &str,
versions: Vec<FileInfoVersions>,
opts: DeleteOptions,
) -> Result<Vec<Option<Error>>> {
match self {
Disk::Local(local_disk) => local_disk.delete_versions(volume, versions, opts).await,
Disk::Remote(remote_disk) => remote_disk.delete_versions(volume, versions, opts).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
match self {
Disk::Local(local_disk) => local_disk.read_multiple(req).await,
Disk::Remote(remote_disk) => remote_disk.read_multiple(req).await,
}
}
#[tracing::instrument(skip(self))]
async fn delete_volume(&self, volume: &str) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete_volume(volume).await,
Disk::Remote(remote_disk) => remote_disk.delete_volume(volume).await,
}
}
#[tracing::instrument(skip(self))]
async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo> {
match self {
Disk::Local(local_disk) => local_disk.disk_info(opts).await,
Disk::Remote(remote_disk) => remote_disk.disk_info(opts).await,
}
}
// #[tracing::instrument(skip(self, cache, we_sleep, scan_mode))]
// async fn ns_scanner(
// &self,
// cache: &DataUsageCache,
// updates: Sender<DataUsageEntry>,
// scan_mode: HealScanMode,
// we_sleep: ShouldSleepFn,
// ) -> Result<DataUsageCache> {
// match self {
// Disk::Local(local_disk) => local_disk.ns_scanner(cache, updates, scan_mode, we_sleep).await,
// Disk::Remote(remote_disk) => remote_disk.ns_scanner(cache, updates, scan_mode, we_sleep).await,
// }
// }
// #[tracing::instrument(skip(self))]
// async fn healing(&self) -> Option<HealingTracker> {
// match self {
// Disk::Local(local_disk) => local_disk.healing().await,
// Disk::Remote(remote_disk) => remote_disk.healing().await,
// }
// }
}
pub async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> Result<DiskStore> {
if ep.is_local {
Ok(Arc::new(Disk::Local(LocalDisk::new(ep, opt.cleanup).await?)))
} else {
Ok(Arc::new(Disk::Remote(RemoteDisk::new(ep, opt).await?)))
}
}
#[async_trait::async_trait]
pub trait DiskAPI: Debug + Send + Sync + 'static {
fn to_string(&self) -> String;
async fn is_online(&self) -> bool;
fn is_local(&self) -> bool;
// LastConn
fn host_name(&self) -> String;
fn endpoint(&self) -> Endpoint;
async fn close(&self) -> Result<()>;
async fn get_disk_id(&self) -> Result<Option<Uuid>>;
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
fn path(&self) -> PathBuf;
fn get_disk_location(&self) -> DiskLocation;
// Healing
// DiskInfo
// NSScanner
// Volume operations.
async fn make_volume(&self, volume: &str) -> Result<()>;
async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>;
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>>;
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo>;
async fn delete_volume(&self, volume: &str) -> Result<()>;
// 并发边读边写 w <- MetaCacheEntry
async fn walk_dir<W: AsyncWrite + Unpin + Send + Sync + 'static>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()>;
// Metadata operations
async fn delete_version(
&self,
volume: &str,
path: &str,
fi: FileInfo,
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<()>;
async fn delete_versions(
&self,
volume: &str,
versions: Vec<FileInfoVersions>,
opts: DeleteOptions,
) -> Result<Vec<Option<Error>>>;
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()>;
async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>;
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()>;
async fn read_version(
&self,
org_volume: &str,
volume: &str,
path: &str,
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo>;
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo>;
async fn rename_data(
&self,
src_volume: &str,
src_path: &str,
file_info: FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp>;
// File operations.
// 读目录下的所有文件、目录
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>>;
async fn read_file(&self, volume: &str, path: &str) -> Result<Box<dyn AsyncRead>>;
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Box<dyn AsyncRead>>;
async fn append_file(&self, volume: &str, path: &str) -> Result<Box<dyn AsyncWrite>>;
async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<Box<dyn AsyncWrite>>;
// ReadFileStream
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>;
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()>;
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
// VerifyFile
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp>;
// CheckParts
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp>;
// StatInfoFile
// ReadParts
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>>;
// CleanAbandonedData
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()>;
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>>;
async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo>;
// async fn ns_scanner(
// &self,
// cache: &DataUsageCache,
// updates: Sender<DataUsageEntry>,
// scan_mode: HealScanMode,
// we_sleep: ShouldSleepFn,
// ) -> Result<DataUsageCache>;
// async fn healing(&self) -> Option<HealingTracker>;
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct CheckPartsResp {
pub results: Vec<usize>,
}
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct UpdateMetadataOpts {
pub no_persistence: bool,
}
pub struct DiskLocation {
pub pool_idx: Option<usize>,
pub set_idx: Option<usize>,
pub disk_idx: Option<usize>,
}
impl DiskLocation {
pub fn valid(&self) -> bool {
self.pool_idx.is_some() && self.set_idx.is_some() && self.disk_idx.is_some()
}
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct DiskInfoOptions {
pub disk_id: String,
pub metrics: bool,
pub noop: bool,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct DiskInfo {
pub total: u64,
pub free: u64,
pub used: u64,
pub used_inodes: u64,
pub free_inodes: u64,
pub major: u64,
pub minor: u64,
pub nr_requests: u64,
pub fs_type: String,
pub root_disk: bool,
pub healing: bool,
pub scanning: bool,
pub endpoint: String,
pub mount_path: String,
pub id: String,
pub rotational: bool,
pub metrics: DiskMetrics,
pub error: String,
}
#[derive(Clone, Debug, Default)]
pub struct Info {
pub total: u64,
pub free: u64,
pub used: u64,
pub files: u64,
pub ffree: u64,
pub fstype: String,
pub major: u64,
pub minor: u64,
pub name: String,
pub rotational: bool,
pub nrrequests: u64,
}
// #[derive(Debug, Default, Clone, Serialize, Deserialize)]
// pub struct FileInfoVersions {
// // Name of the volume.
// pub volume: String,
// // Name of the file.
// pub name: String,
// // Represents the latest mod time of the
// // latest version.
// pub latest_mod_time: Option<OffsetDateTime>,
// pub versions: Vec<FileInfo>,
// pub free_versions: Vec<FileInfo>,
// }
// impl FileInfoVersions {
// pub fn find_version_index(&self, v: &str) -> Option<usize> {
// if v.is_empty() {
// return None;
// }
// let vid = Uuid::parse_str(v).unwrap_or(Uuid::nil());
// self.versions.iter().position(|v| v.version_id == Some(vid))
// }
// }
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct WalkDirOptions {
// Bucket to scanner
pub bucket: String,
// Directory inside the bucket.
pub base_dir: String,
// Do a full recursive scan.
pub recursive: bool,
// ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
pub report_notfound: bool,
// FilterPrefix will only return results with given prefix within folder.
// Should never contain a slash.
pub filter_prefix: Option<String>,
// ForwardTo will forward to the given object path.
pub forward_to: Option<String>,
// Limit the number of returned objects if > 0.
pub limit: i32,
// DiskID contains the disk ID of the disk.
// Leave empty to not check disk ID.
pub disk_id: String,
}
// move metacache to metacache.rs
#[derive(Clone, Debug, Default)]
pub struct DiskOption {
pub cleanup: bool,
pub health_check: bool,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct RenameDataResp {
pub old_data_dir: Option<Uuid>,
pub sign: Option<Vec<u8>>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct DeleteOptions {
pub recursive: bool,
pub immediate: bool,
pub undo_write: bool,
pub old_data_dir: Option<Uuid>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReadMultipleReq {
pub bucket: String,
pub prefix: String,
pub files: Vec<String>,
pub max_size: usize,
pub metadata_only: bool,
pub abort404: bool,
pub max_results: usize,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ReadMultipleResp {
pub bucket: String,
pub prefix: String,
pub file: String,
pub exists: bool,
pub error: String,
pub data: Vec<u8>,
pub mod_time: Option<OffsetDateTime>,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct VolumeInfo {
pub name: String,
pub created: Option<OffsetDateTime>,
}
#[derive(Deserialize, Serialize, Debug, Default)]
pub struct ReadOptions {
pub incl_free_versions: bool,
pub read_data: bool,
pub healing: bool,
}
crates/disk/src/endpoint.rs
-379
View File
@@ -1,379 +0,0 @@
use path_absolutize::Absolutize;
use rustfs_utils::{is_local_host, is_socket_addr};
use std::{fmt::Display, path::Path};
use url::{ParseError, Url};
/// enum for endpoint type.
#[derive(PartialEq, Eq, Debug)]
pub enum EndpointType {
/// path style endpoint type enum.
Path,
/// URL style endpoint type enum.
Url,
}
/// any type of endpoint.
#[derive(Debug, PartialEq, Eq, Clone, Hash)]
pub struct Endpoint {
pub url: url::Url,
pub is_local: bool,
pub pool_idx: i32,
pub set_idx: i32,
pub disk_idx: i32,
}
impl Display for Endpoint {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if self.url.scheme() == "file" {
write!(f, "{}", self.get_file_path())
} else {
write!(f, "{}", self.url)
}
}
}
impl TryFrom<&str> for Endpoint {
/// The type returned in the event of a conversion error.
type Error = std::io::Error;
/// Performs the conversion.
fn try_from(value: &str) -> core::result::Result<Self, Self::Error> {
// check whether given path is not empty.
if ["", "/", "\\"].iter().any(|&v| v.eq(value)) {
return Err(std::io::Error::other("empty or root endpoint is not supported"));
}
let mut is_local = false;
let url = match Url::parse(value) {
#[allow(unused_mut)]
Ok(mut url) if url.has_host() => {
// URL style of endpoint.
// Valid URL style endpoint is
// - Scheme field must contain "http" or "https"
// - All field should be empty except Host and Path.
if !((url.scheme() == "http" || url.scheme() == "https")
&& url.username().is_empty()
&& url.fragment().is_none()
&& url.query().is_none())
{
return Err(std::io::Error::other("invalid URL endpoint format"));
}
let path = url.path().to_string();
#[cfg(not(windows))]
let path = Path::new(&path).absolutize()?;
// On windows having a preceding SlashSeparator will cause problems, if the
// command line already has C:/<export-folder/ in it. Final resulting
// path on windows might become C:/C:/ this will cause problems
// of starting rustfs server properly in distributed mode on windows.
// As a special case make sure to trim the separator.
#[cfg(windows)]
let path = Path::new(&path[1..]).absolutize()?;
if path.parent().is_none() || Path::new("").eq(&path) {
return Err(std::io::Error::other("empty or root path is not supported in URL endpoint"));
}
match path.to_str() {
Some(v) => url.set_path(v),
None => return Err(std::io::Error::other("invalid path")),
}
url
}
Ok(_) => {
// like d:/foo
is_local = true;
url_parse_from_file_path(value)?
}
Err(e) => match e {
ParseError::InvalidPort => {
return Err(std::io::Error::other(
"invalid URL endpoint format: port number must be between 1 to 65535",
))
}
ParseError::EmptyHost => return Err(std::io::Error::other("invalid URL endpoint format: empty host name")),
ParseError::RelativeUrlWithoutBase => {
// like /foo
is_local = true;
url_parse_from_file_path(value)?
}
_ => return Err(std::io::Error::other(format!("invalid URL endpoint format: {}", e))),
},
};
Ok(Endpoint {
url,
is_local,
pool_idx: -1,
set_idx: -1,
disk_idx: -1,
})
}
}
impl Endpoint {
/// returns type of endpoint.
pub fn get_type(&self) -> EndpointType {
if self.url.scheme() == "file" {
EndpointType::Path
} else {
EndpointType::Url
}
}
/// sets a specific pool number to this node
pub fn set_pool_index(&mut self, idx: usize) {
self.pool_idx = idx as i32
}
/// sets a specific set number to this node
pub fn set_set_index(&mut self, idx: usize) {
self.set_idx = idx as i32
}
/// sets a specific disk number to this node
pub fn set_disk_index(&mut self, idx: usize) {
self.disk_idx = idx as i32
}
/// resolves the host and updates if it is local or not.
pub fn update_is_local(&mut self, local_port: u16) -> std::io::Result<()> {
match (self.url.scheme(), self.url.host()) {
(v, Some(host)) if v != "file" => {
self.is_local = is_local_host(host, self.url.port().unwrap_or_default(), local_port)?;
}
_ => {}
}
Ok(())
}
/// returns the host to be used for grid connections.
pub fn grid_host(&self) -> String {
match (self.url.host(), self.url.port()) {
(Some(host), Some(port)) => format!("{}://{}:{}", self.url.scheme(), host, port),
(Some(host), None) => format!("{}://{}", self.url.scheme(), host),
_ => String::new(),
}
}
pub fn host_port(&self) -> String {
match (self.url.host(), self.url.port()) {
(Some(host), Some(port)) => format!("{}:{}", host, port),
(Some(host), None) => format!("{}", host),
_ => String::new(),
}
}
pub fn get_file_path(&self) -> &str {
let path = self.url.path();
#[cfg(windows)]
let path = &path[1..];
path
}
}
/// parse a file path into an URL.
fn url_parse_from_file_path(value: &str) -> std::io::Result<url::Url> {
// Only check if the arg is an ip address and ask for scheme since its absent.
// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
// /mnt/export1. So we go ahead and start the rustfs server in FS modes in these cases.
let addr: Vec<&str> = value.splitn(2, '/').collect();
if is_socket_addr(addr[0]) {
return Err(std::io::Error::other("invalid URL endpoint format: missing scheme http or https"));
}
let file_path = match Path::new(value).absolutize() {
Ok(path) => path,
Err(err) => return Err(std::io::Error::other(format!("absolute path failed: {}", err))),
};
match Url::from_file_path(file_path) {
Ok(url) => Ok(url),
Err(_) => Err(std::io::Error::other("Convert a file path into an URL failed")),
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_new_endpoint() {
#[derive(Default)]
struct TestCase<'a> {
arg: &'a str,
expected_endpoint: Option<Endpoint>,
expected_type: Option<EndpointType>,
expected_err: Option<std::io::Error>,
}
let u2 = url::Url::parse("https://example.org/path").unwrap();
let u4 = url::Url::parse("http://192.168.253.200/path").unwrap();
let u6 = url::Url::parse("http://server:/path").unwrap();
let root_slash_foo = url::Url::from_file_path("/foo").unwrap();
let test_cases = [
TestCase {
arg: "/foo",
expected_endpoint: Some(Endpoint {
url: root_slash_foo,
is_local: true,
pool_idx: -1,
set_idx: -1,
disk_idx: -1,
}),
expected_type: Some(EndpointType::Path),
expected_err: None,
},
TestCase {
arg: "https://example.org/path",
expected_endpoint: Some(Endpoint {
url: u2,
is_local: false,
pool_idx: -1,
set_idx: -1,
disk_idx: -1,
}),
expected_type: Some(EndpointType::Url),
expected_err: None,
},
TestCase {
arg: "http://192.168.253.200/path",
expected_endpoint: Some(Endpoint {
url: u4,
is_local: false,
pool_idx: -1,
set_idx: -1,
disk_idx: -1,
}),
expected_type: Some(EndpointType::Url),
expected_err: None,
},
TestCase {
arg: "",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("empty or root endpoint is not supported")),
},
TestCase {
arg: "/",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("empty or root endpoint is not supported")),
},
TestCase {
arg: "\\",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("empty or root endpoint is not supported")),
},
TestCase {
arg: "c://foo",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("invalid URL endpoint format")),
},
TestCase {
arg: "ftp://foo",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("invalid URL endpoint format")),
},
TestCase {
arg: "http://server/path?location",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("invalid URL endpoint format")),
},
TestCase {
arg: "http://:/path",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("invalid URL endpoint format: empty host name")),
},
TestCase {
arg: "http://:8080/path",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("invalid URL endpoint format: empty host name")),
},
TestCase {
arg: "http://server:/path",
expected_endpoint: Some(Endpoint {
url: u6,
is_local: false,
pool_idx: -1,
set_idx: -1,
disk_idx: -1,
}),
expected_type: Some(EndpointType::Url),
expected_err: None,
},
TestCase {
arg: "https://93.184.216.34:808080/path",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other(
"invalid URL endpoint format: port number must be between 1 to 65535",
)),
},
TestCase {
arg: "http://server:8080//",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("empty or root path is not supported in URL endpoint")),
},
TestCase {
arg: "http://server:8080/",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("empty or root path is not supported in URL endpoint")),
},
TestCase {
arg: "192.168.1.210:9000",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("invalid URL endpoint format: missing scheme http or https")),
},
];
for test_case in test_cases {
let ret = Endpoint::try_from(test_case.arg);
if test_case.expected_err.is_none() && ret.is_err() {
panic!("{}: error: expected = <nil>, got = {:?}", test_case.arg, ret);
}
if test_case.expected_err.is_some() && ret.is_ok() {
panic!("{}: error: expected = {:?}, got = <nil>", test_case.arg, test_case.expected_err);
}
match (test_case.expected_err, ret) {
(None, Err(e)) => panic!("{}: error: expected = <nil>, got = {}", test_case.arg, e),
(None, Ok(mut ep)) => {
let _ = ep.update_is_local(9000);
if test_case.expected_type != Some(ep.get_type()) {
panic!(
"{}: type: expected = {:?}, got = {:?}",
test_case.arg,
test_case.expected_type,
ep.get_type()
);
}
assert_eq!(test_case.expected_endpoint, Some(ep), "{}: endpoint", test_case.arg);
}
(Some(e), Ok(_)) => panic!("{}: error: expected = {}, got = <nil>", test_case.arg, e),
(Some(e), Err(e2)) => {
assert_eq!(e.to_string(), e2.to_string(), "{}: error: expected = {}, got = {}", test_case.arg, e, e2)
}
}
}
}
}
crates/disk/src/error.rs
-594
View File
@@ -1,594 +0,0 @@
use std::io::{self, ErrorKind};
use std::path::PathBuf;
use tracing::error;
use crate::quorum::CheckErrorFn;
use crate::utils::ERROR_TYPE_MASK;
use common::error::{Error, Result};
// DiskError == StorageErr
#[derive(Debug, thiserror::Error)]
pub enum DiskError {
#[error("maximum versions exceeded, please delete few versions to proceed")]
MaxVersionsExceeded,
#[error("unexpected error")]
Unexpected,
#[error("corrupted format")]
CorruptedFormat,
#[error("corrupted backend")]
CorruptedBackend,
#[error("unformatted disk error")]
UnformattedDisk,
#[error("inconsistent drive found")]
InconsistentDisk,
#[error("drive does not support O_DIRECT")]
UnsupportedDisk,
#[error("drive path full")]
DiskFull,
#[error("disk not a dir")]
DiskNotDir,
#[error("disk not found")]
DiskNotFound,
#[error("drive still did not complete the request")]
DiskOngoingReq,
#[error("drive is part of root drive, will not be used")]
DriveIsRoot,
#[error("remote drive is faulty")]
FaultyRemoteDisk,
#[error("drive is faulty")]
FaultyDisk,
#[error("drive access denied")]
DiskAccessDenied,
#[error("file not found")]
FileNotFound,
#[error("file version not found")]
FileVersionNotFound,
#[error("too many open files, please increase 'ulimit -n'")]
TooManyOpenFiles,
#[error("file name too long")]
FileNameTooLong,
#[error("volume already exists")]
VolumeExists,
#[error("not of regular file type")]
IsNotRegular,
#[error("path not found")]
PathNotFound,
#[error("volume not found")]
VolumeNotFound,
#[error("volume is not empty")]
VolumeNotEmpty,
#[error("volume access denied")]
VolumeAccessDenied,
#[error("disk access denied")]
FileAccessDenied,
#[error("file is corrupted")]
FileCorrupt,
#[error("bit-rot hash algorithm is invalid")]
BitrotHashAlgoInvalid,
#[error("Rename across devices not allowed, please fix your backend configuration")]
CrossDeviceLink,
#[error("less data available than what was requested")]
LessData,
#[error("more data was sent than what was advertised")]
MoreData,
#[error("outdated XL meta")]
OutdatedXLMeta,
#[error("part missing or corrupt")]
PartMissingOrCorrupt,
#[error("No healing is required")]
NoHealRequired,
}
impl DiskError {
/// Checks if the given array of errors contains fatal disk errors.
/// If all errors are of the same fatal disk error type, returns the corresponding error.
/// Otherwise, returns Ok.
///
/// # Parameters
/// - `errs`: A slice of optional errors.
///
/// # Returns
/// If all errors are of the same fatal disk error type, returns the corresponding error.
/// Otherwise, returns Ok.
pub fn check_disk_fatal_errs(errs: &[Option<Error>]) -> Result<()> {
if DiskError::UnsupportedDisk.count_errs(errs) == errs.len() {
return Err(DiskError::UnsupportedDisk.into());
}
if DiskError::FileAccessDenied.count_errs(errs) == errs.len() {
return Err(DiskError::FileAccessDenied.into());
}
if DiskError::DiskNotDir.count_errs(errs) == errs.len() {
return Err(DiskError::DiskNotDir.into());
}
Ok(())
}
pub fn count_errs(&self, errs: &[Option<Error>]) -> usize {
errs.iter()
.filter(|&err| match err {
None => false,
Some(e) => self.is(e),
})
.count()
}
pub fn quorum_unformatted_disks(errs: &[Option<Error>]) -> bool {
DiskError::UnformattedDisk.count_errs(errs) > (errs.len() / 2)
}
pub fn should_init_erasure_disks(errs: &[Option<Error>]) -> bool {
DiskError::UnformattedDisk.count_errs(errs) == errs.len()
}
/// Check if the error is a disk error
pub fn is(&self, err: &Error) -> bool {
if let Some(e) = err.downcast_ref::<DiskError>() {
e == self
} else {
false
}
}
}
impl DiskError {
pub fn to_u32(&self) -> u32 {
match self {
DiskError::MaxVersionsExceeded => 0x01,
DiskError::Unexpected => 0x02,
DiskError::CorruptedFormat => 0x03,
DiskError::CorruptedBackend => 0x04,
DiskError::UnformattedDisk => 0x05,
DiskError::InconsistentDisk => 0x06,
DiskError::UnsupportedDisk => 0x07,
DiskError::DiskFull => 0x08,
DiskError::DiskNotDir => 0x09,
DiskError::DiskNotFound => 0x0A,
DiskError::DiskOngoingReq => 0x0B,
DiskError::DriveIsRoot => 0x0C,
DiskError::FaultyRemoteDisk => 0x0D,
DiskError::FaultyDisk => 0x0E,
DiskError::DiskAccessDenied => 0x0F,
DiskError::FileNotFound => 0x10,
DiskError::FileVersionNotFound => 0x11,
DiskError::TooManyOpenFiles => 0x12,
DiskError::FileNameTooLong => 0x13,
DiskError::VolumeExists => 0x14,
DiskError::IsNotRegular => 0x15,
DiskError::PathNotFound => 0x16,
DiskError::VolumeNotFound => 0x17,
DiskError::VolumeNotEmpty => 0x18,
DiskError::VolumeAccessDenied => 0x19,
DiskError::FileAccessDenied => 0x1A,
DiskError::FileCorrupt => 0x1B,
DiskError::BitrotHashAlgoInvalid => 0x1C,
DiskError::CrossDeviceLink => 0x1D,
DiskError::LessData => 0x1E,
DiskError::MoreData => 0x1F,
DiskError::OutdatedXLMeta => 0x20,
DiskError::PartMissingOrCorrupt => 0x21,
DiskError::NoHealRequired => 0x22,
}
}
pub fn from_u32(error: u32) -> Option<Self> {
match error & ERROR_TYPE_MASK {
0x01 => Some(DiskError::MaxVersionsExceeded),
0x02 => Some(DiskError::Unexpected),
0x03 => Some(DiskError::CorruptedFormat),
0x04 => Some(DiskError::CorruptedBackend),
0x05 => Some(DiskError::UnformattedDisk),
0x06 => Some(DiskError::InconsistentDisk),
0x07 => Some(DiskError::UnsupportedDisk),
0x08 => Some(DiskError::DiskFull),
0x09 => Some(DiskError::DiskNotDir),
0x0A => Some(DiskError::DiskNotFound),
0x0B => Some(DiskError::DiskOngoingReq),
0x0C => Some(DiskError::DriveIsRoot),
0x0D => Some(DiskError::FaultyRemoteDisk),
0x0E => Some(DiskError::FaultyDisk),
0x0F => Some(DiskError::DiskAccessDenied),
0x10 => Some(DiskError::FileNotFound),
0x11 => Some(DiskError::FileVersionNotFound),
0x12 => Some(DiskError::TooManyOpenFiles),
0x13 => Some(DiskError::FileNameTooLong),
0x14 => Some(DiskError::VolumeExists),
0x15 => Some(DiskError::IsNotRegular),
0x16 => Some(DiskError::PathNotFound),
0x17 => Some(DiskError::VolumeNotFound),
0x18 => Some(DiskError::VolumeNotEmpty),
0x19 => Some(DiskError::VolumeAccessDenied),
0x1A => Some(DiskError::FileAccessDenied),
0x1B => Some(DiskError::FileCorrupt),
0x1C => Some(DiskError::BitrotHashAlgoInvalid),
0x1D => Some(DiskError::CrossDeviceLink),
0x1E => Some(DiskError::LessData),
0x1F => Some(DiskError::MoreData),
0x20 => Some(DiskError::OutdatedXLMeta),
0x21 => Some(DiskError::PartMissingOrCorrupt),
0x22 => Some(DiskError::NoHealRequired),
_ => None,
}
}
}
impl PartialEq for DiskError {
fn eq(&self, other: &Self) -> bool {
core::mem::discriminant(self) == core::mem::discriminant(other)
}
}
impl CheckErrorFn for DiskError {
fn is(&self, e: &Error) -> bool {
self.is(e)
}
}
pub fn clone_disk_err(e: &DiskError) -> Error {
match e {
DiskError::MaxVersionsExceeded => Error::new(DiskError::MaxVersionsExceeded),
DiskError::Unexpected => Error::new(DiskError::Unexpected),
DiskError::CorruptedFormat => Error::new(DiskError::CorruptedFormat),
DiskError::CorruptedBackend => Error::new(DiskError::CorruptedBackend),
DiskError::UnformattedDisk => Error::new(DiskError::UnformattedDisk),
DiskError::InconsistentDisk => Error::new(DiskError::InconsistentDisk),
DiskError::UnsupportedDisk => Error::new(DiskError::UnsupportedDisk),
DiskError::DiskFull => Error::new(DiskError::DiskFull),
DiskError::DiskNotDir => Error::new(DiskError::DiskNotDir),
DiskError::DiskNotFound => Error::new(DiskError::DiskNotFound),
DiskError::DiskOngoingReq => Error::new(DiskError::DiskOngoingReq),
DiskError::DriveIsRoot => Error::new(DiskError::DriveIsRoot),
DiskError::FaultyRemoteDisk => Error::new(DiskError::FaultyRemoteDisk),
DiskError::FaultyDisk => Error::new(DiskError::FaultyDisk),
DiskError::DiskAccessDenied => Error::new(DiskError::DiskAccessDenied),
DiskError::FileNotFound => Error::new(DiskError::FileNotFound),
DiskError::FileVersionNotFound => Error::new(DiskError::FileVersionNotFound),
DiskError::TooManyOpenFiles => Error::new(DiskError::TooManyOpenFiles),
DiskError::FileNameTooLong => Error::new(DiskError::FileNameTooLong),
DiskError::VolumeExists => Error::new(DiskError::VolumeExists),
DiskError::IsNotRegular => Error::new(DiskError::IsNotRegular),
DiskError::PathNotFound => Error::new(DiskError::PathNotFound),
DiskError::VolumeNotFound => Error::new(DiskError::VolumeNotFound),
DiskError::VolumeNotEmpty => Error::new(DiskError::VolumeNotEmpty),
DiskError::VolumeAccessDenied => Error::new(DiskError::VolumeAccessDenied),
DiskError::FileAccessDenied => Error::new(DiskError::FileAccessDenied),
DiskError::FileCorrupt => Error::new(DiskError::FileCorrupt),
DiskError::BitrotHashAlgoInvalid => Error::new(DiskError::BitrotHashAlgoInvalid),
DiskError::CrossDeviceLink => Error::new(DiskError::CrossDeviceLink),
DiskError::LessData => Error::new(DiskError::LessData),
DiskError::MoreData => Error::new(DiskError::MoreData),
DiskError::OutdatedXLMeta => Error::new(DiskError::OutdatedXLMeta),
DiskError::PartMissingOrCorrupt => Error::new(DiskError::PartMissingOrCorrupt),
DiskError::NoHealRequired => Error::new(DiskError::NoHealRequired),
}
}
pub fn os_err_to_file_err(e: io::Error) -> Error {
match e.kind() {
io::ErrorKind::NotFound => Error::new(DiskError::FileNotFound),
io::ErrorKind::PermissionDenied => Error::new(DiskError::FileAccessDenied),
// io::ErrorKind::ConnectionRefused => todo!(),
// io::ErrorKind::ConnectionReset => todo!(),
// io::ErrorKind::HostUnreachable => todo!(),
// io::ErrorKind::NetworkUnreachable => todo!(),
// io::ErrorKind::ConnectionAborted => todo!(),
// io::ErrorKind::NotConnected => todo!(),
// io::ErrorKind::AddrInUse => todo!(),
// io::ErrorKind::AddrNotAvailable => todo!(),
// io::ErrorKind::NetworkDown => todo!(),
// io::ErrorKind::BrokenPipe => todo!(),
// io::ErrorKind::AlreadyExists => todo!(),
// io::ErrorKind::WouldBlock => todo!(),
// io::ErrorKind::NotADirectory => DiskError::FileNotFound,
// io::ErrorKind::IsADirectory => DiskError::FileNotFound,
// io::ErrorKind::DirectoryNotEmpty => DiskError::VolumeNotEmpty,
// io::ErrorKind::ReadOnlyFilesystem => todo!(),
// io::ErrorKind::FilesystemLoop => todo!(),
// io::ErrorKind::StaleNetworkFileHandle => todo!(),
// io::ErrorKind::InvalidInput => todo!(),
// io::ErrorKind::InvalidData => todo!(),
// io::ErrorKind::TimedOut => todo!(),
// io::ErrorKind::WriteZero => todo!(),
// io::ErrorKind::StorageFull => DiskError::DiskFull,
// io::ErrorKind::NotSeekable => todo!(),
// io::ErrorKind::FilesystemQuotaExceeded => todo!(),
// io::ErrorKind::FileTooLarge => todo!(),
// io::ErrorKind::ResourceBusy => todo!(),
// io::ErrorKind::ExecutableFileBusy => todo!(),
// io::ErrorKind::Deadlock => todo!(),
// io::ErrorKind::CrossesDevices => todo!(),
// io::ErrorKind::TooManyLinks =>DiskError::TooManyOpenFiles,
// io::ErrorKind::InvalidFilename => todo!(),
// io::ErrorKind::ArgumentListTooLong => todo!(),
// io::ErrorKind::Interrupted => todo!(),
// io::ErrorKind::Unsupported => todo!(),
// io::ErrorKind::UnexpectedEof => todo!(),
// io::ErrorKind::OutOfMemory => todo!(),
// io::ErrorKind::Other => todo!(),
// TODO: 把不支持的king用字符串处理
_ => Error::new(e),
}
}
#[derive(Debug, thiserror::Error)]
pub struct FileAccessDeniedWithContext {
pub path: PathBuf,
#[source]
pub source: std::io::Error,
}
impl std::fmt::Display for FileAccessDeniedWithContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "访问文件 '{}' 被拒绝: {}", self.path.display(), self.source)
}
}
pub fn is_unformatted_disk(err: &Error) -> bool {
matches!(err.downcast_ref::<DiskError>(), Some(DiskError::UnformattedDisk))
}
pub fn is_err_file_not_found(err: &Error) -> bool {
if let Some(ioerr) = err.downcast_ref::<io::Error>() {
return ioerr.kind() == ErrorKind::NotFound;
}
matches!(err.downcast_ref::<DiskError>(), Some(DiskError::FileNotFound))
}
pub fn is_err_file_version_not_found(err: &Error) -> bool {
matches!(err.downcast_ref::<DiskError>(), Some(DiskError::FileVersionNotFound))
}
pub fn is_err_volume_not_found(err: &Error) -> bool {
matches!(err.downcast_ref::<DiskError>(), Some(DiskError::VolumeNotFound))
}
pub fn is_err_eof(err: &Error) -> bool {
if let Some(ioerr) = err.downcast_ref::<io::Error>() {
return ioerr.kind() == ErrorKind::UnexpectedEof;
}
false
}
pub fn is_sys_err_no_space(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 28;
}
false
}
pub fn is_sys_err_invalid_arg(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 22;
}
false
}
// TODO: ??
pub fn is_sys_err_io(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 5;
}
false
}
pub fn is_sys_err_is_dir(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 21;
}
false
}
pub fn is_sys_err_not_dir(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 20;
}
false
}
pub fn is_sys_err_too_long(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 63;
}
false
}
pub fn is_sys_err_too_many_symlinks(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 62;
}
false
}
pub fn is_sys_err_not_empty(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
if no == 66 {
return true;
}
if cfg!(target_os = "solaris") && no == 17 {
return true;
}
if cfg!(target_os = "windows") && no == 145 {
return true;
}
}
false
}
pub fn is_sys_err_path_not_found(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
if cfg!(target_os = "windows") {
if no == 3 {
return true;
}
} else if no == 2 {
return true;
}
}
false
}
pub fn is_sys_err_handle_invalid(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
if cfg!(target_os = "windows") {
if no == 6 {
return true;
}
} else {
return false;
}
}
false
}
pub fn is_sys_err_cross_device(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 18;
}
false
}
pub fn is_sys_err_too_many_files(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 23 || no == 24;
}
false
}
// pub fn os_is_not_exist(e: &io::Error) -> bool {
// e.kind() == ErrorKind::NotFound
// }
pub fn os_is_permission(e: &io::Error) -> bool {
if e.kind() == ErrorKind::PermissionDenied {
return true;
}
if let Some(no) = e.raw_os_error() {
if no == 30 {
return true;
}
}
false
}
// pub fn os_is_exist(e: &io::Error) -> bool {
// e.kind() == ErrorKind::AlreadyExists
// }
// // map_err_not_exists
// pub fn map_err_not_exists(e: io::Error) -> Error {
// if os_is_not_exist(&e) {
// return Error::new(DiskError::VolumeNotEmpty);
// } else if is_sys_err_io(&e) {
// return Error::new(DiskError::FaultyDisk);
// }
// Error::new(e)
// }
// pub fn convert_access_error(e: io::Error, per_err: DiskError) -> Error {
// if os_is_not_exist(&e) {
// return Error::new(DiskError::VolumeNotEmpty);
// } else if is_sys_err_io(&e) {
// return Error::new(DiskError::FaultyDisk);
// } else if os_is_permission(&e) {
// return Error::new(per_err);
// }
// Error::new(e)
// }
pub fn is_all_not_found(errs: &[Option<Error>]) -> bool {
for err in errs.iter() {
if let Some(err) = err {
if let Some(err) = err.downcast_ref::<DiskError>() {
match err {
DiskError::FileNotFound | DiskError::VolumeNotFound | &DiskError::FileVersionNotFound => {
continue;
}
_ => return false,
}
}
}
return false;
}
!errs.is_empty()
}
pub fn is_all_volume_not_found(errs: &[Option<Error>]) -> bool {
DiskError::VolumeNotFound.count_errs(errs) == errs.len()
}
pub fn is_all_buckets_not_found(errs: &[Option<Error>]) -> bool {
if errs.is_empty() {
return false;
}
let mut not_found_count = 0;
for err in errs.iter().flatten() {
match err.downcast_ref() {
Some(DiskError::VolumeNotFound) | Some(DiskError::DiskNotFound) => {
not_found_count += 1;
}
_ => {}
}
}
errs.len() == not_found_count
}
pub fn is_err_os_not_exist(err: &Error) -> bool {
if let Some(os_err) = err.downcast_ref::<io::Error>() {
os_err.kind() == ErrorKind::NotFound
} else {
false
}
}
pub fn is_err_os_disk_full(err: &Error) -> bool {
if let Some(os_err) = err.downcast_ref::<io::Error>() {
is_sys_err_no_space(os_err)
} else if let Some(e) = err.downcast_ref::<DiskError>() {
e == &DiskError::DiskFull
} else {
false
}
}
crates/disk/src/format.rs
-273
View File
@@ -1,273 +0,0 @@
// use super::{error::DiskError, DiskInfo};
use rustfs_error::{Error, Result};
use serde::{Deserialize, Serialize};
use serde_json::Error as JsonError;
use uuid::Uuid;
use crate::api::DiskInfo;
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum FormatMetaVersion {
#[serde(rename = "1")]
V1,
#[serde(other)]
Unknown,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum FormatBackend {
#[serde(rename = "xl")]
Erasure,
#[serde(rename = "xl-single")]
ErasureSingle,
#[serde(other)]
Unknown,
}
/// Represents the V3 backend disk structure version
/// under `.rustfs.sys` and actual data namespace.
///
/// FormatErasureV3 - structure holds format config version '3'.
///
/// The V3 format to support "large bucket" support where a bucket
/// can span multiple erasure sets.
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct FormatErasureV3 {
/// Version of 'xl' format.
pub version: FormatErasureVersion,
/// This field carries assigned disk uuid.
pub this: Uuid,
/// Sets field carries the input disk order generated the first
/// time when fresh disks were supplied, it is a two dimensional
/// array second dimension represents list of disks used per set.
pub sets: Vec<Vec<Uuid>>,
/// Distribution algorithm represents the hashing algorithm
/// to pick the right set index for an object.
#[serde(rename = "distributionAlgo")]
pub distribution_algo: DistributionAlgoVersion,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum FormatErasureVersion {
#[serde(rename = "1")]
V1,
#[serde(rename = "2")]
V2,
#[serde(rename = "3")]
V3,
#[serde(other)]
Unknown,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum DistributionAlgoVersion {
#[serde(rename = "CRCMOD")]
V1,
#[serde(rename = "SIPMOD")]
V2,
#[serde(rename = "SIPMOD+PARITY")]
V3,
}
/// format.json currently has the format:
///
/// ```json
/// {
/// "version": "1",
/// "format": "XXXXX",
/// "id": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
/// "XXXXX": {
//
/// }
/// }
/// ```
///
/// Ideally we will never have a situation where we will have to change the
/// fields of this struct and deal with related migration.
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct FormatV3 {
/// Version of the format config.
pub version: FormatMetaVersion,
/// Format indicates the backend format type, supports two values 'xl' and 'xl-single'.
pub format: FormatBackend,
/// ID is the identifier for the rustfs deployment
pub id: Uuid,
#[serde(rename = "xl")]
pub erasure: FormatErasureV3,
// /// DiskInfo is an extended type which returns current
// /// disk usage per path.
#[serde(skip)]
pub disk_info: Option<DiskInfo>,
}
impl TryFrom<&[u8]> for FormatV3 {
type Error = JsonError;
fn try_from(data: &[u8]) -> core::result::Result<Self, JsonError> {
serde_json::from_slice(data)
}
}
impl TryFrom<&str> for FormatV3 {
type Error = JsonError;
fn try_from(data: &str) -> core::result::Result<Self, JsonError> {
serde_json::from_str(data)
}
}
impl FormatV3 {
/// Create a new format config with the given number of sets and set length.
pub fn new(num_sets: usize, set_len: usize) -> Self {
let format = if set_len == 1 {
FormatBackend::ErasureSingle
} else {
FormatBackend::Erasure
};
let erasure = FormatErasureV3 {
version: FormatErasureVersion::V3,
this: Uuid::nil(),
sets: (0..num_sets)
.map(|_| (0..set_len).map(|_| Uuid::new_v4()).collect())
.collect(),
distribution_algo: DistributionAlgoVersion::V3,
};
Self {
version: FormatMetaVersion::V1,
format,
id: Uuid::new_v4(),
erasure,
disk_info: None,
}
}
/// Returns the number of drives in the erasure set.
pub fn drives(&self) -> usize {
self.erasure.sets.iter().map(|v| v.len()).sum()
}
pub fn to_json(&self) -> Result<String> {
Ok(serde_json::to_string(self)?)
}
/// returns the i,j'th position of the input `diskID` against the reference
///
/// format, after successful validation.
/// - i'th position is the set index
/// - j'th position is the disk index in the current set
pub fn find_disk_index_by_disk_id(&self, disk_id: Uuid) -> Result<(usize, usize)> {
if disk_id == Uuid::nil() {
return Err(Error::DiskNotFound);
}
if disk_id == Uuid::max() {
return Err(Error::msg("disk offline"));
}
for (i, set) in self.erasure.sets.iter().enumerate() {
for (j, d) in set.iter().enumerate() {
if disk_id.eq(d) {
return Ok((i, j));
}
}
}
Err(Error::msg(format!("disk id not found {}", disk_id)))
}
pub fn check_other(&self, other: &FormatV3) -> Result<()> {
let mut tmp = other.clone();
let this = tmp.erasure.this;
tmp.erasure.this = Uuid::nil();
if self.erasure.sets.len() != other.erasure.sets.len() {
return Err(Error::msg(format!(
"Expected number of sets {}, got {}",
self.erasure.sets.len(),
other.erasure.sets.len()
)));
}
for i in 0..self.erasure.sets.len() {
if self.erasure.sets[i].len() != other.erasure.sets[i].len() {
return Err(Error::msg(format!(
"Each set should be of same size, expected {}, got {}",
self.erasure.sets[i].len(),
other.erasure.sets[i].len()
)));
}
for j in 0..self.erasure.sets[i].len() {
if self.erasure.sets[i][j] != other.erasure.sets[i][j] {
return Err(Error::msg(format!(
"UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)",
i,
j,
self.erasure.sets[i][j].to_string(),
other.erasure.sets[i][j].to_string(),
)));
}
}
}
for i in 0..tmp.erasure.sets.len() {
for j in 0..tmp.erasure.sets[i].len() {
if this == tmp.erasure.sets[i][j] {
return Ok(());
}
}
}
Err(Error::msg(format!(
"DriveID {:?} not found in any drive sets {:?}",
this, other.erasure.sets
)))
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_format_v1() {
let format = FormatV3::new(1, 4);
let str = serde_json::to_string(&format);
println!("{:?}", str);
let data = r#"
{
"version": "1",
"format": "xl",
"id": "321b3874-987d-4c15-8fa5-757c956b1243",
"xl": {
"version": "1",
"this": null,
"sets": [
[
"8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
"c26315da-05cf-4778-a9ea-b44ea09f58c5",
"fb87a891-18d3-44cf-a46f-bcc15093a038",
"356a925c-57b9-4313-88b3-053edf1104dc"
]
],
"distributionAlgo": "CRCMOD"
}
}"#;
let p = FormatV3::try_from(data);
println!("{:?}", p);
}
}
crates/disk/src/fs.rs
-179
View File
@@ -1,179 +0,0 @@
use std::{fs::Metadata, path::Path};
use tokio::{
fs::{self, File},
io,
};
#[cfg(not(windows))]
pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool {
use std::os::unix::fs::MetadataExt;
if f1.dev() != f2.dev() {
return false;
}
if f1.ino() != f2.ino() {
return false;
}
if f1.size() != f2.size() {
return false;
}
if f1.permissions() != f2.permissions() {
return false;
}
if f1.mtime() != f2.mtime() {
return false;
}
true
}
#[cfg(windows)]
pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool {
if f1.permissions() != f2.permissions() {
return false;
}
if f1.file_type() != f2.file_type() {
return false;
}
if f1.len() != f2.len() {
return false;
}
true
}
type FileMode = usize;
pub const O_RDONLY: FileMode = 0x00000;
pub const O_WRONLY: FileMode = 0x00001;
pub const O_RDWR: FileMode = 0x00002;
pub const O_CREATE: FileMode = 0x00040;
// pub const O_EXCL: FileMode = 0x00080;
// pub const O_NOCTTY: FileMode = 0x00100;
pub const O_TRUNC: FileMode = 0x00200;
// pub const O_NONBLOCK: FileMode = 0x00800;
pub const O_APPEND: FileMode = 0x00400;
// pub const O_SYNC: FileMode = 0x01000;
// pub const O_ASYNC: FileMode = 0x02000;
// pub const O_CLOEXEC: FileMode = 0x80000;
// read: bool,
// write: bool,
// append: bool,
// truncate: bool,
// create: bool,
// create_new: bool,
pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
let mut opts = fs::OpenOptions::new();
match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
O_RDONLY => {
opts.read(true);
}
O_WRONLY => {
opts.write(true);
}
O_RDWR => {
opts.read(true);
opts.write(true);
}
_ => (),
};
if mode & O_CREATE != 0 {
opts.create(true);
}
if mode & O_APPEND != 0 {
opts.append(true);
}
if mode & O_TRUNC != 0 {
opts.truncate(true);
}
opts.open(path.as_ref()).await
}
pub async fn access(path: impl AsRef<Path>) -> io::Result<()> {
fs::metadata(path).await?;
Ok(())
}
pub fn access_std(path: impl AsRef<Path>) -> io::Result<()> {
std::fs::metadata(path)?;
Ok(())
}
pub async fn lstat(path: impl AsRef<Path>) -> io::Result<Metadata> {
fs::metadata(path).await
}
pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
std::fs::metadata(path)
}
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
fs::create_dir_all(path.as_ref()).await
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
let meta = fs::metadata(path.as_ref()).await?;
if meta.is_dir() {
fs::remove_dir(path.as_ref()).await
} else {
fs::remove_file(path.as_ref()).await
}
}
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
let meta = fs::metadata(path.as_ref()).await?;
if meta.is_dir() {
fs::remove_dir_all(path.as_ref()).await
} else {
fs::remove_file(path.as_ref()).await
}
}
#[tracing::instrument(level = "debug", skip_all)]
pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
let meta = std::fs::metadata(path.as_ref())?;
if meta.is_dir() {
std::fs::remove_dir(path.as_ref())
} else {
std::fs::remove_file(path.as_ref())
}
}
pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
let meta = std::fs::metadata(path.as_ref())?;
if meta.is_dir() {
std::fs::remove_dir_all(path.as_ref())
} else {
std::fs::remove_file(path.as_ref())
}
}
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
fs::create_dir(path.as_ref()).await
}
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
fs::rename(from, to).await
}
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
std::fs::rename(from, to)
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
fs::read(path.as_ref()).await
}
crates/disk/src/lib.rs
-12
View File
@@ -1,12 +0,0 @@
pub mod endpoint;
// pub mod error;
pub mod format;
pub mod fs;
pub mod local;
// pub mod metacache;
pub mod api;
pub mod local_list;
pub mod os;
pub mod path;
pub mod remote;
pub mod utils;
crates/disk/src/local.rs
-2048
View File
File diff suppressed because it is too large Load Diff
crates/disk/src/local_bak.rs
-2364
View File
File diff suppressed because it is too large Load Diff
crates/disk/src/local_list.rs
-535
View File
@@ -1,535 +0,0 @@
use crate::{
api::{DiskAPI, DiskStore, WalkDirOptions, STORAGE_FORMAT_FILE},
local::LocalDisk,
os::is_empty_dir,
path::{self, decode_dir_object, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR},
};
use futures::future::join_all;
use rustfs_error::{Error, Result};
use rustfs_metacache::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, MetacacheWriter};
use std::{collections::HashSet, future::Future, pin::Pin, sync::Arc};
use tokio::{io::AsyncWrite, spawn, sync::broadcast::Receiver as B_Receiver};
use tracing::{error, info, warn};
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
pub type PartialFn = Box<dyn Fn(MetaCacheEntries, &[Option<Error>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
type FinishedFn = Box<dyn Fn(&[Option<Error>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
#[derive(Default)]
pub struct ListPathRawOptions {
pub disks: Vec<Option<DiskStore>>,
pub fallback_disks: Vec<Option<DiskStore>>,
pub bucket: String,
pub path: String,
pub recursive: bool,
pub filter_prefix: Option<String>,
pub forward_to: Option<String>,
pub min_disks: usize,
pub report_not_found: bool,
pub per_disk_limit: i32,
pub agreed: Option<AgreedFn>,
pub partial: Option<PartialFn>,
pub finished: Option<FinishedFn>,
}
impl Clone for ListPathRawOptions {
fn clone(&self) -> Self {
Self {
disks: self.disks.clone(),
fallback_disks: self.fallback_disks.clone(),
bucket: self.bucket.clone(),
path: self.path.clone(),
recursive: self.recursive,
filter_prefix: self.filter_prefix.clone(),
forward_to: self.forward_to.clone(),
min_disks: self.min_disks,
report_not_found: self.report_not_found,
per_disk_limit: self.per_disk_limit,
..Default::default()
}
}
}
pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -> Result<()> {
if opts.disks.is_empty() {
return Err(Error::msg("list_path_raw: 0 drives provided"));
}
let mut jobs: Vec<tokio::task::JoinHandle<std::result::Result<(), Error>>> = Vec::new();
let mut readers = Vec::with_capacity(opts.disks.len());
let fds = Arc::new(opts.fallback_disks.clone());
for disk in opts.disks.iter() {
let opdisk = disk.clone();
let opts_clone = opts.clone();
let fds_clone = fds.clone();
let (rd, mut wr) = tokio::io::duplex(64);
readers.push(MetacacheReader::new(rd));
jobs.push(spawn(async move {
let walk_opts = WalkDirOptions {
bucket: opts_clone.bucket.clone(),
base_dir: opts_clone.path.clone(),
recursive: opts_clone.recursive,
report_notfound: opts_clone.report_not_found,
filter_prefix: opts_clone.filter_prefix.clone(),
forward_to: opts_clone.forward_to.clone(),
limit: opts_clone.per_disk_limit,
..Default::default()
};
let mut need_fallback = false;
if let Some(disk) = opdisk {
match disk.walk_dir(walk_opts, &mut wr).await {
Ok(_res) => {}
Err(err) => {
error!("walk dir err {:?}", &err);
need_fallback = true;
}
}
} else {
need_fallback = true;
}
while need_fallback {
let disk = match fds_clone.iter().find(|d| d.is_some()) {
Some(d) => {
if let Some(disk) = d.clone() {
disk
} else {
break;
}
}
None => break,
};
match disk
.walk_dir(
WalkDirOptions {
bucket: opts_clone.bucket.clone(),
base_dir: opts_clone.path.clone(),
recursive: opts_clone.recursive,
report_notfound: opts_clone.report_not_found,
filter_prefix: opts_clone.filter_prefix.clone(),
forward_to: opts_clone.forward_to.clone(),
limit: opts_clone.per_disk_limit,
..Default::default()
},
&mut wr,
)
.await
{
Ok(_r) => {
need_fallback = false;
}
Err(err) => {
error!("walk dir2 err {:?}", &err);
break;
}
}
}
Ok(())
}));
}
let revjob = spawn(async move {
let mut errs: Vec<Option<Error>> = Vec::with_capacity(readers.len());
for _ in 0..readers.len() {
errs.push(None);
}
loop {
let mut current = MetaCacheEntry::default();
if rx.try_recv().is_ok() {
return Err(Error::msg("canceled"));
}
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
let mut at_eof = 0;
let mut fnf = 0;
let mut vnf = 0;
let mut has_err = 0;
let mut agree = 0;
for (i, r) in readers.iter_mut().enumerate() {
if errs[i].is_some() {
has_err += 1;
continue;
}
let entry = match r.peek().await {
Ok(res) => {
if let Some(entry) = res {
entry
} else {
at_eof += 1;
continue;
}
}
Err(err) => {
if err == Error::FaultyDisk {
at_eof += 1;
continue;
} else if err == Error::FileNotFound {
at_eof += 1;
fnf += 1;
continue;
} else if err == Error::VolumeNotFound {
at_eof += 1;
fnf += 1;
vnf += 1;
continue;
} else {
has_err += 1;
errs[i] = Some(err);
continue;
}
}
};
// If no current, add it.
if current.name.is_empty() {
top_entries[i] = Some(entry.clone());
current = entry;
agree += 1;
continue;
}
// If exact match, we agree.
if let (_, true) = current.matches(Some(&entry), true) {
top_entries[i] = Some(entry);
agree += 1;
continue;
}
// If only the name matches we didn't agree, but add it for resolution.
if entry.name == current.name {
top_entries[i] = Some(entry);
continue;
}
// We got different entries
if entry.name > current.name {
continue;
}
for item in top_entries.iter_mut().take(i) {
*item = None;
}
agree = 1;
top_entries[i] = Some(entry.clone());
current = entry;
}
if vnf > 0 && vnf >= (readers.len() - opts.min_disks) {
return Err(Error::VolumeNotFound);
}
if fnf > 0 && fnf >= (readers.len() - opts.min_disks) {
return Err(Error::FileNotFound);
}
if has_err > 0 && has_err > opts.disks.len() - opts.min_disks {
if let Some(finished_fn) = opts.finished.as_ref() {
finished_fn(&errs).await;
}
let mut combined_err = Vec::new();
errs.iter().zip(opts.disks.iter()).for_each(|(err, disk)| match (err, disk) {
(Some(err), Some(disk)) => {
combined_err.push(format!("drive {} returned: {}", disk.to_string(), err));
}
(Some(err), None) => {
combined_err.push(err.to_string());
}
_ => {}
});
return Err(Error::msg(combined_err.join(", ")));
}
// Break if all at EOF or error.
if at_eof + has_err == readers.len() {
if has_err > 0 {
if let Some(finished_fn) = opts.finished.as_ref() {
finished_fn(&errs).await;
}
}
break;
}
if agree == readers.len() {
for r in readers.iter_mut() {
let _ = r.skip(1).await;
}
if let Some(agreed_fn) = opts.agreed.as_ref() {
agreed_fn(current).await;
}
continue;
}
for (i, r) in readers.iter_mut().enumerate() {
if top_entries[i].is_some() {
let _ = r.skip(1).await;
}
}
if let Some(partial_fn) = opts.partial.as_ref() {
partial_fn(MetaCacheEntries(top_entries), &errs).await;
}
}
Ok(())
});
jobs.push(revjob);
let results = join_all(jobs).await;
for result in results {
if let Err(err) = result {
error!("list_path_raw err {:?}", err);
}
}
Ok(())
}
impl LocalDisk {
pub(crate) async fn scan_dir<W: AsyncWrite + Unpin>(
&self,
current: &mut String,
opts: &WalkDirOptions,
out: &mut MetacacheWriter<W>,
objs_returned: &mut i32,
) -> Result<()> {
let forward = {
opts.forward_to.as_ref().filter(|v| v.starts_with(&*current)).map(|v| {
let forward = v.trim_start_matches(&*current);
if let Some(idx) = forward.find('/') {
forward[..idx].to_owned()
} else {
forward.to_owned()
}
})
// if let Some(forward_to) = &opts.forward_to {
// } else {
// None
// }
// if !opts.forward_to.is_empty() && opts.forward_to.starts_with(&*current) {
// let forward = opts.forward_to.trim_start_matches(&*current);
// if let Some(idx) = forward.find('/') {
// &forward[..idx]
// } else {
// forward
// }
// } else {
// ""
// }
};
if opts.limit > 0 && *objs_returned >= opts.limit {
return Ok(());
}
let mut entries = match self.list_dir("", &opts.bucket, current, -1).await {
Ok(res) => res,
Err(e) => {
if e != Error::VolumeNotFound && e != Error::FileNotFound {
info!("scan list_dir {}, err {:?}", &current, &e);
}
if opts.report_notfound && (e == Error::VolumeNotFound || e == Error::FileNotFound) && current == &opts.base_dir {
return Err(Error::FileNotFound);
}
return Ok(());
}
};
if entries.is_empty() {
return Ok(());
}
let s = SLASH_SEPARATOR.chars().next().unwrap_or_default();
*current = current.trim_matches(s).to_owned();
let bucket = opts.bucket.as_str();
let mut dir_objes = HashSet::new();
// 第一层过滤
for item in entries.iter_mut() {
let entry = item.clone();
// check limit
if opts.limit > 0 && *objs_returned >= opts.limit {
return Ok(());
}
// check prefix
if let Some(filter_prefix) = &opts.filter_prefix {
if !entry.starts_with(filter_prefix) {
*item = "".to_owned();
continue;
}
}
if let Some(forward) = &forward {
if &entry < forward {
*item = "".to_owned();
continue;
}
}
if entry.ends_with(SLASH_SEPARATOR) {
if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) {
let entry = format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR);
dir_objes.insert(entry.clone());
*item = entry;
continue;
}
*item = entry.trim_end_matches(SLASH_SEPARATOR).to_owned();
continue;
}
*item = "".to_owned();
if entry.ends_with(STORAGE_FORMAT_FILE) {
//
let metadata = self
.read_metadata(self.get_object_path(bucket, format!("{}/{}", &current, &entry).as_str())?)
.await?;
// 用 strip_suffix 只删除一次
let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned();
let name = entry.trim_end_matches(SLASH_SEPARATOR);
let name = decode_dir_object(format!("{}/{}", &current, &name).as_str());
out.write_obj(&MetaCacheEntry {
name,
metadata,
..Default::default()
})
.await?;
*objs_returned += 1;
return Ok(());
}
}
entries.sort();
let mut entries = entries.as_slice();
if let Some(forward) = &forward {
for (i, entry) in entries.iter().enumerate() {
if entry >= forward || forward.starts_with(entry.as_str()) {
entries = &entries[i..];
break;
}
}
}
let mut dir_stack: Vec<String> = Vec::with_capacity(5);
for entry in entries.iter() {
if opts.limit > 0 && *objs_returned >= opts.limit {
return Ok(());
}
if entry.is_empty() {
continue;
}
let name = path::path_join_buf(&[current, entry]);
if !dir_stack.is_empty() {
if let Some(pop) = dir_stack.pop() {
if pop < name {
//
out.write_obj(&MetaCacheEntry {
name: pop.clone(),
..Default::default()
})
.await?;
if opts.recursive {
let mut opts = opts.clone();
opts.filter_prefix = None;
if let Err(er) = Box::pin(self.scan_dir(&mut pop.clone(), &opts, out, objs_returned)).await {
error!("scan_dir err {:?}", er);
}
}
}
}
}
let mut meta = MetaCacheEntry {
name,
..Default::default()
};
let mut is_dir_obj = false;
if let Some(_dir) = dir_objes.get(entry) {
is_dir_obj = true;
meta.name
.truncate(meta.name.len() - meta.name.chars().last().unwrap().len_utf8());
meta.name.push_str(GLOBAL_DIR_SUFFIX_WITH_SLASH);
}
let fname = format!("{}/{}", &meta.name, STORAGE_FORMAT_FILE);
match self.read_metadata(self.get_object_path(&opts.bucket, fname.as_str())?).await {
Ok(res) => {
if is_dir_obj {
meta.name = meta.name.trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH).to_owned();
meta.name.push_str(SLASH_SEPARATOR);
}
meta.metadata = res;
out.write_obj(&meta).await?;
*objs_returned += 1;
}
Err(err) => {
if err == Error::FileNotFound || err == Error::IsNotRegular {
// NOT an object, append to stack (with slash)
// If dirObject, but no metadata (which is unexpected) we skip it.
if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await {
meta.name.push_str(SLASH_SEPARATOR);
dir_stack.push(meta.name);
}
}
continue;
}
};
}
while let Some(dir) = dir_stack.pop() {
if opts.limit > 0 && *objs_returned >= opts.limit {
return Ok(());
}
out.write_obj(&MetaCacheEntry {
name: dir.clone(),
..Default::default()
})
.await?;
*objs_returned += 1;
if opts.recursive {
let mut dir = dir;
let mut opts = opts.clone();
opts.filter_prefix = None;
if let Err(er) = Box::pin(self.scan_dir(&mut dir, &opts, out, objs_returned)).await {
warn!("scan_dir err {:?}", &er);
}
}
}
Ok(())
}
}
crates/disk/src/metacache.rs
-608
View File
@@ -1,608 +0,0 @@
use crate::bucket::metadata_sys::get_versioning_config;
use crate::bucket::versioning::VersioningApi;
use crate::store_api::ObjectInfo;
use crate::utils::path::SLASH_SEPARATOR;
use rustfs_error::{Error, Result};
use rustfs_filemeta::merge_file_meta_versions;
use rustfs_filemeta::FileInfo;
use rustfs_filemeta::FileInfoVersions;
use rustfs_filemeta::FileMeta;
use rustfs_filemeta::FileMetaShallowVersion;
use rustfs_filemeta::VersionType;
use serde::Deserialize;
use serde::Serialize;
use std::cmp::Ordering;
use time::OffsetDateTime;
use tracing::warn;
#[derive(Clone, Debug, Default)]
pub struct MetadataResolutionParams {
pub dir_quorum: usize,
pub obj_quorum: usize,
pub requested_versions: usize,
pub bucket: String,
pub strict: bool,
pub candidates: Vec<Vec<FileMetaShallowVersion>>,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
pub struct MetaCacheEntry {
// name is the full name of the object including prefixes
pub name: String,
// Metadata. If none is present it is not an object but only a prefix.
// Entries without metadata will only be present in non-recursive scans.
pub metadata: Vec<u8>,
// cached contains the metadata if decoded.
pub cached: Option<FileMeta>,
// Indicates the entry can be reused and only one reference to metadata is expected.
pub reusable: bool,
}
impl MetaCacheEntry {
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut wr = Vec::new();
rmp::encode::write_bool(&mut wr, true)?;
rmp::encode::write_str(&mut wr, &self.name)?;
rmp::encode::write_bin(&mut wr, &self.metadata)?;
Ok(wr)
}
pub fn is_dir(&self) -> bool {
self.metadata.is_empty() && self.name.ends_with('/')
}
pub fn is_in_dir(&self, dir: &str, separator: &str) -> bool {
if dir.is_empty() {
let idx = self.name.find(separator);
return idx.is_none() || idx.unwrap() == self.name.len() - separator.len();
}
let ext = self.name.trim_start_matches(dir);
if ext.len() != self.name.len() {
let idx = ext.find(separator);
return idx.is_none() || idx.unwrap() == ext.len() - separator.len();
}
false
}
pub fn is_object(&self) -> bool {
!self.metadata.is_empty()
}
pub fn is_object_dir(&self) -> bool {
!self.metadata.is_empty() && self.name.ends_with(SLASH_SEPARATOR)
}
pub fn is_latest_delete_marker(&mut self) -> bool {
if let Some(cached) = &self.cached {
if cached.versions.is_empty() {
return true;
}
return cached.versions[0].header.version_type == VersionType::Delete;
}
if !FileMeta::is_xl2_v1_format(&self.metadata) {
return false;
}
match FileMeta::check_xl2_v1(&self.metadata) {
Ok((meta, _, _)) => {
if !meta.is_empty() {
return FileMeta::is_latest_delete_marker(meta);
}
}
Err(_) => return true,
}
match self.xl_meta() {
Ok(res) => {
if res.versions.is_empty() {
return true;
}
res.versions[0].header.version_type == VersionType::Delete
}
Err(_) => true,
}
}
#[tracing::instrument(level = "debug", skip(self))]
pub fn to_fileinfo(&self, bucket: &str) -> Result<FileInfo> {
if self.is_dir() {
return Ok(FileInfo {
volume: bucket.to_owned(),
name: self.name.clone(),
..Default::default()
});
}
if self.cached.is_some() {
let fm = self.cached.as_ref().unwrap();
if fm.versions.is_empty() {
return Ok(FileInfo {
volume: bucket.to_owned(),
name: self.name.clone(),
deleted: true,
is_latest: true,
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
});
}
let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?;
return Ok(fi);
}
let mut fm = FileMeta::new();
fm.unmarshal_msg(&self.metadata)?;
let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?;
return Ok(fi);
}
pub fn file_info_versions(&self, bucket: &str) -> Result<FileInfoVersions> {
if self.is_dir() {
return Ok(FileInfoVersions {
volume: bucket.to_string(),
name: self.name.clone(),
versions: vec![FileInfo {
volume: bucket.to_string(),
name: self.name.clone(),
..Default::default()
}],
..Default::default()
});
}
let mut fm = FileMeta::new();
fm.unmarshal_msg(&self.metadata)?;
fm.into_file_info_versions(bucket, self.name.as_str(), false)
}
pub fn matches(&self, other: Option<&MetaCacheEntry>, strict: bool) -> (Option<MetaCacheEntry>, bool) {
if other.is_none() {
return (None, false);
}
let other = other.unwrap();
let mut prefer = None;
if self.name != other.name {
if self.name < other.name {
return (Some(self.clone()), false);
}
return (Some(other.clone()), false);
}
if other.is_dir() || self.is_dir() {
if self.is_dir() {
return (Some(self.clone()), other.is_dir() == self.is_dir());
}
return (Some(other.clone()), other.is_dir() == self.is_dir());
}
let self_vers = match &self.cached {
Some(file_meta) => file_meta.clone(),
None => match FileMeta::load(&self.metadata) {
Ok(meta) => meta,
Err(_) => {
return (None, false);
}
},
};
let other_vers = match &other.cached {
Some(file_meta) => file_meta.clone(),
None => match FileMeta::load(&other.metadata) {
Ok(meta) => meta,
Err(_) => {
return (None, false);
}
},
};
if self_vers.versions.len() != other_vers.versions.len() {
match self_vers.lastest_mod_time().cmp(&other_vers.lastest_mod_time()) {
Ordering::Greater => {
return (Some(self.clone()), false);
}
Ordering::Less => {
return (Some(other.clone()), false);
}
_ => {}
}
if self_vers.versions.len() > other_vers.versions.len() {
return (Some(self.clone()), false);
}
return (Some(other.clone()), false);
}
for (s_version, o_version) in self_vers.versions.iter().zip(other_vers.versions.iter()) {
if s_version.header != o_version.header {
if s_version.header.has_ec() != o_version.header.has_ec() {
// One version has EC and the other doesn't - may have been written later.
// Compare without considering EC.
let (mut a, mut b) = (s_version.header.clone(), o_version.header.clone());
(a.ec_n, a.ec_m, b.ec_n, b.ec_m) = (0, 0, 0, 0);
if a == b {
continue;
}
}
if !strict && s_version.header.matches_not_strict(&o_version.header) {
if prefer.is_none() {
if s_version.header.sorts_before(&o_version.header) {
prefer = Some(self.clone());
} else {
prefer = Some(other.clone());
}
}
continue;
}
if prefer.is_some() {
return (prefer, false);
}
if s_version.header.sorts_before(&o_version.header) {
return (Some(self.clone()), false);
}
return (Some(other.clone()), false);
}
}
if prefer.is_none() {
prefer = Some(self.clone());
}
(prefer, true)
}
pub fn xl_meta(&mut self) -> Result<FileMeta> {
if self.is_dir() {
return Err(Error::FileNotFound);
}
if let Some(meta) = &self.cached {
Ok(meta.clone())
} else {
if self.metadata.is_empty() {
return Err(Error::FileNotFound);
}
let meta = FileMeta::load(&self.metadata)?;
self.cached = Some(meta.clone());
Ok(meta)
}
}
}
#[derive(Debug, Default)]
pub struct MetaCacheEntries(pub Vec<Option<MetaCacheEntry>>);
impl MetaCacheEntries {
#[allow(clippy::should_implement_trait)]
pub fn as_ref(&self) -> &[Option<MetaCacheEntry>] {
&self.0
}
pub fn resolve(&self, mut params: MetadataResolutionParams) -> Option<MetaCacheEntry> {
if self.0.is_empty() {
warn!("decommission_pool: entries resolve empty");
return None;
}
let mut dir_exists = 0;
let mut selected = None;
params.candidates.clear();
let mut objs_agree = 0;
let mut objs_valid = 0;
for entry in self.0.iter().flatten() {
let mut entry = entry.clone();
warn!("decommission_pool: entries resolve entry {:?}", entry.name);
if entry.name.is_empty() {
continue;
}
if entry.is_dir() {
dir_exists += 1;
selected = Some(entry.clone());
warn!("decommission_pool: entries resolve entry dir {:?}", entry.name);
continue;
}
let xl = match entry.xl_meta() {
Ok(xl) => xl,
Err(e) => {
warn!("decommission_pool: entries resolve entry xl_meta {:?}", e);
continue;
}
};
objs_valid += 1;
params.candidates.push(xl.versions.clone());
if selected.is_none() {
selected = Some(entry.clone());
objs_agree = 1;
warn!("decommission_pool: entries resolve entry selected {:?}", entry.name);
continue;
}
if let (prefer, true) = entry.matches(selected.as_ref(), params.strict) {
selected = prefer;
objs_agree += 1;
warn!("decommission_pool: entries resolve entry prefer {:?}", entry.name);
continue;
}
}
let Some(selected) = selected else {
warn!("decommission_pool: entries resolve entry no selected");
return None;
};
if selected.is_dir() && dir_exists >= params.dir_quorum {
warn!("decommission_pool: entries resolve entry dir selected {:?}", selected.name);
return Some(selected);
}
// If we would never be able to reach read quorum.
if objs_valid < params.obj_quorum {
warn!(
"decommission_pool: entries resolve entry not enough objects {} < {}",
objs_valid, params.obj_quorum
);
return None;
}
if objs_agree == objs_valid {
warn!("decommission_pool: entries resolve entry all agree {} == {}", objs_agree, objs_valid);
return Some(selected);
}
let Some(cached) = selected.cached else {
warn!("decommission_pool: entries resolve entry no cached");
return None;
};
let versions = merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, &params.candidates);
if versions.is_empty() {
warn!("decommission_pool: entries resolve entry no versions");
return None;
}
let metadata = match cached.marshal_msg() {
Ok(meta) => meta,
Err(e) => {
warn!("decommission_pool: entries resolve entry marshal_msg {:?}", e);
return None;
}
};
// Merge if we have disagreement.
// Create a new merged result.
let new_selected = MetaCacheEntry {
name: selected.name.clone(),
cached: Some(FileMeta {
meta_ver: cached.meta_ver,
versions,
..Default::default()
}),
reusable: true,
metadata,
};
warn!("decommission_pool: entries resolve entry selected {:?}", new_selected.name);
Some(new_selected)
}
pub fn first_found(&self) -> (Option<MetaCacheEntry>, usize) {
(self.0.iter().find(|x| x.is_some()).cloned().unwrap_or_default(), self.0.len())
}
}
#[derive(Debug, Default)]
pub struct MetaCacheEntriesSortedResult {
pub entries: Option<MetaCacheEntriesSorted>,
pub err: Option<Error>,
}
// impl MetaCacheEntriesSortedResult {
// pub fn entriy_list(&self) -> Vec<&MetaCacheEntry> {
// if let Some(entries) = &self.entries {
// entries.entries()
// } else {
// Vec::new()
// }
// }
// }
#[derive(Debug, Default)]
pub struct MetaCacheEntriesSorted {
pub o: MetaCacheEntries,
pub list_id: Option<String>,
pub reuse: bool,
pub last_skipped_entry: Option<String>,
}
impl MetaCacheEntriesSorted {
pub fn entries(&self) -> Vec<&MetaCacheEntry> {
let entries: Vec<&MetaCacheEntry> = self.o.0.iter().flatten().collect();
entries
}
pub fn forward_past(&mut self, marker: Option<String>) {
if let Some(val) = marker {
// TODO: reuse
if let Some(idx) = self.o.0.iter().flatten().position(|v| v.name > val) {
self.o.0 = self.o.0.split_off(idx);
}
}
}
pub async fn file_infos(&self, bucket: &str, prefix: &str, delimiter: Option<String>) -> Vec<ObjectInfo> {
let vcfg = get_versioning_config(bucket).await.ok();
let mut objects = Vec::with_capacity(self.o.as_ref().len());
let mut prev_prefix = "";
for entry in self.o.as_ref().iter().flatten() {
if entry.is_object() {
if let Some(delimiter) = &delimiter {
if let Some(idx) = entry.name.trim_start_matches(prefix).find(delimiter) {
let idx = prefix.len() + idx + delimiter.len();
if let Some(curr_prefix) = entry.name.get(0..idx) {
if curr_prefix == prev_prefix {
continue;
}
prev_prefix = curr_prefix;
objects.push(ObjectInfo {
is_dir: true,
bucket: bucket.to_owned(),
name: curr_prefix.to_owned(),
..Default::default()
});
}
continue;
}
}
if let Ok(fi) = entry.to_fileinfo(bucket) {
// TODO:VersionPurgeStatus
let versioned = vcfg.clone().map(|v| v.0.versioned(&entry.name)).unwrap_or_default();
objects.push(ObjectInfo::from_file_info(&fi, bucket, &entry.name, versioned));
}
continue;
}
if entry.is_dir() {
if let Some(delimiter) = &delimiter {
if let Some(idx) = entry.name.trim_start_matches(prefix).find(delimiter) {
let idx = prefix.len() + idx + delimiter.len();
if let Some(curr_prefix) = entry.name.get(0..idx) {
if curr_prefix == prev_prefix {
continue;
}
prev_prefix = curr_prefix;
objects.push(ObjectInfo {
is_dir: true,
bucket: bucket.to_owned(),
name: curr_prefix.to_owned(),
..Default::default()
});
}
}
}
}
}
objects
}
pub async fn file_info_versions(
&self,
bucket: &str,
prefix: &str,
delimiter: Option<String>,
after_v: Option<String>,
) -> Vec<ObjectInfo> {
let vcfg = get_versioning_config(bucket).await.ok();
let mut objects = Vec::with_capacity(self.o.as_ref().len());
let mut prev_prefix = "";
let mut after_v = after_v;
for entry in self.o.as_ref().iter().flatten() {
if entry.is_object() {
if let Some(delimiter) = &delimiter {
if let Some(idx) = entry.name.trim_start_matches(prefix).find(delimiter) {
let idx = prefix.len() + idx + delimiter.len();
if let Some(curr_prefix) = entry.name.get(0..idx) {
if curr_prefix == prev_prefix {
continue;
}
prev_prefix = curr_prefix;
objects.push(ObjectInfo {
is_dir: true,
bucket: bucket.to_owned(),
name: curr_prefix.to_owned(),
..Default::default()
});
}
continue;
}
}
let mut fiv = match entry.file_info_versions(bucket) {
Ok(res) => res,
Err(_err) => {
//
continue;
}
};
let fi_versions = 'c: {
if let Some(after_val) = &after_v {
if let Some(idx) = fiv.find_version_index(after_val) {
after_v = None;
break 'c fiv.versions.split_off(idx + 1);
}
after_v = None;
break 'c fiv.versions;
} else {
break 'c fiv.versions;
}
};
for fi in fi_versions.into_iter() {
// VersionPurgeStatus
let versioned = vcfg.clone().map(|v| v.0.versioned(&entry.name)).unwrap_or_default();
objects.push(ObjectInfo::from_file_info(&fi, bucket, &entry.name, versioned));
}
continue;
}
if entry.is_dir() {
if let Some(delimiter) = &delimiter {
if let Some(idx) = entry.name.trim_start_matches(prefix).find(delimiter) {
let idx = prefix.len() + idx + delimiter.len();
if let Some(curr_prefix) = entry.name.get(0..idx) {
if curr_prefix == prev_prefix {
continue;
}
prev_prefix = curr_prefix;
objects.push(ObjectInfo {
is_dir: true,
bucket: bucket.to_owned(),
name: curr_prefix.to_owned(),
..Default::default()
});
}
}
}
}
}
objects
}
}
crates/disk/src/os.rs
-206
View File
@@ -1,206 +0,0 @@
use super::fs;
use rustfs_error::{to_access_error, Error, Result};
use rustfs_utils::os::same_disk;
use std::{
io,
path::{Component, Path},
};
pub fn check_path_length(path_name: &str) -> Result<()> {
// Apple OS X path length is limited to 1016
if cfg!(target_os = "macos") && path_name.len() > 1016 {
return Err(Error::FileNameTooLong);
}
// Disallow more than 1024 characters on windows, there
// are no known name_max limits on Windows.
if cfg!(target_os = "windows") && path_name.len() > 1024 {
return Err(Error::FileNameTooLong);
}
// On Unix we reject paths if they are just '.', '..' or '/'
let invalid_paths = [".", "..", "/"];
if invalid_paths.contains(&path_name) {
return Err(Error::FileAccessDenied);
}
// Check each path segment length is > 255 on all Unix
// platforms, look for this value as NAME_MAX in
// /usr/include/linux/limits.h
let mut count = 0usize;
for c in path_name.chars() {
match c {
'/' | '\\' if cfg!(target_os = "windows") => count = 0, // Reset
_ => {
count += 1;
if count > 255 {
return Err(Error::FileNameTooLong);
}
}
}
}
// Success.
Ok(())
}
pub fn is_root_disk(disk_path: &str, root_disk: &str) -> Result<bool> {
if cfg!(target_os = "windows") {
return Ok(false);
}
Ok(same_disk(disk_path, root_disk)?)
}
pub async fn make_dir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> Result<()> {
check_path_length(path.as_ref().to_string_lossy().to_string().as_str())?;
if let Err(e) = reliable_mkdir_all(path.as_ref(), base_dir.as_ref()).await {
return Err(to_access_error(e, Error::FileAccessDenied).into());
}
Ok(())
}
pub async fn is_empty_dir(path: impl AsRef<Path>) -> bool {
read_dir(path.as_ref(), 1).await.is_ok_and(|v| v.is_empty())
}
// read_dir count read limit. when count == 0 unlimit.
pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> std::io::Result<Vec<String>> {
let mut entries = tokio::fs::read_dir(path.as_ref()).await?;
let mut volumes = Vec::new();
let mut count = count;
while let Some(entry) = entries.next_entry().await? {
let name = entry.file_name().to_string_lossy().to_string();
if name.is_empty() || name == "." || name == ".." {
continue;
}
let file_type = entry.file_type().await?;
if file_type.is_file() {
volumes.push(name);
} else if file_type.is_dir() {
volumes.push(format!("{}{}", name, super::path::SLASH_SEPARATOR));
}
count -= 1;
if count == 0 {
break;
}
}
Ok(volumes)
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn rename_all(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
base_dir: impl AsRef<Path>,
) -> Result<()> {
reliable_rename(src_file_path, dst_file_path.as_ref(), base_dir)
.await
.map_err(|e| to_access_error(e, Error::FileAccessDenied))?;
Ok(())
}
pub async fn reliable_rename(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
base_dir: impl AsRef<Path>,
) -> io::Result<()> {
if let Some(parent) = dst_file_path.as_ref().parent() {
if !file_exists(parent) {
// info!("reliable_rename reliable_mkdir_all parent: {:?}", parent);
reliable_mkdir_all(parent, base_dir.as_ref()).await?;
}
}
let mut i = 0;
loop {
if let Err(e) = fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
if e.kind() == std::io::ErrorKind::NotFound && i == 0 {
i += 1;
continue;
}
// info!(
// "reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
// src_file_path.as_ref(),
// dst_file_path.as_ref(),
// base_dir.as_ref(),
// e
// );
return Err(e);
}
break;
}
Ok(())
}
pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
let mut i = 0;
let mut base_dir = base_dir.as_ref();
loop {
if let Err(e) = os_mkdir_all(path.as_ref(), base_dir).await {
if e.kind() == std::io::ErrorKind::NotFound && i == 0 {
i += 1;
if let Some(base_parent) = base_dir.parent() {
if let Some(c) = base_parent.components().next() {
if c != Component::RootDir {
base_dir = base_parent
}
}
}
continue;
}
return Err(e);
}
break;
}
Ok(())
}
pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
if !base_dir.as_ref().to_string_lossy().is_empty() && base_dir.as_ref().starts_with(dir_path.as_ref()) {
return Ok(());
}
if let Some(parent) = dir_path.as_ref().parent() {
// 不支持递归，直接create_dir_all了
if let Err(e) = fs::make_dir_all(&parent).await {
if e.kind() == std::io::ErrorKind::AlreadyExists {
return Ok(());
}
return Err(e);
}
// Box::pin(os_mkdir_all(&parent, &base_dir)).await?;
}
if let Err(e) = fs::mkdir(dir_path.as_ref()).await {
if e.kind() == std::io::ErrorKind::AlreadyExists {
return Ok(());
}
return Err(e);
}
Ok(())
}
pub fn file_exists(path: impl AsRef<Path>) -> bool {
std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false)
}
crates/disk/src/path.rs
-308
View File
@@ -1,308 +0,0 @@
use std::path::Path;
use std::path::PathBuf;
pub const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__";
pub const SLASH_SEPARATOR: &str = "/";
pub const GLOBAL_DIR_SUFFIX_WITH_SLASH: &str = "__XLDIR__/";
pub fn has_suffix(s: &str, suffix: &str) -> bool {
if cfg!(target_os = "windows") {
s.to_lowercase().ends_with(&suffix.to_lowercase())
} else {
s.ends_with(suffix)
}
}
pub fn encode_dir_object(object: &str) -> String {
if has_suffix(object, SLASH_SEPARATOR) {
format!("{}{}", object.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX)
} else {
object.to_string()
}
}
pub fn is_dir_object(object: &str) -> bool {
let obj = encode_dir_object(object);
obj.ends_with(GLOBAL_DIR_SUFFIX)
}
#[allow(dead_code)]
pub fn decode_dir_object(object: &str) -> String {
if has_suffix(object, GLOBAL_DIR_SUFFIX) {
format!("{}{}", object.trim_end_matches(GLOBAL_DIR_SUFFIX), SLASH_SEPARATOR)
} else {
object.to_string()
}
}
pub fn retain_slash(s: &str) -> String {
if s.is_empty() {
return s.to_string();
}
if s.ends_with(SLASH_SEPARATOR) {
s.to_string()
} else {
format!("{}{}", s, SLASH_SEPARATOR)
}
}
pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool {
s.len() >= prefix.len() && (s[..prefix.len()] == *prefix || s[..prefix.len()].eq_ignore_ascii_case(prefix))
}
pub fn has_prefix(s: &str, prefix: &str) -> bool {
if cfg!(target_os = "windows") {
return strings_has_prefix_fold(s, prefix);
}
s.starts_with(prefix)
}
pub fn path_join(elem: &[PathBuf]) -> PathBuf {
let mut joined_path = PathBuf::new();
for path in elem {
joined_path.push(path);
}
joined_path
}
pub fn path_join_buf(elements: &[&str]) -> String {
let trailing_slash = !elements.is_empty() && elements.last().unwrap().ends_with(SLASH_SEPARATOR);
let mut dst = String::new();
let mut added = 0;
for e in elements {
if added > 0 || !e.is_empty() {
if added > 0 {
dst.push_str(SLASH_SEPARATOR);
}
dst.push_str(e);
added += e.len();
}
}
let result = dst.to_string();
let cpath = Path::new(&result).components().collect::<PathBuf>();
let clean_path = cpath.to_string_lossy();
if trailing_slash {
return format!("{}{}", clean_path, SLASH_SEPARATOR);
}
clean_path.to_string()
}
pub fn path_to_bucket_object_with_base_path(bash_path: &str, path: &str) -> (String, String) {
let path = path.trim_start_matches(bash_path).trim_start_matches(SLASH_SEPARATOR);
if let Some(m) = path.find(SLASH_SEPARATOR) {
return (path[..m].to_string(), path[m + SLASH_SEPARATOR.len()..].to_string());
}
(path.to_string(), "".to_string())
}
pub fn path_to_bucket_object(s: &str) -> (String, String) {
path_to_bucket_object_with_base_path("", s)
}
pub fn base_dir_from_prefix(prefix: &str) -> String {
let mut base_dir = dir(prefix).to_owned();
if base_dir == "." || base_dir == "./" || base_dir == "/" {
base_dir = "".to_owned();
}
if !prefix.contains('/') {
base_dir = "".to_owned();
}
if !base_dir.is_empty() && !base_dir.ends_with(SLASH_SEPARATOR) {
base_dir.push_str(SLASH_SEPARATOR);
}
base_dir
}
pub struct LazyBuf {
s: String,
buf: Option<Vec<u8>>,
w: usize,
}
impl LazyBuf {
pub fn new(s: String) -> Self {
LazyBuf { s, buf: None, w: 0 }
}
pub fn index(&self, i: usize) -> u8 {
if let Some(ref buf) = self.buf {
buf[i]
} else {
self.s.as_bytes()[i]
}
}
pub fn append(&mut self, c: u8) {
if self.buf.is_none() {
if self.w < self.s.len() && self.s.as_bytes()[self.w] == c {
self.w += 1;
return;
}
let mut new_buf = vec![0; self.s.len()];
new_buf[..self.w].copy_from_slice(&self.s.as_bytes()[..self.w]);
self.buf = Some(new_buf);
}
if let Some(ref mut buf) = self.buf {
buf[self.w] = c;
self.w += 1;
}
}
pub fn string(&self) -> String {
if let Some(ref buf) = self.buf {
String::from_utf8(buf[..self.w].to_vec()).unwrap()
} else {
self.s[..self.w].to_string()
}
}
}
pub fn clean(path: &str) -> String {
if path.is_empty() {
return ".".to_string();
}
let rooted = path.starts_with('/');
let n = path.len();
let mut out = LazyBuf::new(path.to_string());
let mut r = 0;
let mut dotdot = 0;
if rooted {
out.append(b'/');
r = 1;
dotdot = 1;
}
while r < n {
match path.as_bytes()[r] {
b'/' => {
// Empty path element
r += 1;
}
b'.' if r + 1 == n || path.as_bytes()[r + 1] == b'/' => {
// . element
r += 1;
}
b'.' if path.as_bytes()[r + 1] == b'.' && (r + 2 == n || path.as_bytes()[r + 2] == b'/') => {
// .. element: remove to last /
r += 2;
if out.w > dotdot {
// Can backtrack
out.w -= 1;
while out.w > dotdot && out.index(out.w) != b'/' {
out.w -= 1;
}
} else if !rooted {
// Cannot backtrack but not rooted, so append .. element.
if out.w > 0 {
out.append(b'/');
}
out.append(b'.');
out.append(b'.');
dotdot = out.w;
}
}
_ => {
// Real path element.
// Add slash if needed
if (rooted && out.w != 1) || (!rooted && out.w != 0) {
out.append(b'/');
}
// Copy element
while r < n && path.as_bytes()[r] != b'/' {
out.append(path.as_bytes()[r]);
r += 1;
}
}
}
}
// Turn empty string into "."
if out.w == 0 {
return ".".to_string();
}
out.string()
}
pub fn split(path: &str) -> (&str, &str) {
// Find the last occurrence of the '/' character
if let Some(i) = path.rfind('/') {
// Return the directory (up to and including the last '/') and the file name
return (&path[..i + 1], &path[i + 1..]);
}
// If no '/' is found, return an empty string for the directory and the whole path as the file name
(path, "")
}
pub fn dir(path: &str) -> String {
let (a, _) = split(path);
clean(a)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_base_dir_from_prefix() {
let a = "da/";
println!("---- in {}", a);
let a = base_dir_from_prefix(a);
println!("---- out {}", a);
}
#[test]
fn test_clean() {
assert_eq!(clean(""), ".");
assert_eq!(clean("abc"), "abc");
assert_eq!(clean("abc/def"), "abc/def");
assert_eq!(clean("a/b/c"), "a/b/c");
assert_eq!(clean("."), ".");
assert_eq!(clean(".."), "..");
assert_eq!(clean("../.."), "../..");
assert_eq!(clean("../../abc"), "../../abc");
assert_eq!(clean("/abc"), "/abc");
assert_eq!(clean("/"), "/");
assert_eq!(clean("abc/"), "abc");
assert_eq!(clean("abc/def/"), "abc/def");
assert_eq!(clean("a/b/c/"), "a/b/c");
assert_eq!(clean("./"), ".");
assert_eq!(clean("../"), "..");
assert_eq!(clean("../../"), "../..");
assert_eq!(clean("/abc/"), "/abc");
assert_eq!(clean("abc//def//ghi"), "abc/def/ghi");
assert_eq!(clean("//abc"), "/abc");
assert_eq!(clean("///abc"), "/abc");
assert_eq!(clean("//abc//"), "/abc");
assert_eq!(clean("abc//"), "abc");
assert_eq!(clean("abc/./def"), "abc/def");
assert_eq!(clean("/./abc/def"), "/abc/def");
assert_eq!(clean("abc/."), "abc");
assert_eq!(clean("abc/./../def"), "def");
assert_eq!(clean("abc//./../def"), "def");
assert_eq!(clean("abc/../../././../def"), "../../def");
assert_eq!(clean("abc/def/ghi/../jkl"), "abc/def/jkl");
assert_eq!(clean("abc/def/../ghi/../jkl"), "abc/jkl");
assert_eq!(clean("abc/def/.."), "abc");
assert_eq!(clean("abc/def/../.."), ".");
assert_eq!(clean("/abc/def/../.."), "/");
assert_eq!(clean("abc/def/../../.."), "..");
assert_eq!(clean("/abc/def/../../.."), "/");
assert_eq!(clean("abc/def/../../../ghi/jkl/../../../mno"), "../../mno");
}
}
crates/disk/src/remote.rs
-908
View File
@@ -1,908 +0,0 @@
use std::path::PathBuf;
use crate::api::CheckPartsResp;
use crate::api::DeleteOptions;
use crate::api::DiskAPI;
use crate::api::DiskInfo;
use crate::api::DiskInfoOptions;
use crate::api::DiskLocation;
use crate::api::DiskOption;
use crate::api::ReadMultipleReq;
use crate::api::ReadMultipleResp;
use crate::api::ReadOptions;
use crate::api::RenameDataResp;
use crate::api::UpdateMetadataOpts;
use crate::api::VolumeInfo;
use crate::api::WalkDirOptions;
use crate::endpoint::Endpoint;
use futures::StreamExt as _;
use http::HeaderMap;
use http::Method;
use protos::node_service_time_out_client;
use protos::proto_gen::node_service::CheckPartsRequest;
use protos::proto_gen::node_service::DeletePathsRequest;
use protos::proto_gen::node_service::DeleteRequest;
use protos::proto_gen::node_service::DeleteVersionRequest;
use protos::proto_gen::node_service::DeleteVersionsRequest;
use protos::proto_gen::node_service::DeleteVolumeRequest;
use protos::proto_gen::node_service::DiskInfoRequest;
use protos::proto_gen::node_service::ListDirRequest;
use protos::proto_gen::node_service::ListVolumesRequest;
use protos::proto_gen::node_service::MakeVolumeRequest;
use protos::proto_gen::node_service::MakeVolumesRequest;
use protos::proto_gen::node_service::ReadAllRequest;
use protos::proto_gen::node_service::ReadMultipleRequest;
use protos::proto_gen::node_service::ReadVersionRequest;
use protos::proto_gen::node_service::ReadXlRequest;
use protos::proto_gen::node_service::RenameDataRequest;
use protos::proto_gen::node_service::RenameFileRequst;
use protos::proto_gen::node_service::RenamePartRequst;
use protos::proto_gen::node_service::StatVolumeRequest;
use protos::proto_gen::node_service::UpdateMetadataRequest;
use protos::proto_gen::node_service::VerifyFileRequest;
use protos::proto_gen::node_service::WalkDirRequest;
use protos::proto_gen::node_service::WriteAllRequest;
use protos::proto_gen::node_service::WriteMetadataRequest;
use rmp_serde::Serializer;
use rustfs_error::Error;
use rustfs_error::Result;
use rustfs_filemeta::FileInfo;
use rustfs_filemeta::FileInfoVersions;
use rustfs_filemeta::RawFileInfo;
use rustfs_metacache::MetaCacheEntry;
use rustfs_metacache::MetacacheWriter;
use rustfs_rio::HttpReader;
use rustfs_rio::HttpWriter;
use serde::Serialize as _;
use tokio::io::AsyncRead;
use tokio::io::AsyncWrite;
use tokio::sync::Mutex;
use tonic::Request;
use tracing::info;
use uuid::Uuid;
#[derive(Debug)]
pub struct RemoteDisk {
pub id: Mutex<Option<Uuid>>,
pub addr: String,
pub url: url::Url,
pub root: PathBuf,
endpoint: Endpoint,
}
impl RemoteDisk {
pub async fn new(ep: &Endpoint, _opt: &DiskOption) -> Result<Self> {
// let root = fs::canonicalize(ep.url.path()).await?;
let root = PathBuf::from(ep.get_file_path());
let addr = format!("{}://{}:{}", ep.url.scheme(), ep.url.host_str().unwrap(), ep.url.port().unwrap());
Ok(Self {
id: Mutex::new(None),
addr,
url: ep.url.clone(),
root,
endpoint: ep.clone(),
})
}
}
// TODO: all api need to handle errors
#[async_trait::async_trait]
impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
fn to_string(&self) -> String {
self.endpoint.to_string()
}
#[tracing::instrument(skip(self))]
fn is_local(&self) -> bool {
false
}
#[tracing::instrument(skip(self))]
fn host_name(&self) -> String {
self.endpoint.host_port()
}
#[tracing::instrument(skip(self))]
async fn is_online(&self) -> bool {
// TODO: 连接状态
if (node_service_time_out_client(&self.addr).await).is_ok() {
return true;
}
false
}
#[tracing::instrument(skip(self))]
fn endpoint(&self) -> Endpoint {
self.endpoint.clone()
}
#[tracing::instrument(skip(self))]
async fn close(&self) -> Result<()> {
Ok(())
}
#[tracing::instrument(skip(self))]
fn path(&self) -> PathBuf {
self.root.clone()
}
#[tracing::instrument(skip(self))]
fn get_disk_location(&self) -> DiskLocation {
DiskLocation {
pool_idx: {
if self.endpoint.pool_idx < 0 {
None
} else {
Some(self.endpoint.pool_idx as usize)
}
},
set_idx: {
if self.endpoint.set_idx < 0 {
None
} else {
Some(self.endpoint.set_idx as usize)
}
},
disk_idx: {
if self.endpoint.disk_idx < 0 {
None
} else {
Some(self.endpoint.disk_idx as usize)
}
},
}
}
#[tracing::instrument(skip(self))]
async fn get_disk_id(&self) -> Result<Option<Uuid>> {
Ok(*self.id.lock().await)
}
#[tracing::instrument(skip(self))]
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
let mut lock = self.id.lock().await;
*lock = id;
Ok(())
}
#[tracing::instrument(skip(self))]
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
info!("read_all {}/{}", volume, path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadAllRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
});
let response = client.read_all(request).await?.into_inner();
if !response.success {
return Err(Error::FileNotFound);
}
Ok(response.data)
}
#[tracing::instrument(skip(self))]
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
info!("write_all");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(WriteAllRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
data,
});
let response = client.write_all(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
info!("delete {}/{}/{}", self.endpoint.to_string(), volume, path);
let options = serde_json::to_string(&opt)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
options,
});
let response = client.delete(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("verify_file");
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(VerifyFileRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
});
let response = client.verify_file(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let check_parts_resp = serde_json::from_str::<CheckPartsResp>(&response.check_parts_resp)?;
Ok(check_parts_resp)
}
#[tracing::instrument(skip(self))]
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("check_parts");
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(CheckPartsRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
});
let response = client.check_parts(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let check_parts_resp = serde_json::from_str::<CheckPartsResp>(&response.check_parts_resp)?;
Ok(check_parts_resp)
}
#[tracing::instrument(skip(self))]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()> {
info!("rename_part {}/{}", src_volume, src_path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(RenamePartRequst {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
meta,
});
let response = client.rename_part(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
info!("rename_file");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(RenameFileRequst {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
});
let response = client.rename_file(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<Box<dyn AsyncWrite>> {
info!("create_file {}/{}/{}", self.endpoint.to_string(), volume, path);
let url = format!(
"{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
self.endpoint.grid_host(),
urlencoding::encode(&self.endpoint.to_string()),
urlencoding::encode(volume),
urlencoding::encode(path),
false,
file_size
);
let wd = HttpWriter::new(url, Method::PUT, HeaderMap::new()).await?;
Ok(Box::new(wd))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<Box<dyn AsyncWrite>> {
info!("append_file {}/{}", volume, path);
let url = format!(
"{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
self.endpoint.grid_host(),
urlencoding::encode(&self.endpoint.to_string()),
urlencoding::encode(volume),
urlencoding::encode(path),
true,
0
);
let wd = HttpWriter::new(url, Method::PUT, HeaderMap::new()).await?;
Ok(Box::new(wd))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file(&self, volume: &str, path: &str) -> Result<Box<dyn AsyncRead>> {
info!("read_file {}/{}", volume, path);
let url = format!(
"{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
self.endpoint.grid_host(),
urlencoding::encode(&self.endpoint.to_string()),
urlencoding::encode(volume),
urlencoding::encode(path),
0,
0
);
let rd = HttpReader::new(url, Method::GET, HeaderMap::new()).await?;
Ok(Box::new(rd))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Box<dyn AsyncRead>> {
info!("read_file_stream {}/{}/{}", self.endpoint.to_string(), volume, path);
let url = format!(
"{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
self.endpoint.grid_host(),
urlencoding::encode(&self.endpoint.to_string()),
urlencoding::encode(volume),
urlencoding::encode(path),
offset,
length
);
let rd = HttpReader::new(url, Method::GET, HeaderMap::new()).await?;
Ok(Box::new(rd))
}
#[tracing::instrument(skip(self))]
async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
info!("list_dir {}/{}", volume, _dir_path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ListDirRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.list_dir(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(response.volumes)
}
// FIXME: TODO: use writer
#[tracing::instrument(skip(self, wr))]
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
let now = std::time::SystemTime::now();
info!("walk_dir {}/{}/{:?}", self.endpoint.to_string(), opts.bucket, opts.filter_prefix);
let mut wr = wr;
let mut out = MetacacheWriter::new(&mut wr);
let mut buf = Vec::new();
opts.serialize(&mut Serializer::new(&mut buf))?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(WalkDirRequest {
disk: self.endpoint.to_string(),
walk_dir_options: buf,
});
let mut response = client.walk_dir(request).await?.into_inner();
loop {
match response.next().await {
Some(Ok(resp)) => {
if !resp.success {
return Err(Error::msg(resp.error_info.unwrap_or("".to_string())));
}
let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
.map_err(|_| Error::msg(format!("Unexpected response: {:?}", response)))?;
out.write_obj(&entry).await?;
}
None => break,
_ => return Err(Error::msg(format!("Unexpected response: {:?}", response))),
}
}
info!(
"walk_dir {}/{:?} done {:?}",
opts.bucket,
opts.filter_prefix,
now.elapsed().unwrap_or_default()
);
Ok(())
}
#[tracing::instrument(skip(self))]
async fn rename_data(
&self,
src_volume: &str,
src_path: &str,
fi: FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
info!("rename_data {}/{}/{}/{}", self.addr, self.endpoint.to_string(), dst_volume, dst_path);
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(RenameDataRequest {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
file_info,
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
});
let response = client.rename_data(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let rename_data_resp = serde_json::from_str::<RenameDataResp>(&response.rename_data_resp)?;
Ok(rename_data_resp)
}
#[tracing::instrument(skip(self))]
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
info!("make_volumes");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(MakeVolumesRequest {
disk: self.endpoint.to_string(),
volumes: volumes.iter().map(|s| (*s).to_string()).collect(),
});
let response = client.make_volumes(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn make_volume(&self, volume: &str) -> Result<()> {
info!("make_volume");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(MakeVolumeRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.make_volume(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
info!("list_volumes");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ListVolumesRequest {
disk: self.endpoint.to_string(),
});
let response = client.list_volumes(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let infos = response
.volume_infos
.into_iter()
.filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
.collect();
Ok(infos)
}
#[tracing::instrument(skip(self))]
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
info!("stat_volume");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(StatVolumeRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.stat_volume(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let volume_info = serde_json::from_str::<VolumeInfo>(&response.volume_info)?;
Ok(volume_info)
}
#[tracing::instrument(skip(self))]
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
info!("delete_paths");
let paths = paths.to_owned();
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeletePathsRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
paths,
});
let response = client.delete_paths(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
info!("update_metadata");
let file_info = serde_json::to_string(&fi)?;
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(UpdateMetadataRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
opts,
});
let response = client.update_metadata(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
info!("write_metadata {}/{}", volume, path);
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(WriteMetadataRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
});
let response = client.write_metadata(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn read_version(
&self,
_org_volume: &str,
volume: &str,
path: &str,
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo> {
info!("read_version");
let opts = serde_json::to_string(opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadVersionRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
version_id: version_id.to_string(),
opts,
});
let response = client.read_version(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let file_info = serde_json::from_str::<FileInfo>(&response.file_info)?;
Ok(file_info)
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
info!("read_xl {}/{}/{}", self.endpoint.to_string(), volume, path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadXlRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
read_data,
});
let response = client.read_xl(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
Ok(raw_file_info)
}
#[tracing::instrument(skip(self))]
async fn delete_version(
&self,
volume: &str,
path: &str,
fi: FileInfo,
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<()> {
info!("delete_version");
let file_info = serde_json::to_string(&fi)?;
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteVersionRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
force_del_marker,
opts,
});
let response = client.delete_version(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
// let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
Ok(())
}
#[tracing::instrument(skip(self))]
async fn delete_versions(
&self,
volume: &str,
versions: Vec<FileInfoVersions>,
opts: DeleteOptions,
) -> Result<Vec<Option<Error>>> {
info!("delete_versions");
let opts = serde_json::to_string(&opts)?;
let mut versions_str = Vec::with_capacity(versions.len());
for file_info_versions in versions.iter() {
versions_str.push(serde_json::to_string(file_info_versions)?);
}
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteVersionsRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
versions: versions_str,
opts,
});
let response = client.delete_versions(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let errors = response
.errors
.iter()
.map(|error| {
if error.is_empty() {
None
} else {
use std::str::FromStr;
Some(Error::from_str(error).unwrap_or(Error::msg(error)))
}
})
.collect();
Ok(errors)
}
#[tracing::instrument(skip(self))]
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
info!("read_multiple {}/{}/{}", self.endpoint.to_string(), req.bucket, req.prefix);
let read_multiple_req = serde_json::to_string(&req)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadMultipleRequest {
disk: self.endpoint.to_string(),
read_multiple_req,
});
let response = client.read_multiple(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let read_multiple_resps = response
.read_multiple_resps
.into_iter()
.filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
.collect();
Ok(read_multiple_resps)
}
#[tracing::instrument(skip(self))]
async fn delete_volume(&self, volume: &str) -> Result<()> {
info!("delete_volume {}/{}", self.endpoint.to_string(), volume);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteVolumeRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.delete_volume(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo> {
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DiskInfoRequest {
disk: self.endpoint.to_string(),
opts,
});
let response = client.disk_info(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let disk_info = serde_json::from_str::<DiskInfo>(&response.disk_info)?;
Ok(disk_info)
}
// #[tracing::instrument(skip(self, cache, scan_mode, _we_sleep))]
// async fn ns_scanner(
// &self,
// cache: &DataUsageCache,
// updates: Sender<DataUsageEntry>,
// scan_mode: HealScanMode,
// _we_sleep: ShouldSleepFn,
// ) -> Result<DataUsageCache> {
// info!("ns_scanner");
// let cache = serde_json::to_string(cache)?;
// let mut client = node_service_time_out_client(&self.addr)
// .await
// .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
// let (tx, rx) = mpsc::channel(10);
// let in_stream = ReceiverStream::new(rx);
// let mut response = client.ns_scanner(in_stream).await?.into_inner();
// let request = NsScannerRequest {
// disk: self.endpoint.to_string(),
// cache,
// scan_mode: scan_mode as u64,
// };
// tx.send(request)
// .await
// .map_err(|err| Error::msg(format!("can not send request, err: {}", err)))?;
// loop {
// match response.next().await {
// Some(Ok(resp)) => {
// if !resp.update.is_empty() {
// let data_usage_cache = serde_json::from_str::<DataUsageEntry>(&resp.update)?;
// let _ = updates.send(data_usage_cache).await;
// } else if !resp.data_usage_cache.is_empty() {
// let data_usage_cache = serde_json::from_str::<DataUsageCache>(&resp.data_usage_cache)?;
// return Ok(data_usage_cache);
// } else {
// return Err(Error::msg("scan was interrupted"));
// }
// }
// _ => return Err(Error::msg("scan was interrupted")),
// }
// }
// }
// #[tracing::instrument(skip(self))]
// async fn healing(&self) -> Option<HealingTracker> {
// None
// }
}
crates/disk/src/remote_bak.rs
-862
View File
@@ -1,862 +0,0 @@
use std::path::PathBuf;
use super::{
endpoint::Endpoint, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo,
WalkDirOptions,
};
use crate::heal::{
data_scanner::ShouldSleepFn,
data_usage_cache::{DataUsageCache, DataUsageEntry},
heal_commands::{HealScanMode, HealingTracker},
};
use crate::io::{FileReader, FileWriter, HttpFileReader, HttpFileWriter};
use crate::{disk::metacache::MetaCacheEntry, metacache::writer::MetacacheWriter};
use futures::lock::Mutex;
use protos::proto_gen::node_service::RenamePartRequst;
use protos::{
node_service_time_out_client,
proto_gen::node_service::{
CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest,
DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, NsScannerRequest,
ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst,
StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest,
},
};
use rmp_serde::Serializer;
use rustfs_error::{Error, Result};
use rustfs_filemeta::{FileInfo, RawFileInfo};
use serde::Serialize;
use tokio::{
io::AsyncWrite,
sync::mpsc::{self, Sender},
};
use tokio_stream::{wrappers::ReceiverStream, StreamExt};
use tonic::Request;
use tracing::info;
use uuid::Uuid;
#[derive(Debug)]
pub struct RemoteDisk {
pub id: Mutex<Option<Uuid>>,
pub addr: String,
pub url: url::Url,
pub root: PathBuf,
endpoint: Endpoint,
}
impl RemoteDisk {
pub async fn new(ep: &Endpoint, _opt: &DiskOption) -> Result<Self> {
// let root = fs::canonicalize(ep.url.path()).await?;
let root = PathBuf::from(ep.get_file_path());
let addr = format!("{}://{}:{}", ep.url.scheme(), ep.url.host_str().unwrap(), ep.url.port().unwrap());
Ok(Self {
id: Mutex::new(None),
addr,
url: ep.url.clone(),
root,
endpoint: ep.clone(),
})
}
}
// TODO: all api need to handle errors
#[async_trait::async_trait]
impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
fn to_string(&self) -> String {
self.endpoint.to_string()
}
#[tracing::instrument(skip(self))]
fn is_local(&self) -> bool {
false
}
#[tracing::instrument(skip(self))]
fn host_name(&self) -> String {
self.endpoint.host_port()
}
#[tracing::instrument(skip(self))]
async fn is_online(&self) -> bool {
// TODO: 连接状态
if (node_service_time_out_client(&self.addr).await).is_ok() {
return true;
}
false
}
#[tracing::instrument(skip(self))]
fn endpoint(&self) -> Endpoint {
self.endpoint.clone()
}
#[tracing::instrument(skip(self))]
async fn close(&self) -> Result<()> {
Ok(())
}
#[tracing::instrument(skip(self))]
fn path(&self) -> PathBuf {
self.root.clone()
}
#[tracing::instrument(skip(self))]
fn get_disk_location(&self) -> DiskLocation {
DiskLocation {
pool_idx: {
if self.endpoint.pool_idx < 0 {
None
} else {
Some(self.endpoint.pool_idx as usize)
}
},
set_idx: {
if self.endpoint.set_idx < 0 {
None
} else {
Some(self.endpoint.set_idx as usize)
}
},
disk_idx: {
if self.endpoint.disk_idx < 0 {
None
} else {
Some(self.endpoint.disk_idx as usize)
}
},
}
}
#[tracing::instrument(skip(self))]
async fn get_disk_id(&self) -> Result<Option<Uuid>> {
Ok(*self.id.lock().await)
}
#[tracing::instrument(skip(self))]
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
let mut lock = self.id.lock().await;
*lock = id;
Ok(())
}
#[tracing::instrument(skip(self))]
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
info!("read_all {}/{}", volume, path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadAllRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
});
let response = client.read_all(request).await?.into_inner();
if !response.success {
return Err(Error::FileNotFound);
}
Ok(response.data)
}
#[tracing::instrument(skip(self))]
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
info!("write_all");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(WriteAllRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
data,
});
let response = client.write_all(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
info!("delete {}/{}/{}", self.endpoint.to_string(), volume, path);
let options = serde_json::to_string(&opt)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
options,
});
let response = client.delete(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("verify_file");
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(VerifyFileRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
});
let response = client.verify_file(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let check_parts_resp = serde_json::from_str::<CheckPartsResp>(&response.check_parts_resp)?;
Ok(check_parts_resp)
}
#[tracing::instrument(skip(self))]
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("check_parts");
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(CheckPartsRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
});
let response = client.check_parts(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let check_parts_resp = serde_json::from_str::<CheckPartsResp>(&response.check_parts_resp)?;
Ok(check_parts_resp)
}
#[tracing::instrument(skip(self))]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()> {
info!("rename_part {}/{}", src_volume, src_path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(RenamePartRequst {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
meta,
});
let response = client.rename_part(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
info!("rename_file");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(RenameFileRequst {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
});
let response = client.rename_file(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<FileWriter> {
info!("create_file {}/{}/{}", self.endpoint.to_string(), volume, path);
Ok(Box::new(HttpFileWriter::new(
self.endpoint.grid_host().as_str(),
self.endpoint.to_string().as_str(),
volume,
path,
file_size,
false,
)?))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
info!("append_file {}/{}", volume, path);
Ok(Box::new(HttpFileWriter::new(
self.endpoint.grid_host().as_str(),
self.endpoint.to_string().as_str(),
volume,
path,
0,
true,
)?))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
info!("read_file {}/{}", volume, path);
Ok(Box::new(
HttpFileReader::new(self.endpoint.grid_host().as_str(), self.endpoint.to_string().as_str(), volume, path, 0, 0)
.await?,
))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> {
info!("read_file_stream {}/{}/{}", self.endpoint.to_string(), volume, path);
Ok(Box::new(
HttpFileReader::new(
self.endpoint.grid_host().as_str(),
self.endpoint.to_string().as_str(),
volume,
path,
offset,
length,
)
.await?,
))
}
#[tracing::instrument(skip(self))]
async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
info!("list_dir {}/{}", volume, _dir_path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ListDirRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.list_dir(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(response.volumes)
}
// FIXME: TODO: use writer
#[tracing::instrument(skip(self, wr))]
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
let now = std::time::SystemTime::now();
info!("walk_dir {}/{}/{:?}", self.endpoint.to_string(), opts.bucket, opts.filter_prefix);
let mut wr = wr;
let mut out = MetacacheWriter::new(&mut wr);
let mut buf = Vec::new();
opts.serialize(&mut Serializer::new(&mut buf))?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(WalkDirRequest {
disk: self.endpoint.to_string(),
walk_dir_options: buf,
});
let mut response = client.walk_dir(request).await?.into_inner();
loop {
match response.next().await {
Some(Ok(resp)) => {
if !resp.success {
return Err(Error::msg(resp.error_info.unwrap_or("".to_string())));
}
let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
.map_err(|_| Error::msg(format!("Unexpected response: {:?}", response)))?;
out.write_obj(&entry).await?;
}
None => break,
_ => return Err(Error::msg(format!("Unexpected response: {:?}", response))),
}
}
info!(
"walk_dir {}/{:?} done {:?}",
opts.bucket,
opts.filter_prefix,
now.elapsed().unwrap_or_default()
);
Ok(())
}
#[tracing::instrument(skip(self))]
async fn rename_data(
&self,
src_volume: &str,
src_path: &str,
fi: FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
info!("rename_data {}/{}/{}/{}", self.addr, self.endpoint.to_string(), dst_volume, dst_path);
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(RenameDataRequest {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
file_info,
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
});
let response = client.rename_data(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let rename_data_resp = serde_json::from_str::<RenameDataResp>(&response.rename_data_resp)?;
Ok(rename_data_resp)
}
#[tracing::instrument(skip(self))]
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
info!("make_volumes");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(MakeVolumesRequest {
disk: self.endpoint.to_string(),
volumes: volumes.iter().map(|s| (*s).to_string()).collect(),
});
let response = client.make_volumes(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn make_volume(&self, volume: &str) -> Result<()> {
info!("make_volume");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(MakeVolumeRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.make_volume(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
info!("list_volumes");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ListVolumesRequest {
disk: self.endpoint.to_string(),
});
let response = client.list_volumes(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let infos = response
.volume_infos
.into_iter()
.filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
.collect();
Ok(infos)
}
#[tracing::instrument(skip(self))]
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
info!("stat_volume");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(StatVolumeRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.stat_volume(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let volume_info = serde_json::from_str::<VolumeInfo>(&response.volume_info)?;
Ok(volume_info)
}
#[tracing::instrument(skip(self))]
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
info!("delete_paths");
let paths = paths.to_owned();
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeletePathsRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
paths,
});
let response = client.delete_paths(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
info!("update_metadata");
let file_info = serde_json::to_string(&fi)?;
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(UpdateMetadataRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
opts,
});
let response = client.update_metadata(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
info!("write_metadata {}/{}", volume, path);
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(WriteMetadataRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
});
let response = client.write_metadata(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn read_version(
&self,
_org_volume: &str,
volume: &str,
path: &str,
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo> {
info!("read_version");
let opts = serde_json::to_string(opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadVersionRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
version_id: version_id.to_string(),
opts,
});
let response = client.read_version(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let file_info = serde_json::from_str::<FileInfo>(&response.file_info)?;
Ok(file_info)
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
info!("read_xl {}/{}/{}", self.endpoint.to_string(), volume, path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadXlRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
read_data,
});
let response = client.read_xl(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
Ok(raw_file_info)
}
#[tracing::instrument(skip(self))]
async fn delete_version(
&self,
volume: &str,
path: &str,
fi: FileInfo,
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<()> {
info!("delete_version");
let file_info = serde_json::to_string(&fi)?;
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteVersionRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
force_del_marker,
opts,
});
let response = client.delete_version(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
// let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
Ok(())
}
#[tracing::instrument(skip(self))]
async fn delete_versions(
&self,
volume: &str,
versions: Vec<FileInfoVersions>,
opts: DeleteOptions,
) -> Result<Vec<Option<Error>>> {
info!("delete_versions");
let opts = serde_json::to_string(&opts)?;
let mut versions_str = Vec::with_capacity(versions.len());
for file_info_versions in versions.iter() {
versions_str.push(serde_json::to_string(file_info_versions)?);
}
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteVersionsRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
versions: versions_str,
opts,
});
let response = client.delete_versions(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let errors = response
.errors
.iter()
.map(|error| {
if error.is_empty() {
None
} else {
use std::str::FromStr;
Some(Error::from_str(error).unwrap_or(Error::msg(error)))
}
})
.collect();
Ok(errors)
}
#[tracing::instrument(skip(self))]
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
info!("read_multiple {}/{}/{}", self.endpoint.to_string(), req.bucket, req.prefix);
let read_multiple_req = serde_json::to_string(&req)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadMultipleRequest {
disk: self.endpoint.to_string(),
read_multiple_req,
});
let response = client.read_multiple(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let read_multiple_resps = response
.read_multiple_resps
.into_iter()
.filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
.collect();
Ok(read_multiple_resps)
}
#[tracing::instrument(skip(self))]
async fn delete_volume(&self, volume: &str) -> Result<()> {
info!("delete_volume {}/{}", self.endpoint.to_string(), volume);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteVolumeRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.delete_volume(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo> {
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DiskInfoRequest {
disk: self.endpoint.to_string(),
opts,
});
let response = client.disk_info(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let disk_info = serde_json::from_str::<DiskInfo>(&response.disk_info)?;
Ok(disk_info)
}
#[tracing::instrument(skip(self, cache, scan_mode, _we_sleep))]
async fn ns_scanner(
&self,
cache: &DataUsageCache,
updates: Sender<DataUsageEntry>,
scan_mode: HealScanMode,
_we_sleep: ShouldSleepFn,
) -> Result<DataUsageCache> {
info!("ns_scanner");
let cache = serde_json::to_string(cache)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let (tx, rx) = mpsc::channel(10);
let in_stream = ReceiverStream::new(rx);
let mut response = client.ns_scanner(in_stream).await?.into_inner();
let request = NsScannerRequest {
disk: self.endpoint.to_string(),
cache,
scan_mode: scan_mode as u64,
};
tx.send(request)
.await
.map_err(|err| Error::msg(format!("can not send request, err: {}", err)))?;
loop {
match response.next().await {
Some(Ok(resp)) => {
if !resp.update.is_empty() {
let data_usage_cache = serde_json::from_str::<DataUsageEntry>(&resp.update)?;
let _ = updates.send(data_usage_cache).await;
} else if !resp.data_usage_cache.is_empty() {
let data_usage_cache = serde_json::from_str::<DataUsageCache>(&resp.data_usage_cache)?;
return Ok(data_usage_cache);
} else {
return Err(Error::msg("scan was interrupted"));
}
}
_ => return Err(Error::msg("scan was interrupted")),
}
}
}
#[tracing::instrument(skip(self))]
async fn healing(&self) -> Option<HealingTracker> {
None
}
}
crates/disk/src/utils.rs
-35
View File
@@ -1,35 +0,0 @@
use std::{fs::Metadata, path::Path};
use rustfs_error::{to_file_error, Error, Result};
pub async fn read_file_exists(path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<Metadata>)> {
let p = path.as_ref();
let (data, meta) = match read_file_all(&p).await {
Ok((data, meta)) => (data, Some(meta)),
Err(e) => {
if e == Error::FileNotFound {
(Vec::new(), None)
} else {
return Err(e);
}
}
};
Ok((data, meta))
}
pub async fn read_file_all(path: impl AsRef<Path>) -> Result<(Vec<u8>, Metadata)> {
let p = path.as_ref();
let meta = read_file_metadata(&path).await?;
let data = read_all(&p).await?;
Ok((data, meta))
}
pub async fn read_file_metadata(p: impl AsRef<Path>) -> Result<Metadata> {
Ok(tokio::fs::metadata(&p).await.map_err(to_file_error)?)
}
pub async fn read_all(p: impl AsRef<Path>) -> Result<Vec<u8>> {
tokio::fs::read(&p).await.map_err(|e| to_file_error(e).into())
}
crates/error/Cargo.toml
-22
View File
@@ -1,22 +0,0 @@
[package]
name = "rustfs-error"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
[dependencies]
protos.workspace = true
rmp.workspace = true
rmp-serde.workspace = true
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
time.workspace = true
tonic.workspace = true
uuid.workspace = true
[lints]
workspace = true
crates/error/src/bitrot.rs
-27
View File
@@ -1,27 +0,0 @@
use crate::Error;
pub const CHECK_PART_UNKNOWN: usize = 0;
pub const CHECK_PART_SUCCESS: usize = 1;
pub const CHECK_PART_DISK_NOT_FOUND: usize = 2;
pub const CHECK_PART_VOLUME_NOT_FOUND: usize = 3;
pub const CHECK_PART_FILE_NOT_FOUND: usize = 4;
pub const CHECK_PART_FILE_CORRUPT: usize = 5;
pub fn conv_part_err_to_int(err: &Option<Error>) -> usize {
if let Some(err) = err {
match err {
Error::FileNotFound | Error::FileVersionNotFound => CHECK_PART_FILE_NOT_FOUND,
Error::FileCorrupt => CHECK_PART_FILE_CORRUPT,
Error::VolumeNotFound => CHECK_PART_VOLUME_NOT_FOUND,
Error::DiskNotFound => CHECK_PART_DISK_NOT_FOUND,
Error::Nil => CHECK_PART_SUCCESS,
_ => CHECK_PART_UNKNOWN,
}
} else {
CHECK_PART_SUCCESS
}
}
pub fn has_part_err(part_errs: &[usize]) -> bool {
part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS)
}
crates/error/src/convert.rs
-92
View File
@@ -1,92 +0,0 @@
use crate::Error;
pub fn to_file_error(io_err: std::io::Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::NotFound => Error::FileNotFound.into(),
std::io::ErrorKind::PermissionDenied => Error::FileAccessDenied.into(),
std::io::ErrorKind::IsADirectory => Error::IsNotRegular.into(),
std::io::ErrorKind::NotADirectory => Error::FileAccessDenied.into(),
std::io::ErrorKind::DirectoryNotEmpty => Error::FileAccessDenied.into(),
std::io::ErrorKind::UnexpectedEof => Error::FaultyDisk.into(),
std::io::ErrorKind::TooManyLinks => Error::TooManyOpenFiles.into(),
std::io::ErrorKind::InvalidInput => Error::FileNotFound.into(),
std::io::ErrorKind::InvalidData => Error::FileCorrupt.into(),
std::io::ErrorKind::StorageFull => Error::DiskFull.into(),
_ => io_err,
}
}
pub fn to_volume_error(io_err: std::io::Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::NotFound => Error::VolumeNotFound.into(),
std::io::ErrorKind::PermissionDenied => Error::DiskAccessDenied.into(),
std::io::ErrorKind::DirectoryNotEmpty => Error::VolumeNotEmpty.into(),
std::io::ErrorKind::NotADirectory => Error::IsNotRegular.into(),
std::io::ErrorKind::Other => {
let err = Error::from(io_err.to_string());
match err {
Error::FileNotFound => Error::VolumeNotFound.into(),
Error::FileAccessDenied => Error::DiskAccessDenied.into(),
_ => to_file_error(io_err),
}
}
_ => to_file_error(io_err),
}
}
pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::NotFound => Error::DiskNotFound.into(),
std::io::ErrorKind::PermissionDenied => Error::DiskAccessDenied.into(),
std::io::ErrorKind::Other => {
let err = Error::from(io_err.to_string());
match err {
Error::FileNotFound => Error::DiskNotFound.into(),
Error::VolumeNotFound => Error::DiskNotFound.into(),
Error::FileAccessDenied => Error::DiskAccessDenied.into(),
Error::VolumeAccessDenied => Error::DiskAccessDenied.into(),
_ => to_volume_error(io_err),
}
}
_ => to_volume_error(io_err),
}
}
// only errors from FileSystem operations
pub fn to_access_error(io_err: std::io::Error, per_err: Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::PermissionDenied => per_err.into(),
std::io::ErrorKind::NotADirectory => per_err.into(),
std::io::ErrorKind::NotFound => Error::VolumeNotFound.into(),
std::io::ErrorKind::UnexpectedEof => Error::FaultyDisk.into(),
std::io::ErrorKind::Other => {
let err = Error::from(io_err.to_string());
match err {
Error::DiskAccessDenied => per_err.into(),
Error::FileAccessDenied => per_err.into(),
Error::FileNotFound => Error::VolumeNotFound.into(),
_ => to_volume_error(io_err),
}
}
_ => to_volume_error(io_err),
}
}
pub fn to_unformatted_disk_error(io_err: std::io::Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::NotFound => Error::UnformattedDisk.into(),
std::io::ErrorKind::PermissionDenied => Error::DiskAccessDenied.into(),
std::io::ErrorKind::Other => {
let err = Error::from(io_err.to_string());
match err {
Error::FileNotFound => Error::UnformattedDisk.into(),
Error::DiskNotFound => Error::UnformattedDisk.into(),
Error::VolumeNotFound => Error::UnformattedDisk.into(),
Error::FileAccessDenied => Error::DiskAccessDenied.into(),
Error::DiskAccessDenied => Error::DiskAccessDenied.into(),
_ => Error::CorruptedBackend.into(),
}
}
_ => Error::CorruptedBackend.into(),
}
}
crates/error/src/error.rs
-586
View File
@@ -1,586 +0,0 @@
use std::hash::Hash;
use std::str::FromStr;
const ERROR_PREFIX: &str = "[RUSTFS error] ";
pub type Result<T> = core::result::Result<T, Error>;
#[derive(thiserror::Error, Default, Debug)]
pub enum Error {
#[default]
#[error("[RUSTFS error] Nil")]
Nil,
#[error("I/O error: {0}")]
IoError(std::io::Error),
#[error("[RUSTFS error] Erasure Read quorum not met")]
ErasureReadQuorum,
#[error("[RUSTFS error] Erasure Write quorum not met")]
ErasureWriteQuorum,
#[error("[RUSTFS error] Disk not found")]
DiskNotFound,
#[error("[RUSTFS error] Faulty disk")]
FaultyDisk,
#[error("[RUSTFS error] Faulty remote disk")]
FaultyRemoteDisk,
#[error("[RUSTFS error] Unsupported disk")]
UnsupportedDisk,
#[error("[RUSTFS error] Unformatted disk")]
UnformattedDisk,
#[error("[RUSTFS error] Corrupted backend")]
CorruptedBackend,
#[error("[RUSTFS error] Disk access denied")]
DiskAccessDenied,
#[error("[RUSTFS error] Disk ongoing request")]
DiskOngoingReq,
#[error("[RUSTFS error] Disk full")]
DiskFull,
#[error("[RUSTFS error] Volume not found")]
VolumeNotFound,
#[error("[RUSTFS error] Volume not empty")]
VolumeNotEmpty,
#[error("[RUSTFS error] Volume access denied")]
VolumeAccessDenied,
#[error("[RUSTFS error] Volume exists")]
VolumeExists,
#[error("[RUSTFS error] Disk not a directory")]
DiskNotDir,
#[error("[RUSTFS error] File not found")]
FileNotFound,
#[error("[RUSTFS error] File corrupt")]
FileCorrupt,
#[error("[RUSTFS error] File access denied")]
FileAccessDenied,
#[error("[RUSTFS error] Too many open files")]
TooManyOpenFiles,
#[error("[RUSTFS error] Is not a regular file")]
IsNotRegular,
#[error("[RUSTFS error] File version not found")]
FileVersionNotFound,
#[error("[RUSTFS error] Less data than expected")]
LessData,
#[error("[RUSTFS error] Short write")]
ShortWrite,
#[error("[RUSTFS error] Done for now")]
DoneForNow,
#[error("[RUSTFS error] Method not allowed")]
MethodNotAllowed,
#[error("[RUSTFS error] Inconsistent disk")]
InconsistentDisk,
#[error("[RUSTFS error] File name too long")]
FileNameTooLong,
#[error("[RUSTFS error] Scan ignore file contribution")]
ScanIgnoreFileContrib,
#[error("[RUSTFS error] Scan skip file")]
ScanSkipFile,
#[error("[RUSTFS error] Scan heal stop signaled")]
ScanHealStopSignal,
#[error("[RUSTFS error] Scan heal idle timeout")]
ScanHealIdleTimeout,
#[error("[RUSTFS error] Scan retry healing")]
ScanRetryHealing,
#[error("[RUSTFS error] {0}")]
Other(String),
}
// Generic From implementation removed to avoid conflicts with std::convert::From<T> for T
impl FromStr for Error {
type Err = Error;
fn from_str(s: &str) -> core::result::Result<Self, Self::Err> {
// Only strip prefix for non-IoError
let s = if s.starts_with("I/O error: ") {
s
} else {
s.strip_prefix(ERROR_PREFIX).unwrap_or(s)
};
match s {
"Nil" => Ok(Error::Nil),
"ErasureReadQuorum" => Ok(Error::ErasureReadQuorum),
"ErasureWriteQuorum" => Ok(Error::ErasureWriteQuorum),
"DiskNotFound" | "Disk not found" => Ok(Error::DiskNotFound),
"FaultyDisk" | "Faulty disk" => Ok(Error::FaultyDisk),
"FaultyRemoteDisk" | "Faulty remote disk" => Ok(Error::FaultyRemoteDisk),
"UnformattedDisk" | "Unformatted disk" => Ok(Error::UnformattedDisk),
"DiskAccessDenied" | "Disk access denied" => Ok(Error::DiskAccessDenied),
"DiskOngoingReq" | "Disk ongoing request" => Ok(Error::DiskOngoingReq),
"FileNotFound" | "File not found" => Ok(Error::FileNotFound),
"FileCorrupt" | "File corrupt" => Ok(Error::FileCorrupt),
"FileVersionNotFound" | "File version not found" => Ok(Error::FileVersionNotFound),
"LessData" | "Less data than expected" => Ok(Error::LessData),
"ShortWrite" | "Short write" => Ok(Error::ShortWrite),
"VolumeNotFound" | "Volume not found" => Ok(Error::VolumeNotFound),
"VolumeNotEmpty" | "Volume not empty" => Ok(Error::VolumeNotEmpty),
"VolumeExists" | "Volume exists" => Ok(Error::VolumeExists),
"VolumeAccessDenied" | "Volume access denied" => Ok(Error::VolumeAccessDenied),
"DiskNotDir" | "Disk not a directory" => Ok(Error::DiskNotDir),
"FileAccessDenied" | "File access denied" => Ok(Error::FileAccessDenied),
"TooManyOpenFiles" | "Too many open files" => Ok(Error::TooManyOpenFiles),
"IsNotRegular" | "Is not a regular file" => Ok(Error::IsNotRegular),
"CorruptedBackend" | "Corrupted backend" => Ok(Error::CorruptedBackend),
"UnsupportedDisk" | "Unsupported disk" => Ok(Error::UnsupportedDisk),
"InconsistentDisk" | "Inconsistent disk" => Ok(Error::InconsistentDisk),
"DiskFull" | "Disk full" => Ok(Error::DiskFull),
"FileNameTooLong" | "File name too long" => Ok(Error::FileNameTooLong),
"ScanIgnoreFileContrib" | "Scan ignore file contribution" => Ok(Error::ScanIgnoreFileContrib),
"ScanSkipFile" | "Scan skip file" => Ok(Error::ScanSkipFile),
"ScanHealStopSignal" | "Scan heal stop signaled" => Ok(Error::ScanHealStopSignal),
"ScanHealIdleTimeout" | "Scan heal idle timeout" => Ok(Error::ScanHealIdleTimeout),
"ScanRetryHealing" | "Scan retry healing" => Ok(Error::ScanRetryHealing),
s if s.starts_with("I/O error: ") => {
Ok(Error::IoError(std::io::Error::other(s.strip_prefix("I/O error: ").unwrap_or(""))))
}
"DoneForNow" | "Done for now" => Ok(Error::DoneForNow),
"MethodNotAllowed" | "Method not allowed" => Ok(Error::MethodNotAllowed),
str => Err(Error::IoError(std::io::Error::other(str.to_string()))),
}
}
}
impl From<Error> for std::io::Error {
fn from(err: Error) -> Self {
match err {
Error::IoError(e) => e,
e => std::io::Error::other(e),
}
}
}
impl From<std::io::Error> for Error {
fn from(e: std::io::Error) -> Self {
match e.kind() {
// convert Error from string to Error
std::io::ErrorKind::Other => Error::from(e.to_string()),
_ => Error::IoError(e),
}
}
}
impl From<String> for Error {
fn from(s: String) -> Self {
Error::from_str(&s).unwrap_or(Error::IoError(std::io::Error::other(s)))
}
}
impl From<&str> for Error {
fn from(s: &str) -> Self {
Error::from_str(s).unwrap_or(Error::IoError(std::io::Error::other(s)))
}
}
// Common error type conversions for ? operator
impl From<std::num::ParseIntError> for Error {
fn from(e: std::num::ParseIntError) -> Self {
Error::Other(format!("Parse int error: {}", e))
}
}
impl From<std::num::ParseFloatError> for Error {
fn from(e: std::num::ParseFloatError) -> Self {
Error::Other(format!("Parse float error: {}", e))
}
}
impl From<std::str::Utf8Error> for Error {
fn from(e: std::str::Utf8Error) -> Self {
Error::Other(format!("UTF-8 error: {}", e))
}
}
impl From<std::string::FromUtf8Error> for Error {
fn from(e: std::string::FromUtf8Error) -> Self {
Error::Other(format!("UTF-8 conversion error: {}", e))
}
}
impl From<std::fmt::Error> for Error {
fn from(e: std::fmt::Error) -> Self {
Error::Other(format!("Format error: {}", e))
}
}
impl From<Box<dyn std::error::Error + Send + Sync>> for Error {
fn from(e: Box<dyn std::error::Error + Send + Sync>) -> Self {
Error::Other(e.to_string())
}
}
impl From<time::error::ComponentRange> for Error {
fn from(e: time::error::ComponentRange) -> Self {
Error::Other(format!("Time component range error: {}", e))
}
}
impl From<rmp::decode::NumValueReadError<std::io::Error>> for Error {
fn from(e: rmp::decode::NumValueReadError<std::io::Error>) -> Self {
Error::Other(format!("NumValueReadError: {}", e))
}
}
impl From<rmp::encode::ValueWriteError> for Error {
fn from(e: rmp::encode::ValueWriteError) -> Self {
Error::Other(format!("ValueWriteError: {}", e))
}
}
impl From<rmp::decode::ValueReadError> for Error {
fn from(e: rmp::decode::ValueReadError) -> Self {
Error::Other(format!("ValueReadError: {}", e))
}
}
impl From<uuid::Error> for Error {
fn from(e: uuid::Error) -> Self {
Error::Other(format!("UUID error: {}", e))
}
}
impl From<rmp_serde::decode::Error> for Error {
fn from(e: rmp_serde::decode::Error) -> Self {
Error::Other(format!("rmp_serde::decode::Error: {}", e))
}
}
impl From<rmp_serde::encode::Error> for Error {
fn from(e: rmp_serde::encode::Error) -> Self {
Error::Other(format!("rmp_serde::encode::Error: {}", e))
}
}
impl From<serde::de::value::Error> for Error {
fn from(e: serde::de::value::Error) -> Self {
Error::Other(format!("serde::de::value::Error: {}", e))
}
}
impl From<serde_json::Error> for Error {
fn from(e: serde_json::Error) -> Self {
Error::Other(format!("serde_json::Error: {}", e))
}
}
impl From<std::collections::TryReserveError> for Error {
fn from(e: std::collections::TryReserveError) -> Self {
Error::Other(format!("TryReserveError: {}", e))
}
}
impl From<tonic::Status> for Error {
fn from(e: tonic::Status) -> Self {
Error::Other(format!("tonic::Status: {}", e.message()))
}
}
impl From<protos::proto_gen::node_service::Error> for Error {
fn from(e: protos::proto_gen::node_service::Error) -> Self {
Error::from_str(&e.error_info).unwrap_or(Error::Other(format!("Proto_Error: {}", e.error_info)))
}
}
impl From<Error> for protos::proto_gen::node_service::Error {
fn from(val: Error) -> Self {
protos::proto_gen::node_service::Error {
code: 0,
error_info: val.to_string(),
}
}
}
impl Hash for Error {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
match self {
Error::IoError(err) => {
err.kind().hash(state);
err.to_string().hash(state);
}
e => e.to_string().hash(state),
}
}
}
impl Clone for Error {
fn clone(&self) -> Self {
match self {
Error::IoError(err) => Error::IoError(std::io::Error::new(err.kind(), err.to_string())),
Error::ErasureReadQuorum => Error::ErasureReadQuorum,
Error::ErasureWriteQuorum => Error::ErasureWriteQuorum,
Error::DiskNotFound => Error::DiskNotFound,
Error::FaultyDisk => Error::FaultyDisk,
Error::FaultyRemoteDisk => Error::FaultyRemoteDisk,
Error::UnformattedDisk => Error::UnformattedDisk,
Error::DiskAccessDenied => Error::DiskAccessDenied,
Error::DiskOngoingReq => Error::DiskOngoingReq,
Error::FileNotFound => Error::FileNotFound,
Error::FileCorrupt => Error::FileCorrupt,
Error::FileVersionNotFound => Error::FileVersionNotFound,
Error::LessData => Error::LessData,
Error::ShortWrite => Error::ShortWrite,
Error::VolumeNotFound => Error::VolumeNotFound,
Error::VolumeNotEmpty => Error::VolumeNotEmpty,
Error::VolumeAccessDenied => Error::VolumeAccessDenied,
Error::VolumeExists => Error::VolumeExists,
Error::DiskNotDir => Error::DiskNotDir,
Error::FileAccessDenied => Error::FileAccessDenied,
Error::TooManyOpenFiles => Error::TooManyOpenFiles,
Error::IsNotRegular => Error::IsNotRegular,
Error::CorruptedBackend => Error::CorruptedBackend,
Error::UnsupportedDisk => Error::UnsupportedDisk,
Error::DiskFull => Error::DiskFull,
Error::Nil => Error::Nil,
Error::DoneForNow => Error::DoneForNow,
Error::MethodNotAllowed => Error::MethodNotAllowed,
Error::InconsistentDisk => Error::InconsistentDisk,
Error::FileNameTooLong => Error::FileNameTooLong,
Error::ScanIgnoreFileContrib => Error::ScanIgnoreFileContrib,
Error::ScanSkipFile => Error::ScanSkipFile,
Error::ScanHealStopSignal => Error::ScanHealStopSignal,
Error::ScanHealIdleTimeout => Error::ScanHealIdleTimeout,
Error::ScanRetryHealing => Error::ScanRetryHealing,
Error::Other(msg) => Error::Other(msg.clone()),
}
}
}
impl PartialEq for Error {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Error::IoError(e1), Error::IoError(e2)) => e1.kind() == e2.kind() && e1.to_string() == e2.to_string(),
(Error::ErasureReadQuorum, Error::ErasureReadQuorum) => true,
(Error::ErasureWriteQuorum, Error::ErasureWriteQuorum) => true,
(Error::DiskNotFound, Error::DiskNotFound) => true,
(Error::FaultyDisk, Error::FaultyDisk) => true,
(Error::FaultyRemoteDisk, Error::FaultyRemoteDisk) => true,
(Error::UnformattedDisk, Error::UnformattedDisk) => true,
(Error::DiskAccessDenied, Error::DiskAccessDenied) => true,
(Error::DiskOngoingReq, Error::DiskOngoingReq) => true,
(Error::FileNotFound, Error::FileNotFound) => true,
(Error::FileCorrupt, Error::FileCorrupt) => true,
(Error::FileVersionNotFound, Error::FileVersionNotFound) => true,
(Error::LessData, Error::LessData) => true,
(Error::ShortWrite, Error::ShortWrite) => true,
(Error::VolumeNotFound, Error::VolumeNotFound) => true,
(Error::VolumeNotEmpty, Error::VolumeNotEmpty) => true,
(Error::VolumeAccessDenied, Error::VolumeAccessDenied) => true,
(Error::VolumeExists, Error::VolumeExists) => true,
(Error::DiskNotDir, Error::DiskNotDir) => true,
(Error::FileAccessDenied, Error::FileAccessDenied) => true,
(Error::TooManyOpenFiles, Error::TooManyOpenFiles) => true,
(Error::IsNotRegular, Error::IsNotRegular) => true,
(Error::CorruptedBackend, Error::CorruptedBackend) => true,
(Error::UnsupportedDisk, Error::UnsupportedDisk) => true,
(Error::DiskFull, Error::DiskFull) => true,
(Error::Nil, Error::Nil) => true,
(Error::DoneForNow, Error::DoneForNow) => true,
(Error::MethodNotAllowed, Error::MethodNotAllowed) => true,
(Error::InconsistentDisk, Error::InconsistentDisk) => true,
(Error::FileNameTooLong, Error::FileNameTooLong) => true,
(Error::ScanIgnoreFileContrib, Error::ScanIgnoreFileContrib) => true,
(Error::ScanSkipFile, Error::ScanSkipFile) => true,
(Error::ScanHealStopSignal, Error::ScanHealStopSignal) => true,
(Error::ScanHealIdleTimeout, Error::ScanHealIdleTimeout) => true,
(Error::ScanRetryHealing, Error::ScanRetryHealing) => true,
(Error::Other(s1), Error::Other(s2)) => s1 == s2,
_ => false,
}
}
}
impl Eq for Error {}
impl Error {
/// Create an error from a message string (for backward compatibility)
pub fn msg<S: Into<String>>(message: S) -> Self {
Error::Other(message.into())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::io;
#[test]
fn test_display_and_debug() {
let e = Error::DiskNotFound;
assert_eq!(format!("{}", e), format!("{ERROR_PREFIX}Disk not found"));
assert_eq!(format!("{:?}", e), "DiskNotFound");
let io_err = Error::IoError(io::Error::other("fail"));
assert_eq!(format!("{}", io_err), "I/O error: fail");
}
#[test]
fn test_partial_eq_and_eq() {
assert_eq!(Error::DiskNotFound, Error::DiskNotFound);
assert_ne!(Error::DiskNotFound, Error::FaultyDisk);
let e1 = Error::IoError(io::Error::other("fail"));
let e2 = Error::IoError(io::Error::other("fail"));
assert_eq!(e1, e2);
let e3 = Error::IoError(io::Error::new(io::ErrorKind::NotFound, "fail"));
assert_ne!(e1, e3);
}
#[test]
fn test_clone() {
let e = Error::DiskAccessDenied;
let cloned = e.clone();
assert_eq!(e, cloned);
let io_err = Error::IoError(io::Error::other("fail"));
let cloned_io = io_err.clone();
assert_eq!(io_err, cloned_io);
}
#[test]
fn test_hash() {
let e1 = Error::DiskNotFound;
let e2 = Error::DiskNotFound;
let mut h1 = DefaultHasher::new();
let mut h2 = DefaultHasher::new();
e1.hash(&mut h1);
e2.hash(&mut h2);
assert_eq!(h1.finish(), h2.finish());
let io_err1 = Error::IoError(io::Error::other("fail"));
let io_err2 = Error::IoError(io::Error::other("fail"));
let mut h3 = DefaultHasher::new();
let mut h4 = DefaultHasher::new();
io_err1.hash(&mut h3);
io_err2.hash(&mut h4);
assert_eq!(h3.finish(), h4.finish());
}
#[test]
fn test_from_error_for_io_error() {
let e = Error::DiskNotFound;
let io_err: io::Error = e.into();
assert_eq!(io_err.kind(), io::ErrorKind::Other);
assert_eq!(io_err.to_string(), format!("{ERROR_PREFIX}Disk not found"));
assert_eq!(Error::from(io_err.to_string()), Error::DiskNotFound);
let orig = io::Error::other("fail");
let e2 = Error::IoError(orig.kind().into());
let io_err2: io::Error = e2.into();
assert_eq!(io_err2.kind(), io::ErrorKind::Other);
}
#[test]
fn test_from_io_error_for_error() {
let orig = io::Error::other("fail");
let e: Error = orig.into();
match e {
Error::IoError(ioe) => {
assert_eq!(ioe.kind(), io::ErrorKind::Other);
assert_eq!(ioe.to_string(), "fail");
}
_ => panic!("Expected IoError variant"),
}
}
#[test]
fn test_default() {
let e = Error::default();
assert_eq!(e, Error::Nil);
}
#[test]
fn test_from_str() {
use std::str::FromStr;
assert_eq!(Error::from_str("Nil"), Ok(Error::Nil));
assert_eq!(Error::from_str("DiskNotFound"), Ok(Error::DiskNotFound));
assert_eq!(Error::from_str("ErasureReadQuorum"), Ok(Error::ErasureReadQuorum));
assert_eq!(Error::from_str("I/O error: fail"), Ok(Error::IoError(io::Error::other("fail"))));
assert_eq!(Error::from_str(&format!("{ERROR_PREFIX}Disk not found")), Ok(Error::DiskNotFound));
assert_eq!(
Error::from_str("UnknownError"),
Err(Error::IoError(std::io::Error::other("UnknownError")))
);
}
#[test]
fn test_from_string() {
let e: Error = format!("{ERROR_PREFIX}Disk not found").parse().unwrap();
assert_eq!(e, Error::DiskNotFound);
let e2: Error = "I/O error: fail".to_string().parse().unwrap();
assert_eq!(e2, Error::IoError(std::io::Error::other("fail")));
}
#[test]
fn test_from_io_error() {
let e = Error::IoError(io::Error::other("fail"));
let io_err: io::Error = e.clone().into();
assert_eq!(io_err.to_string(), "fail");
let e2: Error = io::Error::other("fail").into();
assert_eq!(e2, Error::IoError(io::Error::other("fail")));
let result = Error::from(io::Error::other("fail"));
assert_eq!(result, Error::IoError(io::Error::other("fail")));
let io_err2: std::io::Error = Error::CorruptedBackend.into();
assert_eq!(io_err2.to_string(), "[RUSTFS error] Corrupted backend");
assert_eq!(Error::from(io_err2), Error::CorruptedBackend);
let io_err3: std::io::Error = Error::DiskNotFound.into();
assert_eq!(io_err3.to_string(), "[RUSTFS error] Disk not found");
assert_eq!(Error::from(io_err3), Error::DiskNotFound);
let io_err4: std::io::Error = Error::DiskAccessDenied.into();
assert_eq!(io_err4.to_string(), "[RUSTFS error] Disk access denied");
assert_eq!(Error::from(io_err4), Error::DiskAccessDenied);
}
#[test]
fn test_question_mark_operator() {
fn parse_number(s: &str) -> Result<i32> {
let num = s.parse::<i32>()?; // ParseIntError automatically converts to Error
Ok(num)
}
fn format_string() -> Result<String> {
use std::fmt::Write;
let mut s = String::new();
write!(&mut s, "test")?; // fmt::Error automatically converts to Error
Ok(s)
}
fn utf8_conversion() -> Result<String> {
let bytes = vec![0xFF, 0xFE]; // Invalid UTF-8
let s = String::from_utf8(bytes)?; // FromUtf8Error automatically converts to Error
Ok(s)
}
// Test successful case
assert_eq!(parse_number("42").unwrap(), 42);
// Test error conversion
let err = parse_number("not_a_number").unwrap_err();
assert!(matches!(err, Error::Other(_)));
assert!(err.to_string().contains("Parse int error"));
// Test format error conversion
assert_eq!(format_string().unwrap(), "test");
// Test UTF-8 error conversion
let err = utf8_conversion().unwrap_err();
assert!(matches!(err, Error::Other(_)));
assert!(err.to_string().contains("UTF-8 conversion error"));
}
}
crates/error/src/ignored.rs
-11
View File
@@ -1,11 +0,0 @@
use crate::Error;
pub static OBJECT_OP_IGNORED_ERRS: &[Error] = &[
Error::DiskNotFound,
Error::FaultyDisk,
Error::FaultyRemoteDisk,
Error::DiskAccessDenied,
Error::DiskOngoingReq,
Error::UnformattedDisk,
];
pub static BASE_IGNORED_ERRS: &[Error] = &[Error::DiskNotFound, Error::FaultyDisk, Error::FaultyRemoteDisk];
crates/error/src/lib.rs
-14
View File
@@ -1,14 +0,0 @@
mod error;
pub use error::*;
mod reduce;
pub use reduce::*;
mod ignored;
pub use ignored::*;
mod convert;
pub use convert::*;
mod bitrot;
pub use bitrot::*;
crates/error/src/reduce.rs
-138
View File
@@ -1,138 +0,0 @@
use crate::error::Error;
pub fn reduce_write_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error], quorun: usize) -> Option<Error> {
reduce_quorum_errs(errors, ignored_errs, quorun, Error::ErasureWriteQuorum)
}
pub fn reduce_read_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error], quorun: usize) -> Option<Error> {
reduce_quorum_errs(errors, ignored_errs, quorun, Error::ErasureReadQuorum)
}
pub fn reduce_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error], quorun: usize, quorun_err: Error) -> Option<Error> {
let (max_count, err) = reduce_errs(errors, ignored_errs);
if max_count >= quorun {
err
} else {
Some(quorun_err)
}
}
pub fn reduce_errs(errors: &[Option<Error>], ignored_errs: &[Error]) -> (usize, Option<Error>) {
let err_counts =
errors
.iter()
.map(|e| e.as_ref().unwrap_or(&Error::Nil))
.fold(std::collections::HashMap::new(), |mut acc, e| {
if is_ignored_err(ignored_errs, e) {
return acc;
}
*acc.entry(e.clone()).or_insert(0) += 1;
acc
});
let (err, max_count) = err_counts
.into_iter()
.max_by(|(e1, c1), (e2, c2)| {
// Prefer Error::Nil if present in a tie
let count_cmp = c1.cmp(c2);
if count_cmp == std::cmp::Ordering::Equal {
match (e1, e2) {
(Error::Nil, _) => std::cmp::Ordering::Greater,
(_, Error::Nil) => std::cmp::Ordering::Less,
_ => format!("{e1:?}").cmp(&format!("{e2:?}")),
}
} else {
count_cmp
}
})
.unwrap_or((Error::Nil, 0));
(max_count, if err == Error::Nil { None } else { Some(err) })
}
pub fn is_ignored_err(ignored_errs: &[Error], err: &Error) -> bool {
ignored_errs.iter().any(|e| e == err)
}
pub fn count_errs(errors: &[Option<Error>], err: Error) -> usize {
errors
.iter()
.map(|e| if e.is_none() { &Error::Nil } else { e.as_ref().unwrap() })
.filter(|&e| e == &err)
.count()
}
#[cfg(test)]
mod tests {
use super::*;
fn err_io(msg: &str) -> Error {
Error::IoError(std::io::Error::other(msg))
}
#[test]
fn test_reduce_errs_basic() {
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None];
let ignored = vec![];
let (count, err) = reduce_errs(&errors, &ignored);
assert_eq!(count, 2);
assert_eq!(err, Some(e1));
}
#[test]
fn test_reduce_errs_ignored() {
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e2.clone()), Some(e1.clone()), Some(e2.clone()), None];
let ignored = vec![e2.clone()];
let (count, err) = reduce_errs(&errors, &ignored);
assert_eq!(count, 2);
assert_eq!(err, Some(e1));
}
#[test]
fn test_reduce_quorum_errs() {
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None];
let ignored = vec![];
// quorum = 2, should return e1
let res = reduce_quorum_errs(&errors, &ignored, 2, Error::ErasureReadQuorum);
assert_eq!(res, Some(e1));
// quorum = 3, should return quorum error
let res = reduce_quorum_errs(&errors, &ignored, 3, Error::ErasureReadQuorum);
assert_eq!(res, Some(Error::ErasureReadQuorum));
}
#[test]
fn test_count_errs() {
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e2.clone()), Some(e1.clone()), None];
assert_eq!(count_errs(&errors, e1.clone()), 2);
assert_eq!(count_errs(&errors, e2.clone()), 1);
}
#[test]
fn test_is_ignored_err() {
let e1 = err_io("a");
let e2 = err_io("b");
let ignored = vec![e1.clone()];
assert!(is_ignored_err(&ignored, &e1));
assert!(!is_ignored_err(&ignored, &e2));
}
#[test]
fn test_reduce_errs_nil_tiebreak() {
// Error::Nil and another error have the same count, should prefer Nil
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e2.clone()), None, Some(e1.clone()), None]; // e1:1, Nil:1
let ignored = vec![];
let (count, err) = reduce_errs(&errors, &ignored);
assert_eq!(count, 2);
assert_eq!(err, None); // None means Error::Nil is preferred
}
}
ecstore/src/bitrot.rs
+719 -719
View File
File diff suppressed because it is too large Load Diff
ecstore/src/bucket/error.rs
+5 -1
View File
@@ -34,13 +34,17 @@ impl BucketMetadataError {
impl From<BucketMetadataError> for Error {
fn from(e: BucketMetadataError) -> Self {
Error::other(e)
match e {
BucketMetadataError::BucketPolicyNotFound => Error::BucketPolicyNotFound,
_ => Error::other(e),
}
}
}
impl From<Error> for BucketMetadataError {
fn from(e: Error) -> Self {
match e {
Error::BucketPolicyNotFound => BucketMetadataError::BucketPolicyNotFound,
Error::Io(e) => e.into(),
_ => BucketMetadataError::other(e),
}
ecstore/src/disk/local.rs
+17 -17
View File
@@ -1,29 +1,29 @@
use super::error::{Error, Result};
use super::os::{is_root_disk, rename_all};
use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
use super::{
os, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, Info,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE_BACKUP,
BUCKET_META_PREFIX, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics,
FileInfoVersions, Info, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, os,
};
use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
use crate::bucket::metadata_sys::{self};
use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::disk::STORAGE_FORMAT_FILE;
use crate::disk::error::FileAccessDeniedWithContext;
use crate::disk::error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error};
use crate::disk::fs::{
access, lstat, lstat_std, remove, remove_all_std, remove_std, rename, O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY,
O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename,
};
use crate::disk::os::{check_path_length, is_empty_dir};
use crate::disk::STORAGE_FORMAT_FILE;
use crate::disk::{
conv_part_err_to_int, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
CHECK_PART_VOLUME_NOT_FOUND,
CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND,
conv_part_err_to_int,
};
use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
use crate::heal::data_scanner::{
lc_has_active_rules, rep_has_active_rules, scan_data_folder, ScannerItem, ShouldSleepFn, SizeSummary,
ScannerItem, ShouldSleepFn, SizeSummary, lc_has_active_rules, rep_has_active_rules, scan_data_folder,
};
use crate::heal::data_scanner_metric::{ScannerMetric, ScannerMetrics};
use crate::heal::data_usage_cache::{DataUsageCache, DataUsageEntry};
@@ -35,23 +35,23 @@ use crate::new_object_layer_fn;
use crate::store_api::{ObjectInfo, StorageAPI};
use crate::utils::os::get_info;
use crate::utils::path::{
clean, decode_dir_object, encode_dir_object, has_suffix, path_join, path_join_buf, GLOBAL_DIR_SUFFIX,
GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR,
GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR, clean, decode_dir_object, encode_dir_object, has_suffix,
path_join, path_join_buf,
};
use common::defer;
use path_absolutize::Absolutize;
use rustfs_filemeta::{
get_file_info, read_xl_meta_no_data, Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, Opts,
RawFileInfo, UpdateFn,
Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, Opts, RawFileInfo, UpdateFn, get_file_info,
read_xl_meta_no_data,
};
use rustfs_rio::{bitrot_verify, Reader};
use rustfs_rio::{Reader, bitrot_verify};
use rustfs_utils::HashAlgorithm;
use std::collections::{HashMap, HashSet};
use std::fmt::Debug;
use std::io::SeekFrom;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
use std::time::{Duration, SystemTime};
use std::{
fs::Metadata,
@@ -60,8 +60,8 @@ use std::{
use time::OffsetDateTime;
use tokio::fs::{self, File};
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind};
use tokio::sync::mpsc::Sender;
use tokio::sync::RwLock;
use tokio::sync::mpsc::Sender;
use tracing::{debug, error, info, warn};
use uuid::Uuid;
@@ -1550,7 +1550,7 @@ impl DiskAPI for LocalDisk {
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
#[tracing::instrument(level = "debug", skip(self, fi))]
async fn rename_data(
&self,
src_volume: &str,
ecstore/src/disks_layout.rs
+13 -13
View File
@@ -1,8 +1,8 @@
use crate::utils::ellipses::*;
use common::error::{Error, Result};
use serde::Deserialize;
use std::collections::HashSet;
use std::env;
use std::io::{Error, Result};
use tracing::debug;
/// Supported set sizes this is used to find the optimal
@@ -89,7 +89,7 @@ pub struct DisksLayout {
impl DisksLayout {
pub fn from_volumes<T: AsRef<str>>(args: &[T]) -> Result<Self> {
if args.is_empty() {
return Err(Error::from_string("Invalid argument"));
return Err(Error::other("Invalid argument"));
}
let is_ellipses = args.iter().any(|v| has_ellipses(&[v]));
@@ -98,7 +98,7 @@ impl DisksLayout {
debug!("{} not set use default:0, {:?}", ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, err);
"0".to_string()
});
let set_drive_count: usize = set_drive_count_env.parse()?;
let set_drive_count: usize = set_drive_count_env.parse().map_err(Error::other)?;
// None of the args have ellipses use the old style.
if !is_ellipses {
@@ -116,7 +116,7 @@ impl DisksLayout {
let mut layout = Vec::with_capacity(args.len());
for arg in args.iter() {
if !has_ellipses(&[arg]) && args.len() > 1 {
return Err(Error::from_string(
return Err(Error::other(
"all args must have ellipses for pool expansion (Invalid arguments specified)",
));
}
@@ -189,7 +189,7 @@ fn get_all_sets<T: AsRef<str>>(set_drive_count: usize, is_ellipses: bool, args:
for args in set_args.iter() {
for arg in args {
if unique_args.contains(arg) {
return Err(Error::from_string(format!("Input args {} has duplicate ellipses", arg)));
return Err(Error::other(format!("Input args {} has duplicate ellipses", arg)));
}
unique_args.insert(arg);
}
@@ -245,7 +245,7 @@ impl EndpointSet {
}
}
pub fn from_volumes<T: AsRef<str>>(args: &[T], set_drive_count: usize) -> Result<Self, Error> {
pub fn from_volumes<T: AsRef<str>>(args: &[T], set_drive_count: usize) -> Result<Self> {
let mut arg_patterns = Vec::with_capacity(args.len());
for arg in args {
arg_patterns.push(find_ellipses_patterns(arg.as_ref())?);
@@ -377,20 +377,20 @@ fn get_set_indexes<T: AsRef<str>>(
arg_patterns: &[ArgPattern],
) -> Result<Vec<Vec<usize>>> {
if args.is_empty() || total_sizes.is_empty() {
return Err(Error::from_string("Invalid argument"));
return Err(Error::other("Invalid argument"));
}
for &size in total_sizes {
// Check if total_sizes has minimum range upto set_size
if size < SET_SIZES[0] || size < set_drive_count {
return Err(Error::from_string(format!("Incorrect number of endpoints provided, size {}", size)));
return Err(Error::other(format!("Incorrect number of endpoints provided, size {}", size)));
}
}
let common_size = get_divisible_size(total_sizes);
let mut set_counts = possible_set_counts(common_size);
if set_counts.is_empty() {
return Err(Error::from_string(format!(
return Err(Error::other(format!(
"Incorrect number of endpoints provided, number of drives {} is not divisible by any supported erasure set sizes {}",
common_size, 0
)));
@@ -399,7 +399,7 @@ fn get_set_indexes<T: AsRef<str>>(
// Returns possible set counts with symmetry.
set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns);
if set_counts.is_empty() {
return Err(Error::from_string("No symmetric distribution detected with input endpoints provided"));
return Err(Error::other("No symmetric distribution detected with input endpoints provided"));
}
let set_size = {
@@ -407,7 +407,7 @@ fn get_set_indexes<T: AsRef<str>>(
let has_set_drive_count = set_counts.contains(&set_drive_count);
if !has_set_drive_count {
return Err(Error::from_string(format!(
return Err(Error::other(format!(
"Invalid set drive count {}. Acceptable values for {:?} number drives are {:?}",
set_drive_count, common_size, &set_counts
)));
@@ -416,7 +416,7 @@ fn get_set_indexes<T: AsRef<str>>(
} else {
set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns);
if set_counts.is_empty() {
return Err(Error::from_string(format!(
return Err(Error::other(format!(
"No symmetric distribution detected with input endpoints , drives {} cannot be spread symmetrically by any supported erasure set sizes {:?}",
common_size, &set_counts
)));
@@ -427,7 +427,7 @@ fn get_set_indexes<T: AsRef<str>>(
};
if !is_valid_set_size(set_size) {
return Err(Error::from_string("Incorrect number of endpoints provided3"));
return Err(Error::other("Incorrect number of endpoints provided3"));
}
Ok(total_sizes
ecstore/src/endpoints.rs
+28 -31
View File
@@ -6,9 +6,9 @@ use crate::{
global::global_rustfs_port,
utils::net::{self, XHost},
};
use common::error::{Error, Result};
use std::io::{Error, Result};
use std::{
collections::{hash_map::Entry, HashMap, HashSet},
collections::{HashMap, HashSet, hash_map::Entry},
net::IpAddr,
};
@@ -76,7 +76,7 @@ impl<T: AsRef<str>> TryFrom<&[T]> for Endpoints {
for (i, arg) in args.iter().enumerate() {
let endpoint = match Endpoint::try_from(arg.as_ref()) {
Ok(ep) => ep,
Err(e) => return Err(Error::from_string(format!("'{}': {}", arg.as_ref(), e))),
Err(e) => return Err(Error::other(format!("'{}': {}", arg.as_ref(), e))),
};
// All endpoints have to be same type and scheme if applicable.
@@ -84,15 +84,15 @@ impl<T: AsRef<str>> TryFrom<&[T]> for Endpoints {
endpoint_type = Some(endpoint.get_type());
schema = Some(endpoint.url.scheme().to_owned());
} else if Some(endpoint.get_type()) != endpoint_type {
return Err(Error::from_string("mixed style endpoints are not supported"));
return Err(Error::other("mixed style endpoints are not supported"));
} else if Some(endpoint.url.scheme()) != schema.as_deref() {
return Err(Error::from_string("mixed scheme is not supported"));
return Err(Error::other("mixed scheme is not supported"));
}
// Check for duplicate endpoints.
let endpoint_str = endpoint.to_string();
if uniq_set.contains(&endpoint_str) {
return Err(Error::from_string("duplicate endpoints found"));
return Err(Error::other("duplicate endpoints found"));
}
uniq_set.insert(endpoint_str);
@@ -156,7 +156,7 @@ impl PoolEndpointList {
/// hostnames and discovers those are local or remote.
fn create_pool_endpoints(server_addr: &str, disks_layout: &DisksLayout) -> Result<Self> {
if disks_layout.is_empty_layout() {
return Err(Error::from_string("invalid number of endpoints"));
return Err(Error::other("invalid number of endpoints"));
}
let server_addr = net::check_local_server_addr(server_addr)?;
@@ -167,7 +167,7 @@ impl PoolEndpointList {
endpoint.update_is_local(server_addr.port())?;
if endpoint.get_type() != EndpointType::Path {
return Err(Error::from_string("use path style endpoint for single node setup"));
return Err(Error::other("use path style endpoint for single node setup"));
}
endpoint.set_pool_index(0);
@@ -201,7 +201,7 @@ impl PoolEndpointList {
}
if endpoints.as_ref().is_empty() {
return Err(Error::from_string("invalid number of endpoints"));
return Err(Error::other("invalid number of endpoints"));
}
pool_endpoints.push(endpoints);
@@ -227,15 +227,14 @@ impl PoolEndpointList {
let host = ep.url.host().unwrap();
let host_ip_set = host_ip_cache.entry(host.clone()).or_insert({
net::get_host_ip(host.clone())
.map_err(|e| Error::from_string(format!("host '{}' cannot resolve: {}", host, e)))?
net::get_host_ip(host.clone()).map_err(|e| Error::other(format!("host '{}' cannot resolve: {}", host, e)))?
});
let path = ep.get_file_path();
match path_ip_map.entry(path) {
Entry::Occupied(mut e) => {
if e.get().intersection(host_ip_set).count() > 0 {
return Err(Error::from_string(format!(
return Err(Error::other(format!(
"same path '{}' can not be served by different port on same address",
path
)));
@@ -257,7 +256,7 @@ impl PoolEndpointList {
let path = ep.get_file_path();
if local_path_set.contains(path) {
return Err(Error::from_string(format!(
return Err(Error::other(format!(
"path '{}' cannot be served by different address on same server",
path
)));
@@ -285,7 +284,7 @@ impl PoolEndpointList {
// If all endpoints have same port number, Just treat it as local erasure setup
// using URL style endpoints.
if local_port_set.len() == 1 && local_server_host_set.len() > 1 {
return Err(Error::from_string("all local endpoints should not have different hostnames/ips"));
return Err(Error::other("all local endpoints should not have different hostnames/ips"));
}
}
@@ -453,7 +452,7 @@ impl EndpointServerPools {
/// both ellipses and without ellipses transparently.
pub fn create_server_endpoints(server_addr: &str, disks_layout: &DisksLayout) -> Result<(EndpointServerPools, SetupType)> {
if disks_layout.pools.is_empty() {
return Err(Error::from_string("Invalid arguments specified"));
return Err(Error::other("Invalid arguments specified"));
}
let pool_eps = PoolEndpointList::create_pool_endpoints(server_addr, disks_layout)?;
@@ -490,7 +489,7 @@ impl EndpointServerPools {
for ep in eps.endpoints.as_ref() {
if exits.contains(&ep.to_string()) {
return Err(Error::from_string("duplicate endpoints found"));
return Err(Error::other("duplicate endpoints found"));
}
}
@@ -664,8 +663,8 @@ mod test {
None,
6,
),
(vec!["d1", "d2", "d3", "d1"], Some(Error::from_string("duplicate endpoints found")), 7),
(vec!["d1", "d2", "d3", "./d1"], Some(Error::from_string("duplicate endpoints found")), 8),
(vec!["d1", "d2", "d3", "d1"], Some(Error::other("duplicate endpoints found")), 7),
(vec!["d1", "d2", "d3", "./d1"], Some(Error::other("duplicate endpoints found")), 8),
(
vec![
"http://localhost/d1",
@@ -673,17 +672,17 @@ mod test {
"http://localhost/d1",
"http://localhost/d4",
],
Some(Error::from_string("duplicate endpoints found")),
Some(Error::other("duplicate endpoints found")),
9,
),
(
vec!["ftp://server/d1", "http://server/d2", "http://server/d3", "http://server/d4"],
Some(Error::from_string("'ftp://server/d1': invalid URL endpoint format")),
Some(Error::other("'ftp://server/d1': invalid URL endpoint format")),
10,
),
(
vec!["d1", "http://localhost/d2", "d3", "d4"],
Some(Error::from_string("mixed style endpoints are not supported")),
Some(Error::other("mixed style endpoints are not supported")),
11,
),
(
@@ -693,7 +692,7 @@ mod test {
"http://example.net/d1",
"https://example.edut/d1",
],
Some(Error::from_string("mixed scheme is not supported")),
Some(Error::other("mixed scheme is not supported")),
12,
),
(
@@ -703,7 +702,7 @@ mod test {
"192.168.1.210:9000/tmp/dir2",
"192.168.110:9000/tmp/dir3",
],
Some(Error::from_string(
Some(Error::other(
"'192.168.1.210:9000/tmp/dir0': invalid URL endpoint format: missing scheme http or https",
)),
13,
@@ -811,7 +810,7 @@ mod test {
TestCase {
num: 1,
server_addr: "localhost",
expected_err: Some(Error::from_string("address localhost: missing port in address")),
expected_err: Some(Error::other("address localhost: missing port in address")),
..Default::default()
},
// Erasure Single Drive
@@ -819,7 +818,7 @@ mod test {
num: 2,
server_addr: "localhost:9000",
args: vec!["http://localhost/d1"],
expected_err: Some(Error::from_string("use path style endpoint for single node setup")),
expected_err: Some(Error::other("use path style endpoint for single node setup")),
..Default::default()
},
TestCase {
@@ -859,7 +858,7 @@ mod test {
"https://example.com/d1",
"https://example.com/d2",
],
expected_err: Some(Error::from_string("same path '/d1' can not be served by different port on same address")),
expected_err: Some(Error::other("same path '/d1' can not be served by different port on same address")),
..Default::default()
},
// Erasure Setup with PathEndpointType
@@ -953,7 +952,7 @@ mod test {
"http://127.0.0.1/d3",
"http://127.0.0.1/d4",
],
expected_err: Some(Error::from_string("all local endpoints should not have different hostnames/ips")),
expected_err: Some(Error::other("all local endpoints should not have different hostnames/ips")),
..Default::default()
},
TestCase {
@@ -965,9 +964,7 @@ mod test {
case7_endpoint1.as_str(),
"http://10.0.0.2:9001/export",
],
expected_err: Some(Error::from_string(
"same path '/export' can not be served by different port on same address",
)),
expected_err: Some(Error::other("same path '/export' can not be served by different port on same address")),
..Default::default()
},
TestCase {
@@ -979,7 +976,7 @@ mod test {
"http://10.0.0.1:9000/export",
"http://10.0.0.2:9000/export",
],
expected_err: Some(Error::from_string("path '/export' cannot be served by different address on same server")),
expected_err: Some(Error::other("path '/export' cannot be served by different address on same server")),
..Default::default()
},
// DistErasure type
ecstore/src/erasure.rs
+2 -6
View File
@@ -1,6 +1,6 @@
use crate::bitrot::{BitrotReader, BitrotWriter};
use crate::disk::error::{Error, Result};
use crate::disk::error_reduce::{reduce_write_quorum_errs, OBJECT_OP_IGNORED_ERRS};
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_write_quorum_errs};
use crate::io::Etag;
use bytes::{Bytes, BytesMut};
use futures::future::join_all;
@@ -72,11 +72,7 @@ impl Erasure {
if total_size > 0 {
let new_len = {
let remain = total_size - total;
if remain > self.block_size {
self.block_size
} else {
remain
}
if remain > self.block_size { self.block_size } else { remain }
};
if new_len == 0 && total > 0 {
ecstore/src/error.rs
+6
View File
@@ -164,6 +164,9 @@ pub enum StorageError {
#[error("first disk wiat")]
FirstDiskWait,
#[error("Bucket policy not found")]
BucketPolicyNotFound,
#[error("Io error: {0}")]
Io(std::io::Error),
}
@@ -376,6 +379,7 @@ impl Clone for StorageError {
StorageError::FirstDiskWait => StorageError::FirstDiskWait,
StorageError::TooManyOpenFiles => StorageError::TooManyOpenFiles,
StorageError::NoHealRequired => StorageError::NoHealRequired,
StorageError::BucketPolicyNotFound => StorageError::BucketPolicyNotFound,
}
}
}
@@ -438,6 +442,7 @@ impl StorageError {
StorageError::ConfigNotFound => 0x35,
StorageError::TooManyOpenFiles => 0x36,
StorageError::NoHealRequired => 0x37,
StorageError::BucketPolicyNotFound => 0x38,
}
}
@@ -502,6 +507,7 @@ impl StorageError {
0x35 => Some(StorageError::ConfigNotFound),
0x36 => Some(StorageError::TooManyOpenFiles),
0x37 => Some(StorageError::NoHealRequired),
0x38 => Some(StorageError::BucketPolicyNotFound),
_ => None,
}
}
ecstore/src/file_meta.rs
+3399 -3399
View File
File diff suppressed because it is too large Load Diff
ecstore/src/file_meta_inline.rs
+1 -5
View File
@@ -27,11 +27,7 @@ impl InlineData {
}
pub fn after_version(&self) -> &[u8] {
if self.0.is_empty() {
&self.0
} else {
&self.0[1..]
}
if self.0.is_empty() { &self.0 } else { &self.0[1..] }
}
pub fn find(&self, key: &str) -> Result<Option<Vec<u8>>> {
ecstore/src/set_disk.rs
+56 -70
View File
@@ -1,7 +1,7 @@
use crate::disk::error_reduce::{reduce_read_quorum_errs, reduce_write_quorum_errs, OBJECT_OP_IGNORED_ERRS};
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs};
use crate::disk::{
self, conv_part_err_to_int, has_part_err, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND,
CHECK_PART_SUCCESS,
self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS,
conv_part_err_to_int, has_part_err,
};
use crate::erasure_coding;
use crate::error::{Error, Result};
@@ -9,24 +9,24 @@ use crate::global::GLOBAL_MRFState;
use crate::heal::data_usage_cache::DataUsageCache;
use crate::store_api::ObjectToDelete;
use crate::{
cache_value::metacache_set::{list_path_raw, ListPathRawOptions},
config::{storageclass, GLOBAL_StorageClass},
cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
config::{GLOBAL_StorageClass, storageclass},
disk::{
endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo,
DiskInfoOptions, DiskOption, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
UpdateMetadataOpts, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, FileInfoVersions,
RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions,
UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk,
},
error::{to_object_err, StorageError},
error::{StorageError, to_object_err},
global::{
get_global_deployment_id, is_dist_erasure, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP,
GLOBAL_LOCAL_DISK_SET_DRIVES,
GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id,
is_dist_erasure,
},
heal::{
data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT},
data_usage_cache::{DataUsageCacheInfo, DataUsageEntry, DataUsageEntryInfo},
heal_commands::{
HealOpts, HealScanMode, HealingTracker, DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE,
DRIVE_STATE_OK, HEAL_DEEP_SCAN, HEAL_ITEM_OBJECT, HEAL_NORMAL_SCAN,
DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, HEAL_DEEP_SCAN, HEAL_ITEM_OBJECT,
HEAL_NORMAL_SCAN, HealOpts, HealScanMode, HealingTracker,
},
heal_ops::BG_HEALING_UUID,
},
@@ -39,13 +39,13 @@ use crate::{
store_init::load_format_erasure,
utils::{
crypto::{base64_decode, base64_encode, hex},
path::{encode_dir_object, has_suffix, SLASH_SEPARATOR},
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix},
},
xhttp,
};
use crate::{disk::STORAGE_FORMAT_FILE, heal::mrf::PartialOperation};
use crate::{
heal::data_scanner::{globalHealConfig, HEAL_DELETE_DANGLING},
heal::data_scanner::{HEAL_DELETE_DANGLING, globalHealConfig},
store_api::ListObjectVersionsInfo,
};
use crate::{
@@ -57,18 +57,18 @@ use chrono::Utc;
use futures::future::join_all;
use glob::Pattern;
use http::HeaderMap;
use lock::{namespace_lock::NsLockMap, LockApi};
use lock::{LockApi, namespace_lock::NsLockMap};
use madmin::heal_commands::{HealDriveInfo, HealResultItem};
use md5::{Digest as Md5Digest, Md5};
use rand::{
thread_rng,
{seq::SliceRandom, Rng},
{Rng, seq::SliceRandom},
};
use rustfs_filemeta::{
file_info_from_raw, merge_file_meta_versions, FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry,
MetadataResolutionParams, ObjectPartInfo, RawFileInfo,
FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo,
RawFileInfo, file_info_from_raw, merge_file_meta_versions,
};
use rustfs_rio::{bitrot_verify, BitrotReader, BitrotWriter, EtagResolvable, HashReader, Writer};
use rustfs_rio::{BitrotReader, BitrotWriter, EtagResolvable, HashReader, Writer, bitrot_verify};
use rustfs_utils::HashAlgorithm;
use sha2::{Digest, Sha256};
use std::hash::Hash;
@@ -84,8 +84,8 @@ use std::{
};
use time::OffsetDateTime;
use tokio::{
io::{empty, AsyncWrite},
sync::{broadcast, RwLock},
io::{AsyncWrite, empty},
sync::{RwLock, broadcast},
};
use tokio::{
select,
@@ -406,11 +406,7 @@ impl SetDisks {
}
}
if max >= write_quorum {
data_dir
} else {
None
}
if max >= write_quorum { data_dir } else { None }
}
#[allow(dead_code)]
@@ -741,11 +737,7 @@ impl SetDisks {
fn common_time(times: &[Option<OffsetDateTime>], quorum: usize) -> Option<OffsetDateTime> {
let (time, count) = Self::common_time_and_occurrence(times);
if count >= quorum {
time
} else {
None
}
if count >= quorum { time } else { None }
}
fn common_time_and_occurrence(times: &[Option<OffsetDateTime>]) -> (Option<OffsetDateTime>, usize) {
@@ -786,11 +778,7 @@ impl SetDisks {
fn common_etag(etags: &[Option<String>], quorum: usize) -> Option<String> {
let (etag, count) = Self::common_etags(etags);
if count >= quorum {
etag
} else {
None
}
if count >= quorum { etag } else { None }
}
fn common_etags(etags: &[Option<String>]) -> (Option<String>, usize) {
@@ -1837,13 +1825,7 @@ impl SetDisks {
let total_size = fi.size;
let length = {
if length == 0 {
total_size - offset
} else {
length
}
};
let length = { if length == 0 { total_size - offset } else { length } };
if offset > total_size || offset + length > total_size {
return Err(Error::other("offset out of range"));
@@ -1896,12 +1878,16 @@ impl SetDisks {
readers.push(Some(reader));
errors.push(None);
} else if let Some(disk) = disk_op {
// Calculate ceiling division of till_offset by shard_size
let till_offset =
till_offset.div_ceil(erasure.shard_size()) * HashAlgorithm::HighwayHash256.size() + till_offset;
let rd = disk
.read_file_stream(
bucket,
&format!("{}/{}/part.{}", object, files[idx].data_dir.unwrap_or(Uuid::nil()), part_number),
part_offset,
till_offset,
part_length,
)
.await?;
let reader = BitrotReader::new(rd, erasure.shard_size(), HashAlgorithm::HighwayHash256);
@@ -2403,8 +2389,10 @@ impl SetDisks {
}
if !lastest_meta.deleted && lastest_meta.erasure.distribution.len() != available_disks.len() {
let err_str = format!("unexpected file distribution ({:?}) from available disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
lastest_meta.erasure.distribution, available_disks, bucket, object, version_id);
let err_str = format!(
"unexpected file distribution ({:?}) from available disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
lastest_meta.erasure.distribution, available_disks, bucket, object, version_id
);
warn!(err_str);
let err = DiskError::other(err_str);
return Ok((
@@ -2416,8 +2404,10 @@ impl SetDisks {
let latest_disks = Self::shuffle_disks(&available_disks, &lastest_meta.erasure.distribution);
if !lastest_meta.deleted && lastest_meta.erasure.distribution.len() != outdate_disks.len() {
let err_str = format!("unexpected file distribution ({:?}) from outdated disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
lastest_meta.erasure.distribution, outdate_disks, bucket, object, version_id);
let err_str = format!(
"unexpected file distribution ({:?}) from outdated disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
lastest_meta.erasure.distribution, outdate_disks, bucket, object, version_id
);
warn!(err_str);
let err = DiskError::other(err_str);
return Ok((
@@ -2428,8 +2418,14 @@ impl SetDisks {
}
if !lastest_meta.deleted && lastest_meta.erasure.distribution.len() != parts_metadata.len() {
let err_str = format!("unexpected file distribution ({:?}) from metadata entries ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
lastest_meta.erasure.distribution, parts_metadata.len(), bucket, object, version_id);
let err_str = format!(
"unexpected file distribution ({:?}) from metadata entries ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
lastest_meta.erasure.distribution,
parts_metadata.len(),
bucket,
object,
version_id
);
warn!(err_str);
let err = DiskError::other(err_str);
return Ok((
@@ -3907,6 +3903,7 @@ impl ObjectIO for SetDisks {
};
writers.push(Some(writer));
errors.push(None);
} else {
errors.push(Some(DiskError::DiskNotFound));
writers.push(None);
@@ -3915,6 +3912,7 @@ impl ObjectIO for SetDisks {
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
if nil_count < write_quorum {
error!("not enough disks to write: {:?}", errors);
if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) {
return Err(to_object_err(write_err.into(), vec![bucket, object]));
}
@@ -3926,7 +3924,7 @@ impl ObjectIO for SetDisks {
let (reader, w_size) = Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?; // TODO: 出错，删除临时目录
mem::replace(&mut data.stream, reader);
let _ = mem::replace(&mut data.stream, reader);
// if let Err(err) = close_bitrot_writers(&mut writers).await {
// error!("close_bitrot_writers err {:?}", err);
// }
@@ -4549,25 +4547,11 @@ impl StorageAPI for SetDisks {
let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let is_inline_buffer = {
if let Some(sc) = GLOBAL_StorageClass.get() {
sc.should_inline(erasure.shard_file_size(data.content_length), opts.versioned)
} else {
false
}
};
let mut writers = Vec::with_capacity(shuffle_disks.len());
let mut errors = Vec::with_capacity(shuffle_disks.len());
for disk_op in shuffle_disks.iter() {
if let Some(disk) = disk_op {
let writer = if is_inline_buffer {
BitrotWriter::new(
Writer::from_cursor(Cursor::new(Vec::new())),
erasure.shard_size(),
HashAlgorithm::HighwayHash256,
)
} else {
let writer = {
let f = match disk
.create_file("", RUSTFS_META_TMP_BUCKET, &tmp_part_path, erasure.shard_file_size(data.content_length))
.await
@@ -4584,6 +4568,7 @@ impl StorageAPI for SetDisks {
};
writers.push(Some(writer));
errors.push(None);
} else {
errors.push(Some(DiskError::DiskNotFound));
writers.push(None);
@@ -4601,8 +4586,9 @@ impl StorageAPI for SetDisks {
let stream = mem::replace(&mut data.stream, HashReader::new(Box::new(Cursor::new(Vec::new())), 0, 0, None, false)?);
let (mut reader, w_size) = Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?; // TODO: 出错，删除临时目录
mem::replace(&mut data.stream, reader);
let (reader, w_size) = Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?; // TODO: 出错，删除临时目录
let _ = mem::replace(&mut data.stream, reader);
let mut etag = data.stream.try_resolve_etag().unwrap_or_default();
@@ -5755,9 +5741,9 @@ fn get_complete_multipart_md5(parts: &[CompletePart]) -> String {
#[cfg(test)]
mod tests {
use super::*;
use crate::disk::error::DiskError;
use crate::disk::CHECK_PART_UNKNOWN;
use crate::disk::CHECK_PART_VOLUME_NOT_FOUND;
use crate::disk::error::DiskError;
use crate::store_api::CompletePart;
use rustfs_filemeta::ErasureInfo;
use std::collections::HashMap;
ecstore/src/utils/bool_flag.rs
+2 -2
View File
@@ -1,9 +1,9 @@
use common::error::{Error, Result};
use std::io::{Error, Result};
pub fn parse_bool(str: &str) -> Result<bool> {
match str {
"1" | "t" | "T" | "true" | "TRUE" | "True" | "on" | "ON" | "On" | "enabled" => Ok(true),
"0" | "f" | "F" | "false" | "FALSE" | "False" | "off" | "OFF" | "Off" | "disabled" => Ok(false),
_ => Err(Error::from_string(format!("ParseBool: parsing {}", str))),
_ => Err(Error::other(format!("ParseBool: parsing {}", str))),
}
}
ecstore/src/utils/ellipses.rs
+15 -9
View File
@@ -1,6 +1,6 @@
use common::error::{Error, Result};
use lazy_static::*;
use regex::Regex;
use std::io::{Error, Result};
lazy_static! {
static ref ELLIPSES_RE: Regex = Regex::new(r"(.*)(\{[0-9a-z]*\.\.\.[0-9a-z]*\})(.*)").unwrap();
@@ -107,7 +107,10 @@ pub fn find_ellipses_patterns(arg: &str) -> Result<ArgPattern> {
let mut parts = match ELLIPSES_RE.captures(arg) {
Some(caps) => caps,
None => {
return Err(Error::from_string(format!("Invalid ellipsis format in ({}), Ellipsis range must be provided in format {{N...M}} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4", arg)));
return Err(Error::other(format!(
"Invalid ellipsis format in ({}), Ellipsis range must be provided in format {{N...M}} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4",
arg
)));
}
};
@@ -144,7 +147,10 @@ pub fn find_ellipses_patterns(arg: &str) -> Result<ArgPattern> {
|| p.suffix.contains(OPEN_BRACES)
|| p.suffix.contains(CLOSE_BRACES)
{
return Err(Error::from_string(format!("Invalid ellipsis format in ({}), Ellipsis range must be provided in format {{N...M}} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4", arg)));
return Err(Error::other(format!(
"Invalid ellipsis format in ({}), Ellipsis range must be provided in format {{N...M}} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4",
arg
)));
}
}
@@ -165,10 +171,10 @@ pub fn has_ellipses<T: AsRef<str>>(s: &[T]) -> bool {
/// {33...64}
pub fn parse_ellipses_range(pattern: &str) -> Result<Vec<String>> {
if !pattern.contains(OPEN_BRACES) {
return Err(Error::from_string("Invalid argument"));
return Err(Error::other("Invalid argument"));
}
if !pattern.contains(OPEN_BRACES) {
return Err(Error::from_string("Invalid argument"));
return Err(Error::other("Invalid argument"));
}
let ellipses_range: Vec<&str> = pattern
@@ -178,15 +184,15 @@ pub fn parse_ellipses_range(pattern: &str) -> Result<Vec<String>> {
.collect();
if ellipses_range.len() != 2 {
return Err(Error::from_string("Invalid argument"));
return Err(Error::other("Invalid argument"));
}
// TODO: Add support for hexadecimals.
let start = ellipses_range[0].parse::<usize>()?;
let end = ellipses_range[1].parse::<usize>()?;
let start = ellipses_range[0].parse::<usize>().map_err(|e| Error::other(e))?;
let end = ellipses_range[1].parse::<usize>().map_err(|e| Error::other(e))?;
if start > end {
return Err(Error::from_string("Invalid argument:range start cannot be bigger than end"));
return Err(Error::other("Invalid argument:range start cannot be bigger than end"));
}
let mut ret: Vec<String> = Vec::with_capacity(end - start + 1);
ecstore/src/utils/net.rs
+11 -14
View File
@@ -1,5 +1,5 @@
use common::error::{Error, Result};
use lazy_static::lazy_static;
use std::io::{Error, Result};
use std::{
collections::HashSet,
fmt::Display,
@@ -23,7 +23,7 @@ pub fn is_socket_addr(addr: &str) -> bool {
pub fn check_local_server_addr(server_addr: &str) -> Result<SocketAddr> {
let addr: Vec<SocketAddr> = match server_addr.to_socket_addrs() {
Ok(addr) => addr.collect(),
Err(err) => return Err(Error::new(Box::new(err))),
Err(err) => return Err(err),
};
// 0.0.0.0 is a wildcard address and refers to local network
@@ -44,7 +44,7 @@ pub fn check_local_server_addr(server_addr: &str) -> Result<SocketAddr> {
}
}
Err(Error::from_string("host in server address should be this server"))
Err(Error::other("host in server address should be this server"))
}
/// checks if the given parameter correspond to one of
@@ -55,7 +55,7 @@ pub fn is_local_host(host: Host<&str>, port: u16, local_port: u16) -> Result<boo
Host::Domain(domain) => {
let ips = match (domain, 0).to_socket_addrs().map(|v| v.map(|v| v.ip()).collect::<Vec<_>>()) {
Ok(ips) => ips,
Err(err) => return Err(Error::new(Box::new(err))),
Err(err) => return Err(err),
};
ips.iter().any(|ip| local_set.contains(ip))
@@ -79,7 +79,7 @@ pub fn get_host_ip(host: Host<&str>) -> Result<HashSet<IpAddr>> {
.map(|v| v.map(|v| v.ip()).collect::<HashSet<_>>())
{
Ok(ips) => Ok(ips),
Err(err) => Err(Error::new(Box::new(err))),
Err(err) => Err(err),
},
Host::Ipv4(ip) => {
let mut set = HashSet::with_capacity(1);
@@ -102,7 +102,7 @@ pub fn get_available_port() -> u16 {
pub(crate) fn must_get_local_ips() -> Result<Vec<IpAddr>> {
match netif::up() {
Ok(up) => Ok(up.map(|x| x.address().to_owned()).collect()),
Err(err) => Err(Error::from_string(format!("Unable to get IP addresses of this host: {}", err))),
Err(err) => Err(Error::other(format!("Unable to get IP addresses of this host: {}", err))),
}
}
@@ -149,7 +149,7 @@ pub fn parse_and_resolve_address(addr_str: &str) -> Result<SocketAddr> {
let port_str = port;
let port: u16 = port_str
.parse()
.map_err(|e| Error::from_string(format!("Invalid port format: {}, err:{:?}", addr_str, e)))?;
.map_err(|e| Error::other(format!("Invalid port format: {}, err:{:?}", addr_str, e)))?;
let final_port = if port == 0 {
get_available_port() // assume get_available_port is available here
} else {
@@ -199,13 +199,10 @@ mod test {
("localhost:54321", Ok(())),
("0.0.0.0:9000", Ok(())),
// (":0", Ok(())),
("localhost", Err(Error::from_string("invalid socket address"))),
("", Err(Error::from_string("invalid socket address"))),
(
"example.org:54321",
Err(Error::from_string("host in server address should be this server")),
),
(":-10", Err(Error::from_string("invalid port value"))),
("localhost", Err(Error::other("invalid socket address"))),
("", Err(Error::other("invalid socket address"))),
("example.org:54321", Err(Error::other("host in server address should be this server"))),
(":-10", Err(Error::other("invalid port value"))),
];
for test_case in test_cases {
ecstore/src/utils/os/linux.rs
+6 -6
View File
@@ -1,11 +1,11 @@
use nix::sys::stat::{self, stat};
use nix::sys::statfs::{self, statfs, FsType};
use nix::sys::statfs::{self, FsType, statfs};
use std::fs::File;
use std::io::{self, BufRead, Error, ErrorKind};
use std::path::Path;
use crate::disk::Info;
use common::error::{Error as e_Error, Result};
use std::io::{Error, Result};
use super::IOStats;
@@ -29,7 +29,7 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<Info> {
bfree,
p.as_ref().display()
),
))
));
}
};
@@ -44,7 +44,7 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<Info> {
blocks,
p.as_ref().display()
),
))
));
}
};
@@ -60,7 +60,7 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<Info> {
total,
p.as_ref().display()
),
))
));
}
};
@@ -122,7 +122,7 @@ pub fn get_drive_stats(major: u32, minor: u32) -> Result<IOStats> {
fn read_drive_stats(stats_file: &str) -> Result<IOStats> {
let stats = read_stat(stats_file)?;
if stats.len() < 11 {
return Err(e_Error::from_string(format!("found invalid format while reading {}", stats_file)));
return Err(Error::new(ErrorKind::Other, format!("found invalid format while reading {}", stats_file)));
}
let mut io_stats = IOStats {
read_ios: stats[0],
ecstore/src/utils/os/unix.rs
+4 -5
View File
@@ -1,8 +1,7 @@
use super::IOStats;
use crate::disk::Info;
use common::error::Result;
use nix::sys::{stat::stat, statfs::statfs};
use std::io::Error;
use std::io::{Error, Result};
use std::path::Path;
/// returns total and free bytes available in a directory, e.g. `/`.
@@ -22,7 +21,7 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<Info> {
bavail,
bfree,
p.as_ref().display()
)))
)));
}
};
@@ -34,7 +33,7 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<Info> {
reserved,
blocks,
p.as_ref().display()
)))
)));
}
};
@@ -47,7 +46,7 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<Info> {
free,
total,
p.as_ref().display()
)))
)));
}
};
ecstore/src/utils/os/windows.rs
+1 -2
View File
@@ -2,8 +2,7 @@
use super::IOStats;
use crate::disk::Info;
use common::error::Result;
use std::io::{Error, ErrorKind};
use std::io::{Error, ErrorKind, Result};
use std::mem;
use std::os::windows::ffi::OsStrExt;
use std::path::Path;
iam/src/error.rs
+15
View File
@@ -115,6 +115,15 @@ impl From<ecstore::error::StorageError> for Error {
}
}
impl From<Error> for ecstore::error::StorageError {
fn from(e: Error) -> Self {
match e {
Error::ConfigNotFound => ecstore::error::StorageError::ConfigNotFound,
_ => ecstore::error::StorageError::other(e),
}
}
}
impl From<policy::error::Error> for Error {
fn from(e: policy::error::Error) -> Self {
match e {
@@ -152,6 +161,12 @@ impl From<policy::error::Error> for Error {
}
}
impl From<Error> for std::io::Error {
fn from(e: Error) -> Self {
std::io::Error::other(e)
}
}
impl From<serde_json::Error> for Error {
fn from(e: serde_json::Error) -> Self {
Error::other(e)
rustfs/Cargo.toml
+1
View File
@@ -88,6 +88,7 @@ tower-http = { workspace = true, features = [
uuid = { workspace = true }
rustfs-filemeta.workspace = true
thiserror.workspace = true
rustfs-rio.workspace = true
[target.'cfg(target_os = "linux")'.dependencies]
libsystemd.workspace = true
rustfs/src/admin/handlers/pools.rs
+5 -5
View File
@@ -7,7 +7,7 @@ use serde_urlencoded::from_bytes;
use tokio::sync::broadcast;
use tracing::warn;
use crate::{admin::router::Operation, storage::error::to_s3_error};
use crate::{admin::router::Operation, error::ApiError};
pub struct ListPools {}
@@ -33,7 +33,7 @@ impl Operation for ListPools {
let mut pools_status = Vec::new();
for (idx, _) in endpoints.as_ref().iter().enumerate() {
let state = store.status(idx).await.map_err(to_s3_error)?;
let state = store.status(idx).await.map_err(ApiError::from)?;
pools_status.push(state);
}
@@ -103,7 +103,7 @@ impl Operation for StatusPool {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let pools_status = store.status(idx).await.map_err(to_s3_error)?;
let pools_status = store.status(idx).await.map_err(ApiError::from)?;
let data = serde_json::to_vec(&pools_status)
.map_err(|_e| S3Error::with_message(S3ErrorCode::InternalError, "parse accountInfo failed"))?;
@@ -191,7 +191,7 @@ impl Operation for StartDecommission {
}
if !pools_indices.is_empty() {
store.decommission(ctx_rx, pools_indices).await.map_err(to_s3_error)?;
store.decommission(ctx_rx, pools_indices).await.map_err(ApiError::from)?;
}
Ok(S3Response::new((StatusCode::OK, Body::default())))
@@ -245,7 +245,7 @@ impl Operation for CancelDecommission {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
store.decommission_cancel(idx).await.map_err(to_s3_error)?;
store.decommission_cancel(idx).await.map_err(ApiError::from)?;
Ok(S3Response::new((StatusCode::OK, Body::default())))
}
rustfs/src/admin/mod.rs
+2 -3
View File
@@ -3,7 +3,6 @@ pub mod router;
mod rpc;
pub mod utils;
use common::error::Result;
// use ecstore::global::{is_dist_erasure, is_erasure};
use handlers::{
group, policys, pools, rebalance,
@@ -18,7 +17,7 @@ use s3s::route::S3Route;
const ADMIN_PREFIX: &str = "/rustfs/admin";
pub fn make_admin_route() -> Result<impl S3Route> {
pub fn make_admin_route() -> std::io::Result<impl S3Route> {
let mut r: S3Router<AdminOperation> = S3Router::new();
// 1
@@ -124,7 +123,7 @@ pub fn make_admin_route() -> Result<impl S3Route> {
Ok(r)
}
fn register_user_route(r: &mut S3Router<AdminOperation>) -> Result<()> {
fn register_user_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
// 1
r.insert(
Method::GET,
rustfs/src/admin/router.rs
+7 -8
View File
@@ -1,21 +1,20 @@
use common::error::Result;
use hyper::http::Extensions;
use hyper::HeaderMap;
use hyper::Method;
use hyper::StatusCode;
use hyper::Uri;
use hyper::http::Extensions;
use matchit::Params;
use matchit::Router;
use s3s::header;
use s3s::route::S3Route;
use s3s::s3_error;
use s3s::Body;
use s3s::S3Request;
use s3s::S3Response;
use s3s::S3Result;
use s3s::header;
use s3s::route::S3Route;
use s3s::s3_error;
use super::rpc::RPC_PREFIX;
use super::ADMIN_PREFIX;
use super::rpc::RPC_PREFIX;
pub struct S3Router<T> {
router: Router<T>,
@@ -28,12 +27,12 @@ impl<T: Operation> S3Router<T> {
Self { router }
}
pub fn insert(&mut self, method: Method, path: &str, operation: T) -> Result<()> {
pub fn insert(&mut self, method: Method, path: &str, operation: T) -> std::io::Result<()> {
let path = Self::make_route_str(method, path);
// warn!("set uri {}", &path);
self.router.insert(path, operation)?;
self.router.insert(path, operation).map_err(|e| std::io::Error::other(e))?;
Ok(())
}
rustfs/src/admin/rpc.rs
+3 -4
View File
@@ -2,7 +2,6 @@ use super::router::AdminOperation;
use super::router::Operation;
use super::router::S3Router;
use crate::storage::ecfs::bytes_stream;
use common::error::Result;
use ecstore::disk::DiskAPI;
use ecstore::io::READ_BUFFER_SIZE;
use ecstore::store::find_local_disk;
@@ -10,19 +9,19 @@ use futures::TryStreamExt;
use http::StatusCode;
use hyper::Method;
use matchit::Params;
use s3s::dto::StreamingBlob;
use s3s::s3_error;
use s3s::Body;
use s3s::S3Request;
use s3s::S3Response;
use s3s::S3Result;
use s3s::dto::StreamingBlob;
use s3s::s3_error;
use serde_urlencoded::from_bytes;
use tokio_util::io::ReaderStream;
use tokio_util::io::StreamReader;
pub const RPC_PREFIX: &str = "/rustfs/rpc";
pub fn regist_rpc_route(r: &mut S3Router<AdminOperation>) -> Result<()> {
pub fn regist_rpc_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
Method::GET,
format!("{}{}", RPC_PREFIX, "/read_file_stream").as_str(),
rustfs/src/error.rs
+87 -1675
View File
File diff suppressed because it is too large Load Diff
rustfs/src/grpc.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use std::{collections::HashMap, io::Cursor, pin::Pin};
use common::error::Error as EcsError;
// use common::error::Error as EcsError;
use ecstore::{
admin_server_info::get_local_server_property,
bucket::{metadata::load_bucket_metadata, metadata_sys},
rustfs/src/license.rs
+5 -5
View File
@@ -1,5 +1,5 @@
use appauth::token::Token;
use common::error::{Error, Result};
use std::io::{Error, Result};
use std::sync::OnceLock;
use std::time::SystemTime;
use std::time::UNIX_EPOCH;
@@ -37,7 +37,7 @@ pub fn license_check() -> Result<()> {
let invalid_license = LICENSE.get().map(|token| {
if token.expired < SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() {
error!("License expired");
return Err(Error::from_string("Incorrect license, please contact RustFS.".to_string()));
return Err(Error::other("Incorrect license, please contact RustFS."));
}
info!("License is valid ! expired at {}", token.expired);
Ok(())
@@ -46,12 +46,12 @@ pub fn license_check() -> Result<()> {
// let invalid_license = config::get_config().license.as_ref().map(|license| {
// if license.is_empty() {
// error!("License is empty");
// return Err(Error::from_string("Incorrect license, please contact RustFS.".to_string()));
// return Err(Error::other("Incorrect license, please contact RustFS.".to_string()));
// }
// let token = appauth::token::parse_license(license)?;
// if token.expired < SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() {
// error!("License expired");
// return Err(Error::from_string("Incorrect license, please contact RustFS.".to_string()));
// return Err(Error::other("Incorrect license, please contact RustFS.".to_string()));
// }
// info!("License is valid ! expired at {}", token.expired);
@@ -59,7 +59,7 @@ pub fn license_check() -> Result<()> {
// });
if invalid_license.is_none() || invalid_license.is_some_and(|v| v.is_err()) {
return Err(Error::from_string("Incorrect license, please contact RustFS.".to_string()));
return Err(Error::other("Incorrect license, please contact RustFS."));
}
Ok(())
rustfs/src/main.rs
+8 -8
View File
@@ -73,7 +73,7 @@ const MI_B: usize = 1024 * 1024;
static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
#[allow(clippy::result_large_err)]
fn check_auth(req: Request<()>) -> Result<Request<()>, Status> {
fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
let token: MetadataValue<_> = "rustfs rpc".parse().unwrap();
match req.metadata().get("authorization") {
@@ -120,7 +120,7 @@ async fn run(opt: config::Opt) -> Result<()> {
// Initialize event notifier
event::init_event_notifier(opt.event_config).await;
let server_addr = net::parse_and_resolve_address(opt.address.as_str())?;
let server_addr = net::parse_and_resolve_address(opt.address.as_str()).map_err(|err| Error::other(err))?;
let server_port = server_addr.port();
let server_address = server_addr.to_string();
@@ -140,7 +140,7 @@ async fn run(opt: config::Opt) -> Result<()> {
// For RPC
let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(server_address.clone().as_str(), opt.volumes.clone())
.map_err(|err| Error::from_string(err.to_string()))?;
.map_err(|err| Error::other(err.to_string()))?;
// Print RustFS-style logging for pool formatting
for (i, eps) in endpoint_pools.as_ref().iter().enumerate() {
@@ -189,7 +189,7 @@ async fn run(opt: config::Opt) -> Result<()> {
// Initialize the local disk
init_local_disks(endpoint_pools.clone())
.await
.map_err(|err| Error::from_string(err.to_string()))?;
.map_err(|err| Error::other(err))?;
// Setup S3 service
// This project uses the S3S library to implement S3 services
@@ -518,7 +518,7 @@ async fn run(opt: config::Opt) -> Result<()> {
..Default::default()
})
.await
.map_err(|err| Error::from_string(err.to_string()))?;
.map_err(|err| Error::other(err))?;
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
@@ -528,7 +528,7 @@ async fn run(opt: config::Opt) -> Result<()> {
new_global_notification_sys(endpoint_pools.clone()).await.map_err(|err| {
error!("new_global_notification_sys failed {:?}", &err);
Error::from_string(err.to_string())
Error::other(err)
})?;
// init scanner
@@ -549,7 +549,7 @@ async fn run(opt: config::Opt) -> Result<()> {
if console_address.is_empty() {
error!("console_address is empty");
return Err(Error::from_string("console_address is empty".to_string()));
return Err(Error::other("console_address is empty".to_string()));
}
tokio::spawn(async move {
@@ -571,7 +571,7 @@ async fn run(opt: config::Opt) -> Result<()> {
// stop event notifier
rustfs_event_notifier::shutdown().await.map_err(|err| {
error!("Failed to shut down the notification system: {}", err);
Error::from_string(err.to_string())
Error::other(err)
})?;
}
rustfs/src/storage/ecfs.rs
+111 -105
View File
@@ -3,8 +3,9 @@ use super::options::del_opts;
use super::options::extract_metadata;
use super::options::put_opts;
use crate::auth::get_condition_values;
use crate::error::ApiError;
use crate::error::Result;
use crate::storage::access::ReqInfo;
use crate::storage::error::to_s3_error;
use crate::storage::options::copy_dst_opts;
use crate::storage::options::copy_src_opts;
use crate::storage::options::{extract_metadata_from_mime, get_opts};
@@ -12,11 +13,9 @@ use api::query::Context;
use api::query::Query;
use api::server::dbms::DatabaseManagerSystem;
use bytes::Bytes;
use common::error::Result;
use datafusion::arrow::csv::WriterBuilder as CsvWriterBuilder;
use datafusion::arrow::json::writer::JsonArray;
use datafusion::arrow::json::WriterBuilder as JsonWriterBuilder;
use ecstore::bucket::error::BucketMetadataError;
use datafusion::arrow::json::writer::JsonArray;
use ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
use ecstore::bucket::metadata::BUCKET_NOTIFICATION_CONFIG;
use ecstore::bucket::metadata::BUCKET_POLICY_CONFIG;
@@ -30,6 +29,7 @@ use ecstore::bucket::policy_sys::PolicySys;
use ecstore::bucket::tagging::decode_tags;
use ecstore::bucket::tagging::encode_tags;
use ecstore::bucket::versioning_sys::BucketVersioningSys;
use ecstore::error::StorageError;
use ecstore::io::READ_BUFFER_SIZE;
use ecstore::new_object_layer_fn;
use ecstore::store_api::BucketOptions;
@@ -42,8 +42,8 @@ use ecstore::store_api::ObjectIO;
use ecstore::store_api::ObjectOptions;
use ecstore::store_api::ObjectToDelete;
use ecstore::store_api::PutObjReader;
use ecstore::store_api::StorageAPI;
use ecstore::store_api::RESERVED_METADATA_PREFIX_LOWER;
use ecstore::store_api::StorageAPI;
use ecstore::utils::path::path_join_buf;
use ecstore::utils::xml;
use ecstore::xhttp;
@@ -52,27 +52,28 @@ use futures::{Stream, StreamExt};
use http::HeaderMap;
use lazy_static::lazy_static;
use policy::auth;
use policy::policy::action::Action;
use policy::policy::action::S3Action;
use policy::policy::BucketPolicy;
use policy::policy::BucketPolicyArgs;
use policy::policy::Validator;
use policy::policy::action::Action;
use policy::policy::action::S3Action;
use query::instance::make_rustfsms;
use rustfs_rio::HashReader;
use rustfs_zip::CompressionFormat;
use s3s::dto::*;
use s3s::s3_error;
use s3s::S3;
use s3s::S3Error;
use s3s::S3ErrorCode;
use s3s::S3Result;
use s3s::S3;
use s3s::dto::*;
use s3s::s3_error;
use s3s::{S3Request, S3Response};
use std::collections::HashMap;
use std::fmt::Debug;
use std::path::Path;
use std::str::FromStr;
use std::sync::Arc;
use time::format_description::well_known::Rfc3339;
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
use tokio::sync::mpsc;
use tokio_stream::wrappers::ReceiverStream;
use tokio_tar::Archive;
@@ -175,12 +176,15 @@ impl FS {
println!("Extracted: {}, size {}", fpath, size);
let mut reader = PutObjReader::new(Box::new(f), size);
// Wrap the tar entry with BufReader to make it compatible with Reader trait
let reader = Box::new(tokio::io::BufReader::new(f));
let hrd = HashReader::new(reader, size as i64, size as i64, None, false).map_err(ApiError::from)?;
let mut reader = PutObjReader::new(hrd, size);
let _obj_info = store
.put_object(&bucket, &fpath, &mut reader, &ObjectOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
// let e_tag = obj_info.etag;
@@ -244,7 +248,7 @@ impl S3 for FS {
},
)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let output = CreateBucketOutput::default();
Ok(S3Response::new(output))
@@ -270,7 +274,7 @@ impl S3 for FS {
// warn!("copy_object {}/{}, to {}/{}", &src_bucket, &src_key, &bucket, &key);
let mut src_opts = copy_src_opts(&src_bucket, &src_key, &req.headers).map_err(to_s3_error)?;
let mut src_opts = copy_src_opts(&src_bucket, &src_key, &req.headers).map_err(ApiError::from)?;
src_opts.version_id = version_id.clone();
@@ -283,7 +287,7 @@ impl S3 for FS {
let dst_opts = copy_dst_opts(&bucket, &key, version_id, &req.headers, None)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let cp_src_dst_same = path_join_buf(&[&src_bucket, &src_key]) == path_join_buf(&[&bucket, &key]);
@@ -300,7 +304,7 @@ impl S3 for FS {
let gr = store
.get_object_reader(&src_bucket, &src_key, None, h, &get_opts)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let mut src_info = gr.object_info.clone();
@@ -308,8 +312,11 @@ impl S3 for FS {
src_info.metadata_only = true;
}
let hrd = HashReader::new(gr.stream, gr.object_info.size as i64, gr.object_info.size as i64, None, false)
.map_err(ApiError::from)?;
src_info.put_object_reader = Some(PutObjReader {
stream: gr.stream,
stream: hrd,
content_length: gr.object_info.size as usize,
});
@@ -320,7 +327,7 @@ impl S3 for FS {
let oi = store
.copy_object(&src_bucket, &src_key, &bucket, &key, &mut src_info, &src_opts, &dst_opts)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
// warn!("copy_object oi {:?}", &oi);
@@ -355,7 +362,7 @@ impl S3 for FS {
},
)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
Ok(S3Response::new(DeleteBucketOutput {}))
}
@@ -371,7 +378,7 @@ impl S3 for FS {
let opts: ObjectOptions = del_opts(&bucket, &key, version_id, &req.headers, Some(metadata))
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let version_id = opts.version_id.as_ref().map(|v| Uuid::parse_str(v).ok()).unwrap_or_default();
let dobj = ObjectToDelete {
@@ -384,7 +391,7 @@ impl S3 for FS {
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let (dobjs, _errs) = store.delete_objects(&bucket, objects, opts).await.map_err(to_s3_error)?;
let (dobjs, _errs) = store.delete_objects(&bucket, objects, opts).await.map_err(ApiError::from)?;
// TODO: let errors;
@@ -392,13 +399,7 @@ impl S3 for FS {
if let Some((a, b)) = dobjs
.iter()
.map(|v| {
let delete_marker = {
if v.delete_marker {
Some(true)
} else {
None
}
};
let delete_marker = { if v.delete_marker { Some(true) } else { None } };
let version_id = v.version_id.clone();
@@ -447,20 +448,14 @@ impl S3 for FS {
let opts: ObjectOptions = del_opts(&bucket, "", None, &req.headers, Some(metadata))
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let (dobjs, errs) = store.delete_objects(&bucket, objects, opts).await.map_err(to_s3_error)?;
let (dobjs, errs) = store.delete_objects(&bucket, objects, opts).await.map_err(ApiError::from)?;
let deleted = dobjs
.iter()
.map(|v| DeletedObject {
delete_marker: {
if v.delete_marker {
Some(true)
} else {
None
}
},
delete_marker: { if v.delete_marker { Some(true) } else { None } },
delete_marker_version_id: v.delete_marker_version_id.clone(),
key: Some(v.object_name.clone()),
version_id: v.version_id.clone(),
@@ -493,7 +488,7 @@ impl S3 for FS {
store
.get_bucket_info(&input.bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let output = GetBucketLocationOutput::default();
Ok(S3Response::new(output))
@@ -550,7 +545,7 @@ impl S3 for FS {
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, part_number, &req.headers)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
@@ -559,7 +554,7 @@ impl S3 for FS {
let reader = store
.get_object_reader(bucket.as_str(), key.as_str(), rs, h, &opts)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let info = reader.object_info;
@@ -606,7 +601,7 @@ impl S3 for FS {
store
.get_bucket_info(&input.bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
// mc cp step 2 GetBucketInfo
Ok(S3Response::new(HeadBucketOutput::default()))
@@ -651,13 +646,13 @@ impl S3 for FS {
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, part_number, &req.headers)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let info = store.get_object_info(&bucket, &key, &opts).await.map_err(to_s3_error)?;
let info = store.get_object_info(&bucket, &key, &opts).await.map_err(ApiError::from)?;
// warn!("head_object info {:?}", &info);
@@ -700,7 +695,7 @@ impl S3 for FS {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let mut bucket_infos = store.list_bucket(&BucketOptions::default()).await.map_err(to_s3_error)?;
let mut bucket_infos = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?;
let mut req = req;
@@ -792,7 +787,7 @@ impl S3 for FS {
start_after,
)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
// warn!("object_infos objects {:?}", object_infos.objects);
@@ -873,7 +868,7 @@ impl S3 for FS {
let object_infos = store
.list_object_versions(&bucket, &prefix, key_marker, version_id_marker, delimiter.clone(), max_keys)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let objects: Vec<ObjectVersion> = object_infos
.objects
@@ -960,9 +955,14 @@ impl S3 for FS {
}
};
let body = Box::new(StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))));
let body = StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string()))));
let body = Box::new(tokio::io::BufReader::new(body));
let hrd = HashReader::new(body, content_length as i64, content_length as i64, None, false).map_err(ApiError::from)?;
let mut reader = PutObjReader::new(hrd, content_length as usize);
let mut reader = PutObjReader::new(body, content_length as usize);
// let body = Box::new(StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))));
// let mut reader = PutObjReader::new(body, content_length as usize);
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
@@ -978,14 +978,14 @@ impl S3 for FS {
let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(metadata))
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
debug!("put_object opts {:?}", &opts);
let obj_info = store
.put_object(&bucket, &key, &mut reader, &opts)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let e_tag = obj_info.etag;
@@ -1027,10 +1027,12 @@ impl S3 for FS {
let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(metadata))
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let MultipartUploadResult { upload_id, .. } =
store.new_multipart_upload(&bucket, &key, &opts).await.map_err(to_s3_error)?;
let MultipartUploadResult { upload_id, .. } = store
.new_multipart_upload(&bucket, &key, &opts)
.await
.map_err(ApiError::from)?;
let output = CreateMultipartUploadOutput {
bucket: Some(bucket),
@@ -1074,10 +1076,12 @@ impl S3 for FS {
}
};
let body = Box::new(StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))));
let body = StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string()))));
let body = Box::new(tokio::io::BufReader::new(body));
let hrd = HashReader::new(body, content_length as i64, content_length as i64, None, false).map_err(ApiError::from)?;
// mc cp step 4
let mut data = PutObjReader::new(body, content_length as usize);
let mut data = PutObjReader::new(hrd, content_length as usize);
let opts = ObjectOptions::default();
let Some(store) = new_object_layer_fn() else {
@@ -1089,7 +1093,7 @@ impl S3 for FS {
let info = store
.put_object_part(&bucket, &key, &upload_id, part_id, &mut data, &opts)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let output = UploadPartOutput {
e_tag: info.etag,
@@ -1155,7 +1159,7 @@ impl S3 for FS {
let oi = store
.complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, opts)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let output = CompleteMultipartUploadOutput {
bucket: Some(bucket),
@@ -1184,7 +1188,7 @@ impl S3 for FS {
store
.abort_multipart_upload(bucket.as_str(), key.as_str(), upload_id.as_str(), opts)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
Ok(S3Response::new(AbortMultipartUploadOutput { ..Default::default() }))
}
@@ -1222,13 +1226,13 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let data = try_!(xml::serialize(&tagging));
metadata_sys::update(&bucket, BUCKET_TAGGING_CONFIG, data)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
Ok(S3Response::new(Default::default()))
}
@@ -1242,7 +1246,7 @@ impl S3 for FS {
metadata_sys::delete(&bucket, BUCKET_TAGGING_CONFIG)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
Ok(S3Response::new(DeleteBucketTaggingOutput {}))
}
@@ -1268,7 +1272,7 @@ impl S3 for FS {
store
.put_object_tags(&bucket, &object, &tags, &ObjectOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
Ok(S3Response::new(PutObjectTaggingOutput { version_id: None }))
}
@@ -1285,7 +1289,7 @@ impl S3 for FS {
let tags = store
.get_object_tags(&bucket, &object, &ObjectOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let tag_set = decode_tags(tags.as_str());
@@ -1311,7 +1315,7 @@ impl S3 for FS {
store
.delete_object_tags(&bucket, &object, &ObjectOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
Ok(S3Response::new(DeleteObjectTaggingOutput { version_id: None }))
}
@@ -1329,9 +1333,9 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let VersioningConfiguration { status, .. } = BucketVersioningSys::get(&bucket).await.map_err(to_s3_error)?;
let VersioningConfiguration { status, .. } = BucketVersioningSys::get(&bucket).await.map_err(ApiError::from)?;
Ok(S3Response::new(GetBucketVersioningOutput {
status,
@@ -1359,7 +1363,7 @@ impl S3 for FS {
metadata_sys::update(&bucket, BUCKET_VERSIONING_CONFIG, data)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
// TODO: globalSiteReplicationSys.BucketMetaHook
@@ -1379,7 +1383,7 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let conditions = get_condition_values(&req.headers, &auth::Credentials::default());
@@ -1426,15 +1430,15 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let cfg = match PolicySys::get(&bucket).await {
Ok(res) => res,
Err(err) => {
if BucketMetadataError::BucketPolicyNotFound.is(&err) {
if StorageError::BucketPolicyNotFound == err {
return Err(s3_error!(NoSuchBucketPolicy));
}
return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", err)));
return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string()));
}
};
@@ -1453,7 +1457,7 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
// warn!("input policy {}", &policy);
@@ -1469,7 +1473,7 @@ impl S3 for FS {
metadata_sys::update(&bucket, BUCKET_POLICY_CONFIG, data)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
Ok(S3Response::new(PutBucketPolicyOutput {}))
}
@@ -1487,11 +1491,11 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
metadata_sys::delete(&bucket, BUCKET_POLICY_CONFIG)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
Ok(S3Response::new(DeleteBucketPolicyOutput {}))
}
@@ -1510,7 +1514,7 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let rules = match metadata_sys::get_lifecycle_config(&bucket).await {
Ok((cfg, _)) => Some(cfg.rules),
@@ -1549,7 +1553,7 @@ impl S3 for FS {
let data = try_!(xml::serialize(&input_cfg));
metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, data)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default()))
}
@@ -1568,11 +1572,11 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
metadata_sys::delete(&bucket, BUCKET_LIFECYCLE_CONFIG)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
Ok(S3Response::new(DeleteBucketLifecycleOutput::default()))
}
@@ -1590,7 +1594,7 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let server_side_encryption_configuration = match metadata_sys::get_sse_config(&bucket).await {
Ok((cfg, _)) => Some(cfg),
@@ -1627,14 +1631,14 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
// TODO: check kms
let data = try_!(xml::serialize(&server_side_encryption_configuration));
metadata_sys::update(&bucket, BUCKET_SSECONFIG, data)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
Ok(S3Response::new(PutBucketEncryptionOutput::default()))
}
@@ -1651,8 +1655,10 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
metadata_sys::delete(&bucket, BUCKET_SSECONFIG).await.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
metadata_sys::delete(&bucket, BUCKET_SSECONFIG)
.await
.map_err(ApiError::from)?;
Ok(S3Response::new(DeleteBucketEncryptionOutput::default()))
}
@@ -1699,13 +1705,13 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let data = try_!(xml::serialize(&input_cfg));
metadata_sys::update(&bucket, OBJECT_LOCK_CONFIG, data)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
Ok(S3Response::new(PutObjectLockConfigurationOutput::default()))
}
@@ -1723,7 +1729,7 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let replication_configuration = match metadata_sys::get_replication_config(&bucket).await {
Ok((cfg, _created)) => Some(cfg),
@@ -1755,14 +1761,14 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
// TODO: check enable, versioning enable
let data = try_!(xml::serialize(&replication_configuration));
metadata_sys::update(&bucket, BUCKET_REPLICATION_CONFIG, data)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
Ok(S3Response::new(PutBucketReplicationOutput::default()))
}
@@ -1780,10 +1786,10 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
metadata_sys::delete(&bucket, BUCKET_REPLICATION_CONFIG)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
// TODO: remove targets
@@ -1803,7 +1809,7 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let has_notification_config = match metadata_sys::get_notification_config(&bucket).await {
Ok(cfg) => cfg,
@@ -1850,13 +1856,13 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let data = try_!(xml::serialize(&notification_configuration));
metadata_sys::update(&bucket, BUCKET_NOTIFICATION_CONFIG, data)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
// TODO: event notice add rule
@@ -1873,7 +1879,7 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let grants = vec![Grant {
grantee: Some(Grantee {
@@ -1909,7 +1915,7 @@ impl S3 for FS {
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
if let Some(canned_acl) = acl {
if canned_acl.as_str() != BucketCannedACL::PRIVATE {
@@ -2085,14 +2091,14 @@ impl S3 for FS {
let _ = store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
// check object lock
let _ = metadata_sys::get_object_lock_config(&bucket).await.map_err(to_s3_error)?;
let _ = metadata_sys::get_object_lock_config(&bucket).await.map_err(ApiError::from)?;
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let object_info = store.get_object_info(&bucket, &key, &opts).await.map_err(|e| {
error!("get_object_info failed, {}", e.to_string());
@@ -2135,14 +2141,14 @@ impl S3 for FS {
let _ = store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
// check object lock
let _ = metadata_sys::get_object_lock_config(&bucket).await.map_err(to_s3_error)?;
let _ = metadata_sys::get_object_lock_config(&bucket).await.map_err(ApiError::from)?;
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
.await
.map_err(to_s3_error)?;
.map_err(ApiError::from)?;
let mut eval_metadata = HashMap::new();
let legal_hold = legal_hold
@@ -2176,9 +2182,9 @@ impl S3 for FS {
}
#[allow(dead_code)]
pub fn bytes_stream<S, E>(stream: S, content_length: usize) -> impl Stream<Item = Result<Bytes, E>> + Send + 'static
pub fn bytes_stream<S, E>(stream: S, content_length: usize) -> impl Stream<Item = std::result::Result<Bytes, E>> + Send + 'static
where
S: Stream<Item = Result<Bytes, E>> + Send + 'static,
S: Stream<Item = std::result::Result<Bytes, E>> + Send + 'static,
E: Send + 'static,
{
AsyncTryStream::<Bytes, E, _>::new(|mut y| async move {
rustfs/src/storage/error.rs
+485 -485
View File
@@ -1,485 +1,485 @@
use common::error::Error;
use ecstore::error::StorageError;
use s3s::{S3Error, S3ErrorCode, s3_error};
pub fn to_s3_error(err: Error) -> S3Error {
if let Some(storage_err) = err.downcast_ref::<StorageError>() {
return match storage_err {
StorageError::NotImplemented => s3_error!(NotImplemented),
StorageError::InvalidArgument(bucket, object, version_id) => {
s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id)
}
StorageError::MethodNotAllowed => s3_error!(MethodNotAllowed),
StorageError::BucketNotFound(bucket) => {
s3_error!(NoSuchBucket, "bucket not found {}", bucket)
}
StorageError::BucketNotEmpty(bucket) => s3_error!(BucketNotEmpty, "bucket not empty {}", bucket),
StorageError::BucketNameInvalid(bucket) => s3_error!(InvalidBucketName, "invalid bucket name {}", bucket),
StorageError::ObjectNameInvalid(bucket, object) => {
s3_error!(InvalidArgument, "invalid object name {}/{}", bucket, object)
}
StorageError::BucketExists(bucket) => s3_error!(BucketAlreadyExists, "{}", bucket),
StorageError::StorageFull => s3_error!(ServiceUnavailable, "Storage reached its minimum free drive threshold."),
StorageError::SlowDown => s3_error!(SlowDown, "Please reduce your request rate"),
StorageError::PrefixAccessDenied(bucket, object) => {
s3_error!(AccessDenied, "PrefixAccessDenied {}/{}", bucket, object)
}
StorageError::InvalidUploadIDKeyCombination(bucket, object) => {
s3_error!(InvalidArgument, "Invalid UploadID KeyCombination: {}/{}", bucket, object)
}
StorageError::MalformedUploadID(bucket) => s3_error!(InvalidArgument, "Malformed UploadID: {}", bucket),
StorageError::ObjectNameTooLong(bucket, object) => {
s3_error!(InvalidArgument, "Object name too long: {}/{}", bucket, object)
}
StorageError::ObjectNamePrefixAsSlash(bucket, object) => {
s3_error!(InvalidArgument, "Object name contains forward slash as prefix: {}/{}", bucket, object)
}
StorageError::ObjectNotFound(bucket, object) => s3_error!(NoSuchKey, "{}/{}", bucket, object),
StorageError::VersionNotFound(bucket, object, version_id) => {
s3_error!(NoSuchVersion, "{}/{}/{}", bucket, object, version_id)
}
StorageError::InvalidUploadID(bucket, object, version_id) => {
s3_error!(InvalidPart, "Invalid upload id: {}/{}-{}", bucket, object, version_id)
}
StorageError::InvalidVersionID(bucket, object, version_id) => {
s3_error!(InvalidArgument, "Invalid version id: {}/{}-{}", bucket, object, version_id)
}
// extended
StorageError::DataMovementOverwriteErr(bucket, object, version_id) => s3_error!(
InvalidArgument,
"invalid data movement operation, source and destination pool are the same for : {}/{}-{}",
bucket,
object,
version_id
),
// extended
StorageError::ObjectExistsAsDirectory(bucket, object) => {
s3_error!(InvalidArgument, "Object exists on :{} as directory {}", bucket, object)
}
StorageError::InvalidPart(bucket, object, version_id) => {
s3_error!(
InvalidPart,
"Specified part could not be found. PartNumber {}, Expected {}, got {}",
bucket,
object,
version_id
)
}
StorageError::DoneForNow => s3_error!(InternalError, "DoneForNow"),
};
}
if is_err_file_not_found(&err) {
return S3Error::with_message(S3ErrorCode::NoSuchKey, format!(" ec err {}", err));
}
S3Error::with_message(S3ErrorCode::InternalError, format!(" ec err {}", err))
}
#[cfg(test)]
mod tests {
use super::*;
use s3s::S3ErrorCode;
#[test]
fn test_to_s3_error_not_implemented() {
let storage_err = StorageError::NotImplemented;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NotImplemented);
}
#[test]
fn test_to_s3_error_invalid_argument() {
let storage_err =
StorageError::InvalidArgument("test-bucket".to_string(), "test-object".to_string(), "test-version".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Invalid arguments provided"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-object"));
assert!(s3_err.message().unwrap().contains("test-version"));
}
#[test]
fn test_to_s3_error_method_not_allowed() {
let storage_err = StorageError::MethodNotAllowed;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::MethodNotAllowed);
}
#[test]
fn test_to_s3_error_bucket_not_found() {
let storage_err = StorageError::BucketNotFound("test-bucket".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
assert!(s3_err.message().unwrap().contains("bucket not found"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
}
#[test]
fn test_to_s3_error_bucket_not_empty() {
let storage_err = StorageError::BucketNotEmpty("test-bucket".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::BucketNotEmpty);
assert!(s3_err.message().unwrap().contains("bucket not empty"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
}
#[test]
fn test_to_s3_error_bucket_name_invalid() {
let storage_err = StorageError::BucketNameInvalid("invalid-bucket-name".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidBucketName);
assert!(s3_err.message().unwrap().contains("invalid bucket name"));
assert!(s3_err.message().unwrap().contains("invalid-bucket-name"));
}
#[test]
fn test_to_s3_error_object_name_invalid() {
let storage_err = StorageError::ObjectNameInvalid("test-bucket".to_string(), "invalid-object".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("invalid object name"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("invalid-object"));
}
#[test]
fn test_to_s3_error_bucket_exists() {
let storage_err = StorageError::BucketExists("existing-bucket".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::BucketAlreadyExists);
assert!(s3_err.message().unwrap().contains("existing-bucket"));
}
#[test]
fn test_to_s3_error_storage_full() {
let storage_err = StorageError::StorageFull;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::ServiceUnavailable);
assert!(
s3_err
.message()
.unwrap()
.contains("Storage reached its minimum free drive threshold")
);
}
#[test]
fn test_to_s3_error_slow_down() {
let storage_err = StorageError::SlowDown;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown);
assert!(s3_err.message().unwrap().contains("Please reduce your request rate"));
}
#[test]
fn test_to_s3_error_prefix_access_denied() {
let storage_err = StorageError::PrefixAccessDenied("test-bucket".to_string(), "test-prefix".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::AccessDenied);
assert!(s3_err.message().unwrap().contains("PrefixAccessDenied"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-prefix"));
}
#[test]
fn test_to_s3_error_invalid_upload_id_key_combination() {
let storage_err = StorageError::InvalidUploadIDKeyCombination("test-bucket".to_string(), "test-object".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Invalid UploadID KeyCombination"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-object"));
}
#[test]
fn test_to_s3_error_malformed_upload_id() {
let storage_err = StorageError::MalformedUploadID("malformed-id".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Malformed UploadID"));
assert!(s3_err.message().unwrap().contains("malformed-id"));
}
#[test]
fn test_to_s3_error_object_name_too_long() {
let storage_err = StorageError::ObjectNameTooLong("test-bucket".to_string(), "very-long-object-name".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Object name too long"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("very-long-object-name"));
}
#[test]
fn test_to_s3_error_object_name_prefix_as_slash() {
let storage_err = StorageError::ObjectNamePrefixAsSlash("test-bucket".to_string(), "/invalid-object".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(
s3_err
.message()
.unwrap()
.contains("Object name contains forward slash as prefix")
);
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("/invalid-object"));
}
#[test]
fn test_to_s3_error_object_not_found() {
let storage_err = StorageError::ObjectNotFound("test-bucket".to_string(), "missing-object".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchKey);
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("missing-object"));
}
#[test]
fn test_to_s3_error_version_not_found() {
let storage_err =
StorageError::VersionNotFound("test-bucket".to_string(), "test-object".to_string(), "missing-version".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchVersion);
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-object"));
assert!(s3_err.message().unwrap().contains("missing-version"));
}
#[test]
fn test_to_s3_error_invalid_upload_id() {
let storage_err =
StorageError::InvalidUploadID("test-bucket".to_string(), "test-object".to_string(), "invalid-upload-id".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidPart);
assert!(s3_err.message().unwrap().contains("Invalid upload id"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-object"));
assert!(s3_err.message().unwrap().contains("invalid-upload-id"));
}
#[test]
fn test_to_s3_error_invalid_version_id() {
let storage_err = StorageError::InvalidVersionID(
"test-bucket".to_string(),
"test-object".to_string(),
"invalid-version-id".to_string(),
);
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Invalid version id"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-object"));
assert!(s3_err.message().unwrap().contains("invalid-version-id"));
}
#[test]
fn test_to_s3_error_data_movement_overwrite_err() {
let storage_err = StorageError::DataMovementOverwriteErr(
"test-bucket".to_string(),
"test-object".to_string(),
"test-version".to_string(),
);
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("invalid data movement operation"));
assert!(s3_err.message().unwrap().contains("source and destination pool are the same"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-object"));
assert!(s3_err.message().unwrap().contains("test-version"));
}
#[test]
fn test_to_s3_error_object_exists_as_directory() {
let storage_err = StorageError::ObjectExistsAsDirectory("test-bucket".to_string(), "directory-object".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Object exists on"));
assert!(s3_err.message().unwrap().contains("as directory"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("directory-object"));
}
#[test]
fn test_to_s3_error_insufficient_read_quorum() {
let storage_err = StorageError::InsufficientReadQuorum;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown);
assert!(
s3_err
.message()
.unwrap()
.contains("Storage resources are insufficient for the read operation")
);
}
#[test]
fn test_to_s3_error_insufficient_write_quorum() {
let storage_err = StorageError::InsufficientWriteQuorum;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown);
assert!(
s3_err
.message()
.unwrap()
.contains("Storage resources are insufficient for the write operation")
);
}
#[test]
fn test_to_s3_error_decommission_not_started() {
let storage_err = StorageError::DecommissionNotStarted;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Decommission Not Started"));
}
#[test]
fn test_to_s3_error_decommission_already_running() {
let storage_err = StorageError::DecommissionAlreadyRunning;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InternalError);
assert!(s3_err.message().unwrap().contains("Decommission already running"));
}
#[test]
fn test_to_s3_error_volume_not_found() {
let storage_err = StorageError::VolumeNotFound("test-volume".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
assert!(s3_err.message().unwrap().contains("bucket not found"));
assert!(s3_err.message().unwrap().contains("test-volume"));
}
#[test]
fn test_to_s3_error_invalid_part() {
let storage_err = StorageError::InvalidPart(1, "expected-part".to_string(), "got-part".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidPart);
assert!(s3_err.message().unwrap().contains("Specified part could not be found"));
assert!(s3_err.message().unwrap().contains("PartNumber"));
assert!(s3_err.message().unwrap().contains("expected-part"));
assert!(s3_err.message().unwrap().contains("got-part"));
}
#[test]
fn test_to_s3_error_done_for_now() {
let storage_err = StorageError::DoneForNow;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InternalError);
assert!(s3_err.message().unwrap().contains("DoneForNow"));
}
#[test]
fn test_to_s3_error_non_storage_error() {
// Test with a non-StorageError
let err = Error::from_string("Generic error message".to_string());
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InternalError);
assert!(s3_err.message().unwrap().contains("ec err"));
assert!(s3_err.message().unwrap().contains("Generic error message"));
}
#[test]
fn test_to_s3_error_with_unicode_strings() {
let storage_err = StorageError::BucketNotFound("测试桶".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
assert!(s3_err.message().unwrap().contains("bucket not found"));
assert!(s3_err.message().unwrap().contains("测试桶"));
}
#[test]
fn test_to_s3_error_with_special_characters() {
let storage_err = StorageError::ObjectNameInvalid("bucket-with-@#$%".to_string(), "object-with-!@#$%^&*()".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("invalid object name"));
assert!(s3_err.message().unwrap().contains("bucket-with-@#$%"));
assert!(s3_err.message().unwrap().contains("object-with-!@#$%^&*()"));
}
#[test]
fn test_to_s3_error_with_empty_strings() {
let storage_err = StorageError::BucketNotFound("".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
assert!(s3_err.message().unwrap().contains("bucket not found"));
}
#[test]
fn test_to_s3_error_with_very_long_strings() {
let long_bucket_name = "a".repeat(1000);
let storage_err = StorageError::BucketNotFound(long_bucket_name.clone());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
assert!(s3_err.message().unwrap().contains("bucket not found"));
assert!(s3_err.message().unwrap().contains(&long_bucket_name));
}
}
// use common::error::Error;
// use ecstore::error::StorageError;
// use s3s::{S3Error, S3ErrorCode, s3_error};
// pub fn to_s3_error(err: Error) -> S3Error {
// if let Some(storage_err) = err.downcast_ref::<StorageError>() {
// return match storage_err {
// StorageError::NotImplemented => s3_error!(NotImplemented),
// StorageError::InvalidArgument(bucket, object, version_id) => {
// s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id)
// }
// StorageError::MethodNotAllowed => s3_error!(MethodNotAllowed),
// StorageError::BucketNotFound(bucket) => {
// s3_error!(NoSuchBucket, "bucket not found {}", bucket)
// }
// StorageError::BucketNotEmpty(bucket) => s3_error!(BucketNotEmpty, "bucket not empty {}", bucket),
// StorageError::BucketNameInvalid(bucket) => s3_error!(InvalidBucketName, "invalid bucket name {}", bucket),
// StorageError::ObjectNameInvalid(bucket, object) => {
// s3_error!(InvalidArgument, "invalid object name {}/{}", bucket, object)
// }
// StorageError::BucketExists(bucket) => s3_error!(BucketAlreadyExists, "{}", bucket),
// StorageError::StorageFull => s3_error!(ServiceUnavailable, "Storage reached its minimum free drive threshold."),
// StorageError::SlowDown => s3_error!(SlowDown, "Please reduce your request rate"),
// StorageError::PrefixAccessDenied(bucket, object) => {
// s3_error!(AccessDenied, "PrefixAccessDenied {}/{}", bucket, object)
// }
// StorageError::InvalidUploadIDKeyCombination(bucket, object) => {
// s3_error!(InvalidArgument, "Invalid UploadID KeyCombination: {}/{}", bucket, object)
// }
// StorageError::MalformedUploadID(bucket) => s3_error!(InvalidArgument, "Malformed UploadID: {}", bucket),
// StorageError::ObjectNameTooLong(bucket, object) => {
// s3_error!(InvalidArgument, "Object name too long: {}/{}", bucket, object)
// }
// StorageError::ObjectNamePrefixAsSlash(bucket, object) => {
// s3_error!(InvalidArgument, "Object name contains forward slash as prefix: {}/{}", bucket, object)
// }
// StorageError::ObjectNotFound(bucket, object) => s3_error!(NoSuchKey, "{}/{}", bucket, object),
// StorageError::VersionNotFound(bucket, object, version_id) => {
// s3_error!(NoSuchVersion, "{}/{}/{}", bucket, object, version_id)
// }
// StorageError::InvalidUploadID(bucket, object, version_id) => {
// s3_error!(InvalidPart, "Invalid upload id: {}/{}-{}", bucket, object, version_id)
// }
// StorageError::InvalidVersionID(bucket, object, version_id) => {
// s3_error!(InvalidArgument, "Invalid version id: {}/{}-{}", bucket, object, version_id)
// }
// // extended
// StorageError::DataMovementOverwriteErr(bucket, object, version_id) => s3_error!(
// InvalidArgument,
// "invalid data movement operation, source and destination pool are the same for : {}/{}-{}",
// bucket,
// object,
// version_id
// ),
// // extended
// StorageError::ObjectExistsAsDirectory(bucket, object) => {
// s3_error!(InvalidArgument, "Object exists on :{} as directory {}", bucket, object)
// }
// StorageError::InvalidPart(bucket, object, version_id) => {
// s3_error!(
// InvalidPart,
// "Specified part could not be found. PartNumber {}, Expected {}, got {}",
// bucket,
// object,
// version_id
// )
// }
// StorageError::DoneForNow => s3_error!(InternalError, "DoneForNow"),
// };
// }
// if is_err_file_not_found(&err) {
// return S3Error::with_message(S3ErrorCode::NoSuchKey, format!(" ec err {}", err));
// }
// S3Error::with_message(S3ErrorCode::InternalError, format!(" ec err {}", err))
// }
// #[cfg(test)]
// mod tests {
// use super::*;
// use s3s::S3ErrorCode;
// #[test]
// fn test_to_s3_error_not_implemented() {
// let storage_err = StorageError::NotImplemented;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NotImplemented);
// }
// #[test]
// fn test_to_s3_error_invalid_argument() {
// let storage_err =
// StorageError::InvalidArgument("test-bucket".to_string(), "test-object".to_string(), "test-version".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Invalid arguments provided"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-object"));
// assert!(s3_err.message().unwrap().contains("test-version"));
// }
// #[test]
// fn test_to_s3_error_method_not_allowed() {
// let storage_err = StorageError::MethodNotAllowed;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::MethodNotAllowed);
// }
// #[test]
// fn test_to_s3_error_bucket_not_found() {
// let storage_err = StorageError::BucketNotFound("test-bucket".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
// assert!(s3_err.message().unwrap().contains("bucket not found"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// }
// #[test]
// fn test_to_s3_error_bucket_not_empty() {
// let storage_err = StorageError::BucketNotEmpty("test-bucket".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::BucketNotEmpty);
// assert!(s3_err.message().unwrap().contains("bucket not empty"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// }
// #[test]
// fn test_to_s3_error_bucket_name_invalid() {
// let storage_err = StorageError::BucketNameInvalid("invalid-bucket-name".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidBucketName);
// assert!(s3_err.message().unwrap().contains("invalid bucket name"));
// assert!(s3_err.message().unwrap().contains("invalid-bucket-name"));
// }
// #[test]
// fn test_to_s3_error_object_name_invalid() {
// let storage_err = StorageError::ObjectNameInvalid("test-bucket".to_string(), "invalid-object".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("invalid object name"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("invalid-object"));
// }
// #[test]
// fn test_to_s3_error_bucket_exists() {
// let storage_err = StorageError::BucketExists("existing-bucket".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::BucketAlreadyExists);
// assert!(s3_err.message().unwrap().contains("existing-bucket"));
// }
// #[test]
// fn test_to_s3_error_storage_full() {
// let storage_err = StorageError::StorageFull;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::ServiceUnavailable);
// assert!(
// s3_err
// .message()
// .unwrap()
// .contains("Storage reached its minimum free drive threshold")
// );
// }
// #[test]
// fn test_to_s3_error_slow_down() {
// let storage_err = StorageError::SlowDown;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown);
// assert!(s3_err.message().unwrap().contains("Please reduce your request rate"));
// }
// #[test]
// fn test_to_s3_error_prefix_access_denied() {
// let storage_err = StorageError::PrefixAccessDenied("test-bucket".to_string(), "test-prefix".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::AccessDenied);
// assert!(s3_err.message().unwrap().contains("PrefixAccessDenied"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-prefix"));
// }
// #[test]
// fn test_to_s3_error_invalid_upload_id_key_combination() {
// let storage_err = StorageError::InvalidUploadIDKeyCombination("test-bucket".to_string(), "test-object".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Invalid UploadID KeyCombination"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-object"));
// }
// #[test]
// fn test_to_s3_error_malformed_upload_id() {
// let storage_err = StorageError::MalformedUploadID("malformed-id".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Malformed UploadID"));
// assert!(s3_err.message().unwrap().contains("malformed-id"));
// }
// #[test]
// fn test_to_s3_error_object_name_too_long() {
// let storage_err = StorageError::ObjectNameTooLong("test-bucket".to_string(), "very-long-object-name".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Object name too long"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("very-long-object-name"));
// }
// #[test]
// fn test_to_s3_error_object_name_prefix_as_slash() {
// let storage_err = StorageError::ObjectNamePrefixAsSlash("test-bucket".to_string(), "/invalid-object".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(
// s3_err
// .message()
// .unwrap()
// .contains("Object name contains forward slash as prefix")
// );
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("/invalid-object"));
// }
// #[test]
// fn test_to_s3_error_object_not_found() {
// let storage_err = StorageError::ObjectNotFound("test-bucket".to_string(), "missing-object".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchKey);
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("missing-object"));
// }
// #[test]
// fn test_to_s3_error_version_not_found() {
// let storage_err =
// StorageError::VersionNotFound("test-bucket".to_string(), "test-object".to_string(), "missing-version".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchVersion);
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-object"));
// assert!(s3_err.message().unwrap().contains("missing-version"));
// }
// #[test]
// fn test_to_s3_error_invalid_upload_id() {
// let storage_err =
// StorageError::InvalidUploadID("test-bucket".to_string(), "test-object".to_string(), "invalid-upload-id".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidPart);
// assert!(s3_err.message().unwrap().contains("Invalid upload id"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-object"));
// assert!(s3_err.message().unwrap().contains("invalid-upload-id"));
// }
// #[test]
// fn test_to_s3_error_invalid_version_id() {
// let storage_err = StorageError::InvalidVersionID(
// "test-bucket".to_string(),
// "test-object".to_string(),
// "invalid-version-id".to_string(),
// );
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Invalid version id"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-object"));
// assert!(s3_err.message().unwrap().contains("invalid-version-id"));
// }
// #[test]
// fn test_to_s3_error_data_movement_overwrite_err() {
// let storage_err = StorageError::DataMovementOverwriteErr(
// "test-bucket".to_string(),
// "test-object".to_string(),
// "test-version".to_string(),
// );
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("invalid data movement operation"));
// assert!(s3_err.message().unwrap().contains("source and destination pool are the same"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-object"));
// assert!(s3_err.message().unwrap().contains("test-version"));
// }
// #[test]
// fn test_to_s3_error_object_exists_as_directory() {
// let storage_err = StorageError::ObjectExistsAsDirectory("test-bucket".to_string(), "directory-object".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Object exists on"));
// assert!(s3_err.message().unwrap().contains("as directory"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("directory-object"));
// }
// #[test]
// fn test_to_s3_error_insufficient_read_quorum() {
// let storage_err = StorageError::InsufficientReadQuorum;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown);
// assert!(
// s3_err
// .message()
// .unwrap()
// .contains("Storage resources are insufficient for the read operation")
// );
// }
// #[test]
// fn test_to_s3_error_insufficient_write_quorum() {
// let storage_err = StorageError::InsufficientWriteQuorum;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown);
// assert!(
// s3_err
// .message()
// .unwrap()
// .contains("Storage resources are insufficient for the write operation")
// );
// }
// #[test]
// fn test_to_s3_error_decommission_not_started() {
// let storage_err = StorageError::DecommissionNotStarted;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Decommission Not Started"));
// }
// #[test]
// fn test_to_s3_error_decommission_already_running() {
// let storage_err = StorageError::DecommissionAlreadyRunning;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InternalError);
// assert!(s3_err.message().unwrap().contains("Decommission already running"));
// }
// #[test]
// fn test_to_s3_error_volume_not_found() {
// let storage_err = StorageError::VolumeNotFound("test-volume".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
// assert!(s3_err.message().unwrap().contains("bucket not found"));
// assert!(s3_err.message().unwrap().contains("test-volume"));
// }
// #[test]
// fn test_to_s3_error_invalid_part() {
// let storage_err = StorageError::InvalidPart(1, "expected-part".to_string(), "got-part".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidPart);
// assert!(s3_err.message().unwrap().contains("Specified part could not be found"));
// assert!(s3_err.message().unwrap().contains("PartNumber"));
// assert!(s3_err.message().unwrap().contains("expected-part"));
// assert!(s3_err.message().unwrap().contains("got-part"));
// }
// #[test]
// fn test_to_s3_error_done_for_now() {
// let storage_err = StorageError::DoneForNow;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InternalError);
// assert!(s3_err.message().unwrap().contains("DoneForNow"));
// }
// #[test]
// fn test_to_s3_error_non_storage_error() {
// // Test with a non-StorageError
// let err = Error::from_string("Generic error message".to_string());
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InternalError);
// assert!(s3_err.message().unwrap().contains("ec err"));
// assert!(s3_err.message().unwrap().contains("Generic error message"));
// }
// #[test]
// fn test_to_s3_error_with_unicode_strings() {
// let storage_err = StorageError::BucketNotFound("测试桶".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
// assert!(s3_err.message().unwrap().contains("bucket not found"));
// assert!(s3_err.message().unwrap().contains("测试桶"));
// }
// #[test]
// fn test_to_s3_error_with_special_characters() {
// let storage_err = StorageError::ObjectNameInvalid("bucket-with-@#$%".to_string(), "object-with-!@#$%^&*()".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("invalid object name"));
// assert!(s3_err.message().unwrap().contains("bucket-with-@#$%"));
// assert!(s3_err.message().unwrap().contains("object-with-!@#$%^&*()"));
// }
// #[test]
// fn test_to_s3_error_with_empty_strings() {
// let storage_err = StorageError::BucketNotFound("".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
// assert!(s3_err.message().unwrap().contains("bucket not found"));
// }
// #[test]
// fn test_to_s3_error_with_very_long_strings() {
// let long_bucket_name = "a".repeat(1000);
// let storage_err = StorageError::BucketNotFound(long_bucket_name.clone());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
// assert!(s3_err.message().unwrap().contains("bucket not found"));
// assert!(s3_err.message().unwrap().contains(&long_bucket_name));
// }
// }
rustfs/src/storage/mod.rs
+1 -1
View File
@@ -1,5 +1,5 @@
pub mod access;
pub mod ecfs;
pub mod error;
// pub mod error;
mod event_notifier;
pub mod options;
rustfs/src/storage/options.rs
+29 -59
View File
@@ -1,7 +1,7 @@
use common::error::{Error, Result};
use ecstore::bucket::versioning_sys::BucketVersioningSys;
use ecstore::store_api::ObjectOptions;
use ecstore::error::Result;
use ecstore::error::StorageError;
use ecstore::store_api::ObjectOptions;
use ecstore::utils::path::is_dir_object;
use http::{HeaderMap, HeaderValue};
use lazy_static::lazy_static;
@@ -25,24 +25,16 @@ pub async fn del_opts(
if let Some(ref id) = vid {
if let Err(_err) = Uuid::parse_str(id.as_str()) {
return Err(Error::new(StorageError::InvalidVersionID(
bucket.to_owned(),
object.to_owned(),
id.clone(),
)));
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
}
if !versioned {
return Err(Error::new(StorageError::InvalidArgument(
bucket.to_owned(),
object.to_owned(),
id.clone(),
)));
return Err(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), id.clone()));
}
}
let mut opts = put_opts_from_headers(headers, metadata)
.map_err(|err| Error::new(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string())))?;
.map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?;
opts.version_id = {
if is_dir_object(object) && vid.is_none() {
@@ -72,24 +64,16 @@ pub async fn get_opts(
if let Some(ref id) = vid {
if let Err(_err) = Uuid::parse_str(id.as_str()) {
return Err(Error::new(StorageError::InvalidVersionID(
bucket.to_owned(),
object.to_owned(),
id.clone(),
)));
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
}
if !versioned {
return Err(Error::new(StorageError::InvalidArgument(
bucket.to_owned(),
object.to_owned(),
id.clone(),
)));
return Err(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), id.clone()));
}
}
let mut opts = get_default_opts(headers, None, false)
.map_err(|err| Error::new(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string())))?;
.map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?;
opts.version_id = {
if is_dir_object(object) && vid.is_none() {
@@ -122,24 +106,16 @@ pub async fn put_opts(
if let Some(ref id) = vid {
if let Err(_err) = Uuid::parse_str(id.as_str()) {
return Err(Error::new(StorageError::InvalidVersionID(
bucket.to_owned(),
object.to_owned(),
id.clone(),
)));
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
}
if !versioned {
return Err(Error::new(StorageError::InvalidArgument(
bucket.to_owned(),
object.to_owned(),
id.clone(),
)));
return Err(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), id.clone()));
}
}
let mut opts = put_opts_from_headers(headers, metadata)
.map_err(|err| Error::new(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string())))?;
.map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?;
opts.version_id = {
if is_dir_object(object) && vid.is_none() {
@@ -317,15 +293,13 @@ mod tests {
assert!(result.is_err());
if let Err(err) = result {
if let Some(storage_err) = err.downcast_ref::<StorageError>() {
match storage_err {
StorageError::InvalidVersionID(bucket, object, version) => {
assert_eq!(bucket, "test-bucket");
assert_eq!(object, "test-object");
assert_eq!(version, "invalid-uuid");
}
_ => panic!("Expected InvalidVersionID error"),
match err {
StorageError::InvalidVersionID(bucket, object, version) => {
assert_eq!(bucket, "test-bucket");
assert_eq!(object, "test-object");
assert_eq!(version, "invalid-uuid");
}
_ => panic!("Expected InvalidVersionID error"),
}
}
}
@@ -373,15 +347,13 @@ mod tests {
assert!(result.is_err());
if let Err(err) = result {
if let Some(storage_err) = err.downcast_ref::<StorageError>() {
match storage_err {
StorageError::InvalidVersionID(bucket, object, version) => {
assert_eq!(bucket, "test-bucket");
assert_eq!(object, "test-object");
assert_eq!(version, "invalid-uuid");
}
_ => panic!("Expected InvalidVersionID error"),
match err {
StorageError::InvalidVersionID(bucket, object, version) => {
assert_eq!(bucket, "test-bucket");
assert_eq!(object, "test-object");
assert_eq!(version, "invalid-uuid");
}
_ => panic!("Expected InvalidVersionID error"),
}
}
}
@@ -419,15 +391,13 @@ mod tests {
assert!(result.is_err());
if let Err(err) = result {
if let Some(storage_err) = err.downcast_ref::<StorageError>() {
match storage_err {
StorageError::InvalidVersionID(bucket, object, version) => {
assert_eq!(bucket, "test-bucket");
assert_eq!(object, "test-object");
assert_eq!(version, "invalid-uuid");
}
_ => panic!("Expected InvalidVersionID error"),
match err {
StorageError::InvalidVersionID(bucket, object, version) => {
assert_eq!(bucket, "test-bucket");
assert_eq!(object, "test-object");
assert_eq!(version, "invalid-uuid");
}
_ => panic!("Expected InvalidVersionID error"),
}
}
}