gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 08:36:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

mc test ok

Browse Source
This commit is contained in:
weisd
2025-06-06 16:15:26 +08:00
parent 3dacde092c
commit c589972fa7
55 changed files with 5053 additions and 17559 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/disk/Cargo.toml
-33
View File
@@ -1,33 +0,0 @@
[package]
name = "rustfs-disk"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
[dependencies]
url.workspace = true
rustfs-filemeta.workspace = true
rustfs-error.workspace = true
rustfs-rio.workspace = true
serde.workspace = true
serde_json.workspace = true
uuid.workspace = true
tracing.workspace = true
tokio.workspace = true
path-absolutize = "3.1.1"
rustfs-utils = {workspace = true, features =["net"]}
async-trait.workspace = true
time.workspace = true
rustfs-metacache.workspace = true
futures.workspace = true
madmin.workspace = true
protos.workspace = true
tonic.workspace = true
urlencoding = "2.1.3"
rmp-serde.workspace = true
http.workspace = true
[lints]
workspace = true
crates/disk/src/api.rs
-667
View File
@@ -1,667 +0,0 @@
use crate::{endpoint::Endpoint, local::LocalDisk, remote::RemoteDisk};
use madmin::DiskMetrics;
use rustfs_error::{Error, Result};
use rustfs_filemeta::{FileInfo, FileInfoVersions, RawFileInfo};
use serde::{Deserialize, Serialize};
use std::{fmt::Debug, path::PathBuf, sync::Arc};
use time::OffsetDateTime;
use tokio::io::{AsyncRead, AsyncWrite};
use uuid::Uuid;
pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp";
pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash";
pub const BUCKET_META_PREFIX: &str = "buckets";
pub const FORMAT_CONFIG_FILE: &str = "format.json";
pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
pub type DiskStore = Arc<Disk>;
#[derive(Debug)]
pub enum Disk {
Local(LocalDisk),
Remote(RemoteDisk),
}
#[async_trait::async_trait]
impl DiskAPI for Disk {
#[tracing::instrument(skip(self))]
fn to_string(&self) -> String {
match self {
Disk::Local(local_disk) => local_disk.to_string(),
Disk::Remote(remote_disk) => remote_disk.to_string(),
}
}
#[tracing::instrument(skip(self))]
fn is_local(&self) -> bool {
match self {
Disk::Local(local_disk) => local_disk.is_local(),
Disk::Remote(remote_disk) => remote_disk.is_local(),
}
}
#[tracing::instrument(skip(self))]
fn host_name(&self) -> String {
match self {
Disk::Local(local_disk) => local_disk.host_name(),
Disk::Remote(remote_disk) => remote_disk.host_name(),
}
}
#[tracing::instrument(skip(self))]
async fn is_online(&self) -> bool {
match self {
Disk::Local(local_disk) => local_disk.is_online().await,
Disk::Remote(remote_disk) => remote_disk.is_online().await,
}
}
#[tracing::instrument(skip(self))]
fn endpoint(&self) -> Endpoint {
match self {
Disk::Local(local_disk) => local_disk.endpoint(),
Disk::Remote(remote_disk) => remote_disk.endpoint(),
}
}
#[tracing::instrument(skip(self))]
async fn close(&self) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.close().await,
Disk::Remote(remote_disk) => remote_disk.close().await,
}
}
#[tracing::instrument(skip(self))]
fn path(&self) -> PathBuf {
match self {
Disk::Local(local_disk) => local_disk.path(),
Disk::Remote(remote_disk) => remote_disk.path(),
}
}
#[tracing::instrument(skip(self))]
fn get_disk_location(&self) -> DiskLocation {
match self {
Disk::Local(local_disk) => local_disk.get_disk_location(),
Disk::Remote(remote_disk) => remote_disk.get_disk_location(),
}
}
#[tracing::instrument(skip(self))]
async fn get_disk_id(&self) -> Result<Option<Uuid>> {
match self {
Disk::Local(local_disk) => local_disk.get_disk_id().await,
Disk::Remote(remote_disk) => remote_disk.get_disk_id().await,
}
}
#[tracing::instrument(skip(self))]
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.set_disk_id(id).await,
Disk::Remote(remote_disk) => remote_disk.set_disk_id(id).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
match self {
Disk::Local(local_disk) => local_disk.read_all(volume, path).await,
Disk::Remote(remote_disk) => remote_disk.read_all(volume, path).await,
}
}
#[tracing::instrument(skip(self))]
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.write_all(volume, path, data).await,
Disk::Remote(remote_disk) => remote_disk.write_all(volume, path, data).await,
}
}
#[tracing::instrument(skip(self))]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete(volume, path, opt).await,
Disk::Remote(remote_disk) => remote_disk.delete(volume, path, opt).await,
}
}
#[tracing::instrument(skip(self))]
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
match self {
Disk::Local(local_disk) => local_disk.verify_file(volume, path, fi).await,
Disk::Remote(remote_disk) => remote_disk.verify_file(volume, path, fi).await,
}
}
#[tracing::instrument(skip(self))]
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
match self {
Disk::Local(local_disk) => local_disk.check_parts(volume, path, fi).await,
Disk::Remote(remote_disk) => remote_disk.check_parts(volume, path, fi).await,
}
}
#[tracing::instrument(skip(self))]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.rename_part(src_volume, src_path, dst_volume, dst_path, meta).await,
Disk::Remote(remote_disk) => {
remote_disk
.rename_part(src_volume, src_path, dst_volume, dst_path, meta)
.await
}
}
}
#[tracing::instrument(skip(self))]
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.rename_file(src_volume, src_path, dst_volume, dst_path).await,
Disk::Remote(remote_disk) => remote_disk.rename_file(src_volume, src_path, dst_volume, dst_path).await,
}
}
#[tracing::instrument(skip(self))]
async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result<Box<dyn AsyncWrite>> {
match self {
Disk::Local(local_disk) => local_disk.create_file(_origvolume, volume, path, _file_size).await,
Disk::Remote(remote_disk) => remote_disk.create_file(_origvolume, volume, path, _file_size).await,
}
}
#[tracing::instrument(skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<Box<dyn AsyncWrite>> {
match self {
Disk::Local(local_disk) => local_disk.append_file(volume, path).await,
Disk::Remote(remote_disk) => remote_disk.append_file(volume, path).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_file(&self, volume: &str, path: &str) -> Result<Box<dyn AsyncRead>> {
match self {
Disk::Local(local_disk) => local_disk.read_file(volume, path).await,
Disk::Remote(remote_disk) => remote_disk.read_file(volume, path).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Box<dyn AsyncRead>> {
match self {
Disk::Local(local_disk) => local_disk.read_file_stream(volume, path, offset, length).await,
Disk::Remote(remote_disk) => remote_disk.read_file_stream(volume, path, offset, length).await,
}
}
#[tracing::instrument(skip(self))]
async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
match self {
Disk::Local(local_disk) => local_disk.list_dir(_origvolume, volume, _dir_path, _count).await,
Disk::Remote(remote_disk) => remote_disk.list_dir(_origvolume, volume, _dir_path, _count).await,
}
}
#[tracing::instrument(skip(self, wr))]
async fn walk_dir<W: AsyncWrite + Unpin + Send + Sync + 'static>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.walk_dir(opts, wr).await,
Disk::Remote(remote_disk) => remote_disk.walk_dir(opts, wr).await,
}
}
#[tracing::instrument(skip(self))]
async fn rename_data(
&self,
src_volume: &str,
src_path: &str,
fi: FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
match self {
Disk::Local(local_disk) => local_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await,
Disk::Remote(remote_disk) => remote_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await,
}
}
#[tracing::instrument(skip(self))]
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.make_volumes(volumes).await,
Disk::Remote(remote_disk) => remote_disk.make_volumes(volumes).await,
}
}
#[tracing::instrument(skip(self))]
async fn make_volume(&self, volume: &str) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.make_volume(volume).await,
Disk::Remote(remote_disk) => remote_disk.make_volume(volume).await,
}
}
#[tracing::instrument(skip(self))]
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
match self {
Disk::Local(local_disk) => local_disk.list_volumes().await,
Disk::Remote(remote_disk) => remote_disk.list_volumes().await,
}
}
#[tracing::instrument(skip(self))]
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
match self {
Disk::Local(local_disk) => local_disk.stat_volume(volume).await,
Disk::Remote(remote_disk) => remote_disk.stat_volume(volume).await,
}
}
#[tracing::instrument(skip(self))]
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete_paths(volume, paths).await,
Disk::Remote(remote_disk) => remote_disk.delete_paths(volume, paths).await,
}
}
#[tracing::instrument(skip(self))]
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.update_metadata(volume, path, fi, opts).await,
Disk::Remote(remote_disk) => remote_disk.update_metadata(volume, path, fi, opts).await,
}
}
#[tracing::instrument(skip(self))]
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.write_metadata(_org_volume, volume, path, fi).await,
Disk::Remote(remote_disk) => remote_disk.write_metadata(_org_volume, volume, path, fi).await,
}
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_version(
&self,
_org_volume: &str,
volume: &str,
path: &str,
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo> {
match self {
Disk::Local(local_disk) => local_disk.read_version(_org_volume, volume, path, version_id, opts).await,
Disk::Remote(remote_disk) => remote_disk.read_version(_org_volume, volume, path, version_id, opts).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
match self {
Disk::Local(local_disk) => local_disk.read_xl(volume, path, read_data).await,
Disk::Remote(remote_disk) => remote_disk.read_xl(volume, path, read_data).await,
}
}
#[tracing::instrument(skip(self))]
async fn delete_version(
&self,
volume: &str,
path: &str,
fi: FileInfo,
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete_version(volume, path, fi, force_del_marker, opts).await,
Disk::Remote(remote_disk) => remote_disk.delete_version(volume, path, fi, force_del_marker, opts).await,
}
}
#[tracing::instrument(skip(self))]
async fn delete_versions(
&self,
volume: &str,
versions: Vec<FileInfoVersions>,
opts: DeleteOptions,
) -> Result<Vec<Option<Error>>> {
match self {
Disk::Local(local_disk) => local_disk.delete_versions(volume, versions, opts).await,
Disk::Remote(remote_disk) => remote_disk.delete_versions(volume, versions, opts).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
match self {
Disk::Local(local_disk) => local_disk.read_multiple(req).await,
Disk::Remote(remote_disk) => remote_disk.read_multiple(req).await,
}
}
#[tracing::instrument(skip(self))]
async fn delete_volume(&self, volume: &str) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete_volume(volume).await,
Disk::Remote(remote_disk) => remote_disk.delete_volume(volume).await,
}
}
#[tracing::instrument(skip(self))]
async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo> {
match self {
Disk::Local(local_disk) => local_disk.disk_info(opts).await,
Disk::Remote(remote_disk) => remote_disk.disk_info(opts).await,
}
}
// #[tracing::instrument(skip(self, cache, we_sleep, scan_mode))]
// async fn ns_scanner(
// &self,
// cache: &DataUsageCache,
// updates: Sender<DataUsageEntry>,
// scan_mode: HealScanMode,
// we_sleep: ShouldSleepFn,
// ) -> Result<DataUsageCache> {
// match self {
// Disk::Local(local_disk) => local_disk.ns_scanner(cache, updates, scan_mode, we_sleep).await,
// Disk::Remote(remote_disk) => remote_disk.ns_scanner(cache, updates, scan_mode, we_sleep).await,
// }
// }
// #[tracing::instrument(skip(self))]
// async fn healing(&self) -> Option<HealingTracker> {
// match self {
// Disk::Local(local_disk) => local_disk.healing().await,
// Disk::Remote(remote_disk) => remote_disk.healing().await,
// }
// }
}
pub async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> Result<DiskStore> {
if ep.is_local {
Ok(Arc::new(Disk::Local(LocalDisk::new(ep, opt.cleanup).await?)))
} else {
Ok(Arc::new(Disk::Remote(RemoteDisk::new(ep, opt).await?)))
}
}
#[async_trait::async_trait]
pub trait DiskAPI: Debug + Send + Sync + 'static {
fn to_string(&self) -> String;
async fn is_online(&self) -> bool;
fn is_local(&self) -> bool;
// LastConn
fn host_name(&self) -> String;
fn endpoint(&self) -> Endpoint;
async fn close(&self) -> Result<()>;
async fn get_disk_id(&self) -> Result<Option<Uuid>>;
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
fn path(&self) -> PathBuf;
fn get_disk_location(&self) -> DiskLocation;
// Healing
// DiskInfo
// NSScanner
// Volume operations.
async fn make_volume(&self, volume: &str) -> Result<()>;
async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>;
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>>;
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo>;
async fn delete_volume(&self, volume: &str) -> Result<()>;
// 并发边读边写 w <- MetaCacheEntry
async fn walk_dir<W: AsyncWrite + Unpin + Send + Sync + 'static>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()>;
// Metadata operations
async fn delete_version(
&self,
volume: &str,
path: &str,
fi: FileInfo,
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<()>;
async fn delete_versions(
&self,
volume: &str,
versions: Vec<FileInfoVersions>,
opts: DeleteOptions,
) -> Result<Vec<Option<Error>>>;
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()>;
async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>;
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()>;
async fn read_version(
&self,
org_volume: &str,
volume: &str,
path: &str,
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo>;
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo>;
async fn rename_data(
&self,
src_volume: &str,
src_path: &str,
file_info: FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp>;
// File operations.
// 读目录下的所有文件、目录
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>>;
async fn read_file(&self, volume: &str, path: &str) -> Result<Box<dyn AsyncRead>>;
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Box<dyn AsyncRead>>;
async fn append_file(&self, volume: &str, path: &str) -> Result<Box<dyn AsyncWrite>>;
async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<Box<dyn AsyncWrite>>;
// ReadFileStream
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>;
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()>;
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
// VerifyFile
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp>;
// CheckParts
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp>;
// StatInfoFile
// ReadParts
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>>;
// CleanAbandonedData
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()>;
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>>;
async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo>;
// async fn ns_scanner(
// &self,
// cache: &DataUsageCache,
// updates: Sender<DataUsageEntry>,
// scan_mode: HealScanMode,
// we_sleep: ShouldSleepFn,
// ) -> Result<DataUsageCache>;
// async fn healing(&self) -> Option<HealingTracker>;
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct CheckPartsResp {
pub results: Vec<usize>,
}
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct UpdateMetadataOpts {
pub no_persistence: bool,
}
pub struct DiskLocation {
pub pool_idx: Option<usize>,
pub set_idx: Option<usize>,
pub disk_idx: Option<usize>,
}
impl DiskLocation {
pub fn valid(&self) -> bool {
self.pool_idx.is_some() && self.set_idx.is_some() && self.disk_idx.is_some()
}
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct DiskInfoOptions {
pub disk_id: String,
pub metrics: bool,
pub noop: bool,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct DiskInfo {
pub total: u64,
pub free: u64,
pub used: u64,
pub used_inodes: u64,
pub free_inodes: u64,
pub major: u64,
pub minor: u64,
pub nr_requests: u64,
pub fs_type: String,
pub root_disk: bool,
pub healing: bool,
pub scanning: bool,
pub endpoint: String,
pub mount_path: String,
pub id: String,
pub rotational: bool,
pub metrics: DiskMetrics,
pub error: String,
}
#[derive(Clone, Debug, Default)]
pub struct Info {
pub total: u64,
pub free: u64,
pub used: u64,
pub files: u64,
pub ffree: u64,
pub fstype: String,
pub major: u64,
pub minor: u64,
pub name: String,
pub rotational: bool,
pub nrrequests: u64,
}
// #[derive(Debug, Default, Clone, Serialize, Deserialize)]
// pub struct FileInfoVersions {
// // Name of the volume.
// pub volume: String,
// // Name of the file.
// pub name: String,
// // Represents the latest mod time of the
// // latest version.
// pub latest_mod_time: Option<OffsetDateTime>,
// pub versions: Vec<FileInfo>,
// pub free_versions: Vec<FileInfo>,
// }
// impl FileInfoVersions {
// pub fn find_version_index(&self, v: &str) -> Option<usize> {
// if v.is_empty() {
// return None;
// }
// let vid = Uuid::parse_str(v).unwrap_or(Uuid::nil());
// self.versions.iter().position(|v| v.version_id == Some(vid))
// }
// }
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct WalkDirOptions {
// Bucket to scanner
pub bucket: String,
// Directory inside the bucket.
pub base_dir: String,
// Do a full recursive scan.
pub recursive: bool,
// ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
pub report_notfound: bool,
// FilterPrefix will only return results with given prefix within folder.
// Should never contain a slash.
pub filter_prefix: Option<String>,
// ForwardTo will forward to the given object path.
pub forward_to: Option<String>,
// Limit the number of returned objects if > 0.
pub limit: i32,
// DiskID contains the disk ID of the disk.
// Leave empty to not check disk ID.
pub disk_id: String,
}
// move metacache to metacache.rs
#[derive(Clone, Debug, Default)]
pub struct DiskOption {
pub cleanup: bool,
pub health_check: bool,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct RenameDataResp {
pub old_data_dir: Option<Uuid>,
pub sign: Option<Vec<u8>>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct DeleteOptions {
pub recursive: bool,
pub immediate: bool,
pub undo_write: bool,
pub old_data_dir: Option<Uuid>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReadMultipleReq {
pub bucket: String,
pub prefix: String,
pub files: Vec<String>,
pub max_size: usize,
pub metadata_only: bool,
pub abort404: bool,
pub max_results: usize,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ReadMultipleResp {
pub bucket: String,
pub prefix: String,
pub file: String,
pub exists: bool,
pub error: String,
pub data: Vec<u8>,
pub mod_time: Option<OffsetDateTime>,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct VolumeInfo {
pub name: String,
pub created: Option<OffsetDateTime>,
}
#[derive(Deserialize, Serialize, Debug, Default)]
pub struct ReadOptions {
pub incl_free_versions: bool,
pub read_data: bool,
pub healing: bool,
}
crates/disk/src/endpoint.rs
-379
View File
@@ -1,379 +0,0 @@
use path_absolutize::Absolutize;
use rustfs_utils::{is_local_host, is_socket_addr};
use std::{fmt::Display, path::Path};
use url::{ParseError, Url};
/// enum for endpoint type.
#[derive(PartialEq, Eq, Debug)]
pub enum EndpointType {
/// path style endpoint type enum.
Path,
/// URL style endpoint type enum.
Url,
}
/// any type of endpoint.
#[derive(Debug, PartialEq, Eq, Clone, Hash)]
pub struct Endpoint {
pub url: url::Url,
pub is_local: bool,
pub pool_idx: i32,
pub set_idx: i32,
pub disk_idx: i32,
}
impl Display for Endpoint {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if self.url.scheme() == "file" {
write!(f, "{}", self.get_file_path())
} else {
write!(f, "{}", self.url)
}
}
}
impl TryFrom<&str> for Endpoint {
/// The type returned in the event of a conversion error.
type Error = std::io::Error;
/// Performs the conversion.
fn try_from(value: &str) -> core::result::Result<Self, Self::Error> {
// check whether given path is not empty.
if ["", "/", "\\"].iter().any(|&v| v.eq(value)) {
return Err(std::io::Error::other("empty or root endpoint is not supported"));
}
let mut is_local = false;
let url = match Url::parse(value) {
#[allow(unused_mut)]
Ok(mut url) if url.has_host() => {
// URL style of endpoint.
// Valid URL style endpoint is
// - Scheme field must contain "http" or "https"
// - All field should be empty except Host and Path.
if !((url.scheme() == "http" || url.scheme() == "https")
&& url.username().is_empty()
&& url.fragment().is_none()
&& url.query().is_none())
{
return Err(std::io::Error::other("invalid URL endpoint format"));
}
let path = url.path().to_string();
#[cfg(not(windows))]
let path = Path::new(&path).absolutize()?;
// On windows having a preceding SlashSeparator will cause problems, if the
// command line already has C:/<export-folder/ in it. Final resulting
// path on windows might become C:/C:/ this will cause problems
// of starting rustfs server properly in distributed mode on windows.
// As a special case make sure to trim the separator.
#[cfg(windows)]
let path = Path::new(&path[1..]).absolutize()?;
if path.parent().is_none() || Path::new("").eq(&path) {
return Err(std::io::Error::other("empty or root path is not supported in URL endpoint"));
}
match path.to_str() {
Some(v) => url.set_path(v),
None => return Err(std::io::Error::other("invalid path")),
}
url
}
Ok(_) => {
// like d:/foo
is_local = true;
url_parse_from_file_path(value)?
}
Err(e) => match e {
ParseError::InvalidPort => {
return Err(std::io::Error::other(
"invalid URL endpoint format: port number must be between 1 to 65535",
))
}
ParseError::EmptyHost => return Err(std::io::Error::other("invalid URL endpoint format: empty host name")),
ParseError::RelativeUrlWithoutBase => {
// like /foo
is_local = true;
url_parse_from_file_path(value)?
}
_ => return Err(std::io::Error::other(format!("invalid URL endpoint format: {}", e))),
},
};
Ok(Endpoint {
url,
is_local,
pool_idx: -1,
set_idx: -1,
disk_idx: -1,
})
}
}
impl Endpoint {
/// returns type of endpoint.
pub fn get_type(&self) -> EndpointType {
if self.url.scheme() == "file" {
EndpointType::Path
} else {
EndpointType::Url
}
}
/// sets a specific pool number to this node
pub fn set_pool_index(&mut self, idx: usize) {
self.pool_idx = idx as i32
}
/// sets a specific set number to this node
pub fn set_set_index(&mut self, idx: usize) {
self.set_idx = idx as i32
}
/// sets a specific disk number to this node
pub fn set_disk_index(&mut self, idx: usize) {
self.disk_idx = idx as i32
}
/// resolves the host and updates if it is local or not.
pub fn update_is_local(&mut self, local_port: u16) -> std::io::Result<()> {
match (self.url.scheme(), self.url.host()) {
(v, Some(host)) if v != "file" => {
self.is_local = is_local_host(host, self.url.port().unwrap_or_default(), local_port)?;
}
_ => {}
}
Ok(())
}
/// returns the host to be used for grid connections.
pub fn grid_host(&self) -> String {
match (self.url.host(), self.url.port()) {
(Some(host), Some(port)) => format!("{}://{}:{}", self.url.scheme(), host, port),
(Some(host), None) => format!("{}://{}", self.url.scheme(), host),
_ => String::new(),
}
}
pub fn host_port(&self) -> String {
match (self.url.host(), self.url.port()) {
(Some(host), Some(port)) => format!("{}:{}", host, port),
(Some(host), None) => format!("{}", host),
_ => String::new(),
}
}
pub fn get_file_path(&self) -> &str {
let path = self.url.path();
#[cfg(windows)]
let path = &path[1..];
path
}
}
/// parse a file path into an URL.
fn url_parse_from_file_path(value: &str) -> std::io::Result<url::Url> {
// Only check if the arg is an ip address and ask for scheme since its absent.
// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
// /mnt/export1. So we go ahead and start the rustfs server in FS modes in these cases.
let addr: Vec<&str> = value.splitn(2, '/').collect();
if is_socket_addr(addr[0]) {
return Err(std::io::Error::other("invalid URL endpoint format: missing scheme http or https"));
}
let file_path = match Path::new(value).absolutize() {
Ok(path) => path,
Err(err) => return Err(std::io::Error::other(format!("absolute path failed: {}", err))),
};
match Url::from_file_path(file_path) {
Ok(url) => Ok(url),
Err(_) => Err(std::io::Error::other("Convert a file path into an URL failed")),
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_new_endpoint() {
#[derive(Default)]
struct TestCase<'a> {
arg: &'a str,
expected_endpoint: Option<Endpoint>,
expected_type: Option<EndpointType>,
expected_err: Option<std::io::Error>,
}
let u2 = url::Url::parse("https://example.org/path").unwrap();
let u4 = url::Url::parse("http://192.168.253.200/path").unwrap();
let u6 = url::Url::parse("http://server:/path").unwrap();
let root_slash_foo = url::Url::from_file_path("/foo").unwrap();
let test_cases = [
TestCase {
arg: "/foo",
expected_endpoint: Some(Endpoint {
url: root_slash_foo,
is_local: true,
pool_idx: -1,
set_idx: -1,
disk_idx: -1,
}),
expected_type: Some(EndpointType::Path),
expected_err: None,
},
TestCase {
arg: "https://example.org/path",
expected_endpoint: Some(Endpoint {
url: u2,
is_local: false,
pool_idx: -1,
set_idx: -1,
disk_idx: -1,
}),
expected_type: Some(EndpointType::Url),
expected_err: None,
},
TestCase {
arg: "http://192.168.253.200/path",
expected_endpoint: Some(Endpoint {
url: u4,
is_local: false,
pool_idx: -1,
set_idx: -1,
disk_idx: -1,
}),
expected_type: Some(EndpointType::Url),
expected_err: None,
},
TestCase {
arg: "",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("empty or root endpoint is not supported")),
},
TestCase {
arg: "/",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("empty or root endpoint is not supported")),
},
TestCase {
arg: "\\",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("empty or root endpoint is not supported")),
},
TestCase {
arg: "c://foo",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("invalid URL endpoint format")),
},
TestCase {
arg: "ftp://foo",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("invalid URL endpoint format")),
},
TestCase {
arg: "http://server/path?location",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("invalid URL endpoint format")),
},
TestCase {
arg: "http://:/path",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("invalid URL endpoint format: empty host name")),
},
TestCase {
arg: "http://:8080/path",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("invalid URL endpoint format: empty host name")),
},
TestCase {
arg: "http://server:/path",
expected_endpoint: Some(Endpoint {
url: u6,
is_local: false,
pool_idx: -1,
set_idx: -1,
disk_idx: -1,
}),
expected_type: Some(EndpointType::Url),
expected_err: None,
},
TestCase {
arg: "https://93.184.216.34:808080/path",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other(
"invalid URL endpoint format: port number must be between 1 to 65535",
)),
},
TestCase {
arg: "http://server:8080//",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("empty or root path is not supported in URL endpoint")),
},
TestCase {
arg: "http://server:8080/",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("empty or root path is not supported in URL endpoint")),
},
TestCase {
arg: "192.168.1.210:9000",
expected_endpoint: None,
expected_type: None,
expected_err: Some(std::io::Error::other("invalid URL endpoint format: missing scheme http or https")),
},
];
for test_case in test_cases {
let ret = Endpoint::try_from(test_case.arg);
if test_case.expected_err.is_none() && ret.is_err() {
panic!("{}: error: expected = <nil>, got = {:?}", test_case.arg, ret);
}
if test_case.expected_err.is_some() && ret.is_ok() {
panic!("{}: error: expected = {:?}, got = <nil>", test_case.arg, test_case.expected_err);
}
match (test_case.expected_err, ret) {
(None, Err(e)) => panic!("{}: error: expected = <nil>, got = {}", test_case.arg, e),
(None, Ok(mut ep)) => {
let _ = ep.update_is_local(9000);
if test_case.expected_type != Some(ep.get_type()) {
panic!(
"{}: type: expected = {:?}, got = {:?}",
test_case.arg,
test_case.expected_type,
ep.get_type()
);
}
assert_eq!(test_case.expected_endpoint, Some(ep), "{}: endpoint", test_case.arg);
}
(Some(e), Ok(_)) => panic!("{}: error: expected = {}, got = <nil>", test_case.arg, e),
(Some(e), Err(e2)) => {
assert_eq!(e.to_string(), e2.to_string(), "{}: error: expected = {}, got = {}", test_case.arg, e, e2)
}
}
}
}
}
crates/disk/src/error.rs
-594
View File
@@ -1,594 +0,0 @@
use std::io::{self, ErrorKind};
use std::path::PathBuf;
use tracing::error;
use crate::quorum::CheckErrorFn;
use crate::utils::ERROR_TYPE_MASK;
use common::error::{Error, Result};
// DiskError == StorageErr
#[derive(Debug, thiserror::Error)]
pub enum DiskError {
#[error("maximum versions exceeded, please delete few versions to proceed")]
MaxVersionsExceeded,
#[error("unexpected error")]
Unexpected,
#[error("corrupted format")]
CorruptedFormat,
#[error("corrupted backend")]
CorruptedBackend,
#[error("unformatted disk error")]
UnformattedDisk,
#[error("inconsistent drive found")]
InconsistentDisk,
#[error("drive does not support O_DIRECT")]
UnsupportedDisk,
#[error("drive path full")]
DiskFull,
#[error("disk not a dir")]
DiskNotDir,
#[error("disk not found")]
DiskNotFound,
#[error("drive still did not complete the request")]
DiskOngoingReq,
#[error("drive is part of root drive, will not be used")]
DriveIsRoot,
#[error("remote drive is faulty")]
FaultyRemoteDisk,
#[error("drive is faulty")]
FaultyDisk,
#[error("drive access denied")]
DiskAccessDenied,
#[error("file not found")]
FileNotFound,
#[error("file version not found")]
FileVersionNotFound,
#[error("too many open files, please increase 'ulimit -n'")]
TooManyOpenFiles,
#[error("file name too long")]
FileNameTooLong,
#[error("volume already exists")]
VolumeExists,
#[error("not of regular file type")]
IsNotRegular,
#[error("path not found")]
PathNotFound,
#[error("volume not found")]
VolumeNotFound,
#[error("volume is not empty")]
VolumeNotEmpty,
#[error("volume access denied")]
VolumeAccessDenied,
#[error("disk access denied")]
FileAccessDenied,
#[error("file is corrupted")]
FileCorrupt,
#[error("bit-rot hash algorithm is invalid")]
BitrotHashAlgoInvalid,
#[error("Rename across devices not allowed, please fix your backend configuration")]
CrossDeviceLink,
#[error("less data available than what was requested")]
LessData,
#[error("more data was sent than what was advertised")]
MoreData,
#[error("outdated XL meta")]
OutdatedXLMeta,
#[error("part missing or corrupt")]
PartMissingOrCorrupt,
#[error("No healing is required")]
NoHealRequired,
}
impl DiskError {
/// Checks if the given array of errors contains fatal disk errors.
/// If all errors are of the same fatal disk error type, returns the corresponding error.
/// Otherwise, returns Ok.
///
/// # Parameters
/// - `errs`: A slice of optional errors.
///
/// # Returns
/// If all errors are of the same fatal disk error type, returns the corresponding error.
/// Otherwise, returns Ok.
pub fn check_disk_fatal_errs(errs: &[Option<Error>]) -> Result<()> {
if DiskError::UnsupportedDisk.count_errs(errs) == errs.len() {
return Err(DiskError::UnsupportedDisk.into());
}
if DiskError::FileAccessDenied.count_errs(errs) == errs.len() {
return Err(DiskError::FileAccessDenied.into());
}
if DiskError::DiskNotDir.count_errs(errs) == errs.len() {
return Err(DiskError::DiskNotDir.into());
}
Ok(())
}
pub fn count_errs(&self, errs: &[Option<Error>]) -> usize {
errs.iter()
.filter(|&err| match err {
None => false,
Some(e) => self.is(e),
})
.count()
}
pub fn quorum_unformatted_disks(errs: &[Option<Error>]) -> bool {
DiskError::UnformattedDisk.count_errs(errs) > (errs.len() / 2)
}
pub fn should_init_erasure_disks(errs: &[Option<Error>]) -> bool {
DiskError::UnformattedDisk.count_errs(errs) == errs.len()
}
/// Check if the error is a disk error
pub fn is(&self, err: &Error) -> bool {
if let Some(e) = err.downcast_ref::<DiskError>() {
e == self
} else {
false
}
}
}
impl DiskError {
pub fn to_u32(&self) -> u32 {
match self {
DiskError::MaxVersionsExceeded => 0x01,
DiskError::Unexpected => 0x02,
DiskError::CorruptedFormat => 0x03,
DiskError::CorruptedBackend => 0x04,
DiskError::UnformattedDisk => 0x05,
DiskError::InconsistentDisk => 0x06,
DiskError::UnsupportedDisk => 0x07,
DiskError::DiskFull => 0x08,
DiskError::DiskNotDir => 0x09,
DiskError::DiskNotFound => 0x0A,
DiskError::DiskOngoingReq => 0x0B,
DiskError::DriveIsRoot => 0x0C,
DiskError::FaultyRemoteDisk => 0x0D,
DiskError::FaultyDisk => 0x0E,
DiskError::DiskAccessDenied => 0x0F,
DiskError::FileNotFound => 0x10,
DiskError::FileVersionNotFound => 0x11,
DiskError::TooManyOpenFiles => 0x12,
DiskError::FileNameTooLong => 0x13,
DiskError::VolumeExists => 0x14,
DiskError::IsNotRegular => 0x15,
DiskError::PathNotFound => 0x16,
DiskError::VolumeNotFound => 0x17,
DiskError::VolumeNotEmpty => 0x18,
DiskError::VolumeAccessDenied => 0x19,
DiskError::FileAccessDenied => 0x1A,
DiskError::FileCorrupt => 0x1B,
DiskError::BitrotHashAlgoInvalid => 0x1C,
DiskError::CrossDeviceLink => 0x1D,
DiskError::LessData => 0x1E,
DiskError::MoreData => 0x1F,
DiskError::OutdatedXLMeta => 0x20,
DiskError::PartMissingOrCorrupt => 0x21,
DiskError::NoHealRequired => 0x22,
}
}
pub fn from_u32(error: u32) -> Option<Self> {
match error & ERROR_TYPE_MASK {
0x01 => Some(DiskError::MaxVersionsExceeded),
0x02 => Some(DiskError::Unexpected),
0x03 => Some(DiskError::CorruptedFormat),
0x04 => Some(DiskError::CorruptedBackend),
0x05 => Some(DiskError::UnformattedDisk),
0x06 => Some(DiskError::InconsistentDisk),
0x07 => Some(DiskError::UnsupportedDisk),
0x08 => Some(DiskError::DiskFull),
0x09 => Some(DiskError::DiskNotDir),
0x0A => Some(DiskError::DiskNotFound),
0x0B => Some(DiskError::DiskOngoingReq),
0x0C => Some(DiskError::DriveIsRoot),
0x0D => Some(DiskError::FaultyRemoteDisk),
0x0E => Some(DiskError::FaultyDisk),
0x0F => Some(DiskError::DiskAccessDenied),
0x10 => Some(DiskError::FileNotFound),
0x11 => Some(DiskError::FileVersionNotFound),
0x12 => Some(DiskError::TooManyOpenFiles),
0x13 => Some(DiskError::FileNameTooLong),
0x14 => Some(DiskError::VolumeExists),
0x15 => Some(DiskError::IsNotRegular),
0x16 => Some(DiskError::PathNotFound),
0x17 => Some(DiskError::VolumeNotFound),
0x18 => Some(DiskError::VolumeNotEmpty),
0x19 => Some(DiskError::VolumeAccessDenied),
0x1A => Some(DiskError::FileAccessDenied),
0x1B => Some(DiskError::FileCorrupt),
0x1C => Some(DiskError::BitrotHashAlgoInvalid),
0x1D => Some(DiskError::CrossDeviceLink),
0x1E => Some(DiskError::LessData),
0x1F => Some(DiskError::MoreData),
0x20 => Some(DiskError::OutdatedXLMeta),
0x21 => Some(DiskError::PartMissingOrCorrupt),
0x22 => Some(DiskError::NoHealRequired),
_ => None,
}
}
}
impl PartialEq for DiskError {
fn eq(&self, other: &Self) -> bool {
core::mem::discriminant(self) == core::mem::discriminant(other)
}
}
impl CheckErrorFn for DiskError {
fn is(&self, e: &Error) -> bool {
self.is(e)
}
}
pub fn clone_disk_err(e: &DiskError) -> Error {
match e {
DiskError::MaxVersionsExceeded => Error::new(DiskError::MaxVersionsExceeded),
DiskError::Unexpected => Error::new(DiskError::Unexpected),
DiskError::CorruptedFormat => Error::new(DiskError::CorruptedFormat),
DiskError::CorruptedBackend => Error::new(DiskError::CorruptedBackend),
DiskError::UnformattedDisk => Error::new(DiskError::UnformattedDisk),
DiskError::InconsistentDisk => Error::new(DiskError::InconsistentDisk),
DiskError::UnsupportedDisk => Error::new(DiskError::UnsupportedDisk),
DiskError::DiskFull => Error::new(DiskError::DiskFull),
DiskError::DiskNotDir => Error::new(DiskError::DiskNotDir),
DiskError::DiskNotFound => Error::new(DiskError::DiskNotFound),
DiskError::DiskOngoingReq => Error::new(DiskError::DiskOngoingReq),
DiskError::DriveIsRoot => Error::new(DiskError::DriveIsRoot),
DiskError::FaultyRemoteDisk => Error::new(DiskError::FaultyRemoteDisk),
DiskError::FaultyDisk => Error::new(DiskError::FaultyDisk),
DiskError::DiskAccessDenied => Error::new(DiskError::DiskAccessDenied),
DiskError::FileNotFound => Error::new(DiskError::FileNotFound),
DiskError::FileVersionNotFound => Error::new(DiskError::FileVersionNotFound),
DiskError::TooManyOpenFiles => Error::new(DiskError::TooManyOpenFiles),
DiskError::FileNameTooLong => Error::new(DiskError::FileNameTooLong),
DiskError::VolumeExists => Error::new(DiskError::VolumeExists),
DiskError::IsNotRegular => Error::new(DiskError::IsNotRegular),
DiskError::PathNotFound => Error::new(DiskError::PathNotFound),
DiskError::VolumeNotFound => Error::new(DiskError::VolumeNotFound),
DiskError::VolumeNotEmpty => Error::new(DiskError::VolumeNotEmpty),
DiskError::VolumeAccessDenied => Error::new(DiskError::VolumeAccessDenied),
DiskError::FileAccessDenied => Error::new(DiskError::FileAccessDenied),
DiskError::FileCorrupt => Error::new(DiskError::FileCorrupt),
DiskError::BitrotHashAlgoInvalid => Error::new(DiskError::BitrotHashAlgoInvalid),
DiskError::CrossDeviceLink => Error::new(DiskError::CrossDeviceLink),
DiskError::LessData => Error::new(DiskError::LessData),
DiskError::MoreData => Error::new(DiskError::MoreData),
DiskError::OutdatedXLMeta => Error::new(DiskError::OutdatedXLMeta),
DiskError::PartMissingOrCorrupt => Error::new(DiskError::PartMissingOrCorrupt),
DiskError::NoHealRequired => Error::new(DiskError::NoHealRequired),
}
}
pub fn os_err_to_file_err(e: io::Error) -> Error {
match e.kind() {
io::ErrorKind::NotFound => Error::new(DiskError::FileNotFound),
io::ErrorKind::PermissionDenied => Error::new(DiskError::FileAccessDenied),
// io::ErrorKind::ConnectionRefused => todo!(),
// io::ErrorKind::ConnectionReset => todo!(),
// io::ErrorKind::HostUnreachable => todo!(),
// io::ErrorKind::NetworkUnreachable => todo!(),
// io::ErrorKind::ConnectionAborted => todo!(),
// io::ErrorKind::NotConnected => todo!(),
// io::ErrorKind::AddrInUse => todo!(),
// io::ErrorKind::AddrNotAvailable => todo!(),
// io::ErrorKind::NetworkDown => todo!(),
// io::ErrorKind::BrokenPipe => todo!(),
// io::ErrorKind::AlreadyExists => todo!(),
// io::ErrorKind::WouldBlock => todo!(),
// io::ErrorKind::NotADirectory => DiskError::FileNotFound,
// io::ErrorKind::IsADirectory => DiskError::FileNotFound,
// io::ErrorKind::DirectoryNotEmpty => DiskError::VolumeNotEmpty,
// io::ErrorKind::ReadOnlyFilesystem => todo!(),
// io::ErrorKind::FilesystemLoop => todo!(),
// io::ErrorKind::StaleNetworkFileHandle => todo!(),
// io::ErrorKind::InvalidInput => todo!(),
// io::ErrorKind::InvalidData => todo!(),
// io::ErrorKind::TimedOut => todo!(),
// io::ErrorKind::WriteZero => todo!(),
// io::ErrorKind::StorageFull => DiskError::DiskFull,
// io::ErrorKind::NotSeekable => todo!(),
// io::ErrorKind::FilesystemQuotaExceeded => todo!(),
// io::ErrorKind::FileTooLarge => todo!(),
// io::ErrorKind::ResourceBusy => todo!(),
// io::ErrorKind::ExecutableFileBusy => todo!(),
// io::ErrorKind::Deadlock => todo!(),
// io::ErrorKind::CrossesDevices => todo!(),
// io::ErrorKind::TooManyLinks =>DiskError::TooManyOpenFiles,
// io::ErrorKind::InvalidFilename => todo!(),
// io::ErrorKind::ArgumentListTooLong => todo!(),
// io::ErrorKind::Interrupted => todo!(),
// io::ErrorKind::Unsupported => todo!(),
// io::ErrorKind::UnexpectedEof => todo!(),
// io::ErrorKind::OutOfMemory => todo!(),
// io::ErrorKind::Other => todo!(),
// TODO: 把不支持的king用字符串处理
_ => Error::new(e),
}
}
#[derive(Debug, thiserror::Error)]
pub struct FileAccessDeniedWithContext {
pub path: PathBuf,
#[source]
pub source: std::io::Error,
}
impl std::fmt::Display for FileAccessDeniedWithContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "访问文件 '{}' 被拒绝: {}", self.path.display(), self.source)
}
}
pub fn is_unformatted_disk(err: &Error) -> bool {
matches!(err.downcast_ref::<DiskError>(), Some(DiskError::UnformattedDisk))
}
pub fn is_err_file_not_found(err: &Error) -> bool {
if let Some(ioerr) = err.downcast_ref::<io::Error>() {
return ioerr.kind() == ErrorKind::NotFound;
}
matches!(err.downcast_ref::<DiskError>(), Some(DiskError::FileNotFound))
}
pub fn is_err_file_version_not_found(err: &Error) -> bool {
matches!(err.downcast_ref::<DiskError>(), Some(DiskError::FileVersionNotFound))
}
pub fn is_err_volume_not_found(err: &Error) -> bool {
matches!(err.downcast_ref::<DiskError>(), Some(DiskError::VolumeNotFound))
}
pub fn is_err_eof(err: &Error) -> bool {
if let Some(ioerr) = err.downcast_ref::<io::Error>() {
return ioerr.kind() == ErrorKind::UnexpectedEof;
}
false
}
pub fn is_sys_err_no_space(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 28;
}
false
}
pub fn is_sys_err_invalid_arg(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 22;
}
false
}
// TODO: ??
pub fn is_sys_err_io(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 5;
}
false
}
pub fn is_sys_err_is_dir(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 21;
}
false
}
pub fn is_sys_err_not_dir(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 20;
}
false
}
pub fn is_sys_err_too_long(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 63;
}
false
}
pub fn is_sys_err_too_many_symlinks(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 62;
}
false
}
pub fn is_sys_err_not_empty(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
if no == 66 {
return true;
}
if cfg!(target_os = "solaris") && no == 17 {
return true;
}
if cfg!(target_os = "windows") && no == 145 {
return true;
}
}
false
}
pub fn is_sys_err_path_not_found(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
if cfg!(target_os = "windows") {
if no == 3 {
return true;
}
} else if no == 2 {
return true;
}
}
false
}
pub fn is_sys_err_handle_invalid(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
if cfg!(target_os = "windows") {
if no == 6 {
return true;
}
} else {
return false;
}
}
false
}
pub fn is_sys_err_cross_device(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 18;
}
false
}
pub fn is_sys_err_too_many_files(e: &io::Error) -> bool {
if let Some(no) = e.raw_os_error() {
return no == 23 || no == 24;
}
false
}
// pub fn os_is_not_exist(e: &io::Error) -> bool {
// e.kind() == ErrorKind::NotFound
// }
pub fn os_is_permission(e: &io::Error) -> bool {
if e.kind() == ErrorKind::PermissionDenied {
return true;
}
if let Some(no) = e.raw_os_error() {
if no == 30 {
return true;
}
}
false
}
// pub fn os_is_exist(e: &io::Error) -> bool {
// e.kind() == ErrorKind::AlreadyExists
// }
// // map_err_not_exists
// pub fn map_err_not_exists(e: io::Error) -> Error {
// if os_is_not_exist(&e) {
// return Error::new(DiskError::VolumeNotEmpty);
// } else if is_sys_err_io(&e) {
// return Error::new(DiskError::FaultyDisk);
// }
// Error::new(e)
// }
// pub fn convert_access_error(e: io::Error, per_err: DiskError) -> Error {
// if os_is_not_exist(&e) {
// return Error::new(DiskError::VolumeNotEmpty);
// } else if is_sys_err_io(&e) {
// return Error::new(DiskError::FaultyDisk);
// } else if os_is_permission(&e) {
// return Error::new(per_err);
// }
// Error::new(e)
// }
pub fn is_all_not_found(errs: &[Option<Error>]) -> bool {
for err in errs.iter() {
if let Some(err) = err {
if let Some(err) = err.downcast_ref::<DiskError>() {
match err {
DiskError::FileNotFound | DiskError::VolumeNotFound | &DiskError::FileVersionNotFound => {
continue;
}
_ => return false,
}
}
}
return false;
}
!errs.is_empty()
}
pub fn is_all_volume_not_found(errs: &[Option<Error>]) -> bool {
DiskError::VolumeNotFound.count_errs(errs) == errs.len()
}
pub fn is_all_buckets_not_found(errs: &[Option<Error>]) -> bool {
if errs.is_empty() {
return false;
}
let mut not_found_count = 0;
for err in errs.iter().flatten() {
match err.downcast_ref() {
Some(DiskError::VolumeNotFound) | Some(DiskError::DiskNotFound) => {
not_found_count += 1;
}
_ => {}
}
}
errs.len() == not_found_count
}
pub fn is_err_os_not_exist(err: &Error) -> bool {
if let Some(os_err) = err.downcast_ref::<io::Error>() {
os_err.kind() == ErrorKind::NotFound
} else {
false
}
}
pub fn is_err_os_disk_full(err: &Error) -> bool {
if let Some(os_err) = err.downcast_ref::<io::Error>() {
is_sys_err_no_space(os_err)
} else if let Some(e) = err.downcast_ref::<DiskError>() {
e == &DiskError::DiskFull
} else {
false
}
}
crates/disk/src/format.rs
-273
View File
@@ -1,273 +0,0 @@
// use super::{error::DiskError, DiskInfo};
use rustfs_error::{Error, Result};
use serde::{Deserialize, Serialize};
use serde_json::Error as JsonError;
use uuid::Uuid;
use crate::api::DiskInfo;
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum FormatMetaVersion {
#[serde(rename = "1")]
V1,
#[serde(other)]
Unknown,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum FormatBackend {
#[serde(rename = "xl")]
Erasure,
#[serde(rename = "xl-single")]
ErasureSingle,
#[serde(other)]
Unknown,
}
/// Represents the V3 backend disk structure version
/// under `.rustfs.sys` and actual data namespace.
///
/// FormatErasureV3 - structure holds format config version '3'.
///
/// The V3 format to support "large bucket" support where a bucket
/// can span multiple erasure sets.
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct FormatErasureV3 {
/// Version of 'xl' format.
pub version: FormatErasureVersion,
/// This field carries assigned disk uuid.
pub this: Uuid,
/// Sets field carries the input disk order generated the first
/// time when fresh disks were supplied, it is a two dimensional
/// array second dimension represents list of disks used per set.
pub sets: Vec<Vec<Uuid>>,
/// Distribution algorithm represents the hashing algorithm
/// to pick the right set index for an object.
#[serde(rename = "distributionAlgo")]
pub distribution_algo: DistributionAlgoVersion,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum FormatErasureVersion {
#[serde(rename = "1")]
V1,
#[serde(rename = "2")]
V2,
#[serde(rename = "3")]
V3,
#[serde(other)]
Unknown,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub enum DistributionAlgoVersion {
#[serde(rename = "CRCMOD")]
V1,
#[serde(rename = "SIPMOD")]
V2,
#[serde(rename = "SIPMOD+PARITY")]
V3,
}
/// format.json currently has the format:
///
/// ```json
/// {
/// "version": "1",
/// "format": "XXXXX",
/// "id": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
/// "XXXXX": {
//
/// }
/// }
/// ```
///
/// Ideally we will never have a situation where we will have to change the
/// fields of this struct and deal with related migration.
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct FormatV3 {
/// Version of the format config.
pub version: FormatMetaVersion,
/// Format indicates the backend format type, supports two values 'xl' and 'xl-single'.
pub format: FormatBackend,
/// ID is the identifier for the rustfs deployment
pub id: Uuid,
#[serde(rename = "xl")]
pub erasure: FormatErasureV3,
// /// DiskInfo is an extended type which returns current
// /// disk usage per path.
#[serde(skip)]
pub disk_info: Option<DiskInfo>,
}
impl TryFrom<&[u8]> for FormatV3 {
type Error = JsonError;
fn try_from(data: &[u8]) -> core::result::Result<Self, JsonError> {
serde_json::from_slice(data)
}
}
impl TryFrom<&str> for FormatV3 {
type Error = JsonError;
fn try_from(data: &str) -> core::result::Result<Self, JsonError> {
serde_json::from_str(data)
}
}
impl FormatV3 {
/// Create a new format config with the given number of sets and set length.
pub fn new(num_sets: usize, set_len: usize) -> Self {
let format = if set_len == 1 {
FormatBackend::ErasureSingle
} else {
FormatBackend::Erasure
};
let erasure = FormatErasureV3 {
version: FormatErasureVersion::V3,
this: Uuid::nil(),
sets: (0..num_sets)
.map(|_| (0..set_len).map(|_| Uuid::new_v4()).collect())
.collect(),
distribution_algo: DistributionAlgoVersion::V3,
};
Self {
version: FormatMetaVersion::V1,
format,
id: Uuid::new_v4(),
erasure,
disk_info: None,
}
}
/// Returns the number of drives in the erasure set.
pub fn drives(&self) -> usize {
self.erasure.sets.iter().map(|v| v.len()).sum()
}
pub fn to_json(&self) -> Result<String> {
Ok(serde_json::to_string(self)?)
}
/// returns the i,j'th position of the input `diskID` against the reference
///
/// format, after successful validation.
/// - i'th position is the set index
/// - j'th position is the disk index in the current set
pub fn find_disk_index_by_disk_id(&self, disk_id: Uuid) -> Result<(usize, usize)> {
if disk_id == Uuid::nil() {
return Err(Error::DiskNotFound);
}
if disk_id == Uuid::max() {
return Err(Error::msg("disk offline"));
}
for (i, set) in self.erasure.sets.iter().enumerate() {
for (j, d) in set.iter().enumerate() {
if disk_id.eq(d) {
return Ok((i, j));
}
}
}
Err(Error::msg(format!("disk id not found {}", disk_id)))
}
pub fn check_other(&self, other: &FormatV3) -> Result<()> {
let mut tmp = other.clone();
let this = tmp.erasure.this;
tmp.erasure.this = Uuid::nil();
if self.erasure.sets.len() != other.erasure.sets.len() {
return Err(Error::msg(format!(
"Expected number of sets {}, got {}",
self.erasure.sets.len(),
other.erasure.sets.len()
)));
}
for i in 0..self.erasure.sets.len() {
if self.erasure.sets[i].len() != other.erasure.sets[i].len() {
return Err(Error::msg(format!(
"Each set should be of same size, expected {}, got {}",
self.erasure.sets[i].len(),
other.erasure.sets[i].len()
)));
}
for j in 0..self.erasure.sets[i].len() {
if self.erasure.sets[i][j] != other.erasure.sets[i][j] {
return Err(Error::msg(format!(
"UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)",
i,
j,
self.erasure.sets[i][j].to_string(),
other.erasure.sets[i][j].to_string(),
)));
}
}
}
for i in 0..tmp.erasure.sets.len() {
for j in 0..tmp.erasure.sets[i].len() {
if this == tmp.erasure.sets[i][j] {
return Ok(());
}
}
}
Err(Error::msg(format!(
"DriveID {:?} not found in any drive sets {:?}",
this, other.erasure.sets
)))
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_format_v1() {
let format = FormatV3::new(1, 4);
let str = serde_json::to_string(&format);
println!("{:?}", str);
let data = r#"
{
"version": "1",
"format": "xl",
"id": "321b3874-987d-4c15-8fa5-757c956b1243",
"xl": {
"version": "1",
"this": null,
"sets": [
[
"8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
"c26315da-05cf-4778-a9ea-b44ea09f58c5",
"fb87a891-18d3-44cf-a46f-bcc15093a038",
"356a925c-57b9-4313-88b3-053edf1104dc"
]
],
"distributionAlgo": "CRCMOD"
}
}"#;
let p = FormatV3::try_from(data);
println!("{:?}", p);
}
}
crates/disk/src/fs.rs
-179
View File
@@ -1,179 +0,0 @@
use std::{fs::Metadata, path::Path};
use tokio::{
fs::{self, File},
io,
};
#[cfg(not(windows))]
pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool {
use std::os::unix::fs::MetadataExt;
if f1.dev() != f2.dev() {
return false;
}
if f1.ino() != f2.ino() {
return false;
}
if f1.size() != f2.size() {
return false;
}
if f1.permissions() != f2.permissions() {
return false;
}
if f1.mtime() != f2.mtime() {
return false;
}
true
}
#[cfg(windows)]
pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool {
if f1.permissions() != f2.permissions() {
return false;
}
if f1.file_type() != f2.file_type() {
return false;
}
if f1.len() != f2.len() {
return false;
}
true
}
type FileMode = usize;
pub const O_RDONLY: FileMode = 0x00000;
pub const O_WRONLY: FileMode = 0x00001;
pub const O_RDWR: FileMode = 0x00002;
pub const O_CREATE: FileMode = 0x00040;
// pub const O_EXCL: FileMode = 0x00080;
// pub const O_NOCTTY: FileMode = 0x00100;
pub const O_TRUNC: FileMode = 0x00200;
// pub const O_NONBLOCK: FileMode = 0x00800;
pub const O_APPEND: FileMode = 0x00400;
// pub const O_SYNC: FileMode = 0x01000;
// pub const O_ASYNC: FileMode = 0x02000;
// pub const O_CLOEXEC: FileMode = 0x80000;
// read: bool,
// write: bool,
// append: bool,
// truncate: bool,
// create: bool,
// create_new: bool,
pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
let mut opts = fs::OpenOptions::new();
match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
O_RDONLY => {
opts.read(true);
}
O_WRONLY => {
opts.write(true);
}
O_RDWR => {
opts.read(true);
opts.write(true);
}
_ => (),
};
if mode & O_CREATE != 0 {
opts.create(true);
}
if mode & O_APPEND != 0 {
opts.append(true);
}
if mode & O_TRUNC != 0 {
opts.truncate(true);
}
opts.open(path.as_ref()).await
}
pub async fn access(path: impl AsRef<Path>) -> io::Result<()> {
fs::metadata(path).await?;
Ok(())
}
pub fn access_std(path: impl AsRef<Path>) -> io::Result<()> {
std::fs::metadata(path)?;
Ok(())
}
pub async fn lstat(path: impl AsRef<Path>) -> io::Result<Metadata> {
fs::metadata(path).await
}
pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
std::fs::metadata(path)
}
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
fs::create_dir_all(path.as_ref()).await
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
let meta = fs::metadata(path.as_ref()).await?;
if meta.is_dir() {
fs::remove_dir(path.as_ref()).await
} else {
fs::remove_file(path.as_ref()).await
}
}
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
let meta = fs::metadata(path.as_ref()).await?;
if meta.is_dir() {
fs::remove_dir_all(path.as_ref()).await
} else {
fs::remove_file(path.as_ref()).await
}
}
#[tracing::instrument(level = "debug", skip_all)]
pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
let meta = std::fs::metadata(path.as_ref())?;
if meta.is_dir() {
std::fs::remove_dir(path.as_ref())
} else {
std::fs::remove_file(path.as_ref())
}
}
pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
let meta = std::fs::metadata(path.as_ref())?;
if meta.is_dir() {
std::fs::remove_dir_all(path.as_ref())
} else {
std::fs::remove_file(path.as_ref())
}
}
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
fs::create_dir(path.as_ref()).await
}
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
fs::rename(from, to).await
}
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
std::fs::rename(from, to)
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
fs::read(path.as_ref()).await
}
crates/disk/src/lib.rs
-12
View File
@@ -1,12 +0,0 @@
pub mod endpoint;
// pub mod error;
pub mod format;
pub mod fs;
pub mod local;
// pub mod metacache;
pub mod api;
pub mod local_list;
pub mod os;
pub mod path;
pub mod remote;
pub mod utils;
crates/disk/src/local.rs
-2048
View File
File diff suppressed because it is too large Load Diff
crates/disk/src/local_bak.rs
-2364
View File
File diff suppressed because it is too large Load Diff
crates/disk/src/local_list.rs
-535
View File
@@ -1,535 +0,0 @@
use crate::{
api::{DiskAPI, DiskStore, WalkDirOptions, STORAGE_FORMAT_FILE},
local::LocalDisk,
os::is_empty_dir,
path::{self, decode_dir_object, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR},
};
use futures::future::join_all;
use rustfs_error::{Error, Result};
use rustfs_metacache::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, MetacacheWriter};
use std::{collections::HashSet, future::Future, pin::Pin, sync::Arc};
use tokio::{io::AsyncWrite, spawn, sync::broadcast::Receiver as B_Receiver};
use tracing::{error, info, warn};
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
pub type PartialFn = Box<dyn Fn(MetaCacheEntries, &[Option<Error>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
type FinishedFn = Box<dyn Fn(&[Option<Error>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
#[derive(Default)]
pub struct ListPathRawOptions {
pub disks: Vec<Option<DiskStore>>,
pub fallback_disks: Vec<Option<DiskStore>>,
pub bucket: String,
pub path: String,
pub recursive: bool,
pub filter_prefix: Option<String>,
pub forward_to: Option<String>,
pub min_disks: usize,
pub report_not_found: bool,
pub per_disk_limit: i32,
pub agreed: Option<AgreedFn>,
pub partial: Option<PartialFn>,
pub finished: Option<FinishedFn>,
}
impl Clone for ListPathRawOptions {
fn clone(&self) -> Self {
Self {
disks: self.disks.clone(),
fallback_disks: self.fallback_disks.clone(),
bucket: self.bucket.clone(),
path: self.path.clone(),
recursive: self.recursive,
filter_prefix: self.filter_prefix.clone(),
forward_to: self.forward_to.clone(),
min_disks: self.min_disks,
report_not_found: self.report_not_found,
per_disk_limit: self.per_disk_limit,
..Default::default()
}
}
}
pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -> Result<()> {
if opts.disks.is_empty() {
return Err(Error::msg("list_path_raw: 0 drives provided"));
}
let mut jobs: Vec<tokio::task::JoinHandle<std::result::Result<(), Error>>> = Vec::new();
let mut readers = Vec::with_capacity(opts.disks.len());
let fds = Arc::new(opts.fallback_disks.clone());
for disk in opts.disks.iter() {
let opdisk = disk.clone();
let opts_clone = opts.clone();
let fds_clone = fds.clone();
let (rd, mut wr) = tokio::io::duplex(64);
readers.push(MetacacheReader::new(rd));
jobs.push(spawn(async move {
let walk_opts = WalkDirOptions {
bucket: opts_clone.bucket.clone(),
base_dir: opts_clone.path.clone(),
recursive: opts_clone.recursive,
report_notfound: opts_clone.report_not_found,
filter_prefix: opts_clone.filter_prefix.clone(),
forward_to: opts_clone.forward_to.clone(),
limit: opts_clone.per_disk_limit,
..Default::default()
};
let mut need_fallback = false;
if let Some(disk) = opdisk {
match disk.walk_dir(walk_opts, &mut wr).await {
Ok(_res) => {}
Err(err) => {
error!("walk dir err {:?}", &err);
need_fallback = true;
}
}
} else {
need_fallback = true;
}
while need_fallback {
let disk = match fds_clone.iter().find(|d| d.is_some()) {
Some(d) => {
if let Some(disk) = d.clone() {
disk
} else {
break;
}
}
None => break,
};
match disk
.walk_dir(
WalkDirOptions {
bucket: opts_clone.bucket.clone(),
base_dir: opts_clone.path.clone(),
recursive: opts_clone.recursive,
report_notfound: opts_clone.report_not_found,
filter_prefix: opts_clone.filter_prefix.clone(),
forward_to: opts_clone.forward_to.clone(),
limit: opts_clone.per_disk_limit,
..Default::default()
},
&mut wr,
)
.await
{
Ok(_r) => {
need_fallback = false;
}
Err(err) => {
error!("walk dir2 err {:?}", &err);
break;
}
}
}
Ok(())
}));
}
let revjob = spawn(async move {
let mut errs: Vec<Option<Error>> = Vec::with_capacity(readers.len());
for _ in 0..readers.len() {
errs.push(None);
}
loop {
let mut current = MetaCacheEntry::default();
if rx.try_recv().is_ok() {
return Err(Error::msg("canceled"));
}
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
let mut at_eof = 0;
let mut fnf = 0;
let mut vnf = 0;
let mut has_err = 0;
let mut agree = 0;
for (i, r) in readers.iter_mut().enumerate() {
if errs[i].is_some() {
has_err += 1;
continue;
}
let entry = match r.peek().await {
Ok(res) => {
if let Some(entry) = res {
entry
} else {
at_eof += 1;
continue;
}
}
Err(err) => {
if err == Error::FaultyDisk {
at_eof += 1;
continue;
} else if err == Error::FileNotFound {
at_eof += 1;
fnf += 1;
continue;
} else if err == Error::VolumeNotFound {
at_eof += 1;
fnf += 1;
vnf += 1;
continue;
} else {
has_err += 1;
errs[i] = Some(err);
continue;
}
}
};
// If no current, add it.
if current.name.is_empty() {
top_entries[i] = Some(entry.clone());
current = entry;
agree += 1;
continue;
}
// If exact match, we agree.
if let (_, true) = current.matches(Some(&entry), true) {
top_entries[i] = Some(entry);
agree += 1;
continue;
}
// If only the name matches we didn't agree, but add it for resolution.
if entry.name == current.name {
top_entries[i] = Some(entry);
continue;
}
// We got different entries
if entry.name > current.name {
continue;
}
for item in top_entries.iter_mut().take(i) {
*item = None;
}
agree = 1;
top_entries[i] = Some(entry.clone());
current = entry;
}
if vnf > 0 && vnf >= (readers.len() - opts.min_disks) {
return Err(Error::VolumeNotFound);
}
if fnf > 0 && fnf >= (readers.len() - opts.min_disks) {
return Err(Error::FileNotFound);
}
if has_err > 0 && has_err > opts.disks.len() - opts.min_disks {
if let Some(finished_fn) = opts.finished.as_ref() {
finished_fn(&errs).await;
}
let mut combined_err = Vec::new();
errs.iter().zip(opts.disks.iter()).for_each(|(err, disk)| match (err, disk) {
(Some(err), Some(disk)) => {
combined_err.push(format!("drive {} returned: {}", disk.to_string(), err));
}
(Some(err), None) => {
combined_err.push(err.to_string());
}
_ => {}
});
return Err(Error::msg(combined_err.join(", ")));
}
// Break if all at EOF or error.
if at_eof + has_err == readers.len() {
if has_err > 0 {
if let Some(finished_fn) = opts.finished.as_ref() {
finished_fn(&errs).await;
}
}
break;
}
if agree == readers.len() {
for r in readers.iter_mut() {
let _ = r.skip(1).await;
}
if let Some(agreed_fn) = opts.agreed.as_ref() {
agreed_fn(current).await;
}
continue;
}
for (i, r) in readers.iter_mut().enumerate() {
if top_entries[i].is_some() {
let _ = r.skip(1).await;
}
}
if let Some(partial_fn) = opts.partial.as_ref() {
partial_fn(MetaCacheEntries(top_entries), &errs).await;
}
}
Ok(())
});
jobs.push(revjob);
let results = join_all(jobs).await;
for result in results {
if let Err(err) = result {
error!("list_path_raw err {:?}", err);
}
}
Ok(())
}
impl LocalDisk {
pub(crate) async fn scan_dir<W: AsyncWrite + Unpin>(
&self,
current: &mut String,
opts: &WalkDirOptions,
out: &mut MetacacheWriter<W>,
objs_returned: &mut i32,
) -> Result<()> {
let forward = {
opts.forward_to.as_ref().filter(|v| v.starts_with(&*current)).map(|v| {
let forward = v.trim_start_matches(&*current);
if let Some(idx) = forward.find('/') {
forward[..idx].to_owned()
} else {
forward.to_owned()
}
})
// if let Some(forward_to) = &opts.forward_to {
// } else {
// None
// }
// if !opts.forward_to.is_empty() && opts.forward_to.starts_with(&*current) {
// let forward = opts.forward_to.trim_start_matches(&*current);
// if let Some(idx) = forward.find('/') {
// &forward[..idx]
// } else {
// forward
// }
// } else {
// ""
// }
};
if opts.limit > 0 && *objs_returned >= opts.limit {
return Ok(());
}
let mut entries = match self.list_dir("", &opts.bucket, current, -1).await {
Ok(res) => res,
Err(e) => {
if e != Error::VolumeNotFound && e != Error::FileNotFound {
info!("scan list_dir {}, err {:?}", &current, &e);
}
if opts.report_notfound && (e == Error::VolumeNotFound || e == Error::FileNotFound) && current == &opts.base_dir {
return Err(Error::FileNotFound);
}
return Ok(());
}
};
if entries.is_empty() {
return Ok(());
}
let s = SLASH_SEPARATOR.chars().next().unwrap_or_default();
*current = current.trim_matches(s).to_owned();
let bucket = opts.bucket.as_str();
let mut dir_objes = HashSet::new();
// 第一层过滤
for item in entries.iter_mut() {
let entry = item.clone();
// check limit
if opts.limit > 0 && *objs_returned >= opts.limit {
return Ok(());
}
// check prefix
if let Some(filter_prefix) = &opts.filter_prefix {
if !entry.starts_with(filter_prefix) {
*item = "".to_owned();
continue;
}
}
if let Some(forward) = &forward {
if &entry < forward {
*item = "".to_owned();
continue;
}
}
if entry.ends_with(SLASH_SEPARATOR) {
if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) {
let entry = format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR);
dir_objes.insert(entry.clone());
*item = entry;
continue;
}
*item = entry.trim_end_matches(SLASH_SEPARATOR).to_owned();
continue;
}
*item = "".to_owned();
if entry.ends_with(STORAGE_FORMAT_FILE) {
//
let metadata = self
.read_metadata(self.get_object_path(bucket, format!("{}/{}", &current, &entry).as_str())?)
.await?;
// 用 strip_suffix 只删除一次
let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned();
let name = entry.trim_end_matches(SLASH_SEPARATOR);
let name = decode_dir_object(format!("{}/{}", &current, &name).as_str());
out.write_obj(&MetaCacheEntry {
name,
metadata,
..Default::default()
})
.await?;
*objs_returned += 1;
return Ok(());
}
}
entries.sort();
let mut entries = entries.as_slice();
if let Some(forward) = &forward {
for (i, entry) in entries.iter().enumerate() {
if entry >= forward || forward.starts_with(entry.as_str()) {
entries = &entries[i..];
break;
}
}
}
let mut dir_stack: Vec<String> = Vec::with_capacity(5);
for entry in entries.iter() {
if opts.limit > 0 && *objs_returned >= opts.limit {
return Ok(());
}
if entry.is_empty() {
continue;
}
let name = path::path_join_buf(&[current, entry]);
if !dir_stack.is_empty() {
if let Some(pop) = dir_stack.pop() {
if pop < name {
//
out.write_obj(&MetaCacheEntry {
name: pop.clone(),
..Default::default()
})
.await?;
if opts.recursive {
let mut opts = opts.clone();
opts.filter_prefix = None;
if let Err(er) = Box::pin(self.scan_dir(&mut pop.clone(), &opts, out, objs_returned)).await {
error!("scan_dir err {:?}", er);
}
}
}
}
}
let mut meta = MetaCacheEntry {
name,
..Default::default()
};
let mut is_dir_obj = false;
if let Some(_dir) = dir_objes.get(entry) {
is_dir_obj = true;
meta.name
.truncate(meta.name.len() - meta.name.chars().last().unwrap().len_utf8());
meta.name.push_str(GLOBAL_DIR_SUFFIX_WITH_SLASH);
}
let fname = format!("{}/{}", &meta.name, STORAGE_FORMAT_FILE);
match self.read_metadata(self.get_object_path(&opts.bucket, fname.as_str())?).await {
Ok(res) => {
if is_dir_obj {
meta.name = meta.name.trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH).to_owned();
meta.name.push_str(SLASH_SEPARATOR);
}
meta.metadata = res;
out.write_obj(&meta).await?;
*objs_returned += 1;
}
Err(err) => {
if err == Error::FileNotFound || err == Error::IsNotRegular {
// NOT an object, append to stack (with slash)
// If dirObject, but no metadata (which is unexpected) we skip it.
if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await {
meta.name.push_str(SLASH_SEPARATOR);
dir_stack.push(meta.name);
}
}
continue;
}
};
}
while let Some(dir) = dir_stack.pop() {
if opts.limit > 0 && *objs_returned >= opts.limit {
return Ok(());
}
out.write_obj(&MetaCacheEntry {
name: dir.clone(),
..Default::default()
})
.await?;
*objs_returned += 1;
if opts.recursive {
let mut dir = dir;
let mut opts = opts.clone();
opts.filter_prefix = None;
if let Err(er) = Box::pin(self.scan_dir(&mut dir, &opts, out, objs_returned)).await {
warn!("scan_dir err {:?}", &er);
}
}
}
Ok(())
}
}
crates/disk/src/metacache.rs
-608
View File
@@ -1,608 +0,0 @@
use crate::bucket::metadata_sys::get_versioning_config;
use crate::bucket::versioning::VersioningApi;
use crate::store_api::ObjectInfo;
use crate::utils::path::SLASH_SEPARATOR;
use rustfs_error::{Error, Result};
use rustfs_filemeta::merge_file_meta_versions;
use rustfs_filemeta::FileInfo;
use rustfs_filemeta::FileInfoVersions;
use rustfs_filemeta::FileMeta;
use rustfs_filemeta::FileMetaShallowVersion;
use rustfs_filemeta::VersionType;
use serde::Deserialize;
use serde::Serialize;
use std::cmp::Ordering;
use time::OffsetDateTime;
use tracing::warn;
#[derive(Clone, Debug, Default)]
pub struct MetadataResolutionParams {
pub dir_quorum: usize,
pub obj_quorum: usize,
pub requested_versions: usize,
pub bucket: String,
pub strict: bool,
pub candidates: Vec<Vec<FileMetaShallowVersion>>,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
pub struct MetaCacheEntry {
// name is the full name of the object including prefixes
pub name: String,
// Metadata. If none is present it is not an object but only a prefix.
// Entries without metadata will only be present in non-recursive scans.
pub metadata: Vec<u8>,
// cached contains the metadata if decoded.
pub cached: Option<FileMeta>,
// Indicates the entry can be reused and only one reference to metadata is expected.
pub reusable: bool,
}
impl MetaCacheEntry {
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut wr = Vec::new();
rmp::encode::write_bool(&mut wr, true)?;
rmp::encode::write_str(&mut wr, &self.name)?;
rmp::encode::write_bin(&mut wr, &self.metadata)?;
Ok(wr)
}
pub fn is_dir(&self) -> bool {
self.metadata.is_empty() && self.name.ends_with('/')
}
pub fn is_in_dir(&self, dir: &str, separator: &str) -> bool {
if dir.is_empty() {
let idx = self.name.find(separator);
return idx.is_none() || idx.unwrap() == self.name.len() - separator.len();
}
let ext = self.name.trim_start_matches(dir);
if ext.len() != self.name.len() {
let idx = ext.find(separator);
return idx.is_none() || idx.unwrap() == ext.len() - separator.len();
}
false
}
pub fn is_object(&self) -> bool {
!self.metadata.is_empty()
}
pub fn is_object_dir(&self) -> bool {
!self.metadata.is_empty() && self.name.ends_with(SLASH_SEPARATOR)
}
pub fn is_latest_delete_marker(&mut self) -> bool {
if let Some(cached) = &self.cached {
if cached.versions.is_empty() {
return true;
}
return cached.versions[0].header.version_type == VersionType::Delete;
}
if !FileMeta::is_xl2_v1_format(&self.metadata) {
return false;
}
match FileMeta::check_xl2_v1(&self.metadata) {
Ok((meta, _, _)) => {
if !meta.is_empty() {
return FileMeta::is_latest_delete_marker(meta);
}
}
Err(_) => return true,
}
match self.xl_meta() {
Ok(res) => {
if res.versions.is_empty() {
return true;
}
res.versions[0].header.version_type == VersionType::Delete
}
Err(_) => true,
}
}
#[tracing::instrument(level = "debug", skip(self))]
pub fn to_fileinfo(&self, bucket: &str) -> Result<FileInfo> {
if self.is_dir() {
return Ok(FileInfo {
volume: bucket.to_owned(),
name: self.name.clone(),
..Default::default()
});
}
if self.cached.is_some() {
let fm = self.cached.as_ref().unwrap();
if fm.versions.is_empty() {
return Ok(FileInfo {
volume: bucket.to_owned(),
name: self.name.clone(),
deleted: true,
is_latest: true,
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
});
}
let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?;
return Ok(fi);
}
let mut fm = FileMeta::new();
fm.unmarshal_msg(&self.metadata)?;
let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?;
return Ok(fi);
}
pub fn file_info_versions(&self, bucket: &str) -> Result<FileInfoVersions> {
if self.is_dir() {
return Ok(FileInfoVersions {
volume: bucket.to_string(),
name: self.name.clone(),
versions: vec![FileInfo {
volume: bucket.to_string(),
name: self.name.clone(),
..Default::default()
}],
..Default::default()
});
}
let mut fm = FileMeta::new();
fm.unmarshal_msg(&self.metadata)?;
fm.into_file_info_versions(bucket, self.name.as_str(), false)
}
pub fn matches(&self, other: Option<&MetaCacheEntry>, strict: bool) -> (Option<MetaCacheEntry>, bool) {
if other.is_none() {
return (None, false);
}
let other = other.unwrap();
let mut prefer = None;
if self.name != other.name {
if self.name < other.name {
return (Some(self.clone()), false);
}
return (Some(other.clone()), false);
}
if other.is_dir() || self.is_dir() {
if self.is_dir() {
return (Some(self.clone()), other.is_dir() == self.is_dir());
}
return (Some(other.clone()), other.is_dir() == self.is_dir());
}
let self_vers = match &self.cached {
Some(file_meta) => file_meta.clone(),
None => match FileMeta::load(&self.metadata) {
Ok(meta) => meta,
Err(_) => {
return (None, false);
}
},
};
let other_vers = match &other.cached {
Some(file_meta) => file_meta.clone(),
None => match FileMeta::load(&other.metadata) {
Ok(meta) => meta,
Err(_) => {
return (None, false);
}
},
};
if self_vers.versions.len() != other_vers.versions.len() {
match self_vers.lastest_mod_time().cmp(&other_vers.lastest_mod_time()) {
Ordering::Greater => {
return (Some(self.clone()), false);
}
Ordering::Less => {
return (Some(other.clone()), false);
}
_ => {}
}
if self_vers.versions.len() > other_vers.versions.len() {
return (Some(self.clone()), false);
}
return (Some(other.clone()), false);
}
for (s_version, o_version) in self_vers.versions.iter().zip(other_vers.versions.iter()) {
if s_version.header != o_version.header {
if s_version.header.has_ec() != o_version.header.has_ec() {
// One version has EC and the other doesn't - may have been written later.
// Compare without considering EC.
let (mut a, mut b) = (s_version.header.clone(), o_version.header.clone());
(a.ec_n, a.ec_m, b.ec_n, b.ec_m) = (0, 0, 0, 0);
if a == b {
continue;
}
}
if !strict && s_version.header.matches_not_strict(&o_version.header) {
if prefer.is_none() {
if s_version.header.sorts_before(&o_version.header) {
prefer = Some(self.clone());
} else {
prefer = Some(other.clone());
}
}
continue;
}
if prefer.is_some() {
return (prefer, false);
}
if s_version.header.sorts_before(&o_version.header) {
return (Some(self.clone()), false);
}
return (Some(other.clone()), false);
}
}
if prefer.is_none() {
prefer = Some(self.clone());
}
(prefer, true)
}
pub fn xl_meta(&mut self) -> Result<FileMeta> {
if self.is_dir() {
return Err(Error::FileNotFound);
}
if let Some(meta) = &self.cached {
Ok(meta.clone())
} else {
if self.metadata.is_empty() {
return Err(Error::FileNotFound);
}
let meta = FileMeta::load(&self.metadata)?;
self.cached = Some(meta.clone());
Ok(meta)
}
}
}
#[derive(Debug, Default)]
pub struct MetaCacheEntries(pub Vec<Option<MetaCacheEntry>>);
impl MetaCacheEntries {
#[allow(clippy::should_implement_trait)]
pub fn as_ref(&self) -> &[Option<MetaCacheEntry>] {
&self.0
}
pub fn resolve(&self, mut params: MetadataResolutionParams) -> Option<MetaCacheEntry> {
if self.0.is_empty() {
warn!("decommission_pool: entries resolve empty");
return None;
}
let mut dir_exists = 0;
let mut selected = None;
params.candidates.clear();
let mut objs_agree = 0;
let mut objs_valid = 0;
for entry in self.0.iter().flatten() {
let mut entry = entry.clone();
warn!("decommission_pool: entries resolve entry {:?}", entry.name);
if entry.name.is_empty() {
continue;
}
if entry.is_dir() {
dir_exists += 1;
selected = Some(entry.clone());
warn!("decommission_pool: entries resolve entry dir {:?}", entry.name);
continue;
}
let xl = match entry.xl_meta() {
Ok(xl) => xl,
Err(e) => {
warn!("decommission_pool: entries resolve entry xl_meta {:?}", e);
continue;
}
};
objs_valid += 1;
params.candidates.push(xl.versions.clone());
if selected.is_none() {
selected = Some(entry.clone());
objs_agree = 1;
warn!("decommission_pool: entries resolve entry selected {:?}", entry.name);
continue;
}
if let (prefer, true) = entry.matches(selected.as_ref(), params.strict) {
selected = prefer;
objs_agree += 1;
warn!("decommission_pool: entries resolve entry prefer {:?}", entry.name);
continue;
}
}
let Some(selected) = selected else {
warn!("decommission_pool: entries resolve entry no selected");
return None;
};
if selected.is_dir() && dir_exists >= params.dir_quorum {
warn!("decommission_pool: entries resolve entry dir selected {:?}", selected.name);
return Some(selected);
}
// If we would never be able to reach read quorum.
if objs_valid < params.obj_quorum {
warn!(
"decommission_pool: entries resolve entry not enough objects {} < {}",
objs_valid, params.obj_quorum
);
return None;
}
if objs_agree == objs_valid {
warn!("decommission_pool: entries resolve entry all agree {} == {}", objs_agree, objs_valid);
return Some(selected);
}
let Some(cached) = selected.cached else {
warn!("decommission_pool: entries resolve entry no cached");
return None;
};
let versions = merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, &params.candidates);
if versions.is_empty() {
warn!("decommission_pool: entries resolve entry no versions");
return None;
}
let metadata = match cached.marshal_msg() {
Ok(meta) => meta,
Err(e) => {
warn!("decommission_pool: entries resolve entry marshal_msg {:?}", e);
return None;
}
};
// Merge if we have disagreement.
// Create a new merged result.
let new_selected = MetaCacheEntry {
name: selected.name.clone(),
cached: Some(FileMeta {
meta_ver: cached.meta_ver,
versions,
..Default::default()
}),
reusable: true,
metadata,
};
warn!("decommission_pool: entries resolve entry selected {:?}", new_selected.name);
Some(new_selected)
}
pub fn first_found(&self) -> (Option<MetaCacheEntry>, usize) {
(self.0.iter().find(|x| x.is_some()).cloned().unwrap_or_default(), self.0.len())
}
}
#[derive(Debug, Default)]
pub struct MetaCacheEntriesSortedResult {
pub entries: Option<MetaCacheEntriesSorted>,
pub err: Option<Error>,
}
// impl MetaCacheEntriesSortedResult {
// pub fn entriy_list(&self) -> Vec<&MetaCacheEntry> {
// if let Some(entries) = &self.entries {
// entries.entries()
// } else {
// Vec::new()
// }
// }
// }
#[derive(Debug, Default)]
pub struct MetaCacheEntriesSorted {
pub o: MetaCacheEntries,
pub list_id: Option<String>,
pub reuse: bool,
pub last_skipped_entry: Option<String>,
}
impl MetaCacheEntriesSorted {
pub fn entries(&self) -> Vec<&MetaCacheEntry> {
let entries: Vec<&MetaCacheEntry> = self.o.0.iter().flatten().collect();
entries
}
pub fn forward_past(&mut self, marker: Option<String>) {
if let Some(val) = marker {
// TODO: reuse
if let Some(idx) = self.o.0.iter().flatten().position(|v| v.name > val) {
self.o.0 = self.o.0.split_off(idx);
}
}
}
pub async fn file_infos(&self, bucket: &str, prefix: &str, delimiter: Option<String>) -> Vec<ObjectInfo> {
let vcfg = get_versioning_config(bucket).await.ok();
let mut objects = Vec::with_capacity(self.o.as_ref().len());
let mut prev_prefix = "";
for entry in self.o.as_ref().iter().flatten() {
if entry.is_object() {
if let Some(delimiter) = &delimiter {
if let Some(idx) = entry.name.trim_start_matches(prefix).find(delimiter) {
let idx = prefix.len() + idx + delimiter.len();
if let Some(curr_prefix) = entry.name.get(0..idx) {
if curr_prefix == prev_prefix {
continue;
}
prev_prefix = curr_prefix;
objects.push(ObjectInfo {
is_dir: true,
bucket: bucket.to_owned(),
name: curr_prefix.to_owned(),
..Default::default()
});
}
continue;
}
}
if let Ok(fi) = entry.to_fileinfo(bucket) {
// TODO:VersionPurgeStatus
let versioned = vcfg.clone().map(|v| v.0.versioned(&entry.name)).unwrap_or_default();
objects.push(ObjectInfo::from_file_info(&fi, bucket, &entry.name, versioned));
}
continue;
}
if entry.is_dir() {
if let Some(delimiter) = &delimiter {
if let Some(idx) = entry.name.trim_start_matches(prefix).find(delimiter) {
let idx = prefix.len() + idx + delimiter.len();
if let Some(curr_prefix) = entry.name.get(0..idx) {
if curr_prefix == prev_prefix {
continue;
}
prev_prefix = curr_prefix;
objects.push(ObjectInfo {
is_dir: true,
bucket: bucket.to_owned(),
name: curr_prefix.to_owned(),
..Default::default()
});
}
}
}
}
}
objects
}
pub async fn file_info_versions(
&self,
bucket: &str,
prefix: &str,
delimiter: Option<String>,
after_v: Option<String>,
) -> Vec<ObjectInfo> {
let vcfg = get_versioning_config(bucket).await.ok();
let mut objects = Vec::with_capacity(self.o.as_ref().len());
let mut prev_prefix = "";
let mut after_v = after_v;
for entry in self.o.as_ref().iter().flatten() {
if entry.is_object() {
if let Some(delimiter) = &delimiter {
if let Some(idx) = entry.name.trim_start_matches(prefix).find(delimiter) {
let idx = prefix.len() + idx + delimiter.len();
if let Some(curr_prefix) = entry.name.get(0..idx) {
if curr_prefix == prev_prefix {
continue;
}
prev_prefix = curr_prefix;
objects.push(ObjectInfo {
is_dir: true,
bucket: bucket.to_owned(),
name: curr_prefix.to_owned(),
..Default::default()
});
}
continue;
}
}
let mut fiv = match entry.file_info_versions(bucket) {
Ok(res) => res,
Err(_err) => {
//
continue;
}
};
let fi_versions = 'c: {
if let Some(after_val) = &after_v {
if let Some(idx) = fiv.find_version_index(after_val) {
after_v = None;
break 'c fiv.versions.split_off(idx + 1);
}
after_v = None;
break 'c fiv.versions;
} else {
break 'c fiv.versions;
}
};
for fi in fi_versions.into_iter() {
// VersionPurgeStatus
let versioned = vcfg.clone().map(|v| v.0.versioned(&entry.name)).unwrap_or_default();
objects.push(ObjectInfo::from_file_info(&fi, bucket, &entry.name, versioned));
}
continue;
}
if entry.is_dir() {
if let Some(delimiter) = &delimiter {
if let Some(idx) = entry.name.trim_start_matches(prefix).find(delimiter) {
let idx = prefix.len() + idx + delimiter.len();
if let Some(curr_prefix) = entry.name.get(0..idx) {
if curr_prefix == prev_prefix {
continue;
}
prev_prefix = curr_prefix;
objects.push(ObjectInfo {
is_dir: true,
bucket: bucket.to_owned(),
name: curr_prefix.to_owned(),
..Default::default()
});
}
}
}
}
}
objects
}
}
crates/disk/src/os.rs
-206
View File
@@ -1,206 +0,0 @@
use super::fs;
use rustfs_error::{to_access_error, Error, Result};
use rustfs_utils::os::same_disk;
use std::{
io,
path::{Component, Path},
};
pub fn check_path_length(path_name: &str) -> Result<()> {
// Apple OS X path length is limited to 1016
if cfg!(target_os = "macos") && path_name.len() > 1016 {
return Err(Error::FileNameTooLong);
}
// Disallow more than 1024 characters on windows, there
// are no known name_max limits on Windows.
if cfg!(target_os = "windows") && path_name.len() > 1024 {
return Err(Error::FileNameTooLong);
}
// On Unix we reject paths if they are just '.', '..' or '/'
let invalid_paths = [".", "..", "/"];
if invalid_paths.contains(&path_name) {
return Err(Error::FileAccessDenied);
}
// Check each path segment length is > 255 on all Unix
// platforms, look for this value as NAME_MAX in
// /usr/include/linux/limits.h
let mut count = 0usize;
for c in path_name.chars() {
match c {
'/' | '\\' if cfg!(target_os = "windows") => count = 0, // Reset
_ => {
count += 1;
if count > 255 {
return Err(Error::FileNameTooLong);
}
}
}
}
// Success.
Ok(())
}
pub fn is_root_disk(disk_path: &str, root_disk: &str) -> Result<bool> {
if cfg!(target_os = "windows") {
return Ok(false);
}
Ok(same_disk(disk_path, root_disk)?)
}
pub async fn make_dir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> Result<()> {
check_path_length(path.as_ref().to_string_lossy().to_string().as_str())?;
if let Err(e) = reliable_mkdir_all(path.as_ref(), base_dir.as_ref()).await {
return Err(to_access_error(e, Error::FileAccessDenied).into());
}
Ok(())
}
pub async fn is_empty_dir(path: impl AsRef<Path>) -> bool {
read_dir(path.as_ref(), 1).await.is_ok_and(|v| v.is_empty())
}
// read_dir count read limit. when count == 0 unlimit.
pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> std::io::Result<Vec<String>> {
let mut entries = tokio::fs::read_dir(path.as_ref()).await?;
let mut volumes = Vec::new();
let mut count = count;
while let Some(entry) = entries.next_entry().await? {
let name = entry.file_name().to_string_lossy().to_string();
if name.is_empty() || name == "." || name == ".." {
continue;
}
let file_type = entry.file_type().await?;
if file_type.is_file() {
volumes.push(name);
} else if file_type.is_dir() {
volumes.push(format!("{}{}", name, super::path::SLASH_SEPARATOR));
}
count -= 1;
if count == 0 {
break;
}
}
Ok(volumes)
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn rename_all(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
base_dir: impl AsRef<Path>,
) -> Result<()> {
reliable_rename(src_file_path, dst_file_path.as_ref(), base_dir)
.await
.map_err(|e| to_access_error(e, Error::FileAccessDenied))?;
Ok(())
}
pub async fn reliable_rename(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
base_dir: impl AsRef<Path>,
) -> io::Result<()> {
if let Some(parent) = dst_file_path.as_ref().parent() {
if !file_exists(parent) {
// info!("reliable_rename reliable_mkdir_all parent: {:?}", parent);
reliable_mkdir_all(parent, base_dir.as_ref()).await?;
}
}
let mut i = 0;
loop {
if let Err(e) = fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
if e.kind() == std::io::ErrorKind::NotFound && i == 0 {
i += 1;
continue;
}
// info!(
// "reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
// src_file_path.as_ref(),
// dst_file_path.as_ref(),
// base_dir.as_ref(),
// e
// );
return Err(e);
}
break;
}
Ok(())
}
pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
let mut i = 0;
let mut base_dir = base_dir.as_ref();
loop {
if let Err(e) = os_mkdir_all(path.as_ref(), base_dir).await {
if e.kind() == std::io::ErrorKind::NotFound && i == 0 {
i += 1;
if let Some(base_parent) = base_dir.parent() {
if let Some(c) = base_parent.components().next() {
if c != Component::RootDir {
base_dir = base_parent
}
}
}
continue;
}
return Err(e);
}
break;
}
Ok(())
}
pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
if !base_dir.as_ref().to_string_lossy().is_empty() && base_dir.as_ref().starts_with(dir_path.as_ref()) {
return Ok(());
}
if let Some(parent) = dir_path.as_ref().parent() {
// 不支持递归，直接create_dir_all了
if let Err(e) = fs::make_dir_all(&parent).await {
if e.kind() == std::io::ErrorKind::AlreadyExists {
return Ok(());
}
return Err(e);
}
// Box::pin(os_mkdir_all(&parent, &base_dir)).await?;
}
if let Err(e) = fs::mkdir(dir_path.as_ref()).await {
if e.kind() == std::io::ErrorKind::AlreadyExists {
return Ok(());
}
return Err(e);
}
Ok(())
}
pub fn file_exists(path: impl AsRef<Path>) -> bool {
std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false)
}
crates/disk/src/path.rs
-308
View File
@@ -1,308 +0,0 @@
use std::path::Path;
use std::path::PathBuf;
pub const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__";
pub const SLASH_SEPARATOR: &str = "/";
pub const GLOBAL_DIR_SUFFIX_WITH_SLASH: &str = "__XLDIR__/";
pub fn has_suffix(s: &str, suffix: &str) -> bool {
if cfg!(target_os = "windows") {
s.to_lowercase().ends_with(&suffix.to_lowercase())
} else {
s.ends_with(suffix)
}
}
pub fn encode_dir_object(object: &str) -> String {
if has_suffix(object, SLASH_SEPARATOR) {
format!("{}{}", object.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX)
} else {
object.to_string()
}
}
pub fn is_dir_object(object: &str) -> bool {
let obj = encode_dir_object(object);
obj.ends_with(GLOBAL_DIR_SUFFIX)
}
#[allow(dead_code)]
pub fn decode_dir_object(object: &str) -> String {
if has_suffix(object, GLOBAL_DIR_SUFFIX) {
format!("{}{}", object.trim_end_matches(GLOBAL_DIR_SUFFIX), SLASH_SEPARATOR)
} else {
object.to_string()
}
}
pub fn retain_slash(s: &str) -> String {
if s.is_empty() {
return s.to_string();
}
if s.ends_with(SLASH_SEPARATOR) {
s.to_string()
} else {
format!("{}{}", s, SLASH_SEPARATOR)
}
}
pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool {
s.len() >= prefix.len() && (s[..prefix.len()] == *prefix || s[..prefix.len()].eq_ignore_ascii_case(prefix))
}
pub fn has_prefix(s: &str, prefix: &str) -> bool {
if cfg!(target_os = "windows") {
return strings_has_prefix_fold(s, prefix);
}
s.starts_with(prefix)
}
pub fn path_join(elem: &[PathBuf]) -> PathBuf {
let mut joined_path = PathBuf::new();
for path in elem {
joined_path.push(path);
}
joined_path
}
pub fn path_join_buf(elements: &[&str]) -> String {
let trailing_slash = !elements.is_empty() && elements.last().unwrap().ends_with(SLASH_SEPARATOR);
let mut dst = String::new();
let mut added = 0;
for e in elements {
if added > 0 || !e.is_empty() {
if added > 0 {
dst.push_str(SLASH_SEPARATOR);
}
dst.push_str(e);
added += e.len();
}
}
let result = dst.to_string();
let cpath = Path::new(&result).components().collect::<PathBuf>();
let clean_path = cpath.to_string_lossy();
if trailing_slash {
return format!("{}{}", clean_path, SLASH_SEPARATOR);
}
clean_path.to_string()
}
pub fn path_to_bucket_object_with_base_path(bash_path: &str, path: &str) -> (String, String) {
let path = path.trim_start_matches(bash_path).trim_start_matches(SLASH_SEPARATOR);
if let Some(m) = path.find(SLASH_SEPARATOR) {
return (path[..m].to_string(), path[m + SLASH_SEPARATOR.len()..].to_string());
}
(path.to_string(), "".to_string())
}
pub fn path_to_bucket_object(s: &str) -> (String, String) {
path_to_bucket_object_with_base_path("", s)
}
pub fn base_dir_from_prefix(prefix: &str) -> String {
let mut base_dir = dir(prefix).to_owned();
if base_dir == "." || base_dir == "./" || base_dir == "/" {
base_dir = "".to_owned();
}
if !prefix.contains('/') {
base_dir = "".to_owned();
}
if !base_dir.is_empty() && !base_dir.ends_with(SLASH_SEPARATOR) {
base_dir.push_str(SLASH_SEPARATOR);
}
base_dir
}
pub struct LazyBuf {
s: String,
buf: Option<Vec<u8>>,
w: usize,
}
impl LazyBuf {
pub fn new(s: String) -> Self {
LazyBuf { s, buf: None, w: 0 }
}
pub fn index(&self, i: usize) -> u8 {
if let Some(ref buf) = self.buf {
buf[i]
} else {
self.s.as_bytes()[i]
}
}
pub fn append(&mut self, c: u8) {
if self.buf.is_none() {
if self.w < self.s.len() && self.s.as_bytes()[self.w] == c {
self.w += 1;
return;
}
let mut new_buf = vec![0; self.s.len()];
new_buf[..self.w].copy_from_slice(&self.s.as_bytes()[..self.w]);
self.buf = Some(new_buf);
}
if let Some(ref mut buf) = self.buf {
buf[self.w] = c;
self.w += 1;
}
}
pub fn string(&self) -> String {
if let Some(ref buf) = self.buf {
String::from_utf8(buf[..self.w].to_vec()).unwrap()
} else {
self.s[..self.w].to_string()
}
}
}
pub fn clean(path: &str) -> String {
if path.is_empty() {
return ".".to_string();
}
let rooted = path.starts_with('/');
let n = path.len();
let mut out = LazyBuf::new(path.to_string());
let mut r = 0;
let mut dotdot = 0;
if rooted {
out.append(b'/');
r = 1;
dotdot = 1;
}
while r < n {
match path.as_bytes()[r] {
b'/' => {
// Empty path element
r += 1;
}
b'.' if r + 1 == n || path.as_bytes()[r + 1] == b'/' => {
// . element
r += 1;
}
b'.' if path.as_bytes()[r + 1] == b'.' && (r + 2 == n || path.as_bytes()[r + 2] == b'/') => {
// .. element: remove to last /
r += 2;
if out.w > dotdot {
// Can backtrack
out.w -= 1;
while out.w > dotdot && out.index(out.w) != b'/' {
out.w -= 1;
}
} else if !rooted {
// Cannot backtrack but not rooted, so append .. element.
if out.w > 0 {
out.append(b'/');
}
out.append(b'.');
out.append(b'.');
dotdot = out.w;
}
}
_ => {
// Real path element.
// Add slash if needed
if (rooted && out.w != 1) || (!rooted && out.w != 0) {
out.append(b'/');
}
// Copy element
while r < n && path.as_bytes()[r] != b'/' {
out.append(path.as_bytes()[r]);
r += 1;
}
}
}
}
// Turn empty string into "."
if out.w == 0 {
return ".".to_string();
}
out.string()
}
pub fn split(path: &str) -> (&str, &str) {
// Find the last occurrence of the '/' character
if let Some(i) = path.rfind('/') {
// Return the directory (up to and including the last '/') and the file name
return (&path[..i + 1], &path[i + 1..]);
}
// If no '/' is found, return an empty string for the directory and the whole path as the file name
(path, "")
}
pub fn dir(path: &str) -> String {
let (a, _) = split(path);
clean(a)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_base_dir_from_prefix() {
let a = "da/";
println!("---- in {}", a);
let a = base_dir_from_prefix(a);
println!("---- out {}", a);
}
#[test]
fn test_clean() {
assert_eq!(clean(""), ".");
assert_eq!(clean("abc"), "abc");
assert_eq!(clean("abc/def"), "abc/def");
assert_eq!(clean("a/b/c"), "a/b/c");
assert_eq!(clean("."), ".");
assert_eq!(clean(".."), "..");
assert_eq!(clean("../.."), "../..");
assert_eq!(clean("../../abc"), "../../abc");
assert_eq!(clean("/abc"), "/abc");
assert_eq!(clean("/"), "/");
assert_eq!(clean("abc/"), "abc");
assert_eq!(clean("abc/def/"), "abc/def");
assert_eq!(clean("a/b/c/"), "a/b/c");
assert_eq!(clean("./"), ".");
assert_eq!(clean("../"), "..");
assert_eq!(clean("../../"), "../..");
assert_eq!(clean("/abc/"), "/abc");
assert_eq!(clean("abc//def//ghi"), "abc/def/ghi");
assert_eq!(clean("//abc"), "/abc");
assert_eq!(clean("///abc"), "/abc");
assert_eq!(clean("//abc//"), "/abc");
assert_eq!(clean("abc//"), "abc");
assert_eq!(clean("abc/./def"), "abc/def");
assert_eq!(clean("/./abc/def"), "/abc/def");
assert_eq!(clean("abc/."), "abc");
assert_eq!(clean("abc/./../def"), "def");
assert_eq!(clean("abc//./../def"), "def");
assert_eq!(clean("abc/../../././../def"), "../../def");
assert_eq!(clean("abc/def/ghi/../jkl"), "abc/def/jkl");
assert_eq!(clean("abc/def/../ghi/../jkl"), "abc/jkl");
assert_eq!(clean("abc/def/.."), "abc");
assert_eq!(clean("abc/def/../.."), ".");
assert_eq!(clean("/abc/def/../.."), "/");
assert_eq!(clean("abc/def/../../.."), "..");
assert_eq!(clean("/abc/def/../../.."), "/");
assert_eq!(clean("abc/def/../../../ghi/jkl/../../../mno"), "../../mno");
}
}
crates/disk/src/remote.rs
-908
View File
@@ -1,908 +0,0 @@
use std::path::PathBuf;
use crate::api::CheckPartsResp;
use crate::api::DeleteOptions;
use crate::api::DiskAPI;
use crate::api::DiskInfo;
use crate::api::DiskInfoOptions;
use crate::api::DiskLocation;
use crate::api::DiskOption;
use crate::api::ReadMultipleReq;
use crate::api::ReadMultipleResp;
use crate::api::ReadOptions;
use crate::api::RenameDataResp;
use crate::api::UpdateMetadataOpts;
use crate::api::VolumeInfo;
use crate::api::WalkDirOptions;
use crate::endpoint::Endpoint;
use futures::StreamExt as _;
use http::HeaderMap;
use http::Method;
use protos::node_service_time_out_client;
use protos::proto_gen::node_service::CheckPartsRequest;
use protos::proto_gen::node_service::DeletePathsRequest;
use protos::proto_gen::node_service::DeleteRequest;
use protos::proto_gen::node_service::DeleteVersionRequest;
use protos::proto_gen::node_service::DeleteVersionsRequest;
use protos::proto_gen::node_service::DeleteVolumeRequest;
use protos::proto_gen::node_service::DiskInfoRequest;
use protos::proto_gen::node_service::ListDirRequest;
use protos::proto_gen::node_service::ListVolumesRequest;
use protos::proto_gen::node_service::MakeVolumeRequest;
use protos::proto_gen::node_service::MakeVolumesRequest;
use protos::proto_gen::node_service::ReadAllRequest;
use protos::proto_gen::node_service::ReadMultipleRequest;
use protos::proto_gen::node_service::ReadVersionRequest;
use protos::proto_gen::node_service::ReadXlRequest;
use protos::proto_gen::node_service::RenameDataRequest;
use protos::proto_gen::node_service::RenameFileRequst;
use protos::proto_gen::node_service::RenamePartRequst;
use protos::proto_gen::node_service::StatVolumeRequest;
use protos::proto_gen::node_service::UpdateMetadataRequest;
use protos::proto_gen::node_service::VerifyFileRequest;
use protos::proto_gen::node_service::WalkDirRequest;
use protos::proto_gen::node_service::WriteAllRequest;
use protos::proto_gen::node_service::WriteMetadataRequest;
use rmp_serde::Serializer;
use rustfs_error::Error;
use rustfs_error::Result;
use rustfs_filemeta::FileInfo;
use rustfs_filemeta::FileInfoVersions;
use rustfs_filemeta::RawFileInfo;
use rustfs_metacache::MetaCacheEntry;
use rustfs_metacache::MetacacheWriter;
use rustfs_rio::HttpReader;
use rustfs_rio::HttpWriter;
use serde::Serialize as _;
use tokio::io::AsyncRead;
use tokio::io::AsyncWrite;
use tokio::sync::Mutex;
use tonic::Request;
use tracing::info;
use uuid::Uuid;
#[derive(Debug)]
pub struct RemoteDisk {
pub id: Mutex<Option<Uuid>>,
pub addr: String,
pub url: url::Url,
pub root: PathBuf,
endpoint: Endpoint,
}
impl RemoteDisk {
pub async fn new(ep: &Endpoint, _opt: &DiskOption) -> Result<Self> {
// let root = fs::canonicalize(ep.url.path()).await?;
let root = PathBuf::from(ep.get_file_path());
let addr = format!("{}://{}:{}", ep.url.scheme(), ep.url.host_str().unwrap(), ep.url.port().unwrap());
Ok(Self {
id: Mutex::new(None),
addr,
url: ep.url.clone(),
root,
endpoint: ep.clone(),
})
}
}
// TODO: all api need to handle errors
#[async_trait::async_trait]
impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
fn to_string(&self) -> String {
self.endpoint.to_string()
}
#[tracing::instrument(skip(self))]
fn is_local(&self) -> bool {
false
}
#[tracing::instrument(skip(self))]
fn host_name(&self) -> String {
self.endpoint.host_port()
}
#[tracing::instrument(skip(self))]
async fn is_online(&self) -> bool {
// TODO: 连接状态
if (node_service_time_out_client(&self.addr).await).is_ok() {
return true;
}
false
}
#[tracing::instrument(skip(self))]
fn endpoint(&self) -> Endpoint {
self.endpoint.clone()
}
#[tracing::instrument(skip(self))]
async fn close(&self) -> Result<()> {
Ok(())
}
#[tracing::instrument(skip(self))]
fn path(&self) -> PathBuf {
self.root.clone()
}
#[tracing::instrument(skip(self))]
fn get_disk_location(&self) -> DiskLocation {
DiskLocation {
pool_idx: {
if self.endpoint.pool_idx < 0 {
None
} else {
Some(self.endpoint.pool_idx as usize)
}
},
set_idx: {
if self.endpoint.set_idx < 0 {
None
} else {
Some(self.endpoint.set_idx as usize)
}
},
disk_idx: {
if self.endpoint.disk_idx < 0 {
None
} else {
Some(self.endpoint.disk_idx as usize)
}
},
}
}
#[tracing::instrument(skip(self))]
async fn get_disk_id(&self) -> Result<Option<Uuid>> {
Ok(*self.id.lock().await)
}
#[tracing::instrument(skip(self))]
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
let mut lock = self.id.lock().await;
*lock = id;
Ok(())
}
#[tracing::instrument(skip(self))]
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
info!("read_all {}/{}", volume, path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadAllRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
});
let response = client.read_all(request).await?.into_inner();
if !response.success {
return Err(Error::FileNotFound);
}
Ok(response.data)
}
#[tracing::instrument(skip(self))]
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
info!("write_all");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(WriteAllRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
data,
});
let response = client.write_all(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
info!("delete {}/{}/{}", self.endpoint.to_string(), volume, path);
let options = serde_json::to_string(&opt)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
options,
});
let response = client.delete(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("verify_file");
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(VerifyFileRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
});
let response = client.verify_file(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let check_parts_resp = serde_json::from_str::<CheckPartsResp>(&response.check_parts_resp)?;
Ok(check_parts_resp)
}
#[tracing::instrument(skip(self))]
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("check_parts");
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(CheckPartsRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
});
let response = client.check_parts(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let check_parts_resp = serde_json::from_str::<CheckPartsResp>(&response.check_parts_resp)?;
Ok(check_parts_resp)
}
#[tracing::instrument(skip(self))]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()> {
info!("rename_part {}/{}", src_volume, src_path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(RenamePartRequst {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
meta,
});
let response = client.rename_part(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
info!("rename_file");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(RenameFileRequst {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
});
let response = client.rename_file(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<Box<dyn AsyncWrite>> {
info!("create_file {}/{}/{}", self.endpoint.to_string(), volume, path);
let url = format!(
"{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
self.endpoint.grid_host(),
urlencoding::encode(&self.endpoint.to_string()),
urlencoding::encode(volume),
urlencoding::encode(path),
false,
file_size
);
let wd = HttpWriter::new(url, Method::PUT, HeaderMap::new()).await?;
Ok(Box::new(wd))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<Box<dyn AsyncWrite>> {
info!("append_file {}/{}", volume, path);
let url = format!(
"{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
self.endpoint.grid_host(),
urlencoding::encode(&self.endpoint.to_string()),
urlencoding::encode(volume),
urlencoding::encode(path),
true,
0
);
let wd = HttpWriter::new(url, Method::PUT, HeaderMap::new()).await?;
Ok(Box::new(wd))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file(&self, volume: &str, path: &str) -> Result<Box<dyn AsyncRead>> {
info!("read_file {}/{}", volume, path);
let url = format!(
"{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
self.endpoint.grid_host(),
urlencoding::encode(&self.endpoint.to_string()),
urlencoding::encode(volume),
urlencoding::encode(path),
0,
0
);
let rd = HttpReader::new(url, Method::GET, HeaderMap::new()).await?;
Ok(Box::new(rd))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Box<dyn AsyncRead>> {
info!("read_file_stream {}/{}/{}", self.endpoint.to_string(), volume, path);
let url = format!(
"{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
self.endpoint.grid_host(),
urlencoding::encode(&self.endpoint.to_string()),
urlencoding::encode(volume),
urlencoding::encode(path),
offset,
length
);
let rd = HttpReader::new(url, Method::GET, HeaderMap::new()).await?;
Ok(Box::new(rd))
}
#[tracing::instrument(skip(self))]
async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
info!("list_dir {}/{}", volume, _dir_path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ListDirRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.list_dir(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(response.volumes)
}
// FIXME: TODO: use writer
#[tracing::instrument(skip(self, wr))]
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
let now = std::time::SystemTime::now();
info!("walk_dir {}/{}/{:?}", self.endpoint.to_string(), opts.bucket, opts.filter_prefix);
let mut wr = wr;
let mut out = MetacacheWriter::new(&mut wr);
let mut buf = Vec::new();
opts.serialize(&mut Serializer::new(&mut buf))?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(WalkDirRequest {
disk: self.endpoint.to_string(),
walk_dir_options: buf,
});
let mut response = client.walk_dir(request).await?.into_inner();
loop {
match response.next().await {
Some(Ok(resp)) => {
if !resp.success {
return Err(Error::msg(resp.error_info.unwrap_or("".to_string())));
}
let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
.map_err(|_| Error::msg(format!("Unexpected response: {:?}", response)))?;
out.write_obj(&entry).await?;
}
None => break,
_ => return Err(Error::msg(format!("Unexpected response: {:?}", response))),
}
}
info!(
"walk_dir {}/{:?} done {:?}",
opts.bucket,
opts.filter_prefix,
now.elapsed().unwrap_or_default()
);
Ok(())
}
#[tracing::instrument(skip(self))]
async fn rename_data(
&self,
src_volume: &str,
src_path: &str,
fi: FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
info!("rename_data {}/{}/{}/{}", self.addr, self.endpoint.to_string(), dst_volume, dst_path);
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(RenameDataRequest {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
file_info,
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
});
let response = client.rename_data(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let rename_data_resp = serde_json::from_str::<RenameDataResp>(&response.rename_data_resp)?;
Ok(rename_data_resp)
}
#[tracing::instrument(skip(self))]
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
info!("make_volumes");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(MakeVolumesRequest {
disk: self.endpoint.to_string(),
volumes: volumes.iter().map(|s| (*s).to_string()).collect(),
});
let response = client.make_volumes(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn make_volume(&self, volume: &str) -> Result<()> {
info!("make_volume");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(MakeVolumeRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.make_volume(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
info!("list_volumes");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ListVolumesRequest {
disk: self.endpoint.to_string(),
});
let response = client.list_volumes(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let infos = response
.volume_infos
.into_iter()
.filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
.collect();
Ok(infos)
}
#[tracing::instrument(skip(self))]
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
info!("stat_volume");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(StatVolumeRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.stat_volume(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let volume_info = serde_json::from_str::<VolumeInfo>(&response.volume_info)?;
Ok(volume_info)
}
#[tracing::instrument(skip(self))]
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
info!("delete_paths");
let paths = paths.to_owned();
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeletePathsRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
paths,
});
let response = client.delete_paths(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
info!("update_metadata");
let file_info = serde_json::to_string(&fi)?;
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(UpdateMetadataRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
opts,
});
let response = client.update_metadata(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
info!("write_metadata {}/{}", volume, path);
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(WriteMetadataRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
});
let response = client.write_metadata(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn read_version(
&self,
_org_volume: &str,
volume: &str,
path: &str,
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo> {
info!("read_version");
let opts = serde_json::to_string(opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadVersionRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
version_id: version_id.to_string(),
opts,
});
let response = client.read_version(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let file_info = serde_json::from_str::<FileInfo>(&response.file_info)?;
Ok(file_info)
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
info!("read_xl {}/{}/{}", self.endpoint.to_string(), volume, path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadXlRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
read_data,
});
let response = client.read_xl(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
Ok(raw_file_info)
}
#[tracing::instrument(skip(self))]
async fn delete_version(
&self,
volume: &str,
path: &str,
fi: FileInfo,
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<()> {
info!("delete_version");
let file_info = serde_json::to_string(&fi)?;
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteVersionRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
force_del_marker,
opts,
});
let response = client.delete_version(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
// let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
Ok(())
}
#[tracing::instrument(skip(self))]
async fn delete_versions(
&self,
volume: &str,
versions: Vec<FileInfoVersions>,
opts: DeleteOptions,
) -> Result<Vec<Option<Error>>> {
info!("delete_versions");
let opts = serde_json::to_string(&opts)?;
let mut versions_str = Vec::with_capacity(versions.len());
for file_info_versions in versions.iter() {
versions_str.push(serde_json::to_string(file_info_versions)?);
}
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteVersionsRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
versions: versions_str,
opts,
});
let response = client.delete_versions(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let errors = response
.errors
.iter()
.map(|error| {
if error.is_empty() {
None
} else {
use std::str::FromStr;
Some(Error::from_str(error).unwrap_or(Error::msg(error)))
}
})
.collect();
Ok(errors)
}
#[tracing::instrument(skip(self))]
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
info!("read_multiple {}/{}/{}", self.endpoint.to_string(), req.bucket, req.prefix);
let read_multiple_req = serde_json::to_string(&req)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadMultipleRequest {
disk: self.endpoint.to_string(),
read_multiple_req,
});
let response = client.read_multiple(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let read_multiple_resps = response
.read_multiple_resps
.into_iter()
.filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
.collect();
Ok(read_multiple_resps)
}
#[tracing::instrument(skip(self))]
async fn delete_volume(&self, volume: &str) -> Result<()> {
info!("delete_volume {}/{}", self.endpoint.to_string(), volume);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteVolumeRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.delete_volume(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo> {
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DiskInfoRequest {
disk: self.endpoint.to_string(),
opts,
});
let response = client.disk_info(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let disk_info = serde_json::from_str::<DiskInfo>(&response.disk_info)?;
Ok(disk_info)
}
// #[tracing::instrument(skip(self, cache, scan_mode, _we_sleep))]
// async fn ns_scanner(
// &self,
// cache: &DataUsageCache,
// updates: Sender<DataUsageEntry>,
// scan_mode: HealScanMode,
// _we_sleep: ShouldSleepFn,
// ) -> Result<DataUsageCache> {
// info!("ns_scanner");
// let cache = serde_json::to_string(cache)?;
// let mut client = node_service_time_out_client(&self.addr)
// .await
// .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
// let (tx, rx) = mpsc::channel(10);
// let in_stream = ReceiverStream::new(rx);
// let mut response = client.ns_scanner(in_stream).await?.into_inner();
// let request = NsScannerRequest {
// disk: self.endpoint.to_string(),
// cache,
// scan_mode: scan_mode as u64,
// };
// tx.send(request)
// .await
// .map_err(|err| Error::msg(format!("can not send request, err: {}", err)))?;
// loop {
// match response.next().await {
// Some(Ok(resp)) => {
// if !resp.update.is_empty() {
// let data_usage_cache = serde_json::from_str::<DataUsageEntry>(&resp.update)?;
// let _ = updates.send(data_usage_cache).await;
// } else if !resp.data_usage_cache.is_empty() {
// let data_usage_cache = serde_json::from_str::<DataUsageCache>(&resp.data_usage_cache)?;
// return Ok(data_usage_cache);
// } else {
// return Err(Error::msg("scan was interrupted"));
// }
// }
// _ => return Err(Error::msg("scan was interrupted")),
// }
// }
// }
// #[tracing::instrument(skip(self))]
// async fn healing(&self) -> Option<HealingTracker> {
// None
// }
}
crates/disk/src/remote_bak.rs
-862
View File
@@ -1,862 +0,0 @@
use std::path::PathBuf;
use super::{
endpoint::Endpoint, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo,
WalkDirOptions,
};
use crate::heal::{
data_scanner::ShouldSleepFn,
data_usage_cache::{DataUsageCache, DataUsageEntry},
heal_commands::{HealScanMode, HealingTracker},
};
use crate::io::{FileReader, FileWriter, HttpFileReader, HttpFileWriter};
use crate::{disk::metacache::MetaCacheEntry, metacache::writer::MetacacheWriter};
use futures::lock::Mutex;
use protos::proto_gen::node_service::RenamePartRequst;
use protos::{
node_service_time_out_client,
proto_gen::node_service::{
CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest,
DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, NsScannerRequest,
ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst,
StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest,
},
};
use rmp_serde::Serializer;
use rustfs_error::{Error, Result};
use rustfs_filemeta::{FileInfo, RawFileInfo};
use serde::Serialize;
use tokio::{
io::AsyncWrite,
sync::mpsc::{self, Sender},
};
use tokio_stream::{wrappers::ReceiverStream, StreamExt};
use tonic::Request;
use tracing::info;
use uuid::Uuid;
#[derive(Debug)]
pub struct RemoteDisk {
pub id: Mutex<Option<Uuid>>,
pub addr: String,
pub url: url::Url,
pub root: PathBuf,
endpoint: Endpoint,
}
impl RemoteDisk {
pub async fn new(ep: &Endpoint, _opt: &DiskOption) -> Result<Self> {
// let root = fs::canonicalize(ep.url.path()).await?;
let root = PathBuf::from(ep.get_file_path());
let addr = format!("{}://{}:{}", ep.url.scheme(), ep.url.host_str().unwrap(), ep.url.port().unwrap());
Ok(Self {
id: Mutex::new(None),
addr,
url: ep.url.clone(),
root,
endpoint: ep.clone(),
})
}
}
// TODO: all api need to handle errors
#[async_trait::async_trait]
impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
fn to_string(&self) -> String {
self.endpoint.to_string()
}
#[tracing::instrument(skip(self))]
fn is_local(&self) -> bool {
false
}
#[tracing::instrument(skip(self))]
fn host_name(&self) -> String {
self.endpoint.host_port()
}
#[tracing::instrument(skip(self))]
async fn is_online(&self) -> bool {
// TODO: 连接状态
if (node_service_time_out_client(&self.addr).await).is_ok() {
return true;
}
false
}
#[tracing::instrument(skip(self))]
fn endpoint(&self) -> Endpoint {
self.endpoint.clone()
}
#[tracing::instrument(skip(self))]
async fn close(&self) -> Result<()> {
Ok(())
}
#[tracing::instrument(skip(self))]
fn path(&self) -> PathBuf {
self.root.clone()
}
#[tracing::instrument(skip(self))]
fn get_disk_location(&self) -> DiskLocation {
DiskLocation {
pool_idx: {
if self.endpoint.pool_idx < 0 {
None
} else {
Some(self.endpoint.pool_idx as usize)
}
},
set_idx: {
if self.endpoint.set_idx < 0 {
None
} else {
Some(self.endpoint.set_idx as usize)
}
},
disk_idx: {
if self.endpoint.disk_idx < 0 {
None
} else {
Some(self.endpoint.disk_idx as usize)
}
},
}
}
#[tracing::instrument(skip(self))]
async fn get_disk_id(&self) -> Result<Option<Uuid>> {
Ok(*self.id.lock().await)
}
#[tracing::instrument(skip(self))]
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
let mut lock = self.id.lock().await;
*lock = id;
Ok(())
}
#[tracing::instrument(skip(self))]
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
info!("read_all {}/{}", volume, path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadAllRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
});
let response = client.read_all(request).await?.into_inner();
if !response.success {
return Err(Error::FileNotFound);
}
Ok(response.data)
}
#[tracing::instrument(skip(self))]
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
info!("write_all");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(WriteAllRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
data,
});
let response = client.write_all(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
info!("delete {}/{}/{}", self.endpoint.to_string(), volume, path);
let options = serde_json::to_string(&opt)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
options,
});
let response = client.delete(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("verify_file");
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(VerifyFileRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
});
let response = client.verify_file(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let check_parts_resp = serde_json::from_str::<CheckPartsResp>(&response.check_parts_resp)?;
Ok(check_parts_resp)
}
#[tracing::instrument(skip(self))]
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("check_parts");
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(CheckPartsRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
});
let response = client.check_parts(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let check_parts_resp = serde_json::from_str::<CheckPartsResp>(&response.check_parts_resp)?;
Ok(check_parts_resp)
}
#[tracing::instrument(skip(self))]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()> {
info!("rename_part {}/{}", src_volume, src_path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(RenamePartRequst {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
meta,
});
let response = client.rename_part(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
info!("rename_file");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(RenameFileRequst {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
});
let response = client.rename_file(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<FileWriter> {
info!("create_file {}/{}/{}", self.endpoint.to_string(), volume, path);
Ok(Box::new(HttpFileWriter::new(
self.endpoint.grid_host().as_str(),
self.endpoint.to_string().as_str(),
volume,
path,
file_size,
false,
)?))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
info!("append_file {}/{}", volume, path);
Ok(Box::new(HttpFileWriter::new(
self.endpoint.grid_host().as_str(),
self.endpoint.to_string().as_str(),
volume,
path,
0,
true,
)?))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
info!("read_file {}/{}", volume, path);
Ok(Box::new(
HttpFileReader::new(self.endpoint.grid_host().as_str(), self.endpoint.to_string().as_str(), volume, path, 0, 0)
.await?,
))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> {
info!("read_file_stream {}/{}/{}", self.endpoint.to_string(), volume, path);
Ok(Box::new(
HttpFileReader::new(
self.endpoint.grid_host().as_str(),
self.endpoint.to_string().as_str(),
volume,
path,
offset,
length,
)
.await?,
))
}
#[tracing::instrument(skip(self))]
async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
info!("list_dir {}/{}", volume, _dir_path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ListDirRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.list_dir(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(response.volumes)
}
// FIXME: TODO: use writer
#[tracing::instrument(skip(self, wr))]
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
let now = std::time::SystemTime::now();
info!("walk_dir {}/{}/{:?}", self.endpoint.to_string(), opts.bucket, opts.filter_prefix);
let mut wr = wr;
let mut out = MetacacheWriter::new(&mut wr);
let mut buf = Vec::new();
opts.serialize(&mut Serializer::new(&mut buf))?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(WalkDirRequest {
disk: self.endpoint.to_string(),
walk_dir_options: buf,
});
let mut response = client.walk_dir(request).await?.into_inner();
loop {
match response.next().await {
Some(Ok(resp)) => {
if !resp.success {
return Err(Error::msg(resp.error_info.unwrap_or("".to_string())));
}
let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
.map_err(|_| Error::msg(format!("Unexpected response: {:?}", response)))?;
out.write_obj(&entry).await?;
}
None => break,
_ => return Err(Error::msg(format!("Unexpected response: {:?}", response))),
}
}
info!(
"walk_dir {}/{:?} done {:?}",
opts.bucket,
opts.filter_prefix,
now.elapsed().unwrap_or_default()
);
Ok(())
}
#[tracing::instrument(skip(self))]
async fn rename_data(
&self,
src_volume: &str,
src_path: &str,
fi: FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
info!("rename_data {}/{}/{}/{}", self.addr, self.endpoint.to_string(), dst_volume, dst_path);
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(RenameDataRequest {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
file_info,
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
});
let response = client.rename_data(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let rename_data_resp = serde_json::from_str::<RenameDataResp>(&response.rename_data_resp)?;
Ok(rename_data_resp)
}
#[tracing::instrument(skip(self))]
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
info!("make_volumes");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(MakeVolumesRequest {
disk: self.endpoint.to_string(),
volumes: volumes.iter().map(|s| (*s).to_string()).collect(),
});
let response = client.make_volumes(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn make_volume(&self, volume: &str) -> Result<()> {
info!("make_volume");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(MakeVolumeRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.make_volume(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
info!("list_volumes");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ListVolumesRequest {
disk: self.endpoint.to_string(),
});
let response = client.list_volumes(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let infos = response
.volume_infos
.into_iter()
.filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
.collect();
Ok(infos)
}
#[tracing::instrument(skip(self))]
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
info!("stat_volume");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(StatVolumeRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.stat_volume(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let volume_info = serde_json::from_str::<VolumeInfo>(&response.volume_info)?;
Ok(volume_info)
}
#[tracing::instrument(skip(self))]
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
info!("delete_paths");
let paths = paths.to_owned();
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeletePathsRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
paths,
});
let response = client.delete_paths(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
info!("update_metadata");
let file_info = serde_json::to_string(&fi)?;
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(UpdateMetadataRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
opts,
});
let response = client.update_metadata(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
info!("write_metadata {}/{}", volume, path);
let file_info = serde_json::to_string(&fi)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(WriteMetadataRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
});
let response = client.write_metadata(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn read_version(
&self,
_org_volume: &str,
volume: &str,
path: &str,
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo> {
info!("read_version");
let opts = serde_json::to_string(opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadVersionRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
version_id: version_id.to_string(),
opts,
});
let response = client.read_version(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let file_info = serde_json::from_str::<FileInfo>(&response.file_info)?;
Ok(file_info)
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
info!("read_xl {}/{}/{}", self.endpoint.to_string(), volume, path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadXlRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
read_data,
});
let response = client.read_xl(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
Ok(raw_file_info)
}
#[tracing::instrument(skip(self))]
async fn delete_version(
&self,
volume: &str,
path: &str,
fi: FileInfo,
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<()> {
info!("delete_version");
let file_info = serde_json::to_string(&fi)?;
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteVersionRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
force_del_marker,
opts,
});
let response = client.delete_version(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
// let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
Ok(())
}
#[tracing::instrument(skip(self))]
async fn delete_versions(
&self,
volume: &str,
versions: Vec<FileInfoVersions>,
opts: DeleteOptions,
) -> Result<Vec<Option<Error>>> {
info!("delete_versions");
let opts = serde_json::to_string(&opts)?;
let mut versions_str = Vec::with_capacity(versions.len());
for file_info_versions in versions.iter() {
versions_str.push(serde_json::to_string(file_info_versions)?);
}
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteVersionsRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
versions: versions_str,
opts,
});
let response = client.delete_versions(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let errors = response
.errors
.iter()
.map(|error| {
if error.is_empty() {
None
} else {
use std::str::FromStr;
Some(Error::from_str(error).unwrap_or(Error::msg(error)))
}
})
.collect();
Ok(errors)
}
#[tracing::instrument(skip(self))]
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
info!("read_multiple {}/{}/{}", self.endpoint.to_string(), req.bucket, req.prefix);
let read_multiple_req = serde_json::to_string(&req)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(ReadMultipleRequest {
disk: self.endpoint.to_string(),
read_multiple_req,
});
let response = client.read_multiple(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let read_multiple_resps = response
.read_multiple_resps
.into_iter()
.filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
.collect();
Ok(read_multiple_resps)
}
#[tracing::instrument(skip(self))]
async fn delete_volume(&self, volume: &str) -> Result<()> {
info!("delete_volume {}/{}", self.endpoint.to_string(), volume);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DeleteVolumeRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
});
let response = client.delete_volume(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo> {
let opts = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let request = Request::new(DiskInfoRequest {
disk: self.endpoint.to_string(),
opts,
});
let response = client.disk_info(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let disk_info = serde_json::from_str::<DiskInfo>(&response.disk_info)?;
Ok(disk_info)
}
#[tracing::instrument(skip(self, cache, scan_mode, _we_sleep))]
async fn ns_scanner(
&self,
cache: &DataUsageCache,
updates: Sender<DataUsageEntry>,
scan_mode: HealScanMode,
_we_sleep: ShouldSleepFn,
) -> Result<DataUsageCache> {
info!("ns_scanner");
let cache = serde_json::to_string(cache)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?;
let (tx, rx) = mpsc::channel(10);
let in_stream = ReceiverStream::new(rx);
let mut response = client.ns_scanner(in_stream).await?.into_inner();
let request = NsScannerRequest {
disk: self.endpoint.to_string(),
cache,
scan_mode: scan_mode as u64,
};
tx.send(request)
.await
.map_err(|err| Error::msg(format!("can not send request, err: {}", err)))?;
loop {
match response.next().await {
Some(Ok(resp)) => {
if !resp.update.is_empty() {
let data_usage_cache = serde_json::from_str::<DataUsageEntry>(&resp.update)?;
let _ = updates.send(data_usage_cache).await;
} else if !resp.data_usage_cache.is_empty() {
let data_usage_cache = serde_json::from_str::<DataUsageCache>(&resp.data_usage_cache)?;
return Ok(data_usage_cache);
} else {
return Err(Error::msg("scan was interrupted"));
}
}
_ => return Err(Error::msg("scan was interrupted")),
}
}
}
#[tracing::instrument(skip(self))]
async fn healing(&self) -> Option<HealingTracker> {
None
}
}
crates/disk/src/utils.rs
-35
View File
@@ -1,35 +0,0 @@
use std::{fs::Metadata, path::Path};
use rustfs_error::{to_file_error, Error, Result};
pub async fn read_file_exists(path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<Metadata>)> {
let p = path.as_ref();
let (data, meta) = match read_file_all(&p).await {
Ok((data, meta)) => (data, Some(meta)),
Err(e) => {
if e == Error::FileNotFound {
(Vec::new(), None)
} else {
return Err(e);
}
}
};
Ok((data, meta))
}
pub async fn read_file_all(path: impl AsRef<Path>) -> Result<(Vec<u8>, Metadata)> {
let p = path.as_ref();
let meta = read_file_metadata(&path).await?;
let data = read_all(&p).await?;
Ok((data, meta))
}
pub async fn read_file_metadata(p: impl AsRef<Path>) -> Result<Metadata> {
Ok(tokio::fs::metadata(&p).await.map_err(to_file_error)?)
}
pub async fn read_all(p: impl AsRef<Path>) -> Result<Vec<u8>> {
tokio::fs::read(&p).await.map_err(|e| to_file_error(e).into())
}
crates/error/Cargo.toml
-22
View File
@@ -1,22 +0,0 @@
[package]
name = "rustfs-error"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
[dependencies]
protos.workspace = true
rmp.workspace = true
rmp-serde.workspace = true
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
time.workspace = true
tonic.workspace = true
uuid.workspace = true
[lints]
workspace = true
crates/error/src/bitrot.rs
-27
View File
@@ -1,27 +0,0 @@
use crate::Error;
pub const CHECK_PART_UNKNOWN: usize = 0;
pub const CHECK_PART_SUCCESS: usize = 1;
pub const CHECK_PART_DISK_NOT_FOUND: usize = 2;
pub const CHECK_PART_VOLUME_NOT_FOUND: usize = 3;
pub const CHECK_PART_FILE_NOT_FOUND: usize = 4;
pub const CHECK_PART_FILE_CORRUPT: usize = 5;
pub fn conv_part_err_to_int(err: &Option<Error>) -> usize {
if let Some(err) = err {
match err {
Error::FileNotFound | Error::FileVersionNotFound => CHECK_PART_FILE_NOT_FOUND,
Error::FileCorrupt => CHECK_PART_FILE_CORRUPT,
Error::VolumeNotFound => CHECK_PART_VOLUME_NOT_FOUND,
Error::DiskNotFound => CHECK_PART_DISK_NOT_FOUND,
Error::Nil => CHECK_PART_SUCCESS,
_ => CHECK_PART_UNKNOWN,
}
} else {
CHECK_PART_SUCCESS
}
}
pub fn has_part_err(part_errs: &[usize]) -> bool {
part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS)
}
crates/error/src/convert.rs
-92
View File
@@ -1,92 +0,0 @@
use crate::Error;
pub fn to_file_error(io_err: std::io::Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::NotFound => Error::FileNotFound.into(),
std::io::ErrorKind::PermissionDenied => Error::FileAccessDenied.into(),
std::io::ErrorKind::IsADirectory => Error::IsNotRegular.into(),
std::io::ErrorKind::NotADirectory => Error::FileAccessDenied.into(),
std::io::ErrorKind::DirectoryNotEmpty => Error::FileAccessDenied.into(),
std::io::ErrorKind::UnexpectedEof => Error::FaultyDisk.into(),
std::io::ErrorKind::TooManyLinks => Error::TooManyOpenFiles.into(),
std::io::ErrorKind::InvalidInput => Error::FileNotFound.into(),
std::io::ErrorKind::InvalidData => Error::FileCorrupt.into(),
std::io::ErrorKind::StorageFull => Error::DiskFull.into(),
_ => io_err,
}
}
pub fn to_volume_error(io_err: std::io::Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::NotFound => Error::VolumeNotFound.into(),
std::io::ErrorKind::PermissionDenied => Error::DiskAccessDenied.into(),
std::io::ErrorKind::DirectoryNotEmpty => Error::VolumeNotEmpty.into(),
std::io::ErrorKind::NotADirectory => Error::IsNotRegular.into(),
std::io::ErrorKind::Other => {
let err = Error::from(io_err.to_string());
match err {
Error::FileNotFound => Error::VolumeNotFound.into(),
Error::FileAccessDenied => Error::DiskAccessDenied.into(),
_ => to_file_error(io_err),
}
}
_ => to_file_error(io_err),
}
}
pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::NotFound => Error::DiskNotFound.into(),
std::io::ErrorKind::PermissionDenied => Error::DiskAccessDenied.into(),
std::io::ErrorKind::Other => {
let err = Error::from(io_err.to_string());
match err {
Error::FileNotFound => Error::DiskNotFound.into(),
Error::VolumeNotFound => Error::DiskNotFound.into(),
Error::FileAccessDenied => Error::DiskAccessDenied.into(),
Error::VolumeAccessDenied => Error::DiskAccessDenied.into(),
_ => to_volume_error(io_err),
}
}
_ => to_volume_error(io_err),
}
}
// only errors from FileSystem operations
pub fn to_access_error(io_err: std::io::Error, per_err: Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::PermissionDenied => per_err.into(),
std::io::ErrorKind::NotADirectory => per_err.into(),
std::io::ErrorKind::NotFound => Error::VolumeNotFound.into(),
std::io::ErrorKind::UnexpectedEof => Error::FaultyDisk.into(),
std::io::ErrorKind::Other => {
let err = Error::from(io_err.to_string());
match err {
Error::DiskAccessDenied => per_err.into(),
Error::FileAccessDenied => per_err.into(),
Error::FileNotFound => Error::VolumeNotFound.into(),
_ => to_volume_error(io_err),
}
}
_ => to_volume_error(io_err),
}
}
pub fn to_unformatted_disk_error(io_err: std::io::Error) -> std::io::Error {
match io_err.kind() {
std::io::ErrorKind::NotFound => Error::UnformattedDisk.into(),
std::io::ErrorKind::PermissionDenied => Error::DiskAccessDenied.into(),
std::io::ErrorKind::Other => {
let err = Error::from(io_err.to_string());
match err {
Error::FileNotFound => Error::UnformattedDisk.into(),
Error::DiskNotFound => Error::UnformattedDisk.into(),
Error::VolumeNotFound => Error::UnformattedDisk.into(),
Error::FileAccessDenied => Error::DiskAccessDenied.into(),
Error::DiskAccessDenied => Error::DiskAccessDenied.into(),
_ => Error::CorruptedBackend.into(),
}
}
_ => Error::CorruptedBackend.into(),
}
}
crates/error/src/error.rs
-586
View File
@@ -1,586 +0,0 @@
use std::hash::Hash;
use std::str::FromStr;
const ERROR_PREFIX: &str = "[RUSTFS error] ";
pub type Result<T> = core::result::Result<T, Error>;
#[derive(thiserror::Error, Default, Debug)]
pub enum Error {
#[default]
#[error("[RUSTFS error] Nil")]
Nil,
#[error("I/O error: {0}")]
IoError(std::io::Error),
#[error("[RUSTFS error] Erasure Read quorum not met")]
ErasureReadQuorum,
#[error("[RUSTFS error] Erasure Write quorum not met")]
ErasureWriteQuorum,
#[error("[RUSTFS error] Disk not found")]
DiskNotFound,
#[error("[RUSTFS error] Faulty disk")]
FaultyDisk,
#[error("[RUSTFS error] Faulty remote disk")]
FaultyRemoteDisk,
#[error("[RUSTFS error] Unsupported disk")]
UnsupportedDisk,
#[error("[RUSTFS error] Unformatted disk")]
UnformattedDisk,
#[error("[RUSTFS error] Corrupted backend")]
CorruptedBackend,
#[error("[RUSTFS error] Disk access denied")]
DiskAccessDenied,
#[error("[RUSTFS error] Disk ongoing request")]
DiskOngoingReq,
#[error("[RUSTFS error] Disk full")]
DiskFull,
#[error("[RUSTFS error] Volume not found")]
VolumeNotFound,
#[error("[RUSTFS error] Volume not empty")]
VolumeNotEmpty,
#[error("[RUSTFS error] Volume access denied")]
VolumeAccessDenied,
#[error("[RUSTFS error] Volume exists")]
VolumeExists,
#[error("[RUSTFS error] Disk not a directory")]
DiskNotDir,
#[error("[RUSTFS error] File not found")]
FileNotFound,
#[error("[RUSTFS error] File corrupt")]
FileCorrupt,
#[error("[RUSTFS error] File access denied")]
FileAccessDenied,
#[error("[RUSTFS error] Too many open files")]
TooManyOpenFiles,
#[error("[RUSTFS error] Is not a regular file")]
IsNotRegular,
#[error("[RUSTFS error] File version not found")]
FileVersionNotFound,
#[error("[RUSTFS error] Less data than expected")]
LessData,
#[error("[RUSTFS error] Short write")]
ShortWrite,
#[error("[RUSTFS error] Done for now")]
DoneForNow,
#[error("[RUSTFS error] Method not allowed")]
MethodNotAllowed,
#[error("[RUSTFS error] Inconsistent disk")]
InconsistentDisk,
#[error("[RUSTFS error] File name too long")]
FileNameTooLong,
#[error("[RUSTFS error] Scan ignore file contribution")]
ScanIgnoreFileContrib,
#[error("[RUSTFS error] Scan skip file")]
ScanSkipFile,
#[error("[RUSTFS error] Scan heal stop signaled")]
ScanHealStopSignal,
#[error("[RUSTFS error] Scan heal idle timeout")]
ScanHealIdleTimeout,
#[error("[RUSTFS error] Scan retry healing")]
ScanRetryHealing,
#[error("[RUSTFS error] {0}")]
Other(String),
}
// Generic From implementation removed to avoid conflicts with std::convert::From<T> for T
impl FromStr for Error {
type Err = Error;
fn from_str(s: &str) -> core::result::Result<Self, Self::Err> {
// Only strip prefix for non-IoError
let s = if s.starts_with("I/O error: ") {
s
} else {
s.strip_prefix(ERROR_PREFIX).unwrap_or(s)
};
match s {
"Nil" => Ok(Error::Nil),
"ErasureReadQuorum" => Ok(Error::ErasureReadQuorum),
"ErasureWriteQuorum" => Ok(Error::ErasureWriteQuorum),
"DiskNotFound" | "Disk not found" => Ok(Error::DiskNotFound),
"FaultyDisk" | "Faulty disk" => Ok(Error::FaultyDisk),
"FaultyRemoteDisk" | "Faulty remote disk" => Ok(Error::FaultyRemoteDisk),
"UnformattedDisk" | "Unformatted disk" => Ok(Error::UnformattedDisk),
"DiskAccessDenied" | "Disk access denied" => Ok(Error::DiskAccessDenied),
"DiskOngoingReq" | "Disk ongoing request" => Ok(Error::DiskOngoingReq),
"FileNotFound" | "File not found" => Ok(Error::FileNotFound),
"FileCorrupt" | "File corrupt" => Ok(Error::FileCorrupt),
"FileVersionNotFound" | "File version not found" => Ok(Error::FileVersionNotFound),
"LessData" | "Less data than expected" => Ok(Error::LessData),
"ShortWrite" | "Short write" => Ok(Error::ShortWrite),
"VolumeNotFound" | "Volume not found" => Ok(Error::VolumeNotFound),
"VolumeNotEmpty" | "Volume not empty" => Ok(Error::VolumeNotEmpty),
"VolumeExists" | "Volume exists" => Ok(Error::VolumeExists),
"VolumeAccessDenied" | "Volume access denied" => Ok(Error::VolumeAccessDenied),
"DiskNotDir" | "Disk not a directory" => Ok(Error::DiskNotDir),
"FileAccessDenied" | "File access denied" => Ok(Error::FileAccessDenied),
"TooManyOpenFiles" | "Too many open files" => Ok(Error::TooManyOpenFiles),
"IsNotRegular" | "Is not a regular file" => Ok(Error::IsNotRegular),
"CorruptedBackend" | "Corrupted backend" => Ok(Error::CorruptedBackend),
"UnsupportedDisk" | "Unsupported disk" => Ok(Error::UnsupportedDisk),
"InconsistentDisk" | "Inconsistent disk" => Ok(Error::InconsistentDisk),
"DiskFull" | "Disk full" => Ok(Error::DiskFull),
"FileNameTooLong" | "File name too long" => Ok(Error::FileNameTooLong),
"ScanIgnoreFileContrib" | "Scan ignore file contribution" => Ok(Error::ScanIgnoreFileContrib),
"ScanSkipFile" | "Scan skip file" => Ok(Error::ScanSkipFile),
"ScanHealStopSignal" | "Scan heal stop signaled" => Ok(Error::ScanHealStopSignal),
"ScanHealIdleTimeout" | "Scan heal idle timeout" => Ok(Error::ScanHealIdleTimeout),
"ScanRetryHealing" | "Scan retry healing" => Ok(Error::ScanRetryHealing),
s if s.starts_with("I/O error: ") => {
Ok(Error::IoError(std::io::Error::other(s.strip_prefix("I/O error: ").unwrap_or(""))))
}
"DoneForNow" | "Done for now" => Ok(Error::DoneForNow),
"MethodNotAllowed" | "Method not allowed" => Ok(Error::MethodNotAllowed),
str => Err(Error::IoError(std::io::Error::other(str.to_string()))),
}
}
}
impl From<Error> for std::io::Error {
fn from(err: Error) -> Self {
match err {
Error::IoError(e) => e,
e => std::io::Error::other(e),
}
}
}
impl From<std::io::Error> for Error {
fn from(e: std::io::Error) -> Self {
match e.kind() {
// convert Error from string to Error
std::io::ErrorKind::Other => Error::from(e.to_string()),
_ => Error::IoError(e),
}
}
}
impl From<String> for Error {
fn from(s: String) -> Self {
Error::from_str(&s).unwrap_or(Error::IoError(std::io::Error::other(s)))
}
}
impl From<&str> for Error {
fn from(s: &str) -> Self {
Error::from_str(s).unwrap_or(Error::IoError(std::io::Error::other(s)))
}
}
// Common error type conversions for ? operator
impl From<std::num::ParseIntError> for Error {
fn from(e: std::num::ParseIntError) -> Self {
Error::Other(format!("Parse int error: {}", e))
}
}
impl From<std::num::ParseFloatError> for Error {
fn from(e: std::num::ParseFloatError) -> Self {
Error::Other(format!("Parse float error: {}", e))
}
}
impl From<std::str::Utf8Error> for Error {
fn from(e: std::str::Utf8Error) -> Self {
Error::Other(format!("UTF-8 error: {}", e))
}
}
impl From<std::string::FromUtf8Error> for Error {
fn from(e: std::string::FromUtf8Error) -> Self {
Error::Other(format!("UTF-8 conversion error: {}", e))
}
}
impl From<std::fmt::Error> for Error {
fn from(e: std::fmt::Error) -> Self {
Error::Other(format!("Format error: {}", e))
}
}
impl From<Box<dyn std::error::Error + Send + Sync>> for Error {
fn from(e: Box<dyn std::error::Error + Send + Sync>) -> Self {
Error::Other(e.to_string())
}
}
impl From<time::error::ComponentRange> for Error {
fn from(e: time::error::ComponentRange) -> Self {
Error::Other(format!("Time component range error: {}", e))
}
}
impl From<rmp::decode::NumValueReadError<std::io::Error>> for Error {
fn from(e: rmp::decode::NumValueReadError<std::io::Error>) -> Self {
Error::Other(format!("NumValueReadError: {}", e))
}
}
impl From<rmp::encode::ValueWriteError> for Error {
fn from(e: rmp::encode::ValueWriteError) -> Self {
Error::Other(format!("ValueWriteError: {}", e))
}
}
impl From<rmp::decode::ValueReadError> for Error {
fn from(e: rmp::decode::ValueReadError) -> Self {
Error::Other(format!("ValueReadError: {}", e))
}
}
impl From<uuid::Error> for Error {
fn from(e: uuid::Error) -> Self {
Error::Other(format!("UUID error: {}", e))
}
}
impl From<rmp_serde::decode::Error> for Error {
fn from(e: rmp_serde::decode::Error) -> Self {
Error::Other(format!("rmp_serde::decode::Error: {}", e))
}
}
impl From<rmp_serde::encode::Error> for Error {
fn from(e: rmp_serde::encode::Error) -> Self {
Error::Other(format!("rmp_serde::encode::Error: {}", e))
}
}
impl From<serde::de::value::Error> for Error {
fn from(e: serde::de::value::Error) -> Self {
Error::Other(format!("serde::de::value::Error: {}", e))
}
}
impl From<serde_json::Error> for Error {
fn from(e: serde_json::Error) -> Self {
Error::Other(format!("serde_json::Error: {}", e))
}
}
impl From<std::collections::TryReserveError> for Error {
fn from(e: std::collections::TryReserveError) -> Self {
Error::Other(format!("TryReserveError: {}", e))
}
}
impl From<tonic::Status> for Error {
fn from(e: tonic::Status) -> Self {
Error::Other(format!("tonic::Status: {}", e.message()))
}
}
impl From<protos::proto_gen::node_service::Error> for Error {
fn from(e: protos::proto_gen::node_service::Error) -> Self {
Error::from_str(&e.error_info).unwrap_or(Error::Other(format!("Proto_Error: {}", e.error_info)))
}
}
impl From<Error> for protos::proto_gen::node_service::Error {
fn from(val: Error) -> Self {
protos::proto_gen::node_service::Error {
code: 0,
error_info: val.to_string(),
}
}
}
impl Hash for Error {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
match self {
Error::IoError(err) => {
err.kind().hash(state);
err.to_string().hash(state);
}
e => e.to_string().hash(state),
}
}
}
impl Clone for Error {
fn clone(&self) -> Self {
match self {
Error::IoError(err) => Error::IoError(std::io::Error::new(err.kind(), err.to_string())),
Error::ErasureReadQuorum => Error::ErasureReadQuorum,
Error::ErasureWriteQuorum => Error::ErasureWriteQuorum,
Error::DiskNotFound => Error::DiskNotFound,
Error::FaultyDisk => Error::FaultyDisk,
Error::FaultyRemoteDisk => Error::FaultyRemoteDisk,
Error::UnformattedDisk => Error::UnformattedDisk,
Error::DiskAccessDenied => Error::DiskAccessDenied,
Error::DiskOngoingReq => Error::DiskOngoingReq,
Error::FileNotFound => Error::FileNotFound,
Error::FileCorrupt => Error::FileCorrupt,
Error::FileVersionNotFound => Error::FileVersionNotFound,
Error::LessData => Error::LessData,
Error::ShortWrite => Error::ShortWrite,
Error::VolumeNotFound => Error::VolumeNotFound,
Error::VolumeNotEmpty => Error::VolumeNotEmpty,
Error::VolumeAccessDenied => Error::VolumeAccessDenied,
Error::VolumeExists => Error::VolumeExists,
Error::DiskNotDir => Error::DiskNotDir,
Error::FileAccessDenied => Error::FileAccessDenied,
Error::TooManyOpenFiles => Error::TooManyOpenFiles,
Error::IsNotRegular => Error::IsNotRegular,
Error::CorruptedBackend => Error::CorruptedBackend,
Error::UnsupportedDisk => Error::UnsupportedDisk,
Error::DiskFull => Error::DiskFull,
Error::Nil => Error::Nil,
Error::DoneForNow => Error::DoneForNow,
Error::MethodNotAllowed => Error::MethodNotAllowed,
Error::InconsistentDisk => Error::InconsistentDisk,
Error::FileNameTooLong => Error::FileNameTooLong,
Error::ScanIgnoreFileContrib => Error::ScanIgnoreFileContrib,
Error::ScanSkipFile => Error::ScanSkipFile,
Error::ScanHealStopSignal => Error::ScanHealStopSignal,
Error::ScanHealIdleTimeout => Error::ScanHealIdleTimeout,
Error::ScanRetryHealing => Error::ScanRetryHealing,
Error::Other(msg) => Error::Other(msg.clone()),
}
}
}
impl PartialEq for Error {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Error::IoError(e1), Error::IoError(e2)) => e1.kind() == e2.kind() && e1.to_string() == e2.to_string(),
(Error::ErasureReadQuorum, Error::ErasureReadQuorum) => true,
(Error::ErasureWriteQuorum, Error::ErasureWriteQuorum) => true,
(Error::DiskNotFound, Error::DiskNotFound) => true,
(Error::FaultyDisk, Error::FaultyDisk) => true,
(Error::FaultyRemoteDisk, Error::FaultyRemoteDisk) => true,
(Error::UnformattedDisk, Error::UnformattedDisk) => true,
(Error::DiskAccessDenied, Error::DiskAccessDenied) => true,
(Error::DiskOngoingReq, Error::DiskOngoingReq) => true,
(Error::FileNotFound, Error::FileNotFound) => true,
(Error::FileCorrupt, Error::FileCorrupt) => true,
(Error::FileVersionNotFound, Error::FileVersionNotFound) => true,
(Error::LessData, Error::LessData) => true,
(Error::ShortWrite, Error::ShortWrite) => true,
(Error::VolumeNotFound, Error::VolumeNotFound) => true,
(Error::VolumeNotEmpty, Error::VolumeNotEmpty) => true,
(Error::VolumeAccessDenied, Error::VolumeAccessDenied) => true,
(Error::VolumeExists, Error::VolumeExists) => true,
(Error::DiskNotDir, Error::DiskNotDir) => true,
(Error::FileAccessDenied, Error::FileAccessDenied) => true,
(Error::TooManyOpenFiles, Error::TooManyOpenFiles) => true,
(Error::IsNotRegular, Error::IsNotRegular) => true,
(Error::CorruptedBackend, Error::CorruptedBackend) => true,
(Error::UnsupportedDisk, Error::UnsupportedDisk) => true,
(Error::DiskFull, Error::DiskFull) => true,
(Error::Nil, Error::Nil) => true,
(Error::DoneForNow, Error::DoneForNow) => true,
(Error::MethodNotAllowed, Error::MethodNotAllowed) => true,
(Error::InconsistentDisk, Error::InconsistentDisk) => true,
(Error::FileNameTooLong, Error::FileNameTooLong) => true,
(Error::ScanIgnoreFileContrib, Error::ScanIgnoreFileContrib) => true,
(Error::ScanSkipFile, Error::ScanSkipFile) => true,
(Error::ScanHealStopSignal, Error::ScanHealStopSignal) => true,
(Error::ScanHealIdleTimeout, Error::ScanHealIdleTimeout) => true,
(Error::ScanRetryHealing, Error::ScanRetryHealing) => true,
(Error::Other(s1), Error::Other(s2)) => s1 == s2,
_ => false,
}
}
}
impl Eq for Error {}
impl Error {
/// Create an error from a message string (for backward compatibility)
pub fn msg<S: Into<String>>(message: S) -> Self {
Error::Other(message.into())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::io;
#[test]
fn test_display_and_debug() {
let e = Error::DiskNotFound;
assert_eq!(format!("{}", e), format!("{ERROR_PREFIX}Disk not found"));
assert_eq!(format!("{:?}", e), "DiskNotFound");
let io_err = Error::IoError(io::Error::other("fail"));
assert_eq!(format!("{}", io_err), "I/O error: fail");
}
#[test]
fn test_partial_eq_and_eq() {
assert_eq!(Error::DiskNotFound, Error::DiskNotFound);
assert_ne!(Error::DiskNotFound, Error::FaultyDisk);
let e1 = Error::IoError(io::Error::other("fail"));
let e2 = Error::IoError(io::Error::other("fail"));
assert_eq!(e1, e2);
let e3 = Error::IoError(io::Error::new(io::ErrorKind::NotFound, "fail"));
assert_ne!(e1, e3);
}
#[test]
fn test_clone() {
let e = Error::DiskAccessDenied;
let cloned = e.clone();
assert_eq!(e, cloned);
let io_err = Error::IoError(io::Error::other("fail"));
let cloned_io = io_err.clone();
assert_eq!(io_err, cloned_io);
}
#[test]
fn test_hash() {
let e1 = Error::DiskNotFound;
let e2 = Error::DiskNotFound;
let mut h1 = DefaultHasher::new();
let mut h2 = DefaultHasher::new();
e1.hash(&mut h1);
e2.hash(&mut h2);
assert_eq!(h1.finish(), h2.finish());
let io_err1 = Error::IoError(io::Error::other("fail"));
let io_err2 = Error::IoError(io::Error::other("fail"));
let mut h3 = DefaultHasher::new();
let mut h4 = DefaultHasher::new();
io_err1.hash(&mut h3);
io_err2.hash(&mut h4);
assert_eq!(h3.finish(), h4.finish());
}
#[test]
fn test_from_error_for_io_error() {
let e = Error::DiskNotFound;
let io_err: io::Error = e.into();
assert_eq!(io_err.kind(), io::ErrorKind::Other);
assert_eq!(io_err.to_string(), format!("{ERROR_PREFIX}Disk not found"));
assert_eq!(Error::from(io_err.to_string()), Error::DiskNotFound);
let orig = io::Error::other("fail");
let e2 = Error::IoError(orig.kind().into());
let io_err2: io::Error = e2.into();
assert_eq!(io_err2.kind(), io::ErrorKind::Other);
}
#[test]
fn test_from_io_error_for_error() {
let orig = io::Error::other("fail");
let e: Error = orig.into();
match e {
Error::IoError(ioe) => {
assert_eq!(ioe.kind(), io::ErrorKind::Other);
assert_eq!(ioe.to_string(), "fail");
}
_ => panic!("Expected IoError variant"),
}
}
#[test]
fn test_default() {
let e = Error::default();
assert_eq!(e, Error::Nil);
}
#[test]
fn test_from_str() {
use std::str::FromStr;
assert_eq!(Error::from_str("Nil"), Ok(Error::Nil));
assert_eq!(Error::from_str("DiskNotFound"), Ok(Error::DiskNotFound));
assert_eq!(Error::from_str("ErasureReadQuorum"), Ok(Error::ErasureReadQuorum));
assert_eq!(Error::from_str("I/O error: fail"), Ok(Error::IoError(io::Error::other("fail"))));
assert_eq!(Error::from_str(&format!("{ERROR_PREFIX}Disk not found")), Ok(Error::DiskNotFound));
assert_eq!(
Error::from_str("UnknownError"),
Err(Error::IoError(std::io::Error::other("UnknownError")))
);
}
#[test]
fn test_from_string() {
let e: Error = format!("{ERROR_PREFIX}Disk not found").parse().unwrap();
assert_eq!(e, Error::DiskNotFound);
let e2: Error = "I/O error: fail".to_string().parse().unwrap();
assert_eq!(e2, Error::IoError(std::io::Error::other("fail")));
}
#[test]
fn test_from_io_error() {
let e = Error::IoError(io::Error::other("fail"));
let io_err: io::Error = e.clone().into();
assert_eq!(io_err.to_string(), "fail");
let e2: Error = io::Error::other("fail").into();
assert_eq!(e2, Error::IoError(io::Error::other("fail")));
let result = Error::from(io::Error::other("fail"));
assert_eq!(result, Error::IoError(io::Error::other("fail")));
let io_err2: std::io::Error = Error::CorruptedBackend.into();
assert_eq!(io_err2.to_string(), "[RUSTFS error] Corrupted backend");
assert_eq!(Error::from(io_err2), Error::CorruptedBackend);
let io_err3: std::io::Error = Error::DiskNotFound.into();
assert_eq!(io_err3.to_string(), "[RUSTFS error] Disk not found");
assert_eq!(Error::from(io_err3), Error::DiskNotFound);
let io_err4: std::io::Error = Error::DiskAccessDenied.into();
assert_eq!(io_err4.to_string(), "[RUSTFS error] Disk access denied");
assert_eq!(Error::from(io_err4), Error::DiskAccessDenied);
}
#[test]
fn test_question_mark_operator() {
fn parse_number(s: &str) -> Result<i32> {
let num = s.parse::<i32>()?; // ParseIntError automatically converts to Error
Ok(num)
}
fn format_string() -> Result<String> {
use std::fmt::Write;
let mut s = String::new();
write!(&mut s, "test")?; // fmt::Error automatically converts to Error
Ok(s)
}
fn utf8_conversion() -> Result<String> {
let bytes = vec![0xFF, 0xFE]; // Invalid UTF-8
let s = String::from_utf8(bytes)?; // FromUtf8Error automatically converts to Error
Ok(s)
}
// Test successful case
assert_eq!(parse_number("42").unwrap(), 42);
// Test error conversion
let err = parse_number("not_a_number").unwrap_err();
assert!(matches!(err, Error::Other(_)));
assert!(err.to_string().contains("Parse int error"));
// Test format error conversion
assert_eq!(format_string().unwrap(), "test");
// Test UTF-8 error conversion
let err = utf8_conversion().unwrap_err();
assert!(matches!(err, Error::Other(_)));
assert!(err.to_string().contains("UTF-8 conversion error"));
}
}
crates/error/src/ignored.rs
-11
View File
@@ -1,11 +0,0 @@
use crate::Error;
pub static OBJECT_OP_IGNORED_ERRS: &[Error] = &[
Error::DiskNotFound,
Error::FaultyDisk,
Error::FaultyRemoteDisk,
Error::DiskAccessDenied,
Error::DiskOngoingReq,
Error::UnformattedDisk,
];
pub static BASE_IGNORED_ERRS: &[Error] = &[Error::DiskNotFound, Error::FaultyDisk, Error::FaultyRemoteDisk];
crates/error/src/lib.rs
-14
View File
@@ -1,14 +0,0 @@
mod error;
pub use error::*;
mod reduce;
pub use reduce::*;
mod ignored;
pub use ignored::*;
mod convert;
pub use convert::*;
mod bitrot;
pub use bitrot::*;
crates/error/src/reduce.rs
-138
View File
@@ -1,138 +0,0 @@
use crate::error::Error;
pub fn reduce_write_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error], quorun: usize) -> Option<Error> {
reduce_quorum_errs(errors, ignored_errs, quorun, Error::ErasureWriteQuorum)
}
pub fn reduce_read_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error], quorun: usize) -> Option<Error> {
reduce_quorum_errs(errors, ignored_errs, quorun, Error::ErasureReadQuorum)
}
pub fn reduce_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error], quorun: usize, quorun_err: Error) -> Option<Error> {
let (max_count, err) = reduce_errs(errors, ignored_errs);
if max_count >= quorun {
err
} else {
Some(quorun_err)
}
}
pub fn reduce_errs(errors: &[Option<Error>], ignored_errs: &[Error]) -> (usize, Option<Error>) {
let err_counts =
errors
.iter()
.map(|e| e.as_ref().unwrap_or(&Error::Nil))
.fold(std::collections::HashMap::new(), |mut acc, e| {
if is_ignored_err(ignored_errs, e) {
return acc;
}
*acc.entry(e.clone()).or_insert(0) += 1;
acc
});
let (err, max_count) = err_counts
.into_iter()
.max_by(|(e1, c1), (e2, c2)| {
// Prefer Error::Nil if present in a tie
let count_cmp = c1.cmp(c2);
if count_cmp == std::cmp::Ordering::Equal {
match (e1, e2) {
(Error::Nil, _) => std::cmp::Ordering::Greater,
(_, Error::Nil) => std::cmp::Ordering::Less,
_ => format!("{e1:?}").cmp(&format!("{e2:?}")),
}
} else {
count_cmp
}
})
.unwrap_or((Error::Nil, 0));
(max_count, if err == Error::Nil { None } else { Some(err) })
}
pub fn is_ignored_err(ignored_errs: &[Error], err: &Error) -> bool {
ignored_errs.iter().any(|e| e == err)
}
pub fn count_errs(errors: &[Option<Error>], err: Error) -> usize {
errors
.iter()
.map(|e| if e.is_none() { &Error::Nil } else { e.as_ref().unwrap() })
.filter(|&e| e == &err)
.count()
}
#[cfg(test)]
mod tests {
use super::*;
fn err_io(msg: &str) -> Error {
Error::IoError(std::io::Error::other(msg))
}
#[test]
fn test_reduce_errs_basic() {
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None];
let ignored = vec![];
let (count, err) = reduce_errs(&errors, &ignored);
assert_eq!(count, 2);
assert_eq!(err, Some(e1));
}
#[test]
fn test_reduce_errs_ignored() {
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e2.clone()), Some(e1.clone()), Some(e2.clone()), None];
let ignored = vec![e2.clone()];
let (count, err) = reduce_errs(&errors, &ignored);
assert_eq!(count, 2);
assert_eq!(err, Some(e1));
}
#[test]
fn test_reduce_quorum_errs() {
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None];
let ignored = vec![];
// quorum = 2, should return e1
let res = reduce_quorum_errs(&errors, &ignored, 2, Error::ErasureReadQuorum);
assert_eq!(res, Some(e1));
// quorum = 3, should return quorum error
let res = reduce_quorum_errs(&errors, &ignored, 3, Error::ErasureReadQuorum);
assert_eq!(res, Some(Error::ErasureReadQuorum));
}
#[test]
fn test_count_errs() {
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e2.clone()), Some(e1.clone()), None];
assert_eq!(count_errs(&errors, e1.clone()), 2);
assert_eq!(count_errs(&errors, e2.clone()), 1);
}
#[test]
fn test_is_ignored_err() {
let e1 = err_io("a");
let e2 = err_io("b");
let ignored = vec![e1.clone()];
assert!(is_ignored_err(&ignored, &e1));
assert!(!is_ignored_err(&ignored, &e2));
}
#[test]
fn test_reduce_errs_nil_tiebreak() {
// Error::Nil and another error have the same count, should prefer Nil
let e1 = err_io("a");
let e2 = err_io("b");
let errors = vec![Some(e1.clone()), Some(e2.clone()), None, Some(e1.clone()), None]; // e1:1, Nil:1
let ignored = vec![];
let (count, err) = reduce_errs(&errors, &ignored);
assert_eq!(count, 2);
assert_eq!(err, None); // None means Error::Nil is preferred
}
}