From c589972fa768f8b38711b9561bf4a5046cc7a57e Mon Sep 17 00:00:00 2001 From: weisd Date: Fri, 6 Jun 2025 16:15:26 +0800 Subject: [PATCH] mc test ok --- Cargo.lock | 1 + common/common/src/error.rs | 7 + crates/disk/Cargo.toml | 33 - crates/disk/src/api.rs | 667 --- crates/disk/src/endpoint.rs | 379 -- crates/disk/src/error.rs | 594 --- crates/disk/src/format.rs | 273 -- crates/disk/src/fs.rs | 179 - crates/disk/src/lib.rs | 12 - crates/disk/src/local.rs | 2048 --------- crates/disk/src/local_bak.rs | 2364 ---------- crates/disk/src/local_list.rs | 535 --- crates/disk/src/metacache.rs | 608 --- crates/disk/src/os.rs | 206 - crates/disk/src/path.rs | 308 -- crates/disk/src/remote.rs | 908 ---- crates/disk/src/remote_bak.rs | 862 ---- crates/disk/src/utils.rs | 35 - crates/error/Cargo.toml | 22 - crates/error/src/bitrot.rs | 27 - crates/error/src/convert.rs | 92 - crates/error/src/error.rs | 586 --- crates/error/src/ignored.rs | 11 - crates/error/src/lib.rs | 14 - crates/error/src/reduce.rs | 138 - ecstore/src/bitrot.rs | 1438 +++--- ecstore/src/bucket/error.rs | 6 +- ecstore/src/disk/local.rs | 34 +- ecstore/src/disks_layout.rs | 26 +- ecstore/src/endpoints.rs | 59 +- ecstore/src/erasure.rs | 8 +- ecstore/src/error.rs | 6 + ecstore/src/file_meta.rs | 6798 ++++++++++++++-------------- ecstore/src/file_meta_inline.rs | 6 +- ecstore/src/set_disk.rs | 126 +- ecstore/src/utils/bool_flag.rs | 4 +- ecstore/src/utils/ellipses.rs | 24 +- ecstore/src/utils/net.rs | 25 +- ecstore/src/utils/os/linux.rs | 12 +- ecstore/src/utils/os/unix.rs | 9 +- ecstore/src/utils/os/windows.rs | 3 +- iam/src/error.rs | 15 + rustfs/Cargo.toml | 1 + rustfs/src/admin/handlers/pools.rs | 10 +- rustfs/src/admin/mod.rs | 5 +- rustfs/src/admin/router.rs | 15 +- rustfs/src/admin/rpc.rs | 7 +- rustfs/src/error.rs | 1762 +------ rustfs/src/grpc.rs | 2 +- rustfs/src/license.rs | 10 +- rustfs/src/main.rs | 16 +- rustfs/src/storage/ecfs.rs | 216 +- rustfs/src/storage/error.rs | 970 ++-- rustfs/src/storage/mod.rs | 2 +- rustfs/src/storage/options.rs | 88 +- 55 files changed, 5053 insertions(+), 17559 deletions(-) delete mode 100644 crates/disk/Cargo.toml delete mode 100644 crates/disk/src/api.rs delete mode 100644 crates/disk/src/endpoint.rs delete mode 100644 crates/disk/src/error.rs delete mode 100644 crates/disk/src/format.rs delete mode 100644 crates/disk/src/fs.rs delete mode 100644 crates/disk/src/lib.rs delete mode 100644 crates/disk/src/local.rs delete mode 100644 crates/disk/src/local_bak.rs delete mode 100644 crates/disk/src/local_list.rs delete mode 100644 crates/disk/src/metacache.rs delete mode 100644 crates/disk/src/os.rs delete mode 100644 crates/disk/src/path.rs delete mode 100644 crates/disk/src/remote.rs delete mode 100644 crates/disk/src/remote_bak.rs delete mode 100644 crates/disk/src/utils.rs delete mode 100644 crates/error/Cargo.toml delete mode 100644 crates/error/src/bitrot.rs delete mode 100644 crates/error/src/convert.rs delete mode 100644 crates/error/src/error.rs delete mode 100644 crates/error/src/ignored.rs delete mode 100644 crates/error/src/lib.rs delete mode 100644 crates/error/src/reduce.rs diff --git a/Cargo.lock b/Cargo.lock index 86e82d6d0..0006dc63b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7710,6 +7710,7 @@ dependencies = [ "rustfs-event-notifier", "rustfs-filemeta", "rustfs-obs", + "rustfs-rio", "rustfs-utils", "rustfs-zip", "rustls 0.23.27", diff --git a/common/common/src/error.rs b/common/common/src/error.rs index 38abb0a80..1f8f52d6e 100644 --- a/common/common/src/error.rs +++ b/common/common/src/error.rs @@ -11,6 +11,13 @@ pub struct Error { } impl Error { + pub fn other(error: E) -> Self + where + E: std::fmt::Display + Into>, + { + Self::from_std_error(error.into()) + } + /// Create a new error from a `std::error::Error`. #[must_use] #[track_caller] diff --git a/crates/disk/Cargo.toml b/crates/disk/Cargo.toml deleted file mode 100644 index cdf7bf8fe..000000000 --- a/crates/disk/Cargo.toml +++ /dev/null @@ -1,33 +0,0 @@ -[package] -name = "rustfs-disk" -edition.workspace = true -license.workspace = true -repository.workspace = true -rust-version.workspace = true -version.workspace = true - -[dependencies] -url.workspace = true -rustfs-filemeta.workspace = true -rustfs-error.workspace = true -rustfs-rio.workspace = true -serde.workspace = true -serde_json.workspace = true -uuid.workspace = true -tracing.workspace = true -tokio.workspace = true -path-absolutize = "3.1.1" -rustfs-utils = {workspace = true, features =["net"]} -async-trait.workspace = true -time.workspace = true -rustfs-metacache.workspace = true -futures.workspace = true -madmin.workspace = true -protos.workspace = true -tonic.workspace = true -urlencoding = "2.1.3" -rmp-serde.workspace = true -http.workspace = true - -[lints] -workspace = true diff --git a/crates/disk/src/api.rs b/crates/disk/src/api.rs deleted file mode 100644 index 1967b1807..000000000 --- a/crates/disk/src/api.rs +++ /dev/null @@ -1,667 +0,0 @@ -use crate::{endpoint::Endpoint, local::LocalDisk, remote::RemoteDisk}; -use madmin::DiskMetrics; -use rustfs_error::{Error, Result}; -use rustfs_filemeta::{FileInfo, FileInfoVersions, RawFileInfo}; -use serde::{Deserialize, Serialize}; -use std::{fmt::Debug, path::PathBuf, sync::Arc}; -use time::OffsetDateTime; -use tokio::io::{AsyncRead, AsyncWrite}; -use uuid::Uuid; - -pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys"; -pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart"; -pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp"; -pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash"; -pub const BUCKET_META_PREFIX: &str = "buckets"; -pub const FORMAT_CONFIG_FILE: &str = "format.json"; -pub const STORAGE_FORMAT_FILE: &str = "xl.meta"; -pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp"; - -pub type DiskStore = Arc; - -#[derive(Debug)] -pub enum Disk { - Local(LocalDisk), - Remote(RemoteDisk), -} - -#[async_trait::async_trait] -impl DiskAPI for Disk { - #[tracing::instrument(skip(self))] - fn to_string(&self) -> String { - match self { - Disk::Local(local_disk) => local_disk.to_string(), - Disk::Remote(remote_disk) => remote_disk.to_string(), - } - } - - #[tracing::instrument(skip(self))] - fn is_local(&self) -> bool { - match self { - Disk::Local(local_disk) => local_disk.is_local(), - Disk::Remote(remote_disk) => remote_disk.is_local(), - } - } - - #[tracing::instrument(skip(self))] - fn host_name(&self) -> String { - match self { - Disk::Local(local_disk) => local_disk.host_name(), - Disk::Remote(remote_disk) => remote_disk.host_name(), - } - } - - #[tracing::instrument(skip(self))] - async fn is_online(&self) -> bool { - match self { - Disk::Local(local_disk) => local_disk.is_online().await, - Disk::Remote(remote_disk) => remote_disk.is_online().await, - } - } - - #[tracing::instrument(skip(self))] - fn endpoint(&self) -> Endpoint { - match self { - Disk::Local(local_disk) => local_disk.endpoint(), - Disk::Remote(remote_disk) => remote_disk.endpoint(), - } - } - - #[tracing::instrument(skip(self))] - async fn close(&self) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.close().await, - Disk::Remote(remote_disk) => remote_disk.close().await, - } - } - - #[tracing::instrument(skip(self))] - fn path(&self) -> PathBuf { - match self { - Disk::Local(local_disk) => local_disk.path(), - Disk::Remote(remote_disk) => remote_disk.path(), - } - } - - #[tracing::instrument(skip(self))] - fn get_disk_location(&self) -> DiskLocation { - match self { - Disk::Local(local_disk) => local_disk.get_disk_location(), - Disk::Remote(remote_disk) => remote_disk.get_disk_location(), - } - } - - #[tracing::instrument(skip(self))] - async fn get_disk_id(&self) -> Result> { - match self { - Disk::Local(local_disk) => local_disk.get_disk_id().await, - Disk::Remote(remote_disk) => remote_disk.get_disk_id().await, - } - } - - #[tracing::instrument(skip(self))] - async fn set_disk_id(&self, id: Option) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.set_disk_id(id).await, - Disk::Remote(remote_disk) => remote_disk.set_disk_id(id).await, - } - } - - #[tracing::instrument(skip(self))] - async fn read_all(&self, volume: &str, path: &str) -> Result> { - match self { - Disk::Local(local_disk) => local_disk.read_all(volume, path).await, - Disk::Remote(remote_disk) => remote_disk.read_all(volume, path).await, - } - } - - #[tracing::instrument(skip(self))] - async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.write_all(volume, path, data).await, - Disk::Remote(remote_disk) => remote_disk.write_all(volume, path, data).await, - } - } - - #[tracing::instrument(skip(self))] - async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.delete(volume, path, opt).await, - Disk::Remote(remote_disk) => remote_disk.delete(volume, path, opt).await, - } - } - - #[tracing::instrument(skip(self))] - async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - match self { - Disk::Local(local_disk) => local_disk.verify_file(volume, path, fi).await, - Disk::Remote(remote_disk) => remote_disk.verify_file(volume, path, fi).await, - } - } - - #[tracing::instrument(skip(self))] - async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - match self { - Disk::Local(local_disk) => local_disk.check_parts(volume, path, fi).await, - Disk::Remote(remote_disk) => remote_disk.check_parts(volume, path, fi).await, - } - } - - #[tracing::instrument(skip(self))] - async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.rename_part(src_volume, src_path, dst_volume, dst_path, meta).await, - Disk::Remote(remote_disk) => { - remote_disk - .rename_part(src_volume, src_path, dst_volume, dst_path, meta) - .await - } - } - } - - #[tracing::instrument(skip(self))] - async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.rename_file(src_volume, src_path, dst_volume, dst_path).await, - Disk::Remote(remote_disk) => remote_disk.rename_file(src_volume, src_path, dst_volume, dst_path).await, - } - } - - #[tracing::instrument(skip(self))] - async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result> { - match self { - Disk::Local(local_disk) => local_disk.create_file(_origvolume, volume, path, _file_size).await, - Disk::Remote(remote_disk) => remote_disk.create_file(_origvolume, volume, path, _file_size).await, - } - } - - #[tracing::instrument(skip(self))] - async fn append_file(&self, volume: &str, path: &str) -> Result> { - match self { - Disk::Local(local_disk) => local_disk.append_file(volume, path).await, - Disk::Remote(remote_disk) => remote_disk.append_file(volume, path).await, - } - } - - #[tracing::instrument(skip(self))] - async fn read_file(&self, volume: &str, path: &str) -> Result> { - match self { - Disk::Local(local_disk) => local_disk.read_file(volume, path).await, - Disk::Remote(remote_disk) => remote_disk.read_file(volume, path).await, - } - } - - #[tracing::instrument(skip(self))] - async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result> { - match self { - Disk::Local(local_disk) => local_disk.read_file_stream(volume, path, offset, length).await, - Disk::Remote(remote_disk) => remote_disk.read_file_stream(volume, path, offset, length).await, - } - } - - #[tracing::instrument(skip(self))] - async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result> { - match self { - Disk::Local(local_disk) => local_disk.list_dir(_origvolume, volume, _dir_path, _count).await, - Disk::Remote(remote_disk) => remote_disk.list_dir(_origvolume, volume, _dir_path, _count).await, - } - } - - #[tracing::instrument(skip(self, wr))] - async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.walk_dir(opts, wr).await, - Disk::Remote(remote_disk) => remote_disk.walk_dir(opts, wr).await, - } - } - - #[tracing::instrument(skip(self))] - async fn rename_data( - &self, - src_volume: &str, - src_path: &str, - fi: FileInfo, - dst_volume: &str, - dst_path: &str, - ) -> Result { - match self { - Disk::Local(local_disk) => local_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await, - Disk::Remote(remote_disk) => remote_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await, - } - } - - #[tracing::instrument(skip(self))] - async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.make_volumes(volumes).await, - Disk::Remote(remote_disk) => remote_disk.make_volumes(volumes).await, - } - } - - #[tracing::instrument(skip(self))] - async fn make_volume(&self, volume: &str) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.make_volume(volume).await, - Disk::Remote(remote_disk) => remote_disk.make_volume(volume).await, - } - } - - #[tracing::instrument(skip(self))] - async fn list_volumes(&self) -> Result> { - match self { - Disk::Local(local_disk) => local_disk.list_volumes().await, - Disk::Remote(remote_disk) => remote_disk.list_volumes().await, - } - } - - #[tracing::instrument(skip(self))] - async fn stat_volume(&self, volume: &str) -> Result { - match self { - Disk::Local(local_disk) => local_disk.stat_volume(volume).await, - Disk::Remote(remote_disk) => remote_disk.stat_volume(volume).await, - } - } - - #[tracing::instrument(skip(self))] - async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.delete_paths(volume, paths).await, - Disk::Remote(remote_disk) => remote_disk.delete_paths(volume, paths).await, - } - } - - #[tracing::instrument(skip(self))] - async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.update_metadata(volume, path, fi, opts).await, - Disk::Remote(remote_disk) => remote_disk.update_metadata(volume, path, fi, opts).await, - } - } - - #[tracing::instrument(skip(self))] - async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.write_metadata(_org_volume, volume, path, fi).await, - Disk::Remote(remote_disk) => remote_disk.write_metadata(_org_volume, volume, path, fi).await, - } - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn read_version( - &self, - _org_volume: &str, - volume: &str, - path: &str, - version_id: &str, - opts: &ReadOptions, - ) -> Result { - match self { - Disk::Local(local_disk) => local_disk.read_version(_org_volume, volume, path, version_id, opts).await, - Disk::Remote(remote_disk) => remote_disk.read_version(_org_volume, volume, path, version_id, opts).await, - } - } - - #[tracing::instrument(skip(self))] - async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { - match self { - Disk::Local(local_disk) => local_disk.read_xl(volume, path, read_data).await, - Disk::Remote(remote_disk) => remote_disk.read_xl(volume, path, read_data).await, - } - } - - #[tracing::instrument(skip(self))] - async fn delete_version( - &self, - volume: &str, - path: &str, - fi: FileInfo, - force_del_marker: bool, - opts: DeleteOptions, - ) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.delete_version(volume, path, fi, force_del_marker, opts).await, - Disk::Remote(remote_disk) => remote_disk.delete_version(volume, path, fi, force_del_marker, opts).await, - } - } - - #[tracing::instrument(skip(self))] - async fn delete_versions( - &self, - volume: &str, - versions: Vec, - opts: DeleteOptions, - ) -> Result>> { - match self { - Disk::Local(local_disk) => local_disk.delete_versions(volume, versions, opts).await, - Disk::Remote(remote_disk) => remote_disk.delete_versions(volume, versions, opts).await, - } - } - - #[tracing::instrument(skip(self))] - async fn read_multiple(&self, req: ReadMultipleReq) -> Result> { - match self { - Disk::Local(local_disk) => local_disk.read_multiple(req).await, - Disk::Remote(remote_disk) => remote_disk.read_multiple(req).await, - } - } - - #[tracing::instrument(skip(self))] - async fn delete_volume(&self, volume: &str) -> Result<()> { - match self { - Disk::Local(local_disk) => local_disk.delete_volume(volume).await, - Disk::Remote(remote_disk) => remote_disk.delete_volume(volume).await, - } - } - - #[tracing::instrument(skip(self))] - async fn disk_info(&self, opts: &DiskInfoOptions) -> Result { - match self { - Disk::Local(local_disk) => local_disk.disk_info(opts).await, - Disk::Remote(remote_disk) => remote_disk.disk_info(opts).await, - } - } - - // #[tracing::instrument(skip(self, cache, we_sleep, scan_mode))] - // async fn ns_scanner( - // &self, - // cache: &DataUsageCache, - // updates: Sender, - // scan_mode: HealScanMode, - // we_sleep: ShouldSleepFn, - // ) -> Result { - // match self { - // Disk::Local(local_disk) => local_disk.ns_scanner(cache, updates, scan_mode, we_sleep).await, - // Disk::Remote(remote_disk) => remote_disk.ns_scanner(cache, updates, scan_mode, we_sleep).await, - // } - // } - - // #[tracing::instrument(skip(self))] - // async fn healing(&self) -> Option { - // match self { - // Disk::Local(local_disk) => local_disk.healing().await, - // Disk::Remote(remote_disk) => remote_disk.healing().await, - // } - // } -} - -pub async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> Result { - if ep.is_local { - Ok(Arc::new(Disk::Local(LocalDisk::new(ep, opt.cleanup).await?))) - } else { - Ok(Arc::new(Disk::Remote(RemoteDisk::new(ep, opt).await?))) - } -} - -#[async_trait::async_trait] -pub trait DiskAPI: Debug + Send + Sync + 'static { - fn to_string(&self) -> String; - async fn is_online(&self) -> bool; - fn is_local(&self) -> bool; - // LastConn - fn host_name(&self) -> String; - fn endpoint(&self) -> Endpoint; - async fn close(&self) -> Result<()>; - async fn get_disk_id(&self) -> Result>; - async fn set_disk_id(&self, id: Option) -> Result<()>; - - fn path(&self) -> PathBuf; - fn get_disk_location(&self) -> DiskLocation; - - // Healing - // DiskInfo - // NSScanner - - // Volume operations. - async fn make_volume(&self, volume: &str) -> Result<()>; - async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>; - async fn list_volumes(&self) -> Result>; - async fn stat_volume(&self, volume: &str) -> Result; - async fn delete_volume(&self, volume: &str) -> Result<()>; - - // 并发边读边写 w <- MetaCacheEntry - async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()>; - - // Metadata operations - async fn delete_version( - &self, - volume: &str, - path: &str, - fi: FileInfo, - force_del_marker: bool, - opts: DeleteOptions, - ) -> Result<()>; - async fn delete_versions( - &self, - volume: &str, - versions: Vec, - opts: DeleteOptions, - ) -> Result>>; - async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()>; - async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>; - async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()>; - async fn read_version( - &self, - org_volume: &str, - volume: &str, - path: &str, - version_id: &str, - opts: &ReadOptions, - ) -> Result; - async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result; - async fn rename_data( - &self, - src_volume: &str, - src_path: &str, - file_info: FileInfo, - dst_volume: &str, - dst_path: &str, - ) -> Result; - - // File operations. - // 读目录下的所有文件、目录 - async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result>; - async fn read_file(&self, volume: &str, path: &str) -> Result>; - async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result>; - async fn append_file(&self, volume: &str, path: &str) -> Result>; - async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result>; - // ReadFileStream - async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>; - async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()>; - async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; - // VerifyFile - async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result; - // CheckParts - async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result; - // StatInfoFile - // ReadParts - async fn read_multiple(&self, req: ReadMultipleReq) -> Result>; - // CleanAbandonedData - async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()>; - async fn read_all(&self, volume: &str, path: &str) -> Result>; - async fn disk_info(&self, opts: &DiskInfoOptions) -> Result; - // async fn ns_scanner( - // &self, - // cache: &DataUsageCache, - // updates: Sender, - // scan_mode: HealScanMode, - // we_sleep: ShouldSleepFn, - // ) -> Result; - // async fn healing(&self) -> Option; -} - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct CheckPartsResp { - pub results: Vec, -} - -#[derive(Debug, Serialize, Deserialize, Default)] -pub struct UpdateMetadataOpts { - pub no_persistence: bool, -} - -pub struct DiskLocation { - pub pool_idx: Option, - pub set_idx: Option, - pub disk_idx: Option, -} - -impl DiskLocation { - pub fn valid(&self) -> bool { - self.pool_idx.is_some() && self.set_idx.is_some() && self.disk_idx.is_some() - } -} - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct DiskInfoOptions { - pub disk_id: String, - pub metrics: bool, - pub noop: bool, -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] -pub struct DiskInfo { - pub total: u64, - pub free: u64, - pub used: u64, - pub used_inodes: u64, - pub free_inodes: u64, - pub major: u64, - pub minor: u64, - pub nr_requests: u64, - pub fs_type: String, - pub root_disk: bool, - pub healing: bool, - pub scanning: bool, - pub endpoint: String, - pub mount_path: String, - pub id: String, - pub rotational: bool, - pub metrics: DiskMetrics, - pub error: String, -} - -#[derive(Clone, Debug, Default)] -pub struct Info { - pub total: u64, - pub free: u64, - pub used: u64, - pub files: u64, - pub ffree: u64, - pub fstype: String, - pub major: u64, - pub minor: u64, - pub name: String, - pub rotational: bool, - pub nrrequests: u64, -} - -// #[derive(Debug, Default, Clone, Serialize, Deserialize)] -// pub struct FileInfoVersions { -// // Name of the volume. -// pub volume: String, - -// // Name of the file. -// pub name: String, - -// // Represents the latest mod time of the -// // latest version. -// pub latest_mod_time: Option, - -// pub versions: Vec, -// pub free_versions: Vec, -// } - -// impl FileInfoVersions { -// pub fn find_version_index(&self, v: &str) -> Option { -// if v.is_empty() { -// return None; -// } - -// let vid = Uuid::parse_str(v).unwrap_or(Uuid::nil()); - -// self.versions.iter().position(|v| v.version_id == Some(vid)) -// } -// } - -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct WalkDirOptions { - // Bucket to scanner - pub bucket: String, - // Directory inside the bucket. - pub base_dir: String, - // Do a full recursive scan. - pub recursive: bool, - - // ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found. - pub report_notfound: bool, - - // FilterPrefix will only return results with given prefix within folder. - // Should never contain a slash. - pub filter_prefix: Option, - - // ForwardTo will forward to the given object path. - pub forward_to: Option, - - // Limit the number of returned objects if > 0. - pub limit: i32, - - // DiskID contains the disk ID of the disk. - // Leave empty to not check disk ID. - pub disk_id: String, -} -// move metacache to metacache.rs - -#[derive(Clone, Debug, Default)] -pub struct DiskOption { - pub cleanup: bool, - pub health_check: bool, -} - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct RenameDataResp { - pub old_data_dir: Option, - pub sign: Option>, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct DeleteOptions { - pub recursive: bool, - pub immediate: bool, - pub undo_write: bool, - pub old_data_dir: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ReadMultipleReq { - pub bucket: String, - pub prefix: String, - pub files: Vec, - pub max_size: usize, - pub metadata_only: bool, - pub abort404: bool, - pub max_results: usize, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ReadMultipleResp { - pub bucket: String, - pub prefix: String, - pub file: String, - pub exists: bool, - pub error: String, - pub data: Vec, - pub mod_time: Option, -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct VolumeInfo { - pub name: String, - pub created: Option, -} - -#[derive(Deserialize, Serialize, Debug, Default)] -pub struct ReadOptions { - pub incl_free_versions: bool, - pub read_data: bool, - pub healing: bool, -} diff --git a/crates/disk/src/endpoint.rs b/crates/disk/src/endpoint.rs deleted file mode 100644 index 9b946e138..000000000 --- a/crates/disk/src/endpoint.rs +++ /dev/null @@ -1,379 +0,0 @@ -use path_absolutize::Absolutize; -use rustfs_utils::{is_local_host, is_socket_addr}; -use std::{fmt::Display, path::Path}; -use url::{ParseError, Url}; - -/// enum for endpoint type. -#[derive(PartialEq, Eq, Debug)] -pub enum EndpointType { - /// path style endpoint type enum. - Path, - - /// URL style endpoint type enum. - Url, -} - -/// any type of endpoint. -#[derive(Debug, PartialEq, Eq, Clone, Hash)] -pub struct Endpoint { - pub url: url::Url, - pub is_local: bool, - - pub pool_idx: i32, - pub set_idx: i32, - pub disk_idx: i32, -} - -impl Display for Endpoint { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if self.url.scheme() == "file" { - write!(f, "{}", self.get_file_path()) - } else { - write!(f, "{}", self.url) - } - } -} - -impl TryFrom<&str> for Endpoint { - /// The type returned in the event of a conversion error. - type Error = std::io::Error; - - /// Performs the conversion. - fn try_from(value: &str) -> core::result::Result { - // check whether given path is not empty. - if ["", "/", "\\"].iter().any(|&v| v.eq(value)) { - return Err(std::io::Error::other("empty or root endpoint is not supported")); - } - - let mut is_local = false; - let url = match Url::parse(value) { - #[allow(unused_mut)] - Ok(mut url) if url.has_host() => { - // URL style of endpoint. - // Valid URL style endpoint is - // - Scheme field must contain "http" or "https" - // - All field should be empty except Host and Path. - if !((url.scheme() == "http" || url.scheme() == "https") - && url.username().is_empty() - && url.fragment().is_none() - && url.query().is_none()) - { - return Err(std::io::Error::other("invalid URL endpoint format")); - } - - let path = url.path().to_string(); - - #[cfg(not(windows))] - let path = Path::new(&path).absolutize()?; - - // On windows having a preceding SlashSeparator will cause problems, if the - // command line already has C:/ url.set_path(v), - None => return Err(std::io::Error::other("invalid path")), - } - - url - } - Ok(_) => { - // like d:/foo - is_local = true; - url_parse_from_file_path(value)? - } - Err(e) => match e { - ParseError::InvalidPort => { - return Err(std::io::Error::other( - "invalid URL endpoint format: port number must be between 1 to 65535", - )) - } - ParseError::EmptyHost => return Err(std::io::Error::other("invalid URL endpoint format: empty host name")), - ParseError::RelativeUrlWithoutBase => { - // like /foo - is_local = true; - url_parse_from_file_path(value)? - } - _ => return Err(std::io::Error::other(format!("invalid URL endpoint format: {}", e))), - }, - }; - - Ok(Endpoint { - url, - is_local, - pool_idx: -1, - set_idx: -1, - disk_idx: -1, - }) - } -} - -impl Endpoint { - /// returns type of endpoint. - pub fn get_type(&self) -> EndpointType { - if self.url.scheme() == "file" { - EndpointType::Path - } else { - EndpointType::Url - } - } - - /// sets a specific pool number to this node - pub fn set_pool_index(&mut self, idx: usize) { - self.pool_idx = idx as i32 - } - - /// sets a specific set number to this node - pub fn set_set_index(&mut self, idx: usize) { - self.set_idx = idx as i32 - } - - /// sets a specific disk number to this node - pub fn set_disk_index(&mut self, idx: usize) { - self.disk_idx = idx as i32 - } - - /// resolves the host and updates if it is local or not. - pub fn update_is_local(&mut self, local_port: u16) -> std::io::Result<()> { - match (self.url.scheme(), self.url.host()) { - (v, Some(host)) if v != "file" => { - self.is_local = is_local_host(host, self.url.port().unwrap_or_default(), local_port)?; - } - _ => {} - } - - Ok(()) - } - - /// returns the host to be used for grid connections. - pub fn grid_host(&self) -> String { - match (self.url.host(), self.url.port()) { - (Some(host), Some(port)) => format!("{}://{}:{}", self.url.scheme(), host, port), - (Some(host), None) => format!("{}://{}", self.url.scheme(), host), - _ => String::new(), - } - } - - pub fn host_port(&self) -> String { - match (self.url.host(), self.url.port()) { - (Some(host), Some(port)) => format!("{}:{}", host, port), - (Some(host), None) => format!("{}", host), - _ => String::new(), - } - } - - pub fn get_file_path(&self) -> &str { - let path = self.url.path(); - - #[cfg(windows)] - let path = &path[1..]; - - path - } -} - -/// parse a file path into an URL. -fn url_parse_from_file_path(value: &str) -> std::io::Result { - // Only check if the arg is an ip address and ask for scheme since its absent. - // localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as - // /mnt/export1. So we go ahead and start the rustfs server in FS modes in these cases. - let addr: Vec<&str> = value.splitn(2, '/').collect(); - if is_socket_addr(addr[0]) { - return Err(std::io::Error::other("invalid URL endpoint format: missing scheme http or https")); - } - - let file_path = match Path::new(value).absolutize() { - Ok(path) => path, - Err(err) => return Err(std::io::Error::other(format!("absolute path failed: {}", err))), - }; - - match Url::from_file_path(file_path) { - Ok(url) => Ok(url), - Err(_) => Err(std::io::Error::other("Convert a file path into an URL failed")), - } -} - -#[cfg(test)] -mod test { - - use super::*; - - #[test] - fn test_new_endpoint() { - #[derive(Default)] - struct TestCase<'a> { - arg: &'a str, - expected_endpoint: Option, - expected_type: Option, - expected_err: Option, - } - - let u2 = url::Url::parse("https://example.org/path").unwrap(); - let u4 = url::Url::parse("http://192.168.253.200/path").unwrap(); - let u6 = url::Url::parse("http://server:/path").unwrap(); - let root_slash_foo = url::Url::from_file_path("/foo").unwrap(); - - let test_cases = [ - TestCase { - arg: "/foo", - expected_endpoint: Some(Endpoint { - url: root_slash_foo, - is_local: true, - pool_idx: -1, - set_idx: -1, - disk_idx: -1, - }), - expected_type: Some(EndpointType::Path), - expected_err: None, - }, - TestCase { - arg: "https://example.org/path", - expected_endpoint: Some(Endpoint { - url: u2, - is_local: false, - pool_idx: -1, - set_idx: -1, - disk_idx: -1, - }), - expected_type: Some(EndpointType::Url), - expected_err: None, - }, - TestCase { - arg: "http://192.168.253.200/path", - expected_endpoint: Some(Endpoint { - url: u4, - is_local: false, - pool_idx: -1, - set_idx: -1, - disk_idx: -1, - }), - expected_type: Some(EndpointType::Url), - expected_err: None, - }, - TestCase { - arg: "", - expected_endpoint: None, - expected_type: None, - expected_err: Some(std::io::Error::other("empty or root endpoint is not supported")), - }, - TestCase { - arg: "/", - expected_endpoint: None, - expected_type: None, - expected_err: Some(std::io::Error::other("empty or root endpoint is not supported")), - }, - TestCase { - arg: "\\", - expected_endpoint: None, - expected_type: None, - expected_err: Some(std::io::Error::other("empty or root endpoint is not supported")), - }, - TestCase { - arg: "c://foo", - expected_endpoint: None, - expected_type: None, - expected_err: Some(std::io::Error::other("invalid URL endpoint format")), - }, - TestCase { - arg: "ftp://foo", - expected_endpoint: None, - expected_type: None, - expected_err: Some(std::io::Error::other("invalid URL endpoint format")), - }, - TestCase { - arg: "http://server/path?location", - expected_endpoint: None, - expected_type: None, - expected_err: Some(std::io::Error::other("invalid URL endpoint format")), - }, - TestCase { - arg: "http://:/path", - expected_endpoint: None, - expected_type: None, - expected_err: Some(std::io::Error::other("invalid URL endpoint format: empty host name")), - }, - TestCase { - arg: "http://:8080/path", - expected_endpoint: None, - expected_type: None, - expected_err: Some(std::io::Error::other("invalid URL endpoint format: empty host name")), - }, - TestCase { - arg: "http://server:/path", - expected_endpoint: Some(Endpoint { - url: u6, - is_local: false, - pool_idx: -1, - set_idx: -1, - disk_idx: -1, - }), - expected_type: Some(EndpointType::Url), - expected_err: None, - }, - TestCase { - arg: "https://93.184.216.34:808080/path", - expected_endpoint: None, - expected_type: None, - expected_err: Some(std::io::Error::other( - "invalid URL endpoint format: port number must be between 1 to 65535", - )), - }, - TestCase { - arg: "http://server:8080//", - expected_endpoint: None, - expected_type: None, - expected_err: Some(std::io::Error::other("empty or root path is not supported in URL endpoint")), - }, - TestCase { - arg: "http://server:8080/", - expected_endpoint: None, - expected_type: None, - expected_err: Some(std::io::Error::other("empty or root path is not supported in URL endpoint")), - }, - TestCase { - arg: "192.168.1.210:9000", - expected_endpoint: None, - expected_type: None, - expected_err: Some(std::io::Error::other("invalid URL endpoint format: missing scheme http or https")), - }, - ]; - - for test_case in test_cases { - let ret = Endpoint::try_from(test_case.arg); - if test_case.expected_err.is_none() && ret.is_err() { - panic!("{}: error: expected = , got = {:?}", test_case.arg, ret); - } - if test_case.expected_err.is_some() && ret.is_ok() { - panic!("{}: error: expected = {:?}, got = ", test_case.arg, test_case.expected_err); - } - match (test_case.expected_err, ret) { - (None, Err(e)) => panic!("{}: error: expected = , got = {}", test_case.arg, e), - (None, Ok(mut ep)) => { - let _ = ep.update_is_local(9000); - if test_case.expected_type != Some(ep.get_type()) { - panic!( - "{}: type: expected = {:?}, got = {:?}", - test_case.arg, - test_case.expected_type, - ep.get_type() - ); - } - - assert_eq!(test_case.expected_endpoint, Some(ep), "{}: endpoint", test_case.arg); - } - (Some(e), Ok(_)) => panic!("{}: error: expected = {}, got = ", test_case.arg, e), - (Some(e), Err(e2)) => { - assert_eq!(e.to_string(), e2.to_string(), "{}: error: expected = {}, got = {}", test_case.arg, e, e2) - } - } - } - } -} diff --git a/crates/disk/src/error.rs b/crates/disk/src/error.rs deleted file mode 100644 index 3f365a88f..000000000 --- a/crates/disk/src/error.rs +++ /dev/null @@ -1,594 +0,0 @@ -use std::io::{self, ErrorKind}; -use std::path::PathBuf; - -use tracing::error; - -use crate::quorum::CheckErrorFn; -use crate::utils::ERROR_TYPE_MASK; -use common::error::{Error, Result}; - -// DiskError == StorageErr -#[derive(Debug, thiserror::Error)] -pub enum DiskError { - #[error("maximum versions exceeded, please delete few versions to proceed")] - MaxVersionsExceeded, - - #[error("unexpected error")] - Unexpected, - - #[error("corrupted format")] - CorruptedFormat, - - #[error("corrupted backend")] - CorruptedBackend, - - #[error("unformatted disk error")] - UnformattedDisk, - - #[error("inconsistent drive found")] - InconsistentDisk, - - #[error("drive does not support O_DIRECT")] - UnsupportedDisk, - - #[error("drive path full")] - DiskFull, - - #[error("disk not a dir")] - DiskNotDir, - - #[error("disk not found")] - DiskNotFound, - - #[error("drive still did not complete the request")] - DiskOngoingReq, - - #[error("drive is part of root drive, will not be used")] - DriveIsRoot, - - #[error("remote drive is faulty")] - FaultyRemoteDisk, - - #[error("drive is faulty")] - FaultyDisk, - - #[error("drive access denied")] - DiskAccessDenied, - - #[error("file not found")] - FileNotFound, - - #[error("file version not found")] - FileVersionNotFound, - - #[error("too many open files, please increase 'ulimit -n'")] - TooManyOpenFiles, - - #[error("file name too long")] - FileNameTooLong, - - #[error("volume already exists")] - VolumeExists, - - #[error("not of regular file type")] - IsNotRegular, - - #[error("path not found")] - PathNotFound, - - #[error("volume not found")] - VolumeNotFound, - - #[error("volume is not empty")] - VolumeNotEmpty, - - #[error("volume access denied")] - VolumeAccessDenied, - - #[error("disk access denied")] - FileAccessDenied, - - #[error("file is corrupted")] - FileCorrupt, - - #[error("bit-rot hash algorithm is invalid")] - BitrotHashAlgoInvalid, - - #[error("Rename across devices not allowed, please fix your backend configuration")] - CrossDeviceLink, - - #[error("less data available than what was requested")] - LessData, - - #[error("more data was sent than what was advertised")] - MoreData, - - #[error("outdated XL meta")] - OutdatedXLMeta, - - #[error("part missing or corrupt")] - PartMissingOrCorrupt, - - #[error("No healing is required")] - NoHealRequired, -} - -impl DiskError { - /// Checks if the given array of errors contains fatal disk errors. - /// If all errors are of the same fatal disk error type, returns the corresponding error. - /// Otherwise, returns Ok. - /// - /// # Parameters - /// - `errs`: A slice of optional errors. - /// - /// # Returns - /// If all errors are of the same fatal disk error type, returns the corresponding error. - /// Otherwise, returns Ok. - pub fn check_disk_fatal_errs(errs: &[Option]) -> Result<()> { - if DiskError::UnsupportedDisk.count_errs(errs) == errs.len() { - return Err(DiskError::UnsupportedDisk.into()); - } - - if DiskError::FileAccessDenied.count_errs(errs) == errs.len() { - return Err(DiskError::FileAccessDenied.into()); - } - - if DiskError::DiskNotDir.count_errs(errs) == errs.len() { - return Err(DiskError::DiskNotDir.into()); - } - - Ok(()) - } - - pub fn count_errs(&self, errs: &[Option]) -> usize { - errs.iter() - .filter(|&err| match err { - None => false, - Some(e) => self.is(e), - }) - .count() - } - - pub fn quorum_unformatted_disks(errs: &[Option]) -> bool { - DiskError::UnformattedDisk.count_errs(errs) > (errs.len() / 2) - } - - pub fn should_init_erasure_disks(errs: &[Option]) -> bool { - DiskError::UnformattedDisk.count_errs(errs) == errs.len() - } - - /// Check if the error is a disk error - pub fn is(&self, err: &Error) -> bool { - if let Some(e) = err.downcast_ref::() { - e == self - } else { - false - } - } -} - -impl DiskError { - pub fn to_u32(&self) -> u32 { - match self { - DiskError::MaxVersionsExceeded => 0x01, - DiskError::Unexpected => 0x02, - DiskError::CorruptedFormat => 0x03, - DiskError::CorruptedBackend => 0x04, - DiskError::UnformattedDisk => 0x05, - DiskError::InconsistentDisk => 0x06, - DiskError::UnsupportedDisk => 0x07, - DiskError::DiskFull => 0x08, - DiskError::DiskNotDir => 0x09, - DiskError::DiskNotFound => 0x0A, - DiskError::DiskOngoingReq => 0x0B, - DiskError::DriveIsRoot => 0x0C, - DiskError::FaultyRemoteDisk => 0x0D, - DiskError::FaultyDisk => 0x0E, - DiskError::DiskAccessDenied => 0x0F, - DiskError::FileNotFound => 0x10, - DiskError::FileVersionNotFound => 0x11, - DiskError::TooManyOpenFiles => 0x12, - DiskError::FileNameTooLong => 0x13, - DiskError::VolumeExists => 0x14, - DiskError::IsNotRegular => 0x15, - DiskError::PathNotFound => 0x16, - DiskError::VolumeNotFound => 0x17, - DiskError::VolumeNotEmpty => 0x18, - DiskError::VolumeAccessDenied => 0x19, - DiskError::FileAccessDenied => 0x1A, - DiskError::FileCorrupt => 0x1B, - DiskError::BitrotHashAlgoInvalid => 0x1C, - DiskError::CrossDeviceLink => 0x1D, - DiskError::LessData => 0x1E, - DiskError::MoreData => 0x1F, - DiskError::OutdatedXLMeta => 0x20, - DiskError::PartMissingOrCorrupt => 0x21, - DiskError::NoHealRequired => 0x22, - } - } - - pub fn from_u32(error: u32) -> Option { - match error & ERROR_TYPE_MASK { - 0x01 => Some(DiskError::MaxVersionsExceeded), - 0x02 => Some(DiskError::Unexpected), - 0x03 => Some(DiskError::CorruptedFormat), - 0x04 => Some(DiskError::CorruptedBackend), - 0x05 => Some(DiskError::UnformattedDisk), - 0x06 => Some(DiskError::InconsistentDisk), - 0x07 => Some(DiskError::UnsupportedDisk), - 0x08 => Some(DiskError::DiskFull), - 0x09 => Some(DiskError::DiskNotDir), - 0x0A => Some(DiskError::DiskNotFound), - 0x0B => Some(DiskError::DiskOngoingReq), - 0x0C => Some(DiskError::DriveIsRoot), - 0x0D => Some(DiskError::FaultyRemoteDisk), - 0x0E => Some(DiskError::FaultyDisk), - 0x0F => Some(DiskError::DiskAccessDenied), - 0x10 => Some(DiskError::FileNotFound), - 0x11 => Some(DiskError::FileVersionNotFound), - 0x12 => Some(DiskError::TooManyOpenFiles), - 0x13 => Some(DiskError::FileNameTooLong), - 0x14 => Some(DiskError::VolumeExists), - 0x15 => Some(DiskError::IsNotRegular), - 0x16 => Some(DiskError::PathNotFound), - 0x17 => Some(DiskError::VolumeNotFound), - 0x18 => Some(DiskError::VolumeNotEmpty), - 0x19 => Some(DiskError::VolumeAccessDenied), - 0x1A => Some(DiskError::FileAccessDenied), - 0x1B => Some(DiskError::FileCorrupt), - 0x1C => Some(DiskError::BitrotHashAlgoInvalid), - 0x1D => Some(DiskError::CrossDeviceLink), - 0x1E => Some(DiskError::LessData), - 0x1F => Some(DiskError::MoreData), - 0x20 => Some(DiskError::OutdatedXLMeta), - 0x21 => Some(DiskError::PartMissingOrCorrupt), - 0x22 => Some(DiskError::NoHealRequired), - _ => None, - } - } -} - -impl PartialEq for DiskError { - fn eq(&self, other: &Self) -> bool { - core::mem::discriminant(self) == core::mem::discriminant(other) - } -} - -impl CheckErrorFn for DiskError { - fn is(&self, e: &Error) -> bool { - self.is(e) - } -} - -pub fn clone_disk_err(e: &DiskError) -> Error { - match e { - DiskError::MaxVersionsExceeded => Error::new(DiskError::MaxVersionsExceeded), - DiskError::Unexpected => Error::new(DiskError::Unexpected), - DiskError::CorruptedFormat => Error::new(DiskError::CorruptedFormat), - DiskError::CorruptedBackend => Error::new(DiskError::CorruptedBackend), - DiskError::UnformattedDisk => Error::new(DiskError::UnformattedDisk), - DiskError::InconsistentDisk => Error::new(DiskError::InconsistentDisk), - DiskError::UnsupportedDisk => Error::new(DiskError::UnsupportedDisk), - DiskError::DiskFull => Error::new(DiskError::DiskFull), - DiskError::DiskNotDir => Error::new(DiskError::DiskNotDir), - DiskError::DiskNotFound => Error::new(DiskError::DiskNotFound), - DiskError::DiskOngoingReq => Error::new(DiskError::DiskOngoingReq), - DiskError::DriveIsRoot => Error::new(DiskError::DriveIsRoot), - DiskError::FaultyRemoteDisk => Error::new(DiskError::FaultyRemoteDisk), - DiskError::FaultyDisk => Error::new(DiskError::FaultyDisk), - DiskError::DiskAccessDenied => Error::new(DiskError::DiskAccessDenied), - DiskError::FileNotFound => Error::new(DiskError::FileNotFound), - DiskError::FileVersionNotFound => Error::new(DiskError::FileVersionNotFound), - DiskError::TooManyOpenFiles => Error::new(DiskError::TooManyOpenFiles), - DiskError::FileNameTooLong => Error::new(DiskError::FileNameTooLong), - DiskError::VolumeExists => Error::new(DiskError::VolumeExists), - DiskError::IsNotRegular => Error::new(DiskError::IsNotRegular), - DiskError::PathNotFound => Error::new(DiskError::PathNotFound), - DiskError::VolumeNotFound => Error::new(DiskError::VolumeNotFound), - DiskError::VolumeNotEmpty => Error::new(DiskError::VolumeNotEmpty), - DiskError::VolumeAccessDenied => Error::new(DiskError::VolumeAccessDenied), - DiskError::FileAccessDenied => Error::new(DiskError::FileAccessDenied), - DiskError::FileCorrupt => Error::new(DiskError::FileCorrupt), - DiskError::BitrotHashAlgoInvalid => Error::new(DiskError::BitrotHashAlgoInvalid), - DiskError::CrossDeviceLink => Error::new(DiskError::CrossDeviceLink), - DiskError::LessData => Error::new(DiskError::LessData), - DiskError::MoreData => Error::new(DiskError::MoreData), - DiskError::OutdatedXLMeta => Error::new(DiskError::OutdatedXLMeta), - DiskError::PartMissingOrCorrupt => Error::new(DiskError::PartMissingOrCorrupt), - DiskError::NoHealRequired => Error::new(DiskError::NoHealRequired), - } -} - -pub fn os_err_to_file_err(e: io::Error) -> Error { - match e.kind() { - io::ErrorKind::NotFound => Error::new(DiskError::FileNotFound), - io::ErrorKind::PermissionDenied => Error::new(DiskError::FileAccessDenied), - // io::ErrorKind::ConnectionRefused => todo!(), - // io::ErrorKind::ConnectionReset => todo!(), - // io::ErrorKind::HostUnreachable => todo!(), - // io::ErrorKind::NetworkUnreachable => todo!(), - // io::ErrorKind::ConnectionAborted => todo!(), - // io::ErrorKind::NotConnected => todo!(), - // io::ErrorKind::AddrInUse => todo!(), - // io::ErrorKind::AddrNotAvailable => todo!(), - // io::ErrorKind::NetworkDown => todo!(), - // io::ErrorKind::BrokenPipe => todo!(), - // io::ErrorKind::AlreadyExists => todo!(), - // io::ErrorKind::WouldBlock => todo!(), - // io::ErrorKind::NotADirectory => DiskError::FileNotFound, - // io::ErrorKind::IsADirectory => DiskError::FileNotFound, - // io::ErrorKind::DirectoryNotEmpty => DiskError::VolumeNotEmpty, - // io::ErrorKind::ReadOnlyFilesystem => todo!(), - // io::ErrorKind::FilesystemLoop => todo!(), - // io::ErrorKind::StaleNetworkFileHandle => todo!(), - // io::ErrorKind::InvalidInput => todo!(), - // io::ErrorKind::InvalidData => todo!(), - // io::ErrorKind::TimedOut => todo!(), - // io::ErrorKind::WriteZero => todo!(), - // io::ErrorKind::StorageFull => DiskError::DiskFull, - // io::ErrorKind::NotSeekable => todo!(), - // io::ErrorKind::FilesystemQuotaExceeded => todo!(), - // io::ErrorKind::FileTooLarge => todo!(), - // io::ErrorKind::ResourceBusy => todo!(), - // io::ErrorKind::ExecutableFileBusy => todo!(), - // io::ErrorKind::Deadlock => todo!(), - // io::ErrorKind::CrossesDevices => todo!(), - // io::ErrorKind::TooManyLinks =>DiskError::TooManyOpenFiles, - // io::ErrorKind::InvalidFilename => todo!(), - // io::ErrorKind::ArgumentListTooLong => todo!(), - // io::ErrorKind::Interrupted => todo!(), - // io::ErrorKind::Unsupported => todo!(), - // io::ErrorKind::UnexpectedEof => todo!(), - // io::ErrorKind::OutOfMemory => todo!(), - // io::ErrorKind::Other => todo!(), - // TODO: 把不支持的king用字符串处理 - _ => Error::new(e), - } -} - -#[derive(Debug, thiserror::Error)] -pub struct FileAccessDeniedWithContext { - pub path: PathBuf, - #[source] - pub source: std::io::Error, -} - -impl std::fmt::Display for FileAccessDeniedWithContext { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "访问文件 '{}' 被拒绝: {}", self.path.display(), self.source) - } -} - -pub fn is_unformatted_disk(err: &Error) -> bool { - matches!(err.downcast_ref::(), Some(DiskError::UnformattedDisk)) -} - -pub fn is_err_file_not_found(err: &Error) -> bool { - if let Some(ioerr) = err.downcast_ref::() { - return ioerr.kind() == ErrorKind::NotFound; - } - - matches!(err.downcast_ref::(), Some(DiskError::FileNotFound)) -} - -pub fn is_err_file_version_not_found(err: &Error) -> bool { - matches!(err.downcast_ref::(), Some(DiskError::FileVersionNotFound)) -} - -pub fn is_err_volume_not_found(err: &Error) -> bool { - matches!(err.downcast_ref::(), Some(DiskError::VolumeNotFound)) -} - -pub fn is_err_eof(err: &Error) -> bool { - if let Some(ioerr) = err.downcast_ref::() { - return ioerr.kind() == ErrorKind::UnexpectedEof; - } - false -} - -pub fn is_sys_err_no_space(e: &io::Error) -> bool { - if let Some(no) = e.raw_os_error() { - return no == 28; - } - false -} - -pub fn is_sys_err_invalid_arg(e: &io::Error) -> bool { - if let Some(no) = e.raw_os_error() { - return no == 22; - } - false -} - -// TODO: ?? -pub fn is_sys_err_io(e: &io::Error) -> bool { - if let Some(no) = e.raw_os_error() { - return no == 5; - } - false -} - -pub fn is_sys_err_is_dir(e: &io::Error) -> bool { - if let Some(no) = e.raw_os_error() { - return no == 21; - } - false -} - -pub fn is_sys_err_not_dir(e: &io::Error) -> bool { - if let Some(no) = e.raw_os_error() { - return no == 20; - } - false -} - -pub fn is_sys_err_too_long(e: &io::Error) -> bool { - if let Some(no) = e.raw_os_error() { - return no == 63; - } - false -} - -pub fn is_sys_err_too_many_symlinks(e: &io::Error) -> bool { - if let Some(no) = e.raw_os_error() { - return no == 62; - } - false -} - -pub fn is_sys_err_not_empty(e: &io::Error) -> bool { - if let Some(no) = e.raw_os_error() { - if no == 66 { - return true; - } - - if cfg!(target_os = "solaris") && no == 17 { - return true; - } - - if cfg!(target_os = "windows") && no == 145 { - return true; - } - } - false -} - -pub fn is_sys_err_path_not_found(e: &io::Error) -> bool { - if let Some(no) = e.raw_os_error() { - if cfg!(target_os = "windows") { - if no == 3 { - return true; - } - } else if no == 2 { - return true; - } - } - false -} - -pub fn is_sys_err_handle_invalid(e: &io::Error) -> bool { - if let Some(no) = e.raw_os_error() { - if cfg!(target_os = "windows") { - if no == 6 { - return true; - } - } else { - return false; - } - } - false -} - -pub fn is_sys_err_cross_device(e: &io::Error) -> bool { - if let Some(no) = e.raw_os_error() { - return no == 18; - } - false -} - -pub fn is_sys_err_too_many_files(e: &io::Error) -> bool { - if let Some(no) = e.raw_os_error() { - return no == 23 || no == 24; - } - false -} - -// pub fn os_is_not_exist(e: &io::Error) -> bool { -// e.kind() == ErrorKind::NotFound -// } - -pub fn os_is_permission(e: &io::Error) -> bool { - if e.kind() == ErrorKind::PermissionDenied { - return true; - } - if let Some(no) = e.raw_os_error() { - if no == 30 { - return true; - } - } - - false -} - -// pub fn os_is_exist(e: &io::Error) -> bool { -// e.kind() == ErrorKind::AlreadyExists -// } - -// // map_err_not_exists -// pub fn map_err_not_exists(e: io::Error) -> Error { -// if os_is_not_exist(&e) { -// return Error::new(DiskError::VolumeNotEmpty); -// } else if is_sys_err_io(&e) { -// return Error::new(DiskError::FaultyDisk); -// } - -// Error::new(e) -// } - -// pub fn convert_access_error(e: io::Error, per_err: DiskError) -> Error { -// if os_is_not_exist(&e) { -// return Error::new(DiskError::VolumeNotEmpty); -// } else if is_sys_err_io(&e) { -// return Error::new(DiskError::FaultyDisk); -// } else if os_is_permission(&e) { -// return Error::new(per_err); -// } - -// Error::new(e) -// } - -pub fn is_all_not_found(errs: &[Option]) -> bool { - for err in errs.iter() { - if let Some(err) = err { - if let Some(err) = err.downcast_ref::() { - match err { - DiskError::FileNotFound | DiskError::VolumeNotFound | &DiskError::FileVersionNotFound => { - continue; - } - _ => return false, - } - } - } - return false; - } - - !errs.is_empty() -} - -pub fn is_all_volume_not_found(errs: &[Option]) -> bool { - DiskError::VolumeNotFound.count_errs(errs) == errs.len() -} - -pub fn is_all_buckets_not_found(errs: &[Option]) -> bool { - if errs.is_empty() { - return false; - } - let mut not_found_count = 0; - for err in errs.iter().flatten() { - match err.downcast_ref() { - Some(DiskError::VolumeNotFound) | Some(DiskError::DiskNotFound) => { - not_found_count += 1; - } - _ => {} - } - } - errs.len() == not_found_count -} - -pub fn is_err_os_not_exist(err: &Error) -> bool { - if let Some(os_err) = err.downcast_ref::() { - os_err.kind() == ErrorKind::NotFound - } else { - false - } -} - -pub fn is_err_os_disk_full(err: &Error) -> bool { - if let Some(os_err) = err.downcast_ref::() { - is_sys_err_no_space(os_err) - } else if let Some(e) = err.downcast_ref::() { - e == &DiskError::DiskFull - } else { - false - } -} diff --git a/crates/disk/src/format.rs b/crates/disk/src/format.rs deleted file mode 100644 index bc20eed6a..000000000 --- a/crates/disk/src/format.rs +++ /dev/null @@ -1,273 +0,0 @@ -// use super::{error::DiskError, DiskInfo}; -use rustfs_error::{Error, Result}; -use serde::{Deserialize, Serialize}; -use serde_json::Error as JsonError; -use uuid::Uuid; - -use crate::api::DiskInfo; - -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub enum FormatMetaVersion { - #[serde(rename = "1")] - V1, - - #[serde(other)] - Unknown, -} - -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub enum FormatBackend { - #[serde(rename = "xl")] - Erasure, - #[serde(rename = "xl-single")] - ErasureSingle, - - #[serde(other)] - Unknown, -} - -/// Represents the V3 backend disk structure version -/// under `.rustfs.sys` and actual data namespace. -/// -/// FormatErasureV3 - structure holds format config version '3'. -/// -/// The V3 format to support "large bucket" support where a bucket -/// can span multiple erasure sets. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct FormatErasureV3 { - /// Version of 'xl' format. - pub version: FormatErasureVersion, - - /// This field carries assigned disk uuid. - pub this: Uuid, - - /// Sets field carries the input disk order generated the first - /// time when fresh disks were supplied, it is a two dimensional - /// array second dimension represents list of disks used per set. - pub sets: Vec>, - - /// Distribution algorithm represents the hashing algorithm - /// to pick the right set index for an object. - #[serde(rename = "distributionAlgo")] - pub distribution_algo: DistributionAlgoVersion, -} - -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub enum FormatErasureVersion { - #[serde(rename = "1")] - V1, - #[serde(rename = "2")] - V2, - #[serde(rename = "3")] - V3, - - #[serde(other)] - Unknown, -} - -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub enum DistributionAlgoVersion { - #[serde(rename = "CRCMOD")] - V1, - #[serde(rename = "SIPMOD")] - V2, - #[serde(rename = "SIPMOD+PARITY")] - V3, -} - -/// format.json currently has the format: -/// -/// ```json -/// { -/// "version": "1", -/// "format": "XXXXX", -/// "id": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX", -/// "XXXXX": { -// -/// } -/// } -/// ``` -/// -/// Ideally we will never have a situation where we will have to change the -/// fields of this struct and deal with related migration. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct FormatV3 { - /// Version of the format config. - pub version: FormatMetaVersion, - - /// Format indicates the backend format type, supports two values 'xl' and 'xl-single'. - pub format: FormatBackend, - - /// ID is the identifier for the rustfs deployment - pub id: Uuid, - - #[serde(rename = "xl")] - pub erasure: FormatErasureV3, - // /// DiskInfo is an extended type which returns current - // /// disk usage per path. - #[serde(skip)] - pub disk_info: Option, -} - -impl TryFrom<&[u8]> for FormatV3 { - type Error = JsonError; - - fn try_from(data: &[u8]) -> core::result::Result { - serde_json::from_slice(data) - } -} - -impl TryFrom<&str> for FormatV3 { - type Error = JsonError; - - fn try_from(data: &str) -> core::result::Result { - serde_json::from_str(data) - } -} - -impl FormatV3 { - /// Create a new format config with the given number of sets and set length. - pub fn new(num_sets: usize, set_len: usize) -> Self { - let format = if set_len == 1 { - FormatBackend::ErasureSingle - } else { - FormatBackend::Erasure - }; - - let erasure = FormatErasureV3 { - version: FormatErasureVersion::V3, - this: Uuid::nil(), - sets: (0..num_sets) - .map(|_| (0..set_len).map(|_| Uuid::new_v4()).collect()) - .collect(), - distribution_algo: DistributionAlgoVersion::V3, - }; - - Self { - version: FormatMetaVersion::V1, - format, - id: Uuid::new_v4(), - erasure, - disk_info: None, - } - } - - /// Returns the number of drives in the erasure set. - pub fn drives(&self) -> usize { - self.erasure.sets.iter().map(|v| v.len()).sum() - } - - pub fn to_json(&self) -> Result { - Ok(serde_json::to_string(self)?) - } - - /// returns the i,j'th position of the input `diskID` against the reference - /// - /// format, after successful validation. - /// - i'th position is the set index - /// - j'th position is the disk index in the current set - pub fn find_disk_index_by_disk_id(&self, disk_id: Uuid) -> Result<(usize, usize)> { - if disk_id == Uuid::nil() { - return Err(Error::DiskNotFound); - } - if disk_id == Uuid::max() { - return Err(Error::msg("disk offline")); - } - - for (i, set) in self.erasure.sets.iter().enumerate() { - for (j, d) in set.iter().enumerate() { - if disk_id.eq(d) { - return Ok((i, j)); - } - } - } - - Err(Error::msg(format!("disk id not found {}", disk_id))) - } - - pub fn check_other(&self, other: &FormatV3) -> Result<()> { - let mut tmp = other.clone(); - let this = tmp.erasure.this; - tmp.erasure.this = Uuid::nil(); - - if self.erasure.sets.len() != other.erasure.sets.len() { - return Err(Error::msg(format!( - "Expected number of sets {}, got {}", - self.erasure.sets.len(), - other.erasure.sets.len() - ))); - } - - for i in 0..self.erasure.sets.len() { - if self.erasure.sets[i].len() != other.erasure.sets[i].len() { - return Err(Error::msg(format!( - "Each set should be of same size, expected {}, got {}", - self.erasure.sets[i].len(), - other.erasure.sets[i].len() - ))); - } - - for j in 0..self.erasure.sets[i].len() { - if self.erasure.sets[i][j] != other.erasure.sets[i][j] { - return Err(Error::msg(format!( - "UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)", - i, - j, - self.erasure.sets[i][j].to_string(), - other.erasure.sets[i][j].to_string(), - ))); - } - } - } - - for i in 0..tmp.erasure.sets.len() { - for j in 0..tmp.erasure.sets[i].len() { - if this == tmp.erasure.sets[i][j] { - return Ok(()); - } - } - } - - Err(Error::msg(format!( - "DriveID {:?} not found in any drive sets {:?}", - this, other.erasure.sets - ))) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_format_v1() { - let format = FormatV3::new(1, 4); - - let str = serde_json::to_string(&format); - println!("{:?}", str); - - let data = r#" - { - "version": "1", - "format": "xl", - "id": "321b3874-987d-4c15-8fa5-757c956b1243", - "xl": { - "version": "1", - "this": null, - "sets": [ - [ - "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5", - "c26315da-05cf-4778-a9ea-b44ea09f58c5", - "fb87a891-18d3-44cf-a46f-bcc15093a038", - "356a925c-57b9-4313-88b3-053edf1104dc" - ] - ], - "distributionAlgo": "CRCMOD" - } - }"#; - - let p = FormatV3::try_from(data); - - println!("{:?}", p); - } -} diff --git a/crates/disk/src/fs.rs b/crates/disk/src/fs.rs deleted file mode 100644 index d8110ca63..000000000 --- a/crates/disk/src/fs.rs +++ /dev/null @@ -1,179 +0,0 @@ -use std::{fs::Metadata, path::Path}; - -use tokio::{ - fs::{self, File}, - io, -}; - -#[cfg(not(windows))] -pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { - use std::os::unix::fs::MetadataExt; - - if f1.dev() != f2.dev() { - return false; - } - - if f1.ino() != f2.ino() { - return false; - } - - if f1.size() != f2.size() { - return false; - } - if f1.permissions() != f2.permissions() { - return false; - } - - if f1.mtime() != f2.mtime() { - return false; - } - - true -} - -#[cfg(windows)] -pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { - if f1.permissions() != f2.permissions() { - return false; - } - - if f1.file_type() != f2.file_type() { - return false; - } - - if f1.len() != f2.len() { - return false; - } - true -} - -type FileMode = usize; - -pub const O_RDONLY: FileMode = 0x00000; -pub const O_WRONLY: FileMode = 0x00001; -pub const O_RDWR: FileMode = 0x00002; -pub const O_CREATE: FileMode = 0x00040; -// pub const O_EXCL: FileMode = 0x00080; -// pub const O_NOCTTY: FileMode = 0x00100; -pub const O_TRUNC: FileMode = 0x00200; -// pub const O_NONBLOCK: FileMode = 0x00800; -pub const O_APPEND: FileMode = 0x00400; -// pub const O_SYNC: FileMode = 0x01000; -// pub const O_ASYNC: FileMode = 0x02000; -// pub const O_CLOEXEC: FileMode = 0x80000; - -// read: bool, -// write: bool, -// append: bool, -// truncate: bool, -// create: bool, -// create_new: bool, - -pub async fn open_file(path: impl AsRef, mode: FileMode) -> io::Result { - let mut opts = fs::OpenOptions::new(); - - match mode & (O_RDONLY | O_WRONLY | O_RDWR) { - O_RDONLY => { - opts.read(true); - } - O_WRONLY => { - opts.write(true); - } - O_RDWR => { - opts.read(true); - opts.write(true); - } - _ => (), - }; - - if mode & O_CREATE != 0 { - opts.create(true); - } - - if mode & O_APPEND != 0 { - opts.append(true); - } - - if mode & O_TRUNC != 0 { - opts.truncate(true); - } - - opts.open(path.as_ref()).await -} - -pub async fn access(path: impl AsRef) -> io::Result<()> { - fs::metadata(path).await?; - Ok(()) -} - -pub fn access_std(path: impl AsRef) -> io::Result<()> { - std::fs::metadata(path)?; - Ok(()) -} - -pub async fn lstat(path: impl AsRef) -> io::Result { - fs::metadata(path).await -} - -pub fn lstat_std(path: impl AsRef) -> io::Result { - std::fs::metadata(path) -} - -pub async fn make_dir_all(path: impl AsRef) -> io::Result<()> { - fs::create_dir_all(path.as_ref()).await -} - -#[tracing::instrument(level = "debug", skip_all)] -pub async fn remove(path: impl AsRef) -> io::Result<()> { - let meta = fs::metadata(path.as_ref()).await?; - if meta.is_dir() { - fs::remove_dir(path.as_ref()).await - } else { - fs::remove_file(path.as_ref()).await - } -} - -pub async fn remove_all(path: impl AsRef) -> io::Result<()> { - let meta = fs::metadata(path.as_ref()).await?; - if meta.is_dir() { - fs::remove_dir_all(path.as_ref()).await - } else { - fs::remove_file(path.as_ref()).await - } -} - -#[tracing::instrument(level = "debug", skip_all)] -pub fn remove_std(path: impl AsRef) -> io::Result<()> { - let meta = std::fs::metadata(path.as_ref())?; - if meta.is_dir() { - std::fs::remove_dir(path.as_ref()) - } else { - std::fs::remove_file(path.as_ref()) - } -} - -pub fn remove_all_std(path: impl AsRef) -> io::Result<()> { - let meta = std::fs::metadata(path.as_ref())?; - if meta.is_dir() { - std::fs::remove_dir_all(path.as_ref()) - } else { - std::fs::remove_file(path.as_ref()) - } -} - -pub async fn mkdir(path: impl AsRef) -> io::Result<()> { - fs::create_dir(path.as_ref()).await -} - -pub async fn rename(from: impl AsRef, to: impl AsRef) -> io::Result<()> { - fs::rename(from, to).await -} - -pub fn rename_std(from: impl AsRef, to: impl AsRef) -> io::Result<()> { - std::fs::rename(from, to) -} - -#[tracing::instrument(level = "debug", skip_all)] -pub async fn read_file(path: impl AsRef) -> io::Result> { - fs::read(path.as_ref()).await -} diff --git a/crates/disk/src/lib.rs b/crates/disk/src/lib.rs deleted file mode 100644 index 32bd48143..000000000 --- a/crates/disk/src/lib.rs +++ /dev/null @@ -1,12 +0,0 @@ -pub mod endpoint; -// pub mod error; -pub mod format; -pub mod fs; -pub mod local; -// pub mod metacache; -pub mod api; -pub mod local_list; -pub mod os; -pub mod path; -pub mod remote; -pub mod utils; diff --git a/crates/disk/src/local.rs b/crates/disk/src/local.rs deleted file mode 100644 index 9c632c4d5..000000000 --- a/crates/disk/src/local.rs +++ /dev/null @@ -1,2048 +0,0 @@ -use std::fs::Metadata; -use std::io::ErrorKind; -use std::io::SeekFrom; -use std::path::Path; -use std::path::PathBuf; -use std::sync::atomic::AtomicU32; -use std::sync::atomic::Ordering; -use std::sync::Arc; -use std::time::Duration; - -use crate::api::CheckPartsResp; -use crate::api::DeleteOptions; -use crate::api::DiskAPI; -use crate::api::DiskInfo; -use crate::api::DiskInfoOptions; -use crate::api::DiskLocation; -use crate::api::ReadMultipleReq; -use crate::api::ReadMultipleResp; -use crate::api::ReadOptions; -use crate::api::RenameDataResp; -use crate::api::UpdateMetadataOpts; -use crate::api::VolumeInfo; -use crate::api::WalkDirOptions; -use crate::api::BUCKET_META_PREFIX; -use crate::api::FORMAT_CONFIG_FILE; -use crate::api::RUSTFS_META_BUCKET; -use crate::api::RUSTFS_META_MULTIPART_BUCKET; -use crate::api::RUSTFS_META_TMP_BUCKET; -use crate::api::RUSTFS_META_TMP_DELETED_BUCKET; -use crate::api::STORAGE_FORMAT_FILE; -use crate::api::STORAGE_FORMAT_FILE_BACKUP; -use crate::endpoint::Endpoint; -use crate::format::FormatV3; -use crate::fs::access; -use crate::fs::lstat; -use crate::fs::lstat_std; -use crate::fs::remove_all_std; -use crate::fs::remove_std; -use crate::fs::O_APPEND; -use crate::fs::O_CREATE; -use crate::fs::O_RDONLY; -use crate::fs::O_TRUNC; -use crate::fs::O_WRONLY; -use crate::os::check_path_length; -use crate::os::is_root_disk; -use crate::os::rename_all; -use crate::path::has_suffix; -use crate::path::path_join_buf; -use crate::path::GLOBAL_DIR_SUFFIX; -use crate::path::SLASH_SEPARATOR; -use crate::utils::read_all; -use crate::utils::read_file_all; -use crate::utils::read_file_exists; -use madmin::DiskMetrics; -use path_absolutize::Absolutize as _; -use rustfs_error::conv_part_err_to_int; -use rustfs_error::to_access_error; -use rustfs_error::to_disk_error; -use rustfs_error::to_file_error; -use rustfs_error::to_unformatted_disk_error; -use rustfs_error::to_volume_error; -use rustfs_error::CHECK_PART_FILE_CORRUPT; -use rustfs_error::CHECK_PART_FILE_NOT_FOUND; -use rustfs_error::CHECK_PART_SUCCESS; -use rustfs_error::CHECK_PART_UNKNOWN; -use rustfs_error::CHECK_PART_VOLUME_NOT_FOUND; -use rustfs_error::{Error, Result}; -use rustfs_filemeta::get_file_info; -use rustfs_filemeta::read_xl_meta_no_data; -use rustfs_filemeta::FileInfo; -use rustfs_filemeta::FileInfoOpts; -use rustfs_filemeta::FileInfoVersions; -use rustfs_filemeta::FileMeta; -use rustfs_filemeta::RawFileInfo; -use rustfs_metacache::Cache; -use rustfs_metacache::MetaCacheEntry; -use rustfs_metacache::MetacacheWriter; -use rustfs_metacache::Opts; -use rustfs_metacache::UpdateFn; -use rustfs_rio::bitrot_verify; -use rustfs_utils::os::get_info; -use rustfs_utils::HashAlgorithm; -use time::OffsetDateTime; -use tokio::fs; -use tokio::fs::File; -use tokio::io::AsyncRead; -use tokio::io::AsyncReadExt as _; -use tokio::io::AsyncSeekExt as _; -use tokio::io::AsyncWrite; -use tokio::io::AsyncWriteExt as _; -use tokio::sync::RwLock; -use tracing::error; -use tracing::info; -use tracing::warn; -use uuid::Uuid; - -#[derive(Debug)] -pub struct FormatInfo { - pub id: Option, - pub data: Vec, - pub file_info: Option, - pub last_check: Option, -} - -impl FormatInfo { - pub fn last_check_valid(&self) -> bool { - let now = OffsetDateTime::now_utc(); - self.file_info.is_some() - && self.id.is_some() - && self.last_check.is_some() - && (now.unix_timestamp() - self.last_check.unwrap().unix_timestamp() <= 1) - } -} - -pub struct LocalDisk { - pub root: PathBuf, - pub format_path: PathBuf, - pub format_info: RwLock, - pub endpoint: Endpoint, - pub disk_info_cache: Arc>, - pub scanning: AtomicU32, - pub rotational: bool, - pub fstype: String, - pub major: u64, - pub minor: u64, - pub nrrequests: u64, -} - -impl std::fmt::Debug for LocalDisk { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("LocalDisk") - .field("root", &self.root) - .field("format_path", &self.format_path) - .field("format_info", &self.format_info) - .field("endpoint", &self.endpoint) - .finish() - } -} - -impl LocalDisk { - pub async fn new(ep: &Endpoint, cleanup: bool) -> Result { - let root = fs::canonicalize(ep.get_file_path()).await?; - - if cleanup { - // TODO: 删除 tmp 数据 - } - - let format_path = Path::new(RUSTFS_META_BUCKET) - .join(Path::new(FORMAT_CONFIG_FILE)) - .absolutize_virtually(&root)? - .into_owned(); - - let (format_data, format_meta) = read_file_exists(&format_path).await?; - - let mut id = None; - // let mut format_legacy = false; - let mut format_last_check = None; - - if !format_data.is_empty() { - let s = format_data.as_slice(); - let fm = FormatV3::try_from(s)?; - let (set_idx, disk_idx) = fm.find_disk_index_by_disk_id(fm.erasure.this)?; - - if set_idx as i32 != ep.set_idx || disk_idx as i32 != ep.disk_idx { - return Err(Error::InconsistentDisk); - } - - id = Some(fm.erasure.this); - // format_legacy = fm.erasure.distribution_algo == DistributionAlgoVersion::V1; - format_last_check = Some(OffsetDateTime::now_utc()); - } - - let format_info = FormatInfo { - id, - data: format_data, - file_info: format_meta, - last_check: format_last_check, - }; - let root_clone = root.clone(); - let update_fn: UpdateFn = Box::new(move || { - let disk_id = id.map_or("".to_string(), |id| id.to_string()); - let root = root_clone.clone(); - let is_erasure_sd = false; // TODO: 从全局变量中获取 - Box::pin(async move { - match get_disk_info(root.clone(), is_erasure_sd).await { - Ok((info, root)) => { - let disk_info = DiskInfo { - total: info.total, - free: info.free, - used: info.used, - used_inodes: info.files - info.ffree, - free_inodes: info.ffree, - major: info.major, - minor: info.minor, - fs_type: info.fstype, - root_disk: root, - id: disk_id.to_string(), - ..Default::default() - }; - // if root { - // return Err(Error::new(DiskError::DriveIsRoot)); - // } - - // disk_info.healing = - Ok(disk_info) - } - Err(err) => Err(err), - } - }) - }); - - let cache = Cache::new(update_fn, Duration::from_secs(1), Opts::default()); - - // TODO: DIRECT suport - // TODD: DiskInfo - let mut disk = Self { - root: root.clone(), - endpoint: ep.clone(), - format_path, - format_info: RwLock::new(format_info), - disk_info_cache: Arc::new(cache), - scanning: AtomicU32::new(0), - rotational: Default::default(), - fstype: Default::default(), - minor: Default::default(), - major: Default::default(), - nrrequests: Default::default(), - // // format_legacy, - // format_file_info: Mutex::new(format_meta), - // format_data: Mutex::new(format_data), - // format_last_check: Mutex::new(format_last_check), - }; - - let info = get_info(&root)?; - // let (info, _root) = get_disk_info(root).await?; - disk.major = info.major; - disk.minor = info.minor; - disk.fstype = info.fstype; - - // if root { - // return Err(Error::new(DiskError::DriveIsRoot)); - // } - - if info.nrrequests > 0 { - disk.nrrequests = info.nrrequests; - } - - if info.rotational { - disk.rotational = true; - } - - disk.make_meta_volumes().await?; - - Ok(disk) - } - - async fn make_meta_volumes(&self) -> Result<()> { - let buckets = format!("{}/{}", RUSTFS_META_BUCKET, BUCKET_META_PREFIX); - let multipart = format!("{}/{}", RUSTFS_META_BUCKET, "multipart"); - let config = format!("{}/{}", RUSTFS_META_BUCKET, "config"); - let tmp = format!("{}/{}", RUSTFS_META_BUCKET, "tmp"); - let defaults = vec![buckets.as_str(), multipart.as_str(), config.as_str(), tmp.as_str()]; - - self.make_volumes(defaults).await - } - - fn is_valid_volname(volname: &str) -> bool { - if volname.len() < 3 { - return false; - } - - if cfg!(target_os = "windows") { - // 在 Windows 上，卷名不应该包含保留字符。 - // 这个正则表达式匹配了不允许的字符。 - if volname.contains('|') - || volname.contains('<') - || volname.contains('>') - || volname.contains('?') - || volname.contains('*') - || volname.contains(':') - || volname.contains('"') - || volname.contains('\\') - { - return false; - } - } else { - // 对于非 Windows 系统，可能需要其他的验证逻辑。 - } - - true - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn check_format_json(&self) -> Result { - let md = std::fs::metadata(&self.format_path).map_err(to_unformatted_disk_error)?; - Ok(md) - } - - pub fn resolve_abs_path(&self, path: impl AsRef) -> Result { - Ok(path.as_ref().absolutize_virtually(&self.root)?.into_owned()) - } - - pub fn get_object_path(&self, bucket: &str, key: &str) -> Result { - let dir = Path::new(&bucket); - let file_path = Path::new(&key); - self.resolve_abs_path(dir.join(file_path)) - } - - pub fn get_bucket_path(&self, bucket: &str) -> Result { - let dir = Path::new(&bucket); - self.resolve_abs_path(dir) - } - pub async fn move_to_trash(&self, delete_path: &PathBuf, recursive: bool, _immediate_purge: bool) -> Result<()> { - if recursive { - remove_all_std(delete_path).map_err(to_file_error)?; - } else { - remove_std(delete_path).map_err(to_file_error)?; - } - - Ok(()) - - // // TODO: 异步通知 检测硬盘空间 清空回收站 - - // let trash_path = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; - // if let Some(parent) = trash_path.parent() { - // if !parent.exists() { - // fs::create_dir_all(parent).await?; - // } - // } - - // let err = if recursive { - // rename_all(delete_path, trash_path, self.get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?) - // .await - // .err() - // } else { - // rename(&delete_path, &trash_path) - // .await - // .map_err(|e| to_file_error(e)) - // .err() - // }; - - // if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR) { - // warn!("move_to_trash immediate_purge {:?}", &delete_path.to_string_lossy()); - // let trash_path2 = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; - // let _ = rename_all( - // encode_dir_object(delete_path.to_string_lossy().as_ref()), - // trash_path2, - // self.get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?, - // ) - // .await; - // } - - // if let Some(err) = err { - // if err == Error::DiskFull { - // if recursive { - // remove_all(delete_path).await.map_err(to_file_error)?; - // } else { - // remove(delete_path).await.map_err(to_file_error)?; - // } - // } - - // return Err(err); - // } - - // Ok(()) - } - - #[tracing::instrument(level = "debug", skip(self))] - pub async fn delete_file( - &self, - base_path: &PathBuf, - delete_path: &PathBuf, - recursive: bool, - immediate_purge: bool, - ) -> Result<()> { - // debug!("delete_file {:?}\n base_path:{:?}", &delete_path, &base_path); - - if is_root_path(base_path) || is_root_path(delete_path) { - // debug!("delete_file skip {:?}", &delete_path); - return Ok(()); - } - - if !delete_path.starts_with(base_path) || base_path == delete_path { - // debug!("delete_file skip {:?}", &delete_path); - return Ok(()); - } - - if recursive { - self.move_to_trash(delete_path, recursive, immediate_purge).await?; - } else if delete_path.is_dir() { - // debug!("delete_file remove_dir {:?}", &delete_path); - if let Err(err) = fs::remove_dir(&delete_path).await { - // debug!("remove_dir err {:?} when {:?}", &err, &delete_path); - match err.kind() { - ErrorKind::NotFound => (), - ErrorKind::DirectoryNotEmpty => { - warn!("delete_file remove_dir {:?} err {}", &delete_path, err.to_string()); - return Err(Error::FileAccessDenied); - } - _ => (), - } - } - // debug!("delete_file remove_dir done {:?}", &delete_path); - } else if let Err(err) = fs::remove_file(&delete_path).await { - // debug!("remove_file err {:?} when {:?}", &err, &delete_path); - match err.kind() { - ErrorKind::NotFound => (), - _ => { - warn!("delete_file remove_file {:?} err {:?}", &delete_path, &err); - return Err(Error::FileAccessDenied); - } - } - } - - if let Some(dir_path) = delete_path.parent() { - Box::pin(self.delete_file(base_path, &PathBuf::from(dir_path), false, false)).await?; - } - - // debug!("delete_file done {:?}", &delete_path); - Ok(()) - } - - /// read xl.meta raw data - #[tracing::instrument(level = "debug", skip(self, volume_dir, file_path))] - async fn read_raw( - &self, - bucket: &str, - volume_dir: impl AsRef, - file_path: impl AsRef, - read_data: bool, - ) -> Result<(Vec, Option)> { - if file_path.as_ref().as_os_str().is_empty() { - return Err(Error::FileNotFound); - } - - let meta_path = file_path.as_ref().join(Path::new(STORAGE_FORMAT_FILE)); - - let res = { - if read_data { - self.read_all_data_with_dmtime(bucket, volume_dir, meta_path).await - } else { - match self.read_metadata_with_dmtime(meta_path).await { - Ok(res) => Ok(res), - Err(err) => { - if err == Error::FileNotFound - && !skip_access_checks(volume_dir.as_ref().to_string_lossy().to_string().as_str()) - { - if let Err(aerr) = access(volume_dir.as_ref()).await { - if aerr.kind() == ErrorKind::NotFound { - warn!("read_metadata_with_dmtime os err {:?}", &aerr); - return Err(Error::VolumeNotFound); - } - } - } - - Err(err) - } - } - } - }; - - let (buf, mtime) = res?; - if buf.is_empty() { - return Err(Error::FileNotFound); - } - - Ok((buf, mtime)) - } - - pub(crate) async fn read_metadata(&self, file_path: impl AsRef) -> Result> { - // TODO: suport timeout - let (data, _) = self.read_metadata_with_dmtime(file_path.as_ref()).await?; - Ok(data) - } - - async fn read_metadata_with_dmtime(&self, file_path: impl AsRef) -> Result<(Vec, Option)> { - check_path_length(file_path.as_ref().to_string_lossy().as_ref())?; - - let mut f = super::fs::open_file(file_path.as_ref(), O_RDONLY) - .await - .map_err(to_file_error)?; - - let meta = f.metadata().await.map_err(to_file_error)?; - - if meta.is_dir() { - // fix use io::Error - return Err(Error::FileNotFound); - } - - let size = meta.len() as usize; - - let data = read_xl_meta_no_data(&mut f, size).await?; - - let modtime = match meta.modified() { - Ok(md) => Some(OffsetDateTime::from(md)), - Err(_) => None, - }; - - Ok((data, modtime)) - } - - async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef, file_path: impl AsRef) -> Result> { - // TODO: timeout suport - let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?; - Ok(data) - } - - #[tracing::instrument(level = "debug", skip(self, volume_dir, file_path))] - async fn read_all_data_with_dmtime( - &self, - volume: &str, - volume_dir: impl AsRef, - file_path: impl AsRef, - ) -> Result<(Vec, Option)> { - let mut f = match super::fs::open_file(file_path.as_ref(), O_RDONLY).await { - Ok(f) => f, - Err(e) => { - if e.kind() == ErrorKind::NotFound { - if !skip_access_checks(volume) { - if let Err(er) = super::fs::access(volume_dir.as_ref()).await { - if er.kind() == ErrorKind::NotFound { - warn!("read_all_data_with_dmtime os err {:?}", &er); - return Err(Error::VolumeNotFound); - } - } - } - - return Err(Error::FileNotFound); - } - - return Err(to_file_error(e).into()); - } - }; - - let meta = f.metadata().await.map_err(to_file_error)?; - - if meta.is_dir() { - return Err(Error::FileNotFound); - } - - let size = meta.len() as usize; - let mut bytes = Vec::new(); - bytes.try_reserve_exact(size)?; - - f.read_to_end(&mut bytes).await.map_err(to_file_error)?; - - let modtime = match meta.modified() { - Ok(md) => Some(OffsetDateTime::from(md)), - Err(_) => None, - }; - - Ok((bytes, modtime)) - } - - async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &Vec) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; - let xlpath = self.get_object_path(volume, format!("{}/{}", path, STORAGE_FORMAT_FILE).as_str())?; - - let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await?; - - let mut fm = FileMeta::default(); - - fm.unmarshal_msg(&data)?; - - for fi in fis { - let data_dir = match fm.delete_version(fi) { - Ok(res) => res, - Err(err) => { - if !fi.deleted && (err == Error::FileVersionNotFound || err == Error::FileNotFound) { - continue; - } - - return Err(err); - } - }; - - if let Some(dir) = data_dir { - let vid = fi.version_id.unwrap_or_default(); - let _ = fm.data.remove(vec![vid, dir]); - - let dir_path = self.get_object_path(volume, format!("{}/{}", path, dir).as_str())?; - if let Err(err) = self.move_to_trash(&dir_path, true, false).await { - if !(err == Error::FileNotFound || err == Error::DiskNotFound) { - return Err(err); - } - }; - } - } - - // 没有版本了，删除 xl.meta - if fm.versions.is_empty() { - self.delete_file(&volume_dir, &xlpath, true, false).await?; - return Ok(()); - } - - // 更新 xl.meta - let buf = fm.marshal_msg()?; - - let volume_dir = self.get_bucket_path(volume)?; - - self.write_all_private(volume, format!("{}/{}", path, STORAGE_FORMAT_FILE).as_str(), &buf, true, volume_dir) - .await?; - - Ok(()) - } - - async fn write_all_meta(&self, volume: &str, path: &str, buf: &[u8], sync: bool) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().as_ref())?; - - let tmp_volume_dir = self.get_bucket_path(RUSTFS_META_TMP_BUCKET)?; - let tmp_file_path = tmp_volume_dir.join(Path::new(Uuid::new_v4().to_string().as_str())); - - self.write_all_internal(&tmp_file_path, buf, sync, tmp_volume_dir).await?; - - super::os::rename_all(tmp_file_path, file_path, volume_dir).await?; - - Ok(()) - } - - // write_all_public for trail - async fn write_all_public(&self, volume: &str, path: &str, data: Vec) -> Result<()> { - if volume == RUSTFS_META_BUCKET && path == FORMAT_CONFIG_FILE { - let mut format_info = self.format_info.write().await; - format_info.data.clone_from(&data); - } - - let volume_dir = self.get_bucket_path(volume)?; - - self.write_all_private(volume, path, &data, true, volume_dir).await?; - - Ok(()) - } - - // write_all_private with check_path_length - #[tracing::instrument(level = "debug", skip_all)] - pub async fn write_all_private( - &self, - volume: &str, - path: &str, - buf: &[u8], - sync: bool, - skip_parent: impl AsRef, - ) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().as_ref())?; - - self.write_all_internal(file_path, buf, sync, skip_parent) - .await - .map_err(to_file_error)?; - - Ok(()) - } - // write_all_internal do write file - pub async fn write_all_internal( - &self, - file_path: impl AsRef, - data: impl AsRef<[u8]>, - sync: bool, - skip_parent: impl AsRef, - ) -> std::io::Result<()> { - let flags = O_CREATE | O_WRONLY | O_TRUNC; - - let mut f = { - if sync { - // TODO: suport sync - self.open_file(file_path.as_ref(), flags, skip_parent.as_ref()).await? - } else { - self.open_file(file_path.as_ref(), flags, skip_parent.as_ref()).await? - } - }; - - f.write_all(data.as_ref()).await?; - - Ok(()) - } - - async fn open_file(&self, path: impl AsRef, mode: usize, skip_parent: impl AsRef) -> Result { - let mut skip_parent = skip_parent.as_ref(); - if skip_parent.as_os_str().is_empty() { - skip_parent = self.root.as_path(); - } - - if let Some(parent) = path.as_ref().parent() { - super::os::make_dir_all(parent, skip_parent).await?; - } - - let f = super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?; - - Ok(f) - } - - #[allow(dead_code)] - fn get_metrics(&self) -> DiskMetrics { - DiskMetrics::default() - } - - async fn bitrot_verify( - &self, - part_path: &PathBuf, - part_size: usize, - algo: HashAlgorithm, - sum: &[u8], - shard_size: usize, - ) -> Result<()> { - let file = super::fs::open_file(part_path, O_CREATE | O_WRONLY) - .await - .map_err(to_file_error)?; - - let meta = file.metadata().await?; - let file_size = meta.len() as usize; - - bitrot_verify(file, file_size, part_size, algo, sum.to_vec(), shard_size) - .await - .map_err(to_file_error)?; - - Ok(()) - } -} - -/// 获取磁盘信息 -async fn get_disk_info(drive_path: PathBuf, is_erasure_sd: bool) -> Result<(rustfs_utils::os::DiskInfo, bool)> { - let drive_path = drive_path.to_string_lossy().to_string(); - check_path_length(&drive_path)?; - - let disk_info = get_info(&drive_path)?; - let root_drive = if !is_erasure_sd { - is_root_disk(&drive_path, SLASH_SEPARATOR).unwrap_or_default() - } else { - false - }; - - Ok((disk_info, root_drive)) -} - -fn is_root_path(path: impl AsRef) -> bool { - path.as_ref().components().count() == 1 && path.as_ref().has_root() -} - -fn skip_access_checks(p: impl AsRef) -> bool { - let vols = [ - RUSTFS_META_TMP_DELETED_BUCKET, - RUSTFS_META_TMP_BUCKET, - RUSTFS_META_MULTIPART_BUCKET, - RUSTFS_META_BUCKET, - ]; - - for v in vols.iter() { - if p.as_ref().starts_with(v) { - return true; - } - } - - false -} - -#[async_trait::async_trait] -impl DiskAPI for LocalDisk { - #[tracing::instrument(skip(self))] - fn to_string(&self) -> String { - self.root.to_string_lossy().to_string() - } - #[tracing::instrument(skip(self))] - fn is_local(&self) -> bool { - true - } - #[tracing::instrument(skip(self))] - fn host_name(&self) -> String { - self.endpoint.host_port() - } - #[tracing::instrument(skip(self))] - async fn is_online(&self) -> bool { - self.check_format_json().await.is_ok() - } - - #[tracing::instrument(skip(self))] - fn endpoint(&self) -> Endpoint { - self.endpoint.clone() - } - - #[tracing::instrument(skip(self))] - async fn close(&self) -> Result<()> { - Ok(()) - } - - #[tracing::instrument(skip(self))] - fn path(&self) -> PathBuf { - self.root.clone() - } - - #[tracing::instrument(skip(self))] - fn get_disk_location(&self) -> DiskLocation { - DiskLocation { - pool_idx: { - if self.endpoint.pool_idx < 0 { - None - } else { - Some(self.endpoint.pool_idx as usize) - } - }, - set_idx: { - if self.endpoint.set_idx < 0 { - None - } else { - Some(self.endpoint.set_idx as usize) - } - }, - disk_idx: { - if self.endpoint.disk_idx < 0 { - None - } else { - Some(self.endpoint.disk_idx as usize) - } - }, - } - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn get_disk_id(&self) -> Result> { - let mut format_info = self.format_info.write().await; - - let id = format_info.id; - - if format_info.last_check_valid() { - return Ok(id); - } - - let file_meta = self.check_format_json().await?; - - if let Some(file_info) = &format_info.file_info { - if super::fs::same_file(&file_meta, file_info) { - format_info.last_check = Some(OffsetDateTime::now_utc()); - - return Ok(id); - } - } - - let b = tokio::fs::read(&self.format_path).await.map_err(to_unformatted_disk_error)?; - - let fm = FormatV3::try_from(b.as_slice()).map_err(|e| { - warn!("decode format.json err {:?}", e); - Error::CorruptedBackend - })?; - - let (m, n) = fm.find_disk_index_by_disk_id(fm.erasure.this)?; - - let disk_id = fm.erasure.this; - - if m as i32 != self.endpoint.set_idx || n as i32 != self.endpoint.disk_idx { - return Err(Error::InconsistentDisk); - } - - format_info.id = Some(disk_id); - format_info.file_info = Some(file_meta); - format_info.data = b; - format_info.last_check = Some(OffsetDateTime::now_utc()); - - Ok(Some(disk_id)) - } - - #[tracing::instrument(skip(self))] - async fn set_disk_id(&self, id: Option) -> Result<()> { - // 本地不需要设置 - // TODO: add check_id_store - let mut format_info = self.format_info.write().await; - format_info.id = id; - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn read_all(&self, volume: &str, path: &str) -> Result> { - if volume == RUSTFS_META_BUCKET && path == FORMAT_CONFIG_FILE { - let format_info = self.format_info.read().await; - if !format_info.data.is_empty() { - return Ok(format_info.data.clone()); - } - } - // TOFIX: - let p = self.get_object_path(volume, path)?; - let data = read_all(&p).await?; - - Ok(data) - } - - #[tracing::instrument(level = "debug", skip_all)] - async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()> { - self.write_all_public(volume, path, data).await - } - - #[tracing::instrument(skip(self))] - async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().to_string().as_str())?; - - self.delete_file(&volume_dir, &file_path, opt.recursive, opt.immediate) - .await?; - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let mut resp = CheckPartsResp { - results: vec![0; fi.parts.len()], - }; - - let erasure = &fi.erasure; - for (i, part) in fi.parts.iter().enumerate() { - let checksum_info = erasure.get_checksum_info(part.number); - let part_path = Path::new(&volume_dir) - .join(path) - .join(fi.data_dir.map_or("".to_string(), |dir| dir.to_string())) - .join(format!("part.{}", part.number)); - let err = (self - .bitrot_verify( - &part_path, - erasure.shard_file_size(part.size), - checksum_info.algorithm, - &checksum_info.hash, - erasure.shard_size(), - ) - .await) - .err(); - resp.results[i] = conv_part_err_to_int(&err); - if resp.results[i] == CHECK_PART_UNKNOWN { - if let Some(err) = err { - match err { - Error::FileAccessDenied => {} - _ => { - info!("part unknown, disk: {}, path: {:?}", self.to_string(), part_path); - } - } - } - } - } - - Ok(resp) - } - - #[tracing::instrument(skip(self))] - async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - let volume_dir = self.get_bucket_path(volume)?; - check_path_length(volume_dir.join(path).to_string_lossy().as_ref())?; - let mut resp = CheckPartsResp { - results: vec![0; fi.parts.len()], - }; - - for (i, part) in fi.parts.iter().enumerate() { - let file_path = Path::new(&volume_dir) - .join(path) - .join(fi.data_dir.map_or("".to_string(), |dir| dir.to_string())) - .join(format!("part.{}", part.number)); - - match lstat(file_path).await { - Ok(st) => { - if st.is_dir() { - resp.results[i] = CHECK_PART_FILE_NOT_FOUND; - continue; - } - if (st.len() as usize) < fi.erasure.shard_file_size(part.size) { - resp.results[i] = CHECK_PART_FILE_CORRUPT; - continue; - } - - resp.results[i] = CHECK_PART_SUCCESS; - } - Err(err) => { - if err.kind() == ErrorKind::NotFound { - if !skip_access_checks(volume) { - if let Err(err) = super::fs::access(&volume_dir).await { - if err.kind() == ErrorKind::NotFound { - resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND; - continue; - } - } - } - resp.results[i] = CHECK_PART_FILE_NOT_FOUND; - } - continue; - } - } - } - - Ok(resp) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()> { - let src_volume_dir = self.get_bucket_path(src_volume)?; - let dst_volume_dir = self.get_bucket_path(dst_volume)?; - if !skip_access_checks(src_volume) { - super::fs::access_std(&src_volume_dir).map_err(to_file_error)? - } - if !skip_access_checks(dst_volume) { - super::fs::access_std(&dst_volume_dir).map_err(to_file_error)? - } - - let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR); - let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR); - - if !src_is_dir && dst_is_dir || src_is_dir && !dst_is_dir { - warn!( - "rename_part src and dst must be both dir or file src_is_dir:{}, dst_is_dir:{}", - src_is_dir, dst_is_dir - ); - return Err(Error::FileAccessDenied); - } - - let src_file_path = src_volume_dir.join(Path::new(src_path)); - let dst_file_path = dst_volume_dir.join(Path::new(dst_path)); - - // warn!("rename_part src_file_path:{:?}, dst_file_path:{:?}", &src_file_path, &dst_file_path); - - check_path_length(src_file_path.to_string_lossy().as_ref())?; - check_path_length(dst_file_path.to_string_lossy().as_ref())?; - - if src_is_dir { - let meta_op = match lstat_std(&src_file_path) { - Ok(meta) => Some(meta), - Err(e) => { - let err = to_file_error(e).into(); - - if err == Error::FaultyDisk { - return Err(err); - } - - if err != Error::FileNotFound { - return Err(err); - } - None - } - }; - - if let Some(meta) = meta_op { - if !meta.is_dir() { - warn!("rename_part src is not dir {:?}", &src_file_path); - return Err(Error::FileAccessDenied); - } - } - - super::fs::remove_std(&dst_file_path).map_err(to_file_error)?; - } - super::os::rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await?; - - self.write_all(dst_volume, format!("{}.meta", dst_path).as_str(), meta) - .await?; - - if let Some(parent) = src_file_path.parent() { - self.delete_file(&src_volume_dir, &parent.to_path_buf(), false, false).await?; - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { - let src_volume_dir = self.get_bucket_path(src_volume)?; - let dst_volume_dir = self.get_bucket_path(dst_volume)?; - if !skip_access_checks(src_volume) { - if let Err(e) = super::fs::access(&src_volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - if !skip_access_checks(dst_volume) { - if let Err(e) = super::fs::access(&dst_volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR); - let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR); - if (dst_is_dir || src_is_dir) && (!dst_is_dir || !src_is_dir) { - return Err(Error::FileAccessDenied); - } - - let src_file_path = src_volume_dir.join(Path::new(&src_path)); - check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; - - let dst_file_path = dst_volume_dir.join(Path::new(&dst_path)); - check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; - - if src_is_dir { - let meta_op = match lstat(&src_file_path).await { - Ok(meta) => Some(meta), - Err(e) => { - if e.kind() != ErrorKind::NotFound { - return Err(to_file_error(e).into()); - } - None - } - }; - - if let Some(meta) = meta_op { - if !meta.is_dir() { - return Err(Error::FileAccessDenied); - } - } - - super::fs::remove(&dst_file_path).await.map_err(to_file_error)?; - } - - super::os::rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await?; - - if let Some(parent) = src_file_path.parent() { - let _ = self.delete_file(&src_volume_dir, &parent.to_path_buf(), false, false).await; - } - - Ok(()) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn create_file(&self, origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result> { - // warn!("disk create_file: origvolume: {}, volume: {}, path: {}", origvolume, volume, path); - - if !origvolume.is_empty() { - let origvolume_dir = self.get_bucket_path(origvolume)?; - if !skip_access_checks(origvolume) { - if let Err(e) = super::fs::access(&origvolume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - } - - let volume_dir = self.get_bucket_path(volume)?; - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().to_string().as_str())?; - - // TODO: writeAllDirect io.copy - // info!("file_path: {:?}", file_path); - if let Some(parent) = file_path.parent() { - super::os::make_dir_all(parent, &volume_dir).await?; - } - let f = super::fs::open_file(&file_path, O_CREATE | O_WRONLY) - .await - .map_err(to_file_error)?; - - Ok(Box::new(f)) - - // Ok(()) - } - - #[tracing::instrument(level = "debug", skip(self))] - // async fn append_file(&self, volume: &str, path: &str, mut r: DuplexStream) -> Result { - async fn append_file(&self, volume: &str, path: &str) -> Result> { - warn!("disk append_file: volume: {}, path: {}", volume, path); - - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().to_string().as_str())?; - - let f = self.open_file(file_path, O_CREATE | O_APPEND | O_WRONLY, volume_dir).await?; - - Ok(Box::new(f)) - } - - // TODO: io verifier - #[tracing::instrument(level = "debug", skip(self))] - async fn read_file(&self, volume: &str, path: &str) -> Result> { - // warn!("disk read_file: volume: {}, path: {}", volume, path); - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().to_string().as_str())?; - - let f = self.open_file(file_path, O_RDONLY, volume_dir).await?; - - Ok(Box::new(f)) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result> { - // warn!( - // "disk read_file_stream: volume: {}, path: {}, offset: {}, length: {}", - // volume, path, offset, length - // ); - - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().to_string().as_str())?; - - let mut f = self.open_file(file_path, O_RDONLY, volume_dir).await?; - - let meta = f.metadata().await?; - if meta.len() < (offset + length) as u64 { - error!( - "read_file_stream: file size is less than offset + length {} + {} = {}", - offset, - length, - meta.len() - ); - return Err(Error::FileCorrupt); - } - - f.seek(SeekFrom::Start(offset as u64)).await?; - - Ok(Box::new(f)) - } - #[tracing::instrument(level = "debug", skip(self))] - async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result> { - if !origvolume.is_empty() { - let origvolume_dir = self.get_bucket_path(origvolume)?; - if !skip_access_checks(origvolume) { - if let Err(e) = super::fs::access(origvolume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - } - - let volume_dir = self.get_bucket_path(volume)?; - let dir_path_abs = volume_dir.join(Path::new(&dir_path.trim_start_matches(SLASH_SEPARATOR))); - - let entries = match super::os::read_dir(&dir_path_abs, count).await { - Ok(res) => res, - Err(e) => { - if e.kind() == ErrorKind::NotFound && !skip_access_checks(volume) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - return Err(to_volume_error(e).into()); - } - }; - - Ok(entries) - } - - // FIXME: TODO: io.writer TODO cancel - #[tracing::instrument(level = "debug", skip(self, wr))] - async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> { - let volume_dir = self.get_bucket_path(&opts.bucket)?; - - if !skip_access_checks(&opts.bucket) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let mut wr = wr; - - let mut out = MetacacheWriter::new(&mut wr); - - let mut objs_returned = 0; - - if opts.base_dir.ends_with(SLASH_SEPARATOR) { - let fpath = self.get_object_path( - &opts.bucket, - path_join_buf(&[ - format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX).as_str(), - STORAGE_FORMAT_FILE, - ]) - .as_str(), - )?; - - if let Ok(data) = self.read_metadata(fpath).await { - let meta = MetaCacheEntry { - name: opts.base_dir.clone(), - metadata: data, - ..Default::default() - }; - out.write_obj(&meta).await?; - objs_returned += 1; - } - } - - let mut current = opts.base_dir.clone(); - self.scan_dir(&mut current, &opts, &mut out, &mut objs_returned).await?; - - Ok(()) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn rename_data( - &self, - src_volume: &str, - src_path: &str, - fi: FileInfo, - dst_volume: &str, - dst_path: &str, - ) -> Result { - let src_volume_dir = self.get_bucket_path(src_volume)?; - if !skip_access_checks(src_volume) { - if let Err(e) = super::fs::access_std(&src_volume_dir) { - info!("access checks failed, src_volume_dir: {:?}, err: {}", src_volume_dir, e.to_string()); - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let dst_volume_dir = self.get_bucket_path(dst_volume)?; - if !skip_access_checks(dst_volume) { - if let Err(e) = super::fs::access_std(&dst_volume_dir) { - info!("access checks failed, dst_volume_dir: {:?}, err: {}", dst_volume_dir, e.to_string()); - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - // xl.meta 路径 - let src_file_path = src_volume_dir.join(Path::new(format!("{}/{}", &src_path, STORAGE_FORMAT_FILE).as_str())); - let dst_file_path = dst_volume_dir.join(Path::new(format!("{}/{}", &dst_path, STORAGE_FORMAT_FILE).as_str())); - - // data_dir 路径 - let has_data_dir_path = { - let has_data_dir = { - if !fi.is_remote() { - fi.data_dir.map(|dir| super::path::retain_slash(dir.to_string().as_str())) - } else { - None - } - }; - - if let Some(data_dir) = has_data_dir { - let src_data_path = src_volume_dir.join(Path::new( - super::path::retain_slash(format!("{}/{}", &src_path, data_dir).as_str()).as_str(), - )); - let dst_data_path = dst_volume_dir.join(Path::new( - super::path::retain_slash(format!("{}/{}", &dst_path, data_dir).as_str()).as_str(), - )); - - Some((src_data_path, dst_data_path)) - } else { - None - } - }; - - check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; - check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; - - // 读旧 xl.meta - - let has_dst_buf = match super::fs::read_file(&dst_file_path).await { - Ok(res) => Some(res), - Err(e) => { - if e.kind() == ErrorKind::NotADirectory && !cfg!(target_os = "windows") { - return Err(Error::FileAccessDenied); - } - - if e.kind() != ErrorKind::NotFound { - return Err(to_file_error(e).into()); - } - - None - } - }; - - let mut xlmeta = FileMeta::new(); - - if let Some(dst_buf) = has_dst_buf.as_ref() { - if FileMeta::is_xl2_v1_format(dst_buf) { - if let Ok(nmeta) = FileMeta::load(dst_buf) { - xlmeta = nmeta - } - } - } - - let mut skip_parent = dst_volume_dir.clone(); - if has_dst_buf.as_ref().is_some() { - if let Some(parent) = dst_file_path.parent() { - skip_parent = parent.to_path_buf(); - } - } - - // TODO: Healing - - let has_old_data_dir = { - if let Ok((_, ver)) = xlmeta.find_version(fi.version_id) { - let has_data_dir = ver.get_data_dir(); - if let Some(data_dir) = has_data_dir { - if xlmeta.shard_data_dir_count(&fi.version_id, &Some(data_dir)) == 0 { - // TODO: Healing - // remove inlinedata\ - Some(data_dir) - } else { - None - } - } else { - None - } - } else { - None - } - }; - - xlmeta.add_version(fi.clone())?; - - if xlmeta.versions.len() <= 10 { - // TODO: Sign - } - - let new_dst_buf = xlmeta.marshal_msg()?; - - self.write_all(src_volume, format!("{}/{}", &src_path, STORAGE_FORMAT_FILE).as_str(), new_dst_buf) - .await?; - - if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() { - let no_inline = fi.data.is_none() && fi.size > 0; - if no_inline { - if let Err(err) = super::os::rename_all(&src_data_path, &dst_data_path, &skip_parent).await { - let _ = self.delete_file(&dst_volume_dir, dst_data_path, false, false).await; - info!( - "rename all failed src_data_path: {:?}, dst_data_path: {:?}, err: {:?}", - src_data_path, dst_data_path, err - ); - return Err(err); - } - } - } - - if let Some(old_data_dir) = has_old_data_dir { - // preserve current xl.meta inside the oldDataDir. - if let Some(dst_buf) = has_dst_buf { - if let Err(err) = self - .write_all_private( - dst_volume, - format!("{}/{}/{}", &dst_path, &old_data_dir.to_string(), STORAGE_FORMAT_FILE).as_str(), - &dst_buf, - true, - &skip_parent, - ) - .await - { - info!("write_all_private failed err: {:?}", err); - return Err(err); - } - } - } - - if let Err(err) = super::os::rename_all(&src_file_path, &dst_file_path, &skip_parent).await { - if let Some((_, dst_data_path)) = has_data_dir_path.as_ref() { - let _ = self.delete_file(&dst_volume_dir, dst_data_path, false, false).await; - } - info!("rename all failed err: {:?}", err); - return Err(err); - } - - if let Some(src_file_path_parent) = src_file_path.parent() { - if src_volume != RUSTFS_META_MULTIPART_BUCKET { - let _ = super::fs::remove_std(src_file_path_parent); - } else { - let _ = self - .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) - .await; - } - } - - Ok(RenameDataResp { - old_data_dir: has_old_data_dir, - sign: None, // TODO: - }) - } - - #[tracing::instrument(skip(self))] - async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> { - for vol in volumes { - if let Err(e) = self.make_volume(vol).await { - if e != Error::VolumeExists { - return Err(e); - } - } - // TODO: health check - } - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn make_volume(&self, volume: &str) -> Result<()> { - if !Self::is_valid_volname(volume) { - return Err(Error::msg("Invalid arguments specified")); - } - - let volume_dir = self.get_bucket_path(volume)?; - - if let Err(e) = super::fs::access(&volume_dir).await { - if e.kind() == std::io::ErrorKind::NotFound { - super::os::make_dir_all(&volume_dir, self.root.as_path()).await?; - return Ok(()); - } - - return Err(to_disk_error(e).into()); - } - - Err(Error::VolumeExists) - } - - #[tracing::instrument(skip(self))] - async fn list_volumes(&self) -> Result> { - let mut volumes = Vec::new(); - - let entries = super::os::read_dir(&self.root, -1) - .await - .map_err(|e| to_access_error(e, Error::DiskAccessDenied))?; - - for entry in entries { - if !super::path::has_suffix(&entry, SLASH_SEPARATOR) || !Self::is_valid_volname(super::path::clean(&entry).as_str()) { - continue; - } - - volumes.push(VolumeInfo { - name: super::path::clean(&entry), - created: None, - }); - } - - Ok(volumes) - } - - #[tracing::instrument(skip(self))] - async fn stat_volume(&self, volume: &str) -> Result { - let volume_dir = self.get_bucket_path(volume)?; - let meta = super::fs::lstat(&volume_dir).await.map_err(to_volume_error)?; - - let modtime = match meta.modified() { - Ok(md) => Some(OffsetDateTime::from(md)), - Err(_) => None, - }; - - Ok(VolumeInfo { - name: volume.to_string(), - created: modtime, - }) - } - - #[tracing::instrument(skip(self))] - async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - super::fs::access(&volume_dir) - .await - .map_err(|e| to_access_error(e, Error::VolumeAccessDenied))?; - } - - for path in paths.iter() { - let file_path = volume_dir.join(Path::new(path)); - - check_path_length(file_path.to_string_lossy().as_ref())?; - - self.move_to_trash(&file_path, false, false).await?; - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> { - if !fi.metadata.is_empty() { - let volume_dir = self.get_bucket_path(volume)?; - let file_path = volume_dir.join(Path::new(&path)); - - check_path_length(file_path.to_string_lossy().as_ref())?; - - let buf = self - .read_all(volume, format!("{}/{}", &path, STORAGE_FORMAT_FILE).as_str()) - .await - .map_err(|e| { - if e == Error::FileNotFound && fi.version_id.is_some() { - Error::FileVersionNotFound - } else { - e - } - })?; - - if !FileMeta::is_xl2_v1_format(buf.as_slice()) { - return Err(Error::FileVersionNotFound); - } - - let mut xl_meta = FileMeta::load(buf.as_slice())?; - - xl_meta.update_object_version(fi)?; - - let wbuf = xl_meta.marshal_msg()?; - - return self - .write_all_meta(volume, format!("{}/{}", path, STORAGE_FORMAT_FILE).as_str(), &wbuf, !opts.no_persistence) - .await; - } - - Err(Error::msg("Invalid Argument")) - } - - #[tracing::instrument(skip(self))] - async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { - let p = self.get_object_path(volume, format!("{}/{}", path, STORAGE_FORMAT_FILE).as_str())?; - - let mut meta = FileMeta::new(); - if !fi.fresh { - let (buf, _) = read_file_exists(&p).await?; - if !buf.is_empty() { - let _ = meta.unmarshal_msg(&buf).map_err(|_| { - meta = FileMeta::new(); - }); - } - } - - meta.add_version(fi)?; - - let fm_data = meta.marshal_msg()?; - - self.write_all(volume, format!("{}/{}", path, STORAGE_FORMAT_FILE).as_str(), fm_data) - .await?; - - return Ok(()); - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn read_version( - &self, - _org_volume: &str, - volume: &str, - path: &str, - version_id: &str, - opts: &ReadOptions, - ) -> Result { - let file_path = self.get_object_path(volume, path)?; - let file_dir = self.get_bucket_path(volume)?; - - let read_data = opts.read_data; - - let (data, _) = self.read_raw(volume, file_dir, file_path, read_data).await?; - - let fi = get_file_info(&data, volume, path, version_id, FileInfoOpts { data: read_data }).await?; - - Ok(fi) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { - let file_path = self.get_object_path(volume, path)?; - let file_dir = self.get_bucket_path(volume)?; - - let (buf, _) = self.read_raw(volume, file_dir, file_path, read_data).await?; - - Ok(RawFileInfo { buf }) - } - - #[tracing::instrument(skip(self))] - async fn delete_version( - &self, - volume: &str, - path: &str, - fi: FileInfo, - force_del_marker: bool, - opts: DeleteOptions, - ) -> Result<()> { - if path.starts_with(SLASH_SEPARATOR) { - return self - .delete( - volume, - path, - DeleteOptions { - recursive: false, - immediate: false, - ..Default::default() - }, - ) - .await; - } - - let volume_dir = self.get_bucket_path(volume)?; - - let file_path = volume_dir.join(Path::new(&path)); - - check_path_length(file_path.to_string_lossy().as_ref())?; - - let xl_path = file_path.join(Path::new(STORAGE_FORMAT_FILE)); - let buf = match self.read_all_data(volume, &volume_dir, &xl_path).await { - Ok(res) => res, - Err(err) => { - // - if err != Error::FileNotFound { - return Err(err); - } - - if fi.deleted && force_del_marker { - return self.write_metadata("", volume, path, fi).await; - } - - if fi.version_id.is_some() { - return Err(Error::FileVersionNotFound); - } else { - return Err(Error::FileNotFound); - } - } - }; - - let mut meta = FileMeta::load(&buf)?; - let old_dir = meta.delete_version(&fi)?; - - if let Some(uuid) = old_dir { - let vid = fi.version_id.unwrap_or(Uuid::nil()); - let _ = meta.data.remove(vec![vid, uuid])?; - - let old_path = file_path.join(Path::new(uuid.to_string().as_str())); - check_path_length(old_path.to_string_lossy().as_ref())?; - - if let Err(err) = self.move_to_trash(&old_path, true, false).await { - if err != Error::FileNotFound { - return Err(err); - } - } - } - - if !meta.versions.is_empty() { - let buf = meta.marshal_msg()?; - return self - .write_all_meta(volume, format!("{}{}{}", path, SLASH_SEPARATOR, STORAGE_FORMAT_FILE).as_str(), &buf, true) - .await; - } - - // opts.undo_write && opts.old_data_dir.is_some_and(f) - if let Some(old_data_dir) = opts.old_data_dir { - if opts.undo_write { - let src_path = file_path.join(Path::new( - format!("{}{}{}", old_data_dir, SLASH_SEPARATOR, STORAGE_FORMAT_FILE_BACKUP).as_str(), - )); - let dst_path = file_path.join(Path::new(format!("{}{}{}", path, SLASH_SEPARATOR, STORAGE_FORMAT_FILE).as_str())); - return rename_all(src_path, dst_path, file_path).await; - } - } - - self.delete_file(&volume_dir, &xl_path, true, false).await - } - #[tracing::instrument(level = "debug", skip(self))] - async fn delete_versions( - &self, - volume: &str, - versions: Vec, - _opts: DeleteOptions, - ) -> Result>> { - let mut errs = Vec::with_capacity(versions.len()); - for _ in 0..versions.len() { - errs.push(None); - } - - for (i, ver) in versions.iter().enumerate() { - if let Err(e) = self.delete_versions_internal(volume, ver.name.as_str(), &ver.versions).await { - errs[i] = Some(e); - } else { - errs[i] = None; - } - } - - Ok(errs) - } - - #[tracing::instrument(skip(self))] - async fn read_multiple(&self, req: ReadMultipleReq) -> Result> { - let mut results = Vec::new(); - let mut found = 0; - - for v in req.files.iter() { - let fpath = self.get_object_path(&req.bucket, format!("{}/{}", &req.prefix, v).as_str())?; - let mut res = ReadMultipleResp { - bucket: req.bucket.clone(), - prefix: req.prefix.clone(), - file: v.clone(), - ..Default::default() - }; - - // if req.metadata_only {} - match read_file_all(&fpath).await { - Ok((data, meta)) => { - found += 1; - - if req.max_size > 0 && data.len() > req.max_size { - res.exists = true; - res.error = format!("max size ({}) exceeded: {}", req.max_size, data.len()); - results.push(res); - break; - } - - res.exists = true; - res.data = data; - res.mod_time = match meta.modified() { - Ok(md) => Some(OffsetDateTime::from(md)), - Err(_) => { - warn!("Not supported modified on this platform"); - None - } - }; - results.push(res); - - if req.max_results > 0 && found >= req.max_results { - break; - } - } - Err(e) => { - if !(e == Error::FileNotFound || e == Error::VolumeNotFound) { - res.exists = true; - res.error = e.to_string(); - } - - if req.abort404 && !res.exists { - results.push(res); - break; - } - - results.push(res); - } - } - } - - Ok(results) - } - - #[tracing::instrument(skip(self))] - async fn delete_volume(&self, volume: &str) -> Result<()> { - let p = self.get_bucket_path(volume)?; - - // TODO: 不能用递归删除，如果目录下面有文件，返回 errVolumeNotEmpty - - if let Err(err) = fs::remove_dir_all(&p).await { - match err.kind() { - ErrorKind::NotFound => (), - // ErrorKind::DirectoryNotEmpty => (), - kind => { - if kind.to_string() == "directory not empty" { - return Err(Error::VolumeNotEmpty); - } - - return Err(Error::from(err)); - } - } - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn disk_info(&self, _: &DiskInfoOptions) -> Result { - let mut info = Cache::get(self.disk_info_cache.clone()).await?; - // TODO: nr_requests, rotational - info.nr_requests = self.nrrequests; - info.rotational = self.rotational; - info.mount_path = self.path().to_str().unwrap().to_string(); - info.endpoint = self.endpoint.to_string(); - info.scanning = self.scanning.load(Ordering::SeqCst) == 1; - - Ok(info) - } - - // #[tracing::instrument(level = "info", skip_all)] - // async fn ns_scanner( - // &self, - // cache: &DataUsageCache, - // updates: Sender, - // scan_mode: HealScanMode, - // we_sleep: ShouldSleepFn, - // ) -> Result { - // self.scanning.fetch_add(1, Ordering::SeqCst); - // defer!(|| { self.scanning.fetch_sub(1, Ordering::SeqCst) }); - - // // must befor metadata_sys - // let Some(store) = new_object_layer_fn() else { return Err(Error::msg("errServerNotInitialized")) }; - - // let mut cache = cache.clone(); - // // Check if the current bucket has a configured lifecycle policy - // if let Ok((lc, _)) = metadata_sys::get_lifecycle_config(&cache.info.name).await { - // if lc_has_active_rules(&lc, "") { - // cache.info.life_cycle = Some(lc); - // } - // } - - // // Check if the current bucket has replication configuration - // if let Ok((rcfg, _)) = metadata_sys::get_replication_config(&cache.info.name).await { - // if rep_has_active_rules(&rcfg, "", true) { - // // TODO: globalBucketTargetSys - // } - // } - - // let vcfg = (BucketVersioningSys::get(&cache.info.name).await).ok(); - - // let loc = self.get_disk_location(); - // let disks = store - // .get_disks(loc.pool_idx.unwrap(), loc.disk_idx.unwrap()) - // .await - // .map_err(Error::from)?; - // let disk = Arc::new(LocalDisk::new(&self.endpoint(), false).await?); - // let disk_clone = disk.clone(); - // cache.info.updates = Some(updates.clone()); - // let mut data_usage_info = scan_data_folder( - // &disks, - // disk, - // &cache, - // Box::new(move |item: &ScannerItem| { - // let mut item = item.clone(); - // let disk = disk_clone.clone(); - // let vcfg = vcfg.clone(); - // Box::pin(async move { - // if !item.path.ends_with(&format!("{}{}", SLASH_SEPARATOR, STORAGE_FORMAT_FILE)) { - // return Err(Error::ScanSkipFile); - // } - // let stop_fn = ScannerMetrics::log(ScannerMetric::ScanObject); - // let mut res = HashMap::new(); - // let done_sz = ScannerMetrics::time_size(ScannerMetric::ReadMetadata).await; - // let buf = match disk.read_metadata(item.path.clone()).await { - // Ok(buf) => buf, - // Err(err) => { - // res.insert("err".to_string(), err.to_string()); - // stop_fn(&res).await; - // return Err(Error::ScanSkipFile); - // } - // }; - // done_sz(buf.len() as u64).await; - // res.insert("metasize".to_string(), buf.len().to_string()); - // item.transform_meda_dir(); - // let meta_cache = MetaCacheEntry { - // name: item.object_path().to_string_lossy().to_string(), - // metadata: buf, - // ..Default::default() - // }; - // let fivs = match meta_cache.file_info_versions(&item.bucket) { - // Ok(fivs) => fivs, - // Err(err) => { - // res.insert("err".to_string(), err.to_string()); - // stop_fn(&res).await; - // return Err(Error::ScanSkipFile); - // } - // }; - // let mut size_s = SizeSummary::default(); - // let done = ScannerMetrics::time(ScannerMetric::ApplyAll); - // let obj_infos = match item.apply_versions_actions(&fivs.versions).await { - // Ok(obj_infos) => obj_infos, - // Err(err) => { - // res.insert("err".to_string(), err.to_string()); - // stop_fn(&res).await; - // return Err(Error::ScanSkipFile); - // } - // }; - - // let versioned = if let Some(vcfg) = vcfg.as_ref() { - // vcfg.versioned(item.object_path().to_str().unwrap_or_default()) - // } else { - // false - // }; - - // let mut obj_deleted = false; - // for info in obj_infos.iter() { - // let done = ScannerMetrics::time(ScannerMetric::ApplyVersion); - // let sz: usize; - // (obj_deleted, sz) = item.apply_actions(info, &size_s).await; - // done().await; - - // if obj_deleted { - // break; - // } - - // let actual_sz = match info.get_actual_size() { - // Ok(size) => size, - // Err(_) => continue, - // }; - - // if info.delete_marker { - // size_s.delete_markers += 1; - // } - - // if info.version_id.is_some() && sz == actual_sz { - // size_s.versions += 1; - // } - - // size_s.total_size += sz; - - // if info.delete_marker { - // continue; - // } - // } - - // for frer_version in fivs.free_versions.iter() { - // let _obj_info = ObjectInfo::from_file_info( - // frer_version, - // &item.bucket, - // &item.object_path().to_string_lossy(), - // versioned, - // ); - // let done = ScannerMetrics::time(ScannerMetric::TierObjSweep); - // done().await; - // } - - // // todo: global trace - // if obj_deleted { - // return Err(Error::ScanIgnoreFileContrib); - // } - // done().await; - // Ok(size_s) - // }) - // }), - // scan_mode, - // we_sleep, - // ) - // .await - // .map_err(|e| Error::from(e.to_string()))?; // TODO: Error::from(e.to_string()) - // data_usage_info.info.last_update = Some(SystemTime::now()); - // info!("ns_scanner completed: {data_usage_info:?}"); - // Ok(data_usage_info) - // } - - // #[tracing::instrument(skip(self))] - // async fn healing(&self) -> Option { - // let healing_file = path_join(&[ - // self.path(), - // PathBuf::from(RUSTFS_META_BUCKET), - // PathBuf::from(BUCKET_META_PREFIX), - // PathBuf::from(HEALING_TRACKER_FILENAME), - // ]); - // let b = match fs::read(healing_file).await { - // Ok(b) => b, - // Err(_) => return None, - // }; - // if b.is_empty() { - // return None; - // } - // match HealingTracker::unmarshal_msg(&b) { - // Ok(h) => Some(h), - // Err(_) => Some(HealingTracker::default()), - // } - // } -} diff --git a/crates/disk/src/local_bak.rs b/crates/disk/src/local_bak.rs deleted file mode 100644 index e2080d635..000000000 --- a/crates/disk/src/local_bak.rs +++ /dev/null @@ -1,2364 +0,0 @@ -use super::error::{ - is_err_file_not_found, is_err_file_version_not_found, is_err_os_disk_full, is_sys_err_io, is_sys_err_not_empty, - is_sys_err_too_many_files, os_is_permission, -}; -use super::metacache::MetaCacheEntry; -use super::os::{is_root_disk, rename_all}; -use super::utils::{self, read_file_all, read_file_exists}; -use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; -use super::{ - os, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, Info, ReadMultipleReq, - ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, BUCKET_META_PREFIX, - RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE_BACKUP, -}; - -use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; -use crate::io::{FileReader, FileWriter}; -// use crate::new_object_layer_fn; - -use crate::utils::path::{ - self, clean, decode_dir_object, encode_dir_object, has_suffix, path_join, path_join_buf, GLOBAL_DIR_SUFFIX, - GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR, -}; - -use path_absolutize::Absolutize; -use rustfs_error::{ - conv_part_err_to_int, to_access_error, to_disk_error, to_file_error, to_unformatted_disk_error, to_volume_error, Error, - Result, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, - CHECK_PART_VOLUME_NOT_FOUND, -}; -use rustfs_filemeta::{get_file_info, read_xl_meta_no_data, FileInfo, FileInfoOpts, FileInfoVersions, FileMeta, RawFileInfo}; -use rustfs_rio::bitrot_verify; -use rustfs_utils::os::get_info; -use rustfs_utils::HashAlgorithm; -use std::collections::{HashMap, HashSet}; -use std::fmt::Debug; -use std::io::SeekFrom; -use std::sync::atomic::{AtomicU32, Ordering}; -use std::sync::Arc; -use std::time::{Duration, SystemTime}; -use std::{ - fs::Metadata, - path::{Path, PathBuf}, -}; -use time::OffsetDateTime; -use tokio::fs::{self, File}; -use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind}; -use tokio::sync::mpsc::Sender; -use tokio::sync::RwLock; -use tracing::{error, info, warn}; -use uuid::Uuid; - -#[derive(Debug)] -pub struct FormatInfo { - pub id: Option, - pub data: Vec, - pub file_info: Option, - pub last_check: Option, -} - -impl FormatInfo { - pub fn last_check_valid(&self) -> bool { - let now = OffsetDateTime::now_utc(); - self.file_info.is_some() - && self.id.is_some() - && self.last_check.is_some() - && (now.unix_timestamp() - self.last_check.unwrap().unix_timestamp() <= 1) - } -} - -pub struct LocalDisk { - pub root: PathBuf, - pub format_path: PathBuf, - pub format_info: RwLock, - pub endpoint: Endpoint, - pub disk_info_cache: Arc>, - pub scanning: AtomicU32, - pub rotational: bool, - pub fstype: String, - pub major: u64, - pub minor: u64, - pub nrrequests: u64, - // pub id: Mutex>, - // pub format_data: Mutex>, - // pub format_file_info: Mutex>, - // pub format_last_check: Mutex>, -} - -impl Debug for LocalDisk { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("LocalDisk") - .field("root", &self.root) - .field("format_path", &self.format_path) - .field("format_info", &self.format_info) - .field("endpoint", &self.endpoint) - .finish() - } -} - -impl LocalDisk { - pub async fn new(ep: &Endpoint, cleanup: bool) -> Result { - let root = fs::canonicalize(ep.get_file_path()).await?; - - if cleanup { - // TODO: 删除 tmp 数据 - } - - let format_path = Path::new(super::RUSTFS_META_BUCKET) - .join(Path::new(super::FORMAT_CONFIG_FILE)) - .absolutize_virtually(&root)? - .into_owned(); - - let (format_data, format_meta) = read_file_exists(&format_path).await?; - - let mut id = None; - // let mut format_legacy = false; - let mut format_last_check = None; - - if !format_data.is_empty() { - let s = format_data.as_slice(); - let fm = FormatV3::try_from(s)?; - let (set_idx, disk_idx) = fm.find_disk_index_by_disk_id(fm.erasure.this)?; - - if set_idx as i32 != ep.set_idx || disk_idx as i32 != ep.disk_idx { - return Err(Error::InconsistentDisk); - } - - id = Some(fm.erasure.this); - // format_legacy = fm.erasure.distribution_algo == DistributionAlgoVersion::V1; - format_last_check = Some(OffsetDateTime::now_utc()); - } - - let format_info = FormatInfo { - id, - data: format_data, - file_info: format_meta, - last_check: format_last_check, - }; - let root_clone = root.clone(); - let update_fn: UpdateFn = Box::new(move || { - let disk_id = id.map_or("".to_string(), |id| id.to_string()); - let root = root_clone.clone(); - Box::pin(async move { - match get_disk_info(root.clone()).await { - Ok((info, root)) => { - let disk_info = DiskInfo { - total: info.total, - free: info.free, - used: info.used, - used_inodes: info.files - info.ffree, - free_inodes: info.ffree, - major: info.major, - minor: info.minor, - fs_type: info.fstype, - root_disk: root, - id: disk_id.to_string(), - ..Default::default() - }; - // if root { - // return Err(Error::new(DiskError::DriveIsRoot)); - // } - - // disk_info.healing = - Ok(disk_info) - } - Err(err) => Err(err), - } - }) - }); - - let cache = Cache::new(update_fn, Duration::from_secs(1), Opts::default()); - - // TODO: DIRECT suport - // TODD: DiskInfo - let mut disk = Self { - root: root.clone(), - endpoint: ep.clone(), - format_path, - format_info: RwLock::new(format_info), - disk_info_cache: Arc::new(cache), - scanning: AtomicU32::new(0), - rotational: Default::default(), - fstype: Default::default(), - minor: Default::default(), - major: Default::default(), - nrrequests: Default::default(), - // // format_legacy, - // format_file_info: Mutex::new(format_meta), - // format_data: Mutex::new(format_data), - // format_last_check: Mutex::new(format_last_check), - }; - let (info, _root) = get_disk_info(root).await?; - disk.major = info.major; - disk.minor = info.minor; - disk.fstype = info.fstype; - - // if root { - // return Err(Error::new(DiskError::DriveIsRoot)); - // } - - if info.nrrequests > 0 { - disk.nrrequests = info.nrrequests; - } - - if info.rotational { - disk.rotational = true; - } - - disk.make_meta_volumes().await?; - - Ok(disk) - } - - fn is_valid_volname(volname: &str) -> bool { - if volname.len() < 3 { - return false; - } - - if cfg!(target_os = "windows") { - // 在 Windows 上，卷名不应该包含保留字符。 - // 这个正则表达式匹配了不允许的字符。 - if volname.contains('|') - || volname.contains('<') - || volname.contains('>') - || volname.contains('?') - || volname.contains('*') - || volname.contains(':') - || volname.contains('"') - || volname.contains('\\') - { - return false; - } - } else { - // 对于非 Windows 系统，可能需要其他的验证逻辑。 - } - - true - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn check_format_json(&self) -> Result { - let md = std::fs::metadata(&self.format_path).map_err(|e| to_unformatted_disk_error(e))?; - Ok(md) - } - async fn make_meta_volumes(&self) -> Result<()> { - let buckets = format!("{}/{}", super::RUSTFS_META_BUCKET, super::BUCKET_META_PREFIX); - let multipart = format!("{}/{}", super::RUSTFS_META_BUCKET, "multipart"); - let config = format!("{}/{}", super::RUSTFS_META_BUCKET, "config"); - let tmp = format!("{}/{}", super::RUSTFS_META_BUCKET, "tmp"); - let defaults = vec![buckets.as_str(), multipart.as_str(), config.as_str(), tmp.as_str()]; - - self.make_volumes(defaults).await - } - - pub fn resolve_abs_path(&self, path: impl AsRef) -> Result { - Ok(path.as_ref().absolutize_virtually(&self.root)?.into_owned()) - } - - pub fn get_object_path(&self, bucket: &str, key: &str) -> Result { - let dir = Path::new(&bucket); - let file_path = Path::new(&key); - self.resolve_abs_path(dir.join(file_path)) - } - - pub fn get_bucket_path(&self, bucket: &str) -> Result { - let dir = Path::new(&bucket); - self.resolve_abs_path(dir) - } - - // /// Write to the filesystem atomically. - // /// This is done by first writing to a temporary location and then moving the file. - // pub(crate) async fn prepare_file_write<'a>(&self, path: &'a PathBuf) -> Result> { - // let tmp_path = self.get_object_path(RUSTFS_META_TMP_BUCKET, Uuid::new_v4().to_string().as_str())?; - - // debug!("prepare_file_write tmp_path:{:?}, path:{:?}", &tmp_path, &path); - - // let file = File::create(&tmp_path).await?; - // let writer = BufWriter::new(file); - // Ok(FileWriter { - // tmp_path, - // dest_path: path, - // writer, - // clean_tmp: true, - // }) - // } - - #[allow(unreachable_code)] - #[allow(unused_variables)] - pub async fn move_to_trash(&self, delete_path: &PathBuf, recursive: bool, immediate_purge: bool) -> Result<()> { - if recursive { - remove_all_std(delete_path).map_err(to_file_error)?; - } else { - remove_std(delete_path).map_err(to_file_error)?; - } - - return Ok(()); - - // TODO: 异步通知 检测硬盘空间 清空回收站 - - let trash_path = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; - if let Some(parent) = trash_path.parent() { - if !parent.exists() { - fs::create_dir_all(parent).await?; - } - } - - let err = if recursive { - rename_all(delete_path, trash_path, self.get_bucket_path(super::RUSTFS_META_TMP_DELETED_BUCKET)?) - .await - .err() - } else { - rename(&delete_path, &trash_path) - .await - .map_err(|e| to_file_error(e).into()) - .err() - }; - - if immediate_purge || delete_path.to_string_lossy().ends_with(path::SLASH_SEPARATOR) { - warn!("move_to_trash immediate_purge {:?}", &delete_path.to_string_lossy()); - let trash_path2 = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; - let _ = rename_all( - encode_dir_object(delete_path.to_string_lossy().as_ref()), - trash_path2, - self.get_bucket_path(super::RUSTFS_META_TMP_DELETED_BUCKET)?, - ) - .await; - } - - if let Some(err) = err { - if err == Error::DiskFull { - if recursive { - remove_all(delete_path).await.map_err(to_file_error)?; - } else { - remove(delete_path).await.map_err(to_file_error)?; - } - } - - return Err(err); - } - - Ok(()) - } - - #[tracing::instrument(level = "debug", skip(self))] - pub async fn delete_file( - &self, - base_path: &PathBuf, - delete_path: &PathBuf, - recursive: bool, - immediate_purge: bool, - ) -> Result<()> { - // debug!("delete_file {:?}\n base_path:{:?}", &delete_path, &base_path); - - if is_root_path(base_path) || is_root_path(delete_path) { - // debug!("delete_file skip {:?}", &delete_path); - return Ok(()); - } - - if !delete_path.starts_with(base_path) || base_path == delete_path { - // debug!("delete_file skip {:?}", &delete_path); - return Ok(()); - } - - if recursive { - self.move_to_trash(delete_path, recursive, immediate_purge).await?; - } else if delete_path.is_dir() { - // debug!("delete_file remove_dir {:?}", &delete_path); - if let Err(err) = fs::remove_dir(&delete_path).await { - // debug!("remove_dir err {:?} when {:?}", &err, &delete_path); - match err.kind() { - ErrorKind::NotFound => (), - ErrorKind::DirectoryNotEmpty => { - warn!("delete_file remove_dir {:?} err {}", &delete_path, err.to_string()); - return Err(Error::FileAccessDenied.into()); - } - _ => (), - } - } - // debug!("delete_file remove_dir done {:?}", &delete_path); - } else if let Err(err) = fs::remove_file(&delete_path).await { - // debug!("remove_file err {:?} when {:?}", &err, &delete_path); - match err.kind() { - ErrorKind::NotFound => (), - _ => { - warn!("delete_file remove_file {:?} err {:?}", &delete_path, &err); - return Err(Error::FileAccessDenied.into()); - } - } - } - - if let Some(dir_path) = delete_path.parent() { - Box::pin(self.delete_file(base_path, &PathBuf::from(dir_path), false, false)).await?; - } - - // debug!("delete_file done {:?}", &delete_path); - Ok(()) - } - - /// read xl.meta raw data - #[tracing::instrument(level = "debug", skip(self, volume_dir, file_path))] - async fn read_raw( - &self, - bucket: &str, - volume_dir: impl AsRef, - file_path: impl AsRef, - read_data: bool, - ) -> Result<(Vec, Option)> { - if file_path.as_ref().as_os_str().is_empty() { - return Err(Error::FileNotFound.into()); - } - - let meta_path = file_path.as_ref().join(Path::new(super::STORAGE_FORMAT_FILE)); - - let res = { - if read_data { - self.read_all_data_with_dmtime(bucket, volume_dir, meta_path).await - } else { - match self.read_metadata_with_dmtime(meta_path).await { - Ok(res) => Ok(res), - Err(err) => { - if err == Error::FileNotFound - && !skip_access_checks(volume_dir.as_ref().to_string_lossy().to_string().as_str()) - { - if let Err(aerr) = access(volume_dir.as_ref()).await { - if aerr.kind() == ErrorKind::NotFound { - warn!("read_metadata_with_dmtime os err {:?}", &aerr); - return Err(Error::VolumeNotFound.into()); - } - } - } - - Err(err) - } - } - } - }; - - let (buf, mtime) = res?; - if buf.is_empty() { - return Err(Error::FileNotFound.into()); - } - - Ok((buf, mtime)) - } - - async fn read_metadata(&self, file_path: impl AsRef) -> Result> { - // TODO: suport timeout - let (data, _) = self.read_metadata_with_dmtime(file_path.as_ref()).await?; - Ok(data) - } - - async fn read_metadata_with_dmtime(&self, file_path: impl AsRef) -> Result<(Vec, Option)> { - check_path_length(file_path.as_ref().to_string_lossy().as_ref())?; - - let mut f = super::fs::open_file(file_path.as_ref(), O_RDONLY) - .await - .map_err(to_file_error)?; - - let meta = f.metadata().await.map_err(to_file_error)?; - - if meta.is_dir() { - // fix use io::Error - return Err(Error::FileNotFound.into()); - } - - let size = meta.len() as usize; - - let data = read_xl_meta_no_data(&mut f, size).await?; - - let modtime = match meta.modified() { - Ok(md) => Some(OffsetDateTime::from(md)), - Err(_) => None, - }; - - Ok((data, modtime)) - } - - async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef, file_path: impl AsRef) -> Result> { - // TODO: timeout suport - let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?; - Ok(data) - } - - #[tracing::instrument(level = "debug", skip(self, volume_dir, file_path))] - async fn read_all_data_with_dmtime( - &self, - volume: &str, - volume_dir: impl AsRef, - file_path: impl AsRef, - ) -> Result<(Vec, Option)> { - let mut f = match super::fs::open_file(file_path.as_ref(), O_RDONLY).await { - Ok(f) => f, - Err(e) => { - if e.kind() == ErrorKind::NotFound { - if !skip_access_checks(volume) { - if let Err(er) = super::fs::access(volume_dir.as_ref()).await { - if er.kind() == ErrorKind::NotFound { - warn!("read_all_data_with_dmtime os err {:?}", &er); - return Err(Error::VolumeNotFound.into()); - } - } - } - - return Err(Error::FileNotFound.into()); - } - - return Err(to_file_error(e).into()); - } - }; - - let meta = f.metadata().await.map_err(to_file_error)?; - - if meta.is_dir() { - return Err(Error::FileNotFound.into()); - } - - let size = meta.len() as usize; - let mut bytes = Vec::new(); - bytes.try_reserve_exact(size)?; - - f.read_to_end(&mut bytes).await.map_err(to_file_error)?; - - let modtime = match meta.modified() { - Ok(md) => Some(OffsetDateTime::from(md)), - Err(_) => None, - }; - - Ok((bytes, modtime)) - } - - async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &Vec) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; - let xlpath = self.get_object_path(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str())?; - - let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await?; - - let mut fm = FileMeta::default(); - - fm.unmarshal_msg(&data)?; - - for fi in fis { - let data_dir = match fm.delete_version(fi) { - Ok(res) => res, - Err(err) => { - if !fi.deleted && (err == Error::FileVersionNotFound || err == Error::FileNotFound) { - continue; - } - - return Err(err); - } - }; - - if let Some(dir) = data_dir { - let vid = fi.version_id.unwrap_or_default(); - let _ = fm.data.remove(vec![vid, dir]); - - let dir_path = self.get_object_path(volume, format!("{}/{}", path, dir).as_str())?; - if let Err(err) = self.move_to_trash(&dir_path, true, false).await { - if !(err == Error::FileNotFound || err == Error::DiskNotFound) { - return Err(err); - } - }; - } - } - - // 没有版本了，删除 xl.meta - if fm.versions.is_empty() { - self.delete_file(&volume_dir, &xlpath, true, false).await?; - return Ok(()); - } - - // 更新 xl.meta - let buf = fm.marshal_msg()?; - - let volume_dir = self.get_bucket_path(volume)?; - - self.write_all_private( - volume, - format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str(), - &buf, - true, - volume_dir, - ) - .await?; - - Ok(()) - } - - async fn write_all_meta(&self, volume: &str, path: &str, buf: &[u8], sync: bool) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().as_ref())?; - - let tmp_volume_dir = self.get_bucket_path(super::RUSTFS_META_TMP_BUCKET)?; - let tmp_file_path = tmp_volume_dir.join(Path::new(Uuid::new_v4().to_string().as_str())); - - self.write_all_internal(&tmp_file_path, buf, sync, tmp_volume_dir).await?; - - super::os::rename_all(tmp_file_path, file_path, volume_dir).await?; - - Ok(()) - } - - // write_all_public for trail - async fn write_all_public(&self, volume: &str, path: &str, data: Vec) -> Result<()> { - if volume == super::RUSTFS_META_BUCKET && path == super::FORMAT_CONFIG_FILE { - let mut format_info = self.format_info.write().await; - format_info.data.clone_from(&data); - } - - let volume_dir = self.get_bucket_path(volume)?; - - self.write_all_private(volume, path, &data, true, volume_dir).await?; - - Ok(()) - } - - // write_all_private with check_path_length - #[tracing::instrument(level = "debug", skip_all)] - pub async fn write_all_private( - &self, - volume: &str, - path: &str, - buf: &[u8], - sync: bool, - skip_parent: impl AsRef, - ) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().as_ref())?; - - self.write_all_internal(file_path, buf, sync, skip_parent) - .await - .map_err(to_file_error)?; - - Ok(()) - } - // write_all_internal do write file - pub async fn write_all_internal( - &self, - file_path: impl AsRef, - data: impl AsRef<[u8]>, - sync: bool, - skip_parent: impl AsRef, - ) -> std::io::Result<()> { - let flags = super::fs::O_CREATE | super::fs::O_WRONLY | super::fs::O_TRUNC; - - let mut f = { - if sync { - // TODO: suport sync - self.open_file(file_path.as_ref(), flags, skip_parent.as_ref()).await? - } else { - self.open_file(file_path.as_ref(), flags, skip_parent.as_ref()).await? - } - }; - - f.write_all(data.as_ref()).await?; - - Ok(()) - } - - async fn open_file(&self, path: impl AsRef, mode: usize, skip_parent: impl AsRef) -> Result { - let mut skip_parent = skip_parent.as_ref(); - if skip_parent.as_os_str().is_empty() { - skip_parent = self.root.as_path(); - } - - if let Some(parent) = path.as_ref().parent() { - os::make_dir_all(parent, skip_parent).await?; - } - - let f = super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?; - - Ok(f) - } - - #[allow(dead_code)] - fn get_metrics(&self) -> DiskMetrics { - DiskMetrics::default() - } - - async fn bitrot_verify( - &self, - part_path: &PathBuf, - part_size: usize, - algo: HashAlgorithm, - sum: &[u8], - shard_size: usize, - ) -> Result<()> { - let file = super::fs::open_file(part_path, O_CREATE | O_WRONLY) - .await - .map_err(to_file_error)?; - - let meta = file.metadata().await?; - let file_size = meta.len() as usize; - - bitrot_verify(file, file_size, part_size, algo, sum.to_vec(), shard_size) - .await - .map_err(to_file_error)?; - - Ok(()) - } - - async fn scan_dir( - &self, - current: &mut String, - opts: &WalkDirOptions, - out: &mut MetacacheWriter, - objs_returned: &mut i32, - ) -> Result<()> { - let forward = { - opts.forward_to.as_ref().filter(|v| v.starts_with(&*current)).map(|v| { - let forward = v.trim_start_matches(&*current); - if let Some(idx) = forward.find('/') { - forward[..idx].to_owned() - } else { - forward.to_owned() - } - }) - // if let Some(forward_to) = &opts.forward_to { - - // } else { - // None - // } - // if !opts.forward_to.is_empty() && opts.forward_to.starts_with(&*current) { - // let forward = opts.forward_to.trim_start_matches(&*current); - // if let Some(idx) = forward.find('/') { - // &forward[..idx] - // } else { - // forward - // } - // } else { - // "" - // } - }; - - if opts.limit > 0 && *objs_returned >= opts.limit { - return Ok(()); - } - - let mut entries = match self.list_dir("", &opts.bucket, current, -1).await { - Ok(res) => res, - Err(e) => { - if e != Error::VolumeNotFound && e != Error::FileNotFound { - info!("scan list_dir {}, err {:?}", ¤t, &e); - } - - if opts.report_notfound && (e == Error::VolumeNotFound || e == Error::FileNotFound) && current == &opts.base_dir { - return Err(Error::FileNotFound.into()); - } - - return Ok(()); - } - }; - - if entries.is_empty() { - return Ok(()); - } - - let s = SLASH_SEPARATOR.chars().next().unwrap_or_default(); - *current = current.trim_matches(s).to_owned(); - - let bucket = opts.bucket.as_str(); - - let mut dir_objes = HashSet::new(); - - // 第一层过滤 - for item in entries.iter_mut() { - let entry = item.clone(); - // check limit - if opts.limit > 0 && *objs_returned >= opts.limit { - return Ok(()); - } - // check prefix - if let Some(filter_prefix) = &opts.filter_prefix { - if !entry.starts_with(filter_prefix) { - *item = "".to_owned(); - continue; - } - } - - if let Some(forward) = &forward { - if &entry < forward { - *item = "".to_owned(); - continue; - } - } - - if entry.ends_with(SLASH_SEPARATOR) { - if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) { - let entry = format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR); - dir_objes.insert(entry.clone()); - *item = entry; - continue; - } - - *item = entry.trim_end_matches(SLASH_SEPARATOR).to_owned(); - continue; - } - - *item = "".to_owned(); - - if entry.ends_with(STORAGE_FORMAT_FILE) { - // - let metadata = self - .read_metadata(self.get_object_path(bucket, format!("{}/{}", ¤t, &entry).as_str())?) - .await?; - - // 用 strip_suffix 只删除一次 - let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned(); - let name = entry.trim_end_matches(SLASH_SEPARATOR); - let name = decode_dir_object(format!("{}/{}", ¤t, &name).as_str()); - - out.write_obj(&MetaCacheEntry { - name, - metadata, - ..Default::default() - }) - .await?; - *objs_returned += 1; - - return Ok(()); - } - } - - entries.sort(); - - let mut entries = entries.as_slice(); - if let Some(forward) = &forward { - for (i, entry) in entries.iter().enumerate() { - if entry >= forward || forward.starts_with(entry.as_str()) { - entries = &entries[i..]; - break; - } - } - } - - let mut dir_stack: Vec = Vec::with_capacity(5); - - for entry in entries.iter() { - if opts.limit > 0 && *objs_returned >= opts.limit { - return Ok(()); - } - - if entry.is_empty() { - continue; - } - - let name = path::path_join_buf(&[current, entry]); - - if !dir_stack.is_empty() { - if let Some(pop) = dir_stack.pop() { - if pop < name { - // - out.write_obj(&MetaCacheEntry { - name: pop.clone(), - ..Default::default() - }) - .await?; - - if opts.recursive { - let mut opts = opts.clone(); - opts.filter_prefix = None; - if let Err(er) = Box::pin(self.scan_dir(&mut pop.clone(), &opts, out, objs_returned)).await { - error!("scan_dir err {:?}", er); - } - } - } - } - } - - let mut meta = MetaCacheEntry { - name, - ..Default::default() - }; - - let mut is_dir_obj = false; - - if let Some(_dir) = dir_objes.get(entry) { - is_dir_obj = true; - meta.name - .truncate(meta.name.len() - meta.name.chars().last().unwrap().len_utf8()); - meta.name.push_str(GLOBAL_DIR_SUFFIX_WITH_SLASH); - } - - let fname = format!("{}/{}", &meta.name, STORAGE_FORMAT_FILE); - - match self.read_metadata(self.get_object_path(&opts.bucket, fname.as_str())?).await { - Ok(res) => { - if is_dir_obj { - meta.name = meta.name.trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH).to_owned(); - meta.name.push_str(SLASH_SEPARATOR); - } - - meta.metadata = res; - - out.write_obj(&meta).await?; - *objs_returned += 1; - } - Err(err) => { - if err == Error::FileNotFound || err == Error::IsNotRegular { - // NOT an object, append to stack (with slash) - // If dirObject, but no metadata (which is unexpected) we skip it. - if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await { - meta.name.push_str(SLASH_SEPARATOR); - dir_stack.push(meta.name); - } - } - - continue; - } - }; - } - - while let Some(dir) = dir_stack.pop() { - if opts.limit > 0 && *objs_returned >= opts.limit { - return Ok(()); - } - - out.write_obj(&MetaCacheEntry { - name: dir.clone(), - ..Default::default() - }) - .await?; - *objs_returned += 1; - - if opts.recursive { - let mut dir = dir; - let mut opts = opts.clone(); - opts.filter_prefix = None; - if let Err(er) = Box::pin(self.scan_dir(&mut dir, &opts, out, objs_returned)).await { - warn!("scan_dir err {:?}", &er); - } - } - } - - Ok(()) - } -} - -fn is_root_path(path: impl AsRef) -> bool { - path.as_ref().components().count() == 1 && path.as_ref().has_root() -} - -fn skip_access_checks(p: impl AsRef) -> bool { - let vols = [ - super::RUSTFS_META_TMP_DELETED_BUCKET, - super::RUSTFS_META_TMP_BUCKET, - super::RUSTFS_META_MULTIPART_BUCKET, - super::RUSTFS_META_BUCKET, - ]; - - for v in vols.iter() { - if p.as_ref().starts_with(v) { - return true; - } - } - - false -} - -#[async_trait::async_trait] -impl DiskAPI for LocalDisk { - #[tracing::instrument(skip(self))] - fn to_string(&self) -> String { - self.root.to_string_lossy().to_string() - } - #[tracing::instrument(skip(self))] - fn is_local(&self) -> bool { - true - } - #[tracing::instrument(skip(self))] - fn host_name(&self) -> String { - self.endpoint.host_port() - } - #[tracing::instrument(skip(self))] - async fn is_online(&self) -> bool { - self.check_format_json().await.is_ok() - } - - #[tracing::instrument(skip(self))] - fn endpoint(&self) -> Endpoint { - self.endpoint.clone() - } - - #[tracing::instrument(skip(self))] - async fn close(&self) -> Result<()> { - Ok(()) - } - - #[tracing::instrument(skip(self))] - fn path(&self) -> PathBuf { - self.root.clone() - } - - #[tracing::instrument(skip(self))] - fn get_disk_location(&self) -> DiskLocation { - DiskLocation { - pool_idx: { - if self.endpoint.pool_idx < 0 { - None - } else { - Some(self.endpoint.pool_idx as usize) - } - }, - set_idx: { - if self.endpoint.set_idx < 0 { - None - } else { - Some(self.endpoint.set_idx as usize) - } - }, - disk_idx: { - if self.endpoint.disk_idx < 0 { - None - } else { - Some(self.endpoint.disk_idx as usize) - } - }, - } - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn get_disk_id(&self) -> Result> { - let mut format_info = self.format_info.write().await; - - let id = format_info.id; - - if format_info.last_check_valid() { - return Ok(id); - } - - let file_meta = self.check_format_json().await?; - - if let Some(file_info) = &format_info.file_info { - if super::fs::same_file(&file_meta, file_info) { - format_info.last_check = Some(OffsetDateTime::now_utc()); - - return Ok(id); - } - } - - let b = tokio::fs::read(&self.format_path) - .await - .map_err(|e| to_unformatted_disk_error(e))?; - - let fm = FormatV3::try_from(b.as_slice()).map_err(|e| { - warn!("decode format.json err {:?}", e); - Error::CorruptedBackend - })?; - - let (m, n) = fm.find_disk_index_by_disk_id(fm.erasure.this)?; - - let disk_id = fm.erasure.this; - - if m as i32 != self.endpoint.set_idx || n as i32 != self.endpoint.disk_idx { - return Err(Error::InconsistentDisk.into()); - } - - format_info.id = Some(disk_id); - format_info.file_info = Some(file_meta); - format_info.data = b; - format_info.last_check = Some(OffsetDateTime::now_utc()); - - Ok(Some(disk_id)) - } - - #[tracing::instrument(skip(self))] - async fn set_disk_id(&self, id: Option) -> Result<()> { - // 本地不需要设置 - // TODO: add check_id_store - let mut format_info = self.format_info.write().await; - format_info.id = id; - Ok(()) - } - - #[must_use] - #[tracing::instrument(skip(self))] - async fn read_all(&self, volume: &str, path: &str) -> Result> { - if volume == super::RUSTFS_META_BUCKET && path == super::FORMAT_CONFIG_FILE { - let format_info = self.format_info.read().await; - if !format_info.data.is_empty() { - return Ok(format_info.data.clone()); - } - } - // TOFIX: - let p = self.get_object_path(volume, path)?; - let data = utils::read_all(&p).await?; - - Ok(data) - } - - #[tracing::instrument(level = "debug", skip_all)] - async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()> { - self.write_all_public(volume, path, data).await - } - - #[tracing::instrument(skip(self))] - async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().to_string().as_str())?; - - self.delete_file(&volume_dir, &file_path, opt.recursive, opt.immediate) - .await?; - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let mut resp = CheckPartsResp { - results: vec![0; fi.parts.len()], - }; - - let erasure = &fi.erasure; - for (i, part) in fi.parts.iter().enumerate() { - let checksum_info = erasure.get_checksum_info(part.number); - let part_path = Path::new(&volume_dir) - .join(path) - .join(fi.data_dir.map_or("".to_string(), |dir| dir.to_string())) - .join(format!("part.{}", part.number)); - let err = (self - .bitrot_verify( - &part_path, - erasure.shard_file_size(part.size), - checksum_info.algorithm, - &checksum_info.hash, - erasure.shard_size(), - ) - .await) - .err(); - resp.results[i] = conv_part_err_to_int(&err); - if resp.results[i] == CHECK_PART_UNKNOWN { - if let Some(err) = err { - match err { - Error::FileAccessDenied => {} - _ => { - info!("part unknown, disk: {}, path: {:?}", self.to_string(), part_path); - } - } - } - } - } - - Ok(resp) - } - - #[tracing::instrument(skip(self))] - async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - let volume_dir = self.get_bucket_path(volume)?; - check_path_length(volume_dir.join(path).to_string_lossy().as_ref())?; - let mut resp = CheckPartsResp { - results: vec![0; fi.parts.len()], - }; - - for (i, part) in fi.parts.iter().enumerate() { - let file_path = Path::new(&volume_dir) - .join(path) - .join(fi.data_dir.map_or("".to_string(), |dir| dir.to_string())) - .join(format!("part.{}", part.number)); - - match lstat(file_path).await { - Ok(st) => { - if st.is_dir() { - resp.results[i] = CHECK_PART_FILE_NOT_FOUND; - continue; - } - if (st.len() as usize) < fi.erasure.shard_file_size(part.size) { - resp.results[i] = CHECK_PART_FILE_CORRUPT; - continue; - } - - resp.results[i] = CHECK_PART_SUCCESS; - } - Err(err) => { - if err.kind() == ErrorKind::NotFound { - if !skip_access_checks(volume) { - if let Err(err) = super::fs::access(&volume_dir).await { - if err.kind() == ErrorKind::NotFound { - resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND; - continue; - } - } - } - resp.results[i] = CHECK_PART_FILE_NOT_FOUND; - } - continue; - } - } - } - - Ok(resp) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()> { - let src_volume_dir = self.get_bucket_path(src_volume)?; - let dst_volume_dir = self.get_bucket_path(dst_volume)?; - if !skip_access_checks(src_volume) { - super::fs::access_std(&src_volume_dir).map_err(to_file_error)? - } - if !skip_access_checks(dst_volume) { - super::fs::access_std(&dst_volume_dir).map_err(to_file_error)? - } - - let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR); - let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR); - - if !src_is_dir && dst_is_dir || src_is_dir && !dst_is_dir { - warn!( - "rename_part src and dst must be both dir or file src_is_dir:{}, dst_is_dir:{}", - src_is_dir, dst_is_dir - ); - return Err(Error::FileAccessDenied.into()); - } - - let src_file_path = src_volume_dir.join(Path::new(src_path)); - let dst_file_path = dst_volume_dir.join(Path::new(dst_path)); - - // warn!("rename_part src_file_path:{:?}, dst_file_path:{:?}", &src_file_path, &dst_file_path); - - check_path_length(src_file_path.to_string_lossy().as_ref())?; - check_path_length(dst_file_path.to_string_lossy().as_ref())?; - - if src_is_dir { - let meta_op = match lstat_std(&src_file_path) { - Ok(meta) => Some(meta), - Err(e) => { - let err = to_file_error(e).into(); - - if err == Error::FaultyDisk { - return Err(err); - } - - if err != Error::FileNotFound { - return Err(err); - } - None - } - }; - - if let Some(meta) = meta_op { - if !meta.is_dir() { - warn!("rename_part src is not dir {:?}", &src_file_path); - return Err(Error::FileAccessDenied.into()); - } - } - - super::fs::remove_std(&dst_file_path).map_err(to_file_error)?; - } - os::rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await?; - - self.write_all(dst_volume, format!("{}.meta", dst_path).as_str(), meta) - .await?; - - if let Some(parent) = src_file_path.parent() { - self.delete_file(&src_volume_dir, &parent.to_path_buf(), false, false).await?; - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { - let src_volume_dir = self.get_bucket_path(src_volume)?; - let dst_volume_dir = self.get_bucket_path(dst_volume)?; - if !skip_access_checks(src_volume) { - if let Err(e) = super::fs::access(&src_volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - if !skip_access_checks(dst_volume) { - if let Err(e) = super::fs::access(&dst_volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR); - let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR); - if (dst_is_dir || src_is_dir) && (!dst_is_dir || !src_is_dir) { - return Err(Error::FileAccessDenied.into()); - } - - let src_file_path = src_volume_dir.join(Path::new(&src_path)); - check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; - - let dst_file_path = dst_volume_dir.join(Path::new(&dst_path)); - check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; - - if src_is_dir { - let meta_op = match lstat(&src_file_path).await { - Ok(meta) => Some(meta), - Err(e) => { - if is_sys_err_io(&e) { - return Err(Error::FaultyDisk.into()); - } - - if e.kind() != ErrorKind::NotFound { - return Err(to_file_error(e).into()); - } - None - } - }; - - if let Some(meta) = meta_op { - if !meta.is_dir() { - return Err(Error::FileAccessDenied.into()); - } - } - - super::fs::remove(&dst_file_path).await.map_err(to_file_error)?; - } - - os::rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await?; - - if let Some(parent) = src_file_path.parent() { - let _ = self.delete_file(&src_volume_dir, &parent.to_path_buf(), false, false).await; - } - - Ok(()) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn create_file(&self, origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result { - // warn!("disk create_file: origvolume: {}, volume: {}, path: {}", origvolume, volume, path); - - if !origvolume.is_empty() { - let origvolume_dir = self.get_bucket_path(origvolume)?; - if !skip_access_checks(origvolume) { - if let Err(e) = super::fs::access(&origvolume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - } - - let volume_dir = self.get_bucket_path(volume)?; - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().to_string().as_str())?; - - // TODO: writeAllDirect io.copy - // info!("file_path: {:?}", file_path); - if let Some(parent) = file_path.parent() { - os::make_dir_all(parent, &volume_dir).await?; - } - let f = super::fs::open_file(&file_path, O_CREATE | O_WRONLY) - .await - .map_err(to_file_error)?; - - Ok(Box::new(f)) - - // Ok(()) - } - - #[tracing::instrument(level = "debug", skip(self))] - // async fn append_file(&self, volume: &str, path: &str, mut r: DuplexStream) -> Result { - async fn append_file(&self, volume: &str, path: &str) -> Result { - warn!("disk append_file: volume: {}, path: {}", volume, path); - - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().to_string().as_str())?; - - let f = self.open_file(file_path, O_CREATE | O_APPEND | O_WRONLY, volume_dir).await?; - - Ok(Box::new(f)) - } - - // TODO: io verifier - #[tracing::instrument(level = "debug", skip(self))] - async fn read_file(&self, volume: &str, path: &str) -> Result { - // warn!("disk read_file: volume: {}, path: {}", volume, path); - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().to_string().as_str())?; - - let f = self.open_file(file_path, O_RDONLY, volume_dir).await?; - - Ok(Box::new(f)) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result { - // warn!( - // "disk read_file_stream: volume: {}, path: {}, offset: {}, length: {}", - // volume, path, offset, length - // ); - - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(file_path.to_string_lossy().to_string().as_str())?; - - let mut f = self.open_file(file_path, O_RDONLY, volume_dir).await?; - - let meta = f.metadata().await?; - if meta.len() < (offset + length) as u64 { - error!( - "read_file_stream: file size is less than offset + length {} + {} = {}", - offset, - length, - meta.len() - ); - return Err(Error::FileCorrupt.into()); - } - - f.seek(SeekFrom::Start(offset as u64)).await?; - - Ok(Box::new(f)) - } - #[tracing::instrument(level = "debug", skip(self))] - async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result> { - if !origvolume.is_empty() { - let origvolume_dir = self.get_bucket_path(origvolume)?; - if !skip_access_checks(origvolume) { - if let Err(e) = super::fs::access(origvolume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - } - - let volume_dir = self.get_bucket_path(volume)?; - let dir_path_abs = volume_dir.join(Path::new(&dir_path.trim_start_matches(SLASH_SEPARATOR))); - - let entries = match os::read_dir(&dir_path_abs, count).await { - Ok(res) => res, - Err(e) => { - if e.kind() == ErrorKind::NotFound && !skip_access_checks(volume) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - return Err(to_volume_error(e).into()); - } - }; - - Ok(entries) - } - - // FIXME: TODO: io.writer TODO cancel - #[tracing::instrument(level = "debug", skip(self, wr))] - async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> { - let volume_dir = self.get_bucket_path(&opts.bucket)?; - - if !skip_access_checks(&opts.bucket) { - if let Err(e) = super::fs::access(&volume_dir).await { - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let mut wr = wr; - - let mut out = MetacacheWriter::new(&mut wr); - - let mut objs_returned = 0; - - if opts.base_dir.ends_with(SLASH_SEPARATOR) { - let fpath = self.get_object_path( - &opts.bucket, - path_join_buf(&[ - format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX).as_str(), - STORAGE_FORMAT_FILE, - ]) - .as_str(), - )?; - - if let Ok(data) = self.read_metadata(fpath).await { - let meta = MetaCacheEntry { - name: opts.base_dir.clone(), - metadata: data, - ..Default::default() - }; - out.write_obj(&meta).await?; - objs_returned += 1; - } - } - - let mut current = opts.base_dir.clone(); - self.scan_dir(&mut current, &opts, &mut out, &mut objs_returned).await?; - - Ok(()) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn rename_data( - &self, - src_volume: &str, - src_path: &str, - fi: FileInfo, - dst_volume: &str, - dst_path: &str, - ) -> Result { - let src_volume_dir = self.get_bucket_path(src_volume)?; - if !skip_access_checks(src_volume) { - if let Err(e) = super::fs::access_std(&src_volume_dir) { - info!("access checks failed, src_volume_dir: {:?}, err: {}", src_volume_dir, e.to_string()); - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - let dst_volume_dir = self.get_bucket_path(dst_volume)?; - if !skip_access_checks(dst_volume) { - if let Err(e) = super::fs::access_std(&dst_volume_dir) { - info!("access checks failed, dst_volume_dir: {:?}, err: {}", dst_volume_dir, e.to_string()); - return Err(to_access_error(e, Error::VolumeAccessDenied).into()); - } - } - - // xl.meta 路径 - let src_file_path = src_volume_dir.join(Path::new(format!("{}/{}", &src_path, super::STORAGE_FORMAT_FILE).as_str())); - let dst_file_path = dst_volume_dir.join(Path::new(format!("{}/{}", &dst_path, super::STORAGE_FORMAT_FILE).as_str())); - - // data_dir 路径 - let has_data_dir_path = { - let has_data_dir = { - if !fi.is_remote() { - fi.data_dir.map(|dir| super::path::retain_slash(dir.to_string().as_str())) - } else { - None - } - }; - - if let Some(data_dir) = has_data_dir { - let src_data_path = src_volume_dir.join(Path::new( - super::path::retain_slash(format!("{}/{}", &src_path, data_dir).as_str()).as_str(), - )); - let dst_data_path = dst_volume_dir.join(Path::new( - super::path::retain_slash(format!("{}/{}", &dst_path, data_dir).as_str()).as_str(), - )); - - Some((src_data_path, dst_data_path)) - } else { - None - } - }; - - check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; - check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; - - // 读旧 xl.meta - - let has_dst_buf = match super::fs::read_file(&dst_file_path).await { - Ok(res) => Some(res), - Err(e) => { - if e.kind() == ErrorKind::NotADirectory && !cfg!(target_os = "windows") { - return Err(Error::FileAccessDenied.into()); - } - - if e.kind() != ErrorKind::NotFound { - return Err(to_file_error(e).into()); - } - - None - } - }; - - let mut xlmeta = FileMeta::new(); - - if let Some(dst_buf) = has_dst_buf.as_ref() { - if FileMeta::is_xl2_v1_format(dst_buf) { - if let Ok(nmeta) = FileMeta::load(dst_buf) { - xlmeta = nmeta - } - } - } - - let mut skip_parent = dst_volume_dir.clone(); - if has_dst_buf.as_ref().is_some() { - if let Some(parent) = dst_file_path.parent() { - skip_parent = parent.to_path_buf(); - } - } - - // TODO: Healing - - let has_old_data_dir = { - if let Ok((_, ver)) = xlmeta.find_version(fi.version_id) { - let has_data_dir = ver.get_data_dir(); - if let Some(data_dir) = has_data_dir { - if xlmeta.shard_data_dir_count(&fi.version_id, &Some(data_dir)) == 0 { - // TODO: Healing - // remove inlinedata\ - Some(data_dir) - } else { - None - } - } else { - None - } - } else { - None - } - }; - - xlmeta.add_version(fi.clone())?; - - if xlmeta.versions.len() <= 10 { - // TODO: Sign - } - - let new_dst_buf = xlmeta.marshal_msg()?; - - self.write_all(src_volume, format!("{}/{}", &src_path, super::STORAGE_FORMAT_FILE).as_str(), new_dst_buf) - .await?; - - if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() { - let no_inline = fi.data.is_none() && fi.size > 0; - if no_inline { - if let Err(err) = os::rename_all(&src_data_path, &dst_data_path, &skip_parent).await { - let _ = self.delete_file(&dst_volume_dir, dst_data_path, false, false).await; - info!( - "rename all failed src_data_path: {:?}, dst_data_path: {:?}, err: {:?}", - src_data_path, dst_data_path, err - ); - return Err(err); - } - } - } - - if let Some(old_data_dir) = has_old_data_dir { - // preserve current xl.meta inside the oldDataDir. - if let Some(dst_buf) = has_dst_buf { - if let Err(err) = self - .write_all_private( - dst_volume, - format!("{}/{}/{}", &dst_path, &old_data_dir.to_string(), super::STORAGE_FORMAT_FILE).as_str(), - &dst_buf, - true, - &skip_parent, - ) - .await - { - info!("write_all_private failed err: {:?}", err); - return Err(err); - } - } - } - - if let Err(err) = os::rename_all(&src_file_path, &dst_file_path, &skip_parent).await { - if let Some((_, dst_data_path)) = has_data_dir_path.as_ref() { - let _ = self.delete_file(&dst_volume_dir, dst_data_path, false, false).await; - } - info!("rename all failed err: {:?}", err); - return Err(err); - } - - if let Some(src_file_path_parent) = src_file_path.parent() { - if src_volume != super::RUSTFS_META_MULTIPART_BUCKET { - let _ = super::fs::remove_std(src_file_path_parent); - } else { - let _ = self - .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) - .await; - } - } - - Ok(RenameDataResp { - old_data_dir: has_old_data_dir, - sign: None, // TODO: - }) - } - - #[tracing::instrument(skip(self))] - async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> { - for vol in volumes { - if let Err(e) = self.make_volume(vol).await { - if e != Error::VolumeExists { - return Err(e); - } - } - // TODO: health check - } - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn make_volume(&self, volume: &str) -> Result<()> { - if !Self::is_valid_volname(volume) { - return Err(Error::msg("Invalid arguments specified")); - } - - let volume_dir = self.get_bucket_path(volume)?; - - if let Err(e) = super::fs::access(&volume_dir).await { - if e.kind() == std::io::ErrorKind::NotFound { - os::make_dir_all(&volume_dir, self.root.as_path()).await?; - return Ok(()); - } - - return Err(to_disk_error(e).into()); - } - - Err(Error::VolumeExists) - } - - #[tracing::instrument(skip(self))] - async fn list_volumes(&self) -> Result> { - let mut volumes = Vec::new(); - - let entries = os::read_dir(&self.root, -1) - .await - .map_err(|e| to_access_error(e, Error::DiskAccessDenied))?; - - for entry in entries { - if !super::path::has_suffix(&entry, SLASH_SEPARATOR) || !Self::is_valid_volname(super::path::clean(&entry).as_str()) { - continue; - } - - volumes.push(VolumeInfo { - name: clean(&entry), - created: None, - }); - } - - Ok(volumes) - } - - #[tracing::instrument(skip(self))] - async fn stat_volume(&self, volume: &str) -> Result { - let volume_dir = self.get_bucket_path(volume)?; - let meta = super::fs::lstat(&volume_dir).await.map_err(to_volume_error)?; - - let modtime = match meta.modified() { - Ok(md) => Some(OffsetDateTime::from(md)), - Err(_) => None, - }; - - Ok(VolumeInfo { - name: volume.to_string(), - created: modtime, - }) - } - - #[tracing::instrument(skip(self))] - async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - super::fs::access(&volume_dir) - .await - .map_err(|e| to_access_error(e, Error::VolumeAccessDenied))?; - } - - for path in paths.iter() { - let file_path = volume_dir.join(Path::new(path)); - - check_path_length(file_path.to_string_lossy().as_ref())?; - - self.move_to_trash(&file_path, false, false).await?; - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> { - if !fi.metadata.is_empty() { - let volume_dir = self.get_bucket_path(volume)?; - let file_path = volume_dir.join(Path::new(&path)); - - check_path_length(file_path.to_string_lossy().as_ref())?; - - let buf = self - .read_all(volume, format!("{}/{}", &path, super::STORAGE_FORMAT_FILE).as_str()) - .await - .map_err(|e| { - if e == Error::FileNotFound && fi.version_id.is_some() { - Error::FileVersionNotFound.into() - } else { - e - } - })?; - - if !FileMeta::is_xl2_v1_format(buf.as_slice()) { - return Err(Error::FileVersionNotFound.into()); - } - - let mut xl_meta = FileMeta::load(buf.as_slice())?; - - xl_meta.update_object_version(fi)?; - - let wbuf = xl_meta.marshal_msg()?; - - return self - .write_all_meta( - volume, - format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str(), - &wbuf, - !opts.no_persistence, - ) - .await; - } - - Err(Error::msg("Invalid Argument")) - } - - #[tracing::instrument(skip(self))] - async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { - let p = self.get_object_path(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str())?; - - let mut meta = FileMeta::new(); - if !fi.fresh { - let (buf, _) = read_file_exists(&p).await?; - if !buf.is_empty() { - let _ = meta.unmarshal_msg(&buf).map_err(|_| { - meta = FileMeta::new(); - }); - } - } - - meta.add_version(fi)?; - - let fm_data = meta.marshal_msg()?; - - self.write_all(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str(), fm_data) - .await?; - - return Ok(()); - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn read_version( - &self, - _org_volume: &str, - volume: &str, - path: &str, - version_id: &str, - opts: &ReadOptions, - ) -> Result { - let file_path = self.get_object_path(volume, path)?; - let file_dir = self.get_bucket_path(volume)?; - - let read_data = opts.read_data; - - let (data, _) = self.read_raw(volume, file_dir, file_path, read_data).await?; - - let fi = get_file_info(&data, volume, path, version_id, FileInfoOpts { data: read_data }).await?; - - Ok(fi) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { - let file_path = self.get_object_path(volume, path)?; - let file_dir = self.get_bucket_path(volume)?; - - let (buf, _) = self.read_raw(volume, file_dir, file_path, read_data).await?; - - Ok(RawFileInfo { buf }) - } - - #[tracing::instrument(skip(self))] - async fn delete_version( - &self, - volume: &str, - path: &str, - fi: FileInfo, - force_del_marker: bool, - opts: DeleteOptions, - ) -> Result<()> { - if path.starts_with(SLASH_SEPARATOR) { - return self - .delete( - volume, - path, - DeleteOptions { - recursive: false, - immediate: false, - ..Default::default() - }, - ) - .await; - } - - let volume_dir = self.get_bucket_path(volume)?; - - let file_path = volume_dir.join(Path::new(&path)); - - check_path_length(file_path.to_string_lossy().as_ref())?; - - let xl_path = file_path.join(Path::new(STORAGE_FORMAT_FILE)); - let buf = match self.read_all_data(volume, &volume_dir, &xl_path).await { - Ok(res) => res, - Err(err) => { - // - if err != Error::FileNotFound { - return Err(err); - } - - if fi.deleted && force_del_marker { - return self.write_metadata("", volume, path, fi).await; - } - - if fi.version_id.is_some() { - return Err(Error::FileVersionNotFound.into()); - } else { - return Err(Error::FileNotFound.into()); - } - } - }; - - let mut meta = FileMeta::load(&buf)?; - let old_dir = meta.delete_version(&fi)?; - - if let Some(uuid) = old_dir { - let vid = fi.version_id.unwrap_or(Uuid::nil()); - let _ = meta.data.remove(vec![vid, uuid])?; - - let old_path = file_path.join(Path::new(uuid.to_string().as_str())); - check_path_length(old_path.to_string_lossy().as_ref())?; - - if let Err(err) = self.move_to_trash(&old_path, true, false).await { - if err != Error::FileNotFound { - return Err(err); - } - } - } - - if !meta.versions.is_empty() { - let buf = meta.marshal_msg()?; - return self - .write_all_meta(volume, format!("{}{}{}", path, SLASH_SEPARATOR, STORAGE_FORMAT_FILE).as_str(), &buf, true) - .await; - } - - // opts.undo_write && opts.old_data_dir.is_some_and(f) - if let Some(old_data_dir) = opts.old_data_dir { - if opts.undo_write { - let src_path = file_path.join(Path::new( - format!("{}{}{}", old_data_dir, SLASH_SEPARATOR, STORAGE_FORMAT_FILE_BACKUP).as_str(), - )); - let dst_path = file_path.join(Path::new(format!("{}{}{}", path, SLASH_SEPARATOR, STORAGE_FORMAT_FILE).as_str())); - return rename_all(src_path, dst_path, file_path).await; - } - } - - self.delete_file(&volume_dir, &xl_path, true, false).await - } - #[tracing::instrument(level = "debug", skip(self))] - async fn delete_versions( - &self, - volume: &str, - versions: Vec, - _opts: DeleteOptions, - ) -> Result>> { - let mut errs = Vec::with_capacity(versions.len()); - for _ in 0..versions.len() { - errs.push(None); - } - - for (i, ver) in versions.iter().enumerate() { - if let Err(e) = self.delete_versions_internal(volume, ver.name.as_str(), &ver.versions).await { - errs[i] = Some(e); - } else { - errs[i] = None; - } - } - - Ok(errs) - } - - #[tracing::instrument(skip(self))] - async fn read_multiple(&self, req: ReadMultipleReq) -> Result> { - let mut results = Vec::new(); - let mut found = 0; - - for v in req.files.iter() { - let fpath = self.get_object_path(&req.bucket, format!("{}/{}", &req.prefix, v).as_str())?; - let mut res = ReadMultipleResp { - bucket: req.bucket.clone(), - prefix: req.prefix.clone(), - file: v.clone(), - ..Default::default() - }; - - // if req.metadata_only {} - match read_file_all(&fpath).await { - Ok((data, meta)) => { - found += 1; - - if req.max_size > 0 && data.len() > req.max_size { - res.exists = true; - res.error = format!("max size ({}) exceeded: {}", req.max_size, data.len()); - results.push(res); - break; - } - - res.exists = true; - res.data = data; - res.mod_time = match meta.modified() { - Ok(md) => Some(OffsetDateTime::from(md)), - Err(_) => { - warn!("Not supported modified on this platform"); - None - } - }; - results.push(res); - - if req.max_results > 0 && found >= req.max_results { - break; - } - } - Err(e) => { - if !(e == Error::FileNotFound || e == Error::VolumeNotFound) { - res.exists = true; - res.error = e.to_string(); - } - - if req.abort404 && !res.exists { - results.push(res); - break; - } - - results.push(res); - } - } - } - - Ok(results) - } - - #[tracing::instrument(skip(self))] - async fn delete_volume(&self, volume: &str) -> Result<()> { - let p = self.get_bucket_path(volume)?; - - // TODO: 不能用递归删除，如果目录下面有文件，返回 errVolumeNotEmpty - - if let Err(err) = fs::remove_dir_all(&p).await { - match err.kind() { - ErrorKind::NotFound => (), - // ErrorKind::DirectoryNotEmpty => (), - kind => { - if kind.to_string() == "directory not empty" { - return Err(Error::VolumeNotEmpty.into()); - } - - return Err(Error::from(err)); - } - } - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn disk_info(&self, _: &DiskInfoOptions) -> Result { - let mut info = Cache::get(self.disk_info_cache.clone()).await?; - // TODO: nr_requests, rotational - info.nr_requests = self.nrrequests; - info.rotational = self.rotational; - info.mount_path = self.path().to_str().unwrap().to_string(); - info.endpoint = self.endpoint.to_string(); - info.scanning = self.scanning.load(Ordering::SeqCst) == 1; - - Ok(info) - } - - // #[tracing::instrument(level = "info", skip_all)] - // async fn ns_scanner( - // &self, - // cache: &DataUsageCache, - // updates: Sender, - // scan_mode: HealScanMode, - // we_sleep: ShouldSleepFn, - // ) -> Result { - // self.scanning.fetch_add(1, Ordering::SeqCst); - // defer!(|| { self.scanning.fetch_sub(1, Ordering::SeqCst) }); - - // // must befor metadata_sys - // let Some(store) = new_object_layer_fn() else { return Err(Error::msg("errServerNotInitialized")) }; - - // let mut cache = cache.clone(); - // // Check if the current bucket has a configured lifecycle policy - // if let Ok((lc, _)) = metadata_sys::get_lifecycle_config(&cache.info.name).await { - // if lc_has_active_rules(&lc, "") { - // cache.info.life_cycle = Some(lc); - // } - // } - - // // Check if the current bucket has replication configuration - // if let Ok((rcfg, _)) = metadata_sys::get_replication_config(&cache.info.name).await { - // if rep_has_active_rules(&rcfg, "", true) { - // // TODO: globalBucketTargetSys - // } - // } - - // let vcfg = (BucketVersioningSys::get(&cache.info.name).await).ok(); - - // let loc = self.get_disk_location(); - // let disks = store - // .get_disks(loc.pool_idx.unwrap(), loc.disk_idx.unwrap()) - // .await - // .map_err(Error::from)?; - // let disk = Arc::new(LocalDisk::new(&self.endpoint(), false).await?); - // let disk_clone = disk.clone(); - // cache.info.updates = Some(updates.clone()); - // let mut data_usage_info = scan_data_folder( - // &disks, - // disk, - // &cache, - // Box::new(move |item: &ScannerItem| { - // let mut item = item.clone(); - // let disk = disk_clone.clone(); - // let vcfg = vcfg.clone(); - // Box::pin(async move { - // if !item.path.ends_with(&format!("{}{}", SLASH_SEPARATOR, STORAGE_FORMAT_FILE)) { - // return Err(Error::ScanSkipFile.into()); - // } - // let stop_fn = ScannerMetrics::log(ScannerMetric::ScanObject); - // let mut res = HashMap::new(); - // let done_sz = ScannerMetrics::time_size(ScannerMetric::ReadMetadata).await; - // let buf = match disk.read_metadata(item.path.clone()).await { - // Ok(buf) => buf, - // Err(err) => { - // res.insert("err".to_string(), err.to_string()); - // stop_fn(&res).await; - // return Err(Error::ScanSkipFile.into()); - // } - // }; - // done_sz(buf.len() as u64).await; - // res.insert("metasize".to_string(), buf.len().to_string()); - // item.transform_meda_dir(); - // let meta_cache = MetaCacheEntry { - // name: item.object_path().to_string_lossy().to_string(), - // metadata: buf, - // ..Default::default() - // }; - // let fivs = match meta_cache.file_info_versions(&item.bucket) { - // Ok(fivs) => fivs, - // Err(err) => { - // res.insert("err".to_string(), err.to_string()); - // stop_fn(&res).await; - // return Err(Error::ScanSkipFile.into()); - // } - // }; - // let mut size_s = SizeSummary::default(); - // let done = ScannerMetrics::time(ScannerMetric::ApplyAll); - // let obj_infos = match item.apply_versions_actions(&fivs.versions).await { - // Ok(obj_infos) => obj_infos, - // Err(err) => { - // res.insert("err".to_string(), err.to_string()); - // stop_fn(&res).await; - // return Err(Error::ScanSkipFile.into()); - // } - // }; - - // let versioned = if let Some(vcfg) = vcfg.as_ref() { - // vcfg.versioned(item.object_path().to_str().unwrap_or_default()) - // } else { - // false - // }; - - // let mut obj_deleted = false; - // for info in obj_infos.iter() { - // let done = ScannerMetrics::time(ScannerMetric::ApplyVersion); - // let sz: usize; - // (obj_deleted, sz) = item.apply_actions(info, &size_s).await; - // done().await; - - // if obj_deleted { - // break; - // } - - // let actual_sz = match info.get_actual_size() { - // Ok(size) => size, - // Err(_) => continue, - // }; - - // if info.delete_marker { - // size_s.delete_markers += 1; - // } - - // if info.version_id.is_some() && sz == actual_sz { - // size_s.versions += 1; - // } - - // size_s.total_size += sz; - - // if info.delete_marker { - // continue; - // } - // } - - // for frer_version in fivs.free_versions.iter() { - // let _obj_info = ObjectInfo::from_file_info( - // frer_version, - // &item.bucket, - // &item.object_path().to_string_lossy(), - // versioned, - // ); - // let done = ScannerMetrics::time(ScannerMetric::TierObjSweep); - // done().await; - // } - - // // todo: global trace - // if obj_deleted { - // return Err(Error::ScanIgnoreFileContrib.into()); - // } - // done().await; - // Ok(size_s) - // }) - // }), - // scan_mode, - // we_sleep, - // ) - // .await - // .map_err(|e| Error::from(e.to_string()))?; // TODO: Error::from(e.to_string()) - // data_usage_info.info.last_update = Some(SystemTime::now()); - // info!("ns_scanner completed: {data_usage_info:?}"); - // Ok(data_usage_info) - // } - - // #[tracing::instrument(skip(self))] - // async fn healing(&self) -> Option { - // let healing_file = path_join(&[ - // self.path(), - // PathBuf::from(RUSTFS_META_BUCKET), - // PathBuf::from(BUCKET_META_PREFIX), - // PathBuf::from(HEALING_TRACKER_FILENAME), - // ]); - // let b = match fs::read(healing_file).await { - // Ok(b) => b, - // Err(_) => return None, - // }; - // if b.is_empty() { - // return None; - // } - // match HealingTracker::unmarshal_msg(&b) { - // Ok(h) => Some(h), - // Err(_) => Some(HealingTracker::default()), - // } - // } -} - -async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> { - let drive_path = drive_path.to_string_lossy().to_string(); - check_path_length(&drive_path)?; - - let disk_info = get_info(&drive_path)?; - let root_drive = if !*GLOBAL_IsErasureSD.read().await { - let root_disk_threshold = *GLOBAL_RootDiskThreshold.read().await; - if root_disk_threshold > 0 { - disk_info.total <= root_disk_threshold - } else { - is_root_disk(&drive_path, SLASH_SEPARATOR).unwrap_or_default() - } - } else { - false - }; - - Ok((disk_info, root_drive)) -} - -#[cfg(test)] -mod test { - use super::*; - - #[tokio::test] - async fn test_skip_access_checks() { - // let arr = Vec::new(); - - let vols = [ - super::super::RUSTFS_META_TMP_DELETED_BUCKET, - super::super::RUSTFS_META_TMP_BUCKET, - super::super::RUSTFS_META_MULTIPART_BUCKET, - super::super::RUSTFS_META_BUCKET, - ]; - - let paths: Vec<_> = vols.iter().map(|v| Path::new(v).join("test")).collect(); - - for p in paths.iter() { - assert!(skip_access_checks(p.to_str().unwrap())); - } - } - - #[tokio::test] - async fn test_make_volume() { - let p = "./testv0"; - fs::create_dir_all(&p).await.unwrap(); - - let ep = match Endpoint::try_from(p) { - Ok(e) => e, - Err(e) => { - println!("{e}"); - return; - } - }; - - let disk = LocalDisk::new(&ep, false).await.unwrap(); - - let tmpp = disk - .resolve_abs_path(Path::new(super::super::RUSTFS_META_TMP_DELETED_BUCKET)) - .unwrap(); - - println!("ppp :{:?}", &tmpp); - - let volumes = vec!["a123", "b123", "c123"]; - - disk.make_volumes(volumes.clone()).await.unwrap(); - - disk.make_volumes(volumes.clone()).await.unwrap(); - - let _ = fs::remove_dir_all(&p).await; - } - - #[tokio::test] - async fn test_delete_volume() { - let p = "./testv1"; - fs::create_dir_all(&p).await.unwrap(); - - let ep = match Endpoint::try_from(p) { - Ok(e) => e, - Err(e) => { - println!("{e}"); - return; - } - }; - - let disk = LocalDisk::new(&ep, false).await.unwrap(); - - let tmpp = disk - .resolve_abs_path(Path::new(super::super::RUSTFS_META_TMP_DELETED_BUCKET)) - .unwrap(); - - println!("ppp :{:?}", &tmpp); - - let volumes = vec!["a123", "b123", "c123"]; - - disk.make_volumes(volumes.clone()).await.unwrap(); - - disk.delete_volume("a").await.unwrap(); - - let _ = fs::remove_dir_all(&p).await; - } -} diff --git a/crates/disk/src/local_list.rs b/crates/disk/src/local_list.rs deleted file mode 100644 index cb890bfb5..000000000 --- a/crates/disk/src/local_list.rs +++ /dev/null @@ -1,535 +0,0 @@ -use crate::{ - api::{DiskAPI, DiskStore, WalkDirOptions, STORAGE_FORMAT_FILE}, - local::LocalDisk, - os::is_empty_dir, - path::{self, decode_dir_object, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR}, -}; -use futures::future::join_all; -use rustfs_error::{Error, Result}; -use rustfs_metacache::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, MetacacheWriter}; -use std::{collections::HashSet, future::Future, pin::Pin, sync::Arc}; -use tokio::{io::AsyncWrite, spawn, sync::broadcast::Receiver as B_Receiver}; -use tracing::{error, info, warn}; - -pub type AgreedFn = Box Pin + Send>> + Send + 'static>; -pub type PartialFn = Box]) -> Pin + Send>> + Send + 'static>; -type FinishedFn = Box]) -> Pin + Send>> + Send + 'static>; - -#[derive(Default)] -pub struct ListPathRawOptions { - pub disks: Vec>, - pub fallback_disks: Vec>, - pub bucket: String, - pub path: String, - pub recursive: bool, - pub filter_prefix: Option, - pub forward_to: Option, - pub min_disks: usize, - pub report_not_found: bool, - pub per_disk_limit: i32, - pub agreed: Option, - pub partial: Option, - pub finished: Option, -} - -impl Clone for ListPathRawOptions { - fn clone(&self) -> Self { - Self { - disks: self.disks.clone(), - fallback_disks: self.fallback_disks.clone(), - bucket: self.bucket.clone(), - path: self.path.clone(), - recursive: self.recursive, - filter_prefix: self.filter_prefix.clone(), - forward_to: self.forward_to.clone(), - min_disks: self.min_disks, - report_not_found: self.report_not_found, - per_disk_limit: self.per_disk_limit, - ..Default::default() - } - } -} - -pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) -> Result<()> { - if opts.disks.is_empty() { - return Err(Error::msg("list_path_raw: 0 drives provided")); - } - - let mut jobs: Vec>> = Vec::new(); - let mut readers = Vec::with_capacity(opts.disks.len()); - let fds = Arc::new(opts.fallback_disks.clone()); - - for disk in opts.disks.iter() { - let opdisk = disk.clone(); - let opts_clone = opts.clone(); - let fds_clone = fds.clone(); - let (rd, mut wr) = tokio::io::duplex(64); - readers.push(MetacacheReader::new(rd)); - - jobs.push(spawn(async move { - let walk_opts = WalkDirOptions { - bucket: opts_clone.bucket.clone(), - base_dir: opts_clone.path.clone(), - recursive: opts_clone.recursive, - report_notfound: opts_clone.report_not_found, - filter_prefix: opts_clone.filter_prefix.clone(), - forward_to: opts_clone.forward_to.clone(), - limit: opts_clone.per_disk_limit, - ..Default::default() - }; - - let mut need_fallback = false; - if let Some(disk) = opdisk { - match disk.walk_dir(walk_opts, &mut wr).await { - Ok(_res) => {} - Err(err) => { - error!("walk dir err {:?}", &err); - need_fallback = true; - } - } - } else { - need_fallback = true; - } - - while need_fallback { - let disk = match fds_clone.iter().find(|d| d.is_some()) { - Some(d) => { - if let Some(disk) = d.clone() { - disk - } else { - break; - } - } - None => break, - }; - - match disk - .walk_dir( - WalkDirOptions { - bucket: opts_clone.bucket.clone(), - base_dir: opts_clone.path.clone(), - recursive: opts_clone.recursive, - report_notfound: opts_clone.report_not_found, - filter_prefix: opts_clone.filter_prefix.clone(), - forward_to: opts_clone.forward_to.clone(), - limit: opts_clone.per_disk_limit, - ..Default::default() - }, - &mut wr, - ) - .await - { - Ok(_r) => { - need_fallback = false; - } - Err(err) => { - error!("walk dir2 err {:?}", &err); - break; - } - } - } - - Ok(()) - })); - } - - let revjob = spawn(async move { - let mut errs: Vec> = Vec::with_capacity(readers.len()); - for _ in 0..readers.len() { - errs.push(None); - } - - loop { - let mut current = MetaCacheEntry::default(); - - if rx.try_recv().is_ok() { - return Err(Error::msg("canceled")); - } - let mut top_entries: Vec> = vec![None; readers.len()]; - - let mut at_eof = 0; - let mut fnf = 0; - let mut vnf = 0; - let mut has_err = 0; - let mut agree = 0; - - for (i, r) in readers.iter_mut().enumerate() { - if errs[i].is_some() { - has_err += 1; - continue; - } - - let entry = match r.peek().await { - Ok(res) => { - if let Some(entry) = res { - entry - } else { - at_eof += 1; - continue; - } - } - Err(err) => { - if err == Error::FaultyDisk { - at_eof += 1; - continue; - } else if err == Error::FileNotFound { - at_eof += 1; - fnf += 1; - continue; - } else if err == Error::VolumeNotFound { - at_eof += 1; - fnf += 1; - vnf += 1; - continue; - } else { - has_err += 1; - errs[i] = Some(err); - continue; - } - } - }; - - // If no current, add it. - if current.name.is_empty() { - top_entries[i] = Some(entry.clone()); - current = entry; - agree += 1; - continue; - } - - // If exact match, we agree. - if let (_, true) = current.matches(Some(&entry), true) { - top_entries[i] = Some(entry); - agree += 1; - continue; - } - - // If only the name matches we didn't agree, but add it for resolution. - if entry.name == current.name { - top_entries[i] = Some(entry); - continue; - } - - // We got different entries - if entry.name > current.name { - continue; - } - - for item in top_entries.iter_mut().take(i) { - *item = None; - } - - agree = 1; - top_entries[i] = Some(entry.clone()); - current = entry; - } - - if vnf > 0 && vnf >= (readers.len() - opts.min_disks) { - return Err(Error::VolumeNotFound); - } - - if fnf > 0 && fnf >= (readers.len() - opts.min_disks) { - return Err(Error::FileNotFound); - } - - if has_err > 0 && has_err > opts.disks.len() - opts.min_disks { - if let Some(finished_fn) = opts.finished.as_ref() { - finished_fn(&errs).await; - } - let mut combined_err = Vec::new(); - errs.iter().zip(opts.disks.iter()).for_each(|(err, disk)| match (err, disk) { - (Some(err), Some(disk)) => { - combined_err.push(format!("drive {} returned: {}", disk.to_string(), err)); - } - (Some(err), None) => { - combined_err.push(err.to_string()); - } - _ => {} - }); - - return Err(Error::msg(combined_err.join(", "))); - } - - // Break if all at EOF or error. - if at_eof + has_err == readers.len() { - if has_err > 0 { - if let Some(finished_fn) = opts.finished.as_ref() { - finished_fn(&errs).await; - } - } - break; - } - - if agree == readers.len() { - for r in readers.iter_mut() { - let _ = r.skip(1).await; - } - - if let Some(agreed_fn) = opts.agreed.as_ref() { - agreed_fn(current).await; - } - continue; - } - - for (i, r) in readers.iter_mut().enumerate() { - if top_entries[i].is_some() { - let _ = r.skip(1).await; - } - } - - if let Some(partial_fn) = opts.partial.as_ref() { - partial_fn(MetaCacheEntries(top_entries), &errs).await; - } - } - Ok(()) - }); - - jobs.push(revjob); - - let results = join_all(jobs).await; - for result in results { - if let Err(err) = result { - error!("list_path_raw err {:?}", err); - } - } - - Ok(()) -} - -impl LocalDisk { - pub(crate) async fn scan_dir( - &self, - current: &mut String, - opts: &WalkDirOptions, - out: &mut MetacacheWriter, - objs_returned: &mut i32, - ) -> Result<()> { - let forward = { - opts.forward_to.as_ref().filter(|v| v.starts_with(&*current)).map(|v| { - let forward = v.trim_start_matches(&*current); - if let Some(idx) = forward.find('/') { - forward[..idx].to_owned() - } else { - forward.to_owned() - } - }) - // if let Some(forward_to) = &opts.forward_to { - - // } else { - // None - // } - // if !opts.forward_to.is_empty() && opts.forward_to.starts_with(&*current) { - // let forward = opts.forward_to.trim_start_matches(&*current); - // if let Some(idx) = forward.find('/') { - // &forward[..idx] - // } else { - // forward - // } - // } else { - // "" - // } - }; - - if opts.limit > 0 && *objs_returned >= opts.limit { - return Ok(()); - } - - let mut entries = match self.list_dir("", &opts.bucket, current, -1).await { - Ok(res) => res, - Err(e) => { - if e != Error::VolumeNotFound && e != Error::FileNotFound { - info!("scan list_dir {}, err {:?}", ¤t, &e); - } - - if opts.report_notfound && (e == Error::VolumeNotFound || e == Error::FileNotFound) && current == &opts.base_dir { - return Err(Error::FileNotFound); - } - - return Ok(()); - } - }; - - if entries.is_empty() { - return Ok(()); - } - - let s = SLASH_SEPARATOR.chars().next().unwrap_or_default(); - *current = current.trim_matches(s).to_owned(); - - let bucket = opts.bucket.as_str(); - - let mut dir_objes = HashSet::new(); - - // 第一层过滤 - for item in entries.iter_mut() { - let entry = item.clone(); - // check limit - if opts.limit > 0 && *objs_returned >= opts.limit { - return Ok(()); - } - // check prefix - if let Some(filter_prefix) = &opts.filter_prefix { - if !entry.starts_with(filter_prefix) { - *item = "".to_owned(); - continue; - } - } - - if let Some(forward) = &forward { - if &entry < forward { - *item = "".to_owned(); - continue; - } - } - - if entry.ends_with(SLASH_SEPARATOR) { - if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) { - let entry = format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR); - dir_objes.insert(entry.clone()); - *item = entry; - continue; - } - - *item = entry.trim_end_matches(SLASH_SEPARATOR).to_owned(); - continue; - } - - *item = "".to_owned(); - - if entry.ends_with(STORAGE_FORMAT_FILE) { - // - let metadata = self - .read_metadata(self.get_object_path(bucket, format!("{}/{}", ¤t, &entry).as_str())?) - .await?; - - // 用 strip_suffix 只删除一次 - let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned(); - let name = entry.trim_end_matches(SLASH_SEPARATOR); - let name = decode_dir_object(format!("{}/{}", ¤t, &name).as_str()); - - out.write_obj(&MetaCacheEntry { - name, - metadata, - ..Default::default() - }) - .await?; - *objs_returned += 1; - - return Ok(()); - } - } - - entries.sort(); - - let mut entries = entries.as_slice(); - if let Some(forward) = &forward { - for (i, entry) in entries.iter().enumerate() { - if entry >= forward || forward.starts_with(entry.as_str()) { - entries = &entries[i..]; - break; - } - } - } - - let mut dir_stack: Vec = Vec::with_capacity(5); - - for entry in entries.iter() { - if opts.limit > 0 && *objs_returned >= opts.limit { - return Ok(()); - } - - if entry.is_empty() { - continue; - } - - let name = path::path_join_buf(&[current, entry]); - - if !dir_stack.is_empty() { - if let Some(pop) = dir_stack.pop() { - if pop < name { - // - out.write_obj(&MetaCacheEntry { - name: pop.clone(), - ..Default::default() - }) - .await?; - - if opts.recursive { - let mut opts = opts.clone(); - opts.filter_prefix = None; - if let Err(er) = Box::pin(self.scan_dir(&mut pop.clone(), &opts, out, objs_returned)).await { - error!("scan_dir err {:?}", er); - } - } - } - } - } - - let mut meta = MetaCacheEntry { - name, - ..Default::default() - }; - - let mut is_dir_obj = false; - - if let Some(_dir) = dir_objes.get(entry) { - is_dir_obj = true; - meta.name - .truncate(meta.name.len() - meta.name.chars().last().unwrap().len_utf8()); - meta.name.push_str(GLOBAL_DIR_SUFFIX_WITH_SLASH); - } - - let fname = format!("{}/{}", &meta.name, STORAGE_FORMAT_FILE); - - match self.read_metadata(self.get_object_path(&opts.bucket, fname.as_str())?).await { - Ok(res) => { - if is_dir_obj { - meta.name = meta.name.trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH).to_owned(); - meta.name.push_str(SLASH_SEPARATOR); - } - - meta.metadata = res; - - out.write_obj(&meta).await?; - *objs_returned += 1; - } - Err(err) => { - if err == Error::FileNotFound || err == Error::IsNotRegular { - // NOT an object, append to stack (with slash) - // If dirObject, but no metadata (which is unexpected) we skip it. - if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await { - meta.name.push_str(SLASH_SEPARATOR); - dir_stack.push(meta.name); - } - } - - continue; - } - }; - } - - while let Some(dir) = dir_stack.pop() { - if opts.limit > 0 && *objs_returned >= opts.limit { - return Ok(()); - } - - out.write_obj(&MetaCacheEntry { - name: dir.clone(), - ..Default::default() - }) - .await?; - *objs_returned += 1; - - if opts.recursive { - let mut dir = dir; - let mut opts = opts.clone(); - opts.filter_prefix = None; - if let Err(er) = Box::pin(self.scan_dir(&mut dir, &opts, out, objs_returned)).await { - warn!("scan_dir err {:?}", &er); - } - } - } - - Ok(()) - } -} diff --git a/crates/disk/src/metacache.rs b/crates/disk/src/metacache.rs deleted file mode 100644 index f0b3c1619..000000000 --- a/crates/disk/src/metacache.rs +++ /dev/null @@ -1,608 +0,0 @@ -use crate::bucket::metadata_sys::get_versioning_config; -use crate::bucket::versioning::VersioningApi; -use crate::store_api::ObjectInfo; -use crate::utils::path::SLASH_SEPARATOR; -use rustfs_error::{Error, Result}; -use rustfs_filemeta::merge_file_meta_versions; -use rustfs_filemeta::FileInfo; -use rustfs_filemeta::FileInfoVersions; -use rustfs_filemeta::FileMeta; -use rustfs_filemeta::FileMetaShallowVersion; -use rustfs_filemeta::VersionType; -use serde::Deserialize; -use serde::Serialize; -use std::cmp::Ordering; -use time::OffsetDateTime; -use tracing::warn; - -#[derive(Clone, Debug, Default)] -pub struct MetadataResolutionParams { - pub dir_quorum: usize, - pub obj_quorum: usize, - pub requested_versions: usize, - pub bucket: String, - pub strict: bool, - pub candidates: Vec>, -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] -pub struct MetaCacheEntry { - // name is the full name of the object including prefixes - pub name: String, - // Metadata. If none is present it is not an object but only a prefix. - // Entries without metadata will only be present in non-recursive scans. - pub metadata: Vec, - - // cached contains the metadata if decoded. - pub cached: Option, - - // Indicates the entry can be reused and only one reference to metadata is expected. - pub reusable: bool, -} - -impl MetaCacheEntry { - pub fn marshal_msg(&self) -> Result> { - let mut wr = Vec::new(); - rmp::encode::write_bool(&mut wr, true)?; - - rmp::encode::write_str(&mut wr, &self.name)?; - - rmp::encode::write_bin(&mut wr, &self.metadata)?; - - Ok(wr) - } - - pub fn is_dir(&self) -> bool { - self.metadata.is_empty() && self.name.ends_with('/') - } - pub fn is_in_dir(&self, dir: &str, separator: &str) -> bool { - if dir.is_empty() { - let idx = self.name.find(separator); - return idx.is_none() || idx.unwrap() == self.name.len() - separator.len(); - } - - let ext = self.name.trim_start_matches(dir); - - if ext.len() != self.name.len() { - let idx = ext.find(separator); - return idx.is_none() || idx.unwrap() == ext.len() - separator.len(); - } - - false - } - pub fn is_object(&self) -> bool { - !self.metadata.is_empty() - } - - pub fn is_object_dir(&self) -> bool { - !self.metadata.is_empty() && self.name.ends_with(SLASH_SEPARATOR) - } - - pub fn is_latest_delete_marker(&mut self) -> bool { - if let Some(cached) = &self.cached { - if cached.versions.is_empty() { - return true; - } - - return cached.versions[0].header.version_type == VersionType::Delete; - } - - if !FileMeta::is_xl2_v1_format(&self.metadata) { - return false; - } - - match FileMeta::check_xl2_v1(&self.metadata) { - Ok((meta, _, _)) => { - if !meta.is_empty() { - return FileMeta::is_latest_delete_marker(meta); - } - } - Err(_) => return true, - } - - match self.xl_meta() { - Ok(res) => { - if res.versions.is_empty() { - return true; - } - res.versions[0].header.version_type == VersionType::Delete - } - Err(_) => true, - } - } - - #[tracing::instrument(level = "debug", skip(self))] - pub fn to_fileinfo(&self, bucket: &str) -> Result { - if self.is_dir() { - return Ok(FileInfo { - volume: bucket.to_owned(), - name: self.name.clone(), - ..Default::default() - }); - } - - if self.cached.is_some() { - let fm = self.cached.as_ref().unwrap(); - if fm.versions.is_empty() { - return Ok(FileInfo { - volume: bucket.to_owned(), - name: self.name.clone(), - deleted: true, - is_latest: true, - mod_time: Some(OffsetDateTime::UNIX_EPOCH), - ..Default::default() - }); - } - - let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?; - - return Ok(fi); - } - - let mut fm = FileMeta::new(); - fm.unmarshal_msg(&self.metadata)?; - - let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?; - - return Ok(fi); - } - - pub fn file_info_versions(&self, bucket: &str) -> Result { - if self.is_dir() { - return Ok(FileInfoVersions { - volume: bucket.to_string(), - name: self.name.clone(), - versions: vec![FileInfo { - volume: bucket.to_string(), - name: self.name.clone(), - ..Default::default() - }], - ..Default::default() - }); - } - - let mut fm = FileMeta::new(); - fm.unmarshal_msg(&self.metadata)?; - - fm.into_file_info_versions(bucket, self.name.as_str(), false) - } - - pub fn matches(&self, other: Option<&MetaCacheEntry>, strict: bool) -> (Option, bool) { - if other.is_none() { - return (None, false); - } - - let other = other.unwrap(); - - let mut prefer = None; - if self.name != other.name { - if self.name < other.name { - return (Some(self.clone()), false); - } - return (Some(other.clone()), false); - } - - if other.is_dir() || self.is_dir() { - if self.is_dir() { - return (Some(self.clone()), other.is_dir() == self.is_dir()); - } - - return (Some(other.clone()), other.is_dir() == self.is_dir()); - } - let self_vers = match &self.cached { - Some(file_meta) => file_meta.clone(), - None => match FileMeta::load(&self.metadata) { - Ok(meta) => meta, - Err(_) => { - return (None, false); - } - }, - }; - let other_vers = match &other.cached { - Some(file_meta) => file_meta.clone(), - None => match FileMeta::load(&other.metadata) { - Ok(meta) => meta, - Err(_) => { - return (None, false); - } - }, - }; - - if self_vers.versions.len() != other_vers.versions.len() { - match self_vers.lastest_mod_time().cmp(&other_vers.lastest_mod_time()) { - Ordering::Greater => { - return (Some(self.clone()), false); - } - Ordering::Less => { - return (Some(other.clone()), false); - } - _ => {} - } - - if self_vers.versions.len() > other_vers.versions.len() { - return (Some(self.clone()), false); - } - return (Some(other.clone()), false); - } - - for (s_version, o_version) in self_vers.versions.iter().zip(other_vers.versions.iter()) { - if s_version.header != o_version.header { - if s_version.header.has_ec() != o_version.header.has_ec() { - // One version has EC and the other doesn't - may have been written later. - // Compare without considering EC. - let (mut a, mut b) = (s_version.header.clone(), o_version.header.clone()); - (a.ec_n, a.ec_m, b.ec_n, b.ec_m) = (0, 0, 0, 0); - if a == b { - continue; - } - } - - if !strict && s_version.header.matches_not_strict(&o_version.header) { - if prefer.is_none() { - if s_version.header.sorts_before(&o_version.header) { - prefer = Some(self.clone()); - } else { - prefer = Some(other.clone()); - } - } - - continue; - } - - if prefer.is_some() { - return (prefer, false); - } - - if s_version.header.sorts_before(&o_version.header) { - return (Some(self.clone()), false); - } - - return (Some(other.clone()), false); - } - } - - if prefer.is_none() { - prefer = Some(self.clone()); - } - - (prefer, true) - } - - pub fn xl_meta(&mut self) -> Result { - if self.is_dir() { - return Err(Error::FileNotFound); - } - - if let Some(meta) = &self.cached { - Ok(meta.clone()) - } else { - if self.metadata.is_empty() { - return Err(Error::FileNotFound); - } - - let meta = FileMeta::load(&self.metadata)?; - - self.cached = Some(meta.clone()); - - Ok(meta) - } - } -} - -#[derive(Debug, Default)] -pub struct MetaCacheEntries(pub Vec>); - -impl MetaCacheEntries { - #[allow(clippy::should_implement_trait)] - pub fn as_ref(&self) -> &[Option] { - &self.0 - } - pub fn resolve(&self, mut params: MetadataResolutionParams) -> Option { - if self.0.is_empty() { - warn!("decommission_pool: entries resolve empty"); - return None; - } - - let mut dir_exists = 0; - let mut selected = None; - - params.candidates.clear(); - let mut objs_agree = 0; - let mut objs_valid = 0; - - for entry in self.0.iter().flatten() { - let mut entry = entry.clone(); - - warn!("decommission_pool: entries resolve entry {:?}", entry.name); - if entry.name.is_empty() { - continue; - } - if entry.is_dir() { - dir_exists += 1; - selected = Some(entry.clone()); - warn!("decommission_pool: entries resolve entry dir {:?}", entry.name); - continue; - } - - let xl = match entry.xl_meta() { - Ok(xl) => xl, - Err(e) => { - warn!("decommission_pool: entries resolve entry xl_meta {:?}", e); - continue; - } - }; - - objs_valid += 1; - - params.candidates.push(xl.versions.clone()); - - if selected.is_none() { - selected = Some(entry.clone()); - objs_agree = 1; - warn!("decommission_pool: entries resolve entry selected {:?}", entry.name); - continue; - } - - if let (prefer, true) = entry.matches(selected.as_ref(), params.strict) { - selected = prefer; - objs_agree += 1; - warn!("decommission_pool: entries resolve entry prefer {:?}", entry.name); - continue; - } - } - - let Some(selected) = selected else { - warn!("decommission_pool: entries resolve entry no selected"); - return None; - }; - - if selected.is_dir() && dir_exists >= params.dir_quorum { - warn!("decommission_pool: entries resolve entry dir selected {:?}", selected.name); - return Some(selected); - } - - // If we would never be able to reach read quorum. - if objs_valid < params.obj_quorum { - warn!( - "decommission_pool: entries resolve entry not enough objects {} < {}", - objs_valid, params.obj_quorum - ); - return None; - } - - if objs_agree == objs_valid { - warn!("decommission_pool: entries resolve entry all agree {} == {}", objs_agree, objs_valid); - return Some(selected); - } - - let Some(cached) = selected.cached else { - warn!("decommission_pool: entries resolve entry no cached"); - return None; - }; - - let versions = merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, ¶ms.candidates); - if versions.is_empty() { - warn!("decommission_pool: entries resolve entry no versions"); - return None; - } - - let metadata = match cached.marshal_msg() { - Ok(meta) => meta, - Err(e) => { - warn!("decommission_pool: entries resolve entry marshal_msg {:?}", e); - return None; - } - }; - - // Merge if we have disagreement. - // Create a new merged result. - let new_selected = MetaCacheEntry { - name: selected.name.clone(), - cached: Some(FileMeta { - meta_ver: cached.meta_ver, - versions, - ..Default::default() - }), - reusable: true, - metadata, - }; - - warn!("decommission_pool: entries resolve entry selected {:?}", new_selected.name); - Some(new_selected) - } - - pub fn first_found(&self) -> (Option, usize) { - (self.0.iter().find(|x| x.is_some()).cloned().unwrap_or_default(), self.0.len()) - } -} - -#[derive(Debug, Default)] -pub struct MetaCacheEntriesSortedResult { - pub entries: Option, - pub err: Option, -} - -// impl MetaCacheEntriesSortedResult { -// pub fn entriy_list(&self) -> Vec<&MetaCacheEntry> { -// if let Some(entries) = &self.entries { -// entries.entries() -// } else { -// Vec::new() -// } -// } -// } - -#[derive(Debug, Default)] -pub struct MetaCacheEntriesSorted { - pub o: MetaCacheEntries, - pub list_id: Option, - pub reuse: bool, - pub last_skipped_entry: Option, -} - -impl MetaCacheEntriesSorted { - pub fn entries(&self) -> Vec<&MetaCacheEntry> { - let entries: Vec<&MetaCacheEntry> = self.o.0.iter().flatten().collect(); - entries - } - pub fn forward_past(&mut self, marker: Option) { - if let Some(val) = marker { - // TODO: reuse - if let Some(idx) = self.o.0.iter().flatten().position(|v| v.name > val) { - self.o.0 = self.o.0.split_off(idx); - } - } - } - pub async fn file_infos(&self, bucket: &str, prefix: &str, delimiter: Option) -> Vec { - let vcfg = get_versioning_config(bucket).await.ok(); - let mut objects = Vec::with_capacity(self.o.as_ref().len()); - let mut prev_prefix = ""; - for entry in self.o.as_ref().iter().flatten() { - if entry.is_object() { - if let Some(delimiter) = &delimiter { - if let Some(idx) = entry.name.trim_start_matches(prefix).find(delimiter) { - let idx = prefix.len() + idx + delimiter.len(); - if let Some(curr_prefix) = entry.name.get(0..idx) { - if curr_prefix == prev_prefix { - continue; - } - - prev_prefix = curr_prefix; - - objects.push(ObjectInfo { - is_dir: true, - bucket: bucket.to_owned(), - name: curr_prefix.to_owned(), - ..Default::default() - }); - } - continue; - } - } - - if let Ok(fi) = entry.to_fileinfo(bucket) { - // TODO:VersionPurgeStatus - let versioned = vcfg.clone().map(|v| v.0.versioned(&entry.name)).unwrap_or_default(); - objects.push(ObjectInfo::from_file_info(&fi, bucket, &entry.name, versioned)); - } - continue; - } - - if entry.is_dir() { - if let Some(delimiter) = &delimiter { - if let Some(idx) = entry.name.trim_start_matches(prefix).find(delimiter) { - let idx = prefix.len() + idx + delimiter.len(); - if let Some(curr_prefix) = entry.name.get(0..idx) { - if curr_prefix == prev_prefix { - continue; - } - - prev_prefix = curr_prefix; - - objects.push(ObjectInfo { - is_dir: true, - bucket: bucket.to_owned(), - name: curr_prefix.to_owned(), - ..Default::default() - }); - } - } - } - } - } - - objects - } - - pub async fn file_info_versions( - &self, - bucket: &str, - prefix: &str, - delimiter: Option, - after_v: Option, - ) -> Vec { - let vcfg = get_versioning_config(bucket).await.ok(); - let mut objects = Vec::with_capacity(self.o.as_ref().len()); - let mut prev_prefix = ""; - let mut after_v = after_v; - for entry in self.o.as_ref().iter().flatten() { - if entry.is_object() { - if let Some(delimiter) = &delimiter { - if let Some(idx) = entry.name.trim_start_matches(prefix).find(delimiter) { - let idx = prefix.len() + idx + delimiter.len(); - if let Some(curr_prefix) = entry.name.get(0..idx) { - if curr_prefix == prev_prefix { - continue; - } - - prev_prefix = curr_prefix; - - objects.push(ObjectInfo { - is_dir: true, - bucket: bucket.to_owned(), - name: curr_prefix.to_owned(), - ..Default::default() - }); - } - continue; - } - } - - let mut fiv = match entry.file_info_versions(bucket) { - Ok(res) => res, - Err(_err) => { - // - continue; - } - }; - - let fi_versions = 'c: { - if let Some(after_val) = &after_v { - if let Some(idx) = fiv.find_version_index(after_val) { - after_v = None; - break 'c fiv.versions.split_off(idx + 1); - } - - after_v = None; - break 'c fiv.versions; - } else { - break 'c fiv.versions; - } - }; - - for fi in fi_versions.into_iter() { - // VersionPurgeStatus - - let versioned = vcfg.clone().map(|v| v.0.versioned(&entry.name)).unwrap_or_default(); - objects.push(ObjectInfo::from_file_info(&fi, bucket, &entry.name, versioned)); - } - - continue; - } - - if entry.is_dir() { - if let Some(delimiter) = &delimiter { - if let Some(idx) = entry.name.trim_start_matches(prefix).find(delimiter) { - let idx = prefix.len() + idx + delimiter.len(); - if let Some(curr_prefix) = entry.name.get(0..idx) { - if curr_prefix == prev_prefix { - continue; - } - - prev_prefix = curr_prefix; - - objects.push(ObjectInfo { - is_dir: true, - bucket: bucket.to_owned(), - name: curr_prefix.to_owned(), - ..Default::default() - }); - } - } - } - } - } - - objects - } -} diff --git a/crates/disk/src/os.rs b/crates/disk/src/os.rs deleted file mode 100644 index 8b608735d..000000000 --- a/crates/disk/src/os.rs +++ /dev/null @@ -1,206 +0,0 @@ -use super::fs; -use rustfs_error::{to_access_error, Error, Result}; -use rustfs_utils::os::same_disk; -use std::{ - io, - path::{Component, Path}, -}; - -pub fn check_path_length(path_name: &str) -> Result<()> { - // Apple OS X path length is limited to 1016 - if cfg!(target_os = "macos") && path_name.len() > 1016 { - return Err(Error::FileNameTooLong); - } - - // Disallow more than 1024 characters on windows, there - // are no known name_max limits on Windows. - if cfg!(target_os = "windows") && path_name.len() > 1024 { - return Err(Error::FileNameTooLong); - } - - // On Unix we reject paths if they are just '.', '..' or '/' - let invalid_paths = [".", "..", "/"]; - if invalid_paths.contains(&path_name) { - return Err(Error::FileAccessDenied); - } - - // Check each path segment length is > 255 on all Unix - // platforms, look for this value as NAME_MAX in - // /usr/include/linux/limits.h - let mut count = 0usize; - for c in path_name.chars() { - match c { - '/' | '\\' if cfg!(target_os = "windows") => count = 0, // Reset - _ => { - count += 1; - if count > 255 { - return Err(Error::FileNameTooLong); - } - } - } - } - - // Success. - Ok(()) -} - -pub fn is_root_disk(disk_path: &str, root_disk: &str) -> Result { - if cfg!(target_os = "windows") { - return Ok(false); - } - - Ok(same_disk(disk_path, root_disk)?) -} - -pub async fn make_dir_all(path: impl AsRef, base_dir: impl AsRef) -> Result<()> { - check_path_length(path.as_ref().to_string_lossy().to_string().as_str())?; - - if let Err(e) = reliable_mkdir_all(path.as_ref(), base_dir.as_ref()).await { - return Err(to_access_error(e, Error::FileAccessDenied).into()); - } - - Ok(()) -} - -pub async fn is_empty_dir(path: impl AsRef) -> bool { - read_dir(path.as_ref(), 1).await.is_ok_and(|v| v.is_empty()) -} - -// read_dir count read limit. when count == 0 unlimit. -pub async fn read_dir(path: impl AsRef, count: i32) -> std::io::Result> { - let mut entries = tokio::fs::read_dir(path.as_ref()).await?; - - let mut volumes = Vec::new(); - - let mut count = count; - - while let Some(entry) = entries.next_entry().await? { - let name = entry.file_name().to_string_lossy().to_string(); - - if name.is_empty() || name == "." || name == ".." { - continue; - } - - let file_type = entry.file_type().await?; - - if file_type.is_file() { - volumes.push(name); - } else if file_type.is_dir() { - volumes.push(format!("{}{}", name, super::path::SLASH_SEPARATOR)); - } - count -= 1; - if count == 0 { - break; - } - } - - Ok(volumes) -} - -#[tracing::instrument(level = "debug", skip_all)] -pub async fn rename_all( - src_file_path: impl AsRef, - dst_file_path: impl AsRef, - base_dir: impl AsRef, -) -> Result<()> { - reliable_rename(src_file_path, dst_file_path.as_ref(), base_dir) - .await - .map_err(|e| to_access_error(e, Error::FileAccessDenied))?; - - Ok(()) -} - -pub async fn reliable_rename( - src_file_path: impl AsRef, - dst_file_path: impl AsRef, - base_dir: impl AsRef, -) -> io::Result<()> { - if let Some(parent) = dst_file_path.as_ref().parent() { - if !file_exists(parent) { - // info!("reliable_rename reliable_mkdir_all parent: {:?}", parent); - reliable_mkdir_all(parent, base_dir.as_ref()).await?; - } - } - - let mut i = 0; - loop { - if let Err(e) = fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) { - if e.kind() == std::io::ErrorKind::NotFound && i == 0 { - i += 1; - continue; - } - // info!( - // "reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}", - // src_file_path.as_ref(), - // dst_file_path.as_ref(), - // base_dir.as_ref(), - // e - // ); - return Err(e); - } - - break; - } - - Ok(()) -} - -pub async fn reliable_mkdir_all(path: impl AsRef, base_dir: impl AsRef) -> io::Result<()> { - let mut i = 0; - - let mut base_dir = base_dir.as_ref(); - loop { - if let Err(e) = os_mkdir_all(path.as_ref(), base_dir).await { - if e.kind() == std::io::ErrorKind::NotFound && i == 0 { - i += 1; - - if let Some(base_parent) = base_dir.parent() { - if let Some(c) = base_parent.components().next() { - if c != Component::RootDir { - base_dir = base_parent - } - } - } - continue; - } - - return Err(e); - } - - break; - } - - Ok(()) -} - -pub async fn os_mkdir_all(dir_path: impl AsRef, base_dir: impl AsRef) -> io::Result<()> { - if !base_dir.as_ref().to_string_lossy().is_empty() && base_dir.as_ref().starts_with(dir_path.as_ref()) { - return Ok(()); - } - - if let Some(parent) = dir_path.as_ref().parent() { - // 不支持递归，直接create_dir_all了 - if let Err(e) = fs::make_dir_all(&parent).await { - if e.kind() == std::io::ErrorKind::AlreadyExists { - return Ok(()); - } - - return Err(e); - } - // Box::pin(os_mkdir_all(&parent, &base_dir)).await?; - } - - if let Err(e) = fs::mkdir(dir_path.as_ref()).await { - if e.kind() == std::io::ErrorKind::AlreadyExists { - return Ok(()); - } - - return Err(e); - } - - Ok(()) -} - -pub fn file_exists(path: impl AsRef) -> bool { - std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false) -} diff --git a/crates/disk/src/path.rs b/crates/disk/src/path.rs deleted file mode 100644 index 0c63b960a..000000000 --- a/crates/disk/src/path.rs +++ /dev/null @@ -1,308 +0,0 @@ -use std::path::Path; -use std::path::PathBuf; - -pub const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__"; - -pub const SLASH_SEPARATOR: &str = "/"; - -pub const GLOBAL_DIR_SUFFIX_WITH_SLASH: &str = "__XLDIR__/"; - -pub fn has_suffix(s: &str, suffix: &str) -> bool { - if cfg!(target_os = "windows") { - s.to_lowercase().ends_with(&suffix.to_lowercase()) - } else { - s.ends_with(suffix) - } -} - -pub fn encode_dir_object(object: &str) -> String { - if has_suffix(object, SLASH_SEPARATOR) { - format!("{}{}", object.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX) - } else { - object.to_string() - } -} - -pub fn is_dir_object(object: &str) -> bool { - let obj = encode_dir_object(object); - obj.ends_with(GLOBAL_DIR_SUFFIX) -} - -#[allow(dead_code)] -pub fn decode_dir_object(object: &str) -> String { - if has_suffix(object, GLOBAL_DIR_SUFFIX) { - format!("{}{}", object.trim_end_matches(GLOBAL_DIR_SUFFIX), SLASH_SEPARATOR) - } else { - object.to_string() - } -} - -pub fn retain_slash(s: &str) -> String { - if s.is_empty() { - return s.to_string(); - } - if s.ends_with(SLASH_SEPARATOR) { - s.to_string() - } else { - format!("{}{}", s, SLASH_SEPARATOR) - } -} - -pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool { - s.len() >= prefix.len() && (s[..prefix.len()] == *prefix || s[..prefix.len()].eq_ignore_ascii_case(prefix)) -} - -pub fn has_prefix(s: &str, prefix: &str) -> bool { - if cfg!(target_os = "windows") { - return strings_has_prefix_fold(s, prefix); - } - - s.starts_with(prefix) -} - -pub fn path_join(elem: &[PathBuf]) -> PathBuf { - let mut joined_path = PathBuf::new(); - - for path in elem { - joined_path.push(path); - } - - joined_path -} - -pub fn path_join_buf(elements: &[&str]) -> String { - let trailing_slash = !elements.is_empty() && elements.last().unwrap().ends_with(SLASH_SEPARATOR); - - let mut dst = String::new(); - let mut added = 0; - - for e in elements { - if added > 0 || !e.is_empty() { - if added > 0 { - dst.push_str(SLASH_SEPARATOR); - } - dst.push_str(e); - added += e.len(); - } - } - - let result = dst.to_string(); - let cpath = Path::new(&result).components().collect::(); - let clean_path = cpath.to_string_lossy(); - - if trailing_slash { - return format!("{}{}", clean_path, SLASH_SEPARATOR); - } - clean_path.to_string() -} - -pub fn path_to_bucket_object_with_base_path(bash_path: &str, path: &str) -> (String, String) { - let path = path.trim_start_matches(bash_path).trim_start_matches(SLASH_SEPARATOR); - if let Some(m) = path.find(SLASH_SEPARATOR) { - return (path[..m].to_string(), path[m + SLASH_SEPARATOR.len()..].to_string()); - } - - (path.to_string(), "".to_string()) -} - -pub fn path_to_bucket_object(s: &str) -> (String, String) { - path_to_bucket_object_with_base_path("", s) -} - -pub fn base_dir_from_prefix(prefix: &str) -> String { - let mut base_dir = dir(prefix).to_owned(); - if base_dir == "." || base_dir == "./" || base_dir == "/" { - base_dir = "".to_owned(); - } - if !prefix.contains('/') { - base_dir = "".to_owned(); - } - if !base_dir.is_empty() && !base_dir.ends_with(SLASH_SEPARATOR) { - base_dir.push_str(SLASH_SEPARATOR); - } - base_dir -} - -pub struct LazyBuf { - s: String, - buf: Option>, - w: usize, -} - -impl LazyBuf { - pub fn new(s: String) -> Self { - LazyBuf { s, buf: None, w: 0 } - } - - pub fn index(&self, i: usize) -> u8 { - if let Some(ref buf) = self.buf { - buf[i] - } else { - self.s.as_bytes()[i] - } - } - - pub fn append(&mut self, c: u8) { - if self.buf.is_none() { - if self.w < self.s.len() && self.s.as_bytes()[self.w] == c { - self.w += 1; - return; - } - let mut new_buf = vec![0; self.s.len()]; - new_buf[..self.w].copy_from_slice(&self.s.as_bytes()[..self.w]); - self.buf = Some(new_buf); - } - - if let Some(ref mut buf) = self.buf { - buf[self.w] = c; - self.w += 1; - } - } - - pub fn string(&self) -> String { - if let Some(ref buf) = self.buf { - String::from_utf8(buf[..self.w].to_vec()).unwrap() - } else { - self.s[..self.w].to_string() - } - } -} - -pub fn clean(path: &str) -> String { - if path.is_empty() { - return ".".to_string(); - } - - let rooted = path.starts_with('/'); - let n = path.len(); - let mut out = LazyBuf::new(path.to_string()); - let mut r = 0; - let mut dotdot = 0; - - if rooted { - out.append(b'/'); - r = 1; - dotdot = 1; - } - - while r < n { - match path.as_bytes()[r] { - b'/' => { - // Empty path element - r += 1; - } - b'.' if r + 1 == n || path.as_bytes()[r + 1] == b'/' => { - // . element - r += 1; - } - b'.' if path.as_bytes()[r + 1] == b'.' && (r + 2 == n || path.as_bytes()[r + 2] == b'/') => { - // .. element: remove to last / - r += 2; - - if out.w > dotdot { - // Can backtrack - out.w -= 1; - while out.w > dotdot && out.index(out.w) != b'/' { - out.w -= 1; - } - } else if !rooted { - // Cannot backtrack but not rooted, so append .. element. - if out.w > 0 { - out.append(b'/'); - } - out.append(b'.'); - out.append(b'.'); - dotdot = out.w; - } - } - _ => { - // Real path element. - // Add slash if needed - if (rooted && out.w != 1) || (!rooted && out.w != 0) { - out.append(b'/'); - } - - // Copy element - while r < n && path.as_bytes()[r] != b'/' { - out.append(path.as_bytes()[r]); - r += 1; - } - } - } - } - - // Turn empty string into "." - if out.w == 0 { - return ".".to_string(); - } - - out.string() -} - -pub fn split(path: &str) -> (&str, &str) { - // Find the last occurrence of the '/' character - if let Some(i) = path.rfind('/') { - // Return the directory (up to and including the last '/') and the file name - return (&path[..i + 1], &path[i + 1..]); - } - // If no '/' is found, return an empty string for the directory and the whole path as the file name - (path, "") -} - -pub fn dir(path: &str) -> String { - let (a, _) = split(path); - clean(a) -} -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_base_dir_from_prefix() { - let a = "da/"; - println!("---- in {}", a); - let a = base_dir_from_prefix(a); - println!("---- out {}", a); - } - - #[test] - fn test_clean() { - assert_eq!(clean(""), "."); - assert_eq!(clean("abc"), "abc"); - assert_eq!(clean("abc/def"), "abc/def"); - assert_eq!(clean("a/b/c"), "a/b/c"); - assert_eq!(clean("."), "."); - assert_eq!(clean(".."), ".."); - assert_eq!(clean("../.."), "../.."); - assert_eq!(clean("../../abc"), "../../abc"); - assert_eq!(clean("/abc"), "/abc"); - assert_eq!(clean("/"), "/"); - assert_eq!(clean("abc/"), "abc"); - assert_eq!(clean("abc/def/"), "abc/def"); - assert_eq!(clean("a/b/c/"), "a/b/c"); - assert_eq!(clean("./"), "."); - assert_eq!(clean("../"), ".."); - assert_eq!(clean("../../"), "../.."); - assert_eq!(clean("/abc/"), "/abc"); - assert_eq!(clean("abc//def//ghi"), "abc/def/ghi"); - assert_eq!(clean("//abc"), "/abc"); - assert_eq!(clean("///abc"), "/abc"); - assert_eq!(clean("//abc//"), "/abc"); - assert_eq!(clean("abc//"), "abc"); - assert_eq!(clean("abc/./def"), "abc/def"); - assert_eq!(clean("/./abc/def"), "/abc/def"); - assert_eq!(clean("abc/."), "abc"); - assert_eq!(clean("abc/./../def"), "def"); - assert_eq!(clean("abc//./../def"), "def"); - assert_eq!(clean("abc/../../././../def"), "../../def"); - - assert_eq!(clean("abc/def/ghi/../jkl"), "abc/def/jkl"); - assert_eq!(clean("abc/def/../ghi/../jkl"), "abc/jkl"); - assert_eq!(clean("abc/def/.."), "abc"); - assert_eq!(clean("abc/def/../.."), "."); - assert_eq!(clean("/abc/def/../.."), "/"); - assert_eq!(clean("abc/def/../../.."), ".."); - assert_eq!(clean("/abc/def/../../.."), "/"); - assert_eq!(clean("abc/def/../../../ghi/jkl/../../../mno"), "../../mno"); - } -} diff --git a/crates/disk/src/remote.rs b/crates/disk/src/remote.rs deleted file mode 100644 index 575c0763a..000000000 --- a/crates/disk/src/remote.rs +++ /dev/null @@ -1,908 +0,0 @@ -use std::path::PathBuf; - -use crate::api::CheckPartsResp; -use crate::api::DeleteOptions; -use crate::api::DiskAPI; -use crate::api::DiskInfo; -use crate::api::DiskInfoOptions; -use crate::api::DiskLocation; -use crate::api::DiskOption; -use crate::api::ReadMultipleReq; -use crate::api::ReadMultipleResp; -use crate::api::ReadOptions; -use crate::api::RenameDataResp; -use crate::api::UpdateMetadataOpts; -use crate::api::VolumeInfo; -use crate::api::WalkDirOptions; -use crate::endpoint::Endpoint; -use futures::StreamExt as _; -use http::HeaderMap; -use http::Method; -use protos::node_service_time_out_client; -use protos::proto_gen::node_service::CheckPartsRequest; -use protos::proto_gen::node_service::DeletePathsRequest; -use protos::proto_gen::node_service::DeleteRequest; -use protos::proto_gen::node_service::DeleteVersionRequest; -use protos::proto_gen::node_service::DeleteVersionsRequest; -use protos::proto_gen::node_service::DeleteVolumeRequest; -use protos::proto_gen::node_service::DiskInfoRequest; -use protos::proto_gen::node_service::ListDirRequest; -use protos::proto_gen::node_service::ListVolumesRequest; -use protos::proto_gen::node_service::MakeVolumeRequest; -use protos::proto_gen::node_service::MakeVolumesRequest; -use protos::proto_gen::node_service::ReadAllRequest; -use protos::proto_gen::node_service::ReadMultipleRequest; -use protos::proto_gen::node_service::ReadVersionRequest; -use protos::proto_gen::node_service::ReadXlRequest; -use protos::proto_gen::node_service::RenameDataRequest; -use protos::proto_gen::node_service::RenameFileRequst; -use protos::proto_gen::node_service::RenamePartRequst; -use protos::proto_gen::node_service::StatVolumeRequest; -use protos::proto_gen::node_service::UpdateMetadataRequest; -use protos::proto_gen::node_service::VerifyFileRequest; -use protos::proto_gen::node_service::WalkDirRequest; -use protos::proto_gen::node_service::WriteAllRequest; -use protos::proto_gen::node_service::WriteMetadataRequest; -use rmp_serde::Serializer; -use rustfs_error::Error; -use rustfs_error::Result; -use rustfs_filemeta::FileInfo; -use rustfs_filemeta::FileInfoVersions; -use rustfs_filemeta::RawFileInfo; -use rustfs_metacache::MetaCacheEntry; -use rustfs_metacache::MetacacheWriter; -use rustfs_rio::HttpReader; -use rustfs_rio::HttpWriter; -use serde::Serialize as _; -use tokio::io::AsyncRead; -use tokio::io::AsyncWrite; -use tokio::sync::Mutex; -use tonic::Request; -use tracing::info; -use uuid::Uuid; - -#[derive(Debug)] -pub struct RemoteDisk { - pub id: Mutex>, - pub addr: String, - pub url: url::Url, - pub root: PathBuf, - endpoint: Endpoint, -} - -impl RemoteDisk { - pub async fn new(ep: &Endpoint, _opt: &DiskOption) -> Result { - // let root = fs::canonicalize(ep.url.path()).await?; - let root = PathBuf::from(ep.get_file_path()); - let addr = format!("{}://{}:{}", ep.url.scheme(), ep.url.host_str().unwrap(), ep.url.port().unwrap()); - Ok(Self { - id: Mutex::new(None), - addr, - url: ep.url.clone(), - root, - endpoint: ep.clone(), - }) - } -} - -// TODO: all api need to handle errors -#[async_trait::async_trait] -impl DiskAPI for RemoteDisk { - #[tracing::instrument(skip(self))] - fn to_string(&self) -> String { - self.endpoint.to_string() - } - - #[tracing::instrument(skip(self))] - fn is_local(&self) -> bool { - false - } - - #[tracing::instrument(skip(self))] - fn host_name(&self) -> String { - self.endpoint.host_port() - } - #[tracing::instrument(skip(self))] - async fn is_online(&self) -> bool { - // TODO: 连接状态 - if (node_service_time_out_client(&self.addr).await).is_ok() { - return true; - } - false - } - #[tracing::instrument(skip(self))] - fn endpoint(&self) -> Endpoint { - self.endpoint.clone() - } - #[tracing::instrument(skip(self))] - async fn close(&self) -> Result<()> { - Ok(()) - } - #[tracing::instrument(skip(self))] - fn path(&self) -> PathBuf { - self.root.clone() - } - - #[tracing::instrument(skip(self))] - fn get_disk_location(&self) -> DiskLocation { - DiskLocation { - pool_idx: { - if self.endpoint.pool_idx < 0 { - None - } else { - Some(self.endpoint.pool_idx as usize) - } - }, - set_idx: { - if self.endpoint.set_idx < 0 { - None - } else { - Some(self.endpoint.set_idx as usize) - } - }, - disk_idx: { - if self.endpoint.disk_idx < 0 { - None - } else { - Some(self.endpoint.disk_idx as usize) - } - }, - } - } - - #[tracing::instrument(skip(self))] - async fn get_disk_id(&self) -> Result> { - Ok(*self.id.lock().await) - } - - #[tracing::instrument(skip(self))] - async fn set_disk_id(&self, id: Option) -> Result<()> { - let mut lock = self.id.lock().await; - *lock = id; - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn read_all(&self, volume: &str, path: &str) -> Result> { - info!("read_all {}/{}", volume, path); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(ReadAllRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - }); - - let response = client.read_all(request).await?.into_inner(); - - if !response.success { - return Err(Error::FileNotFound); - } - - Ok(response.data) - } - - #[tracing::instrument(skip(self))] - async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()> { - info!("write_all"); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(WriteAllRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - data, - }); - - let response = client.write_all(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { - info!("delete {}/{}/{}", self.endpoint.to_string(), volume, path); - let options = serde_json::to_string(&opt)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(DeleteRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - options, - }); - - let response = client.delete(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - info!("verify_file"); - let file_info = serde_json::to_string(&fi)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(VerifyFileRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - file_info, - }); - - let response = client.verify_file(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let check_parts_resp = serde_json::from_str::(&response.check_parts_resp)?; - - Ok(check_parts_resp) - } - - #[tracing::instrument(skip(self))] - async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - info!("check_parts"); - let file_info = serde_json::to_string(&fi)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(CheckPartsRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - file_info, - }); - - let response = client.check_parts(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let check_parts_resp = serde_json::from_str::(&response.check_parts_resp)?; - - Ok(check_parts_resp) - } - - #[tracing::instrument(skip(self))] - async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()> { - info!("rename_part {}/{}", src_volume, src_path); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(RenamePartRequst { - disk: self.endpoint.to_string(), - src_volume: src_volume.to_string(), - src_path: src_path.to_string(), - dst_volume: dst_volume.to_string(), - dst_path: dst_path.to_string(), - meta, - }); - - let response = client.rename_part(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { - info!("rename_file"); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(RenameFileRequst { - disk: self.endpoint.to_string(), - src_volume: src_volume.to_string(), - src_path: src_path.to_string(), - dst_volume: dst_volume.to_string(), - dst_path: dst_path.to_string(), - }); - - let response = client.rename_file(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result> { - info!("create_file {}/{}/{}", self.endpoint.to_string(), volume, path); - - let url = format!( - "{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}", - self.endpoint.grid_host(), - urlencoding::encode(&self.endpoint.to_string()), - urlencoding::encode(volume), - urlencoding::encode(path), - false, - file_size - ); - - let wd = HttpWriter::new(url, Method::PUT, HeaderMap::new()).await?; - - Ok(Box::new(wd)) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn append_file(&self, volume: &str, path: &str) -> Result> { - info!("append_file {}/{}", volume, path); - let url = format!( - "{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}", - self.endpoint.grid_host(), - urlencoding::encode(&self.endpoint.to_string()), - urlencoding::encode(volume), - urlencoding::encode(path), - true, - 0 - ); - - let wd = HttpWriter::new(url, Method::PUT, HeaderMap::new()).await?; - - Ok(Box::new(wd)) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn read_file(&self, volume: &str, path: &str) -> Result> { - info!("read_file {}/{}", volume, path); - let url = format!( - "{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}", - self.endpoint.grid_host(), - urlencoding::encode(&self.endpoint.to_string()), - urlencoding::encode(volume), - urlencoding::encode(path), - 0, - 0 - ); - - let rd = HttpReader::new(url, Method::GET, HeaderMap::new()).await?; - - Ok(Box::new(rd)) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result> { - info!("read_file_stream {}/{}/{}", self.endpoint.to_string(), volume, path); - let url = format!( - "{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}", - self.endpoint.grid_host(), - urlencoding::encode(&self.endpoint.to_string()), - urlencoding::encode(volume), - urlencoding::encode(path), - offset, - length - ); - let rd = HttpReader::new(url, Method::GET, HeaderMap::new()).await?; - - Ok(Box::new(rd)) - } - - #[tracing::instrument(skip(self))] - async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result> { - info!("list_dir {}/{}", volume, _dir_path); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(ListDirRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - }); - - let response = client.list_dir(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(response.volumes) - } - - // FIXME: TODO: use writer - #[tracing::instrument(skip(self, wr))] - async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> { - let now = std::time::SystemTime::now(); - info!("walk_dir {}/{}/{:?}", self.endpoint.to_string(), opts.bucket, opts.filter_prefix); - let mut wr = wr; - let mut out = MetacacheWriter::new(&mut wr); - let mut buf = Vec::new(); - opts.serialize(&mut Serializer::new(&mut buf))?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(WalkDirRequest { - disk: self.endpoint.to_string(), - walk_dir_options: buf, - }); - let mut response = client.walk_dir(request).await?.into_inner(); - - loop { - match response.next().await { - Some(Ok(resp)) => { - if !resp.success { - return Err(Error::msg(resp.error_info.unwrap_or("".to_string()))); - } - let entry = serde_json::from_str::(&resp.meta_cache_entry) - .map_err(|_| Error::msg(format!("Unexpected response: {:?}", response)))?; - out.write_obj(&entry).await?; - } - None => break, - _ => return Err(Error::msg(format!("Unexpected response: {:?}", response))), - } - } - - info!( - "walk_dir {}/{:?} done {:?}", - opts.bucket, - opts.filter_prefix, - now.elapsed().unwrap_or_default() - ); - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn rename_data( - &self, - src_volume: &str, - src_path: &str, - fi: FileInfo, - dst_volume: &str, - dst_path: &str, - ) -> Result { - info!("rename_data {}/{}/{}/{}", self.addr, self.endpoint.to_string(), dst_volume, dst_path); - let file_info = serde_json::to_string(&fi)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(RenameDataRequest { - disk: self.endpoint.to_string(), - src_volume: src_volume.to_string(), - src_path: src_path.to_string(), - file_info, - dst_volume: dst_volume.to_string(), - dst_path: dst_path.to_string(), - }); - - let response = client.rename_data(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let rename_data_resp = serde_json::from_str::(&response.rename_data_resp)?; - - Ok(rename_data_resp) - } - - #[tracing::instrument(skip(self))] - async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> { - info!("make_volumes"); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(MakeVolumesRequest { - disk: self.endpoint.to_string(), - volumes: volumes.iter().map(|s| (*s).to_string()).collect(), - }); - - let response = client.make_volumes(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn make_volume(&self, volume: &str) -> Result<()> { - info!("make_volume"); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(MakeVolumeRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - }); - - let response = client.make_volume(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn list_volumes(&self) -> Result> { - info!("list_volumes"); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(ListVolumesRequest { - disk: self.endpoint.to_string(), - }); - - let response = client.list_volumes(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let infos = response - .volume_infos - .into_iter() - .filter_map(|json_str| serde_json::from_str::(&json_str).ok()) - .collect(); - - Ok(infos) - } - - #[tracing::instrument(skip(self))] - async fn stat_volume(&self, volume: &str) -> Result { - info!("stat_volume"); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(StatVolumeRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - }); - - let response = client.stat_volume(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let volume_info = serde_json::from_str::(&response.volume_info)?; - - Ok(volume_info) - } - - #[tracing::instrument(skip(self))] - async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> { - info!("delete_paths"); - let paths = paths.to_owned(); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(DeletePathsRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - paths, - }); - - let response = client.delete_paths(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> { - info!("update_metadata"); - let file_info = serde_json::to_string(&fi)?; - let opts = serde_json::to_string(&opts)?; - - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(UpdateMetadataRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - file_info, - opts, - }); - - let response = client.update_metadata(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { - info!("write_metadata {}/{}", volume, path); - let file_info = serde_json::to_string(&fi)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(WriteMetadataRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - file_info, - }); - - let response = client.write_metadata(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn read_version( - &self, - _org_volume: &str, - volume: &str, - path: &str, - version_id: &str, - opts: &ReadOptions, - ) -> Result { - info!("read_version"); - let opts = serde_json::to_string(opts)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(ReadVersionRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - version_id: version_id.to_string(), - opts, - }); - - let response = client.read_version(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let file_info = serde_json::from_str::(&response.file_info)?; - - Ok(file_info) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { - info!("read_xl {}/{}/{}", self.endpoint.to_string(), volume, path); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(ReadXlRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - read_data, - }); - - let response = client.read_xl(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let raw_file_info = serde_json::from_str::(&response.raw_file_info)?; - - Ok(raw_file_info) - } - - #[tracing::instrument(skip(self))] - async fn delete_version( - &self, - volume: &str, - path: &str, - fi: FileInfo, - force_del_marker: bool, - opts: DeleteOptions, - ) -> Result<()> { - info!("delete_version"); - let file_info = serde_json::to_string(&fi)?; - let opts = serde_json::to_string(&opts)?; - - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(DeleteVersionRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - file_info, - force_del_marker, - opts, - }); - - let response = client.delete_version(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - // let raw_file_info = serde_json::from_str::(&response.raw_file_info)?; - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn delete_versions( - &self, - volume: &str, - versions: Vec, - opts: DeleteOptions, - ) -> Result>> { - info!("delete_versions"); - let opts = serde_json::to_string(&opts)?; - let mut versions_str = Vec::with_capacity(versions.len()); - for file_info_versions in versions.iter() { - versions_str.push(serde_json::to_string(file_info_versions)?); - } - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(DeleteVersionsRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - versions: versions_str, - opts, - }); - - let response = client.delete_versions(request).await?.into_inner(); - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - let errors = response - .errors - .iter() - .map(|error| { - if error.is_empty() { - None - } else { - use std::str::FromStr; - Some(Error::from_str(error).unwrap_or(Error::msg(error))) - } - }) - .collect(); - - Ok(errors) - } - - #[tracing::instrument(skip(self))] - async fn read_multiple(&self, req: ReadMultipleReq) -> Result> { - info!("read_multiple {}/{}/{}", self.endpoint.to_string(), req.bucket, req.prefix); - let read_multiple_req = serde_json::to_string(&req)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(ReadMultipleRequest { - disk: self.endpoint.to_string(), - read_multiple_req, - }); - - let response = client.read_multiple(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let read_multiple_resps = response - .read_multiple_resps - .into_iter() - .filter_map(|json_str| serde_json::from_str::(&json_str).ok()) - .collect(); - - Ok(read_multiple_resps) - } - - #[tracing::instrument(skip(self))] - async fn delete_volume(&self, volume: &str) -> Result<()> { - info!("delete_volume {}/{}", self.endpoint.to_string(), volume); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(DeleteVolumeRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - }); - - let response = client.delete_volume(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn disk_info(&self, opts: &DiskInfoOptions) -> Result { - let opts = serde_json::to_string(&opts)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(DiskInfoRequest { - disk: self.endpoint.to_string(), - opts, - }); - - let response = client.disk_info(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let disk_info = serde_json::from_str::(&response.disk_info)?; - - Ok(disk_info) - } - - // #[tracing::instrument(skip(self, cache, scan_mode, _we_sleep))] - // async fn ns_scanner( - // &self, - // cache: &DataUsageCache, - // updates: Sender, - // scan_mode: HealScanMode, - // _we_sleep: ShouldSleepFn, - // ) -> Result { - // info!("ns_scanner"); - // let cache = serde_json::to_string(cache)?; - // let mut client = node_service_time_out_client(&self.addr) - // .await - // .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - - // let (tx, rx) = mpsc::channel(10); - // let in_stream = ReceiverStream::new(rx); - // let mut response = client.ns_scanner(in_stream).await?.into_inner(); - // let request = NsScannerRequest { - // disk: self.endpoint.to_string(), - // cache, - // scan_mode: scan_mode as u64, - // }; - // tx.send(request) - // .await - // .map_err(|err| Error::msg(format!("can not send request, err: {}", err)))?; - - // loop { - // match response.next().await { - // Some(Ok(resp)) => { - // if !resp.update.is_empty() { - // let data_usage_cache = serde_json::from_str::(&resp.update)?; - // let _ = updates.send(data_usage_cache).await; - // } else if !resp.data_usage_cache.is_empty() { - // let data_usage_cache = serde_json::from_str::(&resp.data_usage_cache)?; - // return Ok(data_usage_cache); - // } else { - // return Err(Error::msg("scan was interrupted")); - // } - // } - // _ => return Err(Error::msg("scan was interrupted")), - // } - // } - // } - - // #[tracing::instrument(skip(self))] - // async fn healing(&self) -> Option { - // None - // } -} diff --git a/crates/disk/src/remote_bak.rs b/crates/disk/src/remote_bak.rs deleted file mode 100644 index c1ea57b6f..000000000 --- a/crates/disk/src/remote_bak.rs +++ /dev/null @@ -1,862 +0,0 @@ -use std::path::PathBuf; - -use super::{ - endpoint::Endpoint, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, - FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, - WalkDirOptions, -}; -use crate::heal::{ - data_scanner::ShouldSleepFn, - data_usage_cache::{DataUsageCache, DataUsageEntry}, - heal_commands::{HealScanMode, HealingTracker}, -}; -use crate::io::{FileReader, FileWriter, HttpFileReader, HttpFileWriter}; -use crate::{disk::metacache::MetaCacheEntry, metacache::writer::MetacacheWriter}; -use futures::lock::Mutex; -use protos::proto_gen::node_service::RenamePartRequst; -use protos::{ - node_service_time_out_client, - proto_gen::node_service::{ - CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, - DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, NsScannerRequest, - ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst, - StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest, - }, -}; -use rmp_serde::Serializer; -use rustfs_error::{Error, Result}; -use rustfs_filemeta::{FileInfo, RawFileInfo}; -use serde::Serialize; -use tokio::{ - io::AsyncWrite, - sync::mpsc::{self, Sender}, -}; -use tokio_stream::{wrappers::ReceiverStream, StreamExt}; -use tonic::Request; -use tracing::info; -use uuid::Uuid; - -#[derive(Debug)] -pub struct RemoteDisk { - pub id: Mutex>, - pub addr: String, - pub url: url::Url, - pub root: PathBuf, - endpoint: Endpoint, -} - -impl RemoteDisk { - pub async fn new(ep: &Endpoint, _opt: &DiskOption) -> Result { - // let root = fs::canonicalize(ep.url.path()).await?; - let root = PathBuf::from(ep.get_file_path()); - let addr = format!("{}://{}:{}", ep.url.scheme(), ep.url.host_str().unwrap(), ep.url.port().unwrap()); - Ok(Self { - id: Mutex::new(None), - addr, - url: ep.url.clone(), - root, - endpoint: ep.clone(), - }) - } -} - -// TODO: all api need to handle errors -#[async_trait::async_trait] -impl DiskAPI for RemoteDisk { - #[tracing::instrument(skip(self))] - fn to_string(&self) -> String { - self.endpoint.to_string() - } - - #[tracing::instrument(skip(self))] - fn is_local(&self) -> bool { - false - } - - #[tracing::instrument(skip(self))] - fn host_name(&self) -> String { - self.endpoint.host_port() - } - #[tracing::instrument(skip(self))] - async fn is_online(&self) -> bool { - // TODO: 连接状态 - if (node_service_time_out_client(&self.addr).await).is_ok() { - return true; - } - false - } - #[tracing::instrument(skip(self))] - fn endpoint(&self) -> Endpoint { - self.endpoint.clone() - } - #[tracing::instrument(skip(self))] - async fn close(&self) -> Result<()> { - Ok(()) - } - #[tracing::instrument(skip(self))] - fn path(&self) -> PathBuf { - self.root.clone() - } - - #[tracing::instrument(skip(self))] - fn get_disk_location(&self) -> DiskLocation { - DiskLocation { - pool_idx: { - if self.endpoint.pool_idx < 0 { - None - } else { - Some(self.endpoint.pool_idx as usize) - } - }, - set_idx: { - if self.endpoint.set_idx < 0 { - None - } else { - Some(self.endpoint.set_idx as usize) - } - }, - disk_idx: { - if self.endpoint.disk_idx < 0 { - None - } else { - Some(self.endpoint.disk_idx as usize) - } - }, - } - } - - #[tracing::instrument(skip(self))] - async fn get_disk_id(&self) -> Result> { - Ok(*self.id.lock().await) - } - - #[tracing::instrument(skip(self))] - async fn set_disk_id(&self, id: Option) -> Result<()> { - let mut lock = self.id.lock().await; - *lock = id; - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn read_all(&self, volume: &str, path: &str) -> Result> { - info!("read_all {}/{}", volume, path); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(ReadAllRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - }); - - let response = client.read_all(request).await?.into_inner(); - - if !response.success { - return Err(Error::FileNotFound); - } - - Ok(response.data) - } - - #[tracing::instrument(skip(self))] - async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()> { - info!("write_all"); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(WriteAllRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - data, - }); - - let response = client.write_all(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { - info!("delete {}/{}/{}", self.endpoint.to_string(), volume, path); - let options = serde_json::to_string(&opt)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(DeleteRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - options, - }); - - let response = client.delete(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - info!("verify_file"); - let file_info = serde_json::to_string(&fi)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(VerifyFileRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - file_info, - }); - - let response = client.verify_file(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let check_parts_resp = serde_json::from_str::(&response.check_parts_resp)?; - - Ok(check_parts_resp) - } - - #[tracing::instrument(skip(self))] - async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - info!("check_parts"); - let file_info = serde_json::to_string(&fi)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(CheckPartsRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - file_info, - }); - - let response = client.check_parts(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let check_parts_resp = serde_json::from_str::(&response.check_parts_resp)?; - - Ok(check_parts_resp) - } - - #[tracing::instrument(skip(self))] - async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()> { - info!("rename_part {}/{}", src_volume, src_path); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(RenamePartRequst { - disk: self.endpoint.to_string(), - src_volume: src_volume.to_string(), - src_path: src_path.to_string(), - dst_volume: dst_volume.to_string(), - dst_path: dst_path.to_string(), - meta, - }); - - let response = client.rename_part(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { - info!("rename_file"); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(RenameFileRequst { - disk: self.endpoint.to_string(), - src_volume: src_volume.to_string(), - src_path: src_path.to_string(), - dst_volume: dst_volume.to_string(), - dst_path: dst_path.to_string(), - }); - - let response = client.rename_file(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result { - info!("create_file {}/{}/{}", self.endpoint.to_string(), volume, path); - Ok(Box::new(HttpFileWriter::new( - self.endpoint.grid_host().as_str(), - self.endpoint.to_string().as_str(), - volume, - path, - file_size, - false, - )?)) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn append_file(&self, volume: &str, path: &str) -> Result { - info!("append_file {}/{}", volume, path); - Ok(Box::new(HttpFileWriter::new( - self.endpoint.grid_host().as_str(), - self.endpoint.to_string().as_str(), - volume, - path, - 0, - true, - )?)) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn read_file(&self, volume: &str, path: &str) -> Result { - info!("read_file {}/{}", volume, path); - Ok(Box::new( - HttpFileReader::new(self.endpoint.grid_host().as_str(), self.endpoint.to_string().as_str(), volume, path, 0, 0) - .await?, - )) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result { - info!("read_file_stream {}/{}/{}", self.endpoint.to_string(), volume, path); - Ok(Box::new( - HttpFileReader::new( - self.endpoint.grid_host().as_str(), - self.endpoint.to_string().as_str(), - volume, - path, - offset, - length, - ) - .await?, - )) - } - - #[tracing::instrument(skip(self))] - async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result> { - info!("list_dir {}/{}", volume, _dir_path); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(ListDirRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - }); - - let response = client.list_dir(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(response.volumes) - } - - // FIXME: TODO: use writer - #[tracing::instrument(skip(self, wr))] - async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> { - let now = std::time::SystemTime::now(); - info!("walk_dir {}/{}/{:?}", self.endpoint.to_string(), opts.bucket, opts.filter_prefix); - let mut wr = wr; - let mut out = MetacacheWriter::new(&mut wr); - let mut buf = Vec::new(); - opts.serialize(&mut Serializer::new(&mut buf))?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(WalkDirRequest { - disk: self.endpoint.to_string(), - walk_dir_options: buf, - }); - let mut response = client.walk_dir(request).await?.into_inner(); - - loop { - match response.next().await { - Some(Ok(resp)) => { - if !resp.success { - return Err(Error::msg(resp.error_info.unwrap_or("".to_string()))); - } - let entry = serde_json::from_str::(&resp.meta_cache_entry) - .map_err(|_| Error::msg(format!("Unexpected response: {:?}", response)))?; - out.write_obj(&entry).await?; - } - None => break, - _ => return Err(Error::msg(format!("Unexpected response: {:?}", response))), - } - } - - info!( - "walk_dir {}/{:?} done {:?}", - opts.bucket, - opts.filter_prefix, - now.elapsed().unwrap_or_default() - ); - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn rename_data( - &self, - src_volume: &str, - src_path: &str, - fi: FileInfo, - dst_volume: &str, - dst_path: &str, - ) -> Result { - info!("rename_data {}/{}/{}/{}", self.addr, self.endpoint.to_string(), dst_volume, dst_path); - let file_info = serde_json::to_string(&fi)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(RenameDataRequest { - disk: self.endpoint.to_string(), - src_volume: src_volume.to_string(), - src_path: src_path.to_string(), - file_info, - dst_volume: dst_volume.to_string(), - dst_path: dst_path.to_string(), - }); - - let response = client.rename_data(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let rename_data_resp = serde_json::from_str::(&response.rename_data_resp)?; - - Ok(rename_data_resp) - } - - #[tracing::instrument(skip(self))] - async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> { - info!("make_volumes"); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(MakeVolumesRequest { - disk: self.endpoint.to_string(), - volumes: volumes.iter().map(|s| (*s).to_string()).collect(), - }); - - let response = client.make_volumes(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn make_volume(&self, volume: &str) -> Result<()> { - info!("make_volume"); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(MakeVolumeRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - }); - - let response = client.make_volume(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn list_volumes(&self) -> Result> { - info!("list_volumes"); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(ListVolumesRequest { - disk: self.endpoint.to_string(), - }); - - let response = client.list_volumes(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let infos = response - .volume_infos - .into_iter() - .filter_map(|json_str| serde_json::from_str::(&json_str).ok()) - .collect(); - - Ok(infos) - } - - #[tracing::instrument(skip(self))] - async fn stat_volume(&self, volume: &str) -> Result { - info!("stat_volume"); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(StatVolumeRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - }); - - let response = client.stat_volume(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let volume_info = serde_json::from_str::(&response.volume_info)?; - - Ok(volume_info) - } - - #[tracing::instrument(skip(self))] - async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> { - info!("delete_paths"); - let paths = paths.to_owned(); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(DeletePathsRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - paths, - }); - - let response = client.delete_paths(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> { - info!("update_metadata"); - let file_info = serde_json::to_string(&fi)?; - let opts = serde_json::to_string(&opts)?; - - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(UpdateMetadataRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - file_info, - opts, - }); - - let response = client.update_metadata(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { - info!("write_metadata {}/{}", volume, path); - let file_info = serde_json::to_string(&fi)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(WriteMetadataRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - file_info, - }); - - let response = client.write_metadata(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn read_version( - &self, - _org_volume: &str, - volume: &str, - path: &str, - version_id: &str, - opts: &ReadOptions, - ) -> Result { - info!("read_version"); - let opts = serde_json::to_string(opts)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(ReadVersionRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - version_id: version_id.to_string(), - opts, - }); - - let response = client.read_version(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let file_info = serde_json::from_str::(&response.file_info)?; - - Ok(file_info) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { - info!("read_xl {}/{}/{}", self.endpoint.to_string(), volume, path); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(ReadXlRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - read_data, - }); - - let response = client.read_xl(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let raw_file_info = serde_json::from_str::(&response.raw_file_info)?; - - Ok(raw_file_info) - } - - #[tracing::instrument(skip(self))] - async fn delete_version( - &self, - volume: &str, - path: &str, - fi: FileInfo, - force_del_marker: bool, - opts: DeleteOptions, - ) -> Result<()> { - info!("delete_version"); - let file_info = serde_json::to_string(&fi)?; - let opts = serde_json::to_string(&opts)?; - - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(DeleteVersionRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - path: path.to_string(), - file_info, - force_del_marker, - opts, - }); - - let response = client.delete_version(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - // let raw_file_info = serde_json::from_str::(&response.raw_file_info)?; - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn delete_versions( - &self, - volume: &str, - versions: Vec, - opts: DeleteOptions, - ) -> Result>> { - info!("delete_versions"); - let opts = serde_json::to_string(&opts)?; - let mut versions_str = Vec::with_capacity(versions.len()); - for file_info_versions in versions.iter() { - versions_str.push(serde_json::to_string(file_info_versions)?); - } - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(DeleteVersionsRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - versions: versions_str, - opts, - }); - - let response = client.delete_versions(request).await?.into_inner(); - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - let errors = response - .errors - .iter() - .map(|error| { - if error.is_empty() { - None - } else { - use std::str::FromStr; - Some(Error::from_str(error).unwrap_or(Error::msg(error))) - } - }) - .collect(); - - Ok(errors) - } - - #[tracing::instrument(skip(self))] - async fn read_multiple(&self, req: ReadMultipleReq) -> Result> { - info!("read_multiple {}/{}/{}", self.endpoint.to_string(), req.bucket, req.prefix); - let read_multiple_req = serde_json::to_string(&req)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(ReadMultipleRequest { - disk: self.endpoint.to_string(), - read_multiple_req, - }); - - let response = client.read_multiple(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let read_multiple_resps = response - .read_multiple_resps - .into_iter() - .filter_map(|json_str| serde_json::from_str::(&json_str).ok()) - .collect(); - - Ok(read_multiple_resps) - } - - #[tracing::instrument(skip(self))] - async fn delete_volume(&self, volume: &str) -> Result<()> { - info!("delete_volume {}/{}", self.endpoint.to_string(), volume); - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(DeleteVolumeRequest { - disk: self.endpoint.to_string(), - volume: volume.to_string(), - }); - - let response = client.delete_volume(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn disk_info(&self, opts: &DiskInfoOptions) -> Result { - let opts = serde_json::to_string(&opts)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - let request = Request::new(DiskInfoRequest { - disk: self.endpoint.to_string(), - opts, - }); - - let response = client.disk_info(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let disk_info = serde_json::from_str::(&response.disk_info)?; - - Ok(disk_info) - } - - #[tracing::instrument(skip(self, cache, scan_mode, _we_sleep))] - async fn ns_scanner( - &self, - cache: &DataUsageCache, - updates: Sender, - scan_mode: HealScanMode, - _we_sleep: ShouldSleepFn, - ) -> Result { - info!("ns_scanner"); - let cache = serde_json::to_string(cache)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::msg(format!("can not get client, err: {}", err)))?; - - let (tx, rx) = mpsc::channel(10); - let in_stream = ReceiverStream::new(rx); - let mut response = client.ns_scanner(in_stream).await?.into_inner(); - let request = NsScannerRequest { - disk: self.endpoint.to_string(), - cache, - scan_mode: scan_mode as u64, - }; - tx.send(request) - .await - .map_err(|err| Error::msg(format!("can not send request, err: {}", err)))?; - - loop { - match response.next().await { - Some(Ok(resp)) => { - if !resp.update.is_empty() { - let data_usage_cache = serde_json::from_str::(&resp.update)?; - let _ = updates.send(data_usage_cache).await; - } else if !resp.data_usage_cache.is_empty() { - let data_usage_cache = serde_json::from_str::(&resp.data_usage_cache)?; - return Ok(data_usage_cache); - } else { - return Err(Error::msg("scan was interrupted")); - } - } - _ => return Err(Error::msg("scan was interrupted")), - } - } - } - - #[tracing::instrument(skip(self))] - async fn healing(&self) -> Option { - None - } -} diff --git a/crates/disk/src/utils.rs b/crates/disk/src/utils.rs deleted file mode 100644 index 94b3f0bd0..000000000 --- a/crates/disk/src/utils.rs +++ /dev/null @@ -1,35 +0,0 @@ -use std::{fs::Metadata, path::Path}; - -use rustfs_error::{to_file_error, Error, Result}; - -pub async fn read_file_exists(path: impl AsRef) -> Result<(Vec, Option)> { - let p = path.as_ref(); - let (data, meta) = match read_file_all(&p).await { - Ok((data, meta)) => (data, Some(meta)), - Err(e) => { - if e == Error::FileNotFound { - (Vec::new(), None) - } else { - return Err(e); - } - } - }; - - Ok((data, meta)) -} - -pub async fn read_file_all(path: impl AsRef) -> Result<(Vec, Metadata)> { - let p = path.as_ref(); - let meta = read_file_metadata(&path).await?; - - let data = read_all(&p).await?; - - Ok((data, meta)) -} - -pub async fn read_file_metadata(p: impl AsRef) -> Result { - Ok(tokio::fs::metadata(&p).await.map_err(to_file_error)?) -} -pub async fn read_all(p: impl AsRef) -> Result> { - tokio::fs::read(&p).await.map_err(|e| to_file_error(e).into()) -} diff --git a/crates/error/Cargo.toml b/crates/error/Cargo.toml deleted file mode 100644 index b2ec8483c..000000000 --- a/crates/error/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "rustfs-error" -edition.workspace = true -license.workspace = true -repository.workspace = true -rust-version.workspace = true -version.workspace = true - - -[dependencies] -protos.workspace = true -rmp.workspace = true -rmp-serde.workspace = true -serde.workspace = true -serde_json.workspace = true -thiserror.workspace = true -time.workspace = true -tonic.workspace = true -uuid.workspace = true - -[lints] -workspace = true diff --git a/crates/error/src/bitrot.rs b/crates/error/src/bitrot.rs deleted file mode 100644 index bb52021c7..000000000 --- a/crates/error/src/bitrot.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::Error; - -pub const CHECK_PART_UNKNOWN: usize = 0; -pub const CHECK_PART_SUCCESS: usize = 1; -pub const CHECK_PART_DISK_NOT_FOUND: usize = 2; -pub const CHECK_PART_VOLUME_NOT_FOUND: usize = 3; -pub const CHECK_PART_FILE_NOT_FOUND: usize = 4; -pub const CHECK_PART_FILE_CORRUPT: usize = 5; - -pub fn conv_part_err_to_int(err: &Option) -> usize { - if let Some(err) = err { - match err { - Error::FileNotFound | Error::FileVersionNotFound => CHECK_PART_FILE_NOT_FOUND, - Error::FileCorrupt => CHECK_PART_FILE_CORRUPT, - Error::VolumeNotFound => CHECK_PART_VOLUME_NOT_FOUND, - Error::DiskNotFound => CHECK_PART_DISK_NOT_FOUND, - Error::Nil => CHECK_PART_SUCCESS, - _ => CHECK_PART_UNKNOWN, - } - } else { - CHECK_PART_SUCCESS - } -} - -pub fn has_part_err(part_errs: &[usize]) -> bool { - part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS) -} diff --git a/crates/error/src/convert.rs b/crates/error/src/convert.rs deleted file mode 100644 index e73731cb4..000000000 --- a/crates/error/src/convert.rs +++ /dev/null @@ -1,92 +0,0 @@ -use crate::Error; - -pub fn to_file_error(io_err: std::io::Error) -> std::io::Error { - match io_err.kind() { - std::io::ErrorKind::NotFound => Error::FileNotFound.into(), - std::io::ErrorKind::PermissionDenied => Error::FileAccessDenied.into(), - std::io::ErrorKind::IsADirectory => Error::IsNotRegular.into(), - std::io::ErrorKind::NotADirectory => Error::FileAccessDenied.into(), - std::io::ErrorKind::DirectoryNotEmpty => Error::FileAccessDenied.into(), - std::io::ErrorKind::UnexpectedEof => Error::FaultyDisk.into(), - std::io::ErrorKind::TooManyLinks => Error::TooManyOpenFiles.into(), - std::io::ErrorKind::InvalidInput => Error::FileNotFound.into(), - std::io::ErrorKind::InvalidData => Error::FileCorrupt.into(), - std::io::ErrorKind::StorageFull => Error::DiskFull.into(), - _ => io_err, - } -} - -pub fn to_volume_error(io_err: std::io::Error) -> std::io::Error { - match io_err.kind() { - std::io::ErrorKind::NotFound => Error::VolumeNotFound.into(), - std::io::ErrorKind::PermissionDenied => Error::DiskAccessDenied.into(), - std::io::ErrorKind::DirectoryNotEmpty => Error::VolumeNotEmpty.into(), - std::io::ErrorKind::NotADirectory => Error::IsNotRegular.into(), - std::io::ErrorKind::Other => { - let err = Error::from(io_err.to_string()); - match err { - Error::FileNotFound => Error::VolumeNotFound.into(), - Error::FileAccessDenied => Error::DiskAccessDenied.into(), - _ => to_file_error(io_err), - } - } - _ => to_file_error(io_err), - } -} - -pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error { - match io_err.kind() { - std::io::ErrorKind::NotFound => Error::DiskNotFound.into(), - std::io::ErrorKind::PermissionDenied => Error::DiskAccessDenied.into(), - std::io::ErrorKind::Other => { - let err = Error::from(io_err.to_string()); - match err { - Error::FileNotFound => Error::DiskNotFound.into(), - Error::VolumeNotFound => Error::DiskNotFound.into(), - Error::FileAccessDenied => Error::DiskAccessDenied.into(), - Error::VolumeAccessDenied => Error::DiskAccessDenied.into(), - _ => to_volume_error(io_err), - } - } - _ => to_volume_error(io_err), - } -} - -// only errors from FileSystem operations -pub fn to_access_error(io_err: std::io::Error, per_err: Error) -> std::io::Error { - match io_err.kind() { - std::io::ErrorKind::PermissionDenied => per_err.into(), - std::io::ErrorKind::NotADirectory => per_err.into(), - std::io::ErrorKind::NotFound => Error::VolumeNotFound.into(), - std::io::ErrorKind::UnexpectedEof => Error::FaultyDisk.into(), - std::io::ErrorKind::Other => { - let err = Error::from(io_err.to_string()); - match err { - Error::DiskAccessDenied => per_err.into(), - Error::FileAccessDenied => per_err.into(), - Error::FileNotFound => Error::VolumeNotFound.into(), - _ => to_volume_error(io_err), - } - } - _ => to_volume_error(io_err), - } -} - -pub fn to_unformatted_disk_error(io_err: std::io::Error) -> std::io::Error { - match io_err.kind() { - std::io::ErrorKind::NotFound => Error::UnformattedDisk.into(), - std::io::ErrorKind::PermissionDenied => Error::DiskAccessDenied.into(), - std::io::ErrorKind::Other => { - let err = Error::from(io_err.to_string()); - match err { - Error::FileNotFound => Error::UnformattedDisk.into(), - Error::DiskNotFound => Error::UnformattedDisk.into(), - Error::VolumeNotFound => Error::UnformattedDisk.into(), - Error::FileAccessDenied => Error::DiskAccessDenied.into(), - Error::DiskAccessDenied => Error::DiskAccessDenied.into(), - _ => Error::CorruptedBackend.into(), - } - } - _ => Error::CorruptedBackend.into(), - } -} diff --git a/crates/error/src/error.rs b/crates/error/src/error.rs deleted file mode 100644 index 33c4d32d6..000000000 --- a/crates/error/src/error.rs +++ /dev/null @@ -1,586 +0,0 @@ -use std::hash::Hash; -use std::str::FromStr; - -const ERROR_PREFIX: &str = "[RUSTFS error] "; - -pub type Result = core::result::Result; - -#[derive(thiserror::Error, Default, Debug)] -pub enum Error { - #[default] - #[error("[RUSTFS error] Nil")] - Nil, - #[error("I/O error: {0}")] - IoError(std::io::Error), - #[error("[RUSTFS error] Erasure Read quorum not met")] - ErasureReadQuorum, - #[error("[RUSTFS error] Erasure Write quorum not met")] - ErasureWriteQuorum, - - #[error("[RUSTFS error] Disk not found")] - DiskNotFound, - #[error("[RUSTFS error] Faulty disk")] - FaultyDisk, - #[error("[RUSTFS error] Faulty remote disk")] - FaultyRemoteDisk, - #[error("[RUSTFS error] Unsupported disk")] - UnsupportedDisk, - #[error("[RUSTFS error] Unformatted disk")] - UnformattedDisk, - #[error("[RUSTFS error] Corrupted backend")] - CorruptedBackend, - - #[error("[RUSTFS error] Disk access denied")] - DiskAccessDenied, - #[error("[RUSTFS error] Disk ongoing request")] - DiskOngoingReq, - #[error("[RUSTFS error] Disk full")] - DiskFull, - - #[error("[RUSTFS error] Volume not found")] - VolumeNotFound, - #[error("[RUSTFS error] Volume not empty")] - VolumeNotEmpty, - #[error("[RUSTFS error] Volume access denied")] - VolumeAccessDenied, - - #[error("[RUSTFS error] Volume exists")] - VolumeExists, - - #[error("[RUSTFS error] Disk not a directory")] - DiskNotDir, - - #[error("[RUSTFS error] File not found")] - FileNotFound, - #[error("[RUSTFS error] File corrupt")] - FileCorrupt, - #[error("[RUSTFS error] File access denied")] - FileAccessDenied, - #[error("[RUSTFS error] Too many open files")] - TooManyOpenFiles, - #[error("[RUSTFS error] Is not a regular file")] - IsNotRegular, - - #[error("[RUSTFS error] File version not found")] - FileVersionNotFound, - - #[error("[RUSTFS error] Less data than expected")] - LessData, - #[error("[RUSTFS error] Short write")] - ShortWrite, - - #[error("[RUSTFS error] Done for now")] - DoneForNow, - - #[error("[RUSTFS error] Method not allowed")] - MethodNotAllowed, - - #[error("[RUSTFS error] Inconsistent disk")] - InconsistentDisk, - - #[error("[RUSTFS error] File name too long")] - FileNameTooLong, - - #[error("[RUSTFS error] Scan ignore file contribution")] - ScanIgnoreFileContrib, - #[error("[RUSTFS error] Scan skip file")] - ScanSkipFile, - #[error("[RUSTFS error] Scan heal stop signaled")] - ScanHealStopSignal, - #[error("[RUSTFS error] Scan heal idle timeout")] - ScanHealIdleTimeout, - #[error("[RUSTFS error] Scan retry healing")] - ScanRetryHealing, - - #[error("[RUSTFS error] {0}")] - Other(String), -} - -// Generic From implementation removed to avoid conflicts with std::convert::From for T - -impl FromStr for Error { - type Err = Error; - fn from_str(s: &str) -> core::result::Result { - // Only strip prefix for non-IoError - let s = if s.starts_with("I/O error: ") { - s - } else { - s.strip_prefix(ERROR_PREFIX).unwrap_or(s) - }; - - match s { - "Nil" => Ok(Error::Nil), - "ErasureReadQuorum" => Ok(Error::ErasureReadQuorum), - "ErasureWriteQuorum" => Ok(Error::ErasureWriteQuorum), - "DiskNotFound" | "Disk not found" => Ok(Error::DiskNotFound), - "FaultyDisk" | "Faulty disk" => Ok(Error::FaultyDisk), - "FaultyRemoteDisk" | "Faulty remote disk" => Ok(Error::FaultyRemoteDisk), - "UnformattedDisk" | "Unformatted disk" => Ok(Error::UnformattedDisk), - "DiskAccessDenied" | "Disk access denied" => Ok(Error::DiskAccessDenied), - "DiskOngoingReq" | "Disk ongoing request" => Ok(Error::DiskOngoingReq), - "FileNotFound" | "File not found" => Ok(Error::FileNotFound), - "FileCorrupt" | "File corrupt" => Ok(Error::FileCorrupt), - "FileVersionNotFound" | "File version not found" => Ok(Error::FileVersionNotFound), - "LessData" | "Less data than expected" => Ok(Error::LessData), - "ShortWrite" | "Short write" => Ok(Error::ShortWrite), - "VolumeNotFound" | "Volume not found" => Ok(Error::VolumeNotFound), - "VolumeNotEmpty" | "Volume not empty" => Ok(Error::VolumeNotEmpty), - "VolumeExists" | "Volume exists" => Ok(Error::VolumeExists), - "VolumeAccessDenied" | "Volume access denied" => Ok(Error::VolumeAccessDenied), - "DiskNotDir" | "Disk not a directory" => Ok(Error::DiskNotDir), - "FileAccessDenied" | "File access denied" => Ok(Error::FileAccessDenied), - "TooManyOpenFiles" | "Too many open files" => Ok(Error::TooManyOpenFiles), - "IsNotRegular" | "Is not a regular file" => Ok(Error::IsNotRegular), - "CorruptedBackend" | "Corrupted backend" => Ok(Error::CorruptedBackend), - "UnsupportedDisk" | "Unsupported disk" => Ok(Error::UnsupportedDisk), - "InconsistentDisk" | "Inconsistent disk" => Ok(Error::InconsistentDisk), - "DiskFull" | "Disk full" => Ok(Error::DiskFull), - "FileNameTooLong" | "File name too long" => Ok(Error::FileNameTooLong), - "ScanIgnoreFileContrib" | "Scan ignore file contribution" => Ok(Error::ScanIgnoreFileContrib), - "ScanSkipFile" | "Scan skip file" => Ok(Error::ScanSkipFile), - "ScanHealStopSignal" | "Scan heal stop signaled" => Ok(Error::ScanHealStopSignal), - "ScanHealIdleTimeout" | "Scan heal idle timeout" => Ok(Error::ScanHealIdleTimeout), - "ScanRetryHealing" | "Scan retry healing" => Ok(Error::ScanRetryHealing), - s if s.starts_with("I/O error: ") => { - Ok(Error::IoError(std::io::Error::other(s.strip_prefix("I/O error: ").unwrap_or("")))) - } - "DoneForNow" | "Done for now" => Ok(Error::DoneForNow), - "MethodNotAllowed" | "Method not allowed" => Ok(Error::MethodNotAllowed), - str => Err(Error::IoError(std::io::Error::other(str.to_string()))), - } - } -} - -impl From for std::io::Error { - fn from(err: Error) -> Self { - match err { - Error::IoError(e) => e, - e => std::io::Error::other(e), - } - } -} - -impl From for Error { - fn from(e: std::io::Error) -> Self { - match e.kind() { - // convert Error from string to Error - std::io::ErrorKind::Other => Error::from(e.to_string()), - _ => Error::IoError(e), - } - } -} - -impl From for Error { - fn from(s: String) -> Self { - Error::from_str(&s).unwrap_or(Error::IoError(std::io::Error::other(s))) - } -} - -impl From<&str> for Error { - fn from(s: &str) -> Self { - Error::from_str(s).unwrap_or(Error::IoError(std::io::Error::other(s))) - } -} - -// Common error type conversions for ? operator -impl From for Error { - fn from(e: std::num::ParseIntError) -> Self { - Error::Other(format!("Parse int error: {}", e)) - } -} - -impl From for Error { - fn from(e: std::num::ParseFloatError) -> Self { - Error::Other(format!("Parse float error: {}", e)) - } -} - -impl From for Error { - fn from(e: std::str::Utf8Error) -> Self { - Error::Other(format!("UTF-8 error: {}", e)) - } -} - -impl From for Error { - fn from(e: std::string::FromUtf8Error) -> Self { - Error::Other(format!("UTF-8 conversion error: {}", e)) - } -} - -impl From for Error { - fn from(e: std::fmt::Error) -> Self { - Error::Other(format!("Format error: {}", e)) - } -} - -impl From> for Error { - fn from(e: Box) -> Self { - Error::Other(e.to_string()) - } -} - -impl From for Error { - fn from(e: time::error::ComponentRange) -> Self { - Error::Other(format!("Time component range error: {}", e)) - } -} - -impl From> for Error { - fn from(e: rmp::decode::NumValueReadError) -> Self { - Error::Other(format!("NumValueReadError: {}", e)) - } -} - -impl From for Error { - fn from(e: rmp::encode::ValueWriteError) -> Self { - Error::Other(format!("ValueWriteError: {}", e)) - } -} - -impl From for Error { - fn from(e: rmp::decode::ValueReadError) -> Self { - Error::Other(format!("ValueReadError: {}", e)) - } -} - -impl From for Error { - fn from(e: uuid::Error) -> Self { - Error::Other(format!("UUID error: {}", e)) - } -} - -impl From for Error { - fn from(e: rmp_serde::decode::Error) -> Self { - Error::Other(format!("rmp_serde::decode::Error: {}", e)) - } -} - -impl From for Error { - fn from(e: rmp_serde::encode::Error) -> Self { - Error::Other(format!("rmp_serde::encode::Error: {}", e)) - } -} - -impl From for Error { - fn from(e: serde::de::value::Error) -> Self { - Error::Other(format!("serde::de::value::Error: {}", e)) - } -} - -impl From for Error { - fn from(e: serde_json::Error) -> Self { - Error::Other(format!("serde_json::Error: {}", e)) - } -} - -impl From for Error { - fn from(e: std::collections::TryReserveError) -> Self { - Error::Other(format!("TryReserveError: {}", e)) - } -} - -impl From for Error { - fn from(e: tonic::Status) -> Self { - Error::Other(format!("tonic::Status: {}", e.message())) - } -} - -impl From for Error { - fn from(e: protos::proto_gen::node_service::Error) -> Self { - Error::from_str(&e.error_info).unwrap_or(Error::Other(format!("Proto_Error: {}", e.error_info))) - } -} - -impl From for protos::proto_gen::node_service::Error { - fn from(val: Error) -> Self { - protos::proto_gen::node_service::Error { - code: 0, - error_info: val.to_string(), - } - } -} - -impl Hash for Error { - fn hash(&self, state: &mut H) { - match self { - Error::IoError(err) => { - err.kind().hash(state); - err.to_string().hash(state); - } - e => e.to_string().hash(state), - } - } -} - -impl Clone for Error { - fn clone(&self) -> Self { - match self { - Error::IoError(err) => Error::IoError(std::io::Error::new(err.kind(), err.to_string())), - Error::ErasureReadQuorum => Error::ErasureReadQuorum, - Error::ErasureWriteQuorum => Error::ErasureWriteQuorum, - Error::DiskNotFound => Error::DiskNotFound, - Error::FaultyDisk => Error::FaultyDisk, - Error::FaultyRemoteDisk => Error::FaultyRemoteDisk, - Error::UnformattedDisk => Error::UnformattedDisk, - Error::DiskAccessDenied => Error::DiskAccessDenied, - Error::DiskOngoingReq => Error::DiskOngoingReq, - Error::FileNotFound => Error::FileNotFound, - Error::FileCorrupt => Error::FileCorrupt, - Error::FileVersionNotFound => Error::FileVersionNotFound, - Error::LessData => Error::LessData, - Error::ShortWrite => Error::ShortWrite, - Error::VolumeNotFound => Error::VolumeNotFound, - Error::VolumeNotEmpty => Error::VolumeNotEmpty, - Error::VolumeAccessDenied => Error::VolumeAccessDenied, - Error::VolumeExists => Error::VolumeExists, - Error::DiskNotDir => Error::DiskNotDir, - Error::FileAccessDenied => Error::FileAccessDenied, - Error::TooManyOpenFiles => Error::TooManyOpenFiles, - Error::IsNotRegular => Error::IsNotRegular, - Error::CorruptedBackend => Error::CorruptedBackend, - Error::UnsupportedDisk => Error::UnsupportedDisk, - Error::DiskFull => Error::DiskFull, - Error::Nil => Error::Nil, - Error::DoneForNow => Error::DoneForNow, - Error::MethodNotAllowed => Error::MethodNotAllowed, - Error::InconsistentDisk => Error::InconsistentDisk, - Error::FileNameTooLong => Error::FileNameTooLong, - Error::ScanIgnoreFileContrib => Error::ScanIgnoreFileContrib, - Error::ScanSkipFile => Error::ScanSkipFile, - Error::ScanHealStopSignal => Error::ScanHealStopSignal, - Error::ScanHealIdleTimeout => Error::ScanHealIdleTimeout, - Error::ScanRetryHealing => Error::ScanRetryHealing, - Error::Other(msg) => Error::Other(msg.clone()), - } - } -} - -impl PartialEq for Error { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Error::IoError(e1), Error::IoError(e2)) => e1.kind() == e2.kind() && e1.to_string() == e2.to_string(), - (Error::ErasureReadQuorum, Error::ErasureReadQuorum) => true, - (Error::ErasureWriteQuorum, Error::ErasureWriteQuorum) => true, - (Error::DiskNotFound, Error::DiskNotFound) => true, - (Error::FaultyDisk, Error::FaultyDisk) => true, - (Error::FaultyRemoteDisk, Error::FaultyRemoteDisk) => true, - (Error::UnformattedDisk, Error::UnformattedDisk) => true, - (Error::DiskAccessDenied, Error::DiskAccessDenied) => true, - (Error::DiskOngoingReq, Error::DiskOngoingReq) => true, - (Error::FileNotFound, Error::FileNotFound) => true, - (Error::FileCorrupt, Error::FileCorrupt) => true, - (Error::FileVersionNotFound, Error::FileVersionNotFound) => true, - (Error::LessData, Error::LessData) => true, - (Error::ShortWrite, Error::ShortWrite) => true, - (Error::VolumeNotFound, Error::VolumeNotFound) => true, - (Error::VolumeNotEmpty, Error::VolumeNotEmpty) => true, - (Error::VolumeAccessDenied, Error::VolumeAccessDenied) => true, - (Error::VolumeExists, Error::VolumeExists) => true, - (Error::DiskNotDir, Error::DiskNotDir) => true, - (Error::FileAccessDenied, Error::FileAccessDenied) => true, - (Error::TooManyOpenFiles, Error::TooManyOpenFiles) => true, - (Error::IsNotRegular, Error::IsNotRegular) => true, - (Error::CorruptedBackend, Error::CorruptedBackend) => true, - (Error::UnsupportedDisk, Error::UnsupportedDisk) => true, - (Error::DiskFull, Error::DiskFull) => true, - (Error::Nil, Error::Nil) => true, - (Error::DoneForNow, Error::DoneForNow) => true, - (Error::MethodNotAllowed, Error::MethodNotAllowed) => true, - (Error::InconsistentDisk, Error::InconsistentDisk) => true, - (Error::FileNameTooLong, Error::FileNameTooLong) => true, - (Error::ScanIgnoreFileContrib, Error::ScanIgnoreFileContrib) => true, - (Error::ScanSkipFile, Error::ScanSkipFile) => true, - (Error::ScanHealStopSignal, Error::ScanHealStopSignal) => true, - (Error::ScanHealIdleTimeout, Error::ScanHealIdleTimeout) => true, - (Error::ScanRetryHealing, Error::ScanRetryHealing) => true, - (Error::Other(s1), Error::Other(s2)) => s1 == s2, - _ => false, - } - } -} - -impl Eq for Error {} - -impl Error { - /// Create an error from a message string (for backward compatibility) - pub fn msg>(message: S) -> Self { - Error::Other(message.into()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - use std::io; - - #[test] - fn test_display_and_debug() { - let e = Error::DiskNotFound; - assert_eq!(format!("{}", e), format!("{ERROR_PREFIX}Disk not found")); - assert_eq!(format!("{:?}", e), "DiskNotFound"); - let io_err = Error::IoError(io::Error::other("fail")); - assert_eq!(format!("{}", io_err), "I/O error: fail"); - } - - #[test] - fn test_partial_eq_and_eq() { - assert_eq!(Error::DiskNotFound, Error::DiskNotFound); - assert_ne!(Error::DiskNotFound, Error::FaultyDisk); - let e1 = Error::IoError(io::Error::other("fail")); - let e2 = Error::IoError(io::Error::other("fail")); - assert_eq!(e1, e2); - let e3 = Error::IoError(io::Error::new(io::ErrorKind::NotFound, "fail")); - assert_ne!(e1, e3); - } - - #[test] - fn test_clone() { - let e = Error::DiskAccessDenied; - let cloned = e.clone(); - assert_eq!(e, cloned); - let io_err = Error::IoError(io::Error::other("fail")); - let cloned_io = io_err.clone(); - assert_eq!(io_err, cloned_io); - } - - #[test] - fn test_hash() { - let e1 = Error::DiskNotFound; - let e2 = Error::DiskNotFound; - let mut h1 = DefaultHasher::new(); - let mut h2 = DefaultHasher::new(); - e1.hash(&mut h1); - e2.hash(&mut h2); - assert_eq!(h1.finish(), h2.finish()); - let io_err1 = Error::IoError(io::Error::other("fail")); - let io_err2 = Error::IoError(io::Error::other("fail")); - let mut h3 = DefaultHasher::new(); - let mut h4 = DefaultHasher::new(); - io_err1.hash(&mut h3); - io_err2.hash(&mut h4); - assert_eq!(h3.finish(), h4.finish()); - } - - #[test] - fn test_from_error_for_io_error() { - let e = Error::DiskNotFound; - let io_err: io::Error = e.into(); - assert_eq!(io_err.kind(), io::ErrorKind::Other); - assert_eq!(io_err.to_string(), format!("{ERROR_PREFIX}Disk not found")); - - assert_eq!(Error::from(io_err.to_string()), Error::DiskNotFound); - - let orig = io::Error::other("fail"); - let e2 = Error::IoError(orig.kind().into()); - let io_err2: io::Error = e2.into(); - assert_eq!(io_err2.kind(), io::ErrorKind::Other); - } - - #[test] - fn test_from_io_error_for_error() { - let orig = io::Error::other("fail"); - let e: Error = orig.into(); - match e { - Error::IoError(ioe) => { - assert_eq!(ioe.kind(), io::ErrorKind::Other); - assert_eq!(ioe.to_string(), "fail"); - } - _ => panic!("Expected IoError variant"), - } - } - - #[test] - fn test_default() { - let e = Error::default(); - assert_eq!(e, Error::Nil); - } - - #[test] - fn test_from_str() { - use std::str::FromStr; - assert_eq!(Error::from_str("Nil"), Ok(Error::Nil)); - assert_eq!(Error::from_str("DiskNotFound"), Ok(Error::DiskNotFound)); - assert_eq!(Error::from_str("ErasureReadQuorum"), Ok(Error::ErasureReadQuorum)); - assert_eq!(Error::from_str("I/O error: fail"), Ok(Error::IoError(io::Error::other("fail")))); - assert_eq!(Error::from_str(&format!("{ERROR_PREFIX}Disk not found")), Ok(Error::DiskNotFound)); - assert_eq!( - Error::from_str("UnknownError"), - Err(Error::IoError(std::io::Error::other("UnknownError"))) - ); - } - - #[test] - fn test_from_string() { - let e: Error = format!("{ERROR_PREFIX}Disk not found").parse().unwrap(); - assert_eq!(e, Error::DiskNotFound); - let e2: Error = "I/O error: fail".to_string().parse().unwrap(); - assert_eq!(e2, Error::IoError(std::io::Error::other("fail"))); - } - - #[test] - fn test_from_io_error() { - let e = Error::IoError(io::Error::other("fail")); - let io_err: io::Error = e.clone().into(); - assert_eq!(io_err.to_string(), "fail"); - - let e2: Error = io::Error::other("fail").into(); - assert_eq!(e2, Error::IoError(io::Error::other("fail"))); - - let result = Error::from(io::Error::other("fail")); - assert_eq!(result, Error::IoError(io::Error::other("fail"))); - - let io_err2: std::io::Error = Error::CorruptedBackend.into(); - assert_eq!(io_err2.to_string(), "[RUSTFS error] Corrupted backend"); - - assert_eq!(Error::from(io_err2), Error::CorruptedBackend); - - let io_err3: std::io::Error = Error::DiskNotFound.into(); - assert_eq!(io_err3.to_string(), "[RUSTFS error] Disk not found"); - - assert_eq!(Error::from(io_err3), Error::DiskNotFound); - - let io_err4: std::io::Error = Error::DiskAccessDenied.into(); - assert_eq!(io_err4.to_string(), "[RUSTFS error] Disk access denied"); - - assert_eq!(Error::from(io_err4), Error::DiskAccessDenied); - } - - #[test] - fn test_question_mark_operator() { - fn parse_number(s: &str) -> Result { - let num = s.parse::()?; // ParseIntError automatically converts to Error - Ok(num) - } - - fn format_string() -> Result { - use std::fmt::Write; - let mut s = String::new(); - write!(&mut s, "test")?; // fmt::Error automatically converts to Error - Ok(s) - } - - fn utf8_conversion() -> Result { - let bytes = vec![0xFF, 0xFE]; // Invalid UTF-8 - let s = String::from_utf8(bytes)?; // FromUtf8Error automatically converts to Error - Ok(s) - } - - // Test successful case - assert_eq!(parse_number("42").unwrap(), 42); - - // Test error conversion - let err = parse_number("not_a_number").unwrap_err(); - assert!(matches!(err, Error::Other(_))); - assert!(err.to_string().contains("Parse int error")); - - // Test format error conversion - assert_eq!(format_string().unwrap(), "test"); - - // Test UTF-8 error conversion - let err = utf8_conversion().unwrap_err(); - assert!(matches!(err, Error::Other(_))); - assert!(err.to_string().contains("UTF-8 conversion error")); - } -} diff --git a/crates/error/src/ignored.rs b/crates/error/src/ignored.rs deleted file mode 100644 index 6660b2416..000000000 --- a/crates/error/src/ignored.rs +++ /dev/null @@ -1,11 +0,0 @@ -use crate::Error; -pub static OBJECT_OP_IGNORED_ERRS: &[Error] = &[ - Error::DiskNotFound, - Error::FaultyDisk, - Error::FaultyRemoteDisk, - Error::DiskAccessDenied, - Error::DiskOngoingReq, - Error::UnformattedDisk, -]; - -pub static BASE_IGNORED_ERRS: &[Error] = &[Error::DiskNotFound, Error::FaultyDisk, Error::FaultyRemoteDisk]; diff --git a/crates/error/src/lib.rs b/crates/error/src/lib.rs deleted file mode 100644 index e8d7d40fb..000000000 --- a/crates/error/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -mod error; -pub use error::*; - -mod reduce; -pub use reduce::*; - -mod ignored; -pub use ignored::*; - -mod convert; -pub use convert::*; - -mod bitrot; -pub use bitrot::*; diff --git a/crates/error/src/reduce.rs b/crates/error/src/reduce.rs deleted file mode 100644 index e6333a970..000000000 --- a/crates/error/src/reduce.rs +++ /dev/null @@ -1,138 +0,0 @@ -use crate::error::Error; - -pub fn reduce_write_quorum_errs(errors: &[Option], ignored_errs: &[Error], quorun: usize) -> Option { - reduce_quorum_errs(errors, ignored_errs, quorun, Error::ErasureWriteQuorum) -} - -pub fn reduce_read_quorum_errs(errors: &[Option], ignored_errs: &[Error], quorun: usize) -> Option { - reduce_quorum_errs(errors, ignored_errs, quorun, Error::ErasureReadQuorum) -} - -pub fn reduce_quorum_errs(errors: &[Option], ignored_errs: &[Error], quorun: usize, quorun_err: Error) -> Option { - let (max_count, err) = reduce_errs(errors, ignored_errs); - if max_count >= quorun { - err - } else { - Some(quorun_err) - } -} - -pub fn reduce_errs(errors: &[Option], ignored_errs: &[Error]) -> (usize, Option) { - let err_counts = - errors - .iter() - .map(|e| e.as_ref().unwrap_or(&Error::Nil)) - .fold(std::collections::HashMap::new(), |mut acc, e| { - if is_ignored_err(ignored_errs, e) { - return acc; - } - *acc.entry(e.clone()).or_insert(0) += 1; - acc - }); - - let (err, max_count) = err_counts - .into_iter() - .max_by(|(e1, c1), (e2, c2)| { - // Prefer Error::Nil if present in a tie - let count_cmp = c1.cmp(c2); - if count_cmp == std::cmp::Ordering::Equal { - match (e1, e2) { - (Error::Nil, _) => std::cmp::Ordering::Greater, - (_, Error::Nil) => std::cmp::Ordering::Less, - _ => format!("{e1:?}").cmp(&format!("{e2:?}")), - } - } else { - count_cmp - } - }) - .unwrap_or((Error::Nil, 0)); - - (max_count, if err == Error::Nil { None } else { Some(err) }) -} - -pub fn is_ignored_err(ignored_errs: &[Error], err: &Error) -> bool { - ignored_errs.iter().any(|e| e == err) -} - -pub fn count_errs(errors: &[Option], err: Error) -> usize { - errors - .iter() - .map(|e| if e.is_none() { &Error::Nil } else { e.as_ref().unwrap() }) - .filter(|&e| e == &err) - .count() -} - -#[cfg(test)] -mod tests { - use super::*; - - fn err_io(msg: &str) -> Error { - Error::IoError(std::io::Error::other(msg)) - } - - #[test] - fn test_reduce_errs_basic() { - let e1 = err_io("a"); - let e2 = err_io("b"); - let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None]; - let ignored = vec![]; - let (count, err) = reduce_errs(&errors, &ignored); - assert_eq!(count, 2); - assert_eq!(err, Some(e1)); - } - - #[test] - fn test_reduce_errs_ignored() { - let e1 = err_io("a"); - let e2 = err_io("b"); - let errors = vec![Some(e1.clone()), Some(e2.clone()), Some(e1.clone()), Some(e2.clone()), None]; - let ignored = vec![e2.clone()]; - let (count, err) = reduce_errs(&errors, &ignored); - assert_eq!(count, 2); - assert_eq!(err, Some(e1)); - } - - #[test] - fn test_reduce_quorum_errs() { - let e1 = err_io("a"); - let e2 = err_io("b"); - let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None]; - let ignored = vec![]; - // quorum = 2, should return e1 - let res = reduce_quorum_errs(&errors, &ignored, 2, Error::ErasureReadQuorum); - assert_eq!(res, Some(e1)); - // quorum = 3, should return quorum error - let res = reduce_quorum_errs(&errors, &ignored, 3, Error::ErasureReadQuorum); - assert_eq!(res, Some(Error::ErasureReadQuorum)); - } - - #[test] - fn test_count_errs() { - let e1 = err_io("a"); - let e2 = err_io("b"); - let errors = vec![Some(e1.clone()), Some(e2.clone()), Some(e1.clone()), None]; - assert_eq!(count_errs(&errors, e1.clone()), 2); - assert_eq!(count_errs(&errors, e2.clone()), 1); - } - - #[test] - fn test_is_ignored_err() { - let e1 = err_io("a"); - let e2 = err_io("b"); - let ignored = vec![e1.clone()]; - assert!(is_ignored_err(&ignored, &e1)); - assert!(!is_ignored_err(&ignored, &e2)); - } - - #[test] - fn test_reduce_errs_nil_tiebreak() { - // Error::Nil and another error have the same count, should prefer Nil - let e1 = err_io("a"); - let e2 = err_io("b"); - let errors = vec![Some(e1.clone()), Some(e2.clone()), None, Some(e1.clone()), None]; // e1:1, Nil:1 - let ignored = vec![]; - let (count, err) = reduce_errs(&errors, &ignored); - assert_eq!(count, 2); - assert_eq!(err, None); // None means Error::Nil is preferred - } -} diff --git a/ecstore/src/bitrot.rs b/ecstore/src/bitrot.rs index 393bec53d..1874892e5 100644 --- a/ecstore/src/bitrot.rs +++ b/ecstore/src/bitrot.rs @@ -1,435 +1,614 @@ -use crate::disk::error::{Error, Result}; -use crate::{ - disk::{error::DiskError, Disk, DiskAPI}, - erasure::{ReadAt, Writer}, - io::{FileReader, FileWriter}, -}; -use blake2::Blake2b512; -use blake2::Digest as _; -use bytes::Bytes; -use highway::{HighwayHash, HighwayHasher, Key}; -use lazy_static::lazy_static; -use rustfs_utils::HashAlgorithm; -use sha2::{digest::core_api::BlockSizeUser, Digest, Sha256}; -use std::{any::Any, collections::HashMap, io::Cursor, sync::Arc}; -use tokio::io::{AsyncReadExt as _, AsyncWriteExt}; -use tracing::{error, info}; +// use crate::disk::error::{Error, Result}; +// use crate::{ +// disk::{error::DiskError, Disk, DiskAPI}, +// erasure::{ReadAt, Writer}, +// io::{FileReader, FileWriter}, +// }; +// use blake2::Blake2b512; +// use blake2::Digest as _; +// use bytes::Bytes; +// use highway::{HighwayHash, HighwayHasher, Key}; +// use lazy_static::lazy_static; +// use rustfs_utils::HashAlgorithm; +// use sha2::{digest::core_api::BlockSizeUser, Digest, Sha256}; +// use std::{any::Any, collections::HashMap, io::Cursor, sync::Arc}; +// use tokio::io::{AsyncReadExt as _, AsyncWriteExt}; +// use tracing::{error, info}; -lazy_static! { - static ref BITROT_ALGORITHMS: HashMap = { - let mut m = HashMap::new(); - m.insert(BitrotAlgorithm::SHA256, "sha256"); - m.insert(BitrotAlgorithm::BLAKE2b512, "blake2b"); - m.insert(BitrotAlgorithm::HighwayHash256, "highwayhash256"); - m.insert(BitrotAlgorithm::HighwayHash256S, "highwayhash256S"); - m - }; -} - -// const MAGIC_HIGHWAY_HASH256_KEY: &[u8] = &[ -// 0x4b, 0xe7, 0x34, 0xfa, 0x8e, 0x23, 0x8a, 0xcd, 0x26, 0x3e, 0x83, 0xe6, 0xbb, 0x96, 0x85, 0x52, 0x04, 0x0f, 0x93, 0x5d, 0xa3, -// 0x9f, 0x44, 0x14, 0x97, 0xe0, 0x9d, 0x13, 0x22, 0xde, 0x36, 0xa0, -// ]; -const MAGIC_HIGHWAY_HASH256_KEY: &[u64; 4] = &[3, 4, 2, 1]; - -#[derive(Clone, Debug)] -pub enum Hasher { - SHA256(Sha256), - HighwayHash256(HighwayHasher), - BLAKE2b512(Blake2b512), -} - -impl Hasher { - pub fn update(&mut self, data: impl AsRef<[u8]>) { - match self { - Hasher::SHA256(core_wrapper) => { - core_wrapper.update(data); - } - Hasher::HighwayHash256(highway_hasher) => { - highway_hasher.append(data.as_ref()); - } - Hasher::BLAKE2b512(core_wrapper) => { - core_wrapper.update(data); - } - } - } - - pub fn finalize(self) -> Vec { - match self { - Hasher::SHA256(core_wrapper) => core_wrapper.finalize().to_vec(), - Hasher::HighwayHash256(highway_hasher) => highway_hasher - .finalize256() - .iter() - .flat_map(|&n| n.to_le_bytes()) // 使用小端字节序转换 - .collect(), - Hasher::BLAKE2b512(core_wrapper) => core_wrapper.finalize().to_vec(), - } - } - - pub fn size(&self) -> usize { - match self { - Hasher::SHA256(_) => Sha256::output_size(), - Hasher::HighwayHash256(_) => 32, - Hasher::BLAKE2b512(_) => Blake2b512::output_size(), - } - } - - pub fn block_size(&self) -> usize { - match self { - Hasher::SHA256(_) => Sha256::block_size(), - Hasher::HighwayHash256(_) => 64, - Hasher::BLAKE2b512(_) => 64, - } - } - - pub fn reset(&mut self) { - match self { - Hasher::SHA256(core_wrapper) => core_wrapper.reset(), - Hasher::HighwayHash256(highway_hasher) => { - let key = Key(*MAGIC_HIGHWAY_HASH256_KEY); - *highway_hasher = HighwayHasher::new(key); - } - Hasher::BLAKE2b512(core_wrapper) => core_wrapper.reset(), - } - } -} - -impl BitrotAlgorithm { - pub fn new_hasher(&self) -> Hasher { - match self { - BitrotAlgorithm::SHA256 => Hasher::SHA256(Sha256::new()), - BitrotAlgorithm::HighwayHash256 | BitrotAlgorithm::HighwayHash256S => { - let key = Key(*MAGIC_HIGHWAY_HASH256_KEY); - Hasher::HighwayHash256(HighwayHasher::new(key)) - } - BitrotAlgorithm::BLAKE2b512 => Hasher::BLAKE2b512(Blake2b512::new()), - } - } - - pub fn available(&self) -> bool { - BITROT_ALGORITHMS.get(self).is_some() - } - - pub fn string(&self) -> String { - BITROT_ALGORITHMS.get(self).map_or("".to_string(), |s| s.to_string()) - } -} - -#[derive(Debug)] -pub struct BitrotVerifier { - _algorithm: BitrotAlgorithm, - _sum: Vec, -} - -impl BitrotVerifier { - pub fn new(algorithm: BitrotAlgorithm, checksum: &[u8]) -> BitrotVerifier { - BitrotVerifier { - _algorithm: algorithm, - _sum: checksum.to_vec(), - } - } -} - -pub fn bitrot_algorithm_from_string(s: &str) -> BitrotAlgorithm { - for (k, v) in BITROT_ALGORITHMS.iter() { - if *v == s { - return k.clone(); - } - } - - BitrotAlgorithm::HighwayHash256S -} - -pub type BitrotWriter = Box; - -// pub async fn new_bitrot_writer( -// disk: DiskStore, -// orig_volume: &str, -// volume: &str, -// file_path: &str, -// length: usize, -// algo: BitrotAlgorithm, -// shard_size: usize, -// ) -> Result { -// if algo == BitrotAlgorithm::HighwayHash256S { -// return Ok(Box::new( -// StreamingBitrotWriter::new(disk, orig_volume, volume, file_path, length, algo, shard_size).await?, -// )); -// } -// Ok(Box::new(WholeBitrotWriter::new(disk, volume, file_path, algo, shard_size))) +// lazy_static! { +// static ref BITROT_ALGORITHMS: HashMap = { +// let mut m = HashMap::new(); +// m.insert(BitrotAlgorithm::SHA256, "sha256"); +// m.insert(BitrotAlgorithm::BLAKE2b512, "blake2b"); +// m.insert(BitrotAlgorithm::HighwayHash256, "highwayhash256"); +// m.insert(BitrotAlgorithm::HighwayHash256S, "highwayhash256S"); +// m +// }; // } -pub type BitrotReader = Box; +// // const MAGIC_HIGHWAY_HASH256_KEY: &[u8] = &[ +// // 0x4b, 0xe7, 0x34, 0xfa, 0x8e, 0x23, 0x8a, 0xcd, 0x26, 0x3e, 0x83, 0xe6, 0xbb, 0x96, 0x85, 0x52, 0x04, 0x0f, 0x93, 0x5d, 0xa3, +// // 0x9f, 0x44, 0x14, 0x97, 0xe0, 0x9d, 0x13, 0x22, 0xde, 0x36, 0xa0, +// // ]; +// const MAGIC_HIGHWAY_HASH256_KEY: &[u64; 4] = &[3, 4, 2, 1]; -// #[allow(clippy::too_many_arguments)] -// pub fn new_bitrot_reader( -// disk: DiskStore, -// data: &[u8], -// bucket: &str, -// file_path: &str, -// till_offset: usize, -// algo: BitrotAlgorithm, -// sum: &[u8], -// shard_size: usize, -// ) -> BitrotReader { -// if algo == BitrotAlgorithm::HighwayHash256S { -// return Box::new(StreamingBitrotReader::new(disk, data, bucket, file_path, algo, till_offset, shard_size)); -// } -// Box::new(WholeBitrotReader::new(disk, bucket, file_path, algo, till_offset, sum)) +// #[derive(Clone, Debug)] +// pub enum Hasher { +// SHA256(Sha256), +// HighwayHash256(HighwayHasher), +// BLAKE2b512(Blake2b512), // } -pub async fn close_bitrot_writers(writers: &mut [Option]) -> Result<()> { - for w in writers.iter_mut().flatten() { - w.close().await?; - } - - Ok(()) -} - -// pub fn bitrot_writer_sum(w: &BitrotWriter) -> Vec { -// if let Some(w) = w.as_any().downcast_ref::() { -// return w.hash.clone().finalize(); +// impl Hasher { +// pub fn update(&mut self, data: impl AsRef<[u8]>) { +// match self { +// Hasher::SHA256(core_wrapper) => { +// core_wrapper.update(data); +// } +// Hasher::HighwayHash256(highway_hasher) => { +// highway_hasher.append(data.as_ref()); +// } +// Hasher::BLAKE2b512(core_wrapper) => { +// core_wrapper.update(data); +// } +// } // } -// Vec::new() -// } +// pub fn finalize(self) -> Vec { +// match self { +// Hasher::SHA256(core_wrapper) => core_wrapper.finalize().to_vec(), +// Hasher::HighwayHash256(highway_hasher) => highway_hasher +// .finalize256() +// .iter() +// .flat_map(|&n| n.to_le_bytes()) // 使用小端字节序转换 +// .collect(), +// Hasher::BLAKE2b512(core_wrapper) => core_wrapper.finalize().to_vec(), +// } +// } -pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: BitrotAlgorithm) -> usize { - if algo != BitrotAlgorithm::HighwayHash256S { - return size; - } - size.div_ceil(shard_size) * algo.new_hasher().size() + size -} +// pub fn size(&self) -> usize { +// match self { +// Hasher::SHA256(_) => Sha256::output_size(), +// Hasher::HighwayHash256(_) => 32, +// Hasher::BLAKE2b512(_) => Blake2b512::output_size(), +// } +// } -pub async fn bitrot_verify( - r: FileReader, - want_size: usize, - part_size: usize, - algo: BitrotAlgorithm, - _want: Vec, - mut shard_size: usize, -) -> Result<()> { - // if algo != BitrotAlgorithm::HighwayHash256S { - // let mut h = algo.new_hasher(); - // h.update(r.get_ref()); - // let hash = h.finalize(); - // if hash != want { - // info!("bitrot_verify except: {:?}, got: {:?}", want, hash); - // return Err(Error::new(DiskError::FileCorrupt)); - // } +// pub fn block_size(&self) -> usize { +// match self { +// Hasher::SHA256(_) => Sha256::block_size(), +// Hasher::HighwayHash256(_) => 64, +// Hasher::BLAKE2b512(_) => 64, +// } +// } - // return Ok(()); - // } - let mut h = algo.new_hasher(); - let mut hash_buf = vec![0; h.size()]; - let mut left = want_size; - - if left != bitrot_shard_file_size(part_size, shard_size, algo.clone()) { - info!( - "bitrot_shard_file_size failed, left: {}, part_size: {}, shard_size: {}, algo: {:?}", - left, part_size, shard_size, algo - ); - return Err(Error::new(DiskError::FileCorrupt)); - } - - let mut r = r; - - while left > 0 { - h.reset(); - let n = r.read_exact(&mut hash_buf).await?; - left -= n; - - if left < shard_size { - shard_size = left; - } - - let mut buf = vec![0; shard_size]; - let read = r.read_exact(&mut buf).await?; - h.update(buf); - left -= read; - let hash = h.clone().finalize(); - if h.clone().finalize() != hash_buf[0..n] { - info!("bitrot_verify except: {:?}, got: {:?}", hash_buf[0..n].to_vec(), hash); - return Err(Error::new(DiskError::FileCorrupt)); - } - } - - Ok(()) -} - -// pub struct WholeBitrotWriter { -// disk: DiskStore, -// volume: String, -// file_path: String, -// _shard_size: usize, -// pub hash: Hasher, -// } - -// impl WholeBitrotWriter { -// pub fn new(disk: DiskStore, volume: &str, file_path: &str, algo: BitrotAlgorithm, shard_size: usize) -> Self { -// WholeBitrotWriter { -// disk, -// volume: volume.to_string(), -// file_path: file_path.to_string(), -// _shard_size: shard_size, -// hash: algo.new_hasher(), +// pub fn reset(&mut self) { +// match self { +// Hasher::SHA256(core_wrapper) => core_wrapper.reset(), +// Hasher::HighwayHash256(highway_hasher) => { +// let key = Key(*MAGIC_HIGHWAY_HASH256_KEY); +// *highway_hasher = HighwayHasher::new(key); +// } +// Hasher::BLAKE2b512(core_wrapper) => core_wrapper.reset(), // } // } // } -// #[async_trait::async_trait] -// impl Writer for WholeBitrotWriter { -// fn as_any(&self) -> &dyn Any { -// self +// impl BitrotAlgorithm { +// pub fn new_hasher(&self) -> Hasher { +// match self { +// BitrotAlgorithm::SHA256 => Hasher::SHA256(Sha256::new()), +// BitrotAlgorithm::HighwayHash256 | BitrotAlgorithm::HighwayHash256S => { +// let key = Key(*MAGIC_HIGHWAY_HASH256_KEY); +// Hasher::HighwayHash256(HighwayHasher::new(key)) +// } +// BitrotAlgorithm::BLAKE2b512 => Hasher::BLAKE2b512(Blake2b512::new()), +// } // } -// async fn write(&mut self, buf: &[u8]) -> Result<()> { -// let mut file = self.disk.append_file(&self.volume, &self.file_path).await?; -// let _ = file.write(buf).await?; -// self.hash.update(buf); +// pub fn available(&self) -> bool { +// BITROT_ALGORITHMS.get(self).is_some() +// } -// Ok(()) +// pub fn string(&self) -> String { +// BITROT_ALGORITHMS.get(self).map_or("".to_string(), |s| s.to_string()) // } // } // #[derive(Debug)] -// pub struct WholeBitrotReader { -// disk: DiskStore, -// volume: String, -// file_path: String, -// _verifier: BitrotVerifier, -// till_offset: usize, -// buf: Option>, +// pub struct BitrotVerifier { +// _algorithm: BitrotAlgorithm, +// _sum: Vec, // } -// impl WholeBitrotReader { -// pub fn new(disk: DiskStore, volume: &str, file_path: &str, algo: BitrotAlgorithm, till_offset: usize, sum: &[u8]) -> Self { -// Self { -// disk, -// volume: volume.to_string(), -// file_path: file_path.to_string(), -// _verifier: BitrotVerifier::new(algo, sum), -// till_offset, -// buf: None, +// impl BitrotVerifier { +// pub fn new(algorithm: BitrotAlgorithm, checksum: &[u8]) -> BitrotVerifier { +// BitrotVerifier { +// _algorithm: algorithm, +// _sum: checksum.to_vec(), // } // } // } -// #[async_trait::async_trait] -// impl ReadAt for WholeBitrotReader { -// async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { -// if self.buf.is_none() { -// let buf_len = self.till_offset - offset; -// let mut file = self -// .disk -// .read_file_stream(&self.volume, &self.file_path, offset, length) -// .await?; -// let mut buf = vec![0u8; buf_len]; -// file.read_at(offset, &mut buf).await?; -// self.buf = Some(buf); +// pub fn bitrot_algorithm_from_string(s: &str) -> BitrotAlgorithm { +// for (k, v) in BITROT_ALGORITHMS.iter() { +// if *v == s { +// return k.clone(); // } - -// if let Some(buf) = &mut self.buf { -// if buf.len() < length { -// return Err(Error::new(DiskError::LessData)); -// } - -// return Ok((buf.drain(0..length).collect::>(), length)); -// } - -// Err(Error::new(DiskError::LessData)) // } + +// BitrotAlgorithm::HighwayHash256S // } -// struct StreamingBitrotWriter { +// pub type BitrotWriter = Box; + +// // pub async fn new_bitrot_writer( +// // disk: DiskStore, +// // orig_volume: &str, +// // volume: &str, +// // file_path: &str, +// // length: usize, +// // algo: BitrotAlgorithm, +// // shard_size: usize, +// // ) -> Result { +// // if algo == BitrotAlgorithm::HighwayHash256S { +// // return Ok(Box::new( +// // StreamingBitrotWriter::new(disk, orig_volume, volume, file_path, length, algo, shard_size).await?, +// // )); +// // } +// // Ok(Box::new(WholeBitrotWriter::new(disk, volume, file_path, algo, shard_size))) +// // } + +// pub type BitrotReader = Box; + +// // #[allow(clippy::too_many_arguments)] +// // pub fn new_bitrot_reader( +// // disk: DiskStore, +// // data: &[u8], +// // bucket: &str, +// // file_path: &str, +// // till_offset: usize, +// // algo: BitrotAlgorithm, +// // sum: &[u8], +// // shard_size: usize, +// // ) -> BitrotReader { +// // if algo == BitrotAlgorithm::HighwayHash256S { +// // return Box::new(StreamingBitrotReader::new(disk, data, bucket, file_path, algo, till_offset, shard_size)); +// // } +// // Box::new(WholeBitrotReader::new(disk, bucket, file_path, algo, till_offset, sum)) +// // } + +// pub async fn close_bitrot_writers(writers: &mut [Option]) -> Result<()> { +// for w in writers.iter_mut().flatten() { +// w.close().await?; +// } + +// Ok(()) +// } + +// // pub fn bitrot_writer_sum(w: &BitrotWriter) -> Vec { +// // if let Some(w) = w.as_any().downcast_ref::() { +// // return w.hash.clone().finalize(); +// // } + +// // Vec::new() +// // } + +// pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: BitrotAlgorithm) -> usize { +// if algo != BitrotAlgorithm::HighwayHash256S { +// return size; +// } +// size.div_ceil(shard_size) * algo.new_hasher().size() + size +// } + +// pub async fn bitrot_verify( +// r: FileReader, +// want_size: usize, +// part_size: usize, +// algo: BitrotAlgorithm, +// _want: Vec, +// mut shard_size: usize, +// ) -> Result<()> { +// // if algo != BitrotAlgorithm::HighwayHash256S { +// // let mut h = algo.new_hasher(); +// // h.update(r.get_ref()); +// // let hash = h.finalize(); +// // if hash != want { +// // info!("bitrot_verify except: {:?}, got: {:?}", want, hash); +// // return Err(Error::new(DiskError::FileCorrupt)); +// // } + +// // return Ok(()); +// // } +// let mut h = algo.new_hasher(); +// let mut hash_buf = vec![0; h.size()]; +// let mut left = want_size; + +// if left != bitrot_shard_file_size(part_size, shard_size, algo.clone()) { +// info!( +// "bitrot_shard_file_size failed, left: {}, part_size: {}, shard_size: {}, algo: {:?}", +// left, part_size, shard_size, algo +// ); +// return Err(Error::new(DiskError::FileCorrupt)); +// } + +// let mut r = r; + +// while left > 0 { +// h.reset(); +// let n = r.read_exact(&mut hash_buf).await?; +// left -= n; + +// if left < shard_size { +// shard_size = left; +// } + +// let mut buf = vec![0; shard_size]; +// let read = r.read_exact(&mut buf).await?; +// h.update(buf); +// left -= read; +// let hash = h.clone().finalize(); +// if h.clone().finalize() != hash_buf[0..n] { +// info!("bitrot_verify except: {:?}, got: {:?}", hash_buf[0..n].to_vec(), hash); +// return Err(Error::new(DiskError::FileCorrupt)); +// } +// } + +// Ok(()) +// } + +// // pub struct WholeBitrotWriter { +// // disk: DiskStore, +// // volume: String, +// // file_path: String, +// // _shard_size: usize, +// // pub hash: Hasher, +// // } + +// // impl WholeBitrotWriter { +// // pub fn new(disk: DiskStore, volume: &str, file_path: &str, algo: BitrotAlgorithm, shard_size: usize) -> Self { +// // WholeBitrotWriter { +// // disk, +// // volume: volume.to_string(), +// // file_path: file_path.to_string(), +// // _shard_size: shard_size, +// // hash: algo.new_hasher(), +// // } +// // } +// // } + +// // #[async_trait::async_trait] +// // impl Writer for WholeBitrotWriter { +// // fn as_any(&self) -> &dyn Any { +// // self +// // } + +// // async fn write(&mut self, buf: &[u8]) -> Result<()> { +// // let mut file = self.disk.append_file(&self.volume, &self.file_path).await?; +// // let _ = file.write(buf).await?; +// // self.hash.update(buf); + +// // Ok(()) +// // } +// // } + +// // #[derive(Debug)] +// // pub struct WholeBitrotReader { +// // disk: DiskStore, +// // volume: String, +// // file_path: String, +// // _verifier: BitrotVerifier, +// // till_offset: usize, +// // buf: Option>, +// // } + +// // impl WholeBitrotReader { +// // pub fn new(disk: DiskStore, volume: &str, file_path: &str, algo: BitrotAlgorithm, till_offset: usize, sum: &[u8]) -> Self { +// // Self { +// // disk, +// // volume: volume.to_string(), +// // file_path: file_path.to_string(), +// // _verifier: BitrotVerifier::new(algo, sum), +// // till_offset, +// // buf: None, +// // } +// // } +// // } + +// // #[async_trait::async_trait] +// // impl ReadAt for WholeBitrotReader { +// // async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { +// // if self.buf.is_none() { +// // let buf_len = self.till_offset - offset; +// // let mut file = self +// // .disk +// // .read_file_stream(&self.volume, &self.file_path, offset, length) +// // .await?; +// // let mut buf = vec![0u8; buf_len]; +// // file.read_at(offset, &mut buf).await?; +// // self.buf = Some(buf); +// // } + +// // if let Some(buf) = &mut self.buf { +// // if buf.len() < length { +// // return Err(Error::new(DiskError::LessData)); +// // } + +// // return Ok((buf.drain(0..length).collect::>(), length)); +// // } + +// // Err(Error::new(DiskError::LessData)) +// // } +// // } + +// // struct StreamingBitrotWriter { +// // hasher: Hasher, +// // tx: Sender>>, +// // task: Option>, +// // } + +// // impl StreamingBitrotWriter { +// // pub async fn new( +// // disk: DiskStore, +// // orig_volume: &str, +// // volume: &str, +// // file_path: &str, +// // length: usize, +// // algo: BitrotAlgorithm, +// // shard_size: usize, +// // ) -> Result { +// // let hasher = algo.new_hasher(); +// // let (tx, mut rx) = mpsc::channel::>>(10); + +// // let total_file_size = length.div_ceil(shard_size) * hasher.size() + length; +// // let mut writer = disk.create_file(orig_volume, volume, file_path, total_file_size).await?; + +// // let task = spawn(async move { +// // loop { +// // if let Some(Some(buf)) = rx.recv().await { +// // writer.write(&buf).await.unwrap(); +// // continue; +// // } + +// // break; +// // } +// // }); + +// // Ok(StreamingBitrotWriter { +// // hasher, +// // tx, +// // task: Some(task), +// // }) +// // } +// // } + +// // #[async_trait::async_trait] +// // impl Writer for StreamingBitrotWriter { +// // fn as_any(&self) -> &dyn Any { +// // self +// // } + +// // async fn write(&mut self, buf: &[u8]) -> Result<()> { +// // if buf.is_empty() { +// // return Ok(()); +// // } +// // self.hasher.reset(); +// // self.hasher.update(buf); +// // let hash_bytes = self.hasher.clone().finalize(); +// // let _ = self.tx.send(Some(hash_bytes)).await?; +// // let _ = self.tx.send(Some(buf.to_vec())).await?; + +// // Ok(()) +// // } + +// // async fn close(&mut self) -> Result<()> { +// // let _ = self.tx.send(None).await?; +// // if let Some(task) = self.task.take() { +// // let _ = task.await; // 等待任务完成 +// // } +// // Ok(()) +// // } +// // } + +// // #[derive(Debug)] +// // struct StreamingBitrotReader { +// // disk: DiskStore, +// // _data: Vec, +// // volume: String, +// // file_path: String, +// // till_offset: usize, +// // curr_offset: usize, +// // hasher: Hasher, +// // shard_size: usize, +// // buf: Vec, +// // hash_bytes: Vec, +// // } + +// // impl StreamingBitrotReader { +// // pub fn new( +// // disk: DiskStore, +// // data: &[u8], +// // volume: &str, +// // file_path: &str, +// // algo: BitrotAlgorithm, +// // till_offset: usize, +// // shard_size: usize, +// // ) -> Self { +// // let hasher = algo.new_hasher(); +// // Self { +// // disk, +// // _data: data.to_vec(), +// // volume: volume.to_string(), +// // file_path: file_path.to_string(), +// // till_offset: till_offset.div_ceil(shard_size) * hasher.size() + till_offset, +// // curr_offset: 0, +// // hash_bytes: Vec::with_capacity(hasher.size()), +// // hasher, +// // shard_size, +// // buf: Vec::new(), +// // } +// // } +// // } + +// // #[async_trait::async_trait] +// // impl ReadAt for StreamingBitrotReader { +// // async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { +// // if offset % self.shard_size != 0 { +// // return Err(Error::new(DiskError::Unexpected)); +// // } +// // if self.buf.is_empty() { +// // self.curr_offset = offset; +// // let stream_offset = (offset / self.shard_size) * self.hasher.size() + offset; +// // let buf_len = self.till_offset - stream_offset; +// // let mut file = self.disk.read_file(&self.volume, &self.file_path).await?; +// // let mut buf = vec![0u8; buf_len]; +// // file.read_at(stream_offset, &mut buf).await?; +// // self.buf = buf; +// // } +// // if offset != self.curr_offset { +// // return Err(Error::new(DiskError::Unexpected)); +// // } + +// // self.hash_bytes = self.buf.drain(0..self.hash_bytes.capacity()).collect(); +// // let buf = self.buf.drain(0..length).collect::>(); +// // self.hasher.reset(); +// // self.hasher.update(&buf); +// // let actual = self.hasher.clone().finalize(); +// // if actual != self.hash_bytes { +// // return Err(Error::new(DiskError::FileCorrupt)); +// // } + +// // let readed_len = buf.len(); +// // self.curr_offset += readed_len; + +// // Ok((buf, readed_len)) +// // } +// // } + +// pub struct BitrotFileWriter { +// inner: Option, // hasher: Hasher, -// tx: Sender>>, -// task: Option>, +// _shard_size: usize, +// inline: bool, +// inline_data: Vec, // } -// impl StreamingBitrotWriter { +// impl BitrotFileWriter { // pub async fn new( -// disk: DiskStore, -// orig_volume: &str, +// disk: Arc, // volume: &str, -// file_path: &str, -// length: usize, +// path: &str, +// inline: bool, // algo: BitrotAlgorithm, -// shard_size: usize, +// _shard_size: usize, // ) -> Result { +// let inner = if !inline { +// Some(disk.create_file("", volume, path, 0).await?) +// } else { +// None +// }; + // let hasher = algo.new_hasher(); -// let (tx, mut rx) = mpsc::channel::>>(10); -// let total_file_size = length.div_ceil(shard_size) * hasher.size() + length; -// let mut writer = disk.create_file(orig_volume, volume, file_path, total_file_size).await?; - -// let task = spawn(async move { -// loop { -// if let Some(Some(buf)) = rx.recv().await { -// writer.write(&buf).await.unwrap(); -// continue; -// } - -// break; -// } -// }); - -// Ok(StreamingBitrotWriter { +// Ok(Self { +// inner, +// inline, +// inline_data: Vec::new(), // hasher, -// tx, -// task: Some(task), +// _shard_size, // }) // } + +// // pub fn writer(&self) -> &FileWriter { +// // &self.inner +// // } + +// pub fn inline_data(&self) -> &[u8] { +// &self.inline_data +// } // } // #[async_trait::async_trait] -// impl Writer for StreamingBitrotWriter { +// impl Writer for BitrotFileWriter { // fn as_any(&self) -> &dyn Any { // self // } -// async fn write(&mut self, buf: &[u8]) -> Result<()> { +// #[tracing::instrument(level = "info", skip_all)] +// async fn write(&mut self, buf: Bytes) -> Result<()> { // if buf.is_empty() { // return Ok(()); // } -// self.hasher.reset(); -// self.hasher.update(buf); -// let hash_bytes = self.hasher.clone().finalize(); -// let _ = self.tx.send(Some(hash_bytes)).await?; -// let _ = self.tx.send(Some(buf.to_vec())).await?; +// let mut hasher = self.hasher.clone(); +// let h_buf = buf.clone(); +// let hash_bytes = tokio::spawn(async move { +// hasher.reset(); +// hasher.update(h_buf); +// hasher.finalize() +// }) +// .await?; + +// if let Some(f) = self.inner.as_mut() { +// f.write_all(&hash_bytes).await?; +// f.write_all(&buf).await?; +// } else { +// self.inline_data.extend_from_slice(&hash_bytes); +// self.inline_data.extend_from_slice(&buf); +// } // Ok(()) // } - // async fn close(&mut self) -> Result<()> { -// let _ = self.tx.send(None).await?; -// if let Some(task) = self.task.take() { -// let _ = task.await; // 等待任务完成 +// if self.inline { +// return Ok(()); // } + +// if let Some(f) = self.inner.as_mut() { +// f.shutdown().await?; +// } + // Ok(()) // } // } -// #[derive(Debug)] -// struct StreamingBitrotReader { -// disk: DiskStore, -// _data: Vec, +// pub async fn new_bitrot_filewriter( +// disk: Arc, +// volume: &str, +// path: &str, +// inline: bool, +// algo: HashAlgorithm, +// shard_size: usize, +// ) -> Result { +// let w = BitrotFileWriter::new(disk, volume, path, inline, algo, shard_size).await?; + +// Ok(Box::new(w)) +// } + +// struct BitrotFileReader { +// disk: Arc, +// data: Option>, // volume: String, // file_path: String, +// reader: Option, // till_offset: usize, // curr_offset: usize, // hasher: Hasher, // shard_size: usize, -// buf: Vec, +// // buf: Vec, // hash_bytes: Vec, +// read_buf: Vec, // } -// impl StreamingBitrotReader { +// fn ceil(a: usize, b: usize) -> usize { +// a.div_ceil(b) +// } + +// impl BitrotFileReader { // pub fn new( -// disk: DiskStore, -// data: &[u8], -// volume: &str, -// file_path: &str, +// disk: Arc, +// data: Option>, +// volume: String, +// file_path: String, // algo: BitrotAlgorithm, // till_offset: usize, // shard_size: usize, @@ -437,405 +616,226 @@ pub async fn bitrot_verify( // let hasher = algo.new_hasher(); // Self { // disk, -// _data: data.to_vec(), -// volume: volume.to_string(), -// file_path: file_path.to_string(), -// till_offset: till_offset.div_ceil(shard_size) * hasher.size() + till_offset, +// data, +// volume, +// file_path, +// till_offset: ceil(till_offset, shard_size) * hasher.size() + till_offset, // curr_offset: 0, -// hash_bytes: Vec::with_capacity(hasher.size()), +// hash_bytes: vec![0u8; hasher.size()], // hasher, // shard_size, -// buf: Vec::new(), +// // buf: Vec::new(), +// read_buf: Vec::new(), +// reader: None, // } // } // } // #[async_trait::async_trait] -// impl ReadAt for StreamingBitrotReader { +// impl ReadAt for BitrotFileReader { +// // 读取数据 // async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { // if offset % self.shard_size != 0 { -// return Err(Error::new(DiskError::Unexpected)); -// } -// if self.buf.is_empty() { -// self.curr_offset = offset; -// let stream_offset = (offset / self.shard_size) * self.hasher.size() + offset; -// let buf_len = self.till_offset - stream_offset; -// let mut file = self.disk.read_file(&self.volume, &self.file_path).await?; -// let mut buf = vec![0u8; buf_len]; -// file.read_at(stream_offset, &mut buf).await?; -// self.buf = buf; -// } -// if offset != self.curr_offset { +// error!( +// "BitrotFileReader read_at offset % self.shard_size != 0 , {} % {} = {}", +// offset, +// self.shard_size, +// offset % self.shard_size +// ); // return Err(Error::new(DiskError::Unexpected)); // } -// self.hash_bytes = self.buf.drain(0..self.hash_bytes.capacity()).collect(); -// let buf = self.buf.drain(0..length).collect::>(); +// if self.reader.is_none() { +// self.curr_offset = offset; +// let stream_offset = (offset / self.shard_size) * self.hasher.size() + offset; + +// if let Some(data) = self.data.clone() { +// self.reader = Some(Box::new(Cursor::new(data))); +// } else { +// self.reader = Some( +// self.disk +// .read_file_stream(&self.volume, &self.file_path, stream_offset, self.till_offset - stream_offset) +// .await?, +// ); +// } +// } + +// if offset != self.curr_offset { +// error!( +// "BitrotFileReader read_at {}/{} offset != self.curr_offset, {} != {}", +// &self.volume, &self.file_path, offset, self.curr_offset +// ); +// return Err(Error::new(DiskError::Unexpected)); +// } + +// let reader = self.reader.as_mut().unwrap(); +// // let mut hash_buf = self.hash_bytes; + +// self.hash_bytes.clear(); +// self.hash_bytes.resize(self.hasher.size(), 0u8); + +// reader.read_exact(&mut self.hash_bytes).await?; + +// self.read_buf.clear(); +// self.read_buf.resize(length, 0u8); + +// reader.read_exact(&mut self.read_buf).await?; + // self.hasher.reset(); -// self.hasher.update(&buf); +// self.hasher.update(&self.read_buf); // let actual = self.hasher.clone().finalize(); // if actual != self.hash_bytes { +// error!( +// "BitrotFileReader read_at actual != self.hash_bytes, {:?} != {:?}", +// actual, self.hash_bytes +// ); // return Err(Error::new(DiskError::FileCorrupt)); // } -// let readed_len = buf.len(); +// let readed_len = self.read_buf.len(); // self.curr_offset += readed_len; -// Ok((buf, readed_len)) +// Ok((self.read_buf.clone(), readed_len)) + +// // let stream_offset = (offset / self.shard_size) * self.hasher.size() + offset; +// // let buf_len = self.hasher.size() + length; + +// // self.read_buf.clear(); +// // self.read_buf.resize(buf_len, 0u8); + +// // self.inner.read_at(stream_offset, &mut self.read_buf).await?; + +// // let hash_bytes = &self.read_buf.as_slice()[0..self.hash_bytes.capacity()]; + +// // self.hash_bytes.clone_from_slice(hash_bytes); +// // let buf = self.read_buf.as_slice()[self.hash_bytes.capacity()..self.hash_bytes.capacity() + length].to_vec(); + +// // self.hasher.reset(); +// // self.hasher.update(&buf); +// // let actual = self.hasher.clone().finalize(); + +// // if actual != self.hash_bytes { +// // return Err(Error::new(DiskError::FileCorrupt)); +// // } + +// // let readed_len = buf.len(); +// // self.curr_offset += readed_len; + +// // Ok((buf, readed_len)) // } // } -pub struct BitrotFileWriter { - inner: Option, - hasher: Hasher, - _shard_size: usize, - inline: bool, - inline_data: Vec, -} +// pub fn new_bitrot_filereader( +// disk: Arc, +// data: Option>, +// volume: String, +// file_path: String, +// till_offset: usize, +// algo: BitrotAlgorithm, +// shard_size: usize, +// ) -> BitrotReader { +// Box::new(BitrotFileReader::new(disk, data, volume, file_path, algo, till_offset, shard_size)) +// } -impl BitrotFileWriter { - pub async fn new( - disk: Arc, - volume: &str, - path: &str, - inline: bool, - algo: BitrotAlgorithm, - _shard_size: usize, - ) -> Result { - let inner = if !inline { - Some(disk.create_file("", volume, path, 0).await?) - } else { - None - }; +// #[cfg(test)] +// mod test { +// use std::collections::HashMap; - let hasher = algo.new_hasher(); +// use crate::{disk::error::DiskError, store_api::BitrotAlgorithm}; +// use common::error::{Error, Result}; +// use hex_simd::decode_to_vec; - Ok(Self { - inner, - inline, - inline_data: Vec::new(), - hasher, - _shard_size, - }) - } +// // use super::{bitrot_writer_sum, new_bitrot_reader}; - // pub fn writer(&self) -> &FileWriter { - // &self.inner - // } +// #[test] +// fn bitrot_self_test() -> Result<()> { +// let mut checksums = HashMap::new(); +// checksums.insert( +// BitrotAlgorithm::SHA256, +// "a7677ff19e0182e4d52e3a3db727804abc82a5818749336369552e54b838b004", +// ); +// checksums.insert(BitrotAlgorithm::BLAKE2b512, "e519b7d84b1c3c917985f544773a35cf265dcab10948be3550320d156bab612124a5ae2ae5a8c73c0eea360f68b0e28136f26e858756dbfe7375a7389f26c669"); +// checksums.insert( +// BitrotAlgorithm::HighwayHash256, +// "c81c2386a1f565e805513d630d4e50ff26d11269b21c221cf50fc6c29d6ff75b", +// ); +// checksums.insert( +// BitrotAlgorithm::HighwayHash256S, +// "c81c2386a1f565e805513d630d4e50ff26d11269b21c221cf50fc6c29d6ff75b", +// ); - pub fn inline_data(&self) -> &[u8] { - &self.inline_data - } -} +// let iter = [ +// BitrotAlgorithm::SHA256, +// BitrotAlgorithm::BLAKE2b512, +// BitrotAlgorithm::HighwayHash256, +// ]; -#[async_trait::async_trait] -impl Writer for BitrotFileWriter { - fn as_any(&self) -> &dyn Any { - self - } +// for algo in iter.iter() { +// if !algo.available() || *algo != BitrotAlgorithm::HighwayHash256 { +// continue; +// } +// let checksum = decode_to_vec(checksums.get(algo).unwrap())?; - #[tracing::instrument(level = "info", skip_all)] - async fn write(&mut self, buf: Bytes) -> Result<()> { - if buf.is_empty() { - return Ok(()); - } - let mut hasher = self.hasher.clone(); - let h_buf = buf.clone(); - let hash_bytes = tokio::spawn(async move { - hasher.reset(); - hasher.update(h_buf); - hasher.finalize() - }) - .await?; +// let mut h = algo.new_hasher(); +// let mut msg = Vec::with_capacity(h.size() * h.block_size()); +// let mut sum = Vec::with_capacity(h.size()); - if let Some(f) = self.inner.as_mut() { - f.write_all(&hash_bytes).await?; - f.write_all(&buf).await?; - } else { - self.inline_data.extend_from_slice(&hash_bytes); - self.inline_data.extend_from_slice(&buf); - } +// for _ in (0..h.size() * h.block_size()).step_by(h.size()) { +// h.update(&msg); +// sum = h.finalize(); +// msg.extend(sum.clone()); +// h = algo.new_hasher(); +// } - Ok(()) - } - async fn close(&mut self) -> Result<()> { - if self.inline { - return Ok(()); - } +// if checksum != sum { +// return Err(Error::new(DiskError::FileCorrupt)); +// } +// } - if let Some(f) = self.inner.as_mut() { - f.shutdown().await?; - } +// Ok(()) +// } - Ok(()) - } -} +// // #[tokio::test] +// // async fn test_all_bitrot_algorithms() -> Result<()> { +// // for algo in BITROT_ALGORITHMS.keys() { +// // test_bitrot_reader_writer_algo(algo.clone()).await?; +// // } -pub async fn new_bitrot_filewriter( - disk: Arc, - volume: &str, - path: &str, - inline: bool, - algo: HashAlgorithm, - shard_size: usize, -) -> Result { - let w = BitrotFileWriter::new(disk, volume, path, inline, algo, shard_size).await?; +// // Ok(()) +// // } - Ok(Box::new(w)) -} +// // async fn test_bitrot_reader_writer_algo(algo: BitrotAlgorithm) -> Result<()> { +// // let temp_dir = TempDir::new().unwrap().path().to_string_lossy().to_string(); +// // fs::create_dir_all(&temp_dir)?; +// // let volume = "testvol"; +// // let file_path = "testfile"; -struct BitrotFileReader { - disk: Arc, - data: Option>, - volume: String, - file_path: String, - reader: Option, - till_offset: usize, - curr_offset: usize, - hasher: Hasher, - shard_size: usize, - // buf: Vec, - hash_bytes: Vec, - read_buf: Vec, -} +// // let ep = Endpoint::try_from(temp_dir.as_str())?; +// // let opt = DiskOption::default(); +// // let disk = new_disk(&ep, &opt).await?; +// // disk.make_volume(volume).await?; +// // let mut writer = new_bitrot_writer(disk.clone(), "", volume, file_path, 35, algo.clone(), 10).await?; -fn ceil(a: usize, b: usize) -> usize { - a.div_ceil(b) -} +// // writer.write(b"aaaaaaaaaa").await?; +// // writer.write(b"aaaaaaaaaa").await?; +// // writer.write(b"aaaaaaaaaa").await?; +// // writer.write(b"aaaaa").await?; -impl BitrotFileReader { - pub fn new( - disk: Arc, - data: Option>, - volume: String, - file_path: String, - algo: BitrotAlgorithm, - till_offset: usize, - shard_size: usize, - ) -> Self { - let hasher = algo.new_hasher(); - Self { - disk, - data, - volume, - file_path, - till_offset: ceil(till_offset, shard_size) * hasher.size() + till_offset, - curr_offset: 0, - hash_bytes: vec![0u8; hasher.size()], - hasher, - shard_size, - // buf: Vec::new(), - read_buf: Vec::new(), - reader: None, - } - } -} +// // let sum = bitrot_writer_sum(&writer); +// // writer.close().await?; -#[async_trait::async_trait] -impl ReadAt for BitrotFileReader { - // 读取数据 - async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { - if offset % self.shard_size != 0 { - error!( - "BitrotFileReader read_at offset % self.shard_size != 0 , {} % {} = {}", - offset, - self.shard_size, - offset % self.shard_size - ); - return Err(Error::new(DiskError::Unexpected)); - } +// // let mut reader = new_bitrot_reader(disk, b"", volume, file_path, 35, algo, &sum, 10); +// // let read_len = 10; +// // let mut result: Vec; +// // (result, _) = reader.read_at(0, read_len).await?; +// // assert_eq!(result, b"aaaaaaaaaa"); +// // (result, _) = reader.read_at(10, read_len).await?; +// // assert_eq!(result, b"aaaaaaaaaa"); +// // (result, _) = reader.read_at(20, read_len).await?; +// // assert_eq!(result, b"aaaaaaaaaa"); +// // (result, _) = reader.read_at(30, read_len / 2).await?; +// // assert_eq!(result, b"aaaaa"); - if self.reader.is_none() { - self.curr_offset = offset; - let stream_offset = (offset / self.shard_size) * self.hasher.size() + offset; - - if let Some(data) = self.data.clone() { - self.reader = Some(Box::new(Cursor::new(data))); - } else { - self.reader = Some( - self.disk - .read_file_stream(&self.volume, &self.file_path, stream_offset, self.till_offset - stream_offset) - .await?, - ); - } - } - - if offset != self.curr_offset { - error!( - "BitrotFileReader read_at {}/{} offset != self.curr_offset, {} != {}", - &self.volume, &self.file_path, offset, self.curr_offset - ); - return Err(Error::new(DiskError::Unexpected)); - } - - let reader = self.reader.as_mut().unwrap(); - // let mut hash_buf = self.hash_bytes; - - self.hash_bytes.clear(); - self.hash_bytes.resize(self.hasher.size(), 0u8); - - reader.read_exact(&mut self.hash_bytes).await?; - - self.read_buf.clear(); - self.read_buf.resize(length, 0u8); - - reader.read_exact(&mut self.read_buf).await?; - - self.hasher.reset(); - self.hasher.update(&self.read_buf); - let actual = self.hasher.clone().finalize(); - if actual != self.hash_bytes { - error!( - "BitrotFileReader read_at actual != self.hash_bytes, {:?} != {:?}", - actual, self.hash_bytes - ); - return Err(Error::new(DiskError::FileCorrupt)); - } - - let readed_len = self.read_buf.len(); - self.curr_offset += readed_len; - - Ok((self.read_buf.clone(), readed_len)) - - // let stream_offset = (offset / self.shard_size) * self.hasher.size() + offset; - // let buf_len = self.hasher.size() + length; - - // self.read_buf.clear(); - // self.read_buf.resize(buf_len, 0u8); - - // self.inner.read_at(stream_offset, &mut self.read_buf).await?; - - // let hash_bytes = &self.read_buf.as_slice()[0..self.hash_bytes.capacity()]; - - // self.hash_bytes.clone_from_slice(hash_bytes); - // let buf = self.read_buf.as_slice()[self.hash_bytes.capacity()..self.hash_bytes.capacity() + length].to_vec(); - - // self.hasher.reset(); - // self.hasher.update(&buf); - // let actual = self.hasher.clone().finalize(); - - // if actual != self.hash_bytes { - // return Err(Error::new(DiskError::FileCorrupt)); - // } - - // let readed_len = buf.len(); - // self.curr_offset += readed_len; - - // Ok((buf, readed_len)) - } -} - -pub fn new_bitrot_filereader( - disk: Arc, - data: Option>, - volume: String, - file_path: String, - till_offset: usize, - algo: BitrotAlgorithm, - shard_size: usize, -) -> BitrotReader { - Box::new(BitrotFileReader::new(disk, data, volume, file_path, algo, till_offset, shard_size)) -} - -#[cfg(test)] -mod test { - use std::collections::HashMap; - - use crate::{disk::error::DiskError, store_api::BitrotAlgorithm}; - use common::error::{Error, Result}; - use hex_simd::decode_to_vec; - - // use super::{bitrot_writer_sum, new_bitrot_reader}; - - #[test] - fn bitrot_self_test() -> Result<()> { - let mut checksums = HashMap::new(); - checksums.insert( - BitrotAlgorithm::SHA256, - "a7677ff19e0182e4d52e3a3db727804abc82a5818749336369552e54b838b004", - ); - checksums.insert(BitrotAlgorithm::BLAKE2b512, "e519b7d84b1c3c917985f544773a35cf265dcab10948be3550320d156bab612124a5ae2ae5a8c73c0eea360f68b0e28136f26e858756dbfe7375a7389f26c669"); - checksums.insert( - BitrotAlgorithm::HighwayHash256, - "c81c2386a1f565e805513d630d4e50ff26d11269b21c221cf50fc6c29d6ff75b", - ); - checksums.insert( - BitrotAlgorithm::HighwayHash256S, - "c81c2386a1f565e805513d630d4e50ff26d11269b21c221cf50fc6c29d6ff75b", - ); - - let iter = [ - BitrotAlgorithm::SHA256, - BitrotAlgorithm::BLAKE2b512, - BitrotAlgorithm::HighwayHash256, - ]; - - for algo in iter.iter() { - if !algo.available() || *algo != BitrotAlgorithm::HighwayHash256 { - continue; - } - let checksum = decode_to_vec(checksums.get(algo).unwrap())?; - - let mut h = algo.new_hasher(); - let mut msg = Vec::with_capacity(h.size() * h.block_size()); - let mut sum = Vec::with_capacity(h.size()); - - for _ in (0..h.size() * h.block_size()).step_by(h.size()) { - h.update(&msg); - sum = h.finalize(); - msg.extend(sum.clone()); - h = algo.new_hasher(); - } - - if checksum != sum { - return Err(Error::new(DiskError::FileCorrupt)); - } - } - - Ok(()) - } - - // #[tokio::test] - // async fn test_all_bitrot_algorithms() -> Result<()> { - // for algo in BITROT_ALGORITHMS.keys() { - // test_bitrot_reader_writer_algo(algo.clone()).await?; - // } - - // Ok(()) - // } - - // async fn test_bitrot_reader_writer_algo(algo: BitrotAlgorithm) -> Result<()> { - // let temp_dir = TempDir::new().unwrap().path().to_string_lossy().to_string(); - // fs::create_dir_all(&temp_dir)?; - // let volume = "testvol"; - // let file_path = "testfile"; - - // let ep = Endpoint::try_from(temp_dir.as_str())?; - // let opt = DiskOption::default(); - // let disk = new_disk(&ep, &opt).await?; - // disk.make_volume(volume).await?; - // let mut writer = new_bitrot_writer(disk.clone(), "", volume, file_path, 35, algo.clone(), 10).await?; - - // writer.write(b"aaaaaaaaaa").await?; - // writer.write(b"aaaaaaaaaa").await?; - // writer.write(b"aaaaaaaaaa").await?; - // writer.write(b"aaaaa").await?; - - // let sum = bitrot_writer_sum(&writer); - // writer.close().await?; - - // let mut reader = new_bitrot_reader(disk, b"", volume, file_path, 35, algo, &sum, 10); - // let read_len = 10; - // let mut result: Vec; - // (result, _) = reader.read_at(0, read_len).await?; - // assert_eq!(result, b"aaaaaaaaaa"); - // (result, _) = reader.read_at(10, read_len).await?; - // assert_eq!(result, b"aaaaaaaaaa"); - // (result, _) = reader.read_at(20, read_len).await?; - // assert_eq!(result, b"aaaaaaaaaa"); - // (result, _) = reader.read_at(30, read_len / 2).await?; - // assert_eq!(result, b"aaaaa"); - - // Ok(()) - // } -} +// // Ok(()) +// // } +// } diff --git a/ecstore/src/bucket/error.rs b/ecstore/src/bucket/error.rs index c65d7d416..5a5aae381 100644 --- a/ecstore/src/bucket/error.rs +++ b/ecstore/src/bucket/error.rs @@ -34,13 +34,17 @@ impl BucketMetadataError { impl From for Error { fn from(e: BucketMetadataError) -> Self { - Error::other(e) + match e { + BucketMetadataError::BucketPolicyNotFound => Error::BucketPolicyNotFound, + _ => Error::other(e), + } } } impl From for BucketMetadataError { fn from(e: Error) -> Self { match e { + Error::BucketPolicyNotFound => BucketMetadataError::BucketPolicyNotFound, Error::Io(e) => e.into(), _ => BucketMetadataError::other(e), } diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 45451d6ac..75751ec1f 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -1,29 +1,29 @@ use super::error::{Error, Result}; use super::os::{is_root_disk, rename_all}; -use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ - os, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, Info, - ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, - BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE_BACKUP, + BUCKET_META_PREFIX, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, + FileInfoVersions, Info, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, + STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, os, }; +use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use crate::bucket::metadata_sys::{self}; use crate::bucket::versioning::VersioningApi; use crate::bucket::versioning_sys::BucketVersioningSys; +use crate::disk::STORAGE_FORMAT_FILE; use crate::disk::error::FileAccessDeniedWithContext; use crate::disk::error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error}; use crate::disk::fs::{ - access, lstat, lstat_std, remove, remove_all_std, remove_std, rename, O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, + O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename, }; use crate::disk::os::{check_path_length, is_empty_dir}; -use crate::disk::STORAGE_FORMAT_FILE; use crate::disk::{ - conv_part_err_to_int, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, - CHECK_PART_VOLUME_NOT_FOUND, + CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND, + conv_part_err_to_int, }; use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; use crate::heal::data_scanner::{ - lc_has_active_rules, rep_has_active_rules, scan_data_folder, ScannerItem, ShouldSleepFn, SizeSummary, + ScannerItem, ShouldSleepFn, SizeSummary, lc_has_active_rules, rep_has_active_rules, scan_data_folder, }; use crate::heal::data_scanner_metric::{ScannerMetric, ScannerMetrics}; use crate::heal::data_usage_cache::{DataUsageCache, DataUsageEntry}; @@ -35,23 +35,23 @@ use crate::new_object_layer_fn; use crate::store_api::{ObjectInfo, StorageAPI}; use crate::utils::os::get_info; use crate::utils::path::{ - clean, decode_dir_object, encode_dir_object, has_suffix, path_join, path_join_buf, GLOBAL_DIR_SUFFIX, - GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR, + GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR, clean, decode_dir_object, encode_dir_object, has_suffix, + path_join, path_join_buf, }; use common::defer; use path_absolutize::Absolutize; use rustfs_filemeta::{ - get_file_info, read_xl_meta_no_data, Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, Opts, - RawFileInfo, UpdateFn, + Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, Opts, RawFileInfo, UpdateFn, get_file_info, + read_xl_meta_no_data, }; -use rustfs_rio::{bitrot_verify, Reader}; +use rustfs_rio::{Reader, bitrot_verify}; use rustfs_utils::HashAlgorithm; use std::collections::{HashMap, HashSet}; use std::fmt::Debug; use std::io::SeekFrom; -use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; use std::time::{Duration, SystemTime}; use std::{ fs::Metadata, @@ -60,8 +60,8 @@ use std::{ use time::OffsetDateTime; use tokio::fs::{self, File}; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind}; -use tokio::sync::mpsc::Sender; use tokio::sync::RwLock; +use tokio::sync::mpsc::Sender; use tracing::{debug, error, info, warn}; use uuid::Uuid; @@ -1550,7 +1550,7 @@ impl DiskAPI for LocalDisk { Ok(()) } - #[tracing::instrument(level = "debug", skip(self))] + #[tracing::instrument(level = "debug", skip(self, fi))] async fn rename_data( &self, src_volume: &str, diff --git a/ecstore/src/disks_layout.rs b/ecstore/src/disks_layout.rs index 703ae18f8..1fc970e77 100644 --- a/ecstore/src/disks_layout.rs +++ b/ecstore/src/disks_layout.rs @@ -1,8 +1,8 @@ use crate::utils::ellipses::*; -use common::error::{Error, Result}; use serde::Deserialize; use std::collections::HashSet; use std::env; +use std::io::{Error, Result}; use tracing::debug; /// Supported set sizes this is used to find the optimal @@ -89,7 +89,7 @@ pub struct DisksLayout { impl DisksLayout { pub fn from_volumes>(args: &[T]) -> Result { if args.is_empty() { - return Err(Error::from_string("Invalid argument")); + return Err(Error::other("Invalid argument")); } let is_ellipses = args.iter().any(|v| has_ellipses(&[v])); @@ -98,7 +98,7 @@ impl DisksLayout { debug!("{} not set use default:0, {:?}", ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, err); "0".to_string() }); - let set_drive_count: usize = set_drive_count_env.parse()?; + let set_drive_count: usize = set_drive_count_env.parse().map_err(Error::other)?; // None of the args have ellipses use the old style. if !is_ellipses { @@ -116,7 +116,7 @@ impl DisksLayout { let mut layout = Vec::with_capacity(args.len()); for arg in args.iter() { if !has_ellipses(&[arg]) && args.len() > 1 { - return Err(Error::from_string( + return Err(Error::other( "all args must have ellipses for pool expansion (Invalid arguments specified)", )); } @@ -189,7 +189,7 @@ fn get_all_sets>(set_drive_count: usize, is_ellipses: bool, args: for args in set_args.iter() { for arg in args { if unique_args.contains(arg) { - return Err(Error::from_string(format!("Input args {} has duplicate ellipses", arg))); + return Err(Error::other(format!("Input args {} has duplicate ellipses", arg))); } unique_args.insert(arg); } @@ -245,7 +245,7 @@ impl EndpointSet { } } - pub fn from_volumes>(args: &[T], set_drive_count: usize) -> Result { + pub fn from_volumes>(args: &[T], set_drive_count: usize) -> Result { let mut arg_patterns = Vec::with_capacity(args.len()); for arg in args { arg_patterns.push(find_ellipses_patterns(arg.as_ref())?); @@ -377,20 +377,20 @@ fn get_set_indexes>( arg_patterns: &[ArgPattern], ) -> Result>> { if args.is_empty() || total_sizes.is_empty() { - return Err(Error::from_string("Invalid argument")); + return Err(Error::other("Invalid argument")); } for &size in total_sizes { // Check if total_sizes has minimum range upto set_size if size < SET_SIZES[0] || size < set_drive_count { - return Err(Error::from_string(format!("Incorrect number of endpoints provided, size {}", size))); + return Err(Error::other(format!("Incorrect number of endpoints provided, size {}", size))); } } let common_size = get_divisible_size(total_sizes); let mut set_counts = possible_set_counts(common_size); if set_counts.is_empty() { - return Err(Error::from_string(format!( + return Err(Error::other(format!( "Incorrect number of endpoints provided, number of drives {} is not divisible by any supported erasure set sizes {}", common_size, 0 ))); @@ -399,7 +399,7 @@ fn get_set_indexes>( // Returns possible set counts with symmetry. set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns); if set_counts.is_empty() { - return Err(Error::from_string("No symmetric distribution detected with input endpoints provided")); + return Err(Error::other("No symmetric distribution detected with input endpoints provided")); } let set_size = { @@ -407,7 +407,7 @@ fn get_set_indexes>( let has_set_drive_count = set_counts.contains(&set_drive_count); if !has_set_drive_count { - return Err(Error::from_string(format!( + return Err(Error::other(format!( "Invalid set drive count {}. Acceptable values for {:?} number drives are {:?}", set_drive_count, common_size, &set_counts ))); @@ -416,7 +416,7 @@ fn get_set_indexes>( } else { set_counts = possible_set_counts_with_symmetry(&set_counts, arg_patterns); if set_counts.is_empty() { - return Err(Error::from_string(format!( + return Err(Error::other(format!( "No symmetric distribution detected with input endpoints , drives {} cannot be spread symmetrically by any supported erasure set sizes {:?}", common_size, &set_counts ))); @@ -427,7 +427,7 @@ fn get_set_indexes>( }; if !is_valid_set_size(set_size) { - return Err(Error::from_string("Incorrect number of endpoints provided3")); + return Err(Error::other("Incorrect number of endpoints provided3")); } Ok(total_sizes diff --git a/ecstore/src/endpoints.rs b/ecstore/src/endpoints.rs index ea44edf42..37867a4c9 100644 --- a/ecstore/src/endpoints.rs +++ b/ecstore/src/endpoints.rs @@ -6,9 +6,9 @@ use crate::{ global::global_rustfs_port, utils::net::{self, XHost}, }; -use common::error::{Error, Result}; +use std::io::{Error, Result}; use std::{ - collections::{hash_map::Entry, HashMap, HashSet}, + collections::{HashMap, HashSet, hash_map::Entry}, net::IpAddr, }; @@ -76,7 +76,7 @@ impl> TryFrom<&[T]> for Endpoints { for (i, arg) in args.iter().enumerate() { let endpoint = match Endpoint::try_from(arg.as_ref()) { Ok(ep) => ep, - Err(e) => return Err(Error::from_string(format!("'{}': {}", arg.as_ref(), e))), + Err(e) => return Err(Error::other(format!("'{}': {}", arg.as_ref(), e))), }; // All endpoints have to be same type and scheme if applicable. @@ -84,15 +84,15 @@ impl> TryFrom<&[T]> for Endpoints { endpoint_type = Some(endpoint.get_type()); schema = Some(endpoint.url.scheme().to_owned()); } else if Some(endpoint.get_type()) != endpoint_type { - return Err(Error::from_string("mixed style endpoints are not supported")); + return Err(Error::other("mixed style endpoints are not supported")); } else if Some(endpoint.url.scheme()) != schema.as_deref() { - return Err(Error::from_string("mixed scheme is not supported")); + return Err(Error::other("mixed scheme is not supported")); } // Check for duplicate endpoints. let endpoint_str = endpoint.to_string(); if uniq_set.contains(&endpoint_str) { - return Err(Error::from_string("duplicate endpoints found")); + return Err(Error::other("duplicate endpoints found")); } uniq_set.insert(endpoint_str); @@ -156,7 +156,7 @@ impl PoolEndpointList { /// hostnames and discovers those are local or remote. fn create_pool_endpoints(server_addr: &str, disks_layout: &DisksLayout) -> Result { if disks_layout.is_empty_layout() { - return Err(Error::from_string("invalid number of endpoints")); + return Err(Error::other("invalid number of endpoints")); } let server_addr = net::check_local_server_addr(server_addr)?; @@ -167,7 +167,7 @@ impl PoolEndpointList { endpoint.update_is_local(server_addr.port())?; if endpoint.get_type() != EndpointType::Path { - return Err(Error::from_string("use path style endpoint for single node setup")); + return Err(Error::other("use path style endpoint for single node setup")); } endpoint.set_pool_index(0); @@ -201,7 +201,7 @@ impl PoolEndpointList { } if endpoints.as_ref().is_empty() { - return Err(Error::from_string("invalid number of endpoints")); + return Err(Error::other("invalid number of endpoints")); } pool_endpoints.push(endpoints); @@ -227,15 +227,14 @@ impl PoolEndpointList { let host = ep.url.host().unwrap(); let host_ip_set = host_ip_cache.entry(host.clone()).or_insert({ - net::get_host_ip(host.clone()) - .map_err(|e| Error::from_string(format!("host '{}' cannot resolve: {}", host, e)))? + net::get_host_ip(host.clone()).map_err(|e| Error::other(format!("host '{}' cannot resolve: {}", host, e)))? }); let path = ep.get_file_path(); match path_ip_map.entry(path) { Entry::Occupied(mut e) => { if e.get().intersection(host_ip_set).count() > 0 { - return Err(Error::from_string(format!( + return Err(Error::other(format!( "same path '{}' can not be served by different port on same address", path ))); @@ -257,7 +256,7 @@ impl PoolEndpointList { let path = ep.get_file_path(); if local_path_set.contains(path) { - return Err(Error::from_string(format!( + return Err(Error::other(format!( "path '{}' cannot be served by different address on same server", path ))); @@ -285,7 +284,7 @@ impl PoolEndpointList { // If all endpoints have same port number, Just treat it as local erasure setup // using URL style endpoints. if local_port_set.len() == 1 && local_server_host_set.len() > 1 { - return Err(Error::from_string("all local endpoints should not have different hostnames/ips")); + return Err(Error::other("all local endpoints should not have different hostnames/ips")); } } @@ -453,7 +452,7 @@ impl EndpointServerPools { /// both ellipses and without ellipses transparently. pub fn create_server_endpoints(server_addr: &str, disks_layout: &DisksLayout) -> Result<(EndpointServerPools, SetupType)> { if disks_layout.pools.is_empty() { - return Err(Error::from_string("Invalid arguments specified")); + return Err(Error::other("Invalid arguments specified")); } let pool_eps = PoolEndpointList::create_pool_endpoints(server_addr, disks_layout)?; @@ -490,7 +489,7 @@ impl EndpointServerPools { for ep in eps.endpoints.as_ref() { if exits.contains(&ep.to_string()) { - return Err(Error::from_string("duplicate endpoints found")); + return Err(Error::other("duplicate endpoints found")); } } @@ -664,8 +663,8 @@ mod test { None, 6, ), - (vec!["d1", "d2", "d3", "d1"], Some(Error::from_string("duplicate endpoints found")), 7), - (vec!["d1", "d2", "d3", "./d1"], Some(Error::from_string("duplicate endpoints found")), 8), + (vec!["d1", "d2", "d3", "d1"], Some(Error::other("duplicate endpoints found")), 7), + (vec!["d1", "d2", "d3", "./d1"], Some(Error::other("duplicate endpoints found")), 8), ( vec![ "http://localhost/d1", @@ -673,17 +672,17 @@ mod test { "http://localhost/d1", "http://localhost/d4", ], - Some(Error::from_string("duplicate endpoints found")), + Some(Error::other("duplicate endpoints found")), 9, ), ( vec!["ftp://server/d1", "http://server/d2", "http://server/d3", "http://server/d4"], - Some(Error::from_string("'ftp://server/d1': invalid URL endpoint format")), + Some(Error::other("'ftp://server/d1': invalid URL endpoint format")), 10, ), ( vec!["d1", "http://localhost/d2", "d3", "d4"], - Some(Error::from_string("mixed style endpoints are not supported")), + Some(Error::other("mixed style endpoints are not supported")), 11, ), ( @@ -693,7 +692,7 @@ mod test { "http://example.net/d1", "https://example.edut/d1", ], - Some(Error::from_string("mixed scheme is not supported")), + Some(Error::other("mixed scheme is not supported")), 12, ), ( @@ -703,7 +702,7 @@ mod test { "192.168.1.210:9000/tmp/dir2", "192.168.110:9000/tmp/dir3", ], - Some(Error::from_string( + Some(Error::other( "'192.168.1.210:9000/tmp/dir0': invalid URL endpoint format: missing scheme http or https", )), 13, @@ -811,7 +810,7 @@ mod test { TestCase { num: 1, server_addr: "localhost", - expected_err: Some(Error::from_string("address localhost: missing port in address")), + expected_err: Some(Error::other("address localhost: missing port in address")), ..Default::default() }, // Erasure Single Drive @@ -819,7 +818,7 @@ mod test { num: 2, server_addr: "localhost:9000", args: vec!["http://localhost/d1"], - expected_err: Some(Error::from_string("use path style endpoint for single node setup")), + expected_err: Some(Error::other("use path style endpoint for single node setup")), ..Default::default() }, TestCase { @@ -859,7 +858,7 @@ mod test { "https://example.com/d1", "https://example.com/d2", ], - expected_err: Some(Error::from_string("same path '/d1' can not be served by different port on same address")), + expected_err: Some(Error::other("same path '/d1' can not be served by different port on same address")), ..Default::default() }, // Erasure Setup with PathEndpointType @@ -953,7 +952,7 @@ mod test { "http://127.0.0.1/d3", "http://127.0.0.1/d4", ], - expected_err: Some(Error::from_string("all local endpoints should not have different hostnames/ips")), + expected_err: Some(Error::other("all local endpoints should not have different hostnames/ips")), ..Default::default() }, TestCase { @@ -965,9 +964,7 @@ mod test { case7_endpoint1.as_str(), "http://10.0.0.2:9001/export", ], - expected_err: Some(Error::from_string( - "same path '/export' can not be served by different port on same address", - )), + expected_err: Some(Error::other("same path '/export' can not be served by different port on same address")), ..Default::default() }, TestCase { @@ -979,7 +976,7 @@ mod test { "http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", ], - expected_err: Some(Error::from_string("path '/export' cannot be served by different address on same server")), + expected_err: Some(Error::other("path '/export' cannot be served by different address on same server")), ..Default::default() }, // DistErasure type diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index d39f82651..4c61b765a 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -1,6 +1,6 @@ use crate::bitrot::{BitrotReader, BitrotWriter}; use crate::disk::error::{Error, Result}; -use crate::disk::error_reduce::{reduce_write_quorum_errs, OBJECT_OP_IGNORED_ERRS}; +use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_write_quorum_errs}; use crate::io::Etag; use bytes::{Bytes, BytesMut}; use futures::future::join_all; @@ -72,11 +72,7 @@ impl Erasure { if total_size > 0 { let new_len = { let remain = total_size - total; - if remain > self.block_size { - self.block_size - } else { - remain - } + if remain > self.block_size { self.block_size } else { remain } }; if new_len == 0 && total > 0 { diff --git a/ecstore/src/error.rs b/ecstore/src/error.rs index 100bafcee..7af8138c8 100644 --- a/ecstore/src/error.rs +++ b/ecstore/src/error.rs @@ -164,6 +164,9 @@ pub enum StorageError { #[error("first disk wiat")] FirstDiskWait, + #[error("Bucket policy not found")] + BucketPolicyNotFound, + #[error("Io error: {0}")] Io(std::io::Error), } @@ -376,6 +379,7 @@ impl Clone for StorageError { StorageError::FirstDiskWait => StorageError::FirstDiskWait, StorageError::TooManyOpenFiles => StorageError::TooManyOpenFiles, StorageError::NoHealRequired => StorageError::NoHealRequired, + StorageError::BucketPolicyNotFound => StorageError::BucketPolicyNotFound, } } } @@ -438,6 +442,7 @@ impl StorageError { StorageError::ConfigNotFound => 0x35, StorageError::TooManyOpenFiles => 0x36, StorageError::NoHealRequired => 0x37, + StorageError::BucketPolicyNotFound => 0x38, } } @@ -502,6 +507,7 @@ impl StorageError { 0x35 => Some(StorageError::ConfigNotFound), 0x36 => Some(StorageError::TooManyOpenFiles), 0x37 => Some(StorageError::NoHealRequired), + 0x38 => Some(StorageError::BucketPolicyNotFound), _ => None, } } diff --git a/ecstore/src/file_meta.rs b/ecstore/src/file_meta.rs index 947432795..202c8b4fa 100644 --- a/ecstore/src/file_meta.rs +++ b/ecstore/src/file_meta.rs @@ -1,3404 +1,3404 @@ -use crate::disk::FileInfoVersions; -use crate::file_meta_inline::InlineData; -use crate::store_api::RawFileInfo; -use crate::error::StorageError; -use crate::{ - disk::error::DiskError, - store_api::{ErasureInfo, FileInfo, ObjectPartInfo, ERASURE_ALGORITHM}, -}; -use byteorder::ByteOrder; -use common::error::{Error, Result}; -use rmp::Marker; -use serde::{Deserialize, Serialize}; -use std::cmp::Ordering; -use std::fmt::Display; -use std::io::{self, Read, Write}; -use std::{collections::HashMap, io::Cursor}; -use time::OffsetDateTime; -use tokio::io::AsyncRead; -use tracing::{error, warn}; -use uuid::Uuid; -use xxhash_rust::xxh64; - -// XL header specifies the format -pub static XL_FILE_HEADER: [u8; 4] = [b'X', b'L', b'2', b' ']; -// pub static XL_FILE_VERSION_CURRENT: [u8; 4] = [0; 4]; - -// Current version being written. -// static XL_FILE_VERSION: [u8; 4] = [1, 0, 3, 0]; -static XL_FILE_VERSION_MAJOR: u16 = 1; -static XL_FILE_VERSION_MINOR: u16 = 3; -static XL_HEADER_VERSION: u8 = 3; -static XL_META_VERSION: u8 = 2; -static XXHASH_SEED: u64 = 0; - -const XL_FLAG_FREE_VERSION: u8 = 1 << 0; -// const XL_FLAG_USES_DATA_DIR: u8 = 1 << 1; -const _XL_FLAG_INLINE_DATA: u8 = 1 << 2; - -const META_DATA_READ_DEFAULT: usize = 4 << 10; -const MSGP_UINT32_SIZE: usize = 5; - -// type ScanHeaderVersionFn = Box Result<()>>; - -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct FileMeta { - pub versions: Vec, - pub data: InlineData, // TODO: xlMetaInlineData - pub meta_ver: u8, -} - -impl FileMeta { - pub fn new() -> Self { - Self { - meta_ver: XL_META_VERSION, - data: InlineData::new(), - ..Default::default() - } - } - - // isXL2V1Format - #[tracing::instrument(level = "debug", skip_all)] - pub fn is_xl2_v1_format(buf: &[u8]) -> bool { - !matches!(Self::check_xl2_v1(buf), Err(_e)) - } - - #[tracing::instrument(level = "debug", skip_all)] - pub fn load(buf: &[u8]) -> Result { - let mut xl = FileMeta::default(); - xl.unmarshal_msg(buf)?; - - Ok(xl) - } - - // check_xl2_v1 读 xl 文件头，返回后续内容，版本信息 - // checkXL2V1 - #[tracing::instrument(level = "debug", skip_all)] - pub fn check_xl2_v1(buf: &[u8]) -> Result<(&[u8], u16, u16)> { - if buf.len() < 8 { - return Err(Error::msg("xl file header not exists")); - } - - if buf[0..4] != XL_FILE_HEADER { - return Err(Error::msg("xl file header err")); - } - - let major = byteorder::LittleEndian::read_u16(&buf[4..6]); - let minor = byteorder::LittleEndian::read_u16(&buf[6..8]); - if major > XL_FILE_VERSION_MAJOR { - return Err(Error::msg("xl file version err")); - } - - Ok((&buf[8..], major, minor)) - } - - // 固定 u32 - pub fn read_bytes_header(buf: &[u8]) -> Result<(u32, &[u8])> { - if buf.len() < 5 { - return Err(Error::new(io::Error::new( - io::ErrorKind::UnexpectedEof, - format!("Buffer too small: {} bytes, need at least 5", buf.len()), - ))); - } - - let (mut size_buf, _) = buf.split_at(5); - - // 取 meta 数据，buf = crc + data - let bin_len = rmp::decode::read_bin_len(&mut size_buf)?; - - Ok((bin_len, &buf[5..])) - } - - pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let i = buf.len() as u64; - - // check version, buf = buf[8..] - let (buf, _, _) = Self::check_xl2_v1(buf)?; - - let (mut size_buf, buf) = buf.split_at(5); - - // 取 meta 数据，buf = crc + data - let bin_len = rmp::decode::read_bin_len(&mut size_buf)?; - - let (meta, buf) = buf.split_at(bin_len as usize); - - let (mut crc_buf, buf) = buf.split_at(5); - - // crc check - let crc = rmp::decode::read_u32(&mut crc_buf)?; - let meta_crc = xxh64::xxh64(meta, XXHASH_SEED) as u32; - - if crc != meta_crc { - return Err(Error::msg("xl file crc check failed")); - } - - if !buf.is_empty() { - self.data.update(buf); - self.data.validate()?; - } - - // 解析 meta - if !meta.is_empty() { - let (versions_len, _, meta_ver, meta) = Self::decode_xl_headers(meta)?; - - // let (_, meta) = meta.split_at(read_size as usize); - - self.meta_ver = meta_ver; - - self.versions = Vec::with_capacity(versions_len); - - let mut cur: Cursor<&[u8]> = Cursor::new(meta); - for _ in 0..versions_len { - let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; - let start = cur.position() as usize; - let end = start + bin_len; - let header_buf = &meta[start..end]; - - let mut ver = FileMetaShallowVersion::default(); - ver.header.unmarshal_msg(header_buf)?; - - cur.set_position(end as u64); - - let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; - let start = cur.position() as usize; - let end = start + bin_len; - let mut ver_meta_buf = &meta[start..end]; - - ver_meta_buf.read_to_end(&mut ver.meta)?; - - cur.set_position(end as u64); - - self.versions.push(ver); - } - } - - Ok(i) - } - - // decode_xl_headers 解析 meta 头，返回 (versions 数量，xl_header_version, xl_meta_version, 已读数据长度) - #[tracing::instrument(level = "debug", skip_all)] - fn decode_xl_headers(buf: &[u8]) -> Result<(usize, u8, u8, &[u8])> { - let mut cur = Cursor::new(buf); - - let header_ver: u8 = rmp::decode::read_int(&mut cur)?; - - if header_ver > XL_HEADER_VERSION { - return Err(Error::msg("xl header version invalid")); - } - - let meta_ver: u8 = rmp::decode::read_int(&mut cur)?; - if meta_ver > XL_META_VERSION { - return Err(Error::msg("xl meta version invalid")); - } - - let versions_len: usize = rmp::decode::read_int(&mut cur)?; - - Ok((versions_len, header_ver, meta_ver, &buf[cur.position() as usize..])) - } - - fn decode_versions Result<()>>(buf: &[u8], versions: usize, mut fnc: F) -> Result<()> { - let mut cur: Cursor<&[u8]> = Cursor::new(buf); - - for i in 0..versions { - let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; - let start = cur.position() as usize; - let end = start + bin_len; - let header_buf = &buf[start..end]; - - cur.set_position(end as u64); - - let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; - let start = cur.position() as usize; - let end = start + bin_len; - let ver_meta_buf = &buf[start..end]; - - cur.set_position(end as u64); - - if let Err(err) = fnc(i, header_buf, ver_meta_buf) { - if let Some(e) = err.downcast_ref::() { - if e == &StorageError::DoneForNow { - return Ok(()); - } - } - - return Err(err); - } - } - - Ok(()) - } - - pub fn is_latest_delete_marker(buf: &[u8]) -> bool { - let header = Self::decode_xl_headers(buf).ok(); - if let Some((versions, _hdr_v, _meta_v, meta)) = header { - if versions == 0 { - return false; - } - - let mut is_delete_marker = false; - - let _ = Self::decode_versions(meta, versions, |_: usize, hdr: &[u8], _: &[u8]| { - let mut header = FileMetaVersionHeader::default(); - if header.unmarshal_msg(hdr).is_err() { - return Err(Error::new(StorageError::DoneForNow)); - } - - is_delete_marker = header.version_type == VersionType::Delete; - - Err(Error::new(StorageError::DoneForNow)) - }); - - is_delete_marker - } else { - false - } - } - - #[tracing::instrument(level = "debug", skip_all)] - pub fn marshal_msg(&self) -> Result> { - let mut wr = Vec::new(); - - // header - wr.write_all(XL_FILE_HEADER.as_slice())?; - - let mut major = [0u8; 2]; - byteorder::LittleEndian::write_u16(&mut major, XL_FILE_VERSION_MAJOR); - wr.write_all(major.as_slice())?; - - let mut minor = [0u8; 2]; - byteorder::LittleEndian::write_u16(&mut minor, XL_FILE_VERSION_MINOR); - wr.write_all(minor.as_slice())?; - - // size bin32 预留 write_bin_len - wr.write_all(&[0xc6, 0, 0, 0, 0])?; - - let offset = wr.len(); - - rmp::encode::write_uint8(&mut wr, XL_HEADER_VERSION)?; - rmp::encode::write_uint8(&mut wr, XL_META_VERSION)?; - - // versions - rmp::encode::write_sint(&mut wr, self.versions.len() as i64)?; - - for ver in self.versions.iter() { - let hmsg = ver.header.marshal_msg()?; - rmp::encode::write_bin(&mut wr, &hmsg)?; - - rmp::encode::write_bin(&mut wr, &ver.meta)?; - } - - // 更新 bin 长度 - let data_len = wr.len() - offset; - byteorder::BigEndian::write_u32(&mut wr[offset - 4..offset], data_len as u32); - - let crc = xxh64::xxh64(&wr[offset..], XXHASH_SEED) as u32; - let mut crc_buf = [0u8; 5]; - crc_buf[0] = 0xce; // u32 - byteorder::BigEndian::write_u32(&mut crc_buf[1..], crc); - - wr.write_all(&crc_buf)?; - - wr.write_all(self.data.as_slice())?; - - Ok(wr) - } - - // pub fn unmarshal(buf: &[u8]) -> Result { - // let mut s = Self::default(); - // s.unmarshal_msg(buf)?; - // Ok(s) - // // let t: FileMeta = rmp_serde::from_slice(buf)?; - // // Ok(t) - // } - - // pub fn marshal_msg(&self) -> Result> { - // let mut buf = Vec::new(); - - // self.serialize(&mut Serializer::new(&mut buf))?; - - // Ok(buf) - // } - - fn get_idx(&self, idx: usize) -> Result { - if idx > self.versions.len() { - return Err(Error::new(DiskError::FileNotFound)); - } - - FileMetaVersion::try_from(self.versions[idx].meta.as_slice()) - } - - fn set_idx(&mut self, idx: usize, ver: FileMetaVersion) -> Result<()> { - if idx >= self.versions.len() { - return Err(Error::new(DiskError::FileNotFound)); - } - - // TODO: use old buf - let meta_buf = ver.marshal_msg()?; - - let pre_mod_time = self.versions[idx].header.mod_time; - - self.versions[idx].header = ver.header(); - self.versions[idx].meta = meta_buf; - - if pre_mod_time != self.versions[idx].header.mod_time { - self.sort_by_mod_time(); - } - - Ok(()) - } - - fn sort_by_mod_time(&mut self) { - if self.versions.len() <= 1 { - return; - } - - // Sort by mod_time in descending order (latest first) - self.versions.sort_by(|a, b| { - match (a.header.mod_time, b.header.mod_time) { - (Some(a_time), Some(b_time)) => b_time.cmp(&a_time), // Descending order - (Some(_), None) => Ordering::Less, - (None, Some(_)) => Ordering::Greater, - (None, None) => Ordering::Equal, - } - }); - } - - // 查找版本 - pub fn find_version(&self, vid: Option) -> Result<(usize, FileMetaVersion)> { - for (i, fver) in self.versions.iter().enumerate() { - if fver.header.version_id == vid { - let version = self.get_idx(i)?; - return Ok((i, version)); - } - } - - Err(Error::new(DiskError::FileVersionNotFound)) - } - - // shard_data_dir_count 查询 vid 下 data_dir 的数量 - #[tracing::instrument(level = "debug", skip_all)] - pub fn shard_data_dir_count(&self, vid: &Option, data_dir: &Option) -> usize { - self.versions - .iter() - .filter(|v| v.header.version_type == VersionType::Object && v.header.version_id != *vid && v.header.user_data_dir()) - .map(|v| FileMetaVersion::decode_data_dir_from_meta(&v.meta).unwrap_or_default()) - .filter(|v| v == data_dir) - .count() - } - - pub fn update_object_version(&mut self, fi: FileInfo) -> Result<()> { - for version in self.versions.iter_mut() { - match version.header.version_type { - VersionType::Invalid => (), - VersionType::Object => { - if version.header.version_id == fi.version_id { - let mut ver = FileMetaVersion::try_from(version.meta.as_slice())?; - - if let Some(ref mut obj) = ver.object { - if let Some(ref mut meta_user) = obj.meta_user { - if let Some(meta) = &fi.metadata { - for (k, v) in meta { - meta_user.insert(k.clone(), v.clone()); - } - } - obj.meta_user = Some(meta_user.clone()); - } else { - let mut meta_user = HashMap::new(); - if let Some(meta) = &fi.metadata { - for (k, v) in meta { - // TODO: MetaSys - meta_user.insert(k.clone(), v.clone()); - } - } - obj.meta_user = Some(meta_user); - } - - if let Some(mod_time) = fi.mod_time { - obj.mod_time = Some(mod_time); - } - } - - // 更新 - version.header = ver.header(); - version.meta = ver.marshal_msg()?; - } - } - VersionType::Delete => { - if version.header.version_id == fi.version_id { - return Err(Error::msg("method not allowed")); - } - } - } - } - - self.versions.sort_by(|a, b| { - if a.header.mod_time != b.header.mod_time { - a.header.mod_time.cmp(&b.header.mod_time) - } else if a.header.version_type != b.header.version_type { - a.header.version_type.cmp(&b.header.version_type) - } else if a.header.version_id != b.header.version_id { - a.header.version_id.cmp(&b.header.version_id) - } else if a.header.flags != b.header.flags { - a.header.flags.cmp(&b.header.flags) - } else { - a.cmp(b) - } - }); - Ok(()) - } - - // 添加版本 - #[tracing::instrument(level = "debug", skip_all)] - pub fn add_version(&mut self, fi: FileInfo) -> Result<()> { - let vid = fi.version_id; - - if let Some(ref data) = fi.data { - let key = vid.unwrap_or_default().to_string(); - self.data.replace(&key, data.clone())?; - } - - let version = FileMetaVersion::from(fi); - - if !version.valid() { - return Err(Error::msg("file meta version invalid")); - } - - // should replace - for (idx, ver) in self.versions.iter().enumerate() { - if ver.header.version_id != vid { - continue; - } - - return self.set_idx(idx, version); - } - - // TODO: version count limit ! - - let mod_time = version.get_mod_time(); - - // puth a -1 mod time value , so we can relplace this - self.versions.push(FileMetaShallowVersion { - header: FileMetaVersionHeader { - mod_time: Some(OffsetDateTime::from_unix_timestamp(-1)?), - ..Default::default() - }, - ..Default::default() - }); - - for (idx, exist) in self.versions.iter().enumerate() { - if let Some(ref ex_mt) = exist.header.mod_time { - if let Some(ref in_md) = mod_time { - if ex_mt <= in_md { - // insert - self.versions.insert(idx, FileMetaShallowVersion::try_from(version)?); - self.versions.pop(); - return Ok(()); - } - } - } - } - - Err(Error::msg("add_version failed")) - } - - // delete_version 删除版本，返回 data_dir - pub fn delete_version(&mut self, fi: &FileInfo) -> Result> { - let mut ventry = FileMetaVersion::default(); - if fi.deleted { - ventry.version_type = VersionType::Delete; - ventry.delete_marker = Some(MetaDeleteMarker { - version_id: fi.version_id, - mod_time: fi.mod_time, - ..Default::default() - }); - - if !fi.is_valid() { - return Err(Error::msg("invalid file meta version")); - } - } - - for (i, ver) in self.versions.iter().enumerate() { - if ver.header.version_id != fi.version_id { - continue; - } - - return match ver.header.version_type { - VersionType::Invalid => Err(Error::msg("invalid file meta version")), - VersionType::Delete => Ok(None), - VersionType::Object => { - let v = self.get_idx(i)?; - - self.versions.remove(i); - - let a = v.object.map(|v| v.data_dir).unwrap_or_default(); - Ok(a) - } - }; - } - - Err(Error::new(DiskError::FileVersionNotFound)) - } - - // read_data fill fi.dada - #[tracing::instrument(level = "debug", skip(self))] - pub fn into_fileinfo( - &self, - volume: &str, - path: &str, - version_id: &str, - read_data: bool, - all_parts: bool, - ) -> Result { - let has_vid = { - if !version_id.is_empty() { - let id = Uuid::parse_str(version_id)?; - if !id.is_nil() { - Some(id) - } else { - None - } - } else { - None - } - }; - - let mut is_latest = true; - let mut succ_mod_time = None; - for ver in self.versions.iter() { - let header = &ver.header; - - if let Some(vid) = has_vid { - if header.version_id != Some(vid) { - is_latest = false; - succ_mod_time = header.mod_time; - continue; - } - } - - let mut fi = ver.to_fileinfo(volume, path, has_vid, all_parts)?; - fi.is_latest = is_latest; - if let Some(_d) = succ_mod_time { - fi.successor_mod_time = succ_mod_time; - } - if read_data { - fi.data = self.data.find(fi.version_id.unwrap_or_default().to_string().as_str())?; - } - - fi.num_versions = self.versions.len(); - - return Ok(fi); - } - - if has_vid.is_none() { - Err(Error::from(DiskError::FileNotFound)) - } else { - Err(Error::from(DiskError::FileVersionNotFound)) - } - } - - #[tracing::instrument(level = "debug", skip(self))] - pub fn into_file_info_versions(&self, volume: &str, path: &str, all_parts: bool) -> Result { - let mut versions = Vec::new(); - for version in self.versions.iter() { - let mut file_version = FileMetaVersion::default(); - file_version.unmarshal_msg(&version.meta)?; - let fi = file_version.to_fileinfo(volume, path, None, all_parts); - versions.push(fi); - } - - Ok(FileInfoVersions { - volume: volume.to_string(), - name: path.to_string(), - latest_mod_time: versions[0].mod_time, - versions, - ..Default::default() - }) - } - - pub fn lastest_mod_time(&self) -> Option { - if self.versions.is_empty() { - return None; - } - - self.versions.first().unwrap().header.mod_time - } -} - -// impl Display for FileMeta { -// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -// f.write_str("FileMeta:")?; -// for (i, ver) in self.versions.iter().enumerate() { -// let mut meta = FileMetaVersion::default(); -// meta.unmarshal_msg(&ver.meta).unwrap_or_default(); -// f.write_fmt(format_args!("ver:{} header {:?}, meta {:?}", i, ver.header, meta))?; +// use crate::disk::FileInfoVersions; +// use crate::file_meta_inline::InlineData; +// use crate::store_api::RawFileInfo; +// use crate::error::StorageError; +// use crate::{ +// disk::error::DiskError, +// store_api::{ErasureInfo, FileInfo, ObjectPartInfo, ERASURE_ALGORITHM}, +// }; +// use byteorder::ByteOrder; +// use common::error::{Error, Result}; +// use rmp::Marker; +// use serde::{Deserialize, Serialize}; +// use std::cmp::Ordering; +// use std::fmt::Display; +// use std::io::{self, Read, Write}; +// use std::{collections::HashMap, io::Cursor}; +// use time::OffsetDateTime; +// use tokio::io::AsyncRead; +// use tracing::{error, warn}; +// use uuid::Uuid; +// use xxhash_rust::xxh64; + +// // XL header specifies the format +// pub static XL_FILE_HEADER: [u8; 4] = [b'X', b'L', b'2', b' ']; +// // pub static XL_FILE_VERSION_CURRENT: [u8; 4] = [0; 4]; + +// // Current version being written. +// // static XL_FILE_VERSION: [u8; 4] = [1, 0, 3, 0]; +// static XL_FILE_VERSION_MAJOR: u16 = 1; +// static XL_FILE_VERSION_MINOR: u16 = 3; +// static XL_HEADER_VERSION: u8 = 3; +// static XL_META_VERSION: u8 = 2; +// static XXHASH_SEED: u64 = 0; + +// const XL_FLAG_FREE_VERSION: u8 = 1 << 0; +// // const XL_FLAG_USES_DATA_DIR: u8 = 1 << 1; +// const _XL_FLAG_INLINE_DATA: u8 = 1 << 2; + +// const META_DATA_READ_DEFAULT: usize = 4 << 10; +// const MSGP_UINT32_SIZE: usize = 5; + +// // type ScanHeaderVersionFn = Box Result<()>>; + +// #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +// pub struct FileMeta { +// pub versions: Vec, +// pub data: InlineData, // TODO: xlMetaInlineData +// pub meta_ver: u8, +// } + +// impl FileMeta { +// pub fn new() -> Self { +// Self { +// meta_ver: XL_META_VERSION, +// data: InlineData::new(), +// ..Default::default() +// } +// } + +// // isXL2V1Format +// #[tracing::instrument(level = "debug", skip_all)] +// pub fn is_xl2_v1_format(buf: &[u8]) -> bool { +// !matches!(Self::check_xl2_v1(buf), Err(_e)) +// } + +// #[tracing::instrument(level = "debug", skip_all)] +// pub fn load(buf: &[u8]) -> Result { +// let mut xl = FileMeta::default(); +// xl.unmarshal_msg(buf)?; + +// Ok(xl) +// } + +// // check_xl2_v1 读 xl 文件头，返回后续内容，版本信息 +// // checkXL2V1 +// #[tracing::instrument(level = "debug", skip_all)] +// pub fn check_xl2_v1(buf: &[u8]) -> Result<(&[u8], u16, u16)> { +// if buf.len() < 8 { +// return Err(Error::msg("xl file header not exists")); // } -// f.write_str("\n") +// if buf[0..4] != XL_FILE_HEADER { +// return Err(Error::msg("xl file header err")); +// } + +// let major = byteorder::LittleEndian::read_u16(&buf[4..6]); +// let minor = byteorder::LittleEndian::read_u16(&buf[6..8]); +// if major > XL_FILE_VERSION_MAJOR { +// return Err(Error::msg("xl file version err")); +// } + +// Ok((&buf[8..], major, minor)) +// } + +// // 固定 u32 +// pub fn read_bytes_header(buf: &[u8]) -> Result<(u32, &[u8])> { +// if buf.len() < 5 { +// return Err(Error::new(io::Error::new( +// io::ErrorKind::UnexpectedEof, +// format!("Buffer too small: {} bytes, need at least 5", buf.len()), +// ))); +// } + +// let (mut size_buf, _) = buf.split_at(5); + +// // 取 meta 数据，buf = crc + data +// let bin_len = rmp::decode::read_bin_len(&mut size_buf)?; + +// Ok((bin_len, &buf[5..])) +// } + +// pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { +// let i = buf.len() as u64; + +// // check version, buf = buf[8..] +// let (buf, _, _) = Self::check_xl2_v1(buf)?; + +// let (mut size_buf, buf) = buf.split_at(5); + +// // 取 meta 数据，buf = crc + data +// let bin_len = rmp::decode::read_bin_len(&mut size_buf)?; + +// let (meta, buf) = buf.split_at(bin_len as usize); + +// let (mut crc_buf, buf) = buf.split_at(5); + +// // crc check +// let crc = rmp::decode::read_u32(&mut crc_buf)?; +// let meta_crc = xxh64::xxh64(meta, XXHASH_SEED) as u32; + +// if crc != meta_crc { +// return Err(Error::msg("xl file crc check failed")); +// } + +// if !buf.is_empty() { +// self.data.update(buf); +// self.data.validate()?; +// } + +// // 解析 meta +// if !meta.is_empty() { +// let (versions_len, _, meta_ver, meta) = Self::decode_xl_headers(meta)?; + +// // let (_, meta) = meta.split_at(read_size as usize); + +// self.meta_ver = meta_ver; + +// self.versions = Vec::with_capacity(versions_len); + +// let mut cur: Cursor<&[u8]> = Cursor::new(meta); +// for _ in 0..versions_len { +// let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; +// let start = cur.position() as usize; +// let end = start + bin_len; +// let header_buf = &meta[start..end]; + +// let mut ver = FileMetaShallowVersion::default(); +// ver.header.unmarshal_msg(header_buf)?; + +// cur.set_position(end as u64); + +// let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; +// let start = cur.position() as usize; +// let end = start + bin_len; +// let mut ver_meta_buf = &meta[start..end]; + +// ver_meta_buf.read_to_end(&mut ver.meta)?; + +// cur.set_position(end as u64); + +// self.versions.push(ver); +// } +// } + +// Ok(i) +// } + +// // decode_xl_headers 解析 meta 头，返回 (versions 数量，xl_header_version, xl_meta_version, 已读数据长度) +// #[tracing::instrument(level = "debug", skip_all)] +// fn decode_xl_headers(buf: &[u8]) -> Result<(usize, u8, u8, &[u8])> { +// let mut cur = Cursor::new(buf); + +// let header_ver: u8 = rmp::decode::read_int(&mut cur)?; + +// if header_ver > XL_HEADER_VERSION { +// return Err(Error::msg("xl header version invalid")); +// } + +// let meta_ver: u8 = rmp::decode::read_int(&mut cur)?; +// if meta_ver > XL_META_VERSION { +// return Err(Error::msg("xl meta version invalid")); +// } + +// let versions_len: usize = rmp::decode::read_int(&mut cur)?; + +// Ok((versions_len, header_ver, meta_ver, &buf[cur.position() as usize..])) +// } + +// fn decode_versions Result<()>>(buf: &[u8], versions: usize, mut fnc: F) -> Result<()> { +// let mut cur: Cursor<&[u8]> = Cursor::new(buf); + +// for i in 0..versions { +// let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; +// let start = cur.position() as usize; +// let end = start + bin_len; +// let header_buf = &buf[start..end]; + +// cur.set_position(end as u64); + +// let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; +// let start = cur.position() as usize; +// let end = start + bin_len; +// let ver_meta_buf = &buf[start..end]; + +// cur.set_position(end as u64); + +// if let Err(err) = fnc(i, header_buf, ver_meta_buf) { +// if let Some(e) = err.downcast_ref::() { +// if e == &StorageError::DoneForNow { +// return Ok(()); +// } +// } + +// return Err(err); +// } +// } + +// Ok(()) +// } + +// pub fn is_latest_delete_marker(buf: &[u8]) -> bool { +// let header = Self::decode_xl_headers(buf).ok(); +// if let Some((versions, _hdr_v, _meta_v, meta)) = header { +// if versions == 0 { +// return false; +// } + +// let mut is_delete_marker = false; + +// let _ = Self::decode_versions(meta, versions, |_: usize, hdr: &[u8], _: &[u8]| { +// let mut header = FileMetaVersionHeader::default(); +// if header.unmarshal_msg(hdr).is_err() { +// return Err(Error::new(StorageError::DoneForNow)); +// } + +// is_delete_marker = header.version_type == VersionType::Delete; + +// Err(Error::new(StorageError::DoneForNow)) +// }); + +// is_delete_marker +// } else { +// false +// } +// } + +// #[tracing::instrument(level = "debug", skip_all)] +// pub fn marshal_msg(&self) -> Result> { +// let mut wr = Vec::new(); + +// // header +// wr.write_all(XL_FILE_HEADER.as_slice())?; + +// let mut major = [0u8; 2]; +// byteorder::LittleEndian::write_u16(&mut major, XL_FILE_VERSION_MAJOR); +// wr.write_all(major.as_slice())?; + +// let mut minor = [0u8; 2]; +// byteorder::LittleEndian::write_u16(&mut minor, XL_FILE_VERSION_MINOR); +// wr.write_all(minor.as_slice())?; + +// // size bin32 预留 write_bin_len +// wr.write_all(&[0xc6, 0, 0, 0, 0])?; + +// let offset = wr.len(); + +// rmp::encode::write_uint8(&mut wr, XL_HEADER_VERSION)?; +// rmp::encode::write_uint8(&mut wr, XL_META_VERSION)?; + +// // versions +// rmp::encode::write_sint(&mut wr, self.versions.len() as i64)?; + +// for ver in self.versions.iter() { +// let hmsg = ver.header.marshal_msg()?; +// rmp::encode::write_bin(&mut wr, &hmsg)?; + +// rmp::encode::write_bin(&mut wr, &ver.meta)?; +// } + +// // 更新 bin 长度 +// let data_len = wr.len() - offset; +// byteorder::BigEndian::write_u32(&mut wr[offset - 4..offset], data_len as u32); + +// let crc = xxh64::xxh64(&wr[offset..], XXHASH_SEED) as u32; +// let mut crc_buf = [0u8; 5]; +// crc_buf[0] = 0xce; // u32 +// byteorder::BigEndian::write_u32(&mut crc_buf[1..], crc); + +// wr.write_all(&crc_buf)?; + +// wr.write_all(self.data.as_slice())?; + +// Ok(wr) +// } + +// // pub fn unmarshal(buf: &[u8]) -> Result { +// // let mut s = Self::default(); +// // s.unmarshal_msg(buf)?; +// // Ok(s) +// // // let t: FileMeta = rmp_serde::from_slice(buf)?; +// // // Ok(t) +// // } + +// // pub fn marshal_msg(&self) -> Result> { +// // let mut buf = Vec::new(); + +// // self.serialize(&mut Serializer::new(&mut buf))?; + +// // Ok(buf) +// // } + +// fn get_idx(&self, idx: usize) -> Result { +// if idx > self.versions.len() { +// return Err(Error::new(DiskError::FileNotFound)); +// } + +// FileMetaVersion::try_from(self.versions[idx].meta.as_slice()) +// } + +// fn set_idx(&mut self, idx: usize, ver: FileMetaVersion) -> Result<()> { +// if idx >= self.versions.len() { +// return Err(Error::new(DiskError::FileNotFound)); +// } + +// // TODO: use old buf +// let meta_buf = ver.marshal_msg()?; + +// let pre_mod_time = self.versions[idx].header.mod_time; + +// self.versions[idx].header = ver.header(); +// self.versions[idx].meta = meta_buf; + +// if pre_mod_time != self.versions[idx].header.mod_time { +// self.sort_by_mod_time(); +// } + +// Ok(()) +// } + +// fn sort_by_mod_time(&mut self) { +// if self.versions.len() <= 1 { +// return; +// } + +// // Sort by mod_time in descending order (latest first) +// self.versions.sort_by(|a, b| { +// match (a.header.mod_time, b.header.mod_time) { +// (Some(a_time), Some(b_time)) => b_time.cmp(&a_time), // Descending order +// (Some(_), None) => Ordering::Less, +// (None, Some(_)) => Ordering::Greater, +// (None, None) => Ordering::Equal, +// } +// }); +// } + +// // 查找版本 +// pub fn find_version(&self, vid: Option) -> Result<(usize, FileMetaVersion)> { +// for (i, fver) in self.versions.iter().enumerate() { +// if fver.header.version_id == vid { +// let version = self.get_idx(i)?; +// return Ok((i, version)); +// } +// } + +// Err(Error::new(DiskError::FileVersionNotFound)) +// } + +// // shard_data_dir_count 查询 vid 下 data_dir 的数量 +// #[tracing::instrument(level = "debug", skip_all)] +// pub fn shard_data_dir_count(&self, vid: &Option, data_dir: &Option) -> usize { +// self.versions +// .iter() +// .filter(|v| v.header.version_type == VersionType::Object && v.header.version_id != *vid && v.header.user_data_dir()) +// .map(|v| FileMetaVersion::decode_data_dir_from_meta(&v.meta).unwrap_or_default()) +// .filter(|v| v == data_dir) +// .count() +// } + +// pub fn update_object_version(&mut self, fi: FileInfo) -> Result<()> { +// for version in self.versions.iter_mut() { +// match version.header.version_type { +// VersionType::Invalid => (), +// VersionType::Object => { +// if version.header.version_id == fi.version_id { +// let mut ver = FileMetaVersion::try_from(version.meta.as_slice())?; + +// if let Some(ref mut obj) = ver.object { +// if let Some(ref mut meta_user) = obj.meta_user { +// if let Some(meta) = &fi.metadata { +// for (k, v) in meta { +// meta_user.insert(k.clone(), v.clone()); +// } +// } +// obj.meta_user = Some(meta_user.clone()); +// } else { +// let mut meta_user = HashMap::new(); +// if let Some(meta) = &fi.metadata { +// for (k, v) in meta { +// // TODO: MetaSys +// meta_user.insert(k.clone(), v.clone()); +// } +// } +// obj.meta_user = Some(meta_user); +// } + +// if let Some(mod_time) = fi.mod_time { +// obj.mod_time = Some(mod_time); +// } +// } + +// // 更新 +// version.header = ver.header(); +// version.meta = ver.marshal_msg()?; +// } +// } +// VersionType::Delete => { +// if version.header.version_id == fi.version_id { +// return Err(Error::msg("method not allowed")); +// } +// } +// } +// } + +// self.versions.sort_by(|a, b| { +// if a.header.mod_time != b.header.mod_time { +// a.header.mod_time.cmp(&b.header.mod_time) +// } else if a.header.version_type != b.header.version_type { +// a.header.version_type.cmp(&b.header.version_type) +// } else if a.header.version_id != b.header.version_id { +// a.header.version_id.cmp(&b.header.version_id) +// } else if a.header.flags != b.header.flags { +// a.header.flags.cmp(&b.header.flags) +// } else { +// a.cmp(b) +// } +// }); +// Ok(()) +// } + +// // 添加版本 +// #[tracing::instrument(level = "debug", skip_all)] +// pub fn add_version(&mut self, fi: FileInfo) -> Result<()> { +// let vid = fi.version_id; + +// if let Some(ref data) = fi.data { +// let key = vid.unwrap_or_default().to_string(); +// self.data.replace(&key, data.clone())?; +// } + +// let version = FileMetaVersion::from(fi); + +// if !version.valid() { +// return Err(Error::msg("file meta version invalid")); +// } + +// // should replace +// for (idx, ver) in self.versions.iter().enumerate() { +// if ver.header.version_id != vid { +// continue; +// } + +// return self.set_idx(idx, version); +// } + +// // TODO: version count limit ! + +// let mod_time = version.get_mod_time(); + +// // puth a -1 mod time value , so we can relplace this +// self.versions.push(FileMetaShallowVersion { +// header: FileMetaVersionHeader { +// mod_time: Some(OffsetDateTime::from_unix_timestamp(-1)?), +// ..Default::default() +// }, +// ..Default::default() +// }); + +// for (idx, exist) in self.versions.iter().enumerate() { +// if let Some(ref ex_mt) = exist.header.mod_time { +// if let Some(ref in_md) = mod_time { +// if ex_mt <= in_md { +// // insert +// self.versions.insert(idx, FileMetaShallowVersion::try_from(version)?); +// self.versions.pop(); +// return Ok(()); +// } +// } +// } +// } + +// Err(Error::msg("add_version failed")) +// } + +// // delete_version 删除版本，返回 data_dir +// pub fn delete_version(&mut self, fi: &FileInfo) -> Result> { +// let mut ventry = FileMetaVersion::default(); +// if fi.deleted { +// ventry.version_type = VersionType::Delete; +// ventry.delete_marker = Some(MetaDeleteMarker { +// version_id: fi.version_id, +// mod_time: fi.mod_time, +// ..Default::default() +// }); + +// if !fi.is_valid() { +// return Err(Error::msg("invalid file meta version")); +// } +// } + +// for (i, ver) in self.versions.iter().enumerate() { +// if ver.header.version_id != fi.version_id { +// continue; +// } + +// return match ver.header.version_type { +// VersionType::Invalid => Err(Error::msg("invalid file meta version")), +// VersionType::Delete => Ok(None), +// VersionType::Object => { +// let v = self.get_idx(i)?; + +// self.versions.remove(i); + +// let a = v.object.map(|v| v.data_dir).unwrap_or_default(); +// Ok(a) +// } +// }; +// } + +// Err(Error::new(DiskError::FileVersionNotFound)) +// } + +// // read_data fill fi.dada +// #[tracing::instrument(level = "debug", skip(self))] +// pub fn into_fileinfo( +// &self, +// volume: &str, +// path: &str, +// version_id: &str, +// read_data: bool, +// all_parts: bool, +// ) -> Result { +// let has_vid = { +// if !version_id.is_empty() { +// let id = Uuid::parse_str(version_id)?; +// if !id.is_nil() { +// Some(id) +// } else { +// None +// } +// } else { +// None +// } +// }; + +// let mut is_latest = true; +// let mut succ_mod_time = None; +// for ver in self.versions.iter() { +// let header = &ver.header; + +// if let Some(vid) = has_vid { +// if header.version_id != Some(vid) { +// is_latest = false; +// succ_mod_time = header.mod_time; +// continue; +// } +// } + +// let mut fi = ver.to_fileinfo(volume, path, has_vid, all_parts)?; +// fi.is_latest = is_latest; +// if let Some(_d) = succ_mod_time { +// fi.successor_mod_time = succ_mod_time; +// } +// if read_data { +// fi.data = self.data.find(fi.version_id.unwrap_or_default().to_string().as_str())?; +// } + +// fi.num_versions = self.versions.len(); + +// return Ok(fi); +// } + +// if has_vid.is_none() { +// Err(Error::from(DiskError::FileNotFound)) +// } else { +// Err(Error::from(DiskError::FileVersionNotFound)) +// } +// } + +// #[tracing::instrument(level = "debug", skip(self))] +// pub fn into_file_info_versions(&self, volume: &str, path: &str, all_parts: bool) -> Result { +// let mut versions = Vec::new(); +// for version in self.versions.iter() { +// let mut file_version = FileMetaVersion::default(); +// file_version.unmarshal_msg(&version.meta)?; +// let fi = file_version.to_fileinfo(volume, path, None, all_parts); +// versions.push(fi); +// } + +// Ok(FileInfoVersions { +// volume: volume.to_string(), +// name: path.to_string(), +// latest_mod_time: versions[0].mod_time, +// versions, +// ..Default::default() +// }) +// } + +// pub fn lastest_mod_time(&self) -> Option { +// if self.versions.is_empty() { +// return None; +// } + +// self.versions.first().unwrap().header.mod_time // } // } -#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Clone, Eq, PartialOrd, Ord)] -pub struct FileMetaShallowVersion { - pub header: FileMetaVersionHeader, - pub meta: Vec, // FileMetaVersion.marshal_msg -} - -impl FileMetaShallowVersion { - pub fn to_fileinfo(&self, volume: &str, path: &str, version_id: Option, all_parts: bool) -> Result { - let file_version = FileMetaVersion::try_from(self.meta.as_slice())?; - - Ok(file_version.to_fileinfo(volume, path, version_id, all_parts)) - } -} - -impl TryFrom for FileMetaShallowVersion { - type Error = Error; - - fn try_from(value: FileMetaVersion) -> std::result::Result { - let header = value.header(); - let meta = value.marshal_msg()?; - Ok(Self { meta, header }) - } -} - -#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)] -pub struct FileMetaVersion { - pub version_type: VersionType, - pub object: Option, - pub delete_marker: Option, - pub write_version: u64, // rustfs version -} - -impl FileMetaVersion { - pub fn valid(&self) -> bool { - if !self.version_type.valid() { - return false; - } - - match self.version_type { - VersionType::Object => self - .object - .as_ref() - .map(|v| v.erasure_algorithm.valid() && v.bitrot_checksum_algo.valid() && v.mod_time.is_some()) - .unwrap_or_default(), - VersionType::Delete => self - .delete_marker - .as_ref() - .map(|v| v.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) > OffsetDateTime::UNIX_EPOCH) - .unwrap_or_default(), - _ => false, - } - } - - pub fn get_data_dir(&self) -> Option { - self.valid() - .then(|| { - if self.version_type == VersionType::Object { - self.object.as_ref().map(|v| v.data_dir).unwrap_or_default() - } else { - None - } - }) - .unwrap_or_default() - } - - pub fn get_version_id(&self) -> Option { - match self.version_type { - VersionType::Object | VersionType::Delete => self.object.as_ref().map(|v| v.version_id).unwrap_or_default(), - _ => None, - } - } - - pub fn get_mod_time(&self) -> Option { - match self.version_type { - VersionType::Object => self.object.as_ref().map(|v| v.mod_time).unwrap_or_default(), - VersionType::Delete => self.delete_marker.as_ref().map(|v| v.mod_time).unwrap_or_default(), - _ => None, - } - } - - // decode_data_dir_from_meta 从 meta 中读取 data_dir TODO: 直接从 meta buf 中只解析出 data_dir, msg.skip - pub fn decode_data_dir_from_meta(buf: &[u8]) -> Result> { - let mut ver = Self::default(); - ver.unmarshal_msg(buf)?; - - let data_dir = ver.object.map(|v| v.data_dir).unwrap_or_default(); - Ok(data_dir) - } - - pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let mut cur = Cursor::new(buf); - - let mut fields_len = rmp::decode::read_map_len(&mut cur)?; - - while fields_len > 0 { - fields_len -= 1; - - // println!("unmarshal_msg fields idx {}", fields_len); - - let str_len = rmp::decode::read_str_len(&mut cur)?; - - // println!("unmarshal_msg fields name len() {}", &str_len); - - // ！！！Vec::with_capacity(str_len) 失败，vec! 正常 - let mut field_buff = vec![0u8; str_len as usize]; - - cur.read_exact(&mut field_buff)?; - - let field = String::from_utf8(field_buff)?; - - // println!("unmarshal_msg fields name {}", &field); - - match field.as_str() { - "Type" => { - let u: u8 = rmp::decode::read_int(&mut cur)?; - self.version_type = VersionType::from_u8(u); - } - - "V2Obj" => { - // is_nil() - if buf[cur.position() as usize] == 0xc0 { - rmp::decode::read_nil(&mut cur)?; - } else { - // let buf = unsafe { cur.position() }; - let mut obj = MetaObject::default(); - // let start = cur.position(); - - let (_, remain) = buf.split_at(cur.position() as usize); - - let read_len = obj.unmarshal_msg(remain)?; - cur.set_position(cur.position() + read_len); - - self.object = Some(obj); - } - } - "DelObj" => { - if buf[cur.position() as usize] == 0xc0 { - rmp::decode::read_nil(&mut cur)?; - } else { - // let buf = unsafe { cur.position() }; - let mut obj = MetaDeleteMarker::default(); - // let start = cur.position(); - - let (_, remain) = buf.split_at(cur.position() as usize); - let read_len = obj.unmarshal_msg(remain)?; - cur.set_position(cur.position() + read_len); - - self.delete_marker = Some(obj); - } - } - "v" => { - self.write_version = rmp::decode::read_int(&mut cur)?; - } - name => return Err(Error::msg(format!("not suport field name {}", name))), - } - } - - Ok(cur.position()) - } - - pub fn marshal_msg(&self) -> Result> { - let mut len: u32 = 4; - let mut mask: u8 = 0; - - if self.object.is_none() { - len -= 1; - mask |= 0x2; - } - if self.delete_marker.is_none() { - len -= 1; - mask |= 0x4; - } - - let mut wr = Vec::new(); - - // 字段数量 - rmp::encode::write_map_len(&mut wr, len)?; - - // write "Type" - rmp::encode::write_str(&mut wr, "Type")?; - rmp::encode::write_uint(&mut wr, self.version_type.to_u8() as u64)?; - - if (mask & 0x2) == 0 { - // write V2Obj - rmp::encode::write_str(&mut wr, "V2Obj")?; - if self.object.is_none() { - let _ = rmp::encode::write_nil(&mut wr); - } else { - let buf = self.object.as_ref().unwrap().marshal_msg()?; - wr.write_all(&buf)?; - } - } - - if (mask & 0x4) == 0 { - // write "DelObj" - rmp::encode::write_str(&mut wr, "DelObj")?; - if self.delete_marker.is_none() { - let _ = rmp::encode::write_nil(&mut wr); - } else { - let buf = self.delete_marker.as_ref().unwrap().marshal_msg()?; - wr.write_all(&buf)?; - } - } - - // write "v" - rmp::encode::write_str(&mut wr, "v")?; - rmp::encode::write_uint(&mut wr, self.write_version)?; - - Ok(wr) - } - - pub fn free_version(&self) -> bool { - self.version_type == VersionType::Delete && self.delete_marker.as_ref().map(|m| m.free_version()).unwrap_or_default() - } - - pub fn header(&self) -> FileMetaVersionHeader { - FileMetaVersionHeader::from(self.clone()) - } - - pub fn to_fileinfo(&self, volume: &str, path: &str, version_id: Option, all_parts: bool) -> FileInfo { - match self.version_type { - VersionType::Invalid => FileInfo { - name: path.to_string(), - volume: volume.to_string(), - version_id, - ..Default::default() - }, - VersionType::Object => self - .object - .as_ref() - .unwrap() - .clone() - .into_fileinfo(volume, path, version_id, all_parts), - VersionType::Delete => self - .delete_marker - .as_ref() - .unwrap() - .clone() - .into_fileinfo(volume, path, version_id, all_parts), - } - } -} - -impl TryFrom<&[u8]> for FileMetaVersion { - type Error = Error; - - fn try_from(value: &[u8]) -> std::result::Result { - let mut ver = FileMetaVersion::default(); - ver.unmarshal_msg(value)?; - Ok(ver) - } -} - -impl From for FileMetaVersion { - fn from(value: FileInfo) -> Self { - { - if value.deleted { - FileMetaVersion { - version_type: VersionType::Delete, - delete_marker: Some(MetaDeleteMarker::from(value)), - object: None, - write_version: 0, - } - } else { - FileMetaVersion { - version_type: VersionType::Object, - delete_marker: None, - object: Some(MetaObject::from(value)), - write_version: 0, - } - } - } - } -} - -impl TryFrom for FileMetaVersion { - type Error = Error; - - fn try_from(value: FileMetaShallowVersion) -> std::result::Result { - FileMetaVersion::try_from(value.meta.as_slice()) - } -} - -#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone, Eq, Hash)] -pub struct FileMetaVersionHeader { - pub version_id: Option, - pub mod_time: Option, - pub signature: [u8; 4], - pub version_type: VersionType, - pub flags: u8, - pub ec_n: u8, - pub ec_m: u8, -} - -impl FileMetaVersionHeader { - pub fn has_ec(&self) -> bool { - self.ec_m > 0 && self.ec_n > 0 - } - - pub fn matches_not_strict(&self, o: &FileMetaVersionHeader) -> bool { - let mut ok = self.version_id == o.version_id && self.version_type == o.version_type && self.matches_ec(o); - if self.version_id.is_none() { - ok = ok && self.mod_time == o.mod_time; - } - - ok - } - - pub fn matches_ec(&self, o: &FileMetaVersionHeader) -> bool { - if self.has_ec() && o.has_ec() { - return self.ec_n == o.ec_n && self.ec_m == o.ec_m; - } - - true - } - - pub fn free_version(&self) -> bool { - self.flags & XL_FLAG_FREE_VERSION != 0 - } - - pub fn sorts_before(&self, o: &FileMetaVersionHeader) -> bool { - if self == o { - return false; - } - - // Prefer newest modtime. - if self.mod_time != o.mod_time { - return self.mod_time > o.mod_time; - } - - match self.mod_time.cmp(&o.mod_time) { - Ordering::Greater => { - return true; - } - Ordering::Less => { - return false; - } - _ => {} - } - - // The following doesn't make too much sense, but we want sort to be consistent nonetheless. - // Prefer lower types - if self.version_type != o.version_type { - return self.version_type < o.version_type; - } - // Consistent sort on signature - match self.version_id.cmp(&o.version_id) { - Ordering::Greater => { - return true; - } - Ordering::Less => { - return false; - } - _ => {} - } - - if self.flags != o.flags { - return self.flags > o.flags; - } - - false - } - - pub fn user_data_dir(&self) -> bool { - self.flags & Flags::UsesDataDir as u8 != 0 - } - #[tracing::instrument] - pub fn marshal_msg(&self) -> Result> { - let mut wr = Vec::new(); - - // array len 7 - rmp::encode::write_array_len(&mut wr, 7)?; - - // version_id - rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or_default().as_bytes())?; - // mod_time - rmp::encode::write_i64(&mut wr, self.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp_nanos() as i64)?; - // signature - rmp::encode::write_bin(&mut wr, self.signature.as_slice())?; - // version_type - rmp::encode::write_uint8(&mut wr, self.version_type.to_u8())?; - // flags - rmp::encode::write_uint8(&mut wr, self.flags)?; - // ec_n - rmp::encode::write_uint8(&mut wr, self.ec_n)?; - // ec_m - rmp::encode::write_uint8(&mut wr, self.ec_m)?; - - Ok(wr) - } - - pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let mut cur = Cursor::new(buf); - let alen = rmp::decode::read_array_len(&mut cur)?; - if alen != 7 { - return Err(Error::msg(format!("version header array len err need 7 got {}", alen))); - } - - // version_id - rmp::decode::read_bin_len(&mut cur)?; - let mut buf = [0u8; 16]; - cur.read_exact(&mut buf)?; - self.version_id = { - let id = Uuid::from_bytes(buf); - if id.is_nil() { - None - } else { - Some(id) - } - }; - - // mod_time - let unix: i128 = rmp::decode::read_int(&mut cur)?; - - let time = OffsetDateTime::from_unix_timestamp_nanos(unix)?; - if time == OffsetDateTime::UNIX_EPOCH { - self.mod_time = None; - } else { - self.mod_time = Some(time); - } - - // signature - rmp::decode::read_bin_len(&mut cur)?; - cur.read_exact(&mut self.signature)?; - - // version_type - let typ: u8 = rmp::decode::read_int(&mut cur)?; - self.version_type = VersionType::from_u8(typ); - - // flags - self.flags = rmp::decode::read_int(&mut cur)?; - // ec_n - self.ec_n = rmp::decode::read_int(&mut cur)?; - // ec_m - self.ec_m = rmp::decode::read_int(&mut cur)?; - - Ok(cur.position()) - } -} - -impl PartialOrd for FileMetaVersionHeader { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for FileMetaVersionHeader { - fn cmp(&self, other: &Self) -> Ordering { - match self.mod_time.cmp(&other.mod_time) { - Ordering::Equal => {} - ord => return ord, - } - - match self.version_type.cmp(&other.version_type) { - Ordering::Equal => {} - ord => return ord, - } - match self.signature.cmp(&other.signature) { - Ordering::Equal => {} - ord => return ord, - } - match self.version_id.cmp(&other.version_id) { - Ordering::Equal => {} - ord => return ord, - } - self.flags.cmp(&other.flags) - } -} - -impl From for FileMetaVersionHeader { - fn from(value: FileMetaVersion) -> Self { - let flags = { - let mut f: u8 = 0; - if value.free_version() { - f |= Flags::FreeVersion as u8; - } - - if value.version_type == VersionType::Object && value.object.as_ref().map(|v| v.use_data_dir()).unwrap_or_default() { - f |= Flags::UsesDataDir as u8; - } - - if value.version_type == VersionType::Object && value.object.as_ref().map(|v| v.use_inlinedata()).unwrap_or_default() - { - f |= Flags::InlineData as u8; - } - - f - }; - - let (ec_n, ec_m) = { - if value.version_type == VersionType::Object && value.object.is_some() { - ( - value.object.as_ref().unwrap().erasure_n as u8, - value.object.as_ref().unwrap().erasure_m as u8, - ) - } else { - (0, 0) - } - }; - - Self { - version_id: value.get_version_id(), - mod_time: value.get_mod_time(), - signature: [0, 0, 0, 0], - version_type: value.version_type, - flags, - ec_n, - ec_m, - } - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)] -// 因为自定义 message_pack，所以一定要保证字段顺序 -pub struct MetaObject { - pub version_id: Option, // Version ID - pub data_dir: Option, // Data dir ID - pub erasure_algorithm: ErasureAlgo, // Erasure coding algorithm - pub erasure_m: usize, // Erasure data blocks - pub erasure_n: usize, // Erasure parity blocks - pub erasure_block_size: usize, // Erasure block size - pub erasure_index: usize, // Erasure disk index - pub erasure_dist: Vec, // Erasure distribution - pub bitrot_checksum_algo: ChecksumAlgo, // Bitrot checksum algo - pub part_numbers: Vec, // Part Numbers - pub part_etags: Option>, // Part ETags - pub part_sizes: Vec, // Part Sizes - pub part_actual_sizes: Option>, // Part ActualSizes (compression) - pub part_indices: Option>>, // Part Indexes (compression) - pub size: usize, // Object version size - pub mod_time: Option, // Object version modified time - pub meta_sys: Option>>, // Object version internal metadata - pub meta_user: Option>, // Object version metadata set by user -} - -impl MetaObject { - pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let mut cur = Cursor::new(buf); - - let mut fields_len = rmp::decode::read_map_len(&mut cur)?; - - // let mut ret = Self::default(); - - while fields_len > 0 { - fields_len -= 1; - - // println!("unmarshal_msg fields idx {}", fields_len); - - let str_len = rmp::decode::read_str_len(&mut cur)?; - - // println!("unmarshal_msg fields name len() {}", &str_len); - - // ！！！Vec::with_capacity(str_len) 失败，vec! 正常 - let mut field_buff = vec![0u8; str_len as usize]; - - cur.read_exact(&mut field_buff)?; - - let field = String::from_utf8(field_buff)?; - - // println!("unmarshal_msg fields name {}", &field); - - match field.as_str() { - "ID" => { - rmp::decode::read_bin_len(&mut cur)?; - let mut buf = [0u8; 16]; - cur.read_exact(&mut buf)?; - self.version_id = { - let id = Uuid::from_bytes(buf); - if id.is_nil() { - None - } else { - Some(id) - } - }; - } - "DDir" => { - rmp::decode::read_bin_len(&mut cur)?; - let mut buf = [0u8; 16]; - cur.read_exact(&mut buf)?; - self.data_dir = { - let id = Uuid::from_bytes(buf); - if id.is_nil() { - None - } else { - Some(id) - } - }; - } - "EcAlgo" => { - let u: u8 = rmp::decode::read_int(&mut cur)?; - self.erasure_algorithm = ErasureAlgo::from_u8(u) - } - "EcM" => { - self.erasure_m = rmp::decode::read_int(&mut cur)?; - } - "EcN" => { - self.erasure_n = rmp::decode::read_int(&mut cur)?; - } - "EcBSize" => { - self.erasure_block_size = rmp::decode::read_int(&mut cur)?; - } - "EcIndex" => { - self.erasure_index = rmp::decode::read_int(&mut cur)?; - } - "EcDist" => { - let alen = rmp::decode::read_array_len(&mut cur)? as usize; - self.erasure_dist = vec![0u8; alen]; - for i in 0..alen { - self.erasure_dist[i] = rmp::decode::read_int(&mut cur)?; - } - } - "CSumAlgo" => { - let u: u8 = rmp::decode::read_int(&mut cur)?; - self.bitrot_checksum_algo = ChecksumAlgo::from_u8(u) - } - "PartNums" => { - let alen = rmp::decode::read_array_len(&mut cur)? as usize; - self.part_numbers = vec![0; alen]; - for i in 0..alen { - self.part_numbers[i] = rmp::decode::read_int(&mut cur)?; - } - } - "PartETags" => { - let array_len = match rmp::decode::read_nil(&mut cur) { - Ok(_) => None, - Err(e) => match e { - rmp::decode::ValueReadError::TypeMismatch(marker) => match marker { - Marker::FixArray(l) => Some(l as usize), - Marker::Array16 => Some(rmp::decode::read_u16(&mut cur)? as usize), - Marker::Array32 => Some(rmp::decode::read_u16(&mut cur)? as usize), - _ => return Err(Error::msg("PartETags parse failed")), - }, - _ => return Err(Error::msg("PartETags parse failed.")), - }, - }; - - if array_len.is_some() { - let l = array_len.unwrap(); - let mut etags = Vec::with_capacity(l); - for _ in 0..l { - let str_len = rmp::decode::read_str_len(&mut cur)?; - let mut field_buff = vec![0u8; str_len as usize]; - cur.read_exact(&mut field_buff)?; - etags.push(String::from_utf8(field_buff)?); - } - self.part_etags = Some(etags); - } - } - "PartSizes" => { - let alen = rmp::decode::read_array_len(&mut cur)? as usize; - self.part_sizes = vec![0; alen]; - for i in 0..alen { - self.part_sizes[i] = rmp::decode::read_int(&mut cur)?; - } - } - "PartASizes" => { - let array_len = match rmp::decode::read_nil(&mut cur) { - Ok(_) => None, - Err(e) => match e { - rmp::decode::ValueReadError::TypeMismatch(marker) => match marker { - Marker::FixArray(l) => Some(l as usize), - Marker::Array16 => Some(rmp::decode::read_u16(&mut cur)? as usize), - Marker::Array32 => Some(rmp::decode::read_u16(&mut cur)? as usize), - _ => return Err(Error::msg("PartETags parse failed")), - }, - _ => return Err(Error::msg("PartETags parse failed.")), - }, - }; - if let Some(l) = array_len { - let mut sizes = vec![0; l]; - for size in sizes.iter_mut().take(l) { - *size = rmp::decode::read_int(&mut cur)?; - } - // for size in sizes.iter_mut().take(l) { - // let tmp = rmp::decode::read_int(&mut cur)?; - // size = tmp; - // } - self.part_actual_sizes = Some(sizes); - } - } - "PartIdx" => { - let alen = rmp::decode::read_array_len(&mut cur)? as usize; - - if alen == 0 { - self.part_indices = None; - continue; - } - - let mut indices = Vec::with_capacity(alen); - for _ in 0..alen { - let blen = rmp::decode::read_bin_len(&mut cur)?; - let mut buf = vec![0u8; blen as usize]; - cur.read_exact(&mut buf)?; - - indices.push(buf); - } - - self.part_indices = Some(indices); - } - "Size" => { - self.size = rmp::decode::read_int(&mut cur)?; - } - "MTime" => { - let unix: i128 = rmp::decode::read_int(&mut cur)?; - let time = OffsetDateTime::from_unix_timestamp_nanos(unix)?; - if time == OffsetDateTime::UNIX_EPOCH { - self.mod_time = None; - } else { - self.mod_time = Some(time); - } - } - "MetaSys" => { - let len = match rmp::decode::read_nil(&mut cur) { - Ok(_) => None, - Err(e) => match e { - rmp::decode::ValueReadError::TypeMismatch(marker) => match marker { - Marker::FixMap(l) => Some(l as usize), - Marker::Map16 => Some(rmp::decode::read_u16(&mut cur)? as usize), - Marker::Map32 => Some(rmp::decode::read_u16(&mut cur)? as usize), - _ => return Err(Error::msg("MetaSys parse failed")), - }, - _ => return Err(Error::msg("MetaSys parse failed.")), - }, - }; - if len.is_some() { - let l = len.unwrap(); - let mut map = HashMap::new(); - for _ in 0..l { - let str_len = rmp::decode::read_str_len(&mut cur)?; - let mut field_buff = vec![0u8; str_len as usize]; - cur.read_exact(&mut field_buff)?; - let key = String::from_utf8(field_buff)?; - - let blen = rmp::decode::read_bin_len(&mut cur)?; - let mut val = vec![0u8; blen as usize]; - cur.read_exact(&mut val)?; - - map.insert(key, val); - } - - self.meta_sys = Some(map); - } - } - "MetaUsr" => { - let len = match rmp::decode::read_nil(&mut cur) { - Ok(_) => None, - Err(e) => match e { - rmp::decode::ValueReadError::TypeMismatch(marker) => match marker { - Marker::FixMap(l) => Some(l as usize), - Marker::Map16 => Some(rmp::decode::read_u16(&mut cur)? as usize), - Marker::Map32 => Some(rmp::decode::read_u16(&mut cur)? as usize), - _ => return Err(Error::msg("MetaUsr parse failed")), - }, - _ => return Err(Error::msg("MetaUsr parse failed.")), - }, - }; - if len.is_some() { - let l = len.unwrap(); - let mut map = HashMap::new(); - for _ in 0..l { - let str_len = rmp::decode::read_str_len(&mut cur)?; - let mut field_buff = vec![0u8; str_len as usize]; - cur.read_exact(&mut field_buff)?; - let key = String::from_utf8(field_buff)?; - - let blen = rmp::decode::read_str_len(&mut cur)?; - let mut val_buf = vec![0u8; blen as usize]; - cur.read_exact(&mut val_buf)?; - let val = String::from_utf8(val_buf)?; - - map.insert(key, val); - } - - self.meta_user = Some(map); - } - } - - name => return Err(Error::msg(format!("not suport field name {}", name))), - } - } - - Ok(cur.position()) - } - // marshal_msg 自定义 messagepack 命名与 go 一致 - pub fn marshal_msg(&self) -> Result> { - let mut len: u32 = 18; - let mut mask: u32 = 0; - - if self.part_indices.is_none() { - len -= 1; - mask |= 0x2000; - } - - let mut wr = Vec::new(); - - // 字段数量 - rmp::encode::write_map_len(&mut wr, len)?; - - // string "ID" - rmp::encode::write_str(&mut wr, "ID")?; - rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or_default().as_bytes())?; - - // string "DDir" - rmp::encode::write_str(&mut wr, "DDir")?; - rmp::encode::write_bin(&mut wr, self.data_dir.unwrap_or_default().as_bytes())?; - - // string "EcAlgo" - rmp::encode::write_str(&mut wr, "EcAlgo")?; - rmp::encode::write_uint(&mut wr, self.erasure_algorithm.to_u8() as u64)?; - - // string "EcM" - rmp::encode::write_str(&mut wr, "EcM")?; - rmp::encode::write_uint(&mut wr, self.erasure_m.try_into().unwrap())?; - - // string "EcN" - rmp::encode::write_str(&mut wr, "EcN")?; - rmp::encode::write_uint(&mut wr, self.erasure_n.try_into().unwrap())?; - - // string "EcBSize" - rmp::encode::write_str(&mut wr, "EcBSize")?; - rmp::encode::write_uint(&mut wr, self.erasure_block_size.try_into().unwrap())?; - - // string "EcIndex" - rmp::encode::write_str(&mut wr, "EcIndex")?; - rmp::encode::write_uint(&mut wr, self.erasure_index.try_into().unwrap())?; - - // string "EcDist" - rmp::encode::write_str(&mut wr, "EcDist")?; - rmp::encode::write_array_len(&mut wr, self.erasure_dist.len() as u32)?; - for v in self.erasure_dist.iter() { - rmp::encode::write_uint(&mut wr, *v as _)?; - } - - // string "CSumAlgo" - rmp::encode::write_str(&mut wr, "CSumAlgo")?; - rmp::encode::write_uint(&mut wr, self.bitrot_checksum_algo.to_u8() as u64)?; - - // string "PartNums" - rmp::encode::write_str(&mut wr, "PartNums")?; - rmp::encode::write_array_len(&mut wr, self.part_numbers.len() as u32)?; - for v in self.part_numbers.iter() { - rmp::encode::write_uint(&mut wr, *v as _)?; - } - - // string "PartETags" - rmp::encode::write_str(&mut wr, "PartETags")?; - if self.part_etags.is_none() { - rmp::encode::write_nil(&mut wr)?; - } else { - let etags = self.part_etags.as_ref().unwrap(); - rmp::encode::write_array_len(&mut wr, etags.len() as u32)?; - for v in etags.iter() { - rmp::encode::write_str(&mut wr, v.as_str())?; - } - } - - // string "PartSizes" - rmp::encode::write_str(&mut wr, "PartSizes")?; - rmp::encode::write_array_len(&mut wr, self.part_sizes.len() as u32)?; - for v in self.part_sizes.iter() { - rmp::encode::write_uint(&mut wr, *v as _)?; - } - - // string "PartASizes" - rmp::encode::write_str(&mut wr, "PartASizes")?; - if self.part_actual_sizes.is_none() { - rmp::encode::write_nil(&mut wr)?; - } else { - let asizes = self.part_actual_sizes.as_ref().unwrap(); - rmp::encode::write_array_len(&mut wr, asizes.len() as u32)?; - for v in asizes.iter() { - rmp::encode::write_uint(&mut wr, *v as _)?; - } - } - - if (mask & 0x2000) == 0 { - // string "PartIdx" - rmp::encode::write_str(&mut wr, "PartIdx")?; - let indices = self.part_indices.as_ref().unwrap(); - rmp::encode::write_array_len(&mut wr, indices.len() as u32)?; - for v in indices.iter() { - rmp::encode::write_bin(&mut wr, v)?; - } - } - - // string "Size" - rmp::encode::write_str(&mut wr, "Size")?; - rmp::encode::write_uint(&mut wr, self.size.try_into().unwrap())?; - - // string "MTime" - rmp::encode::write_str(&mut wr, "MTime")?; - rmp::encode::write_uint( - &mut wr, - self.mod_time - .unwrap_or(OffsetDateTime::UNIX_EPOCH) - .unix_timestamp_nanos() - .try_into() - .unwrap(), - )?; - - // string "MetaSys" - rmp::encode::write_str(&mut wr, "MetaSys")?; - if self.meta_sys.is_none() { - rmp::encode::write_nil(&mut wr)?; - } else { - let metas = self.meta_sys.as_ref().unwrap(); - rmp::encode::write_map_len(&mut wr, metas.len() as u32)?; - for (k, v) in metas { - rmp::encode::write_str(&mut wr, k.as_str())?; - rmp::encode::write_bin(&mut wr, v)?; - } - } - - // string "MetaUsr" - rmp::encode::write_str(&mut wr, "MetaUsr")?; - if self.meta_user.is_none() { - rmp::encode::write_nil(&mut wr)?; - } else { - let metas = self.meta_user.as_ref().unwrap(); - rmp::encode::write_map_len(&mut wr, metas.len() as u32)?; - for (k, v) in metas { - rmp::encode::write_str(&mut wr, k.as_str())?; - rmp::encode::write_str(&mut wr, v.as_str())?; - } - } - - Ok(wr) - } - pub fn use_data_dir(&self) -> bool { - // TODO: when use inlinedata - true - } - - pub fn use_inlinedata(&self) -> bool { - // TODO: when use inlinedata - false - } - - pub fn into_fileinfo(self, volume: &str, path: &str, _version_id: Option, _all_parts: bool) -> FileInfo { - let version_id = self.version_id; - - let erasure = ErasureInfo { - algorithm: self.erasure_algorithm.to_string(), - data_blocks: self.erasure_m, - parity_blocks: self.erasure_n, - block_size: self.erasure_block_size, - index: self.erasure_index, - distribution: self.erasure_dist.iter().map(|&v| v as usize).collect(), - ..Default::default() - }; - - let mut parts = Vec::new(); - for (i, _) in self.part_numbers.iter().enumerate() { - parts.push(ObjectPartInfo { - number: self.part_numbers[i], - size: self.part_sizes[i], - ..Default::default() - }); - } - - let metadata = { - if let Some(metauser) = self.meta_user.as_ref() { - let mut m = HashMap::new(); - for (k, v) in metauser { - // TODO: skip xhttp x-amz-storage-class - m.insert(k.to_owned(), v.to_owned()); - } - Some(m) - } else { - None - } - }; - - FileInfo { - version_id, - erasure, - data_dir: self.data_dir, - mod_time: self.mod_time, - size: self.size, - name: path.to_string(), - volume: volume.to_string(), - parts, - metadata, - ..Default::default() - } - } -} - -impl From for MetaObject { - fn from(value: FileInfo) -> Self { - let part_numbers: Vec = value.parts.iter().map(|v| v.number).collect(); - let part_sizes: Vec = value.parts.iter().map(|v| v.size).collect(); - - Self { - version_id: value.version_id, - size: value.size, - mod_time: value.mod_time, - data_dir: value.data_dir, - erasure_algorithm: ErasureAlgo::ReedSolomon, - erasure_m: value.erasure.data_blocks, - erasure_n: value.erasure.parity_blocks, - erasure_block_size: value.erasure.block_size, - erasure_index: value.erasure.index, - erasure_dist: value.erasure.distribution.iter().map(|x| *x as u8).collect(), - bitrot_checksum_algo: ChecksumAlgo::HighwayHash, - part_numbers, - part_etags: None, // TODO: add part_etags - part_sizes, - part_actual_sizes: None, // TODO: add part_etags - part_indices: None, - meta_sys: None, - meta_user: value.metadata.clone(), - } - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)] -pub struct MetaDeleteMarker { - pub version_id: Option, // Version ID for delete marker - pub mod_time: Option, // Object delete marker modified time - pub meta_sys: Option>>, // Delete marker internal metadata -} - -impl MetaDeleteMarker { - pub fn free_version(&self) -> bool { - self.meta_sys - .as_ref() - .map(|v| v.get(FREE_VERSION_META_HEADER).is_some()) - .unwrap_or_default() - } - - pub fn into_fileinfo(self, volume: &str, path: &str, version_id: Option, _all_parts: bool) -> FileInfo { - FileInfo { - name: path.to_string(), - volume: volume.to_string(), - version_id, - deleted: true, - mod_time: self.mod_time, - ..Default::default() - } - } - - pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let mut cur = Cursor::new(buf); - - let mut fields_len = rmp::decode::read_map_len(&mut cur)?; - - while fields_len > 0 { - fields_len -= 1; - - let str_len = rmp::decode::read_str_len(&mut cur)?; - - // ！！！Vec::with_capacity(str_len) 失败，vec! 正常 - let mut field_buff = vec![0u8; str_len as usize]; - - cur.read_exact(&mut field_buff)?; - - let field = String::from_utf8(field_buff)?; - - match field.as_str() { - "ID" => { - rmp::decode::read_bin_len(&mut cur)?; - let mut buf = [0u8; 16]; - cur.read_exact(&mut buf)?; - self.version_id = { - let id = Uuid::from_bytes(buf); - if id.is_nil() { - None - } else { - Some(id) - } - }; - } - - "MTime" => { - let unix: i64 = rmp::decode::read_int(&mut cur)?; - let time = OffsetDateTime::from_unix_timestamp(unix)?; - if time == OffsetDateTime::UNIX_EPOCH { - self.mod_time = None; - } else { - self.mod_time = Some(time); - } - } - "MetaSys" => { - let l = rmp::decode::read_map_len(&mut cur)?; - let mut map = HashMap::new(); - for _ in 0..l { - let str_len = rmp::decode::read_str_len(&mut cur)?; - let mut field_buff = vec![0u8; str_len as usize]; - cur.read_exact(&mut field_buff)?; - let key = String::from_utf8(field_buff)?; - - let blen = rmp::decode::read_bin_len(&mut cur)?; - let mut val = vec![0u8; blen as usize]; - cur.read_exact(&mut val)?; - - map.insert(key, val); - } - - self.meta_sys = Some(map); - } - name => return Err(Error::msg(format!("not suport field name {}", name))), - } - } - - Ok(cur.position()) - } - - pub fn marshal_msg(&self) -> Result> { - let mut len: u32 = 3; - let mut mask: u8 = 0; - - if self.meta_sys.is_none() { - len -= 1; - mask |= 0x4; - } - - let mut wr = Vec::new(); - - // 字段数量 - rmp::encode::write_map_len(&mut wr, len)?; - - // string "ID" - rmp::encode::write_str(&mut wr, "ID")?; - rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or_default().as_bytes())?; - - // string "MTime" - rmp::encode::write_str(&mut wr, "MTime")?; - rmp::encode::write_uint( - &mut wr, - self.mod_time - .unwrap_or(OffsetDateTime::UNIX_EPOCH) - .unix_timestamp() - .try_into() - .unwrap(), - )?; - - if (mask & 0x4) == 0 { - let metas = self.meta_sys.as_ref().unwrap(); - rmp::encode::write_map_len(&mut wr, metas.len() as u32)?; - for (k, v) in metas { - rmp::encode::write_str(&mut wr, k.as_str())?; - rmp::encode::write_bin(&mut wr, v)?; - } - } - - Ok(wr) - } -} - -impl From for MetaDeleteMarker { - fn from(value: FileInfo) -> Self { - Self { - version_id: value.version_id, - mod_time: value.mod_time, - meta_sys: None, - } - } -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Default, Clone, PartialOrd, Ord, Hash)] -pub enum VersionType { - #[default] - Invalid = 0, - Object = 1, - Delete = 2, - // Legacy = 3, -} - -impl VersionType { - pub fn valid(&self) -> bool { - matches!(*self, VersionType::Object | VersionType::Delete) - } - - pub fn to_u8(&self) -> u8 { - match self { - VersionType::Invalid => 0, - VersionType::Object => 1, - VersionType::Delete => 2, - } - } - - pub fn from_u8(n: u8) -> Self { - match n { - 1 => VersionType::Object, - 2 => VersionType::Delete, - _ => VersionType::Invalid, - } - } -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Default, Clone)] -pub enum ErasureAlgo { - #[default] - Invalid = 0, - ReedSolomon = 1, -} - -impl ErasureAlgo { - pub fn valid(&self) -> bool { - *self > ErasureAlgo::Invalid - } - pub fn to_u8(&self) -> u8 { - match self { - ErasureAlgo::Invalid => 0, - ErasureAlgo::ReedSolomon => 1, - } - } - - pub fn from_u8(u: u8) -> Self { - match u { - 1 => ErasureAlgo::ReedSolomon, - _ => ErasureAlgo::Invalid, - } - } -} - -impl Display for ErasureAlgo { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ErasureAlgo::Invalid => write!(f, "Invalid"), - ErasureAlgo::ReedSolomon => write!(f, "{}", ERASURE_ALGORITHM), - } - } -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Default, Clone)] -pub enum ChecksumAlgo { - #[default] - Invalid = 0, - HighwayHash = 1, -} - -impl ChecksumAlgo { - pub fn valid(&self) -> bool { - *self > ChecksumAlgo::Invalid - } - pub fn to_u8(&self) -> u8 { - match self { - ChecksumAlgo::Invalid => 0, - ChecksumAlgo::HighwayHash => 1, - } - } - pub fn from_u8(u: u8) -> Self { - match u { - 1 => ChecksumAlgo::HighwayHash, - _ => ChecksumAlgo::Invalid, - } - } -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Default, Clone)] -pub enum Flags { - #[default] - FreeVersion = 1 << 0, - UsesDataDir = 1 << 1, - InlineData = 1 << 2, -} - -const FREE_VERSION_META_HEADER: &str = "free-version"; - -// mergeXLV2Versions -pub fn merge_file_meta_versions( - mut quorum: usize, - mut strict: bool, - requested_versions: usize, - versions: &[Vec], -) -> Vec { - if quorum == 0 { - quorum = 1; - } - - if versions.len() < quorum || versions.is_empty() { - return Vec::new(); - } - - if versions.len() == 1 { - return versions[0].clone(); - } - - if quorum == 1 { - strict = true; - } - - let mut versions = versions.to_owned(); - - let mut n_versions = 0; - - let mut merged = Vec::new(); - loop { - let mut tops = Vec::new(); - let mut top_sig = FileMetaVersionHeader::default(); - let mut consistent = true; - for vers in versions.iter() { - if vers.is_empty() { - consistent = false; - continue; - } - if tops.is_empty() { - consistent = true; - top_sig = vers[0].header.clone(); - } else { - consistent = consistent && vers[0].header == top_sig; - } - tops.push(vers[0].clone()); - } - - // check if done... - if tops.len() < quorum { - break; - } - - let mut latest = FileMetaShallowVersion::default(); - if consistent { - merged.push(tops[0].clone()); - if tops[0].header.free_version() { - n_versions += 1; - } - } else { - let mut lastest_count = 0; - for (i, ver) in tops.iter().enumerate() { - if ver.header == latest.header { - lastest_count += 1; - continue; - } - - if i == 0 || ver.header.sorts_before(&latest.header) { - if i == 0 || lastest_count == 0 { - lastest_count = 1; - } else if !strict && ver.header.matches_not_strict(&latest.header) { - lastest_count += 1; - } else { - lastest_count = 1; - } - latest = ver.clone(); - continue; - } - - // Mismatch, but older. - if lastest_count > 0 && !strict && ver.header.matches_not_strict(&latest.header) { - lastest_count += 1; - continue; - } - - if lastest_count > 0 && ver.header.version_id == latest.header.version_id { - let mut x: HashMap = HashMap::new(); - for a in tops.iter() { - if a.header.version_id != ver.header.version_id { - continue; - } - let mut a_clone = a.clone(); - if !strict { - a_clone.header.signature = [0; 4]; - } - *x.entry(a_clone.header).or_insert(1) += 1; - } - lastest_count = 0; - for (k, v) in x.iter() { - if *v < lastest_count { - continue; - } - if *v == lastest_count && latest.header.sorts_before(k) { - continue; - } - tops.iter().for_each(|a| { - let mut hdr = a.header.clone(); - if !strict { - hdr.signature = [0; 4]; - } - if hdr == *k { - latest = a.clone(); - } - }); - - lastest_count = *v; - } - break; - } - } - if lastest_count >= quorum { - if !latest.header.free_version() { - n_versions += 1; - } - merged.push(latest.clone()); - } - } - - // Remove from all streams up until latest modtime or if selected. - versions.iter_mut().for_each(|vers| { - // // Keep top entry (and remaining)... - let mut bre = false; - vers.retain(|ver| { - if bre { - return true; - } - if let Ordering::Greater = ver.header.mod_time.cmp(&latest.header.mod_time) { - bre = true; - return false; - } - if ver.header == latest.header { - bre = true; - return false; - } - if let Ordering::Equal = latest.header.version_id.cmp(&ver.header.version_id) { - bre = true; - return false; - } - for merged_v in merged.iter() { - if let Ordering::Equal = ver.header.version_id.cmp(&merged_v.header.version_id) { - bre = true; - return false; - } - } - true - }); - }); - if requested_versions > 0 && requested_versions == n_versions { - merged.append(&mut versions[0]); - break; - } - } - - // Sanity check. Enable if duplicates show up. - // todo - merged -} - -pub async fn file_info_from_raw(ri: RawFileInfo, bucket: &str, object: &str, read_data: bool) -> Result { - get_file_info(&ri.buf, bucket, object, "", FileInfoOpts { data: read_data }).await -} - -pub struct FileInfoOpts { - pub data: bool, -} - -pub async fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opts: FileInfoOpts) -> Result { - let vid = { - if version_id.is_empty() { - None - } else { - Some(Uuid::parse_str(version_id)?) - } - }; - - let meta = FileMeta::load(buf)?; - if meta.versions.is_empty() { - return Ok(FileInfo { - volume: volume.to_owned(), - name: path.to_owned(), - version_id: vid, - is_latest: true, - deleted: true, - mod_time: Some(OffsetDateTime::from_unix_timestamp(1)?), - ..Default::default() - }); - } - - let fi = meta.into_fileinfo(volume, path, version_id, opts.data, true)?; - Ok(fi) -} - -async fn read_more( - reader: &mut R, - buf: &mut Vec, - total_size: usize, - read_size: usize, - has_full: bool, -) -> Result<()> { - use tokio::io::AsyncReadExt; - let has = buf.len(); - - if has >= read_size { - return Ok(()); - } - - if has_full || read_size > total_size { - return Err(Error::new(io::Error::new(io::ErrorKind::UnexpectedEof, "Unexpected EOF"))); - } - - let extra = read_size - has; - if buf.capacity() >= read_size { - // Extend the buffer if we have enough space. - buf.resize(read_size, 0); - } else { - buf.extend(vec![0u8; extra]); - } - - reader.read_exact(&mut buf[has..]).await?; - Ok(()) -} - -pub async fn read_xl_meta_no_data(reader: &mut R, size: usize) -> Result> { - use tokio::io::AsyncReadExt; - - let mut initial = size; - let mut has_full = true; - - if initial > META_DATA_READ_DEFAULT { - initial = META_DATA_READ_DEFAULT; - has_full = false; - } - - let mut buf = vec![0u8; initial]; - reader.read_exact(&mut buf).await?; - - let (tmp_buf, major, minor) = FileMeta::check_xl2_v1(&buf)?; - - match major { - 1 => match minor { - 0 => { - read_more(reader, &mut buf, size, size, has_full).await?; - Ok(buf) - } - 1..=3 => { - let (sz, tmp_buf) = FileMeta::read_bytes_header(tmp_buf)?; - let mut want = sz as usize + (buf.len() - tmp_buf.len()); - - if minor < 2 { - read_more(reader, &mut buf, size, want, has_full).await?; - return Ok(buf[..want].to_vec()); - } - - let want_max = usize::min(want + MSGP_UINT32_SIZE, size); - read_more(reader, &mut buf, size, want_max, has_full).await?; - - if buf.len() < want { - error!("read_xl_meta_no_data buffer too small (length: {}, needed: {})", &buf.len(), want); - return Err(Error::new(DiskError::FileCorrupt)); - } - - let tmp = &buf[want..]; - let crc_size = 5; - let other_size = tmp.len() - crc_size; - - want += tmp.len() - other_size; - - Ok(buf[..want].to_vec()) - } - _ => Err(Error::new(io::Error::new(io::ErrorKind::InvalidData, "Unknown minor metadata version"))), - }, - _ => Err(Error::new(io::Error::new(io::ErrorKind::InvalidData, "Unknown major metadata version"))), - } -} -#[cfg(test)] -#[allow(clippy::field_reassign_with_default)] -mod test { - use super::*; - - #[test] - fn test_new_file_meta() { - let mut fm = FileMeta::new(); - - let (m, n) = (3, 2); - - for i in 0..5 { - let mut fi = FileInfo::new(i.to_string().as_str(), m, n); - fi.mod_time = Some(OffsetDateTime::now_utc()); - - fm.add_version(fi).unwrap(); - } - - let buff = fm.marshal_msg().unwrap(); - - let mut newfm = FileMeta::default(); - newfm.unmarshal_msg(&buff).unwrap(); - - assert_eq!(fm, newfm) - } - - #[test] - fn test_marshal_metaobject() { - let obj = MetaObject { - data_dir: Some(Uuid::new_v4()), - ..Default::default() - }; - - // println!("obj {:?}", &obj); - - let encoded = obj.marshal_msg().unwrap(); - - let mut obj2 = MetaObject::default(); - obj2.unmarshal_msg(&encoded).unwrap(); - - // println!("obj2 {:?}", &obj2); - - assert_eq!(obj, obj2); - assert_eq!(obj.data_dir, obj2.data_dir); - } - - #[test] - fn test_marshal_metadeletemarker() { - let obj = MetaDeleteMarker { - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - - // println!("obj {:?}", &obj); - - let encoded = obj.marshal_msg().unwrap(); - - let mut obj2 = MetaDeleteMarker::default(); - obj2.unmarshal_msg(&encoded).unwrap(); - - // println!("obj2 {:?}", &obj2); - - assert_eq!(obj, obj2); - assert_eq!(obj.version_id, obj2.version_id); - } - - #[test] - #[tracing::instrument] - fn test_marshal_metaversion() { - let mut fi = FileInfo::new("test", 3, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(OffsetDateTime::now_utc().unix_timestamp()).unwrap()); - let mut obj = FileMetaVersion::from(fi); - obj.write_version = 110; - - // println!("obj {:?}", &obj); - - let encoded = obj.marshal_msg().unwrap(); - - let mut obj2 = FileMetaVersion::default(); - obj2.unmarshal_msg(&encoded).unwrap(); - - // println!("obj2 {:?}", &obj2); - - // 时间截不一致 - - - assert_eq!(obj, obj2); - assert_eq!(obj.get_version_id(), obj2.get_version_id()); - assert_eq!(obj.write_version, obj2.write_version); - assert_eq!(obj.write_version, 110); - } - - #[test] - #[tracing::instrument] - fn test_marshal_metaversionheader() { - let mut obj = FileMetaVersionHeader::default(); - let vid = Some(Uuid::new_v4()); - obj.version_id = vid; - - let encoded = obj.marshal_msg().unwrap(); - - let mut obj2 = FileMetaVersionHeader::default(); - obj2.unmarshal_msg(&encoded).unwrap(); - - // 时间截不一致 - - - assert_eq!(obj, obj2); - assert_eq!(obj.version_id, obj2.version_id); - assert_eq!(obj.version_id, vid); - } - - // New comprehensive tests for utility functions and validation - - #[test] - fn test_xl_file_header_constants() { - // Test XL file header constants - assert_eq!(XL_FILE_HEADER, [b'X', b'L', b'2', b' ']); - assert_eq!(XL_FILE_VERSION_MAJOR, 1); - assert_eq!(XL_FILE_VERSION_MINOR, 3); - assert_eq!(XL_HEADER_VERSION, 3); - assert_eq!(XL_META_VERSION, 2); - } - - #[test] - fn test_is_xl2_v1_format() { - // Test valid XL2 V1 format - let mut valid_buf = vec![0u8; 20]; - valid_buf[0..4].copy_from_slice(&XL_FILE_HEADER); - byteorder::LittleEndian::write_u16(&mut valid_buf[4..6], 1); - byteorder::LittleEndian::write_u16(&mut valid_buf[6..8], 0); - - assert!(FileMeta::is_xl2_v1_format(&valid_buf)); - - // Test invalid format - wrong header - let invalid_buf = vec![0u8; 20]; - assert!(!FileMeta::is_xl2_v1_format(&invalid_buf)); - - // Test buffer too small - let small_buf = vec![0u8; 4]; - assert!(!FileMeta::is_xl2_v1_format(&small_buf)); - } - - #[test] - fn test_check_xl2_v1() { - // Test valid XL2 V1 check - let mut valid_buf = vec![0u8; 20]; - valid_buf[0..4].copy_from_slice(&XL_FILE_HEADER); - byteorder::LittleEndian::write_u16(&mut valid_buf[4..6], 1); - byteorder::LittleEndian::write_u16(&mut valid_buf[6..8], 2); - - let result = FileMeta::check_xl2_v1(&valid_buf); - assert!(result.is_ok()); - let (remaining, major, minor) = result.unwrap(); - assert_eq!(major, 1); - assert_eq!(minor, 2); - assert_eq!(remaining.len(), 12); // 20 - 8 - - // Test buffer too small - let small_buf = vec![0u8; 4]; - assert!(FileMeta::check_xl2_v1(&small_buf).is_err()); - - // Test wrong header - let mut wrong_header = vec![0u8; 20]; - wrong_header[0..4].copy_from_slice(b"ABCD"); - assert!(FileMeta::check_xl2_v1(&wrong_header).is_err()); - - // Test version too high - let mut high_version = vec![0u8; 20]; - high_version[0..4].copy_from_slice(&XL_FILE_HEADER); - byteorder::LittleEndian::write_u16(&mut high_version[4..6], 99); - byteorder::LittleEndian::write_u16(&mut high_version[6..8], 0); - assert!(FileMeta::check_xl2_v1(&high_version).is_err()); - } - - #[test] - fn test_version_type_enum() { - // Test VersionType enum methods - assert!(VersionType::Object.valid()); - assert!(VersionType::Delete.valid()); - assert!(!VersionType::Invalid.valid()); - - assert_eq!(VersionType::Object.to_u8(), 1); - assert_eq!(VersionType::Delete.to_u8(), 2); - assert_eq!(VersionType::Invalid.to_u8(), 0); - - assert_eq!(VersionType::from_u8(1), VersionType::Object); - assert_eq!(VersionType::from_u8(2), VersionType::Delete); - assert_eq!(VersionType::from_u8(99), VersionType::Invalid); - } - - #[test] - fn test_erasure_algo_enum() { - // Test ErasureAlgo enum methods - assert!(ErasureAlgo::ReedSolomon.valid()); - assert!(!ErasureAlgo::Invalid.valid()); - - assert_eq!(ErasureAlgo::ReedSolomon.to_u8(), 1); - assert_eq!(ErasureAlgo::Invalid.to_u8(), 0); - - assert_eq!(ErasureAlgo::from_u8(1), ErasureAlgo::ReedSolomon); - assert_eq!(ErasureAlgo::from_u8(99), ErasureAlgo::Invalid); - - // Test Display trait - assert_eq!(format!("{}", ErasureAlgo::ReedSolomon), "rs-vandermonde"); - assert_eq!(format!("{}", ErasureAlgo::Invalid), "Invalid"); - } - - #[test] - fn test_checksum_algo_enum() { - // Test ChecksumAlgo enum methods - assert!(ChecksumAlgo::HighwayHash.valid()); - assert!(!ChecksumAlgo::Invalid.valid()); - - assert_eq!(ChecksumAlgo::HighwayHash.to_u8(), 1); - assert_eq!(ChecksumAlgo::Invalid.to_u8(), 0); - - assert_eq!(ChecksumAlgo::from_u8(1), ChecksumAlgo::HighwayHash); - assert_eq!(ChecksumAlgo::from_u8(99), ChecksumAlgo::Invalid); - } - - #[test] - fn test_file_meta_version_header_methods() { - let mut header = FileMetaVersionHeader { - ec_n: 4, - ec_m: 2, - flags: XL_FLAG_FREE_VERSION, - ..Default::default() - }; - - // Test has_ec - assert!(header.has_ec()); - - // Test free_version - assert!(header.free_version()); - - // Test user_data_dir (should be false by default) - assert!(!header.user_data_dir()); - - // Test with different flags - header.flags = 0; - assert!(!header.free_version()); - } - - #[test] - fn test_file_meta_version_header_comparison() { - let mut header1 = FileMetaVersionHeader { - mod_time: Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()), - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - - let mut header2 = FileMetaVersionHeader { - mod_time: Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()), - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - - // Test sorts_before - header2 should sort before header1 (newer mod_time) - assert!(!header1.sorts_before(&header2)); - assert!(header2.sorts_before(&header1)); - - // Test matches_not_strict - let header3 = header1.clone(); - assert!(header1.matches_not_strict(&header3)); - - // Test matches_ec - header1.ec_n = 4; - header1.ec_m = 2; - header2.ec_n = 4; - header2.ec_m = 2; - assert!(header1.matches_ec(&header2)); - - header2.ec_n = 6; - assert!(!header1.matches_ec(&header2)); - } - - #[test] - fn test_file_meta_version_methods() { - // Test with object version - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.data_dir = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - - let version = FileMetaVersion::from(fi.clone()); - - assert!(version.valid()); - assert_eq!(version.get_version_id(), fi.version_id); - assert_eq!(version.get_data_dir(), fi.data_dir); - assert_eq!(version.get_mod_time(), fi.mod_time); - assert!(!version.free_version()); - - // Test with delete marker - let mut delete_fi = FileInfo::new("test", 4, 2); - delete_fi.deleted = true; - delete_fi.version_id = Some(Uuid::new_v4()); - delete_fi.mod_time = Some(OffsetDateTime::now_utc()); - - let delete_version = FileMetaVersion::from(delete_fi); - assert!(delete_version.valid()); - assert_eq!(delete_version.version_type, VersionType::Delete); - } - - #[test] - fn test_meta_object_methods() { - let mut obj = MetaObject { - data_dir: Some(Uuid::new_v4()), - size: 1024, - ..Default::default() - }; - - // Test use_data_dir - assert!(obj.use_data_dir()); - - obj.data_dir = None; - assert!(obj.use_data_dir()); // use_data_dir always returns true - - // Test use_inlinedata (currently always returns false) - obj.size = 100; // Small size - assert!(!obj.use_inlinedata()); - - obj.size = 100000; // Large size - assert!(!obj.use_inlinedata()); - } - - #[test] - fn test_meta_delete_marker_methods() { - let marker = MetaDeleteMarker::default(); - - // Test free_version (should always return false for delete markers) - assert!(!marker.free_version()); - } - - #[test] - fn test_file_meta_latest_mod_time() { - let mut fm = FileMeta::new(); - - // Empty FileMeta should return None - assert!(fm.lastest_mod_time().is_none()); - - // Add versions with different mod times - let time1 = OffsetDateTime::from_unix_timestamp(1000).unwrap(); - let time2 = OffsetDateTime::from_unix_timestamp(2000).unwrap(); - let time3 = OffsetDateTime::from_unix_timestamp(1500).unwrap(); - - let mut fi1 = FileInfo::new("test1", 4, 2); - fi1.mod_time = Some(time1); - fm.add_version(fi1).unwrap(); - - let mut fi2 = FileInfo::new("test2", 4, 2); - fi2.mod_time = Some(time2); - fm.add_version(fi2).unwrap(); - - let mut fi3 = FileInfo::new("test3", 4, 2); - fi3.mod_time = Some(time3); - fm.add_version(fi3).unwrap(); - - // Sort first to ensure latest is at the front - fm.sort_by_mod_time(); - - // Should return the first version's mod time (lastest_mod_time returns first version's time) - assert_eq!(fm.lastest_mod_time(), fm.versions[0].header.mod_time); - } - - #[test] - fn test_file_meta_shard_data_dir_count() { - let mut fm = FileMeta::new(); - let data_dir = Some(Uuid::new_v4()); - - // Add versions with same data_dir - for i in 0..3 { - let mut fi = FileInfo::new(&format!("test{}", i), 4, 2); - fi.data_dir = data_dir; - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - } - - // Add one version with different data_dir - let mut fi_diff = FileInfo::new("test_diff", 4, 2); - fi_diff.data_dir = Some(Uuid::new_v4()); - fi_diff.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi_diff).unwrap(); - - // Count should be 0 because user_data_dir() requires UsesDataDir flag to be set - assert_eq!(fm.shard_data_dir_count(&None, &data_dir), 0); - - // Count should be 0 for non-existent data_dir - assert_eq!(fm.shard_data_dir_count(&None, &Some(Uuid::new_v4())), 0); - } - - #[test] - fn test_file_meta_sort_by_mod_time() { - let mut fm = FileMeta::new(); - - let time1 = OffsetDateTime::from_unix_timestamp(3000).unwrap(); - let time2 = OffsetDateTime::from_unix_timestamp(1000).unwrap(); - let time3 = OffsetDateTime::from_unix_timestamp(2000).unwrap(); - - // Add versions in non-chronological order - let mut fi1 = FileInfo::new("test1", 4, 2); - fi1.mod_time = Some(time1); - fm.add_version(fi1).unwrap(); - - let mut fi2 = FileInfo::new("test2", 4, 2); - fi2.mod_time = Some(time2); - fm.add_version(fi2).unwrap(); - - let mut fi3 = FileInfo::new("test3", 4, 2); - fi3.mod_time = Some(time3); - fm.add_version(fi3).unwrap(); - - // Sort by mod time - fm.sort_by_mod_time(); - - // Verify they are sorted (newest first) - add_version already sorts by insertion - // The actual order depends on how add_version inserts them - // Let's check the first version is the latest - let latest_time = fm.versions.iter().map(|v| v.header.mod_time).max().flatten(); - assert_eq!(fm.versions[0].header.mod_time, latest_time); - } - - #[test] - fn test_file_meta_find_version() { - let mut fm = FileMeta::new(); - let version_id = Some(Uuid::new_v4()); - - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = version_id; - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - // Should find the version - let result = fm.find_version(version_id); - assert!(result.is_ok()); - let (idx, version) = result.unwrap(); - assert_eq!(idx, 0); - assert_eq!(version.get_version_id(), version_id); - - // Should not find non-existent version - let non_existent_id = Some(Uuid::new_v4()); - assert!(fm.find_version(non_existent_id).is_err()); - } - - #[test] - fn test_file_meta_delete_version() { - let mut fm = FileMeta::new(); - let version_id = Some(Uuid::new_v4()); - - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = version_id; - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi.clone()).unwrap(); - - assert_eq!(fm.versions.len(), 1); - - // Delete the version - let result = fm.delete_version(&fi); - assert!(result.is_ok()); - - // Version should be removed - assert_eq!(fm.versions.len(), 0); - } - - #[test] - fn test_file_meta_update_object_version() { - let mut fm = FileMeta::new(); - let version_id = Some(Uuid::new_v4()); - - // Add initial version - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = version_id; - fi.size = 1024; - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi.clone()).unwrap(); - - // Update with new metadata (size is not updated by update_object_version) - let mut metadata = HashMap::new(); - metadata.insert("test-key".to_string(), "test-value".to_string()); - fi.metadata = Some(metadata.clone()); - let result = fm.update_object_version(fi); - assert!(result.is_ok()); - - // Verify the metadata was updated - let (_, updated_version) = fm.find_version(version_id).unwrap(); - if let Some(obj) = updated_version.object { - assert_eq!(obj.size, 1024); // Size remains unchanged - assert_eq!(obj.meta_user, Some(metadata)); // Metadata is updated - } else { - panic!("Expected object version"); - } - } - - #[test] - fn test_file_info_opts() { - let opts = FileInfoOpts { data: true }; - assert!(opts.data); - - let opts_no_data = FileInfoOpts { data: false }; - assert!(!opts_no_data.data); - } - - #[test] - fn test_decode_data_dir_from_meta() { - // Test with valid metadata containing data_dir - let data_dir = Some(Uuid::new_v4()); - let obj = MetaObject { - data_dir, - mod_time: Some(OffsetDateTime::now_utc()), - erasure_algorithm: ErasureAlgo::ReedSolomon, - bitrot_checksum_algo: ChecksumAlgo::HighwayHash, - ..Default::default() - }; - - // Create a valid FileMetaVersion with the object - let version = FileMetaVersion { - version_type: VersionType::Object, - object: Some(obj), - ..Default::default() - }; - - let encoded = version.marshal_msg().unwrap(); - let result = FileMetaVersion::decode_data_dir_from_meta(&encoded); - assert!(result.is_ok()); - assert_eq!(result.unwrap(), data_dir); - - // Test with invalid metadata - let invalid_data = vec![0u8; 10]; - let result = FileMetaVersion::decode_data_dir_from_meta(&invalid_data); - assert!(result.is_err()); - } - - #[test] - fn test_is_latest_delete_marker() { - // Test the is_latest_delete_marker function with simple data - // Since the function is complex and requires specific XL format, - // we'll test with empty data which should return false - let empty_data = vec![]; - assert!(!FileMeta::is_latest_delete_marker(&empty_data)); - - // Test with invalid data - let invalid_data = vec![1, 2, 3, 4, 5]; - assert!(!FileMeta::is_latest_delete_marker(&invalid_data)); - } - - #[test] - fn test_merge_file_meta_versions_basic() { - // Test basic merge functionality - let mut version1 = FileMetaShallowVersion::default(); - version1.header.version_id = Some(Uuid::new_v4()); - version1.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()); - - let mut version2 = FileMetaShallowVersion::default(); - version2.header.version_id = Some(Uuid::new_v4()); - version2.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()); - - let versions = vec![ - vec![version1.clone(), version2.clone()], - vec![version1.clone()], - vec![version2.clone()], - ]; - - let merged = merge_file_meta_versions(2, false, 10, &versions); - - // Should return versions that appear in at least quorum (2) sources - assert!(!merged.is_empty()); - } -} - -#[tokio::test] -async fn test_read_xl_meta_no_data() { - use tokio::fs; - use tokio::fs::File; - use tokio::io::AsyncWriteExt; - - let mut fm = FileMeta::new(); - - let (m, n) = (3, 2); - - for i in 0..5 { - let mut fi = FileInfo::new(i.to_string().as_str(), m, n); - fi.mod_time = Some(OffsetDateTime::now_utc()); - - fm.add_version(fi).unwrap(); - } - - // Use marshal_msg to create properly formatted data with XL headers - let buff = fm.marshal_msg().unwrap(); - - let filepath = "./test_xl.meta"; - - let mut file = File::create(filepath).await.unwrap(); - file.write_all(&buff).await.unwrap(); - - let mut f = File::open(filepath).await.unwrap(); - - let stat = f.metadata().await.unwrap(); - - let data = read_xl_meta_no_data(&mut f, stat.len() as usize).await.unwrap(); - - let mut newfm = FileMeta::default(); - newfm.unmarshal_msg(&data).unwrap(); - - fs::remove_file(filepath).await.unwrap(); - - assert_eq!(fm, newfm) -} - -#[tokio::test] -async fn test_get_file_info() { - // Test get_file_info function - let mut fm = FileMeta::new(); - let version_id = Uuid::new_v4(); - - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(version_id); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - let encoded = fm.marshal_msg().unwrap(); - - let opts = FileInfoOpts { data: false }; - let result = get_file_info(&encoded, "test-volume", "test-path", &version_id.to_string(), opts).await; - - assert!(result.is_ok()); - let file_info = result.unwrap(); - assert_eq!(file_info.volume, "test-volume"); - assert_eq!(file_info.name, "test-path"); -} - -#[tokio::test] -async fn test_file_info_from_raw() { - // Test file_info_from_raw function - let mut fm = FileMeta::new(); - let mut fi = FileInfo::new("test", 4, 2); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - let encoded = fm.marshal_msg().unwrap(); - - let raw_info = RawFileInfo { buf: encoded }; - - let result = file_info_from_raw(raw_info, "test-bucket", "test-object", false).await; - assert!(result.is_ok()); - - let file_info = result.unwrap(); - assert_eq!(file_info.volume, "test-bucket"); - assert_eq!(file_info.name, "test-object"); -} - -// Additional comprehensive tests for better coverage - -#[test] -fn test_file_meta_load_function() { - // Test FileMeta::load function - let mut fm = FileMeta::new(); - let mut fi = FileInfo::new("test", 4, 2); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - let encoded = fm.marshal_msg().unwrap(); - - // Test successful load - let loaded_fm = FileMeta::load(&encoded); - assert!(loaded_fm.is_ok()); - assert_eq!(loaded_fm.unwrap(), fm); - - // Test load with invalid data - let invalid_data = vec![0u8; 10]; - let result = FileMeta::load(&invalid_data); - assert!(result.is_err()); -} - -#[test] -fn test_file_meta_read_bytes_header() { - // Create a real FileMeta and marshal it to get proper format - let mut fm = FileMeta::new(); - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - let marshaled = fm.marshal_msg().unwrap(); - - // First call check_xl2_v1 to get the buffer after XL header validation - let (after_xl_header, _major, _minor) = FileMeta::check_xl2_v1(&marshaled).unwrap(); - - // Ensure we have at least 5 bytes for read_bytes_header - if after_xl_header.len() < 5 { - panic!("Buffer too small: {} bytes, need at least 5", after_xl_header.len()); - } - - // Now call read_bytes_header on the remaining buffer - let result = FileMeta::read_bytes_header(after_xl_header); - assert!(result.is_ok()); - let (length, remaining) = result.unwrap(); - - // The length should be greater than 0 for real data - assert!(length > 0); - // remaining should be everything after the 5-byte header - assert_eq!(remaining.len(), after_xl_header.len() - 5); - - // Test with buffer too small - let small_buf = vec![0u8; 2]; - let result = FileMeta::read_bytes_header(&small_buf); - assert!(result.is_err()); -} - -#[test] -fn test_file_meta_get_set_idx() { - let mut fm = FileMeta::new(); - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - // Test get_idx - let result = fm.get_idx(0); - assert!(result.is_ok()); - - // Test get_idx with invalid index - let result = fm.get_idx(10); - assert!(result.is_err()); - - // Test set_idx - let new_version = FileMetaVersion { - version_type: VersionType::Object, - ..Default::default() - }; - let result = fm.set_idx(0, new_version); - assert!(result.is_ok()); - - // Test set_idx with invalid index - let invalid_version = FileMetaVersion::default(); - let result = fm.set_idx(10, invalid_version); - assert!(result.is_err()); -} - -#[test] -fn test_file_meta_into_fileinfo() { - let mut fm = FileMeta::new(); - let version_id = Uuid::new_v4(); - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(version_id); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - // Test into_fileinfo with valid version_id - let result = fm.into_fileinfo("test-volume", "test-path", &version_id.to_string(), false, false); - assert!(result.is_ok()); - let file_info = result.unwrap(); - assert_eq!(file_info.volume, "test-volume"); - assert_eq!(file_info.name, "test-path"); - - // Test into_fileinfo with invalid version_id - let invalid_id = Uuid::new_v4(); - let result = fm.into_fileinfo("test-volume", "test-path", &invalid_id.to_string(), false, false); - assert!(result.is_err()); - - // Test into_fileinfo with empty version_id (should get latest) - let result = fm.into_fileinfo("test-volume", "test-path", "", false, false); - assert!(result.is_ok()); -} - -#[test] -fn test_file_meta_into_file_info_versions() { - let mut fm = FileMeta::new(); - - // Add multiple versions - for i in 0..3 { - let mut fi = FileInfo::new(&format!("test{}", i), 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(1000 + i).unwrap()); - fm.add_version(fi).unwrap(); - } - - let result = fm.into_file_info_versions("test-volume", "test-path", false); - assert!(result.is_ok()); - let versions = result.unwrap(); - assert_eq!(versions.versions.len(), 3); -} - -#[test] -fn test_file_meta_shallow_version_to_fileinfo() { - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - - let version = FileMetaVersion::from(fi.clone()); - let shallow_version = FileMetaShallowVersion::try_from(version).unwrap(); - - let result = shallow_version.to_fileinfo("test-volume", "test-path", fi.version_id, false); - assert!(result.is_ok()); - let converted_fi = result.unwrap(); - assert_eq!(converted_fi.volume, "test-volume"); - assert_eq!(converted_fi.name, "test-path"); -} - -#[test] -fn test_file_meta_version_try_from_bytes() { - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - let version = FileMetaVersion::from(fi); - let encoded = version.marshal_msg().unwrap(); - - // Test successful conversion - let result = FileMetaVersion::try_from(encoded.as_slice()); - assert!(result.is_ok()); - - // Test with invalid data - let invalid_data = vec![0u8; 5]; - let result = FileMetaVersion::try_from(invalid_data.as_slice()); - assert!(result.is_err()); -} - -#[test] -fn test_file_meta_version_try_from_shallow() { - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - let version = FileMetaVersion::from(fi); - let shallow = FileMetaShallowVersion::try_from(version.clone()).unwrap(); - - let result = FileMetaVersion::try_from(shallow); - assert!(result.is_ok()); - let converted = result.unwrap(); - assert_eq!(converted.get_version_id(), version.get_version_id()); -} - -#[test] -fn test_file_meta_version_header_from_version() { - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - let version = FileMetaVersion::from(fi.clone()); - - let header = FileMetaVersionHeader::from(version); - assert_eq!(header.version_id, fi.version_id); - assert_eq!(header.mod_time, fi.mod_time); -} - -#[test] -fn test_meta_object_into_fileinfo() { - let obj = MetaObject { - version_id: Some(Uuid::new_v4()), - size: 1024, - mod_time: Some(OffsetDateTime::now_utc()), - ..Default::default() - }; - - let version_id = obj.version_id; - let expected_version_id = version_id; - let file_info = obj.into_fileinfo("test-volume", "test-path", version_id, false); - assert_eq!(file_info.volume, "test-volume"); - assert_eq!(file_info.name, "test-path"); - assert_eq!(file_info.size, 1024); - assert_eq!(file_info.version_id, expected_version_id); -} - -#[test] -fn test_meta_object_from_fileinfo() { - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.data_dir = Some(Uuid::new_v4()); - fi.size = 2048; - fi.mod_time = Some(OffsetDateTime::now_utc()); - - let obj = MetaObject::from(fi.clone()); - assert_eq!(obj.version_id, fi.version_id); - assert_eq!(obj.data_dir, fi.data_dir); - assert_eq!(obj.size, fi.size); - assert_eq!(obj.mod_time, fi.mod_time); -} - -#[test] -fn test_meta_delete_marker_into_fileinfo() { - let marker = MetaDeleteMarker { - version_id: Some(Uuid::new_v4()), - mod_time: Some(OffsetDateTime::now_utc()), - ..Default::default() - }; - - let version_id = marker.version_id; - let expected_version_id = version_id; - let file_info = marker.into_fileinfo("test-volume", "test-path", version_id, false); - assert_eq!(file_info.volume, "test-volume"); - assert_eq!(file_info.name, "test-path"); - assert_eq!(file_info.version_id, expected_version_id); - assert!(file_info.deleted); -} - -#[test] -fn test_meta_delete_marker_from_fileinfo() { - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fi.deleted = true; - - let marker = MetaDeleteMarker::from(fi.clone()); - assert_eq!(marker.version_id, fi.version_id); - assert_eq!(marker.mod_time, fi.mod_time); -} - -#[test] -fn test_flags_enum() { - // Test Flags enum values - assert_eq!(Flags::FreeVersion as u8, 1); - assert_eq!(Flags::UsesDataDir as u8, 2); - assert_eq!(Flags::InlineData as u8, 4); -} - -#[test] -fn test_file_meta_version_header_user_data_dir() { - let header = FileMetaVersionHeader { - flags: 0, - ..Default::default() - }; - - // Test without UsesDataDir flag - assert!(!header.user_data_dir()); - - // Test with UsesDataDir flag - let header = FileMetaVersionHeader { - flags: Flags::UsesDataDir as u8, - ..Default::default() - }; - assert!(header.user_data_dir()); - - // Test with multiple flags including UsesDataDir - let header = FileMetaVersionHeader { - flags: Flags::UsesDataDir as u8 | Flags::FreeVersion as u8, - ..Default::default() - }; - assert!(header.user_data_dir()); -} - -#[test] -fn test_file_meta_version_header_ordering() { - let header1 = FileMetaVersionHeader { - mod_time: Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()), - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - - let header2 = FileMetaVersionHeader { - mod_time: Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()), - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - - // Test partial_cmp - assert!(header1.partial_cmp(&header2).is_some()); - - // Test cmp - header2 should be greater (newer) - use std::cmp::Ordering; - assert_eq!(header1.cmp(&header2), Ordering::Less); // header1 has earlier time - assert_eq!(header2.cmp(&header1), Ordering::Greater); // header2 has later time - assert_eq!(header1.cmp(&header1), Ordering::Equal); -} - -#[test] -fn test_merge_file_meta_versions_edge_cases() { - // Test with empty versions - let empty_versions: Vec> = vec![]; - let merged = merge_file_meta_versions(1, false, 10, &empty_versions); - assert!(merged.is_empty()); - - // Test with quorum larger than available sources - let mut version = FileMetaShallowVersion::default(); - version.header.version_id = Some(Uuid::new_v4()); - let versions = vec![vec![version]]; - let merged = merge_file_meta_versions(5, false, 10, &versions); - assert!(merged.is_empty()); - - // Test strict mode - let mut version1 = FileMetaShallowVersion::default(); - version1.header.version_id = Some(Uuid::new_v4()); - version1.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()); - - let mut version2 = FileMetaShallowVersion::default(); - version2.header.version_id = Some(Uuid::new_v4()); - version2.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()); - - let versions = vec![vec![version1.clone()], vec![version2.clone()]]; - - let _merged_strict = merge_file_meta_versions(1, true, 10, &versions); - let merged_non_strict = merge_file_meta_versions(1, false, 10, &versions); - - // In strict mode, behavior might be different - assert!(!merged_non_strict.is_empty()); -} - -#[tokio::test] -async fn test_read_more_function() { - use std::io::Cursor; - - let data = b"Hello, World! This is test data."; - let mut reader = Cursor::new(data); - let mut buf = vec![0u8; 10]; - - // Test reading more data - let result = read_more(&mut reader, &mut buf, 33, 20, false).await; - assert!(result.is_ok()); - assert_eq!(buf.len(), 20); - - // Test with has_full = true and buffer already has enough data - let mut reader2 = Cursor::new(data); - let mut buf2 = vec![0u8; 5]; - let result = read_more(&mut reader2, &mut buf2, 10, 5, true).await; - assert!(result.is_ok()); - assert_eq!(buf2.len(), 5); // Should remain 5 since has >= read_size - - // Test reading beyond available data - let mut reader3 = Cursor::new(b"short"); - let mut buf3 = vec![0u8; 2]; - let result = read_more(&mut reader3, &mut buf3, 100, 98, false).await; - // Should handle gracefully even if not enough data - assert!(result.is_ok() || result.is_err()); // Either is acceptable -} - -#[tokio::test] -async fn test_read_xl_meta_no_data_edge_cases() { - use std::io::Cursor; - - // Test with empty data - let empty_data = vec![]; - let mut reader = Cursor::new(empty_data); - let result = read_xl_meta_no_data(&mut reader, 0).await; - assert!(result.is_err()); // Should fail because buffer is empty - - // Test with very small size (should fail because it's not valid XL format) - let small_data = vec![1, 2, 3]; - let mut reader = Cursor::new(small_data); - let result = read_xl_meta_no_data(&mut reader, 3).await; - assert!(result.is_err()); // Should fail because data is too small for XL format -} - -#[tokio::test] -async fn test_get_file_info_edge_cases() { - // Test with empty buffer - let empty_buf = vec![]; - let opts = FileInfoOpts { data: false }; - let result = get_file_info(&empty_buf, "volume", "path", "version", opts).await; - assert!(result.is_err()); - - // Test with invalid version_id format - let mut fm = FileMeta::new(); - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - let encoded = fm.marshal_msg().unwrap(); - - let opts = FileInfoOpts { data: false }; - let result = get_file_info(&encoded, "volume", "path", "invalid-uuid", opts).await; - assert!(result.is_err()); -} - -#[tokio::test] -async fn test_file_info_from_raw_edge_cases() { - // Test with empty buffer - let empty_raw = RawFileInfo { buf: vec![] }; - let result = file_info_from_raw(empty_raw, "bucket", "object", false).await; - assert!(result.is_err()); - - // Test with invalid buffer - let invalid_raw = RawFileInfo { - buf: vec![1, 2, 3, 4, 5], - }; - let result = file_info_from_raw(invalid_raw, "bucket", "object", false).await; - assert!(result.is_err()); -} - -#[test] -fn test_file_meta_version_invalid_cases() { - // Test invalid version - let version = FileMetaVersion { - version_type: VersionType::Invalid, - ..Default::default() - }; - assert!(!version.valid()); - - // Test version with neither object nor delete marker - let version = FileMetaVersion { - version_type: VersionType::Object, - object: None, - delete_marker: None, - ..Default::default() - }; - assert!(!version.valid()); -} - -#[test] -fn test_meta_object_edge_cases() { - let obj = MetaObject { - data_dir: None, - ..Default::default() - }; - - // Test use_data_dir with None (use_data_dir always returns true) - assert!(obj.use_data_dir()); - - // Test use_inlinedata (always returns false in current implementation) - let obj = MetaObject { - size: 128 * 1024, // 128KB threshold - ..Default::default() - }; - assert!(!obj.use_inlinedata()); // Should be false - - let obj = MetaObject { - size: 128 * 1024 - 1, - ..Default::default() - }; - assert!(!obj.use_inlinedata()); // Should also be false (always false) -} - -#[test] -fn test_file_meta_version_header_edge_cases() { - let header = FileMetaVersionHeader { - ec_n: 0, - ec_m: 0, - ..Default::default() - }; - - // Test has_ec with zero values - assert!(!header.has_ec()); - - // Test matches_not_strict with different signatures but same version_id - let version_id = Some(Uuid::new_v4()); - let header = FileMetaVersionHeader { - version_id, - version_type: VersionType::Object, - signature: [1, 2, 3, 4], - ..Default::default() - }; - let other = FileMetaVersionHeader { - version_id, - version_type: VersionType::Object, - signature: [5, 6, 7, 8], - ..Default::default() - }; - // Should match because they have same version_id and type - assert!(header.matches_not_strict(&other)); - - // Test sorts_before with same mod_time but different version_id - let time = OffsetDateTime::from_unix_timestamp(1000).unwrap(); - let header_time1 = FileMetaVersionHeader { - mod_time: Some(time), - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - let header_time2 = FileMetaVersionHeader { - mod_time: Some(time), - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - - // Should use version_id for comparison when mod_time is same - let sorts_before = header_time1.sorts_before(&header_time2); - assert!(sorts_before || header_time2.sorts_before(&header_time1)); // One should sort before the other -} - -#[test] -fn test_file_meta_add_version_edge_cases() { - let mut fm = FileMeta::new(); - - // Test adding version with same version_id (should update) - let version_id = Some(Uuid::new_v4()); - let mut fi1 = FileInfo::new("test1", 4, 2); - fi1.version_id = version_id; - fi1.size = 1024; - fi1.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi1).unwrap(); - - let mut fi2 = FileInfo::new("test2", 4, 2); - fi2.version_id = version_id; - fi2.size = 2048; - fi2.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi2).unwrap(); - - // Should still have only one version, but updated - assert_eq!(fm.versions.len(), 1); - let (_, version) = fm.find_version(version_id).unwrap(); - if let Some(obj) = version.object { - assert_eq!(obj.size, 2048); // Size gets updated when adding same version_id - } -} - -#[test] -fn test_file_meta_delete_version_edge_cases() { - let mut fm = FileMeta::new(); - - // Test deleting non-existent version - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - - let result = fm.delete_version(&fi); - assert!(result.is_err()); // Should fail for non-existent version -} - -#[test] -fn test_file_meta_shard_data_dir_count_edge_cases() { - let mut fm = FileMeta::new(); - - // Test with None data_dir parameter - let count = fm.shard_data_dir_count(&None, &None); - assert_eq!(count, 0); - - // Test with version_id parameter (not None) - let version_id = Some(Uuid::new_v4()); - let data_dir = Some(Uuid::new_v4()); - - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = version_id; - fi.data_dir = data_dir; - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - let count = fm.shard_data_dir_count(&version_id, &data_dir); - assert_eq!(count, 0); // Should be 0 because user_data_dir() requires flag - - // Test with different version_id - let other_version_id = Some(Uuid::new_v4()); - let count = fm.shard_data_dir_count(&other_version_id, &data_dir); - assert_eq!(count, 1); // Should be 1 because the version has matching data_dir and user_data_dir() is true -} +// // impl Display for FileMeta { +// // fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +// // f.write_str("FileMeta:")?; +// // for (i, ver) in self.versions.iter().enumerate() { +// // let mut meta = FileMetaVersion::default(); +// // meta.unmarshal_msg(&ver.meta).unwrap_or_default(); +// // f.write_fmt(format_args!("ver:{} header {:?}, meta {:?}", i, ver.header, meta))?; +// // } + +// // f.write_str("\n") +// // } +// // } + +// #[derive(Serialize, Deserialize, Debug, Default, PartialEq, Clone, Eq, PartialOrd, Ord)] +// pub struct FileMetaShallowVersion { +// pub header: FileMetaVersionHeader, +// pub meta: Vec, // FileMetaVersion.marshal_msg +// } + +// impl FileMetaShallowVersion { +// pub fn to_fileinfo(&self, volume: &str, path: &str, version_id: Option, all_parts: bool) -> Result { +// let file_version = FileMetaVersion::try_from(self.meta.as_slice())?; + +// Ok(file_version.to_fileinfo(volume, path, version_id, all_parts)) +// } +// } + +// impl TryFrom for FileMetaShallowVersion { +// type Error = Error; + +// fn try_from(value: FileMetaVersion) -> std::result::Result { +// let header = value.header(); +// let meta = value.marshal_msg()?; +// Ok(Self { meta, header }) +// } +// } + +// #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)] +// pub struct FileMetaVersion { +// pub version_type: VersionType, +// pub object: Option, +// pub delete_marker: Option, +// pub write_version: u64, // rustfs version +// } + +// impl FileMetaVersion { +// pub fn valid(&self) -> bool { +// if !self.version_type.valid() { +// return false; +// } + +// match self.version_type { +// VersionType::Object => self +// .object +// .as_ref() +// .map(|v| v.erasure_algorithm.valid() && v.bitrot_checksum_algo.valid() && v.mod_time.is_some()) +// .unwrap_or_default(), +// VersionType::Delete => self +// .delete_marker +// .as_ref() +// .map(|v| v.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) > OffsetDateTime::UNIX_EPOCH) +// .unwrap_or_default(), +// _ => false, +// } +// } + +// pub fn get_data_dir(&self) -> Option { +// self.valid() +// .then(|| { +// if self.version_type == VersionType::Object { +// self.object.as_ref().map(|v| v.data_dir).unwrap_or_default() +// } else { +// None +// } +// }) +// .unwrap_or_default() +// } + +// pub fn get_version_id(&self) -> Option { +// match self.version_type { +// VersionType::Object | VersionType::Delete => self.object.as_ref().map(|v| v.version_id).unwrap_or_default(), +// _ => None, +// } +// } + +// pub fn get_mod_time(&self) -> Option { +// match self.version_type { +// VersionType::Object => self.object.as_ref().map(|v| v.mod_time).unwrap_or_default(), +// VersionType::Delete => self.delete_marker.as_ref().map(|v| v.mod_time).unwrap_or_default(), +// _ => None, +// } +// } + +// // decode_data_dir_from_meta 从 meta 中读取 data_dir TODO: 直接从 meta buf 中只解析出 data_dir, msg.skip +// pub fn decode_data_dir_from_meta(buf: &[u8]) -> Result> { +// let mut ver = Self::default(); +// ver.unmarshal_msg(buf)?; + +// let data_dir = ver.object.map(|v| v.data_dir).unwrap_or_default(); +// Ok(data_dir) +// } + +// pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { +// let mut cur = Cursor::new(buf); + +// let mut fields_len = rmp::decode::read_map_len(&mut cur)?; + +// while fields_len > 0 { +// fields_len -= 1; + +// // println!("unmarshal_msg fields idx {}", fields_len); + +// let str_len = rmp::decode::read_str_len(&mut cur)?; + +// // println!("unmarshal_msg fields name len() {}", &str_len); + +// // ！！！Vec::with_capacity(str_len) 失败，vec! 正常 +// let mut field_buff = vec![0u8; str_len as usize]; + +// cur.read_exact(&mut field_buff)?; + +// let field = String::from_utf8(field_buff)?; + +// // println!("unmarshal_msg fields name {}", &field); + +// match field.as_str() { +// "Type" => { +// let u: u8 = rmp::decode::read_int(&mut cur)?; +// self.version_type = VersionType::from_u8(u); +// } + +// "V2Obj" => { +// // is_nil() +// if buf[cur.position() as usize] == 0xc0 { +// rmp::decode::read_nil(&mut cur)?; +// } else { +// // let buf = unsafe { cur.position() }; +// let mut obj = MetaObject::default(); +// // let start = cur.position(); + +// let (_, remain) = buf.split_at(cur.position() as usize); + +// let read_len = obj.unmarshal_msg(remain)?; +// cur.set_position(cur.position() + read_len); + +// self.object = Some(obj); +// } +// } +// "DelObj" => { +// if buf[cur.position() as usize] == 0xc0 { +// rmp::decode::read_nil(&mut cur)?; +// } else { +// // let buf = unsafe { cur.position() }; +// let mut obj = MetaDeleteMarker::default(); +// // let start = cur.position(); + +// let (_, remain) = buf.split_at(cur.position() as usize); +// let read_len = obj.unmarshal_msg(remain)?; +// cur.set_position(cur.position() + read_len); + +// self.delete_marker = Some(obj); +// } +// } +// "v" => { +// self.write_version = rmp::decode::read_int(&mut cur)?; +// } +// name => return Err(Error::msg(format!("not suport field name {}", name))), +// } +// } + +// Ok(cur.position()) +// } + +// pub fn marshal_msg(&self) -> Result> { +// let mut len: u32 = 4; +// let mut mask: u8 = 0; + +// if self.object.is_none() { +// len -= 1; +// mask |= 0x2; +// } +// if self.delete_marker.is_none() { +// len -= 1; +// mask |= 0x4; +// } + +// let mut wr = Vec::new(); + +// // 字段数量 +// rmp::encode::write_map_len(&mut wr, len)?; + +// // write "Type" +// rmp::encode::write_str(&mut wr, "Type")?; +// rmp::encode::write_uint(&mut wr, self.version_type.to_u8() as u64)?; + +// if (mask & 0x2) == 0 { +// // write V2Obj +// rmp::encode::write_str(&mut wr, "V2Obj")?; +// if self.object.is_none() { +// let _ = rmp::encode::write_nil(&mut wr); +// } else { +// let buf = self.object.as_ref().unwrap().marshal_msg()?; +// wr.write_all(&buf)?; +// } +// } + +// if (mask & 0x4) == 0 { +// // write "DelObj" +// rmp::encode::write_str(&mut wr, "DelObj")?; +// if self.delete_marker.is_none() { +// let _ = rmp::encode::write_nil(&mut wr); +// } else { +// let buf = self.delete_marker.as_ref().unwrap().marshal_msg()?; +// wr.write_all(&buf)?; +// } +// } + +// // write "v" +// rmp::encode::write_str(&mut wr, "v")?; +// rmp::encode::write_uint(&mut wr, self.write_version)?; + +// Ok(wr) +// } + +// pub fn free_version(&self) -> bool { +// self.version_type == VersionType::Delete && self.delete_marker.as_ref().map(|m| m.free_version()).unwrap_or_default() +// } + +// pub fn header(&self) -> FileMetaVersionHeader { +// FileMetaVersionHeader::from(self.clone()) +// } + +// pub fn to_fileinfo(&self, volume: &str, path: &str, version_id: Option, all_parts: bool) -> FileInfo { +// match self.version_type { +// VersionType::Invalid => FileInfo { +// name: path.to_string(), +// volume: volume.to_string(), +// version_id, +// ..Default::default() +// }, +// VersionType::Object => self +// .object +// .as_ref() +// .unwrap() +// .clone() +// .into_fileinfo(volume, path, version_id, all_parts), +// VersionType::Delete => self +// .delete_marker +// .as_ref() +// .unwrap() +// .clone() +// .into_fileinfo(volume, path, version_id, all_parts), +// } +// } +// } + +// impl TryFrom<&[u8]> for FileMetaVersion { +// type Error = Error; + +// fn try_from(value: &[u8]) -> std::result::Result { +// let mut ver = FileMetaVersion::default(); +// ver.unmarshal_msg(value)?; +// Ok(ver) +// } +// } + +// impl From for FileMetaVersion { +// fn from(value: FileInfo) -> Self { +// { +// if value.deleted { +// FileMetaVersion { +// version_type: VersionType::Delete, +// delete_marker: Some(MetaDeleteMarker::from(value)), +// object: None, +// write_version: 0, +// } +// } else { +// FileMetaVersion { +// version_type: VersionType::Object, +// delete_marker: None, +// object: Some(MetaObject::from(value)), +// write_version: 0, +// } +// } +// } +// } +// } + +// impl TryFrom for FileMetaVersion { +// type Error = Error; + +// fn try_from(value: FileMetaShallowVersion) -> std::result::Result { +// FileMetaVersion::try_from(value.meta.as_slice()) +// } +// } + +// #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone, Eq, Hash)] +// pub struct FileMetaVersionHeader { +// pub version_id: Option, +// pub mod_time: Option, +// pub signature: [u8; 4], +// pub version_type: VersionType, +// pub flags: u8, +// pub ec_n: u8, +// pub ec_m: u8, +// } + +// impl FileMetaVersionHeader { +// pub fn has_ec(&self) -> bool { +// self.ec_m > 0 && self.ec_n > 0 +// } + +// pub fn matches_not_strict(&self, o: &FileMetaVersionHeader) -> bool { +// let mut ok = self.version_id == o.version_id && self.version_type == o.version_type && self.matches_ec(o); +// if self.version_id.is_none() { +// ok = ok && self.mod_time == o.mod_time; +// } + +// ok +// } + +// pub fn matches_ec(&self, o: &FileMetaVersionHeader) -> bool { +// if self.has_ec() && o.has_ec() { +// return self.ec_n == o.ec_n && self.ec_m == o.ec_m; +// } + +// true +// } + +// pub fn free_version(&self) -> bool { +// self.flags & XL_FLAG_FREE_VERSION != 0 +// } + +// pub fn sorts_before(&self, o: &FileMetaVersionHeader) -> bool { +// if self == o { +// return false; +// } + +// // Prefer newest modtime. +// if self.mod_time != o.mod_time { +// return self.mod_time > o.mod_time; +// } + +// match self.mod_time.cmp(&o.mod_time) { +// Ordering::Greater => { +// return true; +// } +// Ordering::Less => { +// return false; +// } +// _ => {} +// } + +// // The following doesn't make too much sense, but we want sort to be consistent nonetheless. +// // Prefer lower types +// if self.version_type != o.version_type { +// return self.version_type < o.version_type; +// } +// // Consistent sort on signature +// match self.version_id.cmp(&o.version_id) { +// Ordering::Greater => { +// return true; +// } +// Ordering::Less => { +// return false; +// } +// _ => {} +// } + +// if self.flags != o.flags { +// return self.flags > o.flags; +// } + +// false +// } + +// pub fn user_data_dir(&self) -> bool { +// self.flags & Flags::UsesDataDir as u8 != 0 +// } +// #[tracing::instrument] +// pub fn marshal_msg(&self) -> Result> { +// let mut wr = Vec::new(); + +// // array len 7 +// rmp::encode::write_array_len(&mut wr, 7)?; + +// // version_id +// rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or_default().as_bytes())?; +// // mod_time +// rmp::encode::write_i64(&mut wr, self.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp_nanos() as i64)?; +// // signature +// rmp::encode::write_bin(&mut wr, self.signature.as_slice())?; +// // version_type +// rmp::encode::write_uint8(&mut wr, self.version_type.to_u8())?; +// // flags +// rmp::encode::write_uint8(&mut wr, self.flags)?; +// // ec_n +// rmp::encode::write_uint8(&mut wr, self.ec_n)?; +// // ec_m +// rmp::encode::write_uint8(&mut wr, self.ec_m)?; + +// Ok(wr) +// } + +// pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { +// let mut cur = Cursor::new(buf); +// let alen = rmp::decode::read_array_len(&mut cur)?; +// if alen != 7 { +// return Err(Error::msg(format!("version header array len err need 7 got {}", alen))); +// } + +// // version_id +// rmp::decode::read_bin_len(&mut cur)?; +// let mut buf = [0u8; 16]; +// cur.read_exact(&mut buf)?; +// self.version_id = { +// let id = Uuid::from_bytes(buf); +// if id.is_nil() { +// None +// } else { +// Some(id) +// } +// }; + +// // mod_time +// let unix: i128 = rmp::decode::read_int(&mut cur)?; + +// let time = OffsetDateTime::from_unix_timestamp_nanos(unix)?; +// if time == OffsetDateTime::UNIX_EPOCH { +// self.mod_time = None; +// } else { +// self.mod_time = Some(time); +// } + +// // signature +// rmp::decode::read_bin_len(&mut cur)?; +// cur.read_exact(&mut self.signature)?; + +// // version_type +// let typ: u8 = rmp::decode::read_int(&mut cur)?; +// self.version_type = VersionType::from_u8(typ); + +// // flags +// self.flags = rmp::decode::read_int(&mut cur)?; +// // ec_n +// self.ec_n = rmp::decode::read_int(&mut cur)?; +// // ec_m +// self.ec_m = rmp::decode::read_int(&mut cur)?; + +// Ok(cur.position()) +// } +// } + +// impl PartialOrd for FileMetaVersionHeader { +// fn partial_cmp(&self, other: &Self) -> Option { +// Some(self.cmp(other)) +// } +// } + +// impl Ord for FileMetaVersionHeader { +// fn cmp(&self, other: &Self) -> Ordering { +// match self.mod_time.cmp(&other.mod_time) { +// Ordering::Equal => {} +// ord => return ord, +// } + +// match self.version_type.cmp(&other.version_type) { +// Ordering::Equal => {} +// ord => return ord, +// } +// match self.signature.cmp(&other.signature) { +// Ordering::Equal => {} +// ord => return ord, +// } +// match self.version_id.cmp(&other.version_id) { +// Ordering::Equal => {} +// ord => return ord, +// } +// self.flags.cmp(&other.flags) +// } +// } + +// impl From for FileMetaVersionHeader { +// fn from(value: FileMetaVersion) -> Self { +// let flags = { +// let mut f: u8 = 0; +// if value.free_version() { +// f |= Flags::FreeVersion as u8; +// } + +// if value.version_type == VersionType::Object && value.object.as_ref().map(|v| v.use_data_dir()).unwrap_or_default() { +// f |= Flags::UsesDataDir as u8; +// } + +// if value.version_type == VersionType::Object && value.object.as_ref().map(|v| v.use_inlinedata()).unwrap_or_default() +// { +// f |= Flags::InlineData as u8; +// } + +// f +// }; + +// let (ec_n, ec_m) = { +// if value.version_type == VersionType::Object && value.object.is_some() { +// ( +// value.object.as_ref().unwrap().erasure_n as u8, +// value.object.as_ref().unwrap().erasure_m as u8, +// ) +// } else { +// (0, 0) +// } +// }; + +// Self { +// version_id: value.get_version_id(), +// mod_time: value.get_mod_time(), +// signature: [0, 0, 0, 0], +// version_type: value.version_type, +// flags, +// ec_n, +// ec_m, +// } +// } +// } + +// #[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)] +// // 因为自定义 message_pack，所以一定要保证字段顺序 +// pub struct MetaObject { +// pub version_id: Option, // Version ID +// pub data_dir: Option, // Data dir ID +// pub erasure_algorithm: ErasureAlgo, // Erasure coding algorithm +// pub erasure_m: usize, // Erasure data blocks +// pub erasure_n: usize, // Erasure parity blocks +// pub erasure_block_size: usize, // Erasure block size +// pub erasure_index: usize, // Erasure disk index +// pub erasure_dist: Vec, // Erasure distribution +// pub bitrot_checksum_algo: ChecksumAlgo, // Bitrot checksum algo +// pub part_numbers: Vec, // Part Numbers +// pub part_etags: Option>, // Part ETags +// pub part_sizes: Vec, // Part Sizes +// pub part_actual_sizes: Option>, // Part ActualSizes (compression) +// pub part_indices: Option>>, // Part Indexes (compression) +// pub size: usize, // Object version size +// pub mod_time: Option, // Object version modified time +// pub meta_sys: Option>>, // Object version internal metadata +// pub meta_user: Option>, // Object version metadata set by user +// } + +// impl MetaObject { +// pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { +// let mut cur = Cursor::new(buf); + +// let mut fields_len = rmp::decode::read_map_len(&mut cur)?; + +// // let mut ret = Self::default(); + +// while fields_len > 0 { +// fields_len -= 1; + +// // println!("unmarshal_msg fields idx {}", fields_len); + +// let str_len = rmp::decode::read_str_len(&mut cur)?; + +// // println!("unmarshal_msg fields name len() {}", &str_len); + +// // ！！！Vec::with_capacity(str_len) 失败，vec! 正常 +// let mut field_buff = vec![0u8; str_len as usize]; + +// cur.read_exact(&mut field_buff)?; + +// let field = String::from_utf8(field_buff)?; + +// // println!("unmarshal_msg fields name {}", &field); + +// match field.as_str() { +// "ID" => { +// rmp::decode::read_bin_len(&mut cur)?; +// let mut buf = [0u8; 16]; +// cur.read_exact(&mut buf)?; +// self.version_id = { +// let id = Uuid::from_bytes(buf); +// if id.is_nil() { +// None +// } else { +// Some(id) +// } +// }; +// } +// "DDir" => { +// rmp::decode::read_bin_len(&mut cur)?; +// let mut buf = [0u8; 16]; +// cur.read_exact(&mut buf)?; +// self.data_dir = { +// let id = Uuid::from_bytes(buf); +// if id.is_nil() { +// None +// } else { +// Some(id) +// } +// }; +// } +// "EcAlgo" => { +// let u: u8 = rmp::decode::read_int(&mut cur)?; +// self.erasure_algorithm = ErasureAlgo::from_u8(u) +// } +// "EcM" => { +// self.erasure_m = rmp::decode::read_int(&mut cur)?; +// } +// "EcN" => { +// self.erasure_n = rmp::decode::read_int(&mut cur)?; +// } +// "EcBSize" => { +// self.erasure_block_size = rmp::decode::read_int(&mut cur)?; +// } +// "EcIndex" => { +// self.erasure_index = rmp::decode::read_int(&mut cur)?; +// } +// "EcDist" => { +// let alen = rmp::decode::read_array_len(&mut cur)? as usize; +// self.erasure_dist = vec![0u8; alen]; +// for i in 0..alen { +// self.erasure_dist[i] = rmp::decode::read_int(&mut cur)?; +// } +// } +// "CSumAlgo" => { +// let u: u8 = rmp::decode::read_int(&mut cur)?; +// self.bitrot_checksum_algo = ChecksumAlgo::from_u8(u) +// } +// "PartNums" => { +// let alen = rmp::decode::read_array_len(&mut cur)? as usize; +// self.part_numbers = vec![0; alen]; +// for i in 0..alen { +// self.part_numbers[i] = rmp::decode::read_int(&mut cur)?; +// } +// } +// "PartETags" => { +// let array_len = match rmp::decode::read_nil(&mut cur) { +// Ok(_) => None, +// Err(e) => match e { +// rmp::decode::ValueReadError::TypeMismatch(marker) => match marker { +// Marker::FixArray(l) => Some(l as usize), +// Marker::Array16 => Some(rmp::decode::read_u16(&mut cur)? as usize), +// Marker::Array32 => Some(rmp::decode::read_u16(&mut cur)? as usize), +// _ => return Err(Error::msg("PartETags parse failed")), +// }, +// _ => return Err(Error::msg("PartETags parse failed.")), +// }, +// }; + +// if array_len.is_some() { +// let l = array_len.unwrap(); +// let mut etags = Vec::with_capacity(l); +// for _ in 0..l { +// let str_len = rmp::decode::read_str_len(&mut cur)?; +// let mut field_buff = vec![0u8; str_len as usize]; +// cur.read_exact(&mut field_buff)?; +// etags.push(String::from_utf8(field_buff)?); +// } +// self.part_etags = Some(etags); +// } +// } +// "PartSizes" => { +// let alen = rmp::decode::read_array_len(&mut cur)? as usize; +// self.part_sizes = vec![0; alen]; +// for i in 0..alen { +// self.part_sizes[i] = rmp::decode::read_int(&mut cur)?; +// } +// } +// "PartASizes" => { +// let array_len = match rmp::decode::read_nil(&mut cur) { +// Ok(_) => None, +// Err(e) => match e { +// rmp::decode::ValueReadError::TypeMismatch(marker) => match marker { +// Marker::FixArray(l) => Some(l as usize), +// Marker::Array16 => Some(rmp::decode::read_u16(&mut cur)? as usize), +// Marker::Array32 => Some(rmp::decode::read_u16(&mut cur)? as usize), +// _ => return Err(Error::msg("PartETags parse failed")), +// }, +// _ => return Err(Error::msg("PartETags parse failed.")), +// }, +// }; +// if let Some(l) = array_len { +// let mut sizes = vec![0; l]; +// for size in sizes.iter_mut().take(l) { +// *size = rmp::decode::read_int(&mut cur)?; +// } +// // for size in sizes.iter_mut().take(l) { +// // let tmp = rmp::decode::read_int(&mut cur)?; +// // size = tmp; +// // } +// self.part_actual_sizes = Some(sizes); +// } +// } +// "PartIdx" => { +// let alen = rmp::decode::read_array_len(&mut cur)? as usize; + +// if alen == 0 { +// self.part_indices = None; +// continue; +// } + +// let mut indices = Vec::with_capacity(alen); +// for _ in 0..alen { +// let blen = rmp::decode::read_bin_len(&mut cur)?; +// let mut buf = vec![0u8; blen as usize]; +// cur.read_exact(&mut buf)?; + +// indices.push(buf); +// } + +// self.part_indices = Some(indices); +// } +// "Size" => { +// self.size = rmp::decode::read_int(&mut cur)?; +// } +// "MTime" => { +// let unix: i128 = rmp::decode::read_int(&mut cur)?; +// let time = OffsetDateTime::from_unix_timestamp_nanos(unix)?; +// if time == OffsetDateTime::UNIX_EPOCH { +// self.mod_time = None; +// } else { +// self.mod_time = Some(time); +// } +// } +// "MetaSys" => { +// let len = match rmp::decode::read_nil(&mut cur) { +// Ok(_) => None, +// Err(e) => match e { +// rmp::decode::ValueReadError::TypeMismatch(marker) => match marker { +// Marker::FixMap(l) => Some(l as usize), +// Marker::Map16 => Some(rmp::decode::read_u16(&mut cur)? as usize), +// Marker::Map32 => Some(rmp::decode::read_u16(&mut cur)? as usize), +// _ => return Err(Error::msg("MetaSys parse failed")), +// }, +// _ => return Err(Error::msg("MetaSys parse failed.")), +// }, +// }; +// if len.is_some() { +// let l = len.unwrap(); +// let mut map = HashMap::new(); +// for _ in 0..l { +// let str_len = rmp::decode::read_str_len(&mut cur)?; +// let mut field_buff = vec![0u8; str_len as usize]; +// cur.read_exact(&mut field_buff)?; +// let key = String::from_utf8(field_buff)?; + +// let blen = rmp::decode::read_bin_len(&mut cur)?; +// let mut val = vec![0u8; blen as usize]; +// cur.read_exact(&mut val)?; + +// map.insert(key, val); +// } + +// self.meta_sys = Some(map); +// } +// } +// "MetaUsr" => { +// let len = match rmp::decode::read_nil(&mut cur) { +// Ok(_) => None, +// Err(e) => match e { +// rmp::decode::ValueReadError::TypeMismatch(marker) => match marker { +// Marker::FixMap(l) => Some(l as usize), +// Marker::Map16 => Some(rmp::decode::read_u16(&mut cur)? as usize), +// Marker::Map32 => Some(rmp::decode::read_u16(&mut cur)? as usize), +// _ => return Err(Error::msg("MetaUsr parse failed")), +// }, +// _ => return Err(Error::msg("MetaUsr parse failed.")), +// }, +// }; +// if len.is_some() { +// let l = len.unwrap(); +// let mut map = HashMap::new(); +// for _ in 0..l { +// let str_len = rmp::decode::read_str_len(&mut cur)?; +// let mut field_buff = vec![0u8; str_len as usize]; +// cur.read_exact(&mut field_buff)?; +// let key = String::from_utf8(field_buff)?; + +// let blen = rmp::decode::read_str_len(&mut cur)?; +// let mut val_buf = vec![0u8; blen as usize]; +// cur.read_exact(&mut val_buf)?; +// let val = String::from_utf8(val_buf)?; + +// map.insert(key, val); +// } + +// self.meta_user = Some(map); +// } +// } + +// name => return Err(Error::msg(format!("not suport field name {}", name))), +// } +// } + +// Ok(cur.position()) +// } +// // marshal_msg 自定义 messagepack 命名与 go 一致 +// pub fn marshal_msg(&self) -> Result> { +// let mut len: u32 = 18; +// let mut mask: u32 = 0; + +// if self.part_indices.is_none() { +// len -= 1; +// mask |= 0x2000; +// } + +// let mut wr = Vec::new(); + +// // 字段数量 +// rmp::encode::write_map_len(&mut wr, len)?; + +// // string "ID" +// rmp::encode::write_str(&mut wr, "ID")?; +// rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or_default().as_bytes())?; + +// // string "DDir" +// rmp::encode::write_str(&mut wr, "DDir")?; +// rmp::encode::write_bin(&mut wr, self.data_dir.unwrap_or_default().as_bytes())?; + +// // string "EcAlgo" +// rmp::encode::write_str(&mut wr, "EcAlgo")?; +// rmp::encode::write_uint(&mut wr, self.erasure_algorithm.to_u8() as u64)?; + +// // string "EcM" +// rmp::encode::write_str(&mut wr, "EcM")?; +// rmp::encode::write_uint(&mut wr, self.erasure_m.try_into().unwrap())?; + +// // string "EcN" +// rmp::encode::write_str(&mut wr, "EcN")?; +// rmp::encode::write_uint(&mut wr, self.erasure_n.try_into().unwrap())?; + +// // string "EcBSize" +// rmp::encode::write_str(&mut wr, "EcBSize")?; +// rmp::encode::write_uint(&mut wr, self.erasure_block_size.try_into().unwrap())?; + +// // string "EcIndex" +// rmp::encode::write_str(&mut wr, "EcIndex")?; +// rmp::encode::write_uint(&mut wr, self.erasure_index.try_into().unwrap())?; + +// // string "EcDist" +// rmp::encode::write_str(&mut wr, "EcDist")?; +// rmp::encode::write_array_len(&mut wr, self.erasure_dist.len() as u32)?; +// for v in self.erasure_dist.iter() { +// rmp::encode::write_uint(&mut wr, *v as _)?; +// } + +// // string "CSumAlgo" +// rmp::encode::write_str(&mut wr, "CSumAlgo")?; +// rmp::encode::write_uint(&mut wr, self.bitrot_checksum_algo.to_u8() as u64)?; + +// // string "PartNums" +// rmp::encode::write_str(&mut wr, "PartNums")?; +// rmp::encode::write_array_len(&mut wr, self.part_numbers.len() as u32)?; +// for v in self.part_numbers.iter() { +// rmp::encode::write_uint(&mut wr, *v as _)?; +// } + +// // string "PartETags" +// rmp::encode::write_str(&mut wr, "PartETags")?; +// if self.part_etags.is_none() { +// rmp::encode::write_nil(&mut wr)?; +// } else { +// let etags = self.part_etags.as_ref().unwrap(); +// rmp::encode::write_array_len(&mut wr, etags.len() as u32)?; +// for v in etags.iter() { +// rmp::encode::write_str(&mut wr, v.as_str())?; +// } +// } + +// // string "PartSizes" +// rmp::encode::write_str(&mut wr, "PartSizes")?; +// rmp::encode::write_array_len(&mut wr, self.part_sizes.len() as u32)?; +// for v in self.part_sizes.iter() { +// rmp::encode::write_uint(&mut wr, *v as _)?; +// } + +// // string "PartASizes" +// rmp::encode::write_str(&mut wr, "PartASizes")?; +// if self.part_actual_sizes.is_none() { +// rmp::encode::write_nil(&mut wr)?; +// } else { +// let asizes = self.part_actual_sizes.as_ref().unwrap(); +// rmp::encode::write_array_len(&mut wr, asizes.len() as u32)?; +// for v in asizes.iter() { +// rmp::encode::write_uint(&mut wr, *v as _)?; +// } +// } + +// if (mask & 0x2000) == 0 { +// // string "PartIdx" +// rmp::encode::write_str(&mut wr, "PartIdx")?; +// let indices = self.part_indices.as_ref().unwrap(); +// rmp::encode::write_array_len(&mut wr, indices.len() as u32)?; +// for v in indices.iter() { +// rmp::encode::write_bin(&mut wr, v)?; +// } +// } + +// // string "Size" +// rmp::encode::write_str(&mut wr, "Size")?; +// rmp::encode::write_uint(&mut wr, self.size.try_into().unwrap())?; + +// // string "MTime" +// rmp::encode::write_str(&mut wr, "MTime")?; +// rmp::encode::write_uint( +// &mut wr, +// self.mod_time +// .unwrap_or(OffsetDateTime::UNIX_EPOCH) +// .unix_timestamp_nanos() +// .try_into() +// .unwrap(), +// )?; + +// // string "MetaSys" +// rmp::encode::write_str(&mut wr, "MetaSys")?; +// if self.meta_sys.is_none() { +// rmp::encode::write_nil(&mut wr)?; +// } else { +// let metas = self.meta_sys.as_ref().unwrap(); +// rmp::encode::write_map_len(&mut wr, metas.len() as u32)?; +// for (k, v) in metas { +// rmp::encode::write_str(&mut wr, k.as_str())?; +// rmp::encode::write_bin(&mut wr, v)?; +// } +// } + +// // string "MetaUsr" +// rmp::encode::write_str(&mut wr, "MetaUsr")?; +// if self.meta_user.is_none() { +// rmp::encode::write_nil(&mut wr)?; +// } else { +// let metas = self.meta_user.as_ref().unwrap(); +// rmp::encode::write_map_len(&mut wr, metas.len() as u32)?; +// for (k, v) in metas { +// rmp::encode::write_str(&mut wr, k.as_str())?; +// rmp::encode::write_str(&mut wr, v.as_str())?; +// } +// } + +// Ok(wr) +// } +// pub fn use_data_dir(&self) -> bool { +// // TODO: when use inlinedata +// true +// } + +// pub fn use_inlinedata(&self) -> bool { +// // TODO: when use inlinedata +// false +// } + +// pub fn into_fileinfo(self, volume: &str, path: &str, _version_id: Option, _all_parts: bool) -> FileInfo { +// let version_id = self.version_id; + +// let erasure = ErasureInfo { +// algorithm: self.erasure_algorithm.to_string(), +// data_blocks: self.erasure_m, +// parity_blocks: self.erasure_n, +// block_size: self.erasure_block_size, +// index: self.erasure_index, +// distribution: self.erasure_dist.iter().map(|&v| v as usize).collect(), +// ..Default::default() +// }; + +// let mut parts = Vec::new(); +// for (i, _) in self.part_numbers.iter().enumerate() { +// parts.push(ObjectPartInfo { +// number: self.part_numbers[i], +// size: self.part_sizes[i], +// ..Default::default() +// }); +// } + +// let metadata = { +// if let Some(metauser) = self.meta_user.as_ref() { +// let mut m = HashMap::new(); +// for (k, v) in metauser { +// // TODO: skip xhttp x-amz-storage-class +// m.insert(k.to_owned(), v.to_owned()); +// } +// Some(m) +// } else { +// None +// } +// }; + +// FileInfo { +// version_id, +// erasure, +// data_dir: self.data_dir, +// mod_time: self.mod_time, +// size: self.size, +// name: path.to_string(), +// volume: volume.to_string(), +// parts, +// metadata, +// ..Default::default() +// } +// } +// } + +// impl From for MetaObject { +// fn from(value: FileInfo) -> Self { +// let part_numbers: Vec = value.parts.iter().map(|v| v.number).collect(); +// let part_sizes: Vec = value.parts.iter().map(|v| v.size).collect(); + +// Self { +// version_id: value.version_id, +// size: value.size, +// mod_time: value.mod_time, +// data_dir: value.data_dir, +// erasure_algorithm: ErasureAlgo::ReedSolomon, +// erasure_m: value.erasure.data_blocks, +// erasure_n: value.erasure.parity_blocks, +// erasure_block_size: value.erasure.block_size, +// erasure_index: value.erasure.index, +// erasure_dist: value.erasure.distribution.iter().map(|x| *x as u8).collect(), +// bitrot_checksum_algo: ChecksumAlgo::HighwayHash, +// part_numbers, +// part_etags: None, // TODO: add part_etags +// part_sizes, +// part_actual_sizes: None, // TODO: add part_etags +// part_indices: None, +// meta_sys: None, +// meta_user: value.metadata.clone(), +// } +// } +// } + +// #[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)] +// pub struct MetaDeleteMarker { +// pub version_id: Option, // Version ID for delete marker +// pub mod_time: Option, // Object delete marker modified time +// pub meta_sys: Option>>, // Delete marker internal metadata +// } + +// impl MetaDeleteMarker { +// pub fn free_version(&self) -> bool { +// self.meta_sys +// .as_ref() +// .map(|v| v.get(FREE_VERSION_META_HEADER).is_some()) +// .unwrap_or_default() +// } + +// pub fn into_fileinfo(self, volume: &str, path: &str, version_id: Option, _all_parts: bool) -> FileInfo { +// FileInfo { +// name: path.to_string(), +// volume: volume.to_string(), +// version_id, +// deleted: true, +// mod_time: self.mod_time, +// ..Default::default() +// } +// } + +// pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { +// let mut cur = Cursor::new(buf); + +// let mut fields_len = rmp::decode::read_map_len(&mut cur)?; + +// while fields_len > 0 { +// fields_len -= 1; + +// let str_len = rmp::decode::read_str_len(&mut cur)?; + +// // ！！！Vec::with_capacity(str_len) 失败，vec! 正常 +// let mut field_buff = vec![0u8; str_len as usize]; + +// cur.read_exact(&mut field_buff)?; + +// let field = String::from_utf8(field_buff)?; + +// match field.as_str() { +// "ID" => { +// rmp::decode::read_bin_len(&mut cur)?; +// let mut buf = [0u8; 16]; +// cur.read_exact(&mut buf)?; +// self.version_id = { +// let id = Uuid::from_bytes(buf); +// if id.is_nil() { +// None +// } else { +// Some(id) +// } +// }; +// } + +// "MTime" => { +// let unix: i64 = rmp::decode::read_int(&mut cur)?; +// let time = OffsetDateTime::from_unix_timestamp(unix)?; +// if time == OffsetDateTime::UNIX_EPOCH { +// self.mod_time = None; +// } else { +// self.mod_time = Some(time); +// } +// } +// "MetaSys" => { +// let l = rmp::decode::read_map_len(&mut cur)?; +// let mut map = HashMap::new(); +// for _ in 0..l { +// let str_len = rmp::decode::read_str_len(&mut cur)?; +// let mut field_buff = vec![0u8; str_len as usize]; +// cur.read_exact(&mut field_buff)?; +// let key = String::from_utf8(field_buff)?; + +// let blen = rmp::decode::read_bin_len(&mut cur)?; +// let mut val = vec![0u8; blen as usize]; +// cur.read_exact(&mut val)?; + +// map.insert(key, val); +// } + +// self.meta_sys = Some(map); +// } +// name => return Err(Error::msg(format!("not suport field name {}", name))), +// } +// } + +// Ok(cur.position()) +// } + +// pub fn marshal_msg(&self) -> Result> { +// let mut len: u32 = 3; +// let mut mask: u8 = 0; + +// if self.meta_sys.is_none() { +// len -= 1; +// mask |= 0x4; +// } + +// let mut wr = Vec::new(); + +// // 字段数量 +// rmp::encode::write_map_len(&mut wr, len)?; + +// // string "ID" +// rmp::encode::write_str(&mut wr, "ID")?; +// rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or_default().as_bytes())?; + +// // string "MTime" +// rmp::encode::write_str(&mut wr, "MTime")?; +// rmp::encode::write_uint( +// &mut wr, +// self.mod_time +// .unwrap_or(OffsetDateTime::UNIX_EPOCH) +// .unix_timestamp() +// .try_into() +// .unwrap(), +// )?; + +// if (mask & 0x4) == 0 { +// let metas = self.meta_sys.as_ref().unwrap(); +// rmp::encode::write_map_len(&mut wr, metas.len() as u32)?; +// for (k, v) in metas { +// rmp::encode::write_str(&mut wr, k.as_str())?; +// rmp::encode::write_bin(&mut wr, v)?; +// } +// } + +// Ok(wr) +// } +// } + +// impl From for MetaDeleteMarker { +// fn from(value: FileInfo) -> Self { +// Self { +// version_id: value.version_id, +// mod_time: value.mod_time, +// meta_sys: None, +// } +// } +// } + +// #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Default, Clone, PartialOrd, Ord, Hash)] +// pub enum VersionType { +// #[default] +// Invalid = 0, +// Object = 1, +// Delete = 2, +// // Legacy = 3, +// } + +// impl VersionType { +// pub fn valid(&self) -> bool { +// matches!(*self, VersionType::Object | VersionType::Delete) +// } + +// pub fn to_u8(&self) -> u8 { +// match self { +// VersionType::Invalid => 0, +// VersionType::Object => 1, +// VersionType::Delete => 2, +// } +// } + +// pub fn from_u8(n: u8) -> Self { +// match n { +// 1 => VersionType::Object, +// 2 => VersionType::Delete, +// _ => VersionType::Invalid, +// } +// } +// } + +// #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Default, Clone)] +// pub enum ErasureAlgo { +// #[default] +// Invalid = 0, +// ReedSolomon = 1, +// } + +// impl ErasureAlgo { +// pub fn valid(&self) -> bool { +// *self > ErasureAlgo::Invalid +// } +// pub fn to_u8(&self) -> u8 { +// match self { +// ErasureAlgo::Invalid => 0, +// ErasureAlgo::ReedSolomon => 1, +// } +// } + +// pub fn from_u8(u: u8) -> Self { +// match u { +// 1 => ErasureAlgo::ReedSolomon, +// _ => ErasureAlgo::Invalid, +// } +// } +// } + +// impl Display for ErasureAlgo { +// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +// match self { +// ErasureAlgo::Invalid => write!(f, "Invalid"), +// ErasureAlgo::ReedSolomon => write!(f, "{}", ERASURE_ALGORITHM), +// } +// } +// } + +// #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Default, Clone)] +// pub enum ChecksumAlgo { +// #[default] +// Invalid = 0, +// HighwayHash = 1, +// } + +// impl ChecksumAlgo { +// pub fn valid(&self) -> bool { +// *self > ChecksumAlgo::Invalid +// } +// pub fn to_u8(&self) -> u8 { +// match self { +// ChecksumAlgo::Invalid => 0, +// ChecksumAlgo::HighwayHash => 1, +// } +// } +// pub fn from_u8(u: u8) -> Self { +// match u { +// 1 => ChecksumAlgo::HighwayHash, +// _ => ChecksumAlgo::Invalid, +// } +// } +// } + +// #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Default, Clone)] +// pub enum Flags { +// #[default] +// FreeVersion = 1 << 0, +// UsesDataDir = 1 << 1, +// InlineData = 1 << 2, +// } + +// const FREE_VERSION_META_HEADER: &str = "free-version"; + +// // mergeXLV2Versions +// pub fn merge_file_meta_versions( +// mut quorum: usize, +// mut strict: bool, +// requested_versions: usize, +// versions: &[Vec], +// ) -> Vec { +// if quorum == 0 { +// quorum = 1; +// } + +// if versions.len() < quorum || versions.is_empty() { +// return Vec::new(); +// } + +// if versions.len() == 1 { +// return versions[0].clone(); +// } + +// if quorum == 1 { +// strict = true; +// } + +// let mut versions = versions.to_owned(); + +// let mut n_versions = 0; + +// let mut merged = Vec::new(); +// loop { +// let mut tops = Vec::new(); +// let mut top_sig = FileMetaVersionHeader::default(); +// let mut consistent = true; +// for vers in versions.iter() { +// if vers.is_empty() { +// consistent = false; +// continue; +// } +// if tops.is_empty() { +// consistent = true; +// top_sig = vers[0].header.clone(); +// } else { +// consistent = consistent && vers[0].header == top_sig; +// } +// tops.push(vers[0].clone()); +// } + +// // check if done... +// if tops.len() < quorum { +// break; +// } + +// let mut latest = FileMetaShallowVersion::default(); +// if consistent { +// merged.push(tops[0].clone()); +// if tops[0].header.free_version() { +// n_versions += 1; +// } +// } else { +// let mut lastest_count = 0; +// for (i, ver) in tops.iter().enumerate() { +// if ver.header == latest.header { +// lastest_count += 1; +// continue; +// } + +// if i == 0 || ver.header.sorts_before(&latest.header) { +// if i == 0 || lastest_count == 0 { +// lastest_count = 1; +// } else if !strict && ver.header.matches_not_strict(&latest.header) { +// lastest_count += 1; +// } else { +// lastest_count = 1; +// } +// latest = ver.clone(); +// continue; +// } + +// // Mismatch, but older. +// if lastest_count > 0 && !strict && ver.header.matches_not_strict(&latest.header) { +// lastest_count += 1; +// continue; +// } + +// if lastest_count > 0 && ver.header.version_id == latest.header.version_id { +// let mut x: HashMap = HashMap::new(); +// for a in tops.iter() { +// if a.header.version_id != ver.header.version_id { +// continue; +// } +// let mut a_clone = a.clone(); +// if !strict { +// a_clone.header.signature = [0; 4]; +// } +// *x.entry(a_clone.header).or_insert(1) += 1; +// } +// lastest_count = 0; +// for (k, v) in x.iter() { +// if *v < lastest_count { +// continue; +// } +// if *v == lastest_count && latest.header.sorts_before(k) { +// continue; +// } +// tops.iter().for_each(|a| { +// let mut hdr = a.header.clone(); +// if !strict { +// hdr.signature = [0; 4]; +// } +// if hdr == *k { +// latest = a.clone(); +// } +// }); + +// lastest_count = *v; +// } +// break; +// } +// } +// if lastest_count >= quorum { +// if !latest.header.free_version() { +// n_versions += 1; +// } +// merged.push(latest.clone()); +// } +// } + +// // Remove from all streams up until latest modtime or if selected. +// versions.iter_mut().for_each(|vers| { +// // // Keep top entry (and remaining)... +// let mut bre = false; +// vers.retain(|ver| { +// if bre { +// return true; +// } +// if let Ordering::Greater = ver.header.mod_time.cmp(&latest.header.mod_time) { +// bre = true; +// return false; +// } +// if ver.header == latest.header { +// bre = true; +// return false; +// } +// if let Ordering::Equal = latest.header.version_id.cmp(&ver.header.version_id) { +// bre = true; +// return false; +// } +// for merged_v in merged.iter() { +// if let Ordering::Equal = ver.header.version_id.cmp(&merged_v.header.version_id) { +// bre = true; +// return false; +// } +// } +// true +// }); +// }); +// if requested_versions > 0 && requested_versions == n_versions { +// merged.append(&mut versions[0]); +// break; +// } +// } + +// // Sanity check. Enable if duplicates show up. +// // todo +// merged +// } + +// pub async fn file_info_from_raw(ri: RawFileInfo, bucket: &str, object: &str, read_data: bool) -> Result { +// get_file_info(&ri.buf, bucket, object, "", FileInfoOpts { data: read_data }).await +// } + +// pub struct FileInfoOpts { +// pub data: bool, +// } + +// pub async fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opts: FileInfoOpts) -> Result { +// let vid = { +// if version_id.is_empty() { +// None +// } else { +// Some(Uuid::parse_str(version_id)?) +// } +// }; + +// let meta = FileMeta::load(buf)?; +// if meta.versions.is_empty() { +// return Ok(FileInfo { +// volume: volume.to_owned(), +// name: path.to_owned(), +// version_id: vid, +// is_latest: true, +// deleted: true, +// mod_time: Some(OffsetDateTime::from_unix_timestamp(1)?), +// ..Default::default() +// }); +// } + +// let fi = meta.into_fileinfo(volume, path, version_id, opts.data, true)?; +// Ok(fi) +// } + +// async fn read_more( +// reader: &mut R, +// buf: &mut Vec, +// total_size: usize, +// read_size: usize, +// has_full: bool, +// ) -> Result<()> { +// use tokio::io::AsyncReadExt; +// let has = buf.len(); + +// if has >= read_size { +// return Ok(()); +// } + +// if has_full || read_size > total_size { +// return Err(Error::new(io::Error::new(io::ErrorKind::UnexpectedEof, "Unexpected EOF"))); +// } + +// let extra = read_size - has; +// if buf.capacity() >= read_size { +// // Extend the buffer if we have enough space. +// buf.resize(read_size, 0); +// } else { +// buf.extend(vec![0u8; extra]); +// } + +// reader.read_exact(&mut buf[has..]).await?; +// Ok(()) +// } + +// pub async fn read_xl_meta_no_data(reader: &mut R, size: usize) -> Result> { +// use tokio::io::AsyncReadExt; + +// let mut initial = size; +// let mut has_full = true; + +// if initial > META_DATA_READ_DEFAULT { +// initial = META_DATA_READ_DEFAULT; +// has_full = false; +// } + +// let mut buf = vec![0u8; initial]; +// reader.read_exact(&mut buf).await?; + +// let (tmp_buf, major, minor) = FileMeta::check_xl2_v1(&buf)?; + +// match major { +// 1 => match minor { +// 0 => { +// read_more(reader, &mut buf, size, size, has_full).await?; +// Ok(buf) +// } +// 1..=3 => { +// let (sz, tmp_buf) = FileMeta::read_bytes_header(tmp_buf)?; +// let mut want = sz as usize + (buf.len() - tmp_buf.len()); + +// if minor < 2 { +// read_more(reader, &mut buf, size, want, has_full).await?; +// return Ok(buf[..want].to_vec()); +// } + +// let want_max = usize::min(want + MSGP_UINT32_SIZE, size); +// read_more(reader, &mut buf, size, want_max, has_full).await?; + +// if buf.len() < want { +// error!("read_xl_meta_no_data buffer too small (length: {}, needed: {})", &buf.len(), want); +// return Err(Error::new(DiskError::FileCorrupt)); +// } + +// let tmp = &buf[want..]; +// let crc_size = 5; +// let other_size = tmp.len() - crc_size; + +// want += tmp.len() - other_size; + +// Ok(buf[..want].to_vec()) +// } +// _ => Err(Error::new(io::Error::new(io::ErrorKind::InvalidData, "Unknown minor metadata version"))), +// }, +// _ => Err(Error::new(io::Error::new(io::ErrorKind::InvalidData, "Unknown major metadata version"))), +// } +// } +// #[cfg(test)] +// #[allow(clippy::field_reassign_with_default)] +// mod test { +// use super::*; + +// #[test] +// fn test_new_file_meta() { +// let mut fm = FileMeta::new(); + +// let (m, n) = (3, 2); + +// for i in 0..5 { +// let mut fi = FileInfo::new(i.to_string().as_str(), m, n); +// fi.mod_time = Some(OffsetDateTime::now_utc()); + +// fm.add_version(fi).unwrap(); +// } + +// let buff = fm.marshal_msg().unwrap(); + +// let mut newfm = FileMeta::default(); +// newfm.unmarshal_msg(&buff).unwrap(); + +// assert_eq!(fm, newfm) +// } + +// #[test] +// fn test_marshal_metaobject() { +// let obj = MetaObject { +// data_dir: Some(Uuid::new_v4()), +// ..Default::default() +// }; + +// // println!("obj {:?}", &obj); + +// let encoded = obj.marshal_msg().unwrap(); + +// let mut obj2 = MetaObject::default(); +// obj2.unmarshal_msg(&encoded).unwrap(); + +// // println!("obj2 {:?}", &obj2); + +// assert_eq!(obj, obj2); +// assert_eq!(obj.data_dir, obj2.data_dir); +// } + +// #[test] +// fn test_marshal_metadeletemarker() { +// let obj = MetaDeleteMarker { +// version_id: Some(Uuid::new_v4()), +// ..Default::default() +// }; + +// // println!("obj {:?}", &obj); + +// let encoded = obj.marshal_msg().unwrap(); + +// let mut obj2 = MetaDeleteMarker::default(); +// obj2.unmarshal_msg(&encoded).unwrap(); + +// // println!("obj2 {:?}", &obj2); + +// assert_eq!(obj, obj2); +// assert_eq!(obj.version_id, obj2.version_id); +// } + +// #[test] +// #[tracing::instrument] +// fn test_marshal_metaversion() { +// let mut fi = FileInfo::new("test", 3, 2); +// fi.version_id = Some(Uuid::new_v4()); +// fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(OffsetDateTime::now_utc().unix_timestamp()).unwrap()); +// let mut obj = FileMetaVersion::from(fi); +// obj.write_version = 110; + +// // println!("obj {:?}", &obj); + +// let encoded = obj.marshal_msg().unwrap(); + +// let mut obj2 = FileMetaVersion::default(); +// obj2.unmarshal_msg(&encoded).unwrap(); + +// // println!("obj2 {:?}", &obj2); + +// // 时间截不一致 - - +// assert_eq!(obj, obj2); +// assert_eq!(obj.get_version_id(), obj2.get_version_id()); +// assert_eq!(obj.write_version, obj2.write_version); +// assert_eq!(obj.write_version, 110); +// } + +// #[test] +// #[tracing::instrument] +// fn test_marshal_metaversionheader() { +// let mut obj = FileMetaVersionHeader::default(); +// let vid = Some(Uuid::new_v4()); +// obj.version_id = vid; + +// let encoded = obj.marshal_msg().unwrap(); + +// let mut obj2 = FileMetaVersionHeader::default(); +// obj2.unmarshal_msg(&encoded).unwrap(); + +// // 时间截不一致 - - +// assert_eq!(obj, obj2); +// assert_eq!(obj.version_id, obj2.version_id); +// assert_eq!(obj.version_id, vid); +// } + +// // New comprehensive tests for utility functions and validation + +// #[test] +// fn test_xl_file_header_constants() { +// // Test XL file header constants +// assert_eq!(XL_FILE_HEADER, [b'X', b'L', b'2', b' ']); +// assert_eq!(XL_FILE_VERSION_MAJOR, 1); +// assert_eq!(XL_FILE_VERSION_MINOR, 3); +// assert_eq!(XL_HEADER_VERSION, 3); +// assert_eq!(XL_META_VERSION, 2); +// } + +// #[test] +// fn test_is_xl2_v1_format() { +// // Test valid XL2 V1 format +// let mut valid_buf = vec![0u8; 20]; +// valid_buf[0..4].copy_from_slice(&XL_FILE_HEADER); +// byteorder::LittleEndian::write_u16(&mut valid_buf[4..6], 1); +// byteorder::LittleEndian::write_u16(&mut valid_buf[6..8], 0); + +// assert!(FileMeta::is_xl2_v1_format(&valid_buf)); + +// // Test invalid format - wrong header +// let invalid_buf = vec![0u8; 20]; +// assert!(!FileMeta::is_xl2_v1_format(&invalid_buf)); + +// // Test buffer too small +// let small_buf = vec![0u8; 4]; +// assert!(!FileMeta::is_xl2_v1_format(&small_buf)); +// } + +// #[test] +// fn test_check_xl2_v1() { +// // Test valid XL2 V1 check +// let mut valid_buf = vec![0u8; 20]; +// valid_buf[0..4].copy_from_slice(&XL_FILE_HEADER); +// byteorder::LittleEndian::write_u16(&mut valid_buf[4..6], 1); +// byteorder::LittleEndian::write_u16(&mut valid_buf[6..8], 2); + +// let result = FileMeta::check_xl2_v1(&valid_buf); +// assert!(result.is_ok()); +// let (remaining, major, minor) = result.unwrap(); +// assert_eq!(major, 1); +// assert_eq!(minor, 2); +// assert_eq!(remaining.len(), 12); // 20 - 8 + +// // Test buffer too small +// let small_buf = vec![0u8; 4]; +// assert!(FileMeta::check_xl2_v1(&small_buf).is_err()); + +// // Test wrong header +// let mut wrong_header = vec![0u8; 20]; +// wrong_header[0..4].copy_from_slice(b"ABCD"); +// assert!(FileMeta::check_xl2_v1(&wrong_header).is_err()); + +// // Test version too high +// let mut high_version = vec![0u8; 20]; +// high_version[0..4].copy_from_slice(&XL_FILE_HEADER); +// byteorder::LittleEndian::write_u16(&mut high_version[4..6], 99); +// byteorder::LittleEndian::write_u16(&mut high_version[6..8], 0); +// assert!(FileMeta::check_xl2_v1(&high_version).is_err()); +// } + +// #[test] +// fn test_version_type_enum() { +// // Test VersionType enum methods +// assert!(VersionType::Object.valid()); +// assert!(VersionType::Delete.valid()); +// assert!(!VersionType::Invalid.valid()); + +// assert_eq!(VersionType::Object.to_u8(), 1); +// assert_eq!(VersionType::Delete.to_u8(), 2); +// assert_eq!(VersionType::Invalid.to_u8(), 0); + +// assert_eq!(VersionType::from_u8(1), VersionType::Object); +// assert_eq!(VersionType::from_u8(2), VersionType::Delete); +// assert_eq!(VersionType::from_u8(99), VersionType::Invalid); +// } + +// #[test] +// fn test_erasure_algo_enum() { +// // Test ErasureAlgo enum methods +// assert!(ErasureAlgo::ReedSolomon.valid()); +// assert!(!ErasureAlgo::Invalid.valid()); + +// assert_eq!(ErasureAlgo::ReedSolomon.to_u8(), 1); +// assert_eq!(ErasureAlgo::Invalid.to_u8(), 0); + +// assert_eq!(ErasureAlgo::from_u8(1), ErasureAlgo::ReedSolomon); +// assert_eq!(ErasureAlgo::from_u8(99), ErasureAlgo::Invalid); + +// // Test Display trait +// assert_eq!(format!("{}", ErasureAlgo::ReedSolomon), "rs-vandermonde"); +// assert_eq!(format!("{}", ErasureAlgo::Invalid), "Invalid"); +// } + +// #[test] +// fn test_checksum_algo_enum() { +// // Test ChecksumAlgo enum methods +// assert!(ChecksumAlgo::HighwayHash.valid()); +// assert!(!ChecksumAlgo::Invalid.valid()); + +// assert_eq!(ChecksumAlgo::HighwayHash.to_u8(), 1); +// assert_eq!(ChecksumAlgo::Invalid.to_u8(), 0); + +// assert_eq!(ChecksumAlgo::from_u8(1), ChecksumAlgo::HighwayHash); +// assert_eq!(ChecksumAlgo::from_u8(99), ChecksumAlgo::Invalid); +// } + +// #[test] +// fn test_file_meta_version_header_methods() { +// let mut header = FileMetaVersionHeader { +// ec_n: 4, +// ec_m: 2, +// flags: XL_FLAG_FREE_VERSION, +// ..Default::default() +// }; + +// // Test has_ec +// assert!(header.has_ec()); + +// // Test free_version +// assert!(header.free_version()); + +// // Test user_data_dir (should be false by default) +// assert!(!header.user_data_dir()); + +// // Test with different flags +// header.flags = 0; +// assert!(!header.free_version()); +// } + +// #[test] +// fn test_file_meta_version_header_comparison() { +// let mut header1 = FileMetaVersionHeader { +// mod_time: Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()), +// version_id: Some(Uuid::new_v4()), +// ..Default::default() +// }; + +// let mut header2 = FileMetaVersionHeader { +// mod_time: Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()), +// version_id: Some(Uuid::new_v4()), +// ..Default::default() +// }; + +// // Test sorts_before - header2 should sort before header1 (newer mod_time) +// assert!(!header1.sorts_before(&header2)); +// assert!(header2.sorts_before(&header1)); + +// // Test matches_not_strict +// let header3 = header1.clone(); +// assert!(header1.matches_not_strict(&header3)); + +// // Test matches_ec +// header1.ec_n = 4; +// header1.ec_m = 2; +// header2.ec_n = 4; +// header2.ec_m = 2; +// assert!(header1.matches_ec(&header2)); + +// header2.ec_n = 6; +// assert!(!header1.matches_ec(&header2)); +// } + +// #[test] +// fn test_file_meta_version_methods() { +// // Test with object version +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = Some(Uuid::new_v4()); +// fi.data_dir = Some(Uuid::new_v4()); +// fi.mod_time = Some(OffsetDateTime::now_utc()); + +// let version = FileMetaVersion::from(fi.clone()); + +// assert!(version.valid()); +// assert_eq!(version.get_version_id(), fi.version_id); +// assert_eq!(version.get_data_dir(), fi.data_dir); +// assert_eq!(version.get_mod_time(), fi.mod_time); +// assert!(!version.free_version()); + +// // Test with delete marker +// let mut delete_fi = FileInfo::new("test", 4, 2); +// delete_fi.deleted = true; +// delete_fi.version_id = Some(Uuid::new_v4()); +// delete_fi.mod_time = Some(OffsetDateTime::now_utc()); + +// let delete_version = FileMetaVersion::from(delete_fi); +// assert!(delete_version.valid()); +// assert_eq!(delete_version.version_type, VersionType::Delete); +// } + +// #[test] +// fn test_meta_object_methods() { +// let mut obj = MetaObject { +// data_dir: Some(Uuid::new_v4()), +// size: 1024, +// ..Default::default() +// }; + +// // Test use_data_dir +// assert!(obj.use_data_dir()); + +// obj.data_dir = None; +// assert!(obj.use_data_dir()); // use_data_dir always returns true + +// // Test use_inlinedata (currently always returns false) +// obj.size = 100; // Small size +// assert!(!obj.use_inlinedata()); + +// obj.size = 100000; // Large size +// assert!(!obj.use_inlinedata()); +// } + +// #[test] +// fn test_meta_delete_marker_methods() { +// let marker = MetaDeleteMarker::default(); + +// // Test free_version (should always return false for delete markers) +// assert!(!marker.free_version()); +// } + +// #[test] +// fn test_file_meta_latest_mod_time() { +// let mut fm = FileMeta::new(); + +// // Empty FileMeta should return None +// assert!(fm.lastest_mod_time().is_none()); + +// // Add versions with different mod times +// let time1 = OffsetDateTime::from_unix_timestamp(1000).unwrap(); +// let time2 = OffsetDateTime::from_unix_timestamp(2000).unwrap(); +// let time3 = OffsetDateTime::from_unix_timestamp(1500).unwrap(); + +// let mut fi1 = FileInfo::new("test1", 4, 2); +// fi1.mod_time = Some(time1); +// fm.add_version(fi1).unwrap(); + +// let mut fi2 = FileInfo::new("test2", 4, 2); +// fi2.mod_time = Some(time2); +// fm.add_version(fi2).unwrap(); + +// let mut fi3 = FileInfo::new("test3", 4, 2); +// fi3.mod_time = Some(time3); +// fm.add_version(fi3).unwrap(); + +// // Sort first to ensure latest is at the front +// fm.sort_by_mod_time(); + +// // Should return the first version's mod time (lastest_mod_time returns first version's time) +// assert_eq!(fm.lastest_mod_time(), fm.versions[0].header.mod_time); +// } + +// #[test] +// fn test_file_meta_shard_data_dir_count() { +// let mut fm = FileMeta::new(); +// let data_dir = Some(Uuid::new_v4()); + +// // Add versions with same data_dir +// for i in 0..3 { +// let mut fi = FileInfo::new(&format!("test{}", i), 4, 2); +// fi.data_dir = data_dir; +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi).unwrap(); +// } + +// // Add one version with different data_dir +// let mut fi_diff = FileInfo::new("test_diff", 4, 2); +// fi_diff.data_dir = Some(Uuid::new_v4()); +// fi_diff.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi_diff).unwrap(); + +// // Count should be 0 because user_data_dir() requires UsesDataDir flag to be set +// assert_eq!(fm.shard_data_dir_count(&None, &data_dir), 0); + +// // Count should be 0 for non-existent data_dir +// assert_eq!(fm.shard_data_dir_count(&None, &Some(Uuid::new_v4())), 0); +// } + +// #[test] +// fn test_file_meta_sort_by_mod_time() { +// let mut fm = FileMeta::new(); + +// let time1 = OffsetDateTime::from_unix_timestamp(3000).unwrap(); +// let time2 = OffsetDateTime::from_unix_timestamp(1000).unwrap(); +// let time3 = OffsetDateTime::from_unix_timestamp(2000).unwrap(); + +// // Add versions in non-chronological order +// let mut fi1 = FileInfo::new("test1", 4, 2); +// fi1.mod_time = Some(time1); +// fm.add_version(fi1).unwrap(); + +// let mut fi2 = FileInfo::new("test2", 4, 2); +// fi2.mod_time = Some(time2); +// fm.add_version(fi2).unwrap(); + +// let mut fi3 = FileInfo::new("test3", 4, 2); +// fi3.mod_time = Some(time3); +// fm.add_version(fi3).unwrap(); + +// // Sort by mod time +// fm.sort_by_mod_time(); + +// // Verify they are sorted (newest first) - add_version already sorts by insertion +// // The actual order depends on how add_version inserts them +// // Let's check the first version is the latest +// let latest_time = fm.versions.iter().map(|v| v.header.mod_time).max().flatten(); +// assert_eq!(fm.versions[0].header.mod_time, latest_time); +// } + +// #[test] +// fn test_file_meta_find_version() { +// let mut fm = FileMeta::new(); +// let version_id = Some(Uuid::new_v4()); + +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = version_id; +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi).unwrap(); + +// // Should find the version +// let result = fm.find_version(version_id); +// assert!(result.is_ok()); +// let (idx, version) = result.unwrap(); +// assert_eq!(idx, 0); +// assert_eq!(version.get_version_id(), version_id); + +// // Should not find non-existent version +// let non_existent_id = Some(Uuid::new_v4()); +// assert!(fm.find_version(non_existent_id).is_err()); +// } + +// #[test] +// fn test_file_meta_delete_version() { +// let mut fm = FileMeta::new(); +// let version_id = Some(Uuid::new_v4()); + +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = version_id; +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi.clone()).unwrap(); + +// assert_eq!(fm.versions.len(), 1); + +// // Delete the version +// let result = fm.delete_version(&fi); +// assert!(result.is_ok()); + +// // Version should be removed +// assert_eq!(fm.versions.len(), 0); +// } + +// #[test] +// fn test_file_meta_update_object_version() { +// let mut fm = FileMeta::new(); +// let version_id = Some(Uuid::new_v4()); + +// // Add initial version +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = version_id; +// fi.size = 1024; +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi.clone()).unwrap(); + +// // Update with new metadata (size is not updated by update_object_version) +// let mut metadata = HashMap::new(); +// metadata.insert("test-key".to_string(), "test-value".to_string()); +// fi.metadata = Some(metadata.clone()); +// let result = fm.update_object_version(fi); +// assert!(result.is_ok()); + +// // Verify the metadata was updated +// let (_, updated_version) = fm.find_version(version_id).unwrap(); +// if let Some(obj) = updated_version.object { +// assert_eq!(obj.size, 1024); // Size remains unchanged +// assert_eq!(obj.meta_user, Some(metadata)); // Metadata is updated +// } else { +// panic!("Expected object version"); +// } +// } + +// #[test] +// fn test_file_info_opts() { +// let opts = FileInfoOpts { data: true }; +// assert!(opts.data); + +// let opts_no_data = FileInfoOpts { data: false }; +// assert!(!opts_no_data.data); +// } + +// #[test] +// fn test_decode_data_dir_from_meta() { +// // Test with valid metadata containing data_dir +// let data_dir = Some(Uuid::new_v4()); +// let obj = MetaObject { +// data_dir, +// mod_time: Some(OffsetDateTime::now_utc()), +// erasure_algorithm: ErasureAlgo::ReedSolomon, +// bitrot_checksum_algo: ChecksumAlgo::HighwayHash, +// ..Default::default() +// }; + +// // Create a valid FileMetaVersion with the object +// let version = FileMetaVersion { +// version_type: VersionType::Object, +// object: Some(obj), +// ..Default::default() +// }; + +// let encoded = version.marshal_msg().unwrap(); +// let result = FileMetaVersion::decode_data_dir_from_meta(&encoded); +// assert!(result.is_ok()); +// assert_eq!(result.unwrap(), data_dir); + +// // Test with invalid metadata +// let invalid_data = vec![0u8; 10]; +// let result = FileMetaVersion::decode_data_dir_from_meta(&invalid_data); +// assert!(result.is_err()); +// } + +// #[test] +// fn test_is_latest_delete_marker() { +// // Test the is_latest_delete_marker function with simple data +// // Since the function is complex and requires specific XL format, +// // we'll test with empty data which should return false +// let empty_data = vec![]; +// assert!(!FileMeta::is_latest_delete_marker(&empty_data)); + +// // Test with invalid data +// let invalid_data = vec![1, 2, 3, 4, 5]; +// assert!(!FileMeta::is_latest_delete_marker(&invalid_data)); +// } + +// #[test] +// fn test_merge_file_meta_versions_basic() { +// // Test basic merge functionality +// let mut version1 = FileMetaShallowVersion::default(); +// version1.header.version_id = Some(Uuid::new_v4()); +// version1.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()); + +// let mut version2 = FileMetaShallowVersion::default(); +// version2.header.version_id = Some(Uuid::new_v4()); +// version2.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()); + +// let versions = vec![ +// vec![version1.clone(), version2.clone()], +// vec![version1.clone()], +// vec![version2.clone()], +// ]; + +// let merged = merge_file_meta_versions(2, false, 10, &versions); + +// // Should return versions that appear in at least quorum (2) sources +// assert!(!merged.is_empty()); +// } +// } + +// #[tokio::test] +// async fn test_read_xl_meta_no_data() { +// use tokio::fs; +// use tokio::fs::File; +// use tokio::io::AsyncWriteExt; + +// let mut fm = FileMeta::new(); + +// let (m, n) = (3, 2); + +// for i in 0..5 { +// let mut fi = FileInfo::new(i.to_string().as_str(), m, n); +// fi.mod_time = Some(OffsetDateTime::now_utc()); + +// fm.add_version(fi).unwrap(); +// } + +// // Use marshal_msg to create properly formatted data with XL headers +// let buff = fm.marshal_msg().unwrap(); + +// let filepath = "./test_xl.meta"; + +// let mut file = File::create(filepath).await.unwrap(); +// file.write_all(&buff).await.unwrap(); + +// let mut f = File::open(filepath).await.unwrap(); + +// let stat = f.metadata().await.unwrap(); + +// let data = read_xl_meta_no_data(&mut f, stat.len() as usize).await.unwrap(); + +// let mut newfm = FileMeta::default(); +// newfm.unmarshal_msg(&data).unwrap(); + +// fs::remove_file(filepath).await.unwrap(); + +// assert_eq!(fm, newfm) +// } + +// #[tokio::test] +// async fn test_get_file_info() { +// // Test get_file_info function +// let mut fm = FileMeta::new(); +// let version_id = Uuid::new_v4(); + +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = Some(version_id); +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi).unwrap(); + +// let encoded = fm.marshal_msg().unwrap(); + +// let opts = FileInfoOpts { data: false }; +// let result = get_file_info(&encoded, "test-volume", "test-path", &version_id.to_string(), opts).await; + +// assert!(result.is_ok()); +// let file_info = result.unwrap(); +// assert_eq!(file_info.volume, "test-volume"); +// assert_eq!(file_info.name, "test-path"); +// } + +// #[tokio::test] +// async fn test_file_info_from_raw() { +// // Test file_info_from_raw function +// let mut fm = FileMeta::new(); +// let mut fi = FileInfo::new("test", 4, 2); +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi).unwrap(); + +// let encoded = fm.marshal_msg().unwrap(); + +// let raw_info = RawFileInfo { buf: encoded }; + +// let result = file_info_from_raw(raw_info, "test-bucket", "test-object", false).await; +// assert!(result.is_ok()); + +// let file_info = result.unwrap(); +// assert_eq!(file_info.volume, "test-bucket"); +// assert_eq!(file_info.name, "test-object"); +// } + +// // Additional comprehensive tests for better coverage + +// #[test] +// fn test_file_meta_load_function() { +// // Test FileMeta::load function +// let mut fm = FileMeta::new(); +// let mut fi = FileInfo::new("test", 4, 2); +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi).unwrap(); + +// let encoded = fm.marshal_msg().unwrap(); + +// // Test successful load +// let loaded_fm = FileMeta::load(&encoded); +// assert!(loaded_fm.is_ok()); +// assert_eq!(loaded_fm.unwrap(), fm); + +// // Test load with invalid data +// let invalid_data = vec![0u8; 10]; +// let result = FileMeta::load(&invalid_data); +// assert!(result.is_err()); +// } + +// #[test] +// fn test_file_meta_read_bytes_header() { +// // Create a real FileMeta and marshal it to get proper format +// let mut fm = FileMeta::new(); +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = Some(Uuid::new_v4()); +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi).unwrap(); + +// let marshaled = fm.marshal_msg().unwrap(); + +// // First call check_xl2_v1 to get the buffer after XL header validation +// let (after_xl_header, _major, _minor) = FileMeta::check_xl2_v1(&marshaled).unwrap(); + +// // Ensure we have at least 5 bytes for read_bytes_header +// if after_xl_header.len() < 5 { +// panic!("Buffer too small: {} bytes, need at least 5", after_xl_header.len()); +// } + +// // Now call read_bytes_header on the remaining buffer +// let result = FileMeta::read_bytes_header(after_xl_header); +// assert!(result.is_ok()); +// let (length, remaining) = result.unwrap(); + +// // The length should be greater than 0 for real data +// assert!(length > 0); +// // remaining should be everything after the 5-byte header +// assert_eq!(remaining.len(), after_xl_header.len() - 5); + +// // Test with buffer too small +// let small_buf = vec![0u8; 2]; +// let result = FileMeta::read_bytes_header(&small_buf); +// assert!(result.is_err()); +// } + +// #[test] +// fn test_file_meta_get_set_idx() { +// let mut fm = FileMeta::new(); +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = Some(Uuid::new_v4()); +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi).unwrap(); + +// // Test get_idx +// let result = fm.get_idx(0); +// assert!(result.is_ok()); + +// // Test get_idx with invalid index +// let result = fm.get_idx(10); +// assert!(result.is_err()); + +// // Test set_idx +// let new_version = FileMetaVersion { +// version_type: VersionType::Object, +// ..Default::default() +// }; +// let result = fm.set_idx(0, new_version); +// assert!(result.is_ok()); + +// // Test set_idx with invalid index +// let invalid_version = FileMetaVersion::default(); +// let result = fm.set_idx(10, invalid_version); +// assert!(result.is_err()); +// } + +// #[test] +// fn test_file_meta_into_fileinfo() { +// let mut fm = FileMeta::new(); +// let version_id = Uuid::new_v4(); +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = Some(version_id); +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi).unwrap(); + +// // Test into_fileinfo with valid version_id +// let result = fm.into_fileinfo("test-volume", "test-path", &version_id.to_string(), false, false); +// assert!(result.is_ok()); +// let file_info = result.unwrap(); +// assert_eq!(file_info.volume, "test-volume"); +// assert_eq!(file_info.name, "test-path"); + +// // Test into_fileinfo with invalid version_id +// let invalid_id = Uuid::new_v4(); +// let result = fm.into_fileinfo("test-volume", "test-path", &invalid_id.to_string(), false, false); +// assert!(result.is_err()); + +// // Test into_fileinfo with empty version_id (should get latest) +// let result = fm.into_fileinfo("test-volume", "test-path", "", false, false); +// assert!(result.is_ok()); +// } + +// #[test] +// fn test_file_meta_into_file_info_versions() { +// let mut fm = FileMeta::new(); + +// // Add multiple versions +// for i in 0..3 { +// let mut fi = FileInfo::new(&format!("test{}", i), 4, 2); +// fi.version_id = Some(Uuid::new_v4()); +// fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(1000 + i).unwrap()); +// fm.add_version(fi).unwrap(); +// } + +// let result = fm.into_file_info_versions("test-volume", "test-path", false); +// assert!(result.is_ok()); +// let versions = result.unwrap(); +// assert_eq!(versions.versions.len(), 3); +// } + +// #[test] +// fn test_file_meta_shallow_version_to_fileinfo() { +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = Some(Uuid::new_v4()); +// fi.mod_time = Some(OffsetDateTime::now_utc()); + +// let version = FileMetaVersion::from(fi.clone()); +// let shallow_version = FileMetaShallowVersion::try_from(version).unwrap(); + +// let result = shallow_version.to_fileinfo("test-volume", "test-path", fi.version_id, false); +// assert!(result.is_ok()); +// let converted_fi = result.unwrap(); +// assert_eq!(converted_fi.volume, "test-volume"); +// assert_eq!(converted_fi.name, "test-path"); +// } + +// #[test] +// fn test_file_meta_version_try_from_bytes() { +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = Some(Uuid::new_v4()); +// let version = FileMetaVersion::from(fi); +// let encoded = version.marshal_msg().unwrap(); + +// // Test successful conversion +// let result = FileMetaVersion::try_from(encoded.as_slice()); +// assert!(result.is_ok()); + +// // Test with invalid data +// let invalid_data = vec![0u8; 5]; +// let result = FileMetaVersion::try_from(invalid_data.as_slice()); +// assert!(result.is_err()); +// } + +// #[test] +// fn test_file_meta_version_try_from_shallow() { +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = Some(Uuid::new_v4()); +// let version = FileMetaVersion::from(fi); +// let shallow = FileMetaShallowVersion::try_from(version.clone()).unwrap(); + +// let result = FileMetaVersion::try_from(shallow); +// assert!(result.is_ok()); +// let converted = result.unwrap(); +// assert_eq!(converted.get_version_id(), version.get_version_id()); +// } + +// #[test] +// fn test_file_meta_version_header_from_version() { +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = Some(Uuid::new_v4()); +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// let version = FileMetaVersion::from(fi.clone()); + +// let header = FileMetaVersionHeader::from(version); +// assert_eq!(header.version_id, fi.version_id); +// assert_eq!(header.mod_time, fi.mod_time); +// } + +// #[test] +// fn test_meta_object_into_fileinfo() { +// let obj = MetaObject { +// version_id: Some(Uuid::new_v4()), +// size: 1024, +// mod_time: Some(OffsetDateTime::now_utc()), +// ..Default::default() +// }; + +// let version_id = obj.version_id; +// let expected_version_id = version_id; +// let file_info = obj.into_fileinfo("test-volume", "test-path", version_id, false); +// assert_eq!(file_info.volume, "test-volume"); +// assert_eq!(file_info.name, "test-path"); +// assert_eq!(file_info.size, 1024); +// assert_eq!(file_info.version_id, expected_version_id); +// } + +// #[test] +// fn test_meta_object_from_fileinfo() { +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = Some(Uuid::new_v4()); +// fi.data_dir = Some(Uuid::new_v4()); +// fi.size = 2048; +// fi.mod_time = Some(OffsetDateTime::now_utc()); + +// let obj = MetaObject::from(fi.clone()); +// assert_eq!(obj.version_id, fi.version_id); +// assert_eq!(obj.data_dir, fi.data_dir); +// assert_eq!(obj.size, fi.size); +// assert_eq!(obj.mod_time, fi.mod_time); +// } + +// #[test] +// fn test_meta_delete_marker_into_fileinfo() { +// let marker = MetaDeleteMarker { +// version_id: Some(Uuid::new_v4()), +// mod_time: Some(OffsetDateTime::now_utc()), +// ..Default::default() +// }; + +// let version_id = marker.version_id; +// let expected_version_id = version_id; +// let file_info = marker.into_fileinfo("test-volume", "test-path", version_id, false); +// assert_eq!(file_info.volume, "test-volume"); +// assert_eq!(file_info.name, "test-path"); +// assert_eq!(file_info.version_id, expected_version_id); +// assert!(file_info.deleted); +// } + +// #[test] +// fn test_meta_delete_marker_from_fileinfo() { +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = Some(Uuid::new_v4()); +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// fi.deleted = true; + +// let marker = MetaDeleteMarker::from(fi.clone()); +// assert_eq!(marker.version_id, fi.version_id); +// assert_eq!(marker.mod_time, fi.mod_time); +// } + +// #[test] +// fn test_flags_enum() { +// // Test Flags enum values +// assert_eq!(Flags::FreeVersion as u8, 1); +// assert_eq!(Flags::UsesDataDir as u8, 2); +// assert_eq!(Flags::InlineData as u8, 4); +// } + +// #[test] +// fn test_file_meta_version_header_user_data_dir() { +// let header = FileMetaVersionHeader { +// flags: 0, +// ..Default::default() +// }; + +// // Test without UsesDataDir flag +// assert!(!header.user_data_dir()); + +// // Test with UsesDataDir flag +// let header = FileMetaVersionHeader { +// flags: Flags::UsesDataDir as u8, +// ..Default::default() +// }; +// assert!(header.user_data_dir()); + +// // Test with multiple flags including UsesDataDir +// let header = FileMetaVersionHeader { +// flags: Flags::UsesDataDir as u8 | Flags::FreeVersion as u8, +// ..Default::default() +// }; +// assert!(header.user_data_dir()); +// } + +// #[test] +// fn test_file_meta_version_header_ordering() { +// let header1 = FileMetaVersionHeader { +// mod_time: Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()), +// version_id: Some(Uuid::new_v4()), +// ..Default::default() +// }; + +// let header2 = FileMetaVersionHeader { +// mod_time: Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()), +// version_id: Some(Uuid::new_v4()), +// ..Default::default() +// }; + +// // Test partial_cmp +// assert!(header1.partial_cmp(&header2).is_some()); + +// // Test cmp - header2 should be greater (newer) +// use std::cmp::Ordering; +// assert_eq!(header1.cmp(&header2), Ordering::Less); // header1 has earlier time +// assert_eq!(header2.cmp(&header1), Ordering::Greater); // header2 has later time +// assert_eq!(header1.cmp(&header1), Ordering::Equal); +// } + +// #[test] +// fn test_merge_file_meta_versions_edge_cases() { +// // Test with empty versions +// let empty_versions: Vec> = vec![]; +// let merged = merge_file_meta_versions(1, false, 10, &empty_versions); +// assert!(merged.is_empty()); + +// // Test with quorum larger than available sources +// let mut version = FileMetaShallowVersion::default(); +// version.header.version_id = Some(Uuid::new_v4()); +// let versions = vec![vec![version]]; +// let merged = merge_file_meta_versions(5, false, 10, &versions); +// assert!(merged.is_empty()); + +// // Test strict mode +// let mut version1 = FileMetaShallowVersion::default(); +// version1.header.version_id = Some(Uuid::new_v4()); +// version1.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()); + +// let mut version2 = FileMetaShallowVersion::default(); +// version2.header.version_id = Some(Uuid::new_v4()); +// version2.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()); + +// let versions = vec![vec![version1.clone()], vec![version2.clone()]]; + +// let _merged_strict = merge_file_meta_versions(1, true, 10, &versions); +// let merged_non_strict = merge_file_meta_versions(1, false, 10, &versions); + +// // In strict mode, behavior might be different +// assert!(!merged_non_strict.is_empty()); +// } + +// #[tokio::test] +// async fn test_read_more_function() { +// use std::io::Cursor; + +// let data = b"Hello, World! This is test data."; +// let mut reader = Cursor::new(data); +// let mut buf = vec![0u8; 10]; + +// // Test reading more data +// let result = read_more(&mut reader, &mut buf, 33, 20, false).await; +// assert!(result.is_ok()); +// assert_eq!(buf.len(), 20); + +// // Test with has_full = true and buffer already has enough data +// let mut reader2 = Cursor::new(data); +// let mut buf2 = vec![0u8; 5]; +// let result = read_more(&mut reader2, &mut buf2, 10, 5, true).await; +// assert!(result.is_ok()); +// assert_eq!(buf2.len(), 5); // Should remain 5 since has >= read_size + +// // Test reading beyond available data +// let mut reader3 = Cursor::new(b"short"); +// let mut buf3 = vec![0u8; 2]; +// let result = read_more(&mut reader3, &mut buf3, 100, 98, false).await; +// // Should handle gracefully even if not enough data +// assert!(result.is_ok() || result.is_err()); // Either is acceptable +// } + +// #[tokio::test] +// async fn test_read_xl_meta_no_data_edge_cases() { +// use std::io::Cursor; + +// // Test with empty data +// let empty_data = vec![]; +// let mut reader = Cursor::new(empty_data); +// let result = read_xl_meta_no_data(&mut reader, 0).await; +// assert!(result.is_err()); // Should fail because buffer is empty + +// // Test with very small size (should fail because it's not valid XL format) +// let small_data = vec![1, 2, 3]; +// let mut reader = Cursor::new(small_data); +// let result = read_xl_meta_no_data(&mut reader, 3).await; +// assert!(result.is_err()); // Should fail because data is too small for XL format +// } + +// #[tokio::test] +// async fn test_get_file_info_edge_cases() { +// // Test with empty buffer +// let empty_buf = vec![]; +// let opts = FileInfoOpts { data: false }; +// let result = get_file_info(&empty_buf, "volume", "path", "version", opts).await; +// assert!(result.is_err()); + +// // Test with invalid version_id format +// let mut fm = FileMeta::new(); +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = Some(Uuid::new_v4()); +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi).unwrap(); +// let encoded = fm.marshal_msg().unwrap(); + +// let opts = FileInfoOpts { data: false }; +// let result = get_file_info(&encoded, "volume", "path", "invalid-uuid", opts).await; +// assert!(result.is_err()); +// } + +// #[tokio::test] +// async fn test_file_info_from_raw_edge_cases() { +// // Test with empty buffer +// let empty_raw = RawFileInfo { buf: vec![] }; +// let result = file_info_from_raw(empty_raw, "bucket", "object", false).await; +// assert!(result.is_err()); + +// // Test with invalid buffer +// let invalid_raw = RawFileInfo { +// buf: vec![1, 2, 3, 4, 5], +// }; +// let result = file_info_from_raw(invalid_raw, "bucket", "object", false).await; +// assert!(result.is_err()); +// } + +// #[test] +// fn test_file_meta_version_invalid_cases() { +// // Test invalid version +// let version = FileMetaVersion { +// version_type: VersionType::Invalid, +// ..Default::default() +// }; +// assert!(!version.valid()); + +// // Test version with neither object nor delete marker +// let version = FileMetaVersion { +// version_type: VersionType::Object, +// object: None, +// delete_marker: None, +// ..Default::default() +// }; +// assert!(!version.valid()); +// } + +// #[test] +// fn test_meta_object_edge_cases() { +// let obj = MetaObject { +// data_dir: None, +// ..Default::default() +// }; + +// // Test use_data_dir with None (use_data_dir always returns true) +// assert!(obj.use_data_dir()); + +// // Test use_inlinedata (always returns false in current implementation) +// let obj = MetaObject { +// size: 128 * 1024, // 128KB threshold +// ..Default::default() +// }; +// assert!(!obj.use_inlinedata()); // Should be false + +// let obj = MetaObject { +// size: 128 * 1024 - 1, +// ..Default::default() +// }; +// assert!(!obj.use_inlinedata()); // Should also be false (always false) +// } + +// #[test] +// fn test_file_meta_version_header_edge_cases() { +// let header = FileMetaVersionHeader { +// ec_n: 0, +// ec_m: 0, +// ..Default::default() +// }; + +// // Test has_ec with zero values +// assert!(!header.has_ec()); + +// // Test matches_not_strict with different signatures but same version_id +// let version_id = Some(Uuid::new_v4()); +// let header = FileMetaVersionHeader { +// version_id, +// version_type: VersionType::Object, +// signature: [1, 2, 3, 4], +// ..Default::default() +// }; +// let other = FileMetaVersionHeader { +// version_id, +// version_type: VersionType::Object, +// signature: [5, 6, 7, 8], +// ..Default::default() +// }; +// // Should match because they have same version_id and type +// assert!(header.matches_not_strict(&other)); + +// // Test sorts_before with same mod_time but different version_id +// let time = OffsetDateTime::from_unix_timestamp(1000).unwrap(); +// let header_time1 = FileMetaVersionHeader { +// mod_time: Some(time), +// version_id: Some(Uuid::new_v4()), +// ..Default::default() +// }; +// let header_time2 = FileMetaVersionHeader { +// mod_time: Some(time), +// version_id: Some(Uuid::new_v4()), +// ..Default::default() +// }; + +// // Should use version_id for comparison when mod_time is same +// let sorts_before = header_time1.sorts_before(&header_time2); +// assert!(sorts_before || header_time2.sorts_before(&header_time1)); // One should sort before the other +// } + +// #[test] +// fn test_file_meta_add_version_edge_cases() { +// let mut fm = FileMeta::new(); + +// // Test adding version with same version_id (should update) +// let version_id = Some(Uuid::new_v4()); +// let mut fi1 = FileInfo::new("test1", 4, 2); +// fi1.version_id = version_id; +// fi1.size = 1024; +// fi1.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi1).unwrap(); + +// let mut fi2 = FileInfo::new("test2", 4, 2); +// fi2.version_id = version_id; +// fi2.size = 2048; +// fi2.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi2).unwrap(); + +// // Should still have only one version, but updated +// assert_eq!(fm.versions.len(), 1); +// let (_, version) = fm.find_version(version_id).unwrap(); +// if let Some(obj) = version.object { +// assert_eq!(obj.size, 2048); // Size gets updated when adding same version_id +// } +// } + +// #[test] +// fn test_file_meta_delete_version_edge_cases() { +// let mut fm = FileMeta::new(); + +// // Test deleting non-existent version +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = Some(Uuid::new_v4()); + +// let result = fm.delete_version(&fi); +// assert!(result.is_err()); // Should fail for non-existent version +// } + +// #[test] +// fn test_file_meta_shard_data_dir_count_edge_cases() { +// let mut fm = FileMeta::new(); + +// // Test with None data_dir parameter +// let count = fm.shard_data_dir_count(&None, &None); +// assert_eq!(count, 0); + +// // Test with version_id parameter (not None) +// let version_id = Some(Uuid::new_v4()); +// let data_dir = Some(Uuid::new_v4()); + +// let mut fi = FileInfo::new("test", 4, 2); +// fi.version_id = version_id; +// fi.data_dir = data_dir; +// fi.mod_time = Some(OffsetDateTime::now_utc()); +// fm.add_version(fi).unwrap(); + +// let count = fm.shard_data_dir_count(&version_id, &data_dir); +// assert_eq!(count, 0); // Should be 0 because user_data_dir() requires flag + +// // Test with different version_id +// let other_version_id = Some(Uuid::new_v4()); +// let count = fm.shard_data_dir_count(&other_version_id, &data_dir); +// assert_eq!(count, 1); // Should be 1 because the version has matching data_dir and user_data_dir() is true +// } diff --git a/ecstore/src/file_meta_inline.rs b/ecstore/src/file_meta_inline.rs index 083d1b4ff..d7c6af6f9 100644 --- a/ecstore/src/file_meta_inline.rs +++ b/ecstore/src/file_meta_inline.rs @@ -27,11 +27,7 @@ impl InlineData { } pub fn after_version(&self) -> &[u8] { - if self.0.is_empty() { - &self.0 - } else { - &self.0[1..] - } + if self.0.is_empty() { &self.0 } else { &self.0[1..] } } pub fn find(&self, key: &str) -> Result>> { diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs index a46f7fccd..e2f07796d 100644 --- a/ecstore/src/set_disk.rs +++ b/ecstore/src/set_disk.rs @@ -1,7 +1,7 @@ -use crate::disk::error_reduce::{reduce_read_quorum_errs, reduce_write_quorum_errs, OBJECT_OP_IGNORED_ERRS}; +use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs}; use crate::disk::{ - self, conv_part_err_to_int, has_part_err, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, - CHECK_PART_SUCCESS, + self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, + conv_part_err_to_int, has_part_err, }; use crate::erasure_coding; use crate::error::{Error, Result}; @@ -9,24 +9,24 @@ use crate::global::GLOBAL_MRFState; use crate::heal::data_usage_cache::DataUsageCache; use crate::store_api::ObjectToDelete; use crate::{ - cache_value::metacache_set::{list_path_raw, ListPathRawOptions}, - config::{storageclass, GLOBAL_StorageClass}, + cache_value::metacache_set::{ListPathRawOptions, list_path_raw}, + config::{GLOBAL_StorageClass, storageclass}, disk::{ - endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, - DiskInfoOptions, DiskOption, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, - UpdateMetadataOpts, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, + CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, FileInfoVersions, + RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, + UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk, }, - error::{to_object_err, StorageError}, + error::{StorageError, to_object_err}, global::{ - get_global_deployment_id, is_dist_erasure, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, - GLOBAL_LOCAL_DISK_SET_DRIVES, + GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, + is_dist_erasure, }, heal::{ data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT}, data_usage_cache::{DataUsageCacheInfo, DataUsageEntry, DataUsageEntryInfo}, heal_commands::{ - HealOpts, HealScanMode, HealingTracker, DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, - DRIVE_STATE_OK, HEAL_DEEP_SCAN, HEAL_ITEM_OBJECT, HEAL_NORMAL_SCAN, + DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, HEAL_DEEP_SCAN, HEAL_ITEM_OBJECT, + HEAL_NORMAL_SCAN, HealOpts, HealScanMode, HealingTracker, }, heal_ops::BG_HEALING_UUID, }, @@ -39,13 +39,13 @@ use crate::{ store_init::load_format_erasure, utils::{ crypto::{base64_decode, base64_encode, hex}, - path::{encode_dir_object, has_suffix, SLASH_SEPARATOR}, + path::{SLASH_SEPARATOR, encode_dir_object, has_suffix}, }, xhttp, }; use crate::{disk::STORAGE_FORMAT_FILE, heal::mrf::PartialOperation}; use crate::{ - heal::data_scanner::{globalHealConfig, HEAL_DELETE_DANGLING}, + heal::data_scanner::{HEAL_DELETE_DANGLING, globalHealConfig}, store_api::ListObjectVersionsInfo, }; use crate::{ @@ -57,18 +57,18 @@ use chrono::Utc; use futures::future::join_all; use glob::Pattern; use http::HeaderMap; -use lock::{namespace_lock::NsLockMap, LockApi}; +use lock::{LockApi, namespace_lock::NsLockMap}; use madmin::heal_commands::{HealDriveInfo, HealResultItem}; use md5::{Digest as Md5Digest, Md5}; use rand::{ thread_rng, - {seq::SliceRandom, Rng}, + {Rng, seq::SliceRandom}, }; use rustfs_filemeta::{ - file_info_from_raw, merge_file_meta_versions, FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, - MetadataResolutionParams, ObjectPartInfo, RawFileInfo, + FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo, + RawFileInfo, file_info_from_raw, merge_file_meta_versions, }; -use rustfs_rio::{bitrot_verify, BitrotReader, BitrotWriter, EtagResolvable, HashReader, Writer}; +use rustfs_rio::{BitrotReader, BitrotWriter, EtagResolvable, HashReader, Writer, bitrot_verify}; use rustfs_utils::HashAlgorithm; use sha2::{Digest, Sha256}; use std::hash::Hash; @@ -84,8 +84,8 @@ use std::{ }; use time::OffsetDateTime; use tokio::{ - io::{empty, AsyncWrite}, - sync::{broadcast, RwLock}, + io::{AsyncWrite, empty}, + sync::{RwLock, broadcast}, }; use tokio::{ select, @@ -406,11 +406,7 @@ impl SetDisks { } } - if max >= write_quorum { - data_dir - } else { - None - } + if max >= write_quorum { data_dir } else { None } } #[allow(dead_code)] @@ -741,11 +737,7 @@ impl SetDisks { fn common_time(times: &[Option], quorum: usize) -> Option { let (time, count) = Self::common_time_and_occurrence(times); - if count >= quorum { - time - } else { - None - } + if count >= quorum { time } else { None } } fn common_time_and_occurrence(times: &[Option]) -> (Option, usize) { @@ -786,11 +778,7 @@ impl SetDisks { fn common_etag(etags: &[Option], quorum: usize) -> Option { let (etag, count) = Self::common_etags(etags); - if count >= quorum { - etag - } else { - None - } + if count >= quorum { etag } else { None } } fn common_etags(etags: &[Option]) -> (Option, usize) { @@ -1837,13 +1825,7 @@ impl SetDisks { let total_size = fi.size; - let length = { - if length == 0 { - total_size - offset - } else { - length - } - }; + let length = { if length == 0 { total_size - offset } else { length } }; if offset > total_size || offset + length > total_size { return Err(Error::other("offset out of range")); @@ -1896,12 +1878,16 @@ impl SetDisks { readers.push(Some(reader)); errors.push(None); } else if let Some(disk) = disk_op { + // Calculate ceiling division of till_offset by shard_size + let till_offset = + till_offset.div_ceil(erasure.shard_size()) * HashAlgorithm::HighwayHash256.size() + till_offset; + let rd = disk .read_file_stream( bucket, &format!("{}/{}/part.{}", object, files[idx].data_dir.unwrap_or(Uuid::nil()), part_number), + part_offset, till_offset, - part_length, ) .await?; let reader = BitrotReader::new(rd, erasure.shard_size(), HashAlgorithm::HighwayHash256); @@ -2403,8 +2389,10 @@ impl SetDisks { } if !lastest_meta.deleted && lastest_meta.erasure.distribution.len() != available_disks.len() { - let err_str = format!("unexpected file distribution ({:?}) from available disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})", - lastest_meta.erasure.distribution, available_disks, bucket, object, version_id); + let err_str = format!( + "unexpected file distribution ({:?}) from available disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})", + lastest_meta.erasure.distribution, available_disks, bucket, object, version_id + ); warn!(err_str); let err = DiskError::other(err_str); return Ok(( @@ -2416,8 +2404,10 @@ impl SetDisks { let latest_disks = Self::shuffle_disks(&available_disks, &lastest_meta.erasure.distribution); if !lastest_meta.deleted && lastest_meta.erasure.distribution.len() != outdate_disks.len() { - let err_str = format!("unexpected file distribution ({:?}) from outdated disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})", - lastest_meta.erasure.distribution, outdate_disks, bucket, object, version_id); + let err_str = format!( + "unexpected file distribution ({:?}) from outdated disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})", + lastest_meta.erasure.distribution, outdate_disks, bucket, object, version_id + ); warn!(err_str); let err = DiskError::other(err_str); return Ok(( @@ -2428,8 +2418,14 @@ impl SetDisks { } if !lastest_meta.deleted && lastest_meta.erasure.distribution.len() != parts_metadata.len() { - let err_str = format!("unexpected file distribution ({:?}) from metadata entries ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})", - lastest_meta.erasure.distribution, parts_metadata.len(), bucket, object, version_id); + let err_str = format!( + "unexpected file distribution ({:?}) from metadata entries ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})", + lastest_meta.erasure.distribution, + parts_metadata.len(), + bucket, + object, + version_id + ); warn!(err_str); let err = DiskError::other(err_str); return Ok(( @@ -3907,6 +3903,7 @@ impl ObjectIO for SetDisks { }; writers.push(Some(writer)); + errors.push(None); } else { errors.push(Some(DiskError::DiskNotFound)); writers.push(None); @@ -3915,6 +3912,7 @@ impl ObjectIO for SetDisks { let nil_count = errors.iter().filter(|&e| e.is_none()).count(); if nil_count < write_quorum { + error!("not enough disks to write: {:?}", errors); if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) { return Err(to_object_err(write_err.into(), vec![bucket, object])); } @@ -3926,7 +3924,7 @@ impl ObjectIO for SetDisks { let (reader, w_size) = Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?; // TODO: 出错，删除临时目录 - mem::replace(&mut data.stream, reader); + let _ = mem::replace(&mut data.stream, reader); // if let Err(err) = close_bitrot_writers(&mut writers).await { // error!("close_bitrot_writers err {:?}", err); // } @@ -4549,25 +4547,11 @@ impl StorageAPI for SetDisks { let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); - let is_inline_buffer = { - if let Some(sc) = GLOBAL_StorageClass.get() { - sc.should_inline(erasure.shard_file_size(data.content_length), opts.versioned) - } else { - false - } - }; - let mut writers = Vec::with_capacity(shuffle_disks.len()); let mut errors = Vec::with_capacity(shuffle_disks.len()); for disk_op in shuffle_disks.iter() { if let Some(disk) = disk_op { - let writer = if is_inline_buffer { - BitrotWriter::new( - Writer::from_cursor(Cursor::new(Vec::new())), - erasure.shard_size(), - HashAlgorithm::HighwayHash256, - ) - } else { + let writer = { let f = match disk .create_file("", RUSTFS_META_TMP_BUCKET, &tmp_part_path, erasure.shard_file_size(data.content_length)) .await @@ -4584,6 +4568,7 @@ impl StorageAPI for SetDisks { }; writers.push(Some(writer)); + errors.push(None); } else { errors.push(Some(DiskError::DiskNotFound)); writers.push(None); @@ -4601,8 +4586,9 @@ impl StorageAPI for SetDisks { let stream = mem::replace(&mut data.stream, HashReader::new(Box::new(Cursor::new(Vec::new())), 0, 0, None, false)?); - let (mut reader, w_size) = Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?; // TODO: 出错，删除临时目录 - mem::replace(&mut data.stream, reader); + let (reader, w_size) = Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?; // TODO: 出错，删除临时目录 + + let _ = mem::replace(&mut data.stream, reader); let mut etag = data.stream.try_resolve_etag().unwrap_or_default(); @@ -5755,9 +5741,9 @@ fn get_complete_multipart_md5(parts: &[CompletePart]) -> String { #[cfg(test)] mod tests { use super::*; - use crate::disk::error::DiskError; use crate::disk::CHECK_PART_UNKNOWN; use crate::disk::CHECK_PART_VOLUME_NOT_FOUND; + use crate::disk::error::DiskError; use crate::store_api::CompletePart; use rustfs_filemeta::ErasureInfo; use std::collections::HashMap; diff --git a/ecstore/src/utils/bool_flag.rs b/ecstore/src/utils/bool_flag.rs index 1a042af2d..d073af1b5 100644 --- a/ecstore/src/utils/bool_flag.rs +++ b/ecstore/src/utils/bool_flag.rs @@ -1,9 +1,9 @@ -use common::error::{Error, Result}; +use std::io::{Error, Result}; pub fn parse_bool(str: &str) -> Result { match str { "1" | "t" | "T" | "true" | "TRUE" | "True" | "on" | "ON" | "On" | "enabled" => Ok(true), "0" | "f" | "F" | "false" | "FALSE" | "False" | "off" | "OFF" | "Off" | "disabled" => Ok(false), - _ => Err(Error::from_string(format!("ParseBool: parsing {}", str))), + _ => Err(Error::other(format!("ParseBool: parsing {}", str))), } } diff --git a/ecstore/src/utils/ellipses.rs b/ecstore/src/utils/ellipses.rs index f052236ae..f323fd2a2 100644 --- a/ecstore/src/utils/ellipses.rs +++ b/ecstore/src/utils/ellipses.rs @@ -1,6 +1,6 @@ -use common::error::{Error, Result}; use lazy_static::*; use regex::Regex; +use std::io::{Error, Result}; lazy_static! { static ref ELLIPSES_RE: Regex = Regex::new(r"(.*)(\{[0-9a-z]*\.\.\.[0-9a-z]*\})(.*)").unwrap(); @@ -107,7 +107,10 @@ pub fn find_ellipses_patterns(arg: &str) -> Result { let mut parts = match ELLIPSES_RE.captures(arg) { Some(caps) => caps, None => { - return Err(Error::from_string(format!("Invalid ellipsis format in ({}), Ellipsis range must be provided in format {{N...M}} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4", arg))); + return Err(Error::other(format!( + "Invalid ellipsis format in ({}), Ellipsis range must be provided in format {{N...M}} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4", + arg + ))); } }; @@ -144,7 +147,10 @@ pub fn find_ellipses_patterns(arg: &str) -> Result { || p.suffix.contains(OPEN_BRACES) || p.suffix.contains(CLOSE_BRACES) { - return Err(Error::from_string(format!("Invalid ellipsis format in ({}), Ellipsis range must be provided in format {{N...M}} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4", arg))); + return Err(Error::other(format!( + "Invalid ellipsis format in ({}), Ellipsis range must be provided in format {{N...M}} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4", + arg + ))); } } @@ -165,10 +171,10 @@ pub fn has_ellipses>(s: &[T]) -> bool { /// {33...64} pub fn parse_ellipses_range(pattern: &str) -> Result> { if !pattern.contains(OPEN_BRACES) { - return Err(Error::from_string("Invalid argument")); + return Err(Error::other("Invalid argument")); } if !pattern.contains(OPEN_BRACES) { - return Err(Error::from_string("Invalid argument")); + return Err(Error::other("Invalid argument")); } let ellipses_range: Vec<&str> = pattern @@ -178,15 +184,15 @@ pub fn parse_ellipses_range(pattern: &str) -> Result> { .collect(); if ellipses_range.len() != 2 { - return Err(Error::from_string("Invalid argument")); + return Err(Error::other("Invalid argument")); } // TODO: Add support for hexadecimals. - let start = ellipses_range[0].parse::()?; - let end = ellipses_range[1].parse::()?; + let start = ellipses_range[0].parse::().map_err(|e| Error::other(e))?; + let end = ellipses_range[1].parse::().map_err(|e| Error::other(e))?; if start > end { - return Err(Error::from_string("Invalid argument:range start cannot be bigger than end")); + return Err(Error::other("Invalid argument:range start cannot be bigger than end")); } let mut ret: Vec = Vec::with_capacity(end - start + 1); diff --git a/ecstore/src/utils/net.rs b/ecstore/src/utils/net.rs index bcd2c80d1..831ae617e 100644 --- a/ecstore/src/utils/net.rs +++ b/ecstore/src/utils/net.rs @@ -1,5 +1,5 @@ -use common::error::{Error, Result}; use lazy_static::lazy_static; +use std::io::{Error, Result}; use std::{ collections::HashSet, fmt::Display, @@ -23,7 +23,7 @@ pub fn is_socket_addr(addr: &str) -> bool { pub fn check_local_server_addr(server_addr: &str) -> Result { let addr: Vec = match server_addr.to_socket_addrs() { Ok(addr) => addr.collect(), - Err(err) => return Err(Error::new(Box::new(err))), + Err(err) => return Err(err), }; // 0.0.0.0 is a wildcard address and refers to local network @@ -44,7 +44,7 @@ pub fn check_local_server_addr(server_addr: &str) -> Result { } } - Err(Error::from_string("host in server address should be this server")) + Err(Error::other("host in server address should be this server")) } /// checks if the given parameter correspond to one of @@ -55,7 +55,7 @@ pub fn is_local_host(host: Host<&str>, port: u16, local_port: u16) -> Result { let ips = match (domain, 0).to_socket_addrs().map(|v| v.map(|v| v.ip()).collect::>()) { Ok(ips) => ips, - Err(err) => return Err(Error::new(Box::new(err))), + Err(err) => return Err(err), }; ips.iter().any(|ip| local_set.contains(ip)) @@ -79,7 +79,7 @@ pub fn get_host_ip(host: Host<&str>) -> Result> { .map(|v| v.map(|v| v.ip()).collect::>()) { Ok(ips) => Ok(ips), - Err(err) => Err(Error::new(Box::new(err))), + Err(err) => Err(err), }, Host::Ipv4(ip) => { let mut set = HashSet::with_capacity(1); @@ -102,7 +102,7 @@ pub fn get_available_port() -> u16 { pub(crate) fn must_get_local_ips() -> Result> { match netif::up() { Ok(up) => Ok(up.map(|x| x.address().to_owned()).collect()), - Err(err) => Err(Error::from_string(format!("Unable to get IP addresses of this host: {}", err))), + Err(err) => Err(Error::other(format!("Unable to get IP addresses of this host: {}", err))), } } @@ -149,7 +149,7 @@ pub fn parse_and_resolve_address(addr_str: &str) -> Result { let port_str = port; let port: u16 = port_str .parse() - .map_err(|e| Error::from_string(format!("Invalid port format: {}, err:{:?}", addr_str, e)))?; + .map_err(|e| Error::other(format!("Invalid port format: {}, err:{:?}", addr_str, e)))?; let final_port = if port == 0 { get_available_port() // assume get_available_port is available here } else { @@ -199,13 +199,10 @@ mod test { ("localhost:54321", Ok(())), ("0.0.0.0:9000", Ok(())), // (":0", Ok(())), - ("localhost", Err(Error::from_string("invalid socket address"))), - ("", Err(Error::from_string("invalid socket address"))), - ( - "example.org:54321", - Err(Error::from_string("host in server address should be this server")), - ), - (":-10", Err(Error::from_string("invalid port value"))), + ("localhost", Err(Error::other("invalid socket address"))), + ("", Err(Error::other("invalid socket address"))), + ("example.org:54321", Err(Error::other("host in server address should be this server"))), + (":-10", Err(Error::other("invalid port value"))), ]; for test_case in test_cases { diff --git a/ecstore/src/utils/os/linux.rs b/ecstore/src/utils/os/linux.rs index 064b74ae5..e43fc5f7d 100644 --- a/ecstore/src/utils/os/linux.rs +++ b/ecstore/src/utils/os/linux.rs @@ -1,11 +1,11 @@ use nix::sys::stat::{self, stat}; -use nix::sys::statfs::{self, statfs, FsType}; +use nix::sys::statfs::{self, FsType, statfs}; use std::fs::File; use std::io::{self, BufRead, Error, ErrorKind}; use std::path::Path; use crate::disk::Info; -use common::error::{Error as e_Error, Result}; +use std::io::{Error, Result}; use super::IOStats; @@ -29,7 +29,7 @@ pub fn get_info(p: impl AsRef) -> std::io::Result { bfree, p.as_ref().display() ), - )) + )); } }; @@ -44,7 +44,7 @@ pub fn get_info(p: impl AsRef) -> std::io::Result { blocks, p.as_ref().display() ), - )) + )); } }; @@ -60,7 +60,7 @@ pub fn get_info(p: impl AsRef) -> std::io::Result { total, p.as_ref().display() ), - )) + )); } }; @@ -122,7 +122,7 @@ pub fn get_drive_stats(major: u32, minor: u32) -> Result { fn read_drive_stats(stats_file: &str) -> Result { let stats = read_stat(stats_file)?; if stats.len() < 11 { - return Err(e_Error::from_string(format!("found invalid format while reading {}", stats_file))); + return Err(Error::new(ErrorKind::Other, format!("found invalid format while reading {}", stats_file))); } let mut io_stats = IOStats { read_ios: stats[0], diff --git a/ecstore/src/utils/os/unix.rs b/ecstore/src/utils/os/unix.rs index 98b4e187c..117d2f02c 100644 --- a/ecstore/src/utils/os/unix.rs +++ b/ecstore/src/utils/os/unix.rs @@ -1,8 +1,7 @@ use super::IOStats; use crate::disk::Info; -use common::error::Result; use nix::sys::{stat::stat, statfs::statfs}; -use std::io::Error; +use std::io::{Error, Result}; use std::path::Path; /// returns total and free bytes available in a directory, e.g. `/`. @@ -22,7 +21,7 @@ pub fn get_info(p: impl AsRef) -> std::io::Result { bavail, bfree, p.as_ref().display() - ))) + ))); } }; @@ -34,7 +33,7 @@ pub fn get_info(p: impl AsRef) -> std::io::Result { reserved, blocks, p.as_ref().display() - ))) + ))); } }; @@ -47,7 +46,7 @@ pub fn get_info(p: impl AsRef) -> std::io::Result { free, total, p.as_ref().display() - ))) + ))); } }; diff --git a/ecstore/src/utils/os/windows.rs b/ecstore/src/utils/os/windows.rs index a99d7001c..946271504 100644 --- a/ecstore/src/utils/os/windows.rs +++ b/ecstore/src/utils/os/windows.rs @@ -2,8 +2,7 @@ use super::IOStats; use crate::disk::Info; -use common::error::Result; -use std::io::{Error, ErrorKind}; +use std::io::{Error, ErrorKind, Result}; use std::mem; use std::os::windows::ffi::OsStrExt; use std::path::Path; diff --git a/iam/src/error.rs b/iam/src/error.rs index 2e6e094c9..758df3178 100644 --- a/iam/src/error.rs +++ b/iam/src/error.rs @@ -115,6 +115,15 @@ impl From for Error { } } +impl From for ecstore::error::StorageError { + fn from(e: Error) -> Self { + match e { + Error::ConfigNotFound => ecstore::error::StorageError::ConfigNotFound, + _ => ecstore::error::StorageError::other(e), + } + } +} + impl From for Error { fn from(e: policy::error::Error) -> Self { match e { @@ -152,6 +161,12 @@ impl From for Error { } } +impl From for std::io::Error { + fn from(e: Error) -> Self { + std::io::Error::other(e) + } +} + impl From for Error { fn from(e: serde_json::Error) -> Self { Error::other(e) diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 2d90ce5e0..5abb23e29 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -88,6 +88,7 @@ tower-http = { workspace = true, features = [ uuid = { workspace = true } rustfs-filemeta.workspace = true thiserror.workspace = true +rustfs-rio.workspace = true [target.'cfg(target_os = "linux")'.dependencies] libsystemd.workspace = true diff --git a/rustfs/src/admin/handlers/pools.rs b/rustfs/src/admin/handlers/pools.rs index e59015c90..66f85c1e9 100644 --- a/rustfs/src/admin/handlers/pools.rs +++ b/rustfs/src/admin/handlers/pools.rs @@ -7,7 +7,7 @@ use serde_urlencoded::from_bytes; use tokio::sync::broadcast; use tracing::warn; -use crate::{admin::router::Operation, storage::error::to_s3_error}; +use crate::{admin::router::Operation, error::ApiError}; pub struct ListPools {} @@ -33,7 +33,7 @@ impl Operation for ListPools { let mut pools_status = Vec::new(); for (idx, _) in endpoints.as_ref().iter().enumerate() { - let state = store.status(idx).await.map_err(to_s3_error)?; + let state = store.status(idx).await.map_err(ApiError::from)?; pools_status.push(state); } @@ -103,7 +103,7 @@ impl Operation for StatusPool { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - let pools_status = store.status(idx).await.map_err(to_s3_error)?; + let pools_status = store.status(idx).await.map_err(ApiError::from)?; let data = serde_json::to_vec(&pools_status) .map_err(|_e| S3Error::with_message(S3ErrorCode::InternalError, "parse accountInfo failed"))?; @@ -191,7 +191,7 @@ impl Operation for StartDecommission { } if !pools_indices.is_empty() { - store.decommission(ctx_rx, pools_indices).await.map_err(to_s3_error)?; + store.decommission(ctx_rx, pools_indices).await.map_err(ApiError::from)?; } Ok(S3Response::new((StatusCode::OK, Body::default()))) @@ -245,7 +245,7 @@ impl Operation for CancelDecommission { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - store.decommission_cancel(idx).await.map_err(to_s3_error)?; + store.decommission_cancel(idx).await.map_err(ApiError::from)?; Ok(S3Response::new((StatusCode::OK, Body::default()))) } diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index f0229f201..f34c1f1f5 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -3,7 +3,6 @@ pub mod router; mod rpc; pub mod utils; -use common::error::Result; // use ecstore::global::{is_dist_erasure, is_erasure}; use handlers::{ group, policys, pools, rebalance, @@ -18,7 +17,7 @@ use s3s::route::S3Route; const ADMIN_PREFIX: &str = "/rustfs/admin"; -pub fn make_admin_route() -> Result { +pub fn make_admin_route() -> std::io::Result { let mut r: S3Router = S3Router::new(); // 1 @@ -124,7 +123,7 @@ pub fn make_admin_route() -> Result { Ok(r) } -fn register_user_route(r: &mut S3Router) -> Result<()> { +fn register_user_route(r: &mut S3Router) -> std::io::Result<()> { // 1 r.insert( Method::GET, diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index d19d4da2c..7a58ca557 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -1,21 +1,20 @@ -use common::error::Result; -use hyper::http::Extensions; use hyper::HeaderMap; use hyper::Method; use hyper::StatusCode; use hyper::Uri; +use hyper::http::Extensions; use matchit::Params; use matchit::Router; -use s3s::header; -use s3s::route::S3Route; -use s3s::s3_error; use s3s::Body; use s3s::S3Request; use s3s::S3Response; use s3s::S3Result; +use s3s::header; +use s3s::route::S3Route; +use s3s::s3_error; -use super::rpc::RPC_PREFIX; use super::ADMIN_PREFIX; +use super::rpc::RPC_PREFIX; pub struct S3Router { router: Router, @@ -28,12 +27,12 @@ impl S3Router { Self { router } } - pub fn insert(&mut self, method: Method, path: &str, operation: T) -> Result<()> { + pub fn insert(&mut self, method: Method, path: &str, operation: T) -> std::io::Result<()> { let path = Self::make_route_str(method, path); // warn!("set uri {}", &path); - self.router.insert(path, operation)?; + self.router.insert(path, operation).map_err(|e| std::io::Error::other(e))?; Ok(()) } diff --git a/rustfs/src/admin/rpc.rs b/rustfs/src/admin/rpc.rs index c3f9847fc..46959489b 100644 --- a/rustfs/src/admin/rpc.rs +++ b/rustfs/src/admin/rpc.rs @@ -2,7 +2,6 @@ use super::router::AdminOperation; use super::router::Operation; use super::router::S3Router; use crate::storage::ecfs::bytes_stream; -use common::error::Result; use ecstore::disk::DiskAPI; use ecstore::io::READ_BUFFER_SIZE; use ecstore::store::find_local_disk; @@ -10,19 +9,19 @@ use futures::TryStreamExt; use http::StatusCode; use hyper::Method; use matchit::Params; -use s3s::dto::StreamingBlob; -use s3s::s3_error; use s3s::Body; use s3s::S3Request; use s3s::S3Response; use s3s::S3Result; +use s3s::dto::StreamingBlob; +use s3s::s3_error; use serde_urlencoded::from_bytes; use tokio_util::io::ReaderStream; use tokio_util::io::StreamReader; pub const RPC_PREFIX: &str = "/rustfs/rpc"; -pub fn regist_rpc_route(r: &mut S3Router) -> Result<()> { +pub fn regist_rpc_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( Method::GET, format!("{}{}", RPC_PREFIX, "/read_file_stream").as_str(), diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index e9270771e..5c01e4270 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -1,1687 +1,99 @@ use ecstore::error::StorageError; use s3s::{S3Error, S3ErrorCode}; -pub struct Error { +pub type Error = ApiError; +pub type Result = core::result::Result; + +#[derive(Debug)] +pub struct ApiError { pub code: S3ErrorCode, pub message: String, pub source: Option>, } -impl From for Error { - fn from(err: StorageError) -> Self { - Error { - code: S3ErrorCode::Custom(err.to_string()), - message: err.to_string(), +impl std::fmt::Display for ApiError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +impl std::error::Error for ApiError {} + +impl ApiError { + pub fn other(error: E) -> Self + where + E: std::fmt::Display + Into>, + { + ApiError { + code: S3ErrorCode::InternalError, + message: error.to_string(), + source: Some(error.into()), } } } -// /// copy from s3s::S3ErrorCode -// #[derive(thiserror::Error)] -// pub enum Error { -// /// The bucket does not allow ACLs. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// AccessControlListNotSupported, - -// /// Access Denied -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// AccessDenied, - -// /// An access point with an identical name already exists in your account. -// /// -// /// HTTP Status Code: 409 Conflict -// /// -// AccessPointAlreadyOwnedByYou, - -// /// There is a problem with your Amazon Web Services account that prevents the action from completing successfully. Contact Amazon Web Services Support for further assistance. -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// AccountProblem, - -// /// All access to this Amazon S3 resource has been disabled. Contact Amazon Web Services Support for further assistance. -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// AllAccessDisabled, - -// /// The field name matches to multiple fields in the file. Check the SQL expression and the file, and try again. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// AmbiguousFieldName, - -// /// The email address you provided is associated with more than one account. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// AmbiguousGrantByEmailAddress, - -// /// The authorization header you provided is invalid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// AuthorizationHeaderMalformed, - -// /// The authorization query parameters that you provided are not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// AuthorizationQueryParametersError, - -// /// The Content-MD5 you specified did not match what we received. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// BadDigest, - -// /// The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again. -// /// -// /// HTTP Status Code: 409 Conflict -// /// -// BucketAlreadyExists, - -// /// The bucket you tried to create already exists, and you own it. Amazon S3 returns this error in all Amazon Web Services Regions except in the North Virginia Region. For legacy compatibility, if you re-create an existing bucket that you already own in the North Virginia Region, Amazon S3 returns 200 OK and resets the bucket access control lists (ACLs). -// /// -// /// HTTP Status Code: 409 Conflict -// /// -// BucketAlreadyOwnedByYou, - -// /// The bucket you tried to delete has access points attached. Delete your access points before deleting your bucket. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// BucketHasAccessPointsAttached, - -// /// The bucket you tried to delete is not empty. -// /// -// /// HTTP Status Code: 409 Conflict -// /// -// BucketNotEmpty, - -// /// The service is unavailable. Try again later. -// /// -// /// HTTP Status Code: 503 Service Unavailable -// /// -// Busy, - -// /// A quoted record delimiter was found in the file. To allow quoted record delimiters, set AllowQuotedRecordDelimiter to 'TRUE'. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// CSVEscapingRecordDelimiter, - -// /// An error occurred while parsing the CSV file. Check the file and try again. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// CSVParsingError, - -// /// An unescaped quote was found while parsing the CSV file. To allow quoted record delimiters, set AllowQuotedRecordDelimiter to 'TRUE'. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// CSVUnescapedQuote, - -// /// An attempt to convert from one data type to another using CAST failed in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// CastFailed, - -// /// Your Multi-Region Access Point idempotency token was already used for a different request. -// /// -// /// HTTP Status Code: 409 Conflict -// /// -// ClientTokenConflict, - -// /// The length of a column in the result is greater than maxCharsPerColumn of 1 MB. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ColumnTooLong, - -// /// A conflicting operation occurred. If using PutObject you can retry the request. If using multipart upload you should initiate another CreateMultipartUpload request and re-upload each part. -// /// -// /// HTTP Status Code: 409 Conflict -// /// -// ConditionalRequestConflict, - -// /// Returned to the original caller when an error is encountered while reading the WriteGetObjectResponse body. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ConnectionClosedByRequester, - -// /// This request does not support credentials. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// CredentialsNotSupported, - -// /// Cross-location logging not allowed. Buckets in one geographic location cannot log information to a bucket in another location. -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// CrossLocationLoggingProhibited, - -// /// The device is not currently active. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// DeviceNotActiveError, - -// /// The request body cannot be empty. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EmptyRequestBody, - -// /// Direct requests to the correct endpoint. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EndpointNotFound, - -// /// Your proposed upload exceeds the maximum allowed object size. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EntityTooLarge, - -// /// Your proposed upload is smaller than the minimum allowed object size. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EntityTooSmall, - -// /// A column name or a path provided does not exist in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EvaluatorBindingDoesNotExist, - -// /// There is an incorrect number of arguments in the function call in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EvaluatorInvalidArguments, - -// /// The timestamp format string in the SQL expression is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EvaluatorInvalidTimestampFormatPattern, - -// /// The timestamp format pattern contains a symbol in the SQL expression that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EvaluatorInvalidTimestampFormatPatternSymbol, - -// /// The timestamp format pattern contains a valid format symbol that cannot be applied to timestamp parsing in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EvaluatorInvalidTimestampFormatPatternSymbolForParsing, - -// /// The timestamp format pattern contains a token in the SQL expression that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EvaluatorInvalidTimestampFormatPatternToken, - -// /// An argument given to the LIKE expression was not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EvaluatorLikePatternInvalidEscapeSequence, - -// /// LIMIT must not be negative. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EvaluatorNegativeLimit, - -// /// The timestamp format pattern contains multiple format specifiers representing the timestamp field in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EvaluatorTimestampFormatPatternDuplicateFields, - -// /// The timestamp format pattern contains a 12-hour hour of day format symbol but doesn't also contain an AM/PM field, or it contains a 24-hour hour of day format specifier and contains an AM/PM field in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EvaluatorTimestampFormatPatternHourClockAmPmMismatch, - -// /// The timestamp format pattern contains an unterminated token in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// EvaluatorUnterminatedTimestampFormatPatternToken, - -// /// The provided token has expired. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ExpiredToken, - -// /// The SQL expression is too long. The maximum byte-length for an SQL expression is 256 KB. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ExpressionTooLong, - -// /// The query cannot be evaluated. Check the file and try again. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ExternalEvalException, - -// /// This error might occur for the following reasons: -// /// -// /// -// /// You are trying to access a bucket from a different Region than where the bucket exists. -// /// -// /// You attempt to create a bucket with a location constraint that corresponds to a different region than the regional endpoint the request was sent to. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// IllegalLocationConstraintException, - -// /// An illegal argument was used in the SQL function. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// IllegalSqlFunctionArgument, - -// /// Indicates that the versioning configuration specified in the request is invalid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// IllegalVersioningConfigurationException, - -// /// You did not provide the number of bytes specified by the Content-Length HTTP header -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// IncompleteBody, - -// /// The specified bucket exists in another Region. Direct requests to the correct endpoint. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// IncorrectEndpoint, - -// /// POST requires exactly one file upload per request. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// IncorrectNumberOfFilesInPostRequest, - -// /// An incorrect argument type was specified in a function call in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// IncorrectSqlFunctionArgumentType, - -// /// Inline data exceeds the maximum allowed size. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InlineDataTooLarge, - -// /// An integer overflow or underflow occurred in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// IntegerOverflow, - -// /// We encountered an internal error. Please try again. -// /// -// /// HTTP Status Code: 500 Internal Server Error -// /// -// InternalError, - -// /// The Amazon Web Services access key ID you provided does not exist in our records. -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// InvalidAccessKeyId, - -// /// The specified access point name or account is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidAccessPoint, - -// /// The specified access point alias name is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidAccessPointAliasError, - -// /// You must specify the Anonymous role. -// /// -// InvalidAddressingHeader, - -// /// Invalid Argument -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidArgument, - -// /// Bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidBucketAclWithObjectOwnership, - -// /// The specified bucket is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidBucketName, - -// /// The value of the expected bucket owner parameter must be an AWS account ID. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidBucketOwnerAWSAccountID, - -// /// The request is not valid with the current state of the bucket. -// /// -// /// HTTP Status Code: 409 Conflict -// /// -// InvalidBucketState, - -// /// An attempt to convert from one data type to another using CAST failed in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidCast, - -// /// The column index in the SQL expression is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidColumnIndex, - -// /// The file is not in a supported compression format. Only GZIP and BZIP2 are supported. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidCompressionFormat, - -// /// The data source type is not valid. Only CSV, JSON, and Parquet are supported. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidDataSource, - -// /// The SQL expression contains a data type that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidDataType, - -// /// The Content-MD5 you specified is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidDigest, - -// /// The encryption request you specified is not valid. The valid value is AES256. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidEncryptionAlgorithmError, - -// /// The ExpressionType value is not valid. Only SQL expressions are supported. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidExpressionType, - -// /// The FileHeaderInfo value is not valid. Only NONE, USE, and IGNORE are supported. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidFileHeaderInfo, - -// /// The host headers provided in the request used the incorrect style addressing. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidHostHeader, - -// /// The request is made using an unexpected HTTP method. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidHttpMethod, - -// /// The JsonType value is not valid. Only DOCUMENT and LINES are supported. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidJsonType, - -// /// The key path in the SQL expression is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidKeyPath, - -// /// The specified location constraint is not valid. For more information about Regions, see How to Select a Region for Your Buckets. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidLocationConstraint, - -// /// The action is not valid for the current state of the object. -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// InvalidObjectState, - -// /// One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidPart, - -// /// The list of parts was not in ascending order. Parts list must be specified in order by part number. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidPartOrder, - -// /// All access to this object has been disabled. Please contact Amazon Web Services Support for further assistance. -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// InvalidPayer, - -// /// The content of the form does not meet the conditions specified in the policy document. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidPolicyDocument, - -// /// The QuoteFields value is not valid. Only ALWAYS and ASNEEDED are supported. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidQuoteFields, - -// /// The requested range cannot be satisfied. -// /// -// /// HTTP Status Code: 416 Requested Range NotSatisfiable -// /// -// InvalidRange, - -// /// + Please use AWS4-HMAC-SHA256. -// /// + SOAP requests must be made over an HTTPS connection. -// /// + Amazon S3 Transfer Acceleration is not supported for buckets with non-DNS compliant names. -// /// + Amazon S3 Transfer Acceleration is not supported for buckets with periods (.) in their names. -// /// + Amazon S3 Transfer Accelerate endpoint only supports virtual style requests. -// /// + Amazon S3 Transfer Accelerate is not configured on this bucket. -// /// + Amazon S3 Transfer Accelerate is disabled on this bucket. -// /// + Amazon S3 Transfer Acceleration is not supported on this bucket. Contact Amazon Web Services Support for more information. -// /// + Amazon S3 Transfer Acceleration cannot be enabled on this bucket. Contact Amazon Web Services Support for more information. -// /// -// /// HTTP Status Code: 400 Bad Request -// InvalidRequest, - -// /// The value of a parameter in the SelectRequest element is not valid. Check the service API documentation and try again. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidRequestParameter, - -// /// The SOAP request body is invalid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidSOAPRequest, - -// /// The provided scan range is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidScanRange, - -// /// The provided security credentials are not valid. -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// InvalidSecurity, - -// /// Returned if the session doesn't exist anymore because it timed out or expired. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidSessionException, - -// /// The request signature that the server calculated does not match the signature that you provided. Check your AWS secret access key and signing method. For more information, see Signing and authenticating REST requests. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidSignature, - -// /// The storage class you specified is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidStorageClass, - -// /// The SQL expression contains a table alias that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidTableAlias, - -// /// Your request contains tag input that is not valid. For example, your request might contain duplicate keys, keys or values that are too long, or system tags. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidTag, - -// /// The target bucket for logging does not exist, is not owned by you, or does not have the appropriate grants for the log-delivery group. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidTargetBucketForLogging, - -// /// The encoding type is not valid. Only UTF-8 encoding is supported. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidTextEncoding, - -// /// The provided token is malformed or otherwise invalid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidToken, - -// /// Couldn't parse the specified URI. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// InvalidURI, - -// /// An error occurred while parsing the JSON file. Check the file and try again. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// JSONParsingError, - -// /// Your key is too long. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// KeyTooLongError, - -// /// The SQL expression contains a character that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// LexerInvalidChar, - -// /// The SQL expression contains an operator that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// LexerInvalidIONLiteral, - -// /// The SQL expression contains an operator that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// LexerInvalidLiteral, - -// /// The SQL expression contains a literal that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// LexerInvalidOperator, - -// /// The argument given to the LIKE clause in the SQL expression is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// LikeInvalidInputs, - -// /// The XML you provided was not well-formed or did not validate against our published schema. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// MalformedACLError, - -// /// The body of your POST request is not well-formed multipart/form-data. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// MalformedPOSTRequest, - -// /// Your policy contains a principal that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// MalformedPolicy, - -// /// This happens when the user sends malformed XML (XML that doesn't conform to the published XSD) for the configuration. The error message is, "The XML you provided was not well-formed or did not validate against our published schema." -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// MalformedXML, - -// /// Your request was too big. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// MaxMessageLengthExceeded, - -// /// Failed to parse SQL expression, try reducing complexity. For example, reduce number of operators used. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// MaxOperatorsExceeded, - -// /// Your POST request fields preceding the upload file were too large. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// MaxPostPreDataLengthExceededError, - -// /// Your metadata headers exceed the maximum allowed metadata size. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// MetadataTooLarge, - -// /// The specified method is not allowed against this resource. -// /// -// /// HTTP Status Code: 405 Method Not Allowed -// /// -// MethodNotAllowed, - -// /// A SOAP attachment was expected, but none were found. -// /// -// MissingAttachment, - -// /// The request was not signed. -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// MissingAuthenticationToken, - -// /// You must provide the Content-Length HTTP header. -// /// -// /// HTTP Status Code: 411 Length Required -// /// -// MissingContentLength, - -// /// This happens when the user sends an empty XML document as a request. The error message is, "Request body is empty." -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// MissingRequestBodyError, - -// /// The SelectRequest entity is missing a required parameter. Check the service documentation and try again. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// MissingRequiredParameter, - -// /// The SOAP 1.1 request is missing a security element. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// MissingSecurityElement, - -// /// Your request is missing a required header. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// MissingSecurityHeader, - -// /// Multiple data sources are not supported. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// MultipleDataSourcesUnsupported, - -// /// There is no such thing as a logging status subresource for a key. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// NoLoggingStatusForKey, - -// /// The specified access point does not exist. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchAccessPoint, - -// /// The specified request was not found. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchAsyncRequest, - -// /// The specified bucket does not exist. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchBucket, - -// /// The specified bucket does not have a bucket policy. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchBucketPolicy, - -// /// The specified bucket does not have a CORS configuration. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchCORSConfiguration, - -// /// The specified key does not exist. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchKey, - -// /// The lifecycle configuration does not exist. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchLifecycleConfiguration, - -// /// The specified Multi-Region Access Point does not exist. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchMultiRegionAccessPoint, - -// /// The specified object does not have an ObjectLock configuration. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchObjectLockConfiguration, - -// /// The specified resource doesn't exist. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchResource, - -// /// The specified tag does not exist. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchTagSet, - -// /// The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchUpload, - -// /// Indicates that the version ID specified in the request does not match an existing version. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchVersion, - -// /// The specified bucket does not have a website configuration. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoSuchWebsiteConfiguration, - -// /// No transformation found for this Object Lambda Access Point. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// NoTransformationDefined, - -// /// The device that generated the token is not owned by the authenticated user. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// NotDeviceOwnerError, - -// /// A header you provided implies functionality that is not implemented. -// /// -// /// HTTP Status Code: 501 Not Implemented -// /// -// NotImplemented, - -// /// The resource was not changed. -// /// -// /// HTTP Status Code: 304 Not Modified -// /// -// NotModified, - -// /// Your account is not signed up for the Amazon S3 service. You must sign up before you can use Amazon S3. You can sign up at the following URL: Amazon S3 -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// NotSignedUp, - -// /// An error occurred while parsing a number. This error can be caused by underflow or overflow of integers. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// NumberFormatError, - -// /// The Object Lock configuration does not exist for this bucket. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// ObjectLockConfigurationNotFoundError, - -// /// InputSerialization specifies more than one format (CSV, JSON, or Parquet), or OutputSerialization specifies more than one format (CSV or JSON). For InputSerialization and OutputSerialization, you can specify only one format for each. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ObjectSerializationConflict, - -// /// A conflicting conditional action is currently in progress against this resource. Try again. -// /// -// /// HTTP Status Code: 409 Conflict -// /// -// OperationAborted, - -// /// The number of columns in the result is greater than the maximum allowable number of columns. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// OverMaxColumn, - -// /// The Parquet file is above the max row group size. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// OverMaxParquetBlockSize, - -// /// The length of a record in the input or result is greater than the maxCharsPerRecord limit of 1 MB. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// OverMaxRecordSize, - -// /// The bucket ownership controls were not found. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// OwnershipControlsNotFoundError, - -// /// An error occurred while parsing the Parquet file. Check the file and try again. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParquetParsingError, - -// /// The specified Parquet compression codec is not supported. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParquetUnsupportedCompressionCodec, - -// /// Other expressions are not allowed in the SELECT list when * is used without dot notation in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseAsteriskIsNotAloneInSelectList, - -// /// Cannot mix [] and * in the same expression in a SELECT list in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseCannotMixSqbAndWildcardInSelectList, - -// /// The SQL expression CAST has incorrect arity. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseCastArity, - -// /// The SQL expression contains an empty SELECT clause. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseEmptySelect, - -// /// The expected token in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpected2TokenTypes, - -// /// The expected argument delimiter in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedArgumentDelimiter, - -// /// The expected date part in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedDatePart, - -// /// The expected SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedExpression, - -// /// The expected identifier for the alias in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedIdentForAlias, - -// /// The expected identifier for AT name in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedIdentForAt, - -// /// GROUP is not supported in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedIdentForGroupName, - -// /// The expected keyword in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedKeyword, - -// /// The expected left parenthesis after CAST in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedLeftParenAfterCast, - -// /// The expected left parenthesis in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedLeftParenBuiltinFunctionCall, - -// /// The expected left parenthesis in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedLeftParenValueConstructor, - -// /// The SQL expression contains an unsupported use of MEMBER. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedMember, - -// /// The expected number in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedNumber, - -// /// The expected right parenthesis character in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedRightParenBuiltinFunctionCall, - -// /// The expected token in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedTokenType, - -// /// The expected type name in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedTypeName, - -// /// The expected WHEN clause in the SQL expression was not found. CASE is not supported. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseExpectedWhenClause, - -// /// The use of * in the SELECT list in the SQL expression is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseInvalidContextForWildcardInSelectList, - -// /// The SQL expression contains a path component that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseInvalidPathComponent, - -// /// The SQL expression contains a parameter value that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseInvalidTypeParam, - -// /// JOIN is not supported in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseMalformedJoin, - -// /// The expected identifier after the @ symbol in the SQL expression was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseMissingIdentAfterAt, - -// /// Only one argument is supported for aggregate functions in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseNonUnaryAgregateFunctionCall, - -// /// The SQL expression contains a missing FROM after the SELECT list. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseSelectMissingFrom, - -// /// The SQL expression contains an unexpected keyword. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseUnExpectedKeyword, - -// /// The SQL expression contains an unexpected operator. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseUnexpectedOperator, - -// /// The SQL expression contains an unexpected term. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseUnexpectedTerm, - -// /// The SQL expression contains an unexpected token. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseUnexpectedToken, - -// /// The SQL expression contains an operator that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseUnknownOperator, - -// /// The SQL expression contains an unsupported use of ALIAS. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseUnsupportedAlias, - -// /// Only COUNT with (*) as a parameter is supported in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseUnsupportedCallWithStar, - -// /// The SQL expression contains an unsupported use of CASE. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseUnsupportedCase, - -// /// The SQL expression contains an unsupported use of CASE. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseUnsupportedCaseClause, - -// /// The SQL expression contains an unsupported use of GROUP BY. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseUnsupportedLiteralsGroupBy, - -// /// The SQL expression contains an unsupported use of SELECT. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseUnsupportedSelect, - -// /// The SQL expression contains unsupported syntax. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseUnsupportedSyntax, - -// /// The SQL expression contains an unsupported token. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ParseUnsupportedToken, - -// /// The bucket you are attempting to access must be addressed using the specified endpoint. Send all future requests to this endpoint. -// /// -// /// HTTP Status Code: 301 Moved Permanently -// /// -// PermanentRedirect, - -// /// The API operation you are attempting to access must be addressed using the specified endpoint. Send all future requests to this endpoint. -// /// -// /// HTTP Status Code: 301 Moved Permanently -// /// -// PermanentRedirectControlError, - -// /// At least one of the preconditions you specified did not hold. -// /// -// /// HTTP Status Code: 412 Precondition Failed -// /// -// PreconditionFailed, - -// /// Temporary redirect. -// /// -// /// HTTP Status Code: 307 Moved Temporarily -// /// -// Redirect, - -// /// There is no replication configuration for this bucket. -// /// -// /// HTTP Status Code: 404 Not Found -// /// -// ReplicationConfigurationNotFoundError, - -// /// The request header and query parameters used to make the request exceed the maximum allowed size. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// RequestHeaderSectionTooLarge, - -// /// Bucket POST must be of the enclosure-type multipart/form-data. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// RequestIsNotMultiPartContent, - -// /// The difference between the request time and the server's time is too large. -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// RequestTimeTooSkewed, - -// /// Your socket connection to the server was not read from or written to within the timeout period. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// RequestTimeout, - -// /// Requesting the torrent file of a bucket is not permitted. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// RequestTorrentOfBucketError, - -// /// Returned to the original caller when an error is encountered while reading the WriteGetObjectResponse body. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ResponseInterrupted, - -// /// Object restore is already in progress. -// /// -// /// HTTP Status Code: 409 Conflict -// /// -// RestoreAlreadyInProgress, - -// /// The server-side encryption configuration was not found. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ServerSideEncryptionConfigurationNotFoundError, - -// /// Service is unable to handle request. -// /// -// /// HTTP Status Code: 503 Service Unavailable -// /// -// ServiceUnavailable, - -// /// The request signature we calculated does not match the signature you provided. Check your Amazon Web Services secret access key and signing method. For more information, see REST Authentication and SOAP Authentication for details. -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// SignatureDoesNotMatch, - -// /// Reduce your request rate. -// /// -// /// HTTP Status Code: 503 Slow Down -// /// -// SlowDown, - -// /// You are being redirected to the bucket while DNS updates. -// /// -// /// HTTP Status Code: 307 Moved Temporarily -// /// -// TemporaryRedirect, - -// /// The serial number and/or token code you provided is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// TokenCodeInvalidError, - -// /// The provided token must be refreshed. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// TokenRefreshRequired, - -// /// You have attempted to create more access points than are allowed for an account. For more information, see Amazon Simple Storage Service endpoints and quotas in the AWS General Reference. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// TooManyAccessPoints, - -// /// You have attempted to create more buckets than allowed. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// TooManyBuckets, - -// /// You have attempted to create a Multi-Region Access Point with more Regions than are allowed for an account. For more information, see Amazon Simple Storage Service endpoints and quotas in the AWS General Reference. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// TooManyMultiRegionAccessPointregionsError, - -// /// You have attempted to create more Multi-Region Access Points than are allowed for an account. For more information, see Amazon Simple Storage Service endpoints and quotas in the AWS General Reference. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// TooManyMultiRegionAccessPoints, - -// /// The number of tags exceeds the limit of 50 tags. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// TooManyTags, - -// /// Object decompression failed. Check that the object is properly compressed using the format specified in the request. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// TruncatedInput, - -// /// You are not authorized to perform this operation. -// /// -// /// HTTP Status Code: 401 Unauthorized -// /// -// UnauthorizedAccess, - -// /// Applicable in China Regions only. Returned when a request is made to a bucket that doesn't have an ICP license. For more information, see ICP Recordal. -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// UnauthorizedAccessError, - -// /// This request does not support content. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnexpectedContent, - -// /// Applicable in China Regions only. This request was rejected because the IP was unexpected. -// /// -// /// HTTP Status Code: 403 Forbidden -// /// -// UnexpectedIPError, - -// /// We encountered a record type that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnrecognizedFormatException, - -// /// The email address you provided does not match any account on record. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnresolvableGrantByEmailAddress, - -// /// The request contained an unsupported argument. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnsupportedArgument, - -// /// We encountered an unsupported SQL function. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnsupportedFunction, - -// /// The specified Parquet type is not supported. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnsupportedParquetType, - -// /// A range header is not supported for this operation. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnsupportedRangeHeader, - -// /// Scan range queries are not supported on this type of object. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnsupportedScanRangeInput, - -// /// The provided request is signed with an unsupported STS Token version or the signature version is not supported. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnsupportedSignature, - -// /// We encountered an unsupported SQL operation. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnsupportedSqlOperation, - -// /// We encountered an unsupported SQL structure. Check the SQL Reference. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnsupportedSqlStructure, - -// /// We encountered a storage class that is not supported. Only STANDARD, STANDARD_IA, and ONEZONE_IA storage classes are supported. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnsupportedStorageClass, - -// /// We encountered syntax that is not valid. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnsupportedSyntax, - -// /// Your query contains an unsupported type for comparison (e.g. verifying that a Parquet INT96 column type is greater than 0). -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UnsupportedTypeForQuerying, - -// /// The bucket POST must contain the specified field name. If it is specified, check the order of the fields. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// UserKeyMustBeSpecified, - -// /// A timestamp parse failure occurred in the SQL expression. -// /// -// /// HTTP Status Code: 400 Bad Request -// /// -// ValueParseFailure, - -// Custom(String), -// } - -// #[derive(Debug, thiserror::Error)] -// pub enum Error { -// #[error("Faulty disk")] -// FaultyDisk, - -// #[error("Disk full")] -// DiskFull, - -// #[error("Volume not found")] -// VolumeNotFound, - -// #[error("Volume exists")] -// VolumeExists, - -// #[error("File not found")] -// FileNotFound, - -// #[error("File version not found")] -// FileVersionNotFound, - -// #[error("File name too long")] -// FileNameTooLong, - -// #[error("File access denied")] -// FileAccessDenied, - -// #[error("File is corrupted")] -// FileCorrupt, - -// #[error("Not a regular file")] -// IsNotRegular, - -// #[error("Volume not empty")] -// VolumeNotEmpty, - -// #[error("Volume access denied")] -// VolumeAccessDenied, - -// #[error("Corrupted format")] -// CorruptedFormat, - -// #[error("Corrupted backend")] -// CorruptedBackend, - -// #[error("Unformatted disk")] -// UnformattedDisk, - -// #[error("Disk not found")] -// DiskNotFound, - -// #[error("Drive is root")] -// DriveIsRoot, - -// #[error("Faulty remote disk")] -// FaultyRemoteDisk, - -// #[error("Disk access denied")] -// DiskAccessDenied, - -// #[error("Unexpected error")] -// Unexpected, - -// #[error("Too many open files")] -// TooManyOpenFiles, - -// #[error("No heal required")] -// NoHealRequired, - -// #[error("Config not found")] -// ConfigNotFound, - -// #[error("not implemented")] -// NotImplemented, - -// #[error("Invalid arguments provided for {0}/{1}-{2}")] -// InvalidArgument(String, String, String), - -// #[error("method not allowed")] -// MethodNotAllowed, - -// #[error("Bucket not found: {0}")] -// BucketNotFound(String), - -// #[error("Bucket not empty: {0}")] -// BucketNotEmpty(String), - -// #[error("Bucket name invalid: {0}")] -// BucketNameInvalid(String), - -// #[error("Object name invalid: {0}/{1}")] -// ObjectNameInvalid(String, String), - -// #[error("Bucket exists: {0}")] -// BucketExists(String), -// #[error("Storage reached its minimum free drive threshold.")] -// StorageFull, -// #[error("Please reduce your request rate")] -// SlowDown, - -// #[error("Prefix access is denied:{0}/{1}")] -// PrefixAccessDenied(String, String), - -// #[error("Invalid UploadID KeyCombination: {0}/{1}")] -// InvalidUploadIDKeyCombination(String, String), - -// #[error("Malformed UploadID: {0}")] -// MalformedUploadID(String), - -// #[error("Object name too long: {0}/{1}")] -// ObjectNameTooLong(String, String), - -// #[error("Object name contains forward slash as prefix: {0}/{1}")] -// ObjectNamePrefixAsSlash(String, String), - -// #[error("Object not found: {0}/{1}")] -// ObjectNotFound(String, String), - -// #[error("Version not found: {0}/{1}-{2}")] -// VersionNotFound(String, String, String), - -// #[error("Invalid upload id: {0}/{1}-{2}")] -// InvalidUploadID(String, String, String), - -// #[error("Specified part could not be found. PartNumber {0}, Expected {1}, got {2}")] -// InvalidPart(usize, String, String), - -// #[error("Invalid version id: {0}/{1}-{2}")] -// InvalidVersionID(String, String, String), -// #[error("invalid data movement operation, source and destination pool are the same for : {0}/{1}-{2}")] -// DataMovementOverwriteErr(String, String, String), - -// #[error("Object exists on :{0} as directory {1}")] -// ObjectExistsAsDirectory(String, String), - -// // #[error("Storage resources are insufficient for the read operation")] -// // InsufficientReadQuorum, - -// // #[error("Storage resources are insufficient for the write operation")] -// // InsufficientWriteQuorum, -// #[error("Decommission not started")] -// DecommissionNotStarted, -// #[error("Decommission already running")] -// DecommissionAlreadyRunning, - -// #[error("DoneForNow")] -// DoneForNow, - -// #[error("erasure read quorum")] -// ErasureReadQuorum, - -// #[error("erasure write quorum")] -// ErasureWriteQuorum, - -// #[error("not first disk")] -// NotFirstDisk, - -// #[error("first disk wiat")] -// FirstDiskWait, - -// #[error("Io error: {0}")] -// Io(std::io::Error), -// } - -// impl Error { -// pub fn other(error: E) -> Self -// where -// E: Into>, -// { -// Error::Io(std::io::Error::other(error)) -// } -// } - -// impl From for Error { -// fn from(err: StorageError) -> Self { -// match err { -// StorageError::FaultyDisk => Error::FaultyDisk, -// StorageError::DiskFull => Error::DiskFull, -// StorageError::VolumeNotFound => Error::VolumeNotFound, -// StorageError::VolumeExists => Error::VolumeExists, -// StorageError::FileNotFound => Error::FileNotFound, -// StorageError::FileVersionNotFound => Error::FileVersionNotFound, -// StorageError::FileNameTooLong => Error::FileNameTooLong, -// StorageError::FileAccessDenied => Error::FileAccessDenied, -// StorageError::FileCorrupt => Error::FileCorrupt, -// StorageError::IsNotRegular => Error::IsNotRegular, -// StorageError::VolumeNotEmpty => Error::VolumeNotEmpty, -// StorageError::VolumeAccessDenied => Error::VolumeAccessDenied, -// StorageError::CorruptedFormat => Error::CorruptedFormat, -// StorageError::CorruptedBackend => Error::CorruptedBackend, -// StorageError::UnformattedDisk => Error::UnformattedDisk, -// StorageError::DiskNotFound => Error::DiskNotFound, -// StorageError::DriveIsRoot => Error::DriveIsRoot, -// StorageError::FaultyRemoteDisk => Error::FaultyRemoteDisk, -// StorageError::DiskAccessDenied => Error::DiskAccessDenied, -// StorageError::Unexpected => Error::Unexpected, -// StorageError::TooManyOpenFiles => Error::TooManyOpenFiles, -// StorageError::NoHealRequired => Error::NoHealRequired, -// StorageError::ConfigNotFound => Error::ConfigNotFound, -// StorageError::NotImplemented => Error::NotImplemented, -// StorageError::InvalidArgument(bucket, object, version_id) => Error::InvalidArgument(bucket, object, version_id), -// StorageError::MethodNotAllowed => Error::MethodNotAllowed, -// StorageError::BucketNotFound(bucket) => Error::BucketNotFound(bucket), -// StorageError::BucketNotEmpty(bucket) => Error::BucketNotEmpty(bucket), -// StorageError::BucketNameInvalid(bucket) => Error::BucketNameInvalid(bucket), -// StorageError::ObjectNameInvalid(bucket, object) => Error::ObjectNameInvalid(bucket, object), -// StorageError::BucketExists(bucket) => Error::BucketExists(bucket), -// StorageError::StorageFull => Error::StorageFull, -// StorageError::SlowDown => Error::SlowDown, -// StorageError::PrefixAccessDenied(bucket, object) => Error::PrefixAccessDenied(bucket, object), -// StorageError::InvalidUploadIDKeyCombination(bucket, object) => Error::InvalidUploadIDKeyCombination(bucket, object), -// StorageError::MalformedUploadID(upload_id) => Error::MalformedUploadID(upload_id), -// StorageError::ObjectNameTooLong(bucket, object) => Error::ObjectNameTooLong(bucket, object), -// StorageError::ObjectNamePrefixAsSlash(bucket, object) => Error::ObjectNamePrefixAsSlash(bucket, object), -// StorageError::ObjectNotFound(bucket, object) => Error::ObjectNotFound(bucket, object), -// StorageError::VersionNotFound(bucket, object, version_id) => Error::VersionNotFound(bucket, object, version_id), -// StorageError::InvalidUploadID(bucket, object, version_id) => Error::InvalidUploadID(bucket, object, version_id), -// StorageError::InvalidPart(part_number, bucket, object) => Error::InvalidPart(part_number, bucket, object), -// StorageError::InvalidVersionID(bucket, object, version_id) => Error::InvalidVersionID(bucket, object, version_id), -// StorageError::DataMovementOverwriteErr(bucket, object, version_id) => { -// Error::DataMovementOverwriteErr(bucket, object, version_id) -// } -// StorageError::ObjectExistsAsDirectory(bucket, object) => Error::ObjectExistsAsDirectory(bucket, object), -// StorageError::DecommissionNotStarted => Error::DecommissionNotStarted, -// StorageError::DecommissionAlreadyRunning => Error::DecommissionAlreadyRunning, -// StorageError::DoneForNow => Error::DoneForNow, -// StorageError::ErasureReadQuorum => Error::ErasureReadQuorum, -// StorageError::ErasureWriteQuorum => Error::ErasureWriteQuorum, -// StorageError::NotFirstDisk => Error::NotFirstDisk, -// StorageError::FirstDiskWait => Error::FirstDiskWait, -// StorageError::Io(io_error) => Error::Io(io_error), -// } -// } -// } +impl From for S3Error { + fn from(err: ApiError) -> Self { + let mut s3e = S3Error::with_message(err.code, err.message); + if let Some(source) = err.source { + s3e.set_source(source); + } + s3e + } +} + +impl From for ApiError { + fn from(err: StorageError) -> Self { + let code = match &err { + StorageError::NotImplemented => S3ErrorCode::NotImplemented, + StorageError::InvalidArgument(_, _, _) => S3ErrorCode::InvalidArgument, + StorageError::MethodNotAllowed => S3ErrorCode::MethodNotAllowed, + StorageError::BucketNotFound(_) => S3ErrorCode::NoSuchBucket, + StorageError::BucketNotEmpty(_) => S3ErrorCode::BucketNotEmpty, + StorageError::BucketNameInvalid(_) => S3ErrorCode::InvalidBucketName, + StorageError::ObjectNameInvalid(_, _) => S3ErrorCode::InvalidArgument, + StorageError::BucketExists(_) => S3ErrorCode::BucketAlreadyExists, + StorageError::StorageFull => S3ErrorCode::ServiceUnavailable, + StorageError::SlowDown => S3ErrorCode::SlowDown, + StorageError::PrefixAccessDenied(_, _) => S3ErrorCode::AccessDenied, + StorageError::InvalidUploadIDKeyCombination(_, _) => S3ErrorCode::InvalidArgument, + StorageError::ObjectNameTooLong(_, _) => S3ErrorCode::InvalidArgument, + StorageError::ObjectNamePrefixAsSlash(_, _) => S3ErrorCode::InvalidArgument, + StorageError::ObjectNotFound(_, _) => S3ErrorCode::NoSuchKey, + StorageError::ConfigNotFound => S3ErrorCode::NoSuchKey, + StorageError::VolumeNotFound => S3ErrorCode::NoSuchBucket, + StorageError::FileNotFound => S3ErrorCode::NoSuchKey, + StorageError::FileVersionNotFound => S3ErrorCode::NoSuchVersion, + StorageError::VersionNotFound(_, _, _) => S3ErrorCode::NoSuchVersion, + StorageError::InvalidUploadID(_, _, _) => S3ErrorCode::InvalidPart, + StorageError::InvalidVersionID(_, _, _) => S3ErrorCode::InvalidArgument, + StorageError::DataMovementOverwriteErr(_, _, _) => S3ErrorCode::InvalidArgument, + StorageError::ObjectExistsAsDirectory(_, _) => S3ErrorCode::InvalidArgument, + StorageError::InvalidPart(_, _, _) => S3ErrorCode::InvalidPart, + _ => S3ErrorCode::InternalError, + }; + + ApiError { + code, + message: err.to_string(), + source: Some(Box::new(err)), + } + } +} + +impl From for ApiError { + fn from(err: std::io::Error) -> Self { + ApiError { + code: S3ErrorCode::InternalError, + message: err.to_string(), + source: Some(Box::new(err)), + } + } +} + +impl From for ApiError { + fn from(err: iam::error::Error) -> Self { + let serr: StorageError = err.into(); + serr.into() + } +} diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index 3fd8649c6..c063edc92 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -1,6 +1,6 @@ use std::{collections::HashMap, io::Cursor, pin::Pin}; -use common::error::Error as EcsError; +// use common::error::Error as EcsError; use ecstore::{ admin_server_info::get_local_server_property, bucket::{metadata::load_bucket_metadata, metadata_sys}, diff --git a/rustfs/src/license.rs b/rustfs/src/license.rs index d805dc159..43b893d03 100644 --- a/rustfs/src/license.rs +++ b/rustfs/src/license.rs @@ -1,5 +1,5 @@ use appauth::token::Token; -use common::error::{Error, Result}; +use std::io::{Error, Result}; use std::sync::OnceLock; use std::time::SystemTime; use std::time::UNIX_EPOCH; @@ -37,7 +37,7 @@ pub fn license_check() -> Result<()> { let invalid_license = LICENSE.get().map(|token| { if token.expired < SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() { error!("License expired"); - return Err(Error::from_string("Incorrect license, please contact RustFS.".to_string())); + return Err(Error::other("Incorrect license, please contact RustFS.")); } info!("License is valid ! expired at {}", token.expired); Ok(()) @@ -46,12 +46,12 @@ pub fn license_check() -> Result<()> { // let invalid_license = config::get_config().license.as_ref().map(|license| { // if license.is_empty() { // error!("License is empty"); - // return Err(Error::from_string("Incorrect license, please contact RustFS.".to_string())); + // return Err(Error::other("Incorrect license, please contact RustFS.".to_string())); // } // let token = appauth::token::parse_license(license)?; // if token.expired < SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() { // error!("License expired"); - // return Err(Error::from_string("Incorrect license, please contact RustFS.".to_string())); + // return Err(Error::other("Incorrect license, please contact RustFS.".to_string())); // } // info!("License is valid ! expired at {}", token.expired); @@ -59,7 +59,7 @@ pub fn license_check() -> Result<()> { // }); if invalid_license.is_none() || invalid_license.is_some_and(|v| v.is_err()) { - return Err(Error::from_string("Incorrect license, please contact RustFS.".to_string())); + return Err(Error::other("Incorrect license, please contact RustFS.")); } Ok(()) diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 4a1696321..a38fa1959 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -73,7 +73,7 @@ const MI_B: usize = 1024 * 1024; static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; #[allow(clippy::result_large_err)] -fn check_auth(req: Request<()>) -> Result, Status> { +fn check_auth(req: Request<()>) -> std::result::Result, Status> { let token: MetadataValue<_> = "rustfs rpc".parse().unwrap(); match req.metadata().get("authorization") { @@ -120,7 +120,7 @@ async fn run(opt: config::Opt) -> Result<()> { // Initialize event notifier event::init_event_notifier(opt.event_config).await; - let server_addr = net::parse_and_resolve_address(opt.address.as_str())?; + let server_addr = net::parse_and_resolve_address(opt.address.as_str()).map_err(|err| Error::other(err))?; let server_port = server_addr.port(); let server_address = server_addr.to_string(); @@ -140,7 +140,7 @@ async fn run(opt: config::Opt) -> Result<()> { // For RPC let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(server_address.clone().as_str(), opt.volumes.clone()) - .map_err(|err| Error::from_string(err.to_string()))?; + .map_err(|err| Error::other(err.to_string()))?; // Print RustFS-style logging for pool formatting for (i, eps) in endpoint_pools.as_ref().iter().enumerate() { @@ -189,7 +189,7 @@ async fn run(opt: config::Opt) -> Result<()> { // Initialize the local disk init_local_disks(endpoint_pools.clone()) .await - .map_err(|err| Error::from_string(err.to_string()))?; + .map_err(|err| Error::other(err))?; // Setup S3 service // This project uses the S3S library to implement S3 services @@ -518,7 +518,7 @@ async fn run(opt: config::Opt) -> Result<()> { ..Default::default() }) .await - .map_err(|err| Error::from_string(err.to_string()))?; + .map_err(|err| Error::other(err))?; let buckets = buckets_list.into_iter().map(|v| v.name).collect(); @@ -528,7 +528,7 @@ async fn run(opt: config::Opt) -> Result<()> { new_global_notification_sys(endpoint_pools.clone()).await.map_err(|err| { error!("new_global_notification_sys failed {:?}", &err); - Error::from_string(err.to_string()) + Error::other(err) })?; // init scanner @@ -549,7 +549,7 @@ async fn run(opt: config::Opt) -> Result<()> { if console_address.is_empty() { error!("console_address is empty"); - return Err(Error::from_string("console_address is empty".to_string())); + return Err(Error::other("console_address is empty".to_string())); } tokio::spawn(async move { @@ -571,7 +571,7 @@ async fn run(opt: config::Opt) -> Result<()> { // stop event notifier rustfs_event_notifier::shutdown().await.map_err(|err| { error!("Failed to shut down the notification system: {}", err); - Error::from_string(err.to_string()) + Error::other(err) })?; } diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 950717d62..e0e26bfa4 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -3,8 +3,9 @@ use super::options::del_opts; use super::options::extract_metadata; use super::options::put_opts; use crate::auth::get_condition_values; +use crate::error::ApiError; +use crate::error::Result; use crate::storage::access::ReqInfo; -use crate::storage::error::to_s3_error; use crate::storage::options::copy_dst_opts; use crate::storage::options::copy_src_opts; use crate::storage::options::{extract_metadata_from_mime, get_opts}; @@ -12,11 +13,9 @@ use api::query::Context; use api::query::Query; use api::server::dbms::DatabaseManagerSystem; use bytes::Bytes; -use common::error::Result; use datafusion::arrow::csv::WriterBuilder as CsvWriterBuilder; -use datafusion::arrow::json::writer::JsonArray; use datafusion::arrow::json::WriterBuilder as JsonWriterBuilder; -use ecstore::bucket::error::BucketMetadataError; +use datafusion::arrow::json::writer::JsonArray; use ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG; use ecstore::bucket::metadata::BUCKET_NOTIFICATION_CONFIG; use ecstore::bucket::metadata::BUCKET_POLICY_CONFIG; @@ -30,6 +29,7 @@ use ecstore::bucket::policy_sys::PolicySys; use ecstore::bucket::tagging::decode_tags; use ecstore::bucket::tagging::encode_tags; use ecstore::bucket::versioning_sys::BucketVersioningSys; +use ecstore::error::StorageError; use ecstore::io::READ_BUFFER_SIZE; use ecstore::new_object_layer_fn; use ecstore::store_api::BucketOptions; @@ -42,8 +42,8 @@ use ecstore::store_api::ObjectIO; use ecstore::store_api::ObjectOptions; use ecstore::store_api::ObjectToDelete; use ecstore::store_api::PutObjReader; -use ecstore::store_api::StorageAPI; use ecstore::store_api::RESERVED_METADATA_PREFIX_LOWER; +use ecstore::store_api::StorageAPI; use ecstore::utils::path::path_join_buf; use ecstore::utils::xml; use ecstore::xhttp; @@ -52,27 +52,28 @@ use futures::{Stream, StreamExt}; use http::HeaderMap; use lazy_static::lazy_static; use policy::auth; -use policy::policy::action::Action; -use policy::policy::action::S3Action; use policy::policy::BucketPolicy; use policy::policy::BucketPolicyArgs; use policy::policy::Validator; +use policy::policy::action::Action; +use policy::policy::action::S3Action; use query::instance::make_rustfsms; +use rustfs_rio::HashReader; use rustfs_zip::CompressionFormat; -use s3s::dto::*; -use s3s::s3_error; +use s3s::S3; use s3s::S3Error; use s3s::S3ErrorCode; use s3s::S3Result; -use s3s::S3; +use s3s::dto::*; +use s3s::s3_error; use s3s::{S3Request, S3Response}; use std::collections::HashMap; use std::fmt::Debug; use std::path::Path; use std::str::FromStr; use std::sync::Arc; -use time::format_description::well_known::Rfc3339; use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tokio_tar::Archive; @@ -175,12 +176,15 @@ impl FS { println!("Extracted: {}, size {}", fpath, size); - let mut reader = PutObjReader::new(Box::new(f), size); + // Wrap the tar entry with BufReader to make it compatible with Reader trait + let reader = Box::new(tokio::io::BufReader::new(f)); + let hrd = HashReader::new(reader, size as i64, size as i64, None, false).map_err(ApiError::from)?; + let mut reader = PutObjReader::new(hrd, size); let _obj_info = store .put_object(&bucket, &fpath, &mut reader, &ObjectOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; // let e_tag = obj_info.etag; @@ -244,7 +248,7 @@ impl S3 for FS { }, ) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let output = CreateBucketOutput::default(); Ok(S3Response::new(output)) @@ -270,7 +274,7 @@ impl S3 for FS { // warn!("copy_object {}/{}, to {}/{}", &src_bucket, &src_key, &bucket, &key); - let mut src_opts = copy_src_opts(&src_bucket, &src_key, &req.headers).map_err(to_s3_error)?; + let mut src_opts = copy_src_opts(&src_bucket, &src_key, &req.headers).map_err(ApiError::from)?; src_opts.version_id = version_id.clone(); @@ -283,7 +287,7 @@ impl S3 for FS { let dst_opts = copy_dst_opts(&bucket, &key, version_id, &req.headers, None) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let cp_src_dst_same = path_join_buf(&[&src_bucket, &src_key]) == path_join_buf(&[&bucket, &key]); @@ -300,7 +304,7 @@ impl S3 for FS { let gr = store .get_object_reader(&src_bucket, &src_key, None, h, &get_opts) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let mut src_info = gr.object_info.clone(); @@ -308,8 +312,11 @@ impl S3 for FS { src_info.metadata_only = true; } + let hrd = HashReader::new(gr.stream, gr.object_info.size as i64, gr.object_info.size as i64, None, false) + .map_err(ApiError::from)?; + src_info.put_object_reader = Some(PutObjReader { - stream: gr.stream, + stream: hrd, content_length: gr.object_info.size as usize, }); @@ -320,7 +327,7 @@ impl S3 for FS { let oi = store .copy_object(&src_bucket, &src_key, &bucket, &key, &mut src_info, &src_opts, &dst_opts) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; // warn!("copy_object oi {:?}", &oi); @@ -355,7 +362,7 @@ impl S3 for FS { }, ) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; Ok(S3Response::new(DeleteBucketOutput {})) } @@ -371,7 +378,7 @@ impl S3 for FS { let opts: ObjectOptions = del_opts(&bucket, &key, version_id, &req.headers, Some(metadata)) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let version_id = opts.version_id.as_ref().map(|v| Uuid::parse_str(v).ok()).unwrap_or_default(); let dobj = ObjectToDelete { @@ -384,7 +391,7 @@ impl S3 for FS { let Some(store) = new_object_layer_fn() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - let (dobjs, _errs) = store.delete_objects(&bucket, objects, opts).await.map_err(to_s3_error)?; + let (dobjs, _errs) = store.delete_objects(&bucket, objects, opts).await.map_err(ApiError::from)?; // TODO: let errors; @@ -392,13 +399,7 @@ impl S3 for FS { if let Some((a, b)) = dobjs .iter() .map(|v| { - let delete_marker = { - if v.delete_marker { - Some(true) - } else { - None - } - }; + let delete_marker = { if v.delete_marker { Some(true) } else { None } }; let version_id = v.version_id.clone(); @@ -447,20 +448,14 @@ impl S3 for FS { let opts: ObjectOptions = del_opts(&bucket, "", None, &req.headers, Some(metadata)) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; - let (dobjs, errs) = store.delete_objects(&bucket, objects, opts).await.map_err(to_s3_error)?; + let (dobjs, errs) = store.delete_objects(&bucket, objects, opts).await.map_err(ApiError::from)?; let deleted = dobjs .iter() .map(|v| DeletedObject { - delete_marker: { - if v.delete_marker { - Some(true) - } else { - None - } - }, + delete_marker: { if v.delete_marker { Some(true) } else { None } }, delete_marker_version_id: v.delete_marker_version_id.clone(), key: Some(v.object_name.clone()), version_id: v.version_id.clone(), @@ -493,7 +488,7 @@ impl S3 for FS { store .get_bucket_info(&input.bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let output = GetBucketLocationOutput::default(); Ok(S3Response::new(output)) @@ -550,7 +545,7 @@ impl S3 for FS { let opts: ObjectOptions = get_opts(&bucket, &key, version_id, part_number, &req.headers) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let Some(store) = new_object_layer_fn() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); @@ -559,7 +554,7 @@ impl S3 for FS { let reader = store .get_object_reader(bucket.as_str(), key.as_str(), rs, h, &opts) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let info = reader.object_info; @@ -606,7 +601,7 @@ impl S3 for FS { store .get_bucket_info(&input.bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; // mc cp step 2 GetBucketInfo Ok(S3Response::new(HeadBucketOutput::default())) @@ -651,13 +646,13 @@ impl S3 for FS { let opts: ObjectOptions = get_opts(&bucket, &key, version_id, part_number, &req.headers) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let Some(store) = new_object_layer_fn() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - let info = store.get_object_info(&bucket, &key, &opts).await.map_err(to_s3_error)?; + let info = store.get_object_info(&bucket, &key, &opts).await.map_err(ApiError::from)?; // warn!("head_object info {:?}", &info); @@ -700,7 +695,7 @@ impl S3 for FS { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - let mut bucket_infos = store.list_bucket(&BucketOptions::default()).await.map_err(to_s3_error)?; + let mut bucket_infos = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?; let mut req = req; @@ -792,7 +787,7 @@ impl S3 for FS { start_after, ) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; // warn!("object_infos objects {:?}", object_infos.objects); @@ -873,7 +868,7 @@ impl S3 for FS { let object_infos = store .list_object_versions(&bucket, &prefix, key_marker, version_id_marker, delimiter.clone(), max_keys) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let objects: Vec = object_infos .objects @@ -960,9 +955,14 @@ impl S3 for FS { } }; - let body = Box::new(StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string()))))); + let body = StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))); + let body = Box::new(tokio::io::BufReader::new(body)); + let hrd = HashReader::new(body, content_length as i64, content_length as i64, None, false).map_err(ApiError::from)?; + let mut reader = PutObjReader::new(hrd, content_length as usize); - let mut reader = PutObjReader::new(body, content_length as usize); + // let body = Box::new(StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string()))))); + + // let mut reader = PutObjReader::new(body, content_length as usize); let Some(store) = new_object_layer_fn() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); @@ -978,14 +978,14 @@ impl S3 for FS { let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(metadata)) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; debug!("put_object opts {:?}", &opts); let obj_info = store .put_object(&bucket, &key, &mut reader, &opts) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let e_tag = obj_info.etag; @@ -1027,10 +1027,12 @@ impl S3 for FS { let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(metadata)) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; - let MultipartUploadResult { upload_id, .. } = - store.new_multipart_upload(&bucket, &key, &opts).await.map_err(to_s3_error)?; + let MultipartUploadResult { upload_id, .. } = store + .new_multipart_upload(&bucket, &key, &opts) + .await + .map_err(ApiError::from)?; let output = CreateMultipartUploadOutput { bucket: Some(bucket), @@ -1074,10 +1076,12 @@ impl S3 for FS { } }; - let body = Box::new(StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string()))))); + let body = StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))); + let body = Box::new(tokio::io::BufReader::new(body)); + let hrd = HashReader::new(body, content_length as i64, content_length as i64, None, false).map_err(ApiError::from)?; // mc cp step 4 - let mut data = PutObjReader::new(body, content_length as usize); + let mut data = PutObjReader::new(hrd, content_length as usize); let opts = ObjectOptions::default(); let Some(store) = new_object_layer_fn() else { @@ -1089,7 +1093,7 @@ impl S3 for FS { let info = store .put_object_part(&bucket, &key, &upload_id, part_id, &mut data, &opts) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let output = UploadPartOutput { e_tag: info.etag, @@ -1155,7 +1159,7 @@ impl S3 for FS { let oi = store .complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, opts) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let output = CompleteMultipartUploadOutput { bucket: Some(bucket), @@ -1184,7 +1188,7 @@ impl S3 for FS { store .abort_multipart_upload(bucket.as_str(), key.as_str(), upload_id.as_str(), opts) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; Ok(S3Response::new(AbortMultipartUploadOutput { ..Default::default() })) } @@ -1222,13 +1226,13 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let data = try_!(xml::serialize(&tagging)); metadata_sys::update(&bucket, BUCKET_TAGGING_CONFIG, data) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; Ok(S3Response::new(Default::default())) } @@ -1242,7 +1246,7 @@ impl S3 for FS { metadata_sys::delete(&bucket, BUCKET_TAGGING_CONFIG) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; Ok(S3Response::new(DeleteBucketTaggingOutput {})) } @@ -1268,7 +1272,7 @@ impl S3 for FS { store .put_object_tags(&bucket, &object, &tags, &ObjectOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; Ok(S3Response::new(PutObjectTaggingOutput { version_id: None })) } @@ -1285,7 +1289,7 @@ impl S3 for FS { let tags = store .get_object_tags(&bucket, &object, &ObjectOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let tag_set = decode_tags(tags.as_str()); @@ -1311,7 +1315,7 @@ impl S3 for FS { store .delete_object_tags(&bucket, &object, &ObjectOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; Ok(S3Response::new(DeleteObjectTaggingOutput { version_id: None })) } @@ -1329,9 +1333,9 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; - let VersioningConfiguration { status, .. } = BucketVersioningSys::get(&bucket).await.map_err(to_s3_error)?; + let VersioningConfiguration { status, .. } = BucketVersioningSys::get(&bucket).await.map_err(ApiError::from)?; Ok(S3Response::new(GetBucketVersioningOutput { status, @@ -1359,7 +1363,7 @@ impl S3 for FS { metadata_sys::update(&bucket, BUCKET_VERSIONING_CONFIG, data) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; // TODO: globalSiteReplicationSys.BucketMetaHook @@ -1379,7 +1383,7 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let conditions = get_condition_values(&req.headers, &auth::Credentials::default()); @@ -1426,15 +1430,15 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let cfg = match PolicySys::get(&bucket).await { Ok(res) => res, Err(err) => { - if BucketMetadataError::BucketPolicyNotFound.is(&err) { + if StorageError::BucketPolicyNotFound == err { return Err(s3_error!(NoSuchBucketPolicy)); } - return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", err))); + return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string())); } }; @@ -1453,7 +1457,7 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; // warn!("input policy {}", &policy); @@ -1469,7 +1473,7 @@ impl S3 for FS { metadata_sys::update(&bucket, BUCKET_POLICY_CONFIG, data) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; Ok(S3Response::new(PutBucketPolicyOutput {})) } @@ -1487,11 +1491,11 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; metadata_sys::delete(&bucket, BUCKET_POLICY_CONFIG) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; Ok(S3Response::new(DeleteBucketPolicyOutput {})) } @@ -1510,7 +1514,7 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let rules = match metadata_sys::get_lifecycle_config(&bucket).await { Ok((cfg, _)) => Some(cfg.rules), @@ -1549,7 +1553,7 @@ impl S3 for FS { let data = try_!(xml::serialize(&input_cfg)); metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, data) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default())) } @@ -1568,11 +1572,11 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; metadata_sys::delete(&bucket, BUCKET_LIFECYCLE_CONFIG) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; Ok(S3Response::new(DeleteBucketLifecycleOutput::default())) } @@ -1590,7 +1594,7 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let server_side_encryption_configuration = match metadata_sys::get_sse_config(&bucket).await { Ok((cfg, _)) => Some(cfg), @@ -1627,14 +1631,14 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; // TODO: check kms let data = try_!(xml::serialize(&server_side_encryption_configuration)); metadata_sys::update(&bucket, BUCKET_SSECONFIG, data) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; Ok(S3Response::new(PutBucketEncryptionOutput::default())) } @@ -1651,8 +1655,10 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; - metadata_sys::delete(&bucket, BUCKET_SSECONFIG).await.map_err(to_s3_error)?; + .map_err(ApiError::from)?; + metadata_sys::delete(&bucket, BUCKET_SSECONFIG) + .await + .map_err(ApiError::from)?; Ok(S3Response::new(DeleteBucketEncryptionOutput::default())) } @@ -1699,13 +1705,13 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let data = try_!(xml::serialize(&input_cfg)); metadata_sys::update(&bucket, OBJECT_LOCK_CONFIG, data) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; Ok(S3Response::new(PutObjectLockConfigurationOutput::default())) } @@ -1723,7 +1729,7 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let replication_configuration = match metadata_sys::get_replication_config(&bucket).await { Ok((cfg, _created)) => Some(cfg), @@ -1755,14 +1761,14 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; // TODO: check enable, versioning enable let data = try_!(xml::serialize(&replication_configuration)); metadata_sys::update(&bucket, BUCKET_REPLICATION_CONFIG, data) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; Ok(S3Response::new(PutBucketReplicationOutput::default())) } @@ -1780,10 +1786,10 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; metadata_sys::delete(&bucket, BUCKET_REPLICATION_CONFIG) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; // TODO: remove targets @@ -1803,7 +1809,7 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let has_notification_config = match metadata_sys::get_notification_config(&bucket).await { Ok(cfg) => cfg, @@ -1850,13 +1856,13 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let data = try_!(xml::serialize(¬ification_configuration)); metadata_sys::update(&bucket, BUCKET_NOTIFICATION_CONFIG, data) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; // TODO: event notice add rule @@ -1873,7 +1879,7 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let grants = vec![Grant { grantee: Some(Grantee { @@ -1909,7 +1915,7 @@ impl S3 for FS { store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; if let Some(canned_acl) = acl { if canned_acl.as_str() != BucketCannedACL::PRIVATE { @@ -2085,14 +2091,14 @@ impl S3 for FS { let _ = store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; // check object lock - let _ = metadata_sys::get_object_lock_config(&bucket).await.map_err(to_s3_error)?; + let _ = metadata_sys::get_object_lock_config(&bucket).await.map_err(ApiError::from)?; let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let object_info = store.get_object_info(&bucket, &key, &opts).await.map_err(|e| { error!("get_object_info failed, {}", e.to_string()); @@ -2135,14 +2141,14 @@ impl S3 for FS { let _ = store .get_bucket_info(&bucket, &BucketOptions::default()) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; // check object lock - let _ = metadata_sys::get_object_lock_config(&bucket).await.map_err(to_s3_error)?; + let _ = metadata_sys::get_object_lock_config(&bucket).await.map_err(ApiError::from)?; let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers) .await - .map_err(to_s3_error)?; + .map_err(ApiError::from)?; let mut eval_metadata = HashMap::new(); let legal_hold = legal_hold @@ -2176,9 +2182,9 @@ impl S3 for FS { } #[allow(dead_code)] -pub fn bytes_stream(stream: S, content_length: usize) -> impl Stream> + Send + 'static +pub fn bytes_stream(stream: S, content_length: usize) -> impl Stream> + Send + 'static where - S: Stream> + Send + 'static, + S: Stream> + Send + 'static, E: Send + 'static, { AsyncTryStream::::new(|mut y| async move { diff --git a/rustfs/src/storage/error.rs b/rustfs/src/storage/error.rs index fd4c95c5c..c14eb1cfd 100644 --- a/rustfs/src/storage/error.rs +++ b/rustfs/src/storage/error.rs @@ -1,485 +1,485 @@ -use common::error::Error; -use ecstore::error::StorageError; -use s3s::{S3Error, S3ErrorCode, s3_error}; -pub fn to_s3_error(err: Error) -> S3Error { - if let Some(storage_err) = err.downcast_ref::() { - return match storage_err { - StorageError::NotImplemented => s3_error!(NotImplemented), - StorageError::InvalidArgument(bucket, object, version_id) => { - s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id) - } - StorageError::MethodNotAllowed => s3_error!(MethodNotAllowed), - StorageError::BucketNotFound(bucket) => { - s3_error!(NoSuchBucket, "bucket not found {}", bucket) - } - StorageError::BucketNotEmpty(bucket) => s3_error!(BucketNotEmpty, "bucket not empty {}", bucket), - StorageError::BucketNameInvalid(bucket) => s3_error!(InvalidBucketName, "invalid bucket name {}", bucket), - StorageError::ObjectNameInvalid(bucket, object) => { - s3_error!(InvalidArgument, "invalid object name {}/{}", bucket, object) - } - StorageError::BucketExists(bucket) => s3_error!(BucketAlreadyExists, "{}", bucket), - StorageError::StorageFull => s3_error!(ServiceUnavailable, "Storage reached its minimum free drive threshold."), - StorageError::SlowDown => s3_error!(SlowDown, "Please reduce your request rate"), - StorageError::PrefixAccessDenied(bucket, object) => { - s3_error!(AccessDenied, "PrefixAccessDenied {}/{}", bucket, object) - } - StorageError::InvalidUploadIDKeyCombination(bucket, object) => { - s3_error!(InvalidArgument, "Invalid UploadID KeyCombination: {}/{}", bucket, object) - } - StorageError::MalformedUploadID(bucket) => s3_error!(InvalidArgument, "Malformed UploadID: {}", bucket), - StorageError::ObjectNameTooLong(bucket, object) => { - s3_error!(InvalidArgument, "Object name too long: {}/{}", bucket, object) - } - StorageError::ObjectNamePrefixAsSlash(bucket, object) => { - s3_error!(InvalidArgument, "Object name contains forward slash as prefix: {}/{}", bucket, object) - } - StorageError::ObjectNotFound(bucket, object) => s3_error!(NoSuchKey, "{}/{}", bucket, object), - StorageError::VersionNotFound(bucket, object, version_id) => { - s3_error!(NoSuchVersion, "{}/{}/{}", bucket, object, version_id) - } - StorageError::InvalidUploadID(bucket, object, version_id) => { - s3_error!(InvalidPart, "Invalid upload id: {}/{}-{}", bucket, object, version_id) - } - StorageError::InvalidVersionID(bucket, object, version_id) => { - s3_error!(InvalidArgument, "Invalid version id: {}/{}-{}", bucket, object, version_id) - } - // extended - StorageError::DataMovementOverwriteErr(bucket, object, version_id) => s3_error!( - InvalidArgument, - "invalid data movement operation, source and destination pool are the same for : {}/{}-{}", - bucket, - object, - version_id - ), - - // extended - StorageError::ObjectExistsAsDirectory(bucket, object) => { - s3_error!(InvalidArgument, "Object exists on :{} as directory {}", bucket, object) - } - StorageError::InvalidPart(bucket, object, version_id) => { - s3_error!( - InvalidPart, - "Specified part could not be found. PartNumber {}, Expected {}, got {}", - bucket, - object, - version_id - ) - } - StorageError::DoneForNow => s3_error!(InternalError, "DoneForNow"), - }; - } - - if is_err_file_not_found(&err) { - return S3Error::with_message(S3ErrorCode::NoSuchKey, format!(" ec err {}", err)); - } - - S3Error::with_message(S3ErrorCode::InternalError, format!(" ec err {}", err)) -} - -#[cfg(test)] -mod tests { - use super::*; - use s3s::S3ErrorCode; - - #[test] - fn test_to_s3_error_not_implemented() { - let storage_err = StorageError::NotImplemented; - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::NotImplemented); - } - - #[test] - fn test_to_s3_error_invalid_argument() { - let storage_err = - StorageError::InvalidArgument("test-bucket".to_string(), "test-object".to_string(), "test-version".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); - assert!(s3_err.message().unwrap().contains("Invalid arguments provided")); - assert!(s3_err.message().unwrap().contains("test-bucket")); - assert!(s3_err.message().unwrap().contains("test-object")); - assert!(s3_err.message().unwrap().contains("test-version")); - } - - #[test] - fn test_to_s3_error_method_not_allowed() { - let storage_err = StorageError::MethodNotAllowed; - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::MethodNotAllowed); - } - - #[test] - fn test_to_s3_error_bucket_not_found() { - let storage_err = StorageError::BucketNotFound("test-bucket".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket); - assert!(s3_err.message().unwrap().contains("bucket not found")); - assert!(s3_err.message().unwrap().contains("test-bucket")); - } - - #[test] - fn test_to_s3_error_bucket_not_empty() { - let storage_err = StorageError::BucketNotEmpty("test-bucket".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::BucketNotEmpty); - assert!(s3_err.message().unwrap().contains("bucket not empty")); - assert!(s3_err.message().unwrap().contains("test-bucket")); - } - - #[test] - fn test_to_s3_error_bucket_name_invalid() { - let storage_err = StorageError::BucketNameInvalid("invalid-bucket-name".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidBucketName); - assert!(s3_err.message().unwrap().contains("invalid bucket name")); - assert!(s3_err.message().unwrap().contains("invalid-bucket-name")); - } - - #[test] - fn test_to_s3_error_object_name_invalid() { - let storage_err = StorageError::ObjectNameInvalid("test-bucket".to_string(), "invalid-object".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); - assert!(s3_err.message().unwrap().contains("invalid object name")); - assert!(s3_err.message().unwrap().contains("test-bucket")); - assert!(s3_err.message().unwrap().contains("invalid-object")); - } - - #[test] - fn test_to_s3_error_bucket_exists() { - let storage_err = StorageError::BucketExists("existing-bucket".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::BucketAlreadyExists); - assert!(s3_err.message().unwrap().contains("existing-bucket")); - } - - #[test] - fn test_to_s3_error_storage_full() { - let storage_err = StorageError::StorageFull; - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::ServiceUnavailable); - assert!( - s3_err - .message() - .unwrap() - .contains("Storage reached its minimum free drive threshold") - ); - } - - #[test] - fn test_to_s3_error_slow_down() { - let storage_err = StorageError::SlowDown; - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown); - assert!(s3_err.message().unwrap().contains("Please reduce your request rate")); - } - - #[test] - fn test_to_s3_error_prefix_access_denied() { - let storage_err = StorageError::PrefixAccessDenied("test-bucket".to_string(), "test-prefix".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::AccessDenied); - assert!(s3_err.message().unwrap().contains("PrefixAccessDenied")); - assert!(s3_err.message().unwrap().contains("test-bucket")); - assert!(s3_err.message().unwrap().contains("test-prefix")); - } - - #[test] - fn test_to_s3_error_invalid_upload_id_key_combination() { - let storage_err = StorageError::InvalidUploadIDKeyCombination("test-bucket".to_string(), "test-object".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); - assert!(s3_err.message().unwrap().contains("Invalid UploadID KeyCombination")); - assert!(s3_err.message().unwrap().contains("test-bucket")); - assert!(s3_err.message().unwrap().contains("test-object")); - } - - #[test] - fn test_to_s3_error_malformed_upload_id() { - let storage_err = StorageError::MalformedUploadID("malformed-id".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); - assert!(s3_err.message().unwrap().contains("Malformed UploadID")); - assert!(s3_err.message().unwrap().contains("malformed-id")); - } - - #[test] - fn test_to_s3_error_object_name_too_long() { - let storage_err = StorageError::ObjectNameTooLong("test-bucket".to_string(), "very-long-object-name".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); - assert!(s3_err.message().unwrap().contains("Object name too long")); - assert!(s3_err.message().unwrap().contains("test-bucket")); - assert!(s3_err.message().unwrap().contains("very-long-object-name")); - } - - #[test] - fn test_to_s3_error_object_name_prefix_as_slash() { - let storage_err = StorageError::ObjectNamePrefixAsSlash("test-bucket".to_string(), "/invalid-object".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); - assert!( - s3_err - .message() - .unwrap() - .contains("Object name contains forward slash as prefix") - ); - assert!(s3_err.message().unwrap().contains("test-bucket")); - assert!(s3_err.message().unwrap().contains("/invalid-object")); - } - - #[test] - fn test_to_s3_error_object_not_found() { - let storage_err = StorageError::ObjectNotFound("test-bucket".to_string(), "missing-object".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchKey); - assert!(s3_err.message().unwrap().contains("test-bucket")); - assert!(s3_err.message().unwrap().contains("missing-object")); - } - - #[test] - fn test_to_s3_error_version_not_found() { - let storage_err = - StorageError::VersionNotFound("test-bucket".to_string(), "test-object".to_string(), "missing-version".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchVersion); - assert!(s3_err.message().unwrap().contains("test-bucket")); - assert!(s3_err.message().unwrap().contains("test-object")); - assert!(s3_err.message().unwrap().contains("missing-version")); - } - - #[test] - fn test_to_s3_error_invalid_upload_id() { - let storage_err = - StorageError::InvalidUploadID("test-bucket".to_string(), "test-object".to_string(), "invalid-upload-id".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidPart); - assert!(s3_err.message().unwrap().contains("Invalid upload id")); - assert!(s3_err.message().unwrap().contains("test-bucket")); - assert!(s3_err.message().unwrap().contains("test-object")); - assert!(s3_err.message().unwrap().contains("invalid-upload-id")); - } - - #[test] - fn test_to_s3_error_invalid_version_id() { - let storage_err = StorageError::InvalidVersionID( - "test-bucket".to_string(), - "test-object".to_string(), - "invalid-version-id".to_string(), - ); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); - assert!(s3_err.message().unwrap().contains("Invalid version id")); - assert!(s3_err.message().unwrap().contains("test-bucket")); - assert!(s3_err.message().unwrap().contains("test-object")); - assert!(s3_err.message().unwrap().contains("invalid-version-id")); - } - - #[test] - fn test_to_s3_error_data_movement_overwrite_err() { - let storage_err = StorageError::DataMovementOverwriteErr( - "test-bucket".to_string(), - "test-object".to_string(), - "test-version".to_string(), - ); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); - assert!(s3_err.message().unwrap().contains("invalid data movement operation")); - assert!(s3_err.message().unwrap().contains("source and destination pool are the same")); - assert!(s3_err.message().unwrap().contains("test-bucket")); - assert!(s3_err.message().unwrap().contains("test-object")); - assert!(s3_err.message().unwrap().contains("test-version")); - } - - #[test] - fn test_to_s3_error_object_exists_as_directory() { - let storage_err = StorageError::ObjectExistsAsDirectory("test-bucket".to_string(), "directory-object".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); - assert!(s3_err.message().unwrap().contains("Object exists on")); - assert!(s3_err.message().unwrap().contains("as directory")); - assert!(s3_err.message().unwrap().contains("test-bucket")); - assert!(s3_err.message().unwrap().contains("directory-object")); - } - - #[test] - fn test_to_s3_error_insufficient_read_quorum() { - let storage_err = StorageError::InsufficientReadQuorum; - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown); - assert!( - s3_err - .message() - .unwrap() - .contains("Storage resources are insufficient for the read operation") - ); - } - - #[test] - fn test_to_s3_error_insufficient_write_quorum() { - let storage_err = StorageError::InsufficientWriteQuorum; - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown); - assert!( - s3_err - .message() - .unwrap() - .contains("Storage resources are insufficient for the write operation") - ); - } - - #[test] - fn test_to_s3_error_decommission_not_started() { - let storage_err = StorageError::DecommissionNotStarted; - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); - assert!(s3_err.message().unwrap().contains("Decommission Not Started")); - } - - #[test] - fn test_to_s3_error_decommission_already_running() { - let storage_err = StorageError::DecommissionAlreadyRunning; - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InternalError); - assert!(s3_err.message().unwrap().contains("Decommission already running")); - } - - #[test] - fn test_to_s3_error_volume_not_found() { - let storage_err = StorageError::VolumeNotFound("test-volume".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket); - assert!(s3_err.message().unwrap().contains("bucket not found")); - assert!(s3_err.message().unwrap().contains("test-volume")); - } - - #[test] - fn test_to_s3_error_invalid_part() { - let storage_err = StorageError::InvalidPart(1, "expected-part".to_string(), "got-part".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidPart); - assert!(s3_err.message().unwrap().contains("Specified part could not be found")); - assert!(s3_err.message().unwrap().contains("PartNumber")); - assert!(s3_err.message().unwrap().contains("expected-part")); - assert!(s3_err.message().unwrap().contains("got-part")); - } - - #[test] - fn test_to_s3_error_done_for_now() { - let storage_err = StorageError::DoneForNow; - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InternalError); - assert!(s3_err.message().unwrap().contains("DoneForNow")); - } - - #[test] - fn test_to_s3_error_non_storage_error() { - // Test with a non-StorageError - let err = Error::from_string("Generic error message".to_string()); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InternalError); - assert!(s3_err.message().unwrap().contains("ec err")); - assert!(s3_err.message().unwrap().contains("Generic error message")); - } - - #[test] - fn test_to_s3_error_with_unicode_strings() { - let storage_err = StorageError::BucketNotFound("测试桶".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket); - assert!(s3_err.message().unwrap().contains("bucket not found")); - assert!(s3_err.message().unwrap().contains("测试桶")); - } - - #[test] - fn test_to_s3_error_with_special_characters() { - let storage_err = StorageError::ObjectNameInvalid("bucket-with-@#$%".to_string(), "object-with-!@#$%^&*()".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); - assert!(s3_err.message().unwrap().contains("invalid object name")); - assert!(s3_err.message().unwrap().contains("bucket-with-@#$%")); - assert!(s3_err.message().unwrap().contains("object-with-!@#$%^&*()")); - } - - #[test] - fn test_to_s3_error_with_empty_strings() { - let storage_err = StorageError::BucketNotFound("".to_string()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket); - assert!(s3_err.message().unwrap().contains("bucket not found")); - } - - #[test] - fn test_to_s3_error_with_very_long_strings() { - let long_bucket_name = "a".repeat(1000); - let storage_err = StorageError::BucketNotFound(long_bucket_name.clone()); - let err = Error::new(storage_err); - let s3_err = to_s3_error(err); - - assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket); - assert!(s3_err.message().unwrap().contains("bucket not found")); - assert!(s3_err.message().unwrap().contains(&long_bucket_name)); - } -} +// use common::error::Error; +// use ecstore::error::StorageError; +// use s3s::{S3Error, S3ErrorCode, s3_error}; +// pub fn to_s3_error(err: Error) -> S3Error { +// if let Some(storage_err) = err.downcast_ref::() { +// return match storage_err { +// StorageError::NotImplemented => s3_error!(NotImplemented), +// StorageError::InvalidArgument(bucket, object, version_id) => { +// s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id) +// } +// StorageError::MethodNotAllowed => s3_error!(MethodNotAllowed), +// StorageError::BucketNotFound(bucket) => { +// s3_error!(NoSuchBucket, "bucket not found {}", bucket) +// } +// StorageError::BucketNotEmpty(bucket) => s3_error!(BucketNotEmpty, "bucket not empty {}", bucket), +// StorageError::BucketNameInvalid(bucket) => s3_error!(InvalidBucketName, "invalid bucket name {}", bucket), +// StorageError::ObjectNameInvalid(bucket, object) => { +// s3_error!(InvalidArgument, "invalid object name {}/{}", bucket, object) +// } +// StorageError::BucketExists(bucket) => s3_error!(BucketAlreadyExists, "{}", bucket), +// StorageError::StorageFull => s3_error!(ServiceUnavailable, "Storage reached its minimum free drive threshold."), +// StorageError::SlowDown => s3_error!(SlowDown, "Please reduce your request rate"), +// StorageError::PrefixAccessDenied(bucket, object) => { +// s3_error!(AccessDenied, "PrefixAccessDenied {}/{}", bucket, object) +// } +// StorageError::InvalidUploadIDKeyCombination(bucket, object) => { +// s3_error!(InvalidArgument, "Invalid UploadID KeyCombination: {}/{}", bucket, object) +// } +// StorageError::MalformedUploadID(bucket) => s3_error!(InvalidArgument, "Malformed UploadID: {}", bucket), +// StorageError::ObjectNameTooLong(bucket, object) => { +// s3_error!(InvalidArgument, "Object name too long: {}/{}", bucket, object) +// } +// StorageError::ObjectNamePrefixAsSlash(bucket, object) => { +// s3_error!(InvalidArgument, "Object name contains forward slash as prefix: {}/{}", bucket, object) +// } +// StorageError::ObjectNotFound(bucket, object) => s3_error!(NoSuchKey, "{}/{}", bucket, object), +// StorageError::VersionNotFound(bucket, object, version_id) => { +// s3_error!(NoSuchVersion, "{}/{}/{}", bucket, object, version_id) +// } +// StorageError::InvalidUploadID(bucket, object, version_id) => { +// s3_error!(InvalidPart, "Invalid upload id: {}/{}-{}", bucket, object, version_id) +// } +// StorageError::InvalidVersionID(bucket, object, version_id) => { +// s3_error!(InvalidArgument, "Invalid version id: {}/{}-{}", bucket, object, version_id) +// } +// // extended +// StorageError::DataMovementOverwriteErr(bucket, object, version_id) => s3_error!( +// InvalidArgument, +// "invalid data movement operation, source and destination pool are the same for : {}/{}-{}", +// bucket, +// object, +// version_id +// ), + +// // extended +// StorageError::ObjectExistsAsDirectory(bucket, object) => { +// s3_error!(InvalidArgument, "Object exists on :{} as directory {}", bucket, object) +// } +// StorageError::InvalidPart(bucket, object, version_id) => { +// s3_error!( +// InvalidPart, +// "Specified part could not be found. PartNumber {}, Expected {}, got {}", +// bucket, +// object, +// version_id +// ) +// } +// StorageError::DoneForNow => s3_error!(InternalError, "DoneForNow"), +// }; +// } + +// if is_err_file_not_found(&err) { +// return S3Error::with_message(S3ErrorCode::NoSuchKey, format!(" ec err {}", err)); +// } + +// S3Error::with_message(S3ErrorCode::InternalError, format!(" ec err {}", err)) +// } + +// #[cfg(test)] +// mod tests { +// use super::*; +// use s3s::S3ErrorCode; + +// #[test] +// fn test_to_s3_error_not_implemented() { +// let storage_err = StorageError::NotImplemented; +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::NotImplemented); +// } + +// #[test] +// fn test_to_s3_error_invalid_argument() { +// let storage_err = +// StorageError::InvalidArgument("test-bucket".to_string(), "test-object".to_string(), "test-version".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); +// assert!(s3_err.message().unwrap().contains("Invalid arguments provided")); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// assert!(s3_err.message().unwrap().contains("test-object")); +// assert!(s3_err.message().unwrap().contains("test-version")); +// } + +// #[test] +// fn test_to_s3_error_method_not_allowed() { +// let storage_err = StorageError::MethodNotAllowed; +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::MethodNotAllowed); +// } + +// #[test] +// fn test_to_s3_error_bucket_not_found() { +// let storage_err = StorageError::BucketNotFound("test-bucket".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket); +// assert!(s3_err.message().unwrap().contains("bucket not found")); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// } + +// #[test] +// fn test_to_s3_error_bucket_not_empty() { +// let storage_err = StorageError::BucketNotEmpty("test-bucket".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::BucketNotEmpty); +// assert!(s3_err.message().unwrap().contains("bucket not empty")); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// } + +// #[test] +// fn test_to_s3_error_bucket_name_invalid() { +// let storage_err = StorageError::BucketNameInvalid("invalid-bucket-name".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidBucketName); +// assert!(s3_err.message().unwrap().contains("invalid bucket name")); +// assert!(s3_err.message().unwrap().contains("invalid-bucket-name")); +// } + +// #[test] +// fn test_to_s3_error_object_name_invalid() { +// let storage_err = StorageError::ObjectNameInvalid("test-bucket".to_string(), "invalid-object".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); +// assert!(s3_err.message().unwrap().contains("invalid object name")); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// assert!(s3_err.message().unwrap().contains("invalid-object")); +// } + +// #[test] +// fn test_to_s3_error_bucket_exists() { +// let storage_err = StorageError::BucketExists("existing-bucket".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::BucketAlreadyExists); +// assert!(s3_err.message().unwrap().contains("existing-bucket")); +// } + +// #[test] +// fn test_to_s3_error_storage_full() { +// let storage_err = StorageError::StorageFull; +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::ServiceUnavailable); +// assert!( +// s3_err +// .message() +// .unwrap() +// .contains("Storage reached its minimum free drive threshold") +// ); +// } + +// #[test] +// fn test_to_s3_error_slow_down() { +// let storage_err = StorageError::SlowDown; +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown); +// assert!(s3_err.message().unwrap().contains("Please reduce your request rate")); +// } + +// #[test] +// fn test_to_s3_error_prefix_access_denied() { +// let storage_err = StorageError::PrefixAccessDenied("test-bucket".to_string(), "test-prefix".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::AccessDenied); +// assert!(s3_err.message().unwrap().contains("PrefixAccessDenied")); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// assert!(s3_err.message().unwrap().contains("test-prefix")); +// } + +// #[test] +// fn test_to_s3_error_invalid_upload_id_key_combination() { +// let storage_err = StorageError::InvalidUploadIDKeyCombination("test-bucket".to_string(), "test-object".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); +// assert!(s3_err.message().unwrap().contains("Invalid UploadID KeyCombination")); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// assert!(s3_err.message().unwrap().contains("test-object")); +// } + +// #[test] +// fn test_to_s3_error_malformed_upload_id() { +// let storage_err = StorageError::MalformedUploadID("malformed-id".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); +// assert!(s3_err.message().unwrap().contains("Malformed UploadID")); +// assert!(s3_err.message().unwrap().contains("malformed-id")); +// } + +// #[test] +// fn test_to_s3_error_object_name_too_long() { +// let storage_err = StorageError::ObjectNameTooLong("test-bucket".to_string(), "very-long-object-name".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); +// assert!(s3_err.message().unwrap().contains("Object name too long")); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// assert!(s3_err.message().unwrap().contains("very-long-object-name")); +// } + +// #[test] +// fn test_to_s3_error_object_name_prefix_as_slash() { +// let storage_err = StorageError::ObjectNamePrefixAsSlash("test-bucket".to_string(), "/invalid-object".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); +// assert!( +// s3_err +// .message() +// .unwrap() +// .contains("Object name contains forward slash as prefix") +// ); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// assert!(s3_err.message().unwrap().contains("/invalid-object")); +// } + +// #[test] +// fn test_to_s3_error_object_not_found() { +// let storage_err = StorageError::ObjectNotFound("test-bucket".to_string(), "missing-object".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchKey); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// assert!(s3_err.message().unwrap().contains("missing-object")); +// } + +// #[test] +// fn test_to_s3_error_version_not_found() { +// let storage_err = +// StorageError::VersionNotFound("test-bucket".to_string(), "test-object".to_string(), "missing-version".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchVersion); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// assert!(s3_err.message().unwrap().contains("test-object")); +// assert!(s3_err.message().unwrap().contains("missing-version")); +// } + +// #[test] +// fn test_to_s3_error_invalid_upload_id() { +// let storage_err = +// StorageError::InvalidUploadID("test-bucket".to_string(), "test-object".to_string(), "invalid-upload-id".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidPart); +// assert!(s3_err.message().unwrap().contains("Invalid upload id")); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// assert!(s3_err.message().unwrap().contains("test-object")); +// assert!(s3_err.message().unwrap().contains("invalid-upload-id")); +// } + +// #[test] +// fn test_to_s3_error_invalid_version_id() { +// let storage_err = StorageError::InvalidVersionID( +// "test-bucket".to_string(), +// "test-object".to_string(), +// "invalid-version-id".to_string(), +// ); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); +// assert!(s3_err.message().unwrap().contains("Invalid version id")); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// assert!(s3_err.message().unwrap().contains("test-object")); +// assert!(s3_err.message().unwrap().contains("invalid-version-id")); +// } + +// #[test] +// fn test_to_s3_error_data_movement_overwrite_err() { +// let storage_err = StorageError::DataMovementOverwriteErr( +// "test-bucket".to_string(), +// "test-object".to_string(), +// "test-version".to_string(), +// ); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); +// assert!(s3_err.message().unwrap().contains("invalid data movement operation")); +// assert!(s3_err.message().unwrap().contains("source and destination pool are the same")); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// assert!(s3_err.message().unwrap().contains("test-object")); +// assert!(s3_err.message().unwrap().contains("test-version")); +// } + +// #[test] +// fn test_to_s3_error_object_exists_as_directory() { +// let storage_err = StorageError::ObjectExistsAsDirectory("test-bucket".to_string(), "directory-object".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); +// assert!(s3_err.message().unwrap().contains("Object exists on")); +// assert!(s3_err.message().unwrap().contains("as directory")); +// assert!(s3_err.message().unwrap().contains("test-bucket")); +// assert!(s3_err.message().unwrap().contains("directory-object")); +// } + +// #[test] +// fn test_to_s3_error_insufficient_read_quorum() { +// let storage_err = StorageError::InsufficientReadQuorum; +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown); +// assert!( +// s3_err +// .message() +// .unwrap() +// .contains("Storage resources are insufficient for the read operation") +// ); +// } + +// #[test] +// fn test_to_s3_error_insufficient_write_quorum() { +// let storage_err = StorageError::InsufficientWriteQuorum; +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown); +// assert!( +// s3_err +// .message() +// .unwrap() +// .contains("Storage resources are insufficient for the write operation") +// ); +// } + +// #[test] +// fn test_to_s3_error_decommission_not_started() { +// let storage_err = StorageError::DecommissionNotStarted; +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); +// assert!(s3_err.message().unwrap().contains("Decommission Not Started")); +// } + +// #[test] +// fn test_to_s3_error_decommission_already_running() { +// let storage_err = StorageError::DecommissionAlreadyRunning; +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InternalError); +// assert!(s3_err.message().unwrap().contains("Decommission already running")); +// } + +// #[test] +// fn test_to_s3_error_volume_not_found() { +// let storage_err = StorageError::VolumeNotFound("test-volume".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket); +// assert!(s3_err.message().unwrap().contains("bucket not found")); +// assert!(s3_err.message().unwrap().contains("test-volume")); +// } + +// #[test] +// fn test_to_s3_error_invalid_part() { +// let storage_err = StorageError::InvalidPart(1, "expected-part".to_string(), "got-part".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidPart); +// assert!(s3_err.message().unwrap().contains("Specified part could not be found")); +// assert!(s3_err.message().unwrap().contains("PartNumber")); +// assert!(s3_err.message().unwrap().contains("expected-part")); +// assert!(s3_err.message().unwrap().contains("got-part")); +// } + +// #[test] +// fn test_to_s3_error_done_for_now() { +// let storage_err = StorageError::DoneForNow; +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InternalError); +// assert!(s3_err.message().unwrap().contains("DoneForNow")); +// } + +// #[test] +// fn test_to_s3_error_non_storage_error() { +// // Test with a non-StorageError +// let err = Error::from_string("Generic error message".to_string()); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InternalError); +// assert!(s3_err.message().unwrap().contains("ec err")); +// assert!(s3_err.message().unwrap().contains("Generic error message")); +// } + +// #[test] +// fn test_to_s3_error_with_unicode_strings() { +// let storage_err = StorageError::BucketNotFound("测试桶".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket); +// assert!(s3_err.message().unwrap().contains("bucket not found")); +// assert!(s3_err.message().unwrap().contains("测试桶")); +// } + +// #[test] +// fn test_to_s3_error_with_special_characters() { +// let storage_err = StorageError::ObjectNameInvalid("bucket-with-@#$%".to_string(), "object-with-!@#$%^&*()".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument); +// assert!(s3_err.message().unwrap().contains("invalid object name")); +// assert!(s3_err.message().unwrap().contains("bucket-with-@#$%")); +// assert!(s3_err.message().unwrap().contains("object-with-!@#$%^&*()")); +// } + +// #[test] +// fn test_to_s3_error_with_empty_strings() { +// let storage_err = StorageError::BucketNotFound("".to_string()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket); +// assert!(s3_err.message().unwrap().contains("bucket not found")); +// } + +// #[test] +// fn test_to_s3_error_with_very_long_strings() { +// let long_bucket_name = "a".repeat(1000); +// let storage_err = StorageError::BucketNotFound(long_bucket_name.clone()); +// let err = Error::new(storage_err); +// let s3_err = to_s3_error(err); + +// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket); +// assert!(s3_err.message().unwrap().contains("bucket not found")); +// assert!(s3_err.message().unwrap().contains(&long_bucket_name)); +// } +// } diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 2f8ec8b88..90f7135d2 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -1,5 +1,5 @@ pub mod access; pub mod ecfs; -pub mod error; +// pub mod error; mod event_notifier; pub mod options; diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 4b9768cfb..cc812a23d 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -1,7 +1,7 @@ -use common::error::{Error, Result}; use ecstore::bucket::versioning_sys::BucketVersioningSys; -use ecstore::store_api::ObjectOptions; +use ecstore::error::Result; use ecstore::error::StorageError; +use ecstore::store_api::ObjectOptions; use ecstore::utils::path::is_dir_object; use http::{HeaderMap, HeaderValue}; use lazy_static::lazy_static; @@ -25,24 +25,16 @@ pub async fn del_opts( if let Some(ref id) = vid { if let Err(_err) = Uuid::parse_str(id.as_str()) { - return Err(Error::new(StorageError::InvalidVersionID( - bucket.to_owned(), - object.to_owned(), - id.clone(), - ))); + return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone())); } if !versioned { - return Err(Error::new(StorageError::InvalidArgument( - bucket.to_owned(), - object.to_owned(), - id.clone(), - ))); + return Err(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), id.clone())); } } let mut opts = put_opts_from_headers(headers, metadata) - .map_err(|err| Error::new(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string())))?; + .map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?; opts.version_id = { if is_dir_object(object) && vid.is_none() { @@ -72,24 +64,16 @@ pub async fn get_opts( if let Some(ref id) = vid { if let Err(_err) = Uuid::parse_str(id.as_str()) { - return Err(Error::new(StorageError::InvalidVersionID( - bucket.to_owned(), - object.to_owned(), - id.clone(), - ))); + return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone())); } if !versioned { - return Err(Error::new(StorageError::InvalidArgument( - bucket.to_owned(), - object.to_owned(), - id.clone(), - ))); + return Err(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), id.clone())); } } let mut opts = get_default_opts(headers, None, false) - .map_err(|err| Error::new(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string())))?; + .map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?; opts.version_id = { if is_dir_object(object) && vid.is_none() { @@ -122,24 +106,16 @@ pub async fn put_opts( if let Some(ref id) = vid { if let Err(_err) = Uuid::parse_str(id.as_str()) { - return Err(Error::new(StorageError::InvalidVersionID( - bucket.to_owned(), - object.to_owned(), - id.clone(), - ))); + return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone())); } if !versioned { - return Err(Error::new(StorageError::InvalidArgument( - bucket.to_owned(), - object.to_owned(), - id.clone(), - ))); + return Err(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), id.clone())); } } let mut opts = put_opts_from_headers(headers, metadata) - .map_err(|err| Error::new(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string())))?; + .map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?; opts.version_id = { if is_dir_object(object) && vid.is_none() { @@ -317,15 +293,13 @@ mod tests { assert!(result.is_err()); if let Err(err) = result { - if let Some(storage_err) = err.downcast_ref::() { - match storage_err { - StorageError::InvalidVersionID(bucket, object, version) => { - assert_eq!(bucket, "test-bucket"); - assert_eq!(object, "test-object"); - assert_eq!(version, "invalid-uuid"); - } - _ => panic!("Expected InvalidVersionID error"), + match err { + StorageError::InvalidVersionID(bucket, object, version) => { + assert_eq!(bucket, "test-bucket"); + assert_eq!(object, "test-object"); + assert_eq!(version, "invalid-uuid"); } + _ => panic!("Expected InvalidVersionID error"), } } } @@ -373,15 +347,13 @@ mod tests { assert!(result.is_err()); if let Err(err) = result { - if let Some(storage_err) = err.downcast_ref::() { - match storage_err { - StorageError::InvalidVersionID(bucket, object, version) => { - assert_eq!(bucket, "test-bucket"); - assert_eq!(object, "test-object"); - assert_eq!(version, "invalid-uuid"); - } - _ => panic!("Expected InvalidVersionID error"), + match err { + StorageError::InvalidVersionID(bucket, object, version) => { + assert_eq!(bucket, "test-bucket"); + assert_eq!(object, "test-object"); + assert_eq!(version, "invalid-uuid"); } + _ => panic!("Expected InvalidVersionID error"), } } } @@ -419,15 +391,13 @@ mod tests { assert!(result.is_err()); if let Err(err) = result { - if let Some(storage_err) = err.downcast_ref::() { - match storage_err { - StorageError::InvalidVersionID(bucket, object, version) => { - assert_eq!(bucket, "test-bucket"); - assert_eq!(object, "test-object"); - assert_eq!(version, "invalid-uuid"); - } - _ => panic!("Expected InvalidVersionID error"), + match err { + StorageError::InvalidVersionID(bucket, object, version) => { + assert_eq!(bucket, "test-bucket"); + assert_eq!(object, "test-object"); + assert_eq!(version, "invalid-uuid"); } + _ => panic!("Expected InvalidVersionID error"), } } }