merge main

ecstore/src/disk/error.rs

@@ -1,36 +1,105 @@
-use crate::error::{Error, Result};
+use std::io::{self, ErrorKind};
 
+use crate::{
+    error::{Error, Result},
+    quorum::CheckErrorFn,
+};
+
+// DiskError == StorageErr
 #[derive(Debug, thiserror::Error)]
 pub enum DiskError {
+    #[error("maximum versions exceeded, please delete few versions to proceed")]
+    MaxVersionsExceeded,
+
+    #[error("unexpected error")]
+    Unexpected,
+
+    #[error("corrupted format")]
+    CorruptedFormat,
+
+    #[error("corrupted backend")]
+    CorruptedBackend,
+
+    #[error("unformatted disk error")]
+    UnformattedDisk,
+
+    #[error("inconsistent drive found")]
+    InconsistentDisk,
+
+    #[error("drive does not support O_DIRECT")]
+    UnsupportedDisk,
+
+    #[error("drive path full")]
+    DiskFull,
+
+    #[error("disk not a dir")]
+    DiskNotDir,
+
+    #[error("disk not found")]
+    DiskNotFound,
+
+    #[error("drive still did not complete the request")]
+    DiskOngoingReq,
+
+    #[error("drive is part of root drive, will not be used")]
+    DriveIsRoot,
+
+    #[error("remote drive is faulty")]
+    FaultyRemoteDisk,
+
+    #[error("drive is faulty")]
+    FaultyDisk,
+
+    #[error("drive access denied")]
+    DiskAccessDenied,
+
     #[error("file not found")]
     FileNotFound,
 
     #[error("file version not found")]
     FileVersionNotFound,
 
-    #[error("disk not found")]
-    DiskNotFound,
+    #[error("too many open files, please increase 'ulimit -n'")]
+    TooManyOpenFiles,
 
-    #[error("disk access denied")]
-    FileAccessDenied,
-
-    #[error("InconsistentDisk")]
-    InconsistentDisk,
+    #[error("file name too long")]
+    FileNameTooLong,
 
     #[error("volume already exists")]
     VolumeExists,
 
-    #[error("unformatted disk error")]
-    UnformattedDisk,
+    #[error("not of regular file type")]
+    IsNotRegular,
 
-    #[error("unsupport disk")]
-    UnsupportedDisk,
-
-    #[error("disk not a dir")]
-    DiskNotDir,
+    #[error("path not found")]
+    PathNotFound,
 
     #[error("volume not found")]
     VolumeNotFound,
+
+    #[error("volume is not empty")]
+    VolumeNotEmpty,
+
+    #[error("volume access denied")]
+    VolumeAccessDenied,
+
+    #[error("disk access denied")]
+    FileAccessDenied,
+
+    #[error("file is corrupted")]
+    FileCorrupt,
+
+    #[error("bit-rot hash algorithm is invalid")]
+    BitrotHashAlgoInvalid,
+
+    #[error("Rename across devices not allowed, please fix your backend configuration")]
+    CrossDeviceLink,
+
+    #[error("less data available than what was requested")]
+    LessData,
+
+    #[error("more data was sent than what was advertised")]
+    MoreData,
 }
 
 impl DiskError {
@@ -93,3 +162,244 @@ impl PartialEq for DiskError {
         core::mem::discriminant(self) == core::mem::discriminant(other)
     }
 }
+
+impl CheckErrorFn for DiskError {
+    fn is(&self, e: &Error) -> bool {
+        self.is(e)
+    }
+}
+
+pub fn clone_disk_err(e: &DiskError) -> Error {
+    match e {
+        DiskError::MaxVersionsExceeded => Error::new(DiskError::MaxVersionsExceeded),
+        DiskError::Unexpected => Error::new(DiskError::Unexpected),
+        DiskError::CorruptedFormat => Error::new(DiskError::CorruptedFormat),
+        DiskError::CorruptedBackend => Error::new(DiskError::CorruptedBackend),
+        DiskError::UnformattedDisk => Error::new(DiskError::UnformattedDisk),
+        DiskError::InconsistentDisk => Error::new(DiskError::InconsistentDisk),
+        DiskError::UnsupportedDisk => Error::new(DiskError::UnsupportedDisk),
+        DiskError::DiskFull => Error::new(DiskError::DiskFull),
+        DiskError::DiskNotDir => Error::new(DiskError::DiskNotDir),
+        DiskError::DiskNotFound => Error::new(DiskError::DiskNotFound),
+        DiskError::DiskOngoingReq => Error::new(DiskError::DiskOngoingReq),
+        DiskError::DriveIsRoot => Error::new(DiskError::DriveIsRoot),
+        DiskError::FaultyRemoteDisk => Error::new(DiskError::FaultyRemoteDisk),
+        DiskError::FaultyDisk => Error::new(DiskError::FaultyDisk),
+        DiskError::DiskAccessDenied => Error::new(DiskError::DiskAccessDenied),
+        DiskError::FileNotFound => Error::new(DiskError::FileNotFound),
+        DiskError::FileVersionNotFound => Error::new(DiskError::FileVersionNotFound),
+        DiskError::TooManyOpenFiles => Error::new(DiskError::TooManyOpenFiles),
+        DiskError::FileNameTooLong => Error::new(DiskError::FileNameTooLong),
+        DiskError::VolumeExists => Error::new(DiskError::VolumeExists),
+        DiskError::IsNotRegular => Error::new(DiskError::IsNotRegular),
+        DiskError::PathNotFound => Error::new(DiskError::PathNotFound),
+        DiskError::VolumeNotFound => Error::new(DiskError::VolumeNotFound),
+        DiskError::VolumeNotEmpty => Error::new(DiskError::VolumeNotEmpty),
+        DiskError::VolumeAccessDenied => Error::new(DiskError::VolumeAccessDenied),
+        DiskError::FileAccessDenied => Error::new(DiskError::FileAccessDenied),
+        DiskError::FileCorrupt => Error::new(DiskError::FileCorrupt),
+        DiskError::BitrotHashAlgoInvalid => Error::new(DiskError::BitrotHashAlgoInvalid),
+        DiskError::CrossDeviceLink => Error::new(DiskError::CrossDeviceLink),
+        DiskError::LessData => Error::new(DiskError::LessData),
+        DiskError::MoreData => Error::new(DiskError::MoreData),
+    }
+}
+
+pub fn os_err_to_file_err(e: io::Error) -> Error {
+    match e.kind() {
+        io::ErrorKind::NotFound => Error::new(DiskError::FileNotFound),
+        io::ErrorKind::PermissionDenied => Error::new(DiskError::FileAccessDenied),
+        // io::ErrorKind::ConnectionRefused => todo!(),
+        // io::ErrorKind::ConnectionReset => todo!(),
+        // io::ErrorKind::HostUnreachable => todo!(),
+        // io::ErrorKind::NetworkUnreachable => todo!(),
+        // io::ErrorKind::ConnectionAborted => todo!(),
+        // io::ErrorKind::NotConnected => todo!(),
+        // io::ErrorKind::AddrInUse => todo!(),
+        // io::ErrorKind::AddrNotAvailable => todo!(),
+        // io::ErrorKind::NetworkDown => todo!(),
+        // io::ErrorKind::BrokenPipe => todo!(),
+        // io::ErrorKind::AlreadyExists => todo!(),
+        // io::ErrorKind::WouldBlock => todo!(),
+        // io::ErrorKind::NotADirectory => DiskError::FileNotFound,
+        // io::ErrorKind::IsADirectory => DiskError::FileNotFound,
+        // io::ErrorKind::DirectoryNotEmpty => DiskError::VolumeNotEmpty,
+        // io::ErrorKind::ReadOnlyFilesystem => todo!(),
+        // io::ErrorKind::FilesystemLoop => todo!(),
+        // io::ErrorKind::StaleNetworkFileHandle => todo!(),
+        // io::ErrorKind::InvalidInput => todo!(),
+        // io::ErrorKind::InvalidData => todo!(),
+        // io::ErrorKind::TimedOut => todo!(),
+        // io::ErrorKind::WriteZero => todo!(),
+        // io::ErrorKind::StorageFull => DiskError::DiskFull,
+        // io::ErrorKind::NotSeekable => todo!(),
+        // io::ErrorKind::FilesystemQuotaExceeded => todo!(),
+        // io::ErrorKind::FileTooLarge => todo!(),
+        // io::ErrorKind::ResourceBusy => todo!(),
+        // io::ErrorKind::ExecutableFileBusy => todo!(),
+        // io::ErrorKind::Deadlock => todo!(),
+        // io::ErrorKind::CrossesDevices => todo!(),
+        // io::ErrorKind::TooManyLinks =>DiskError::TooManyOpenFiles,
+        // io::ErrorKind::InvalidFilename => todo!(),
+        // io::ErrorKind::ArgumentListTooLong => todo!(),
+        // io::ErrorKind::Interrupted => todo!(),
+        // io::ErrorKind::Unsupported => todo!(),
+        // io::ErrorKind::UnexpectedEof => todo!(),
+        // io::ErrorKind::OutOfMemory => todo!(),
+        // io::ErrorKind::Other => todo!(),
+        // TODO: 把不支持的king用字符串处理
+        _ => Error::new(e),
+    }
+}
+
+pub fn is_sys_err_no_space(e: &io::Error) -> bool {
+    if let Some(no) = e.raw_os_error() {
+        return no == 28;
+    }
+    false
+}
+
+pub fn is_sys_err_invalid_arg(e: &io::Error) -> bool {
+    if let Some(no) = e.raw_os_error() {
+        return no == 22;
+    }
+    false
+}
+
+pub fn is_sys_err_io(e: &io::Error) -> bool {
+    if let Some(no) = e.raw_os_error() {
+        return no == 5;
+    }
+    false
+}
+
+pub fn is_sys_err_is_dir(e: &io::Error) -> bool {
+    if let Some(no) = e.raw_os_error() {
+        return no == 21;
+    }
+    false
+}
+
+pub fn is_sys_err_not_dir(e: &io::Error) -> bool {
+    if let Some(no) = e.raw_os_error() {
+        return no == 20;
+    }
+    false
+}
+
+pub fn is_sys_err_too_long(e: &io::Error) -> bool {
+    if let Some(no) = e.raw_os_error() {
+        return no == 63;
+    }
+    false
+}
+
+pub fn is_sys_err_too_many_symlinks(e: &io::Error) -> bool {
+    if let Some(no) = e.raw_os_error() {
+        return no == 62;
+    }
+    false
+}
+
+pub fn is_sys_err_not_empty(e: &io::Error) -> bool {
+    if let Some(no) = e.raw_os_error() {
+        if no == 66 {
+            return true;
+        }
+
+        if cfg!(target_os = "solaris") && no == 17 {
+            return true;
+        }
+
+        if cfg!(target_os = "windows") && no == 145 {
+            return true;
+        }
+    }
+    false
+}
+
+pub fn is_sys_err_path_not_found(e: &io::Error) -> bool {
+    if let Some(no) = e.raw_os_error() {
+        if cfg!(target_os = "windows") {
+            if no == 3 {
+                return true;
+            }
+        } else {
+            if no == 2 {
+                return true;
+            }
+        }
+    }
+    false
+}
+
+pub fn is_sys_err_handle_invalid(e: &io::Error) -> bool {
+    if let Some(no) = e.raw_os_error() {
+        if cfg!(target_os = "windows") {
+            if no == 6 {
+                return true;
+            }
+        } else {
+            return false;
+        }
+    }
+    false
+}
+
+pub fn is_sys_err_cross_device(e: &io::Error) -> bool {
+    if let Some(no) = e.raw_os_error() {
+        return no == 18;
+    }
+    false
+}
+
+pub fn is_sys_err_too_many_files(e: &io::Error) -> bool {
+    if let Some(no) = e.raw_os_error() {
+        return no == 23 || no == 24;
+    }
+    false
+}
+
+pub fn os_is_not_exist(e: &io::Error) -> bool {
+    e.kind() == ErrorKind::NotFound
+}
+
+pub fn os_is_permission(e: &io::Error) -> bool {
+    if e.kind() == ErrorKind::PermissionDenied {
+        return true;
+    }
+    if let Some(no) = e.raw_os_error() {
+        if no == 30 {
+            return true;
+        }
+    }
+
+    false
+}
+
+pub fn os_is_exist(e: &io::Error) -> bool {
+    e.kind() == ErrorKind::AlreadyExists
+}
+
+// map_err_not_exists
+pub fn map_err_not_exists(e: io::Error) -> Error {
+    if os_is_not_exist(&e) {
+        return Error::new(DiskError::VolumeNotEmpty);
+    } else if is_sys_err_io(&e) {
+        return Error::new(DiskError::FaultyDisk);
+    }
+
+    Error::new(e)
+}
+
+pub fn convert_access_error(e: io::Error, per_err: DiskError) -> Error {
+    if os_is_not_exist(&e) {
+        return Error::new(DiskError::VolumeNotEmpty);
+    } else if is_sys_err_io(&e) {
+        return Error::new(DiskError::FaultyDisk);
+    } else if os_is_permission(&e) {
+        return Error::new(per_err);
+    }
+
+    Error::new(e)
+}

ecstore/src/disk/format.rs

@@ -182,6 +182,55 @@ impl FormatV3 {
 
         Err(Error::msg(format!("disk id not found {}", disk_id)))
     }
+
+    pub fn check_other(&self, other: &FormatV3) -> Result<()> {
+        let mut tmp = other.clone();
+        let this = tmp.erasure.this;
+        tmp.erasure.this = Uuid::nil();
+
+        if self.erasure.sets.len() != other.erasure.sets.len() {
+            return Err(Error::from_string(format!(
+                "Expected number of sets {}, got {}",
+                self.erasure.sets.len(),
+                other.erasure.sets.len()
+            )));
+        }
+
+        for i in 0..self.erasure.sets.len() {
+            if self.erasure.sets[i].len() != other.erasure.sets[i].len() {
+                return Err(Error::from_string(format!(
+                    "Each set should be of same size, expected {}, got {}",
+                    self.erasure.sets[i].len(),
+                    other.erasure.sets[i].len()
+                )));
+            }
+
+            for j in 0..self.erasure.sets[i].len() {
+                if self.erasure.sets[i][j] != self.erasure.sets[i][j] {
+                    return Err(Error::from_string(format!(
+                        "UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)",
+                        i,
+                        j,
+                        self.erasure.sets[i][j].to_string(),
+                        other.erasure.sets[i][j].to_string(),
+                    )));
+                }
+            }
+        }
+
+        for i in 0..tmp.erasure.sets.len() {
+            for j in 0..tmp.erasure.sets[i].len() {
+                if this == tmp.erasure.sets[i][j] {
+                    return Ok(());
+                }
+            }
+        }
+
+        Err(Error::msg(format!(
+            "DriveID {:?} not found in any drive sets {:?}",
+            this, other.erasure.sets
+        )))
+    }
 }
 
 #[cfg(test)]

ecstore/src/disk/mod.rs

@@ -2,6 +2,7 @@ pub mod endpoint;
 pub mod error;
 pub mod format;
 mod local;
+pub mod os;
 pub mod remote;
 
 pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
@@ -18,7 +19,8 @@ use crate::{
     file_meta::FileMeta,
     store_api::{FileInfo, RawFileInfo},
 };
-use bytes::Bytes;
+
+use endpoint::Endpoint;
 use futures::StreamExt;
 use protos::proto_gen::node_service::{
     node_service_client::NodeServiceClient, ReadAtRequest, ReadAtResponse, WriteRequest, WriteResponse,
@@ -50,39 +52,51 @@ pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result<DiskS
 
 #[async_trait::async_trait]
 pub trait DiskAPI: Debug + Send + Sync + 'static {
+    fn to_string(&self) -> String;
+    async fn is_online(&self) -> bool;
     fn is_local(&self) -> bool;
-    fn path(&self) -> PathBuf;
+    // LastConn
+    fn host_name(&self) -> String;
+    fn endpoint(&self) -> Endpoint;
     async fn close(&self) -> Result<()>;
-    async fn get_disk_id(&self) -> Option<Uuid>;
+    async fn get_disk_id(&self) -> Result<Option<Uuid>>;
     async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
 
-    async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
-    async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes>;
-    async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()>;
-    async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>;
-    async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<FileWriter>;
-    async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter>;
-    async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader>;
-    // 读目录下的所有文件、目录
-    async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>>;
+    fn path(&self) -> PathBuf;
+    fn get_disk_location(&self) -> DiskLocation;
+
+    // Healing
+    // DiskInfo
+    // NSScanner
+
+    // Volume operations.
+    async fn make_volume(&self, volume: &str) -> Result<()>;
+    async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>;
+    async fn list_volumes(&self) -> Result<Vec<VolumeInfo>>;
+    async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo>;
+    async fn delete_volume(&self, volume: &str) -> Result<()>;
+
     // 并发边读边写 TODO: wr io.Writer
     async fn walk_dir(&self, opts: WalkDirOptions) -> Result<Vec<MetaCacheEntry>>;
-    async fn rename_data(
+
+    // Metadata operations
+    async fn delete_version(
         &self,
-        src_volume: &str,
-        src_path: &str,
-        file_info: FileInfo,
-        dst_volume: &str,
-        dst_path: &str,
-    ) -> Result<RenameDataResp>;
-
-    async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>;
-    async fn delete_volume(&self, volume: &str) -> Result<()>;
-    async fn list_volumes(&self) -> Result<Vec<VolumeInfo>>;
-    async fn make_volume(&self, volume: &str) -> Result<()>;
-    async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo>;
-
+        volume: &str,
+        path: &str,
+        fi: FileInfo,
+        force_del_marker: bool,
+        opts: DeleteOptions,
+    ) -> Result<RawFileInfo>;
+    async fn delete_versions(
+        &self,
+        volume: &str,
+        versions: Vec<FileInfoVersions>,
+        opts: DeleteOptions,
+    ) -> Result<Vec<Option<Error>>>;
+    async fn delete_paths(&self, volume: &str, paths: &[&str]) -> Result<()>;
     async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>;
+    async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts) -> Result<()>;
     async fn read_version(
         &self,
         org_volume: &str,
@@ -92,13 +106,50 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
         opts: &ReadOptions,
     ) -> Result<FileInfo>;
     async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo>;
-    async fn delete_versions(
+    async fn rename_data(
         &self,
-        volume: &str,
-        versions: Vec<FileInfoVersions>,
-        opts: DeleteOptions,
-    ) -> Result<Vec<Option<Error>>>;
+        src_volume: &str,
+        src_path: &str,
+        file_info: FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+    ) -> Result<RenameDataResp>;
+
+    // File operations.
+    // 读目录下的所有文件、目录
+    async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>>;
+    async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader>;
+    async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter>;
+    async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<FileWriter>;
+    // ReadFileStream
+    async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>;
+    async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()>;
+    // CheckParts
+    async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
+    // VerifyFile
+    // StatInfoFile
+    // ReadParts
     async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>>;
+    // CleanAbandonedData
+    async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()>;
+    async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>>;
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct UpdateMetadataOpts {
+    pub no_persistence: bool,
+}
+
+pub struct DiskLocation {
+    pub pool_idx: Option<usize>,
+    pub set_idx: Option<usize>,
+    pub disk_idx: Option<usize>,
+}
+
+impl DiskLocation {
+    pub fn valid(&self) -> bool {
+        self.pool_idx.is_some() && self.set_idx.is_some() && self.disk_idx.is_some()
+    }
 }
 
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
@@ -214,20 +265,24 @@ impl MetaCacheEntry {
     }
 }
 
+#[derive(Debug, Default)]
 pub struct DiskOption {
     pub cleanup: bool,
     pub health_check: bool,
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Default, Serialize, Deserialize)]
 pub struct RenameDataResp {
     pub old_data_dir: Option<Uuid>,
+    pub sign: Option<Vec<u8>>,
 }
 
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct DeleteOptions {
     pub recursive: bool,
     pub immediate: bool,
+    pub undo_write: bool,
+    pub old_data_dir: Option<Uuid>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

ecstore/src/disk/os.rs

@@ -0,0 +1,204 @@
+use std::{
+    io,
+    path::{Component, Path},
+};
+
+use tokio::fs;
+
+use crate::{
+    disk::error::{is_sys_err_not_dir, is_sys_err_path_not_found, os_is_not_exist},
+    error::{Error, Result},
+    utils,
+};
+
+use super::error::{os_err_to_file_err, os_is_exist, DiskError};
+
+pub fn check_path_length(path_name: &str) -> Result<()> {
+    // Apple OS X path length is limited to 1016
+    if cfg!(target_os = "macos") && path_name.len() > 1016 {
+        return Err(Error::new(DiskError::FileNameTooLong));
+    }
+
+    // Disallow more than 1024 characters on windows, there
+    // are no known name_max limits on Windows.
+    if cfg!(target_os = "windows") && path_name.len() > 1024 {
+        return Err(Error::new(DiskError::FileNameTooLong));
+    }
+
+    // On Unix we reject paths if they are just '.', '..' or '/'
+    let invalid_paths = [".", "..", "/"];
+    if invalid_paths.contains(&path_name) {
+        return Err(Error::new(DiskError::FileAccessDenied));
+    }
+
+    // Check each path segment length is > 255 on all Unix
+    // platforms, look for this value as NAME_MAX in
+    // /usr/include/linux/limits.h
+    let mut count = 0usize;
+    for c in path_name.chars() {
+        match c {
+            '/' | '\\' if cfg!(target_os = "windows") => count = 0, // Reset
+            _ => {
+                count += 1;
+                if count > 255 {
+                    return Err(Error::new(DiskError::FileNameTooLong));
+                }
+            }
+        }
+    }
+
+    // Success.
+    Ok(())
+}
+
+pub async fn make_dir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> Result<()> {
+    check_path_length(path.as_ref().to_string_lossy().to_string().as_str())?;
+
+    if let Err(e) = reliable_mkdir_all(path.as_ref(), base_dir.as_ref()).await {
+        if is_sys_err_not_dir(&e) {
+            return Err(Error::new(DiskError::FileAccessDenied));
+        } else if is_sys_err_path_not_found(&e) {
+            return Err(Error::new(DiskError::FileAccessDenied));
+        }
+
+        return Err(os_err_to_file_err(e));
+    }
+
+    Ok(())
+}
+
+// read_dir  count read limit. when count == 0 unlimit.
+pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> Result<Vec<String>> {
+    let mut entries = fs::read_dir(path.as_ref()).await?;
+
+    let mut volumes = Vec::new();
+
+    let mut count = count;
+
+    while let Some(entry) = entries.next_entry().await? {
+        let name = entry.file_name().to_string_lossy().to_string();
+
+        if name == "" || name == "." || name == ".." {
+            continue;
+        }
+
+        let file_type = entry.file_type().await?;
+
+        if file_type.is_dir() {
+            count -= 1;
+            volumes.push(format!("{}{}", name, utils::path::SLASH_SEPARATOR));
+
+            if count == 0 {
+                break;
+            }
+        }
+    }
+
+    Ok(volumes)
+}
+
+pub async fn rename_all(
+    src_file_path: impl AsRef<Path>,
+    dst_file_path: impl AsRef<Path>,
+    base_dir: impl AsRef<Path>,
+) -> Result<()> {
+    reliable_rename(src_file_path, dst_file_path, base_dir).await.map_err(|e| {
+        if is_sys_err_not_dir(&e) || !os_is_not_exist(&e) {
+            Error::new(DiskError::FileAccessDenied)
+        } else if is_sys_err_path_not_found(&e) {
+            Error::new(DiskError::FileAccessDenied)
+        } else if os_is_not_exist(&e) {
+            Error::new(DiskError::FileNotFound)
+        } else if os_is_exist(&e) {
+            Error::new(DiskError::IsNotRegular)
+        } else {
+            Error::new(e)
+        }
+    })?;
+
+    Ok(())
+}
+
+pub async fn reliable_rename(
+    src_file_path: impl AsRef<Path>,
+    dst_file_path: impl AsRef<Path>,
+    base_dir: impl AsRef<Path>,
+) -> io::Result<()> {
+    if let Some(parent) = dst_file_path.as_ref().parent() {
+        reliable_mkdir_all(parent, base_dir.as_ref()).await?;
+    }
+
+    let mut i = 0;
+    loop {
+        if let Err(e) = utils::fs::rename(src_file_path.as_ref(), dst_file_path.as_ref()).await {
+            if os_is_not_exist(&e) && i == 0 {
+                i += 1;
+                continue;
+            }
+
+            return Err(e);
+        }
+
+        break;
+    }
+
+    Ok(())
+}
+
+pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
+    let mut i = 0;
+
+    let mut base_dir = base_dir.as_ref();
+    loop {
+        if let Err(e) = os_mkdir_all(path.as_ref(), base_dir).await {
+            if os_is_not_exist(&e) && i == 0 {
+                i += 1;
+
+                if let Some(base_parent) = base_dir.parent() {
+                    if let Some(c) = base_parent.components().next() {
+                        if c != Component::RootDir {
+                            base_dir = base_parent
+                        }
+                    }
+                }
+                continue;
+            }
+
+            return Err(e);
+        }
+
+        break;
+    }
+
+    Ok(())
+}
+
+pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
+    if !base_dir.as_ref().to_string_lossy().is_empty() {
+        if base_dir.as_ref().starts_with(dir_path.as_ref()) {
+            return Ok(());
+        }
+    }
+
+    if let Some(parent) = dir_path.as_ref().parent() {
+        // 不支持递归，直接create_dir_all了
+        if let Err(e) = utils::fs::make_dir_all(&parent).await {
+            if os_is_exist(&e) {
+                return Ok(());
+            }
+
+            return Err(e);
+        }
+        // Box::pin(os_mkdir_all(&parent, &base_dir)).await?;
+    }
+
+    if let Err(e) = utils::fs::mkdir(dir_path.as_ref()).await {
+        if os_is_exist(&e) {
+            return Ok(());
+        }
+
+        return Err(e);
+    }
+
+    Ok(())
+}

ecstore/src/disk/remote.rs

@@ -1,30 +1,30 @@
 use std::path::PathBuf;
 
-use bytes::Bytes;
 use futures::lock::Mutex;
 use protos::{
     node_service_time_out_client,
     proto_gen::node_service::{
-        DeleteRequest, DeleteVersionsRequest, DeleteVolumeRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest,
-        MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
-        RenameFileRequst, StatVolumeRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest,
+        DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, ListDirRequest,
+        ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, ReadVersionRequest,
+        ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest, UpdateMetadataRequest, WalkDirRequest,
+        WriteAllRequest, WriteMetadataRequest,
     },
 };
 use tonic::Request;
 use tracing::info;
 use uuid::Uuid;
 
+use super::{
+    endpoint::Endpoint, DeleteOptions, DiskAPI, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter,
+    MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp,
+    UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
+};
 use crate::{
     disk::error::DiskError,
     error::{Error, Result},
     store_api::{FileInfo, RawFileInfo},
 };
-
-use super::{
-    endpoint::Endpoint, DeleteOptions, DiskAPI, DiskOption, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry,
-    ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp, VolumeInfo,
-    WalkDirOptions,
-};
+use protos::proto_gen::node_service::RenamePartRequst;
 
 #[derive(Debug)]
 pub struct RemoteDisk {
@@ -32,6 +32,7 @@ pub struct RemoteDisk {
     pub addr: String,
     pub url: url::Url,
     pub root: PathBuf,
+    endpoint: Endpoint,
 }
 
 impl RemoteDisk {
@@ -44,6 +45,7 @@ impl RemoteDisk {
             addr,
             url: ep.url.clone(),
             root,
+            endpoint: ep.clone(),
         })
     }
 }
@@ -51,9 +53,27 @@ impl RemoteDisk {
 // TODO: all api need to handle errors
 #[async_trait::async_trait]
 impl DiskAPI for RemoteDisk {
+    fn to_string(&self) -> String {
+        self.endpoint.to_string()
+    }
+
     fn is_local(&self) -> bool {
         false
     }
+
+    fn host_name(&self) -> String {
+        self.endpoint.host_port()
+    }
+    async fn is_online(&self) -> bool {
+        // TODO: 连接状态
+        if let Ok(_) = node_service_time_out_client(&self.addr).await {
+            return true;
+        }
+        false
+    }
+    fn endpoint(&self) -> Endpoint {
+        self.endpoint.clone()
+    }
     async fn close(&self) -> Result<()> {
         Ok(())
     }
@@ -61,8 +81,16 @@ impl DiskAPI for RemoteDisk {
         self.root.clone()
     }
 
-    async fn get_disk_id(&self) -> Option<Uuid> {
-        self.id.lock().await.clone()
+    fn get_disk_location(&self) -> DiskLocation {
+        DiskLocation {
+            pool_idx: self.endpoint.pool_idx,
+            set_idx: self.endpoint.set_idx,
+            disk_idx: self.endpoint.pool_idx,
+        }
+    }
+
+    async fn get_disk_id(&self) -> Result<Option<Uuid>> {
+        Ok(self.id.lock().await.clone())
     }
     async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
         let mut lock = self.id.lock().await;
@@ -71,7 +99,7 @@ impl DiskAPI for RemoteDisk {
         Ok(())
     }
 
-    async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes> {
+    async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
         info!("read_all");
         let mut client = node_service_time_out_client(&self.addr)
             .await
@@ -90,7 +118,7 @@ impl DiskAPI for RemoteDisk {
             return Err(DiskError::FileNotFound.into());
         }
 
-        Ok(Bytes::from(response.data))
+        Ok(response.data)
     }
 
     async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
@@ -135,7 +163,28 @@ impl DiskAPI for RemoteDisk {
 
         Ok(())
     }
+    async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()> {
+        info!("rename_part");
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::from_string(format!("can not get client, err: {}", err.to_string())))?;
+        let request = Request::new(RenamePartRequst {
+            disk: self.root.to_string_lossy().to_string(),
+            src_volume: src_volume.to_string(),
+            src_path: src_path.to_string(),
+            dst_volume: dst_volume.to_string(),
+            dst_path: dst_path.to_string(),
+            meta,
+        });
 
+        let response = client.rename_part(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        Ok(())
+    }
     async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
         info!("rename_file");
         let mut client = node_service_time_out_client(&self.addr)
@@ -366,6 +415,51 @@ impl DiskAPI for RemoteDisk {
         Ok(volume_info)
     }
 
+    async fn delete_paths(&self, volume: &str, paths: &[&str]) -> Result<()> {
+        info!("delete_paths");
+        let paths = paths.iter().map(|s| s.to_string()).collect::<Vec<String>>();
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::from_string(format!("can not get client, err: {}", err.to_string())))?;
+        let request = Request::new(DeletePathsRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+            paths,
+        });
+
+        let response = client.delete_paths(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        Ok(())
+    }
+    async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts) -> Result<()> {
+        info!("update_metadata");
+        let file_info = serde_json::to_string(&fi)?;
+        let opts = serde_json::to_string(&opts)?;
+
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::from_string(format!("can not get client, err: {}", err.to_string())))?;
+        let request = Request::new(UpdateMetadataRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+            path: path.to_string(),
+            file_info,
+            opts,
+        });
+
+        let response = client.update_metadata(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        Ok(())
+    }
+
     async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
         info!("write_metadata");
         let file_info = serde_json::to_string(&fi)?;
@@ -442,7 +536,40 @@ impl DiskAPI for RemoteDisk {
 
         Ok(raw_file_info)
     }
+    async fn delete_version(
+        &self,
+        volume: &str,
+        path: &str,
+        fi: FileInfo,
+        force_del_marker: bool,
+        opts: DeleteOptions,
+    ) -> Result<RawFileInfo> {
+        info!("delete_version");
+        let file_info = serde_json::to_string(&fi)?;
+        let opts = serde_json::to_string(&opts)?;
 
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::from_string(format!("can not get client, err: {}", err.to_string())))?;
+        let request = Request::new(DeleteVersionRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+            path: path.to_string(),
+            file_info,
+            force_del_marker,
+            opts,
+        });
+
+        let response = client.delete_version(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
+
+        Ok(raw_file_info)
+    }
     async fn delete_versions(
         &self,
         volume: &str,