merge main

ecstore/src/disk/local.rs

@@ -3,7 +3,7 @@ use super::{
     DeleteOptions, DiskAPI, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp,
     ReadOptions, RenameDataResp, VolumeInfo, WalkDirOptions,
 };
-use crate::disk::STORAGE_FORMAT_FILE;
+use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE};
 use crate::{
     error::{Error, Result},
     file_meta::FileMeta,
@@ -19,18 +19,29 @@ use std::{
 use time::OffsetDateTime;
 use tokio::fs::{self, File};
 use tokio::io::ErrorKind;
-use tracing::{debug, error, warn};
+use tokio::sync::Mutex;
+use tracing::{debug, warn};
 use uuid::Uuid;
 
+#[derive(Debug)]
+pub struct FormatInfo {
+    pub id: Option<Uuid>,
+    pub _data: Vec<u8>,
+    pub _file_info: Option<Metadata>,
+    pub _last_check: Option<OffsetDateTime>,
+}
+
+impl FormatInfo {}
+
 #[derive(Debug)]
 pub struct LocalDisk {
     pub root: PathBuf,
-    pub id: Uuid,
-    pub _format_data: Vec<u8>,
-    pub _format_meta: Option<Metadata>,
     pub _format_path: PathBuf,
-    // pub format_legacy: bool, // drop
-    pub _format_last_check: Option<OffsetDateTime>,
+    pub format_info: Mutex<FormatInfo>,
+    // pub id: Mutex<Option<Uuid>>,
+    // pub format_data: Mutex<Vec<u8>>,
+    // pub format_file_info: Mutex<Option<Metadata>>,
+    // pub format_last_check: Mutex<Option<OffsetDateTime>>,
 }
 
 impl LocalDisk {
@@ -48,7 +59,7 @@ impl LocalDisk {
 
         let (format_data, format_meta) = read_file_exists(&format_path).await?;
 
-        let mut id = Uuid::nil();
+        let mut id = None;
         // let mut format_legacy = false;
         let mut format_last_check = None;
 
@@ -61,19 +72,26 @@ impl LocalDisk {
                 return Err(Error::from(DiskError::InconsistentDisk));
             }
 
-            id = fm.erasure.this;
+            id = Some(fm.erasure.this);
             // format_legacy = fm.erasure.distribution_algo == DistributionAlgoVersion::V1;
             format_last_check = Some(OffsetDateTime::now_utc());
         }
 
+        let format_info = FormatInfo {
+            id,
+            _data: format_data,
+            _file_info: format_meta,
+            _last_check: format_last_check,
+        };
+
         let disk = Self {
             root,
-            id,
-            _format_meta: format_meta,
-            _format_data: format_data,
             _format_path: format_path,
-            // format_legacy,
-            _format_last_check: format_last_check,
+            format_info: Mutex::new(format_info),
+            // // format_legacy,
+            // format_file_info: Mutex::new(format_meta),
+            // format_data: Mutex::new(format_data),
+            // format_last_check: Mutex::new(format_last_check),
         };
 
         disk.make_meta_volumes().await?;
@@ -204,7 +222,7 @@ impl LocalDisk {
         } else {
             if delete_path.is_dir() {
                 if let Err(err) = fs::remove_dir(&delete_path).await {
-                    error!("remove_dir err {:?} when {:?}", &err, &delete_path);
+                    debug!("remove_dir err {:?} when {:?}", &err, &delete_path);
                     match err.kind() {
                         ErrorKind::NotFound => (),
                         // ErrorKind::DirectoryNotEmpty => (),
@@ -218,7 +236,7 @@ impl LocalDisk {
                 }
             } else {
                 if let Err(err) = fs::remove_file(&delete_path).await {
-                    error!("remove_file err {:?} when {:?}", &err, &delete_path);
+                    debug!("remove_file err {:?} when {:?}", &err, &delete_path);
                     match err.kind() {
                         ErrorKind::NotFound => (),
                         _ => {
@@ -413,9 +431,24 @@ impl DiskAPI for LocalDisk {
     fn is_local(&self) -> bool {
         true
     }
+    async fn close(&self) -> Result<()> {
+        Ok(())
+    }
+    fn path(&self) -> PathBuf {
+        self.root.clone()
+    }
 
-    fn id(&self) -> Uuid {
-        self.id
+    async fn get_disk_id(&self) -> Option<Uuid> {
+        // TODO: check format file
+        let format_info = self.format_info.lock().await;
+
+        format_info.id.clone()
+        // TODO: 判断源文件id,是否有效
+    }
+
+    async fn set_disk_id(&self, _id: Option<Uuid>) -> Result<()> {
+        // 本地不需要设置
+        Ok(())
     }
 
     #[must_use]
@@ -517,7 +550,8 @@ impl DiskAPI for LocalDisk {
 
         let file = File::create(&fpath).await?;
 
-        Ok(FileWriter::new(file))
+        Ok(FileWriter::Local(LocalFileWriter::new(file)))
+        // Ok(FileWriter::new(file))
 
         // let mut writer = BufWriter::new(file);
 
@@ -543,7 +577,8 @@ impl DiskAPI for LocalDisk {
             .open(&p)
             .await?;
 
-        Ok(FileWriter::new(file))
+        Ok(FileWriter::Local(LocalFileWriter::new(file)))
+        // Ok(FileWriter::new(file))
 
         // let mut writer = BufWriter::new(file);
 
@@ -560,7 +595,7 @@ impl DiskAPI for LocalDisk {
         debug!("read_file {:?}", &p);
         let file = File::options().read(true).open(&p).await?;
 
-        Ok(FileReader::new(file))
+        Ok(FileReader::Local(LocalFileReader::new(file)))
 
         // file.seek(SeekFrom::Start(offset as u64)).await?;
 
@@ -756,9 +791,7 @@ impl DiskAPI for LocalDisk {
                 .await?;
         }
 
-        Ok(RenameDataResp {
-            old_data_dir: old_data_dir,
-        })
+        Ok(RenameDataResp { old_data_dir })
     }
 
     async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {

ecstore/src/disk/mod.rs

@@ -2,6 +2,7 @@ pub mod endpoint;
 pub mod error;
 pub mod format;
 mod local;
+mod remote;
 
 pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
 pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
@@ -12,18 +13,21 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
 const STORAGE_FORMAT_FILE: &str = "xl.meta";
 
 use crate::{
-    erasure::ReadAt,
+    erasure::{ReadAt, Write},
     error::{Error, Result},
     file_meta::FileMeta,
     store_api::{FileInfo, RawFileInfo},
 };
 use bytes::Bytes;
-use std::{fmt::Debug, io::SeekFrom, pin::Pin, sync::Arc};
+use protos::proto_gen::node_service::{node_service_client::NodeServiceClient, ReadAtRequest, WriteRequest};
+use serde::{Deserialize, Serialize};
+use std::{fmt::Debug, io::SeekFrom, path::PathBuf, sync::Arc};
 use time::OffsetDateTime;
 use tokio::{
     fs::File,
-    io::{AsyncReadExt, AsyncSeekExt, AsyncWrite},
+    io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt},
 };
+use tonic::{transport::Channel, Request};
 use uuid::Uuid;
 
 pub type DiskStore = Arc<Box<dyn DiskAPI>>;
@@ -33,15 +37,18 @@ pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result<DiskS
         let s = local::LocalDisk::new(ep, opt.cleanup).await?;
         Ok(Arc::new(Box::new(s)))
     } else {
-        let _ = opt.health_check;
-        unimplemented!()
+        let remote_disk = remote::RemoteDisk::new(ep, opt).await?;
+        Ok(Arc::new(Box::new(remote_disk)))
     }
 }
 
 #[async_trait::async_trait]
 pub trait DiskAPI: Debug + Send + Sync + 'static {
     fn is_local(&self) -> bool;
-    fn id(&self) -> Uuid;
+    fn path(&self) -> PathBuf;
+    async fn close(&self) -> Result<()>;
+    async fn get_disk_id(&self) -> Option<Uuid>;
+    async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
 
     async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
     async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes>;
@@ -88,7 +95,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>>;
 }
 
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct FileInfoVersions {
     // Name of the volume.
     pub volume: String,
@@ -104,7 +111,7 @@ pub struct FileInfoVersions {
     pub free_versions: Vec<FileInfo>,
 }
 
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct WalkDirOptions {
     // Bucket to scanner
     pub bucket: String,
@@ -131,7 +138,7 @@ pub struct WalkDirOptions {
     pub disk_id: String,
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Serialize, Deserialize)]
 pub struct MetaCacheEntry {
     // name is the full name of the object including prefixes
     pub name: String,
@@ -206,17 +213,18 @@ pub struct DiskOption {
     pub health_check: bool,
 }
 
+#[derive(Serialize, Deserialize)]
 pub struct RenameDataResp {
     pub old_data_dir: Option<Uuid>,
 }
 
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct DeleteOptions {
     pub recursive: bool,
     pub immediate: bool,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ReadMultipleReq {
     pub bucket: String,
     pub prefix: String,
@@ -227,7 +235,7 @@ pub struct ReadMultipleReq {
     pub max_results: usize,
 }
 
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct ReadMultipleResp {
     pub bucket: String,
     pub prefix: String,
@@ -252,65 +260,154 @@ pub struct ReadMultipleResp {
 //     }
 // }
 
+#[derive(Debug, Deserialize, Serialize)]
 pub struct VolumeInfo {
     pub name: String,
     pub created: Option<OffsetDateTime>,
 }
 
+#[derive(Deserialize, Serialize)]
 pub struct ReadOptions {
     pub read_data: bool,
     pub healing: bool,
 }
 
-pub struct FileWriter {
-    pub inner: Pin<Box<dyn AsyncWrite + Send + Sync + 'static>>,
+// pub struct FileWriter {
+//     pub inner: Pin<Box<dyn AsyncWrite + Send + Sync + 'static>>,
+// }
+
+// impl AsyncWrite for FileWriter {
+//     fn poll_write(
+//         mut self: Pin<&mut Self>,
+//         cx: &mut std::task::Context<'_>,
+//         buf: &[u8],
+//     ) -> std::task::Poll<std::result::Result<usize, std::io::Error>> {
+//         Pin::new(&mut self.inner).poll_write(cx, buf)
+//     }
+
+//     fn poll_flush(
+//         mut self: Pin<&mut Self>,
+//         cx: &mut std::task::Context<'_>,
+//     ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
+//         Pin::new(&mut self.inner).poll_flush(cx)
+//     }
+
+//     fn poll_shutdown(
+//         mut self: Pin<&mut Self>,
+//         cx: &mut std::task::Context<'_>,
+//     ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
+//         Pin::new(&mut self.inner).poll_shutdown(cx)
+//     }
+// }
+
+// impl FileWriter {
+//     pub fn new<W>(inner: W) -> Self
+//     where
+//         W: AsyncWrite + Send + Sync + 'static,
+//     {
+//         Self { inner: Box::pin(inner) }
+//     }
+// }
+
+pub enum FileWriter {
+    Local(LocalFileWriter),
+    Remote(RemoteFileWriter),
 }
 
-impl AsyncWrite for FileWriter {
-    fn poll_write(
-        mut self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-        buf: &[u8],
-    ) -> std::task::Poll<std::result::Result<usize, std::io::Error>> {
-        Pin::new(&mut self.inner).poll_write(cx, buf)
-    }
-
-    fn poll_flush(
-        mut self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
-        Pin::new(&mut self.inner).poll_flush(cx)
-    }
-
-    fn poll_shutdown(
-        mut self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
-        Pin::new(&mut self.inner).poll_shutdown(cx)
+#[async_trait::async_trait]
+impl Write for FileWriter {
+    async fn write(&mut self, buf: &[u8]) -> Result<()> {
+        match self {
+            Self::Local(local_file_writer) => local_file_writer.write(buf).await,
+            Self::Remote(remote_file_writer) => remote_file_writer.write(buf).await,
+        }
     }
 }
 
-impl FileWriter {
-    pub fn new<W>(inner: W) -> Self
-    where
-        W: AsyncWrite + Send + Sync + 'static,
-    {
-        Self { inner: Box::pin(inner) }
-    }
-}
-
-#[derive(Debug)]
-pub struct FileReader {
+pub struct LocalFileWriter {
     pub inner: File,
 }
 
-impl FileReader {
+impl LocalFileWriter {
     pub fn new(inner: File) -> Self {
         Self { inner }
     }
 }
 
+#[async_trait::async_trait]
+impl Write for LocalFileWriter {
+    async fn write(&mut self, buf: &[u8]) -> Result<()> {
+        self.inner.write(buf).await?;
+        self.inner.flush().await?;
+
+        Ok(())
+    }
+}
+
+pub struct RemoteFileWriter {
+    pub root: PathBuf,
+    pub volume: String,
+    pub path: String,
+    pub is_append: bool,
+    client: NodeServiceClient<Channel>,
+}
+
+impl RemoteFileWriter {
+    pub fn new(root: PathBuf, volume: String, path: String, is_append: bool, client: NodeServiceClient<Channel>) -> Self {
+        Self {
+            root,
+            volume,
+            path,
+            is_append,
+            client,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl Write for RemoteFileWriter {
+    async fn write(&mut self, buf: &[u8]) -> Result<()> {
+        let request = Request::new(WriteRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: self.volume.to_string(),
+            path: self.path.to_string(),
+            is_append: self.is_append,
+            data: buf.to_vec(),
+        });
+        let _response = self.client.write(request).await?.into_inner();
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+pub enum FileReader {
+    Local(LocalFileReader),
+    Remote(RemoteFileReader),
+}
+
+#[async_trait::async_trait]
 impl ReadAt for FileReader {
+    async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec<u8>, usize)> {
+        match self {
+            Self::Local(local_file_writer) => local_file_writer.read_at(offset, length).await,
+            Self::Remote(remote_file_writer) => remote_file_writer.read_at(offset, length).await,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct LocalFileReader {
+    pub inner: File,
+}
+
+impl LocalFileReader {
+    pub fn new(inner: File) -> Self {
+        Self { inner }
+    }
+}
+
+#[async_trait::async_trait]
+impl ReadAt for LocalFileReader {
     async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec<u8>, usize)> {
         self.inner.seek(SeekFrom::Start(offset as u64)).await?;
 
@@ -323,3 +420,38 @@ impl ReadAt for FileReader {
         Ok((buffer, bytes_read))
     }
 }
+
+#[derive(Debug)]
+pub struct RemoteFileReader {
+    pub root: PathBuf,
+    pub volume: String,
+    pub path: String,
+    client: NodeServiceClient<Channel>,
+}
+
+impl RemoteFileReader {
+    pub fn new(root: PathBuf, volume: String, path: String, client: NodeServiceClient<Channel>) -> Self {
+        Self {
+            root,
+            volume,
+            path,
+            client,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl ReadAt for RemoteFileReader {
+    async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec<u8>, usize)> {
+        let request = Request::new(ReadAtRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: self.volume.to_string(),
+            path: self.path.to_string(),
+            offset: offset.try_into().unwrap(),
+            length: length.try_into().unwrap(),
+        });
+        let response = self.client.read_at(request).await?.into_inner();
+
+        Ok((response.data, response.read_size.try_into().unwrap()))
+    }
+}

ecstore/src/disk/remote.rs

@@ -0,0 +1,529 @@
+use std::{path::PathBuf, sync::Arc, time::Duration};
+
+use bytes::Bytes;
+use futures::lock::Mutex;
+use protos::{
+    node_service_time_out_client,
+    proto_gen::node_service::{
+        node_service_client::NodeServiceClient, DeleteRequest, DeleteVersionsRequest, DeleteVolumeRequest, ListDirRequest,
+        ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, ReadVersionRequest,
+        ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest, WalkDirRequest, WriteAllRequest,
+        WriteMetadataRequest,
+    },
+    DEFAULT_GRPC_SERVER_MESSAGE_LEN,
+};
+use tokio::{fs, sync::RwLock};
+use tonic::{
+    transport::{Channel, Endpoint as tonic_Endpoint},
+    Request,
+};
+use tower::timeout::Timeout;
+use tracing::info;
+use uuid::Uuid;
+
+use crate::{
+    disk::error::DiskError,
+    error::{Error, Result},
+    store_api::{FileInfo, RawFileInfo},
+};
+
+use super::{
+    endpoint::Endpoint, DeleteOptions, DiskAPI, DiskOption, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry,
+    ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp, VolumeInfo,
+    WalkDirOptions,
+};
+
+#[derive(Debug)]
+pub struct RemoteDisk {
+    id: Mutex<Option<Uuid>>,
+    channel: Arc<RwLock<Option<Channel>>>,
+    url: url::Url,
+    pub root: PathBuf,
+}
+
+impl RemoteDisk {
+    pub async fn new(ep: &Endpoint, _opt: &DiskOption) -> Result<Self> {
+        let root = fs::canonicalize(ep.url.path()).await?;
+
+        Ok(Self {
+            channel: Arc::new(RwLock::new(None)),
+            url: ep.url.clone(),
+            root,
+            id: Mutex::new(None),
+        })
+    }
+
+    #[allow(dead_code)]
+    async fn get_client(&self) -> Result<NodeServiceClient<Timeout<Channel>>> {
+        let channel_clone = self.channel.clone();
+        let channel = {
+            let read_lock = channel_clone.read().await;
+
+            if let Some(ref channel) = *read_lock {
+                channel.clone()
+            } else {
+                let addr = format!("{}://{}:{}", self.url.scheme(), self.url.host_str().unwrap(), self.url.port().unwrap());
+                info!("disk url: {}", addr);
+                let connector = tonic_Endpoint::from_shared(addr.clone())?;
+
+                let new_channel = connector.connect().await.map_err(|_err| DiskError::DiskNotFound)?;
+
+                info!("get channel success");
+
+                *self.channel.write().await = Some(new_channel.clone());
+
+                new_channel
+            }
+        };
+
+        Ok(node_service_time_out_client(
+            channel,
+            Duration::new(30, 0), // TODO: use config setting
+            DEFAULT_GRPC_SERVER_MESSAGE_LEN,
+            // grpc_enable_gzip,
+            false, // TODO: use config setting
+        ))
+    }
+
+    async fn get_client_v2(&self) -> Result<NodeServiceClient<tonic::transport::Channel>> {
+        // Ok(NodeServiceClient::connect("http://220.181.1.138:9000").await?)
+        let addr = format!("{}://{}:{}", self.url.scheme(), self.url.host_str().unwrap(), self.url.port().unwrap());
+        Ok(NodeServiceClient::connect(addr).await?)
+    }
+}
+
+// TODO: all api need to handle errors
+#[async_trait::async_trait]
+impl DiskAPI for RemoteDisk {
+    fn is_local(&self) -> bool {
+        false
+    }
+    async fn close(&self) -> Result<()> {
+        Ok(())
+    }
+    fn path(&self) -> PathBuf {
+        self.root.clone()
+    }
+
+    async fn get_disk_id(&self) -> Option<Uuid> {
+        self.id.lock().await.clone()
+    }
+    async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
+        let mut lock = self.id.lock().await;
+        *lock = id;
+
+        Ok(())
+    }
+
+    async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes> {
+        info!("read_all");
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(ReadAllRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+            path: path.to_string(),
+        });
+
+        let response = client.read_all(request).await?.into_inner();
+
+        info!("read_all success");
+
+        if !response.success {
+            return Err(DiskError::FileNotFound.into());
+        }
+
+        Ok(Bytes::from(response.data))
+    }
+
+    async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
+        info!("write_all");
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(WriteAllRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+            path: path.to_string(),
+            data,
+        });
+
+        let response = client.write_all(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        Ok(())
+    }
+
+    async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
+        info!("delete");
+        let options = serde_json::to_string(&opt)?;
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(DeleteRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+            path: path.to_string(),
+            options,
+        });
+
+        let response = client.delete(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        Ok(())
+    }
+
+    async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
+        info!("rename_file");
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(RenameFileRequst {
+            disk: self.root.to_string_lossy().to_string(),
+            src_volume: src_volume.to_string(),
+            src_path: src_path.to_string(),
+            dst_volume: dst_volume.to_string(),
+            dst_path: dst_path.to_string(),
+        });
+
+        let response = client.rename_file(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        Ok(())
+    }
+
+    async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result<FileWriter> {
+        info!("create_file");
+        Ok(FileWriter::Remote(RemoteFileWriter::new(
+            self.root.clone(),
+            volume.to_string(),
+            path.to_string(),
+            false,
+            self.get_client_v2().await?,
+        )))
+    }
+
+    async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
+        info!("append_file");
+        Ok(FileWriter::Remote(RemoteFileWriter::new(
+            self.root.clone(),
+            volume.to_string(),
+            path.to_string(),
+            true,
+            self.get_client_v2().await?,
+        )))
+    }
+
+    async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
+        info!("read_file");
+        Ok(FileReader::Remote(RemoteFileReader::new(
+            self.root.clone(),
+            volume.to_string(),
+            path.to_string(),
+            self.get_client_v2().await?,
+        )))
+    }
+
+    async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
+        info!("list_dir");
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(ListDirRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+        });
+
+        let response = client.list_dir(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        Ok(response.volumes)
+    }
+
+    async fn walk_dir(&self, opts: WalkDirOptions) -> Result<Vec<MetaCacheEntry>> {
+        info!("walk_dir");
+        let walk_dir_options = serde_json::to_string(&opts)?;
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(WalkDirRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            walk_dir_options,
+        });
+
+        let response = client.walk_dir(request).await?.into_inner();
+        
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+        
+        let entries = response
+            .meta_cache_entry
+            .into_iter()
+            .filter_map(|json_str| serde_json::from_str::<MetaCacheEntry>(&json_str).ok())
+            .collect();
+
+        Ok(entries)
+    }
+
+    async fn rename_data(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+    ) -> Result<RenameDataResp> {
+        info!("rename_data");
+        let file_info = serde_json::to_string(&fi)?;
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(RenameDataRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            src_volume: src_volume.to_string(),
+            src_path: src_path.to_string(),
+            file_info,
+            dst_volume: dst_volume.to_string(),
+            dst_path: dst_path.to_string(),
+        });
+
+        let response = client.rename_data(request).await?.into_inner();
+        
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        let rename_data_resp = serde_json::from_str::<RenameDataResp>(&response.rename_data_resp)?;
+
+        Ok(rename_data_resp)
+    }
+
+    async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
+        info!("make_volumes");
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(MakeVolumesRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volumes: volumes.iter().map(|s| (*s).to_string()).collect(),
+        });
+
+        let response = client.make_volumes(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        Ok(())
+    }
+
+    async fn make_volume(&self, volume: &str) -> Result<()> {
+        info!("make_volume");
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(MakeVolumeRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+        });
+
+        let response = client.make_volume(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        Ok(())
+    }
+
+    async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
+        info!("list_volumes");
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(ListVolumesRequest {
+            disk: self.root.to_string_lossy().to_string(),
+        });
+
+        let response = client.list_volumes(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        let infos = response
+            .volume_infos
+            .into_iter()
+            .filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
+            .collect();
+
+        Ok(infos)
+    }
+
+    async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
+        info!("stat_volume");
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(StatVolumeRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+        });
+
+        let response = client.stat_volume(request).await?.into_inner();
+        
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        let volume_info = serde_json::from_str::<VolumeInfo>(&response.volume_info)?;
+
+        Ok(volume_info)
+    }
+
+    async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
+        info!("write_metadata");
+        let file_info = serde_json::to_string(&fi)?;
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(WriteMetadataRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+            path: path.to_string(),
+            file_info,
+        });
+
+        let response = client.write_metadata(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        Ok(())
+    }
+
+    async fn read_version(
+        &self,
+        _org_volume: &str,
+        volume: &str,
+        path: &str,
+        version_id: &str,
+        opts: &ReadOptions,
+    ) -> Result<FileInfo> {
+        info!("read_version");
+        let opts = serde_json::to_string(opts)?;
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(ReadVersionRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+            path: path.to_string(),
+            version_id: version_id.to_string(),
+            opts,
+        });
+
+        let response = client.read_version(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        let file_info = serde_json::from_str::<FileInfo>(&response.file_info)?;
+
+        Ok(file_info)
+    }
+
+    async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
+        info!("read_xl");
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(ReadXlRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+            path: path.to_string(),
+            read_data,
+        });
+
+        let response = client.read_xl(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
+
+        Ok(raw_file_info)
+    }
+
+    async fn delete_versions(
+        &self,
+        volume: &str,
+        versions: Vec<FileInfoVersions>,
+        opts: DeleteOptions,
+    ) -> Result<Vec<Option<Error>>> {
+        info!("delete_versions");
+        let opts = serde_json::to_string(&opts)?;
+        let mut versions_str = Vec::with_capacity(versions.len());
+        for file_info_versions in versions.iter() {
+            versions_str.push(serde_json::to_string(file_info_versions)?);
+        }
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(DeleteVersionsRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+            versions: versions_str,
+            opts,
+        });
+
+        let response = client.delete_versions(request).await?.into_inner();
+        if !response.success {
+            return Err(Error::from_string(format!(
+                "delete versions remote err: {}",
+                response.error_info.unwrap_or("None".to_string())
+            )));
+        }
+        let errors = response
+            .errors
+            .iter()
+            .map(|error| {
+                if error.is_empty() {
+                    None
+                } else {
+                    Some(Error::from_string(error))
+                }
+            })
+            .collect();
+
+        Ok(errors)
+    }
+
+    async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
+        info!("read_multiple");
+        let read_multiple_req = serde_json::to_string(&req)?;
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(ReadMultipleRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            read_multiple_req,
+        });
+
+        let response = client.read_multiple(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        let read_multiple_resps = response
+            .read_multiple_resps
+            .into_iter()
+            .filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
+            .collect();
+
+        Ok(read_multiple_resps)
+    }
+
+    async fn delete_volume(&self, volume: &str) -> Result<()> {
+        info!("delete_volume");
+        let mut client = self.get_client_v2().await?;
+        let request = Request::new(DeleteVolumeRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            volume: volume.to_string(),
+        });
+
+        let response = client.delete_volume(request).await?.into_inner();
+
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+        }
+
+        Ok(())
+    }
+}