use http for remote read/write

ecstore/src/disk/io.rs

@@ -0,0 +1,229 @@
+use crate::error::Result;
+use futures::TryStreamExt;
+use std::io::Cursor;
+use std::pin::Pin;
+use std::task::Poll;
+use tokio::fs::File;
+use tokio::io::{AsyncRead, AsyncWrite};
+use tokio_util::io::ReaderStream;
+use tokio_util::io::StreamReader;
+use tracing::error;
+use tracing::warn;
+
+#[derive(Debug)]
+pub enum FileReader {
+    Local(File),
+    // Remote(RemoteFileReader),
+    Buffer(Cursor<Vec<u8>>),
+    Http(HttpFileReader),
+}
+
+impl AsyncRead for FileReader {
+    #[tracing::instrument(level = "debug", skip(self, buf))]
+    fn poll_read(
+        mut self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut tokio::io::ReadBuf<'_>,
+    ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
+        match &mut *self {
+            Self::Local(reader) => Pin::new(reader).poll_read(cx, buf),
+            Self::Buffer(reader) => Pin::new(reader).poll_read(cx, buf),
+            Self::Http(reader) => Pin::new(reader).poll_read(cx, buf),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct HttpFileReader {
+    // client: reqwest::Client,
+    // url: String,
+    // disk: String,
+    // volume: String,
+    // path: String,
+    // offset: usize,
+    // length: usize,
+    inner: tokio::io::DuplexStream,
+    // buf: Vec<u8>,
+    // pos: usize,
+}
+
+impl HttpFileReader {
+    pub fn new(url: &str, disk: &str, volume: &str, path: &str, offset: usize, length: usize) -> Result<Self> {
+        warn!("http read start {}", path);
+        let url = url.to_owned();
+        let disk = disk.to_owned();
+        let volume = volume.to_owned();
+        let path = path.to_owned();
+
+        // let (reader, mut writer) = tokio::io::simplex(1024);
+        let (reader, mut writer) = tokio::io::duplex(1024 * 1024 * 10);
+
+        tokio::spawn(async move {
+            let client = reqwest::Client::new();
+            let resp = match client
+                .get(format!(
+                    "{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
+                    url,
+                    urlencoding::encode(&disk),
+                    urlencoding::encode(&volume),
+                    urlencoding::encode(&path),
+                    offset,
+                    length
+                ))
+                .send()
+                .await
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))
+            {
+                Ok(resp) => resp,
+                Err(err) => {
+                    warn!("http file reader error: {}", err);
+                    return;
+                }
+            };
+
+            let mut rd = StreamReader::new(
+                resp.bytes_stream()
+                    .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)),
+            );
+
+            if let Err(err) = tokio::io::copy(&mut rd, &mut writer).await {
+                error!("http file reader copy error: {}", err);
+            };
+        });
+        Ok(Self {
+            // client: reqwest::Client::new(),
+            // url: url.to_string(),
+            // disk: disk.to_string(),
+            // volume: volume.to_string(),
+            // path: path.to_string(),
+            // offset,
+            // length,
+            inner: reader,
+            // buf: Vec::new(),
+            // pos: 0,
+        })
+    }
+}
+
+impl AsyncRead for HttpFileReader {
+    #[tracing::instrument(level = "debug", skip(self, buf))]
+    fn poll_read(
+        mut self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut tokio::io::ReadBuf<'_>,
+    ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
+        Pin::new(&mut self.inner).poll_read(cx, buf)
+    }
+}
+
+#[derive(Debug)]
+pub enum FileWriter {
+    Local(File),
+    Http(HttpFileWriter),
+    Buffer(Cursor<Vec<u8>>),
+}
+
+impl AsyncWrite for FileWriter {
+    #[tracing::instrument(level = "debug", skip(self, buf))]
+    fn poll_write(
+        mut self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &[u8],
+    ) -> Poll<std::result::Result<usize, std::io::Error>> {
+        match &mut *self {
+            Self::Local(writer) => Pin::new(writer).poll_write(cx, buf),
+            Self::Buffer(writer) => Pin::new(writer).poll_write(cx, buf),
+            Self::Http(writer) => Pin::new(writer).poll_write(cx, buf),
+        }
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    fn poll_flush(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<std::result::Result<(), std::io::Error>> {
+        match &mut *self {
+            Self::Local(writer) => Pin::new(writer).poll_flush(cx),
+            Self::Buffer(writer) => Pin::new(writer).poll_flush(cx),
+            Self::Http(writer) => Pin::new(writer).poll_flush(cx),
+        }
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<std::result::Result<(), std::io::Error>> {
+        match &mut *self {
+            Self::Local(writer) => Pin::new(writer).poll_shutdown(cx),
+            Self::Buffer(writer) => Pin::new(writer).poll_shutdown(cx),
+            Self::Http(writer) => Pin::new(writer).poll_shutdown(cx),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct HttpFileWriter {
+    wd: tokio::io::WriteHalf<tokio::io::SimplexStream>,
+}
+
+impl HttpFileWriter {
+    pub fn new(url: &str, disk: &str, volume: &str, path: &str, size: usize, append: bool) -> Result<Self> {
+        let (rd, wd) = tokio::io::simplex(1024 * 1024 * 10);
+
+        let body = reqwest::Body::wrap_stream(ReaderStream::new(rd));
+
+        let url = url.to_owned();
+        let disk = disk.to_owned();
+        let volume = volume.to_owned();
+        let path = path.to_owned();
+
+        tokio::spawn(async move {
+            let client = reqwest::Client::new();
+            if let Err(err) = client
+                .put(format!(
+                    "{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
+                    url,
+                    urlencoding::encode(&disk),
+                    urlencoding::encode(&volume),
+                    urlencoding::encode(&path),
+                    append,
+                    size
+                ))
+                .body(body)
+                .send()
+                .await
+            {
+                error!("HttpFileWriter put file err: {:?}", err);
+                // return;
+            }
+
+            // TODO: handle response
+
+            // debug!("http write done {}", path);
+        });
+
+        Ok(Self {
+            wd,
+            // client: reqwest::Client::new(),
+            // url: url.to_string(),
+            // disk: disk.to_string(),
+            // volume: volume.to_string(),
+        })
+    }
+}
+
+impl AsyncWrite for HttpFileWriter {
+    #[tracing::instrument(level = "debug", skip(self, buf))]
+    fn poll_write(
+        mut self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &[u8],
+    ) -> Poll<std::result::Result<usize, std::io::Error>> {
+        Pin::new(&mut self.wd).poll_write(cx, buf)
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    fn poll_flush(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<std::result::Result<(), std::io::Error>> {
+        Pin::new(&mut self.wd).poll_flush(cx)
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<std::result::Result<(), std::io::Error>> {
+        Pin::new(&mut self.wd).poll_shutdown(cx)
+    }
+}

ecstore/src/disk/local.rs

@@ -17,7 +17,7 @@ use crate::disk::error::{
     is_sys_err_not_dir, map_err_not_exists, os_err_to_file_err,
 };
 use crate::disk::os::{check_path_length, is_empty_dir};
-use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE};
+use crate::disk::STORAGE_FORMAT_FILE;
 use crate::error::{Error, Result};
 use crate::file_meta::{get_file_info, read_xl_meta_no_data, FileInfoOpts};
 use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
@@ -745,15 +745,7 @@ impl LocalDisk {
 
         let meta = file.metadata().await?;
 
-        bitrot_verify(
-            FileReader::Local(LocalFileReader::new(file)),
-            meta.size() as usize,
-            part_size,
-            algo,
-            sum.to_vec(),
-            shard_size,
-        )
-        .await
+        bitrot_verify(FileReader::Local(file), meta.size() as usize, part_size, algo, sum.to_vec(), shard_size).await
     }
 
     async fn scan_dir<W: AsyncWrite + Unpin>(
@@ -1297,6 +1289,7 @@ impl DiskAPI for LocalDisk {
         Ok(resp)
     }
 
+    #[tracing::instrument(level = "debug", skip(self))]
     async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()> {
         let src_volume_dir = self.get_bucket_path(src_volume)?;
         let dst_volume_dir = self.get_bucket_path(dst_volume)?;
@@ -1311,12 +1304,18 @@ impl DiskAPI for LocalDisk {
         let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR);
 
         if !src_is_dir && dst_is_dir || src_is_dir && !dst_is_dir {
+            warn!(
+                "rename_part src and dst must be both dir or file src_is_dir:{}, dst_is_dir:{}",
+                src_is_dir, dst_is_dir
+            );
             return Err(Error::from(DiskError::FileAccessDenied));
         }
 
         let src_file_path = src_volume_dir.join(Path::new(src_path));
         let dst_file_path = dst_volume_dir.join(Path::new(dst_path));
 
+        warn!("rename_part src_file_path:{:?}, dst_file_path:{:?}", &src_file_path, &dst_file_path);
+
         check_path_length(src_file_path.to_string_lossy().as_ref())?;
         check_path_length(dst_file_path.to_string_lossy().as_ref())?;
 
@@ -1337,12 +1336,14 @@ impl DiskAPI for LocalDisk {
 
             if let Some(meta) = meta_op {
                 if !meta.is_dir() {
+                    warn!("rename_part src is not dir {:?}", &src_file_path);
                     return Err(Error::new(DiskError::FileAccessDenied));
                 }
             }
 
             if let Err(e) = utils::fs::remove(&dst_file_path).await {
                 if is_sys_err_not_empty(&e) || is_sys_err_not_dir(&e) {
+                    warn!("rename_part remove dst failed {:?} err {:?}", &dst_file_path, e);
                     return Err(Error::new(DiskError::FileAccessDenied));
                 } else if is_sys_err_io(&e) {
                     return Err(Error::new(DiskError::FaultyDisk));
@@ -1355,6 +1356,7 @@ impl DiskAPI for LocalDisk {
         if let Err(err) = os::rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await {
             if let Some(e) = err.to_io_err() {
                 if is_sys_err_not_empty(&e) || is_sys_err_not_dir(&e) {
+                    warn!("rename_part rename all failed {:?} err {:?}", &dst_file_path, e);
                     return Err(Error::new(DiskError::FileAccessDenied));
                 }
 
@@ -1467,8 +1469,10 @@ impl DiskAPI for LocalDisk {
         Ok(())
     }
 
-    // TODO: use io.reader
+    #[tracing::instrument(level = "debug", skip(self))]
     async fn create_file(&self, origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result<FileWriter> {
+        warn!("disk create_file: origvolume: {}, volume: {}, path: {}", origvolume, volume, path);
+
         if !origvolume.is_empty() {
             let origvolume_dir = self.get_bucket_path(origvolume)?;
             if !skip_access_checks(origvolume) {
@@ -1491,12 +1495,16 @@ impl DiskAPI for LocalDisk {
             .await
             .map_err(os_err_to_file_err)?;
 
-        Ok(FileWriter::Local(LocalFileWriter::new(f)))
+        Ok(FileWriter::Local(f))
 
         // Ok(())
     }
+
+    #[tracing::instrument(level = "debug", skip(self))]
     // async fn append_file(&self, volume: &str, path: &str, mut r: DuplexStream) -> Result<File> {
     async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
+        warn!("disk append_file:  volume: {}, path: {}", volume, path);
+
         let volume_dir = self.get_bucket_path(volume)?;
         if !skip_access_checks(volume) {
             if let Err(e) = utils::fs::access(&volume_dir).await {
@@ -1509,11 +1517,13 @@ impl DiskAPI for LocalDisk {
 
         let f = self.open_file(file_path, O_CREATE | O_APPEND | O_WRONLY, volume_dir).await?;
 
-        Ok(FileWriter::Local(LocalFileWriter::new(f)))
+        Ok(FileWriter::Local(f))
     }
 
     // TODO: io verifier
+    #[tracing::instrument(level = "debug", skip(self))]
     async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
+        warn!("disk read_file: volume: {}, path: {}", volume, path);
         let volume_dir = self.get_bucket_path(volume)?;
         if !skip_access_checks(volume) {
             if let Err(e) = utils::fs::access(&volume_dir).await {
@@ -1542,10 +1552,16 @@ impl DiskAPI for LocalDisk {
             }
         })?;
 
-        Ok(FileReader::Local(LocalFileReader::new(f)))
+        Ok(FileReader::Local(f))
     }
 
+    #[tracing::instrument(level = "debug", skip(self))]
     async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> {
+        warn!(
+            "disk read_file_stream: volume: {}, path: {}, offset: {}, length: {}",
+            volume, path, offset, length
+        );
+
         let volume_dir = self.get_bucket_path(volume)?;
         if !skip_access_checks(volume) {
             if let Err(e) = utils::fs::access(&volume_dir).await {
@@ -1587,7 +1603,7 @@ impl DiskAPI for LocalDisk {
 
         f.seek(SeekFrom::Start(offset as u64)).await?;
 
-        Ok(FileReader::Local(LocalFileReader::new(f)))
+        Ok(FileReader::Local(f))
     }
     #[tracing::instrument(level = "debug", skip(self))]
     async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {

ecstore/src/disk/mod.rs

@@ -1,6 +1,7 @@
 pub mod endpoint;
 pub mod error;
 pub mod format;
+pub mod io;
 pub mod local;
 pub mod os;
 pub mod remote;
@@ -14,10 +15,8 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
 pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
 pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
 
-use crate::utils::proto_err_to_err;
 use crate::{
     bucket::{metadata_sys::get_versioning_config, versioning::VersioningApi},
-    erasure::Writer,
     error::{Error, Result},
     file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion, VersionType},
     heal::{
@@ -28,28 +27,16 @@ use crate::{
     store_api::{FileInfo, ObjectInfo, RawFileInfo},
     utils::path::SLASH_SEPARATOR,
 };
-
 use endpoint::Endpoint;
 use error::DiskError;
-use futures::StreamExt;
+use io::{FileReader, FileWriter};
 use local::LocalDisk;
 use madmin::info_commands::DiskMetrics;
-use protos::proto_gen::node_service::{node_service_client::NodeServiceClient, WriteRequest, WriteResponse};
 use remote::RemoteDisk;
 use serde::{Deserialize, Serialize};
-use std::io::Read as _;
-use std::pin::Pin;
-use std::task::Poll;
-use std::{any::Any, cmp::Ordering, fmt::Debug, io::Cursor, path::PathBuf, sync::Arc};
+use std::{cmp::Ordering, fmt::Debug, path::PathBuf, sync::Arc};
 use time::OffsetDateTime;
-use tokio::io::AsyncRead;
-use tokio::{
-    fs::File,
-    io::{AsyncWrite, AsyncWriteExt},
-    sync::mpsc::{self, Sender},
-};
-use tokio_stream::wrappers::ReceiverStream;
-use tonic::{service::interceptor::InterceptedService, transport::Channel, Request, Status, Streaming};
+use tokio::{io::AsyncWrite, sync::mpsc::Sender};
 use tracing::info;
 use tracing::warn;
 use uuid::Uuid;
@@ -1256,164 +1243,142 @@ pub struct ReadOptions {
 //     }
 // }
 
-#[derive(Debug)]
-pub enum FileWriter {
-    Local(LocalFileWriter),
-    Remote(RemoteFileWriter),
-    Buffer(BufferWriter),
-}
+// #[derive(Debug)]
+// pub struct BufferWriter {
+//     pub inner: Vec<u8>,
+// }
 
-#[async_trait::async_trait]
-impl Writer for FileWriter {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
+// impl BufferWriter {
+//     pub fn new(inner: Vec<u8>) -> Self {
+//         Self { inner }
+//     }
+//     #[allow(clippy::should_implement_trait)]
+//     pub fn as_ref(&self) -> &[u8] {
+//         self.inner.as_ref()
+//     }
+// }
 
-    async fn write(&mut self, buf: &[u8]) -> Result<()> {
-        match self {
-            Self::Local(writer) => writer.write(buf).await,
-            Self::Remote(writter) => writter.write(buf).await,
-            Self::Buffer(writer) => writer.write(buf).await,
-        }
-    }
-}
+// #[async_trait::async_trait]
+// impl Writer for BufferWriter {
+//     fn as_any(&self) -> &dyn Any {
+//         self
+//     }
 
-#[derive(Debug)]
-pub struct BufferWriter {
-    pub inner: Vec<u8>,
-}
+//     async fn write(&mut self, buf: &[u8]) -> Result<()> {
+//         let _ = self.inner.write(buf).await?;
+//         self.inner.flush().await?;
 
-impl BufferWriter {
-    pub fn new(inner: Vec<u8>) -> Self {
-        Self { inner }
-    }
-    #[allow(clippy::should_implement_trait)]
-    pub fn as_ref(&self) -> &[u8] {
-        self.inner.as_ref()
-    }
-}
+//         Ok(())
+//     }
+// }
 
-#[async_trait::async_trait]
-impl Writer for BufferWriter {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
+// #[derive(Debug)]
+// pub struct LocalFileWriter {
+//     pub inner: File,
+// }
 
-    async fn write(&mut self, buf: &[u8]) -> Result<()> {
-        let _ = self.inner.write(buf).await?;
-        self.inner.flush().await?;
+// impl LocalFileWriter {
+//     pub fn new(inner: File) -> Self {
+//         Self { inner }
+//     }
+// }
 
-        Ok(())
-    }
-}
+// #[async_trait::async_trait]
+// impl Writer for LocalFileWriter {
+//     fn as_any(&self) -> &dyn Any {
+//         self
+//     }
 
-#[derive(Debug)]
-pub struct LocalFileWriter {
-    pub inner: File,
-}
+//     async fn write(&mut self, buf: &[u8]) -> Result<()> {
+//         let _ = self.inner.write(buf).await?;
+//         self.inner.flush().await?;
 
-impl LocalFileWriter {
-    pub fn new(inner: File) -> Self {
-        Self { inner }
-    }
-}
+//         Ok(())
+//     }
+// }
 
-#[async_trait::async_trait]
-impl Writer for LocalFileWriter {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
+// type NodeClient = NodeServiceClient<
+//     InterceptedService<Channel, Box<dyn Fn(Request<()>) -> Result<Request<()>, Status> + Send + Sync + 'static>>,
+// >;
 
-    async fn write(&mut self, buf: &[u8]) -> Result<()> {
-        let _ = self.inner.write(buf).await?;
-        self.inner.flush().await?;
+// #[derive(Debug)]
+// pub struct RemoteFileWriter {
+//     pub endpoint: Endpoint,
+//     pub volume: String,
+//     pub path: String,
+//     pub is_append: bool,
+//     tx: Sender<WriteRequest>,
+//     resp_stream: Streaming<WriteResponse>,
+// }
 
-        Ok(())
-    }
-}
+// impl RemoteFileWriter {
+//     pub async fn new(endpoint: Endpoint, volume: String, path: String, is_append: bool, mut client: NodeClient) -> Result<Self> {
+//         let (tx, rx) = mpsc::channel(128);
+//         let in_stream = ReceiverStream::new(rx);
 
-type NodeClient = NodeServiceClient<
-    InterceptedService<Channel, Box<dyn Fn(Request<()>) -> Result<Request<()>, Status> + Send + Sync + 'static>>,
->;
+//         let response = client.write_stream(in_stream).await.unwrap();
 
-#[derive(Debug)]
-pub struct RemoteFileWriter {
-    pub endpoint: Endpoint,
-    pub volume: String,
-    pub path: String,
-    pub is_append: bool,
-    tx: Sender<WriteRequest>,
-    resp_stream: Streaming<WriteResponse>,
-}
+//         let resp_stream = response.into_inner();
 
-impl RemoteFileWriter {
-    pub async fn new(endpoint: Endpoint, volume: String, path: String, is_append: bool, mut client: NodeClient) -> Result<Self> {
-        let (tx, rx) = mpsc::channel(128);
-        let in_stream = ReceiverStream::new(rx);
+//         Ok(Self {
+//             endpoint,
+//             volume,
+//             path,
+//             is_append,
+//             tx,
+//             resp_stream,
+//         })
+//     }
+// }
 
-        let response = client.write_stream(in_stream).await.unwrap();
+// #[async_trait::async_trait]
+// impl Writer for RemoteFileWriter {
+//     fn as_any(&self) -> &dyn Any {
+//         self
+//     }
 
-        let resp_stream = response.into_inner();
+//     async fn write(&mut self, buf: &[u8]) -> Result<()> {
+//         let request = WriteRequest {
+//             disk: self.endpoint.to_string(),
+//             volume: self.volume.to_string(),
+//             path: self.path.to_string(),
+//             is_append: self.is_append,
+//             data: buf.to_vec(),
+//         };
+//         self.tx.send(request).await?;
 
-        Ok(Self {
-            endpoint,
-            volume,
-            path,
-            is_append,
-            tx,
-            resp_stream,
-        })
-    }
-}
+//         if let Some(resp) = self.resp_stream.next().await {
+//             // match resp {
+//             //     Ok(resp) => {
+//             //         if resp.success {
+//             //             info!("write stream success");
+//             //         } else {
+//             //             info!("write stream failed: {}", resp.error_info.unwrap_or("".to_string()));
+//             //         }
+//             //     }
+//             //     Err(_err) => {
 
-#[async_trait::async_trait]
-impl Writer for RemoteFileWriter {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
+//             //     }
+//             // }
+//             let resp = resp?;
+//             if resp.success {
+//                 info!("write stream success");
+//             } else {
+//                 return if let Some(err) = &resp.error {
+//                     Err(proto_err_to_err(err))
+//                 } else {
+//                     Err(Error::from_string(""))
+//                 };
+//             }
+//         } else {
+//             let error_info = "can not get response";
+//             info!("write stream failed: {}", error_info);
+//             return Err(Error::from_string(error_info));
+//         }
 
-    async fn write(&mut self, buf: &[u8]) -> Result<()> {
-        let request = WriteRequest {
-            disk: self.endpoint.to_string(),
-            volume: self.volume.to_string(),
-            path: self.path.to_string(),
-            is_append: self.is_append,
-            data: buf.to_vec(),
-        };
-        self.tx.send(request).await?;
-
-        if let Some(resp) = self.resp_stream.next().await {
-            // match resp {
-            //     Ok(resp) => {
-            //         if resp.success {
-            //             info!("write stream success");
-            //         } else {
-            //             info!("write stream failed: {}", resp.error_info.unwrap_or("".to_string()));
-            //         }
-            //     }
-            //     Err(_err) => {
-
-            //     }
-            // }
-            let resp = resp?;
-            if resp.success {
-                info!("write stream success");
-            } else {
-                return if let Some(err) = &resp.error {
-                    Err(proto_err_to_err(err))
-                } else {
-                    Err(Error::from_string(""))
-                };
-            }
-        } else {
-            let error_info = "can not get response";
-            info!("write stream failed: {}", error_info);
-            return Err(Error::from_string(error_info));
-        }
-
-        Ok(())
-    }
-}
+//         Ok(())
+//     }
+// }
 
 // #[async_trait::async_trait]
 // pub trait Reader {
@@ -1422,29 +1387,6 @@ impl Writer for RemoteFileWriter {
 //     // async fn read_exact(&mut self, buf: &mut [u8]) -> Result<usize>;
 // }
 
-#[derive(Debug)]
-pub enum FileReader {
-    Local(LocalFileReader),
-    // Remote(RemoteFileReader),
-    Buffer(BufferReader),
-    Http(HttpFileReader),
-}
-
-impl AsyncRead for FileReader {
-    #[tracing::instrument(level = "debug", skip(self, buf))]
-    fn poll_read(
-        mut self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-        buf: &mut tokio::io::ReadBuf<'_>,
-    ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
-        match &mut *self {
-            Self::Local(reader) => Pin::new(&mut reader.inner).poll_read(cx, buf),
-            Self::Buffer(reader) => Pin::new(&mut reader.inner).poll_read(cx, buf),
-            Self::Http(reader) => Pin::new(reader).poll_read(cx, buf),
-        }
-    }
-}
-
 // #[async_trait::async_trait]
 // impl Reader for FileReader {
 //     async fn read_at(&mut self, offset: usize, buf: &mut [u8]) -> Result<usize> {
@@ -1471,44 +1413,44 @@ impl AsyncRead for FileReader {
 //     // }
 // }
 
-#[derive(Debug)]
-pub struct BufferReader {
-    pub inner: Cursor<Vec<u8>>,
-    remaining: usize,
-}
+// #[derive(Debug)]
+// pub struct BufferReader {
+//     pub inner: Cursor<Vec<u8>>,
+//     remaining: usize,
+// }
 
-impl BufferReader {
-    pub fn new(inner: Vec<u8>, offset: usize, read_length: usize) -> Self {
-        let mut cur = Cursor::new(inner);
-        cur.set_position(offset as u64);
-        Self {
-            inner: cur,
-            remaining: offset + read_length,
-        }
-    }
-}
+// impl BufferReader {
+//     pub fn new(inner: Vec<u8>, offset: usize, read_length: usize) -> Self {
+//         let mut cur = Cursor::new(inner);
+//         cur.set_position(offset as u64);
+//         Self {
+//             inner: cur,
+//             remaining: offset + read_length,
+//         }
+//     }
+// }
 
-impl AsyncRead for BufferReader {
-    #[tracing::instrument(level = "debug", skip(self, buf))]
-    fn poll_read(
-        mut self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-        buf: &mut tokio::io::ReadBuf<'_>,
-    ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
-        match Pin::new(&mut self.inner).poll_read(cx, buf) {
-            Poll::Ready(Ok(_)) => {
-                if self.inner.position() as usize >= self.remaining {
-                    self.remaining -= buf.filled().len();
-                    Poll::Ready(Ok(()))
-                } else {
-                    Poll::Pending
-                }
-            }
-            Poll::Ready(Err(err)) => Poll::Ready(Err(err)),
-            Poll::Pending => Poll::Pending,
-        }
-    }
-}
+// impl AsyncRead for BufferReader {
+//     #[tracing::instrument(level = "debug", skip(self, buf))]
+//     fn poll_read(
+//         mut self: Pin<&mut Self>,
+//         cx: &mut std::task::Context<'_>,
+//         buf: &mut tokio::io::ReadBuf<'_>,
+//     ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
+//         match Pin::new(&mut self.inner).poll_read(cx, buf) {
+//             Poll::Ready(Ok(_)) => {
+//                 if self.inner.position() as usize >= self.remaining {
+//                     self.remaining -= buf.filled().len();
+//                     Poll::Ready(Ok(()))
+//                 } else {
+//                     Poll::Pending
+//                 }
+//             }
+//             Poll::Ready(Err(err)) => Poll::Ready(Err(err)),
+//             Poll::Pending => Poll::Pending,
+//         }
+//     }
+// }
 
 // #[async_trait::async_trait]
 // impl Reader for BufferReader {
@@ -1537,17 +1479,17 @@ impl AsyncRead for BufferReader {
 //     // }
 // }
 
-#[derive(Debug)]
-pub struct LocalFileReader {
-    pub inner: File,
-    // pos: usize,
-}
+// #[derive(Debug)]
+// pub struct LocalFileReader {
+//     pub inner: File,
+//     // pos: usize,
+// }
 
-impl LocalFileReader {
-    pub fn new(inner: File) -> Self {
-        Self { inner }
-    }
-}
+// impl LocalFileReader {
+//     pub fn new(inner: File) -> Self {
+//         Self { inner }
+//     }
+// }
 
 // #[async_trait::async_trait]
 // impl Reader for LocalFileReader {
@@ -1579,16 +1521,16 @@ impl LocalFileReader {
 //     // }
 // }
 
-impl AsyncRead for LocalFileReader {
-    #[tracing::instrument(level = "debug", skip(self, buf))]
-    fn poll_read(
-        mut self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-        buf: &mut tokio::io::ReadBuf<'_>,
-    ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
-        Pin::new(&mut self.inner).poll_read(cx, buf)
-    }
-}
+// impl AsyncRead for LocalFileReader {
+//     #[tracing::instrument(level = "debug", skip(self, buf))]
+//     fn poll_read(
+//         mut self: Pin<&mut Self>,
+//         cx: &mut std::task::Context<'_>,
+//         buf: &mut tokio::io::ReadBuf<'_>,
+//     ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
+//         Pin::new(&mut self.inner).poll_read(cx, buf)
+//     }
+// }
 
 // #[derive(Debug)]
 // pub struct RemoteFileReader {
@@ -1670,85 +1612,3 @@ impl AsyncRead for LocalFileReader {
 //         unimplemented!("poll_read")
 //     }
 // }
-
-#[derive(Debug)]
-pub struct HttpFileReader {
-    // client: reqwest::Client,
-    // url: String,
-    // disk: String,
-    // volume: String,
-    // path: String,
-    // offset: usize,
-    // length: usize,
-    inner: reqwest::blocking::Response,
-    // buf: Vec<u8>,
-    pos: usize,
-}
-
-impl HttpFileReader {
-    pub async fn new(url: &str, disk: &str, volume: &str, path: &str, offset: usize, length: usize) -> Result<Self> {
-        let client = reqwest::blocking::Client::new();
-        let resp = client
-            .get(format!(
-                "{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
-                url, disk, volume, path, offset, length
-            ))
-            .send()?;
-        Ok(Self {
-            // client: reqwest::Client::new(),
-            // url: url.to_string(),
-            // disk: disk.to_string(),
-            // volume: volume.to_string(),
-            // path: path.to_string(),
-            // offset,
-            // length,
-            inner: resp,
-            // buf: Vec::new(),
-            pos: 0,
-        })
-    }
-
-    // pub async fn get_response(&self) -> Result<&Response, std::io::Error> {
-    //     if let Some(resp) = self.inner.get() {
-    //         return Ok(resp);
-    //     } else {
-    //         let client = reqwest::Client::new();
-    //         let resp = client
-    //             .get(&format!(
-    //                 "{}/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
-    //                 self.url, self.disk, self.volume, self.path, self.offset, self.length
-    //             ))
-    //             .send()
-    //             .await
-    //             .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
-    //         self.inner.set(resp);
-    //         Ok(self.inner.get().unwrap())
-    //     }
-    // }
-}
-
-impl AsyncRead for HttpFileReader {
-    #[tracing::instrument(level = "debug", skip(self, buf))]
-    fn poll_read(
-        mut self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-        buf: &mut tokio::io::ReadBuf<'_>,
-    ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
-        let buf = buf.initialize_unfilled();
-        self.inner.read_exact(buf)?;
-        self.pos += buf.len();
-        Poll::Ready(Ok(()))
-    }
-}
-
-// impl Reader for HttpFileReader {
-//     async fn read_at(&mut self, offset: usize, buf: &mut [u8]) -> Result<usize> {
-//         if self.pos != offset {
-//             self.inner.seek(SeekFrom::Start(offset as u64))?;
-//             self.pos = offset;
-//         }
-//         let bytes_read = self.inner.read(buf)?;
-//         self.pos += bytes_read;
-//         Ok(bytes_read)
-//     }
-// }

ecstore/src/disk/remote.rs

@@ -22,8 +22,8 @@ use tracing::info;
 use uuid::Uuid;
 
 use super::{
-    endpoint::Endpoint, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
-    FileInfoVersions, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileWriter, RenameDataResp,
+    endpoint::Endpoint, io::HttpFileReader, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
+    DiskOption, FileInfoVersions, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
     UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
 };
 use crate::{
@@ -36,7 +36,7 @@ use crate::{
     },
     store_api::{FileInfo, RawFileInfo},
 };
-use crate::{disk::HttpFileReader, utils::proto_err_to_err};
+use crate::{disk::io::HttpFileWriter, utils::proto_err_to_err};
 use crate::{disk::MetaCacheEntry, metacache::writer::MetacacheWriter};
 use protos::proto_gen::node_service::RenamePartRequst;
 
@@ -286,6 +286,7 @@ impl DiskAPI for RemoteDisk {
 
         Ok(())
     }
+    #[tracing::instrument(level = "debug", skip(self))]
     async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
         info!("rename_file");
         let mut client = node_service_time_out_client(&self.addr)
@@ -312,58 +313,55 @@ impl DiskAPI for RemoteDisk {
         Ok(())
     }
 
-    async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result<FileWriter> {
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<FileWriter> {
         info!("create_file");
-        Ok(FileWriter::Remote(
-            RemoteFileWriter::new(
-                self.endpoint.clone(),
-                volume.to_string(),
-                path.to_string(),
-                false,
-                node_service_time_out_client(&self.addr)
-                    .await
-                    .map_err(|err| Error::from_string(format!("can not get client, err: {}", err)))?,
-            )
-            .await?,
-        ))
+        Ok(FileWriter::Http(HttpFileWriter::new(
+            self.endpoint.grid_host().as_str(),
+            self.endpoint.to_string().as_str(),
+            volume,
+            path,
+            file_size,
+            false,
+        )?))
     }
 
+    #[tracing::instrument(level = "debug", skip(self))]
     async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
         info!("append_file");
-        Ok(FileWriter::Remote(
-            RemoteFileWriter::new(
-                self.endpoint.clone(),
-                volume.to_string(),
-                path.to_string(),
-                true,
-                node_service_time_out_client(&self.addr)
-                    .await
-                    .map_err(|err| Error::from_string(format!("can not get client, err: {}", err)))?,
-            )
-            .await?,
-        ))
+        Ok(FileWriter::Http(HttpFileWriter::new(
+            self.endpoint.grid_host().as_str(),
+            self.endpoint.to_string().as_str(),
+            volume,
+            path,
+            0,
+            true,
+        )?))
     }
 
+    #[tracing::instrument(level = "debug", skip(self))]
     async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
         info!("read_file");
-        Ok(FileReader::Http(
-            HttpFileReader::new(self.endpoint.grid_host().as_str(), self.endpoint.to_string().as_str(), volume, path, 0, 0)
-                .await?,
-        ))
+        Ok(FileReader::Http(HttpFileReader::new(
+            self.endpoint.grid_host().as_str(),
+            self.endpoint.to_string().as_str(),
+            volume,
+            path,
+            0,
+            0,
+        )?))
     }
 
+    #[tracing::instrument(level = "debug", skip(self))]
     async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> {
-        Ok(FileReader::Http(
-            HttpFileReader::new(
-                self.endpoint.grid_host().as_str(),
-                self.endpoint.to_string().as_str(),
-                volume,
-                path,
-                offset,
-                length,
-            )
-            .await?,
-        ))
+        Ok(FileReader::Http(HttpFileReader::new(
+            self.endpoint.grid_host().as_str(),
+            self.endpoint.to_string().as_str(),
+            volume,
+            path,
+            offset,
+            length,
+        )?))
     }
 
     async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {