opt network io

rustfs/src/admin/rpc.rs

@@ -4,6 +4,7 @@ use super::router::S3Router;
 use crate::storage::ecfs::bytes_stream;
 use common::error::Result;
 use ecstore::disk::DiskAPI;
+use ecstore::io::READ_BUFFER_SIZE;
 use ecstore::store::find_local_disk;
 use futures::TryStreamExt;
 use http::StatusCode;
@@ -71,7 +72,10 @@ impl Operation for ReadFile {
 
         Ok(S3Response::new((
             StatusCode::OK,
-            Body::from(StreamingBlob::wrap(bytes_stream(ReaderStream::new(file), query.length))),
+            Body::from(StreamingBlob::wrap(bytes_stream(
+                ReaderStream::with_capacity(file, READ_BUFFER_SIZE),
+                query.length,
+            ))),
         )))
     }
 }

rustfs/src/main.rs

@@ -152,9 +152,13 @@ async fn run(opt: config::Opt) -> Result<()> {
 
     for (i, eps) in endpoint_pools.as_ref().iter().enumerate() {
         info!(
-            "created endpoints {}, set_count:{}, drives_per_set: {}, cmd: {:?}, \n{:?}",
-            i, eps.set_count, eps.drives_per_set, eps.cmd_line, eps
+            "created endpoints {}, set_count:{}, drives_per_set: {}, cmd: {:?}",
+            i, eps.set_count, eps.drives_per_set, eps.cmd_line
         );
+
+        for ep in eps.endpoints.as_ref().iter() {
+            info!("  - {}", ep);
+        }
     }
 
     set_global_addr(&opt.address).await;

rustfs/src/storage/ecfs.rs

@@ -20,6 +20,7 @@ use ecstore::bucket::policy_sys::PolicySys;
 use ecstore::bucket::tagging::decode_tags;
 use ecstore::bucket::tagging::encode_tags;
 use ecstore::bucket::versioning_sys::BucketVersioningSys;
+use ecstore::io::READ_BUFFER_SIZE;
 use ecstore::new_object_layer_fn;
 use ecstore::store_api::BucketOptions;
 use ecstore::store_api::CompletePart;
@@ -51,6 +52,8 @@ use s3s::S3;
 use s3s::{S3Request, S3Response};
 use std::fmt::Debug;
 use std::str::FromStr;
+use tokio_util::io::ReaderStream;
+use tokio_util::io::StreamReader;
 use tracing::debug;
 use tracing::error;
 use tracing::info;
@@ -464,8 +467,13 @@ impl S3 for FS {
         };
         let last_modified = info.mod_time.map(Timestamp::from);
 
+        let body = Some(StreamingBlob::wrap(bytes_stream(
+            ReaderStream::with_capacity(reader.stream, READ_BUFFER_SIZE),
+            info.size,
+        )));
+
         let output = GetObjectOutput {
-            body: Some(reader.stream),
+            body,
             content_length: Some(info.size as i64),
             last_modified,
             content_type,
@@ -799,6 +807,10 @@ impl S3 for FS {
             }
         };
 
+        let body = Box::new(StreamReader::new(
+            body.map(|f| f.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))),
+        ));
+
         let mut reader = PutObjReader::new(body, content_length as usize);
 
         let Some(store) = new_object_layer_fn() else {
@@ -911,6 +923,10 @@ impl S3 for FS {
             }
         };
 
+        let body = Box::new(StreamReader::new(
+            body.map(|f| f.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))),
+        ));
+
         // mc cp step 4
         let mut data = PutObjReader::new(body, content_length as usize);
         let opts = ObjectOptions::default();