fix: address correctness, safety, and concurrency issues (#2327)

Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: houseme <housemecn@gmail.com>

crates/ecstore/src/bitrot.rs

@@ -14,21 +14,26 @@
 
 use crate::disk::{self, DiskAPI as _, DiskStore, error::DiskError};
 use crate::erasure_coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
+use bytes::Bytes;
 use rustfs_utils::HashAlgorithm;
 use std::io::Cursor;
+use std::time::Instant;
 use tokio::io::AsyncRead;
+use tracing::debug;
 
 /// Create a BitrotReader from either inline data or disk file stream
 ///
 /// # Parameters
 /// * `inline_data` - Optional inline data, if present, will use Cursor to read from memory
-/// * `disk` - Optional disk reference for file stream reading  
+/// * `disk` - Optional disk reference for file stream reading
 /// * `bucket` - Bucket name for file path
 /// * `path` - File path within the bucket
 /// * `offset` - Starting offset for reading
 /// * `length` - Length to read
 /// * `shard_size` - Shard size for erasure coding
 /// * `checksum_algo` - Hash algorithm for bitrot verification
+/// * `skip_verify` - If true, skip checksum verification
+/// * `use_zero_copy` - If true, use zero-copy read (mmap on Unix)
 #[allow(clippy::too_many_arguments)]
 pub async fn create_bitrot_reader(
     inline_data: Option<&[u8]>,
@@ -40,13 +45,15 @@ pub async fn create_bitrot_reader(
     shard_size: usize,
     checksum_algo: HashAlgorithm,
     skip_verify: bool,
+    use_zero_copy: bool,
 ) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
     // Calculate the total length to read, including the checksum overhead
     let length = length.div_ceil(shard_size) * checksum_algo.size() + length;
     let offset = offset.div_ceil(shard_size) * checksum_algo.size() + offset;
     if let Some(data) = inline_data {
         // Use inline data
-        let mut rd = Cursor::new(data.to_vec());
+        let mut rd = Cursor::new(Bytes::copy_from_slice(data));
+        // Apply the computed offset so inline data matches disk read behavior
         rd.set_position(offset as u64);
         let reader = BitrotReader::new(
             Box::new(rd) as Box<dyn AsyncRead + Send + Sync + Unpin>,
@@ -57,12 +64,67 @@ pub async fn create_bitrot_reader(
         Ok(Some(reader))
     } else if let Some(disk) = disk {
         // Read from disk
-        match disk.read_file_stream(bucket, path, offset, length - offset).await {
-            Ok(rd) => {
-                let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify);
-                Ok(Some(reader))
+        if use_zero_copy {
+            // Try zero-copy read first (uses mmap on Unix)
+            let start = Instant::now();
+            match disk.read_file_zero_copy(bucket, path, offset, length).await {
+                Ok(bytes) => {
+                    let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
+
+                    // Record zero-copy metrics
+                    rustfs_io_metrics::record_zero_copy_read(bytes.len(), duration_ms);
+
+                    // Log successful zero-copy read
+                    debug!(
+                        size = bytes.len(),
+                        path = %path,
+                        "zero_copy_read_success"
+                    );
+
+                    // Wrap Bytes in Cursor for AsyncRead
+                    // The Bytes is reference-counted, so this is zero-copy
+                    let rd = Cursor::new(bytes);
+                    let reader = BitrotReader::new(
+                        Box::new(rd) as Box<dyn AsyncRead + Send + Sync + Unpin>,
+                        shard_size,
+                        checksum_algo,
+                        skip_verify,
+                    );
+                    Ok(Some(reader))
+                }
+                Err(e) => {
+                    // Record zero-copy fallback
+                    rustfs_io_metrics::record_zero_copy_fallback(&format!("{:?}", e));
+
+                    // Log zero-copy fallback
+                    debug!(
+                        reason = %format!("{:?}", e),
+                        path = %path,
+                        "zero_copy_fallback"
+                    );
+
+                    // Fall back to regular stream read on error
+                    match disk.read_file_stream(bucket, path, offset, length).await {
+                        Ok(rd) => {
+                            let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify);
+                            Ok(Some(reader))
+                        }
+                        Err(_e2) => {
+                            // Return the original error from zero-copy attempt
+                            Err(e)
+                        }
+                    }
+                }
+            }
+        } else {
+            // Use regular stream read
+            match disk.read_file_stream(bucket, path, offset, length).await {
+                Ok(rd) => {
+                    let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify);
+                    Ok(Some(reader))
+                }
+                Err(e) => Err(e),
             }
-            Err(e) => Err(e),
         }
     } else {
         // Neither inline data nor disk available
@@ -121,8 +183,44 @@ mod tests {
         let shard_size = 16;
         let checksum_algo = HashAlgorithm::HighwayHash256S;
 
-        let result =
-            create_bitrot_reader(Some(test_data), None, "test-bucket", "test-path", 0, 0, shard_size, checksum_algo, false).await;
+        let result = create_bitrot_reader(
+            Some(test_data),
+            None,
+            "test-bucket",
+            "test-path",
+            0,
+            0,
+            shard_size,
+            checksum_algo,
+            false,
+            false,
+        )
+        .await;
+
+        assert!(result.is_ok());
+        assert!(result.unwrap().is_some());
+    }
+
+    #[tokio::test]
+    async fn test_create_bitrot_reader_with_zero_copy_enabled() {
+        let test_data = b"hello world test data";
+        let shard_size = 16;
+        let checksum_algo = HashAlgorithm::HighwayHash256S;
+
+        // Test with zero-copy enabled (should work the same for inline data)
+        let result = create_bitrot_reader(
+            Some(test_data),
+            None,
+            "test-bucket",
+            "test-path",
+            0,
+            0,
+            shard_size,
+            checksum_algo,
+            false,
+            true,
+        )
+        .await;
 
         assert!(result.is_ok());
         assert!(result.unwrap().is_some());
@@ -134,7 +232,7 @@ mod tests {
         let checksum_algo = HashAlgorithm::HighwayHash256S;
 
         let result =
-            create_bitrot_reader(None, None, "test-bucket", "test-path", 0, 1024, shard_size, checksum_algo, false).await;
+            create_bitrot_reader(None, None, "test-bucket", "test-path", 0, 1024, shard_size, checksum_algo, false, false).await;
 
         assert!(result.is_ok());
         assert!(result.unwrap().is_none());

crates/ecstore/src/disk/disk_store.rs

@@ -730,6 +730,14 @@ impl DiskAPI for LocalDiskWrapper {
         .await
     }
 
+    async fn read_file_zero_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<bytes::Bytes> {
+        self.track_disk_health(
+            || async { self.disk.read_file_zero_copy(volume, path, offset, length).await },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
     async fn append_file(&self, volume: &str, path: &str) -> Result<crate::disk::FileWriter> {
         self.track_disk_health(|| async { self.disk.append_file(volume, path).await }, Duration::ZERO)
             .await

crates/ecstore/src/disk/local.rs

@@ -1809,7 +1809,8 @@ impl DiskAPI for LocalDisk {
         let mut f = self.open_file(file_path, O_RDONLY, volume_dir).await?;
 
         let meta = f.metadata().await?;
-        if meta.len() < (offset + length) as u64 {
+        let end_offset = offset.checked_add(length).ok_or(DiskError::FileCorrupt)?;
+        if meta.len() < end_offset as u64 {
             error!(
                 "read_file_stream: file size is less than offset + length {} + {} = {}",
                 offset,
@@ -1825,6 +1826,99 @@ impl DiskAPI for LocalDisk {
 
         Ok(Box::new(f))
     }
+
+    /// Zero-copy file read using memory mapping (Unix) or efficient read (non-Unix).
+    /// Returns Bytes that can be shared without copying.
+    #[allow(unsafe_code)]
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn read_file_zero_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Bytes> {
+        use std::time::Instant;
+
+        let start = Instant::now();
+        let volume_dir = self.get_bucket_path(volume)?;
+        if !skip_access_checks(volume) {
+            access(&volume_dir)
+                .await
+                .map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
+        }
+
+        let file_path = self.get_object_path(volume, path)?;
+        check_path_length(file_path.to_string_lossy().as_ref())?;
+
+        // Verify file exists and get metadata
+        let file_path_clone = file_path.clone();
+        let meta = tokio::task::spawn_blocking(move || std::fs::metadata(&file_path_clone).map_err(DiskError::from))
+            .await
+            .map_err(DiskError::from)??;
+
+        let end_offset = offset.checked_add(length).ok_or(DiskError::FileCorrupt)?;
+        if meta.len() < end_offset as u64 {
+            error!(
+                "read_file_zero_copy: file size is less than offset + length {} + {} = {}",
+                offset,
+                length,
+                meta.len()
+            );
+            return Err(DiskError::FileCorrupt);
+        }
+
+        // Unix: use mmap to read the data (copies into Bytes for safe ownership)
+        // Non-Unix: fall back to efficient read
+        #[cfg(unix)]
+        {
+            use memmap2::MmapOptions;
+            let file_path_clone = file_path.clone();
+            let offset_u64 = offset as u64;
+
+            let bytes = tokio::task::spawn_blocking(move || {
+                let file = std::fs::File::open(&file_path_clone).map_err(DiskError::from)?;
+
+                // Create memory map for the specified region
+                // SAFETY: The file is opened as read-only, and we're mapping a region
+                // that we've already verified exists and is within file bounds.
+                let mmap = unsafe { MmapOptions::new().offset(offset_u64).len(length).map(&file) }.map_err(DiskError::other)?;
+
+                // Copy the mapped region into a Bytes buffer. This avoids undefined
+                // behavior from treating OS-managed mmap memory as allocator-managed
+                // Vec storage, at the cost of an extra copy.
+                Ok::<Bytes, DiskError>(Bytes::copy_from_slice(&mmap))
+            })
+            .await
+            .map_err(DiskError::from)??;
+
+            // Log successful mmap read metrics
+            let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
+
+            // Record mmap read metrics
+            rustfs_io_metrics::record_zero_copy_read(length, duration_ms);
+
+            debug!(size = length, duration_ms = duration_ms, "mmap_read_success");
+
+            return Ok(bytes);
+        }
+
+        // Non-Unix fallback: efficient read into Bytes
+        #[cfg(not(unix))]
+        {
+            // Record zero-copy fallback
+            rustfs_io_metrics::record_zero_copy_fallback("non_unix_platform");
+
+            debug!(reason = "non_unix_platform", "zero_copy_fallback");
+
+            let mut f = self.open_file(file_path, O_RDONLY, volume_dir).await?;
+
+            if offset > 0 {
+                f.seek(SeekFrom::Start(offset as u64)).await?;
+            }
+
+            let mut buffer = Vec::with_capacity(length);
+            buffer.resize(length, 0);
+            f.read_exact(&mut buffer).await?;
+
+            Ok(Bytes::from(buffer))
+        }
+    }
+
     #[tracing::instrument(level = "debug", skip(self))]
     async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
         if !origvolume.is_empty() {

crates/ecstore/src/disk/mod.rs

@@ -287,6 +287,14 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
+    async fn read_file_zero_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Bytes> {
+        match self {
+            Disk::Local(local_disk) => local_disk.read_file_zero_copy(volume, path, offset, length).await,
+            Disk::Remote(remote_disk) => remote_disk.read_file_zero_copy(volume, path, offset, length).await,
+        }
+    }
+
     #[tracing::instrument(skip(self))]
     async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
         match self {
@@ -490,6 +498,13 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>>;
     async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader>;
     async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader>;
+
+    /// Zero-copy file read using memory mapping (Unix) or efficient read (non-Unix).
+    /// Returns Bytes that can be shared without copying.
+    /// On Unix, this uses mmap for true zero-copy access.
+    /// On other platforms, falls back to efficient read operations.
+    async fn read_file_zero_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Bytes>;
+
     async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter>;
     async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> Result<FileWriter>;
     // ReadFileStream

crates/ecstore/src/rpc/remote_disk.rs

@@ -1053,6 +1053,24 @@ impl DiskAPI for RemoteDisk {
         Ok(Box::new(HttpReader::new(url, Method::GET, headers, None).await?))
     }
 
+    /// Zero-copy read for remote disks falls back to efficient network read.
+    /// Note: True zero-copy is not possible over network, but we avoid extra copies
+    /// by reading directly into Bytes.
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn read_file_zero_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Bytes> {
+        // For remote disks, use the regular reader and read into Bytes
+        let reader = self.read_file_stream(volume, path, offset, length).await?;
+
+        use tokio::io::AsyncReadExt;
+        let mut reader = reader;
+
+        // Read all data into Bytes (single allocation)
+        let mut buffer = Vec::with_capacity(length);
+        reader.read_to_end(&mut buffer).await?;
+
+        Ok(Bytes::from(buffer))
+    }
+
     #[tracing::instrument(level = "debug", skip(self))]
     async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
         info!("append_file {}/{}", volume, path);

crates/ecstore/src/set_disk/heal.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::*;
+use rustfs_config::{DEFAULT_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE};
 
 impl SetDisks {
     #[tracing::instrument(skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
@@ -357,6 +358,12 @@ impl SetDisks {
                                 } else {
                                     checksum_info.algorithm
                                 };
+
+                                // Read zero-copy configuration from environment variable
+                                // Default: enabled (true) for performance
+                                let use_zero_copy =
+                                    rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE);
+
                                 let mut readers = Vec::with_capacity(latest_disks.len());
                                 let mut writers = Vec::with_capacity(out_dated_disks.len());
                                 // let mut errors = Vec::with_capacity(out_dated_disks.len());
@@ -385,6 +392,7 @@ impl SetDisks {
                                             erasure.shard_size(),
                                             checksum_algo.clone(),
                                             false,
+                                            use_zero_copy,
                                         )
                                         .await
                                         {

crates/ecstore/src/set_disk/read.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::*;
+use rustfs_config::{DEFAULT_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE};
 
 impl SetDisks {
     pub(super) async fn read_parts(
@@ -667,6 +668,10 @@ impl SetDisks {
                     checksum_info.algorithm
                 };
 
+            // Read zero-copy configuration from environment variable
+            // Default: enabled (true) for performance
+            let use_zero_copy = rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE);
+
             let mut readers = Vec::with_capacity(disks.len());
             let mut errors = Vec::with_capacity(disks.len());
             for (idx, disk_op) in disks.iter().enumerate() {
@@ -680,6 +685,7 @@ impl SetDisks {
                     erasure.shard_size(),
                     checksum_algo.clone(),
                     skip_verify_bitrot,
+                    use_zero_copy,
                 )
                 .await
                 {