mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-10 15:16:56 +00:00
docs(io): clarify zero copy metric transition (#4029)
This commit is contained in:
@@ -36,6 +36,8 @@ rustfs-io-metrics = { workspace = true }
|
||||
|
||||
# Enable tokio's io_uring-based runtime backend on Linux.
|
||||
# Note: this is a tokio runtime feature, not application-level io_uring usage.
|
||||
# See the Tokio 1.52.0 release notes and tokio-rs/tokio#7907 for the upstream
|
||||
# runtime feature context.
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
tokio = { workspace = true, features = ["io-uring"] }
|
||||
|
||||
|
||||
@@ -155,10 +155,10 @@ impl DirectIoReader {
|
||||
})
|
||||
}
|
||||
|
||||
/// Read a chunk of data using Direct I/O.
|
||||
/// Read a chunk of data using aligned pread.
|
||||
///
|
||||
/// This method performs aligned reads and handles the buffering
|
||||
/// required for Direct I/O operations.
|
||||
/// This method performs aligned reads and handles the buffering required
|
||||
/// by this aligned pread implementation. It does not use `O_DIRECT`.
|
||||
fn read_chunk(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||
// If buffer is exhausted, read more data
|
||||
if self.buffer_pos >= self.buffer_len {
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
//! This crate provides buffered readers and writers for I/O operations.
|
||||
//! Note: despite "ZeroCopy" naming in type names (kept for backward compatibility),
|
||||
//! the actual implementations perform mmap-then-copy or aligned pread, not true
|
||||
//! zero-copy file I/O or true Direct I/O.
|
||||
//!
|
||||
//! # Features
|
||||
//!
|
||||
@@ -36,7 +37,7 @@
|
||||
//! let data = Bytes::from("hello world");
|
||||
//! let reader = ZeroCopyObjectReader::from_bytes(data);
|
||||
//!
|
||||
//! // Create from file using mmap (Unix only)
|
||||
//! // Create from file using mmap-then-copy (Unix only)
|
||||
//! #[cfg(unix)]
|
||||
//! let reader = ZeroCopyObjectReader::from_file_mmap(&file, 0, 1024).await?;
|
||||
//!
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Zero-copy object reader implementation.
|
||||
//! Bytes-backed object reader implementation.
|
||||
|
||||
use bytes::Bytes;
|
||||
use std::io;
|
||||
@@ -20,7 +20,7 @@ use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
/// Errors that can occur during zero-copy read operations.
|
||||
/// Errors that can occur during Bytes-backed read operations.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum ZeroCopyReadError {
|
||||
/// I/O error occurred.
|
||||
@@ -49,12 +49,11 @@ impl From<io::Error> for ZeroCopyReadError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Zero-copy object reader.
|
||||
/// Bytes-backed object reader.
|
||||
///
|
||||
/// This reader provides zero-copy access to object data by using:
|
||||
/// - Memory-mapped files for on-disk data
|
||||
/// - Bytes wrapping for in-memory data
|
||||
/// - Reference counting to avoid copies
|
||||
/// This reader keeps the historical `ZeroCopyObjectReader` name for public API
|
||||
/// compatibility. `from_bytes` wraps existing `Bytes` without copying, but file
|
||||
/// constructors copy file data into owned `Bytes` after mmap or normal reads.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
@@ -94,10 +93,10 @@ impl ZeroCopyObjectReader {
|
||||
Self { data, pos: 0 }
|
||||
}
|
||||
|
||||
/// Create a zero-copy reader from a file using mmap.
|
||||
/// Create a Bytes-backed reader from a file using mmap-then-copy.
|
||||
///
|
||||
/// This uses memory mapping to avoid loading the entire file into memory.
|
||||
/// Only the accessed pages are loaded on demand.
|
||||
/// This maps the requested file range and copies it into owned `Bytes`
|
||||
/// before returning. It does not expose the mmap as a zero-copy buffer.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
@@ -107,7 +106,7 @@ impl ZeroCopyObjectReader {
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A reader that provides zero-copy access to the file data.
|
||||
/// A reader backed by copied file data.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
@@ -148,10 +147,10 @@ impl ZeroCopyObjectReader {
|
||||
.map_err(|e| ZeroCopyReadError::Io(e.to_string()))?
|
||||
}
|
||||
|
||||
/// Create a zero-copy reader from a file using mmap.
|
||||
/// Create a Bytes-backed reader from a file using normal reads.
|
||||
///
|
||||
/// This uses memory mapping to avoid loading the entire file into memory.
|
||||
/// Only the accessed pages are loaded on demand.
|
||||
/// This path reads the requested range into an owned buffer and wraps it in
|
||||
/// `Bytes`. It does not perform mmap or zero-copy file I/O.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
@@ -161,7 +160,7 @@ impl ZeroCopyObjectReader {
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A reader that provides zero-copy access to the file data.
|
||||
/// A reader backed by copied file data.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
@@ -191,7 +190,7 @@ impl ZeroCopyObjectReader {
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a zero-copy reader from a file (non-Unix fallback).
|
||||
/// Create a Bytes-backed reader from a file (non-Unix fallback).
|
||||
///
|
||||
/// On platforms that don't support mmap, this falls back to regular file I/O.
|
||||
#[cfg(not(unix))]
|
||||
|
||||
@@ -12,21 +12,22 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Zero-copy object writer for optimized write operations.
|
||||
//! BytesMut-backed object writer for optimized write operations.
|
||||
//!
|
||||
//! This module provides a zero-copy writer that minimizes memory allocations
|
||||
//! and data copying during write operations.
|
||||
//! This module keeps the historical `ZeroCopyObjectWriter` name for public API
|
||||
//! compatibility. It uses `BytesMut` for efficient buffering; writes into that
|
||||
//! buffer may still copy input bytes.
|
||||
|
||||
use bytes::{BufMut, Bytes, BytesMut};
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::AsyncWrite;
|
||||
|
||||
/// Zero-copy object writer for optimized write operations.
|
||||
/// BytesMut-backed object writer for optimized write operations.
|
||||
///
|
||||
/// This writer minimizes memory allocations by:
|
||||
/// - Using BytesMut for efficient buffer growth
|
||||
/// - Supporting zero-copy data transfer via Bytes
|
||||
/// - Accepting `Bytes` inputs for efficient buffer handling
|
||||
/// - Optional integration with BytesPool for buffer reuse
|
||||
///
|
||||
/// # Example
|
||||
@@ -89,11 +90,10 @@ impl ZeroCopyObjectWriter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Write data with zero-copy if possible.
|
||||
/// Write data into the internal buffer.
|
||||
///
|
||||
/// This method attempts to write data without copying:
|
||||
/// - If `data` is a Bytes slice, it may be appended without copying
|
||||
/// - If `data` shares the same underlying buffer, no copy occurs
|
||||
/// This method accepts `Bytes` for API compatibility, then appends the
|
||||
/// bytes into the internal `BytesMut` buffer.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
@@ -116,8 +116,6 @@ impl ZeroCopyObjectWriter {
|
||||
}
|
||||
|
||||
let len = data.len();
|
||||
// Zero-copy: put Bytes into BytesMut
|
||||
// If data shares the same underlying buffer, no copy occurs
|
||||
self.buffer.put(data);
|
||||
|
||||
self.bytes_written += len;
|
||||
|
||||
Reference in New Issue
Block a user