mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
docs(io): clarify zero copy metric transition (#4029)
This commit is contained in:
@@ -5211,10 +5211,11 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "rate(rustfs_s3_put_object_total{job=~\"$job\"}[5m])",
|
||||
"legendFormat": "PutObject - {{zero_copy_enabled}}",
|
||||
"legendFormat": "PutObject - legacy mmap flag {{zero_copy_enabled}}",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"description": "The zero_copy_enabled label is a legacy mmap-read compatibility flag retained until the metric rename transition.",
|
||||
"title": "S3 Operations Rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
@@ -5480,11 +5481,12 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "sum(increase(rustfs_zero_copy_memory_saved_bytes_total{job=~\"$job\"}[$__rate_interval])) or sum(increase(rustfs_zero_copy_bytes_saved_total{job=~\"$job\"}[$__rate_interval])) or vector(0)",
|
||||
"legendFormat": "Memory Saved",
|
||||
"legendFormat": "Legacy mmap metric",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Zero-Copy Memory Savings",
|
||||
"description": "Legacy zero_copy metric names are retained for dashboard compatibility until the Phase 3 metric rename can dual-write replacement metrics.",
|
||||
"title": "Mmap Read Memory Savings",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
|
||||
+2
-2
@@ -53,8 +53,8 @@ members = [
|
||||
"crates/trusted-proxies", # Trusted proxies management
|
||||
"crates/tls-runtime", # Project-wide TLS runtime foundation
|
||||
"crates/utils", # Utility functions and helpers
|
||||
"crates/io-metrics", # Zero-copy metrics collection for performance analysis
|
||||
"crates/io-core", # Zero-copy core reader and writer implementations
|
||||
"crates/io-metrics", # I/O metrics collection for performance analysis
|
||||
"crates/io-core", # Buffered Bytes, mmap-then-copy, and aligned pread I/O helpers
|
||||
"crates/zip", # ZIP file handling and compression
|
||||
]
|
||||
resolver = "3"
|
||||
|
||||
@@ -36,6 +36,8 @@ rustfs-io-metrics = { workspace = true }
|
||||
|
||||
# Enable tokio's io_uring-based runtime backend on Linux.
|
||||
# Note: this is a tokio runtime feature, not application-level io_uring usage.
|
||||
# See the Tokio 1.52.0 release notes and tokio-rs/tokio#7907 for the upstream
|
||||
# runtime feature context.
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
tokio = { workspace = true, features = ["io-uring"] }
|
||||
|
||||
|
||||
@@ -155,10 +155,10 @@ impl DirectIoReader {
|
||||
})
|
||||
}
|
||||
|
||||
/// Read a chunk of data using Direct I/O.
|
||||
/// Read a chunk of data using aligned pread.
|
||||
///
|
||||
/// This method performs aligned reads and handles the buffering
|
||||
/// required for Direct I/O operations.
|
||||
/// This method performs aligned reads and handles the buffering required
|
||||
/// by this aligned pread implementation. It does not use `O_DIRECT`.
|
||||
fn read_chunk(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||
// If buffer is exhausted, read more data
|
||||
if self.buffer_pos >= self.buffer_len {
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
//! This crate provides buffered readers and writers for I/O operations.
|
||||
//! Note: despite "ZeroCopy" naming in type names (kept for backward compatibility),
|
||||
//! the actual implementations perform mmap-then-copy or aligned pread, not true
|
||||
//! zero-copy file I/O or true Direct I/O.
|
||||
//!
|
||||
//! # Features
|
||||
//!
|
||||
@@ -36,7 +37,7 @@
|
||||
//! let data = Bytes::from("hello world");
|
||||
//! let reader = ZeroCopyObjectReader::from_bytes(data);
|
||||
//!
|
||||
//! // Create from file using mmap (Unix only)
|
||||
//! // Create from file using mmap-then-copy (Unix only)
|
||||
//! #[cfg(unix)]
|
||||
//! let reader = ZeroCopyObjectReader::from_file_mmap(&file, 0, 1024).await?;
|
||||
//!
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Zero-copy object reader implementation.
|
||||
//! Bytes-backed object reader implementation.
|
||||
|
||||
use bytes::Bytes;
|
||||
use std::io;
|
||||
@@ -20,7 +20,7 @@ use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
/// Errors that can occur during zero-copy read operations.
|
||||
/// Errors that can occur during Bytes-backed read operations.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum ZeroCopyReadError {
|
||||
/// I/O error occurred.
|
||||
@@ -49,12 +49,11 @@ impl From<io::Error> for ZeroCopyReadError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Zero-copy object reader.
|
||||
/// Bytes-backed object reader.
|
||||
///
|
||||
/// This reader provides zero-copy access to object data by using:
|
||||
/// - Memory-mapped files for on-disk data
|
||||
/// - Bytes wrapping for in-memory data
|
||||
/// - Reference counting to avoid copies
|
||||
/// This reader keeps the historical `ZeroCopyObjectReader` name for public API
|
||||
/// compatibility. `from_bytes` wraps existing `Bytes` without copying, but file
|
||||
/// constructors copy file data into owned `Bytes` after mmap or normal reads.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
@@ -94,10 +93,10 @@ impl ZeroCopyObjectReader {
|
||||
Self { data, pos: 0 }
|
||||
}
|
||||
|
||||
/// Create a zero-copy reader from a file using mmap.
|
||||
/// Create a Bytes-backed reader from a file using mmap-then-copy.
|
||||
///
|
||||
/// This uses memory mapping to avoid loading the entire file into memory.
|
||||
/// Only the accessed pages are loaded on demand.
|
||||
/// This maps the requested file range and copies it into owned `Bytes`
|
||||
/// before returning. It does not expose the mmap as a zero-copy buffer.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
@@ -107,7 +106,7 @@ impl ZeroCopyObjectReader {
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A reader that provides zero-copy access to the file data.
|
||||
/// A reader backed by copied file data.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
@@ -148,10 +147,10 @@ impl ZeroCopyObjectReader {
|
||||
.map_err(|e| ZeroCopyReadError::Io(e.to_string()))?
|
||||
}
|
||||
|
||||
/// Create a zero-copy reader from a file using mmap.
|
||||
/// Create a Bytes-backed reader from a file using normal reads.
|
||||
///
|
||||
/// This uses memory mapping to avoid loading the entire file into memory.
|
||||
/// Only the accessed pages are loaded on demand.
|
||||
/// This path reads the requested range into an owned buffer and wraps it in
|
||||
/// `Bytes`. It does not perform mmap or zero-copy file I/O.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
@@ -161,7 +160,7 @@ impl ZeroCopyObjectReader {
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A reader that provides zero-copy access to the file data.
|
||||
/// A reader backed by copied file data.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
@@ -191,7 +190,7 @@ impl ZeroCopyObjectReader {
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a zero-copy reader from a file (non-Unix fallback).
|
||||
/// Create a Bytes-backed reader from a file (non-Unix fallback).
|
||||
///
|
||||
/// On platforms that don't support mmap, this falls back to regular file I/O.
|
||||
#[cfg(not(unix))]
|
||||
|
||||
@@ -12,21 +12,22 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Zero-copy object writer for optimized write operations.
|
||||
//! BytesMut-backed object writer for optimized write operations.
|
||||
//!
|
||||
//! This module provides a zero-copy writer that minimizes memory allocations
|
||||
//! and data copying during write operations.
|
||||
//! This module keeps the historical `ZeroCopyObjectWriter` name for public API
|
||||
//! compatibility. It uses `BytesMut` for efficient buffering; writes into that
|
||||
//! buffer may still copy input bytes.
|
||||
|
||||
use bytes::{BufMut, Bytes, BytesMut};
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::AsyncWrite;
|
||||
|
||||
/// Zero-copy object writer for optimized write operations.
|
||||
/// BytesMut-backed object writer for optimized write operations.
|
||||
///
|
||||
/// This writer minimizes memory allocations by:
|
||||
/// - Using BytesMut for efficient buffer growth
|
||||
/// - Supporting zero-copy data transfer via Bytes
|
||||
/// - Accepting `Bytes` inputs for efficient buffer handling
|
||||
/// - Optional integration with BytesPool for buffer reuse
|
||||
///
|
||||
/// # Example
|
||||
@@ -89,11 +90,10 @@ impl ZeroCopyObjectWriter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Write data with zero-copy if possible.
|
||||
/// Write data into the internal buffer.
|
||||
///
|
||||
/// This method attempts to write data without copying:
|
||||
/// - If `data` is a Bytes slice, it may be appended without copying
|
||||
/// - If `data` shares the same underlying buffer, no copy occurs
|
||||
/// This method accepts `Bytes` for API compatibility, then appends the
|
||||
/// bytes into the internal `BytesMut` buffer.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
@@ -116,8 +116,6 @@ impl ZeroCopyObjectWriter {
|
||||
}
|
||||
|
||||
let len = data.len();
|
||||
// Zero-copy: put Bytes into BytesMut
|
||||
// If data shares the same underlying buffer, no copy occurs
|
||||
self.buffer.put(data);
|
||||
|
||||
self.bytes_written += len;
|
||||
|
||||
Reference in New Issue
Block a user