mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-28 16:07:05 +00:00
This commit is contained in:
@@ -12,38 +12,37 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
//! Zero-copy I/O configuration constants.
|
//! Mmap-based read I/O configuration constants.
|
||||||
//!
|
//!
|
||||||
//! This module defines environment variables and default values for zero-copy
|
//! This module defines environment variables and default values for mmap-based
|
||||||
//! read operations, which use memory mapping (mmap) to avoid data copying.
|
//! read operations. Note: despite the "zero_copy" naming in env vars (kept for
|
||||||
|
//! backward compatibility), the actual implementation performs mmap-then-copy,
|
||||||
|
//! not true zero-copy. See https://github.com/rustfs/backlog/issues/733
|
||||||
|
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
// Zero-Copy Configuration
|
// Mmap Read Configuration
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
|
|
||||||
/// Environment variable for zero-copy read enable.
|
/// Environment variable for mmap-based read enable.
|
||||||
///
|
///
|
||||||
/// When enabled, uses mmap (Unix) or optimized reads for zero-copy data access.
|
/// When enabled, uses mmap (Unix) or optimized reads for file access.
|
||||||
/// This reduces memory copies from 3-4 to 1, lowering CPU usage by 20-30%
|
/// This reduces memory copies from 3-4 to 1, lowering CPU usage and
|
||||||
/// and improving P95 latency by 15-25%.
|
/// improving latency for large object reads.
|
||||||
///
|
///
|
||||||
/// - Purpose: Enable or disable zero-copy read operations
|
/// - Purpose: Enable or disable mmap-based read operations
|
||||||
/// - Acceptable values: `"true"` / `"false"` (case-insensitive) or a boolean typed config
|
/// - Acceptable values: `"true"` / `"false"` (case-insensitive) or a boolean typed config
|
||||||
/// - Semantics: When enabled, uses mmap on Unix systems for memory-mapped file reads;
|
/// - Semantics: When enabled, uses mmap on Unix systems for memory-mapped file reads;
|
||||||
/// falls back to regular I/O on non-Unix platforms or when mmap fails
|
/// falls back to regular I/O on non-Unix platforms or when mmap fails
|
||||||
/// - Example: `export RUSTFS_OBJECT_ZERO_COPY_ENABLE=true`
|
/// - Example: `export RUSTFS_OBJECT_ZERO_COPY_ENABLE=true`
|
||||||
/// - Note: Zero-copy is safe for all workloads and provides significant performance
|
/// - Note: The env var name is kept as ZERO_COPY for backward compatibility.
|
||||||
/// benefits with minimal risk. Disable only if mmap-related issues are encountered.
|
/// The actual behavior is mmap-then-copy, not true zero-copy.
|
||||||
pub const ENV_OBJECT_ZERO_COPY_ENABLE: &str = "RUSTFS_OBJECT_ZERO_COPY_ENABLE";
|
pub const ENV_OBJECT_ZERO_COPY_ENABLE: &str = "RUSTFS_OBJECT_ZERO_COPY_ENABLE";
|
||||||
|
|
||||||
/// Default: zero-copy reads are enabled.
|
/// Default: mmap-based reads are enabled.
|
||||||
///
|
///
|
||||||
/// Zero-copy uses memory mapping (mmap) on Unix systems to avoid data copying
|
/// Uses memory mapping (mmap) on Unix systems, then copies data into owned
|
||||||
/// between kernel and user space. This provides:
|
/// Bytes. This is faster than multiple read+copy passes but is NOT true
|
||||||
/// - Reduced memory copies: from 3-4 copies to 1 copy
|
/// zero-copy (the data is still copied once from the mmap region).
|
||||||
/// - Lower CPU usage: 20-30% reduction expected
|
|
||||||
/// - Improved latency P95: 15-25% reduction expected
|
|
||||||
/// - Increased throughput: 10-20% improvement expected
|
|
||||||
///
|
///
|
||||||
/// On non-Unix platforms or when mmap fails, the system automatically falls back
|
/// On non-Unix platforms or when mmap fails, the system automatically falls back
|
||||||
/// to regular I/O without errors.
|
/// to regular I/O without errors.
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ struct BitrotReaderSource {
|
|||||||
path: String,
|
path: String,
|
||||||
offset: usize,
|
offset: usize,
|
||||||
length: usize,
|
length: usize,
|
||||||
use_zero_copy: bool,
|
use_mmap_read: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BitrotReaderSource {
|
impl BitrotReaderSource {
|
||||||
@@ -47,7 +47,7 @@ impl BitrotReaderSource {
|
|||||||
rd.set_position(offset);
|
rd.set_position(offset);
|
||||||
Ok(Some(Box::new(rd)))
|
Ok(Some(Box::new(rd)))
|
||||||
} else if let Some(disk) = self.disk {
|
} else if let Some(disk) = self.disk {
|
||||||
open_disk_reader(&disk, &self.bucket, &self.path, self.offset, self.length, self.use_zero_copy)
|
open_disk_reader(&disk, &self.bucket, &self.path, self.offset, self.length, self.use_mmap_read)
|
||||||
.await
|
.await
|
||||||
.map(Some)
|
.map(Some)
|
||||||
} else {
|
} else {
|
||||||
@@ -136,9 +136,9 @@ async fn open_disk_reader(
|
|||||||
path: &str,
|
path: &str,
|
||||||
offset: usize,
|
offset: usize,
|
||||||
length: usize,
|
length: usize,
|
||||||
use_zero_copy: bool,
|
use_mmap_read: bool,
|
||||||
) -> disk::error::Result<FileReader> {
|
) -> disk::error::Result<FileReader> {
|
||||||
if use_zero_copy && disk.is_local() {
|
if use_mmap_read && disk.is_local() {
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
match disk.read_file_zero_copy(bucket, path, offset, length).await {
|
match disk.read_file_zero_copy(bucket, path, offset, length).await {
|
||||||
Ok(bytes) => {
|
Ok(bytes) => {
|
||||||
@@ -192,7 +192,7 @@ fn bitrot_encoded_range(offset: usize, length: usize, shard_size: usize, checksu
|
|||||||
/// * `shard_size` - Shard size for erasure coding
|
/// * `shard_size` - Shard size for erasure coding
|
||||||
/// * `checksum_algo` - Hash algorithm for bitrot verification
|
/// * `checksum_algo` - Hash algorithm for bitrot verification
|
||||||
/// * `skip_verify` - If true, skip checksum verification
|
/// * `skip_verify` - If true, skip checksum verification
|
||||||
/// * `use_zero_copy` - If true, use zero-copy read (mmap on Unix)
|
/// * `use_mmap_read` - If true, use zero-copy read (mmap on Unix)
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub async fn create_bitrot_reader(
|
pub async fn create_bitrot_reader(
|
||||||
inline_data: Option<&[u8]>,
|
inline_data: Option<&[u8]>,
|
||||||
@@ -204,7 +204,7 @@ pub async fn create_bitrot_reader(
|
|||||||
shard_size: usize,
|
shard_size: usize,
|
||||||
checksum_algo: HashAlgorithm,
|
checksum_algo: HashAlgorithm,
|
||||||
skip_verify: bool,
|
skip_verify: bool,
|
||||||
use_zero_copy: bool,
|
use_mmap_read: bool,
|
||||||
) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
|
) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
|
||||||
let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
|
let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
|
||||||
let source = BitrotReaderSource {
|
let source = BitrotReaderSource {
|
||||||
@@ -214,7 +214,7 @@ pub async fn create_bitrot_reader(
|
|||||||
path: path.to_string(),
|
path: path.to_string(),
|
||||||
offset,
|
offset,
|
||||||
length,
|
length,
|
||||||
use_zero_copy,
|
use_mmap_read,
|
||||||
};
|
};
|
||||||
|
|
||||||
source
|
source
|
||||||
@@ -234,7 +234,7 @@ pub fn create_deferred_bitrot_reader(
|
|||||||
shard_size: usize,
|
shard_size: usize,
|
||||||
checksum_algo: HashAlgorithm,
|
checksum_algo: HashAlgorithm,
|
||||||
skip_verify: bool,
|
skip_verify: bool,
|
||||||
use_zero_copy: bool,
|
use_mmap_read: bool,
|
||||||
) -> BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>> {
|
) -> BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>> {
|
||||||
let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
|
let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
|
||||||
let source = BitrotReaderSource {
|
let source = BitrotReaderSource {
|
||||||
@@ -244,7 +244,7 @@ pub fn create_deferred_bitrot_reader(
|
|||||||
path: path.to_string(),
|
path: path.to_string(),
|
||||||
offset,
|
offset,
|
||||||
length,
|
length,
|
||||||
use_zero_copy,
|
use_mmap_read,
|
||||||
};
|
};
|
||||||
|
|
||||||
BitrotReader::new(Box::new(DeferredObjectReader::new(source)), shard_size, checksum_algo, skip_verify)
|
BitrotReader::new(Box::new(DeferredObjectReader::new(source)), shard_size, checksum_algo, skip_verify)
|
||||||
|
|||||||
@@ -362,7 +362,7 @@ impl SetDisks {
|
|||||||
let till_offset = erasure.shard_file_offset(0, part.size, part.size);
|
let till_offset = erasure.shard_file_offset(0, part.size, part.size);
|
||||||
// Read zero-copy configuration from environment variable
|
// Read zero-copy configuration from environment variable
|
||||||
// Default: enabled (true) for performance
|
// Default: enabled (true) for performance
|
||||||
let use_zero_copy =
|
let use_mmap_read =
|
||||||
rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE);
|
rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE);
|
||||||
|
|
||||||
let mut readers = Vec::with_capacity(latest_disks.len());
|
let mut readers = Vec::with_capacity(latest_disks.len());
|
||||||
@@ -402,7 +402,7 @@ impl SetDisks {
|
|||||||
erasure.shard_size(),
|
erasure.shard_size(),
|
||||||
checksum_algo.clone(),
|
checksum_algo.clone(),
|
||||||
false,
|
false,
|
||||||
use_zero_copy,
|
use_mmap_read,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -819,7 +819,7 @@ async fn create_bitrot_readers_until_quorum(
|
|||||||
shard_size: usize,
|
shard_size: usize,
|
||||||
checksum_algo: HashAlgorithm,
|
checksum_algo: HashAlgorithm,
|
||||||
skip_verify_bitrot: bool,
|
skip_verify_bitrot: bool,
|
||||||
use_zero_copy: bool,
|
use_mmap_read: bool,
|
||||||
data_shards: usize,
|
data_shards: usize,
|
||||||
parity_shards: usize,
|
parity_shards: usize,
|
||||||
mode: BitrotReaderSetupMode,
|
mode: BitrotReaderSetupMode,
|
||||||
@@ -850,7 +850,7 @@ async fn create_bitrot_readers_until_quorum(
|
|||||||
shard_size,
|
shard_size,
|
||||||
checksum_algo,
|
checksum_algo,
|
||||||
skip_verify_bitrot,
|
skip_verify_bitrot,
|
||||||
use_zero_copy,
|
use_mmap_read,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
(idx, result)
|
(idx, result)
|
||||||
@@ -902,7 +902,7 @@ async fn create_bitrot_readers_until_quorum(
|
|||||||
shard_size,
|
shard_size,
|
||||||
checksum_algo.clone(),
|
checksum_algo.clone(),
|
||||||
skip_verify_bitrot,
|
skip_verify_bitrot,
|
||||||
use_zero_copy,
|
use_mmap_read,
|
||||||
));
|
));
|
||||||
setup.errors[idx] = None;
|
setup.errors[idx] = None;
|
||||||
}
|
}
|
||||||
@@ -1972,7 +1972,7 @@ impl SetDisks {
|
|||||||
|
|
||||||
// Read zero-copy configuration from environment variable
|
// Read zero-copy configuration from environment variable
|
||||||
// Default: enabled (true) for performance
|
// Default: enabled (true) for performance
|
||||||
let use_zero_copy = rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE);
|
let use_mmap_read = rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE);
|
||||||
|
|
||||||
let reader_setup_stage_start = Instant::now();
|
let reader_setup_stage_start = Instant::now();
|
||||||
let read_costs = disks
|
let read_costs = disks
|
||||||
@@ -1990,7 +1990,7 @@ impl SetDisks {
|
|||||||
erasure.shard_size(),
|
erasure.shard_size(),
|
||||||
checksum_algo,
|
checksum_algo,
|
||||||
skip_verify_bitrot,
|
skip_verify_bitrot,
|
||||||
use_zero_copy,
|
use_mmap_read,
|
||||||
erasure.data_shards,
|
erasure.data_shards,
|
||||||
erasure.parity_shards,
|
erasure.parity_shards,
|
||||||
BitrotReaderSetupMode::ReadQuorum,
|
BitrotReaderSetupMode::ReadQuorum,
|
||||||
@@ -2277,7 +2277,7 @@ impl SetDisks {
|
|||||||
} else {
|
} else {
|
||||||
checksum_info.algorithm
|
checksum_info.algorithm
|
||||||
};
|
};
|
||||||
let use_zero_copy = rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE);
|
let use_mmap_read = rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE);
|
||||||
let till_offset = erasure.shard_file_offset(0, part_length, part_size);
|
let till_offset = erasure.shard_file_offset(0, part_length, part_size);
|
||||||
|
|
||||||
let reader_setup_stage_start = Instant::now();
|
let reader_setup_stage_start = Instant::now();
|
||||||
@@ -2296,7 +2296,7 @@ impl SetDisks {
|
|||||||
erasure.shard_size(),
|
erasure.shard_size(),
|
||||||
checksum_algo,
|
checksum_algo,
|
||||||
skip_verify_bitrot,
|
skip_verify_bitrot,
|
||||||
use_zero_copy,
|
use_mmap_read,
|
||||||
erasure.data_shards,
|
erasure.data_shards,
|
||||||
erasure.parity_shards,
|
erasure.parity_shards,
|
||||||
BitrotReaderSetupMode::VerifyReconstruction,
|
BitrotReaderSetupMode::VerifyReconstruction,
|
||||||
|
|||||||
@@ -20,8 +20,8 @@ license.workspace = true
|
|||||||
repository.workspace = true
|
repository.workspace = true
|
||||||
rust-version.workspace = true
|
rust-version.workspace = true
|
||||||
homepage.workspace = true
|
homepage.workspace = true
|
||||||
description = "Zero-copy core reader and writer implementations for RustFS"
|
description = "Buffered I/O reader and writer implementations for RustFS (mmap-then-copy, aligned pread)"
|
||||||
keywords = ["zero-copy", "reader", "writer", "rustfs"]
|
keywords = ["io", "reader", "writer", "rustfs", "mmap"]
|
||||||
categories = ["development-tools", "filesystem"]
|
categories = ["development-tools", "filesystem"]
|
||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
@@ -34,7 +34,9 @@ tokio = { workspace = true, features = ["io-util", "fs", "rt", "sync"] }
|
|||||||
memmap2 = { workspace = true }
|
memmap2 = { workspace = true }
|
||||||
rustfs-io-metrics = { workspace = true }
|
rustfs-io-metrics = { workspace = true }
|
||||||
|
|
||||||
# io-uring is Linux-only. Scope the feature to Linux targets.
|
# Enable tokio's io_uring-based runtime backend on Linux.
|
||||||
|
# Note: this is a tokio runtime feature, not application-level io_uring usage.
|
||||||
|
# See https://github.com/rustfs/backlog/issues/733
|
||||||
[target.'cfg(target_os = "linux")'.dependencies]
|
[target.'cfg(target_os = "linux")'.dependencies]
|
||||||
tokio = { workspace = true, features = ["io-uring"] }
|
tokio = { workspace = true, features = ["io-uring"] }
|
||||||
|
|
||||||
|
|||||||
@@ -12,20 +12,20 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
//! Zero-copy core reader and writer implementations for RustFS.
|
//! Buffered I/O reader and writer implementations for RustFS.
|
||||||
//!
|
//!
|
||||||
//! This crate provides zero-copy readers and writers that minimize memory
|
//! This crate provides buffered readers and writers for I/O operations.
|
||||||
//! allocations and data copying during I/O operations. It depends on
|
//! Note: despite "ZeroCopy" naming in type names (kept for backward compatibility),
|
||||||
//! `rustfs-io-metrics` for metrics reporting and is designed to avoid
|
//! the actual implementations perform mmap-then-copy or aligned pread, not true
|
||||||
//! introducing cyclic dependencies in the RustFS crate graph.
|
//! zero-copy. See https://github.com/rustfs/backlog/issues/733
|
||||||
//!
|
//!
|
||||||
//! # Features
|
//! # Features
|
||||||
//!
|
//!
|
||||||
//! - Memory-mapped file reading (mmap) on Unix platforms
|
//! - Memory-mapped file reading (mmap-then-copy) on Unix platforms
|
||||||
//! - Bytes-based zero-copy wrapping
|
//! - Bytes-based buffered wrapping
|
||||||
//! - AsyncRead trait implementations
|
//! - AsyncRead trait implementations
|
||||||
//! - Tiered BytesPool for buffer management
|
//! - Tiered BytesPool for buffer management
|
||||||
//! - Optional Direct I/O support (Linux only)
|
//! - Aligned pread-based reader (NOT true Direct I/O / O_DIRECT)
|
||||||
//!
|
//!
|
||||||
//! # Example
|
//! # Example
|
||||||
//!
|
//!
|
||||||
|
|||||||
Reference in New Issue
Block a user