diff --git a/crates/config/src/constants/zero_copy.rs b/crates/config/src/constants/zero_copy.rs index e931bd02c..9a62ed326 100644 --- a/crates/config/src/constants/zero_copy.rs +++ b/crates/config/src/constants/zero_copy.rs @@ -12,38 +12,37 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Zero-copy I/O configuration constants. +//! Mmap-based read I/O configuration constants. //! -//! This module defines environment variables and default values for zero-copy -//! read operations, which use memory mapping (mmap) to avoid data copying. +//! This module defines environment variables and default values for mmap-based +//! read operations. Note: despite the "zero_copy" naming in env vars (kept for +//! backward compatibility), the actual implementation performs mmap-then-copy, +//! not true zero-copy. See https://github.com/rustfs/backlog/issues/733 // ============================================================================= -// Zero-Copy Configuration +// Mmap Read Configuration // ============================================================================= -/// Environment variable for zero-copy read enable. +/// Environment variable for mmap-based read enable. /// -/// When enabled, uses mmap (Unix) or optimized reads for zero-copy data access. -/// This reduces memory copies from 3-4 to 1, lowering CPU usage by 20-30% -/// and improving P95 latency by 15-25%. +/// When enabled, uses mmap (Unix) or optimized reads for file access. +/// This reduces memory copies from 3-4 to 1, lowering CPU usage and +/// improving latency for large object reads. /// -/// - Purpose: Enable or disable zero-copy read operations +/// - Purpose: Enable or disable mmap-based read operations /// - Acceptable values: `"true"` / `"false"` (case-insensitive) or a boolean typed config /// - Semantics: When enabled, uses mmap on Unix systems for memory-mapped file reads; /// falls back to regular I/O on non-Unix platforms or when mmap fails /// - Example: `export RUSTFS_OBJECT_ZERO_COPY_ENABLE=true` -/// - Note: Zero-copy is safe for all workloads and provides significant performance -/// benefits with minimal risk. Disable only if mmap-related issues are encountered. +/// - Note: The env var name is kept as ZERO_COPY for backward compatibility. +/// The actual behavior is mmap-then-copy, not true zero-copy. pub const ENV_OBJECT_ZERO_COPY_ENABLE: &str = "RUSTFS_OBJECT_ZERO_COPY_ENABLE"; -/// Default: zero-copy reads are enabled. +/// Default: mmap-based reads are enabled. /// -/// Zero-copy uses memory mapping (mmap) on Unix systems to avoid data copying -/// between kernel and user space. This provides: -/// - Reduced memory copies: from 3-4 copies to 1 copy -/// - Lower CPU usage: 20-30% reduction expected -/// - Improved latency P95: 15-25% reduction expected -/// - Increased throughput: 10-20% improvement expected +/// Uses memory mapping (mmap) on Unix systems, then copies data into owned +/// Bytes. This is faster than multiple read+copy passes but is NOT true +/// zero-copy (the data is still copied once from the mmap region). /// /// On non-Unix platforms or when mmap fails, the system automatically falls back /// to regular I/O without errors. diff --git a/crates/ecstore/src/io_support/bitrot.rs b/crates/ecstore/src/io_support/bitrot.rs index 8bd1130c2..571862288 100644 --- a/crates/ecstore/src/io_support/bitrot.rs +++ b/crates/ecstore/src/io_support/bitrot.rs @@ -36,7 +36,7 @@ struct BitrotReaderSource { path: String, offset: usize, length: usize, - use_zero_copy: bool, + use_mmap_read: bool, } impl BitrotReaderSource { @@ -47,7 +47,7 @@ impl BitrotReaderSource { rd.set_position(offset); Ok(Some(Box::new(rd))) } else if let Some(disk) = self.disk { - open_disk_reader(&disk, &self.bucket, &self.path, self.offset, self.length, self.use_zero_copy) + open_disk_reader(&disk, &self.bucket, &self.path, self.offset, self.length, self.use_mmap_read) .await .map(Some) } else { @@ -136,9 +136,9 @@ async fn open_disk_reader( path: &str, offset: usize, length: usize, - use_zero_copy: bool, + use_mmap_read: bool, ) -> disk::error::Result { - if use_zero_copy && disk.is_local() { + if use_mmap_read && disk.is_local() { let start = Instant::now(); match disk.read_file_zero_copy(bucket, path, offset, length).await { Ok(bytes) => { @@ -192,7 +192,7 @@ fn bitrot_encoded_range(offset: usize, length: usize, shard_size: usize, checksu /// * `shard_size` - Shard size for erasure coding /// * `checksum_algo` - Hash algorithm for bitrot verification /// * `skip_verify` - If true, skip checksum verification -/// * `use_zero_copy` - If true, use zero-copy read (mmap on Unix) +/// * `use_mmap_read` - If true, use zero-copy read (mmap on Unix) #[allow(clippy::too_many_arguments)] pub async fn create_bitrot_reader( inline_data: Option<&[u8]>, @@ -204,7 +204,7 @@ pub async fn create_bitrot_reader( shard_size: usize, checksum_algo: HashAlgorithm, skip_verify: bool, - use_zero_copy: bool, + use_mmap_read: bool, ) -> disk::error::Result>>> { let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone()); let source = BitrotReaderSource { @@ -214,7 +214,7 @@ pub async fn create_bitrot_reader( path: path.to_string(), offset, length, - use_zero_copy, + use_mmap_read, }; source @@ -234,7 +234,7 @@ pub fn create_deferred_bitrot_reader( shard_size: usize, checksum_algo: HashAlgorithm, skip_verify: bool, - use_zero_copy: bool, + use_mmap_read: bool, ) -> BitrotReader> { let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone()); let source = BitrotReaderSource { @@ -244,7 +244,7 @@ pub fn create_deferred_bitrot_reader( path: path.to_string(), offset, length, - use_zero_copy, + use_mmap_read, }; BitrotReader::new(Box::new(DeferredObjectReader::new(source)), shard_size, checksum_algo, skip_verify) diff --git a/crates/ecstore/src/set_disk/heal.rs b/crates/ecstore/src/set_disk/heal.rs index 751794af0..f193b1c0a 100644 --- a/crates/ecstore/src/set_disk/heal.rs +++ b/crates/ecstore/src/set_disk/heal.rs @@ -362,7 +362,7 @@ impl SetDisks { let till_offset = erasure.shard_file_offset(0, part.size, part.size); // Read zero-copy configuration from environment variable // Default: enabled (true) for performance - let use_zero_copy = + let use_mmap_read = rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE); let mut readers = Vec::with_capacity(latest_disks.len()); @@ -402,7 +402,7 @@ impl SetDisks { erasure.shard_size(), checksum_algo.clone(), false, - use_zero_copy, + use_mmap_read, ) .await { diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 25b9ffedc..6c4158de1 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -819,7 +819,7 @@ async fn create_bitrot_readers_until_quorum( shard_size: usize, checksum_algo: HashAlgorithm, skip_verify_bitrot: bool, - use_zero_copy: bool, + use_mmap_read: bool, data_shards: usize, parity_shards: usize, mode: BitrotReaderSetupMode, @@ -850,7 +850,7 @@ async fn create_bitrot_readers_until_quorum( shard_size, checksum_algo, skip_verify_bitrot, - use_zero_copy, + use_mmap_read, ) .await; (idx, result) @@ -902,7 +902,7 @@ async fn create_bitrot_readers_until_quorum( shard_size, checksum_algo.clone(), skip_verify_bitrot, - use_zero_copy, + use_mmap_read, )); setup.errors[idx] = None; } @@ -1972,7 +1972,7 @@ impl SetDisks { // Read zero-copy configuration from environment variable // Default: enabled (true) for performance - let use_zero_copy = rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE); + let use_mmap_read = rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE); let reader_setup_stage_start = Instant::now(); let read_costs = disks @@ -1990,7 +1990,7 @@ impl SetDisks { erasure.shard_size(), checksum_algo, skip_verify_bitrot, - use_zero_copy, + use_mmap_read, erasure.data_shards, erasure.parity_shards, BitrotReaderSetupMode::ReadQuorum, @@ -2277,7 +2277,7 @@ impl SetDisks { } else { checksum_info.algorithm }; - let use_zero_copy = rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE); + let use_mmap_read = rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE); let till_offset = erasure.shard_file_offset(0, part_length, part_size); let reader_setup_stage_start = Instant::now(); @@ -2296,7 +2296,7 @@ impl SetDisks { erasure.shard_size(), checksum_algo, skip_verify_bitrot, - use_zero_copy, + use_mmap_read, erasure.data_shards, erasure.parity_shards, BitrotReaderSetupMode::VerifyReconstruction, diff --git a/crates/io-core/Cargo.toml b/crates/io-core/Cargo.toml index 08b1cb529..ba37b1797 100644 --- a/crates/io-core/Cargo.toml +++ b/crates/io-core/Cargo.toml @@ -20,8 +20,8 @@ license.workspace = true repository.workspace = true rust-version.workspace = true homepage.workspace = true -description = "Zero-copy core reader and writer implementations for RustFS" -keywords = ["zero-copy", "reader", "writer", "rustfs"] +description = "Buffered I/O reader and writer implementations for RustFS (mmap-then-copy, aligned pread)" +keywords = ["io", "reader", "writer", "rustfs", "mmap"] categories = ["development-tools", "filesystem"] [lints] @@ -34,7 +34,9 @@ tokio = { workspace = true, features = ["io-util", "fs", "rt", "sync"] } memmap2 = { workspace = true } rustfs-io-metrics = { workspace = true } -# io-uring is Linux-only. Scope the feature to Linux targets. +# Enable tokio's io_uring-based runtime backend on Linux. +# Note: this is a tokio runtime feature, not application-level io_uring usage. +# See https://github.com/rustfs/backlog/issues/733 [target.'cfg(target_os = "linux")'.dependencies] tokio = { workspace = true, features = ["io-uring"] } diff --git a/crates/io-core/src/lib.rs b/crates/io-core/src/lib.rs index fe0ee031f..6cf25f198 100644 --- a/crates/io-core/src/lib.rs +++ b/crates/io-core/src/lib.rs @@ -12,20 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Zero-copy core reader and writer implementations for RustFS. +//! Buffered I/O reader and writer implementations for RustFS. //! -//! This crate provides zero-copy readers and writers that minimize memory -//! allocations and data copying during I/O operations. It depends on -//! `rustfs-io-metrics` for metrics reporting and is designed to avoid -//! introducing cyclic dependencies in the RustFS crate graph. +//! This crate provides buffered readers and writers for I/O operations. +//! Note: despite "ZeroCopy" naming in type names (kept for backward compatibility), +//! the actual implementations perform mmap-then-copy or aligned pread, not true +//! zero-copy. See https://github.com/rustfs/backlog/issues/733 //! //! # Features //! -//! - Memory-mapped file reading (mmap) on Unix platforms -//! - Bytes-based zero-copy wrapping +//! - Memory-mapped file reading (mmap-then-copy) on Unix platforms +//! - Bytes-based buffered wrapping //! - AsyncRead trait implementations //! - Tiered BytesPool for buffer management -//! - Optional Direct I/O support (Linux only) +//! - Aligned pread-based reader (NOT true Direct I/O / O_DIRECT) //! //! # Example //!