perf(ecstore): erasure read path for large GETs (#9304)

Lands the contributor's pull requests #9302 (opt-in lockless KV cache
buckets), #9303 (per-bucket XXH3 ETags; MD5 off the request task) and
#9304 (erasure read path for large GETs) as one change, because #9304
was stacked on the other two.

Maintainer changes on top of the contributed work:
- Merged main. crates/ecstore/src/erasure/coding/bitrot.rs: kept the
  PR's verify_chunked_frame/append_chunked_data restructuring and routed
  its hash-mismatch log through main's throttled log_hash_mismatch.
- rustfs/src/app/multipart_usecase.rs: took main's per-operation split
  and re-applied the PR's changes in place: multipart_etag_algorithm and
  its test in the parent module, the XXH3 pin in create.rs, the stage
  timer and from_stream_with_etag in upload_part.rs.
- Reworded one doc comment in crates/config/src/constants/object.rs that
  spelled an ecstore import path, which the architecture migration guard
  counts as a direct import.
- fix(config): made the MD5 offload opt-in. Each offloaded upload holds
  one blocking-pool thread until its body ends, and that pool also
  serves disk I/O, so many concurrent large uploads could starve disk
  writes. The default threshold is now 0 (disabled); operators enable
  it with RUSTFS_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES (for example 8388608).

crates/config/src/constants/object.rs

@@ -117,6 +117,30 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
 /// Default: bitrot verification is enabled on GetObject reads (do not skip).
 pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
 
+/// Comma-separated buckets that hold a content-addressed cache (an LLM KV
+/// cache) and serve PutObject, GetObject and HeadObject without distributed
+/// locks or authoritative bucket-metadata reads. See the ecstore
+/// `bucket::lockless_cache` module for what such a bucket gives up.
+///
+/// Default is empty: every bucket keeps the locked paths.
+pub const ENV_LOCKLESS_CACHE_BUCKETS: &str = "RUSTFS_LOCKLESS_CACHE_BUCKETS";
+
+/// Smallest declared body, in bytes, whose ETag MD5 is computed on a
+/// blocking-pool thread while the body streams into the erasure encoder,
+/// instead of inline on the request's task. The ETag and Content-MD5
+/// checks are unchanged; `0` keeps every MD5 inline.
+///
+/// Each offloaded upload holds one blocking-pool thread until its body ends,
+/// and that pool also serves disk I/O, so size the threshold against the
+/// expected number of concurrent large uploads before enabling it.
+pub const ENV_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES: &str = "RUSTFS_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES";
+
+/// Default: disabled; every ETag MD5 is computed inline. Set
+/// `RUSTFS_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES` (for example `8388608`) to opt in.
+pub const DEFAULT_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES: u64 = 0;
+
+const _: () = assert!(DEFAULT_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES == 0);
+
 /// Create independent shard commitments for new writes after the fleet is upgraded.
 /// Existing protected objects and multipart uploads retain their protection.
 pub const ENV_SHARD_INTEGRITY_WRITE: &str = "RUSTFS_SHARD_INTEGRITY_WRITE";
@@ -298,6 +322,37 @@ pub const ENV_OBJECT_DISK_READ_TIMEOUT: &str = "RUSTFS_OBJECT_DISK_READ_TIMEOUT"
 /// Default disk read timeout in seconds.
 pub const DEFAULT_OBJECT_DISK_READ_TIMEOUT: u64 = 10;
 
+/// Environment variable for the smallest bitrot frame (hash + shard bytes)
+/// that is verified on a blocking thread instead of inline on the decode task
+/// (`0` disables off-thread verification).
+///
+/// The shard reads of one stripe complete on one task, so with large shards
+/// their hashes were computed one after another on one core: at 4 MiB shards
+/// a 12+4 stripe spent about 10 ms hashing serially. Frames at or above this
+/// size are hashed on the blocking pool so the shards of a stripe verify in
+/// parallel; small frames stay inline because a thread hop would cost more
+/// than the hash.
+pub const ENV_BITROT_VERIFY_OFFLOAD_MIN_BYTES: &str = "RUSTFS_BITROT_VERIFY_OFFLOAD_MIN_BYTES";
+/// Default smallest off-thread bitrot frame: 2 MiB (frames of the legacy 1 MiB
+/// block, and 1 MiB shards, stay inline).
+pub const DEFAULT_BITROT_VERIFY_OFFLOAD_MIN_BYTES: usize = 2 * 1024 * 1024;
+
+/// Environment variable for the erasure shard size, in bytes, of newly written
+/// objects (`0`, the default, keeps the historical 1 MiB erasure block).
+///
+/// When set, the erasure block of a new object is `data shards x shard bytes`,
+/// so every shard read and write is exactly this large and aligned: 4 MiB on a
+/// 12+4 layout gives a 48 MiB block, 48x fewer stripes per object than today.
+/// Existing objects keep the block recorded in their `xl.meta`. System
+/// metadata under `.rustfs.sys` always uses the 1 MiB block. The value must be
+/// a multiple of 4 KiB and at most `MAX_ERASURE_SHARD_BYTES`; invalid values
+/// are ignored with a warning.
+pub const ENV_ERASURE_SHARD_BYTES: &str = "RUSTFS_ERASURE_SHARD_BYTES";
+/// Default erasure shard size override (`0` = derive from the 1 MiB block).
+pub const DEFAULT_ERASURE_SHARD_BYTES: usize = 0;
+/// Largest accepted erasure shard size override (64 MiB).
+pub const MAX_ERASURE_SHARD_BYTES: usize = 64 * 1024 * 1024;
+
 /// Environment variable for the per-shard erasure write stall timeout (seconds).
 ///
 /// A single shard write (or shard-writer shutdown) that makes no forward

crates/config/src/constants/runtime.rs

@@ -114,6 +114,14 @@ pub const ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: &str = "RUSTFS_OBJECT_FILE_
 pub const ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD";
 pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE: bool = false;
 pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: bool = true;
+
+/// Bytes of page-cache read-ahead a ranged file stream keeps queued in front
+/// of its read position (`posix_fadvise(WILLNEED)`, Linux only; `0` disables).
+/// A shard stream served to a peer reads 4 MiB chunks one after another; the
+/// kernel's own read-ahead window ramps up slowly for large reads, so without
+/// this each chunk waits for its own device round trips.
+pub const ENV_OBJECT_STREAM_READ_AHEAD_BYTES: &str = "RUSTFS_OBJECT_STREAM_READ_AHEAD_BYTES";
+pub const DEFAULT_OBJECT_STREAM_READ_AHEAD_BYTES: usize = 16 * 1024 * 1024;
 pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: usize = 4 * 1024 * 1024;
 
 /// Threshold for small object seek support in bytes.
@@ -158,3 +166,10 @@ pub const ENV_OBJECT_DATA_CACHE_MIN_FREE_MEMORY_PERCENT: &str = "RUSTFS_OBJECT_D
 pub const ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_PER_CPU: &str = "RUSTFS_OBJECT_DATA_CACHE_FILL_CONCURRENCY_PER_CPU";
 pub const ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_MAX: &str = "RUSTFS_OBJECT_DATA_CACHE_FILL_CONCURRENCY_MAX";
 pub const ENV_OBJECT_DATA_CACHE_IDENTITY_KEYS_MAX: &str = "RUSTFS_OBJECT_DATA_CACHE_IDENTITY_KEYS_MAX";
+/// Fill the object data cache from the PUT body as it streams in, so the first
+/// GET after a write is a hit (write-once KV-cache chunks). Only objects that
+/// fit the cache entry limit and take the plain streaming PUT path (no
+/// compression, no SSE) are filled; the body is kept in memory once more
+/// during the PUT.
+pub const ENV_OBJECT_DATA_CACHE_FILL_ON_PUT: &str = "RUSTFS_OBJECT_DATA_CACHE_FILL_ON_PUT";
+pub const DEFAULT_OBJECT_DATA_CACHE_FILL_ON_PUT: bool = false;

crates/config/src/observability/mod.rs

@@ -55,6 +55,9 @@ pub const ENV_OBS_PUT_STAGE_METRICS_ENABLED: &str = "RUSTFS_OBS_PUT_STAGE_METRIC
 pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED";
 pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE";
 pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS";
+/// Path of the sampled GetObject stage summary file (defaults to `/var/log/rustfs/get-stage-summary.log`).
+pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_FILE: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_FILE";
+pub const DEFAULT_OBS_GET_STAGE_LOCAL_SUMMARY_FILE: &str = "/var/log/rustfs/get-stage-summary.log";
 
 pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
 pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
@@ -178,6 +181,7 @@ mod tests {
             ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS,
             "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS"
         );
+        assert_eq!(ENV_OBS_GET_STAGE_LOCAL_SUMMARY_FILE, "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_FILE");
         // Test log cleanup related env keys
         assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
         assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");

crates/ecstore/src/api/mod.rs

@@ -194,8 +194,8 @@ pub mod bucket {
             table_catalog_path_hash, unreadable_config_refusal,
         };
         pub use crate::bucket::metadata::{
-            BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG, MAX_SWIFT_METADATA_COUNT, MAX_SWIFT_METADATA_VALUE_SIZE,
-            SWIFT_ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX, SWIFT_TAGGING_MARKER,
+            BUCKET_DURABILITY_CONFIG, BUCKET_ETAG_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG, MAX_SWIFT_METADATA_COUNT,
+            MAX_SWIFT_METADATA_VALUE_SIZE, SWIFT_ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX, SWIFT_TAGGING_MARKER,
         };
         pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, is_swift_bucket_metadata_tag_key, validate_tag_set_entries};
     }
@@ -212,6 +212,16 @@ pub mod bucket {
         };
     }
 
+    pub mod etag {
+        pub use crate::bucket::etag::{
+            BUCKET_ETAG_ALGORITHM_MD5, BUCKET_ETAG_ALGORITHM_XXH3, BucketEtagConfig, bucket_uses_xxh3_etag,
+        };
+    }
+
+    pub mod lockless_cache {
+        pub use crate::bucket::lockless_cache::is_lockless_cache_bucket;
+    }
+
     pub mod metadata_sys {
         pub use crate::bucket::metadata_sys::{
             BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
@@ -220,7 +230,7 @@ pub mod bucket {
             acquire_scanner_bucket_incarnation_read_fence, capture_bucket_metadata_incarnation, delete, delete_if_incarnation,
             delete_if_incarnation_at, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
             get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
-            get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
+            get_etag_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
             get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
             get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
             get_request_payment_config, get_s3_tagging_config, get_sse_config, get_tagging_config, get_versioning_config,
@@ -594,9 +604,14 @@ pub mod integrity {
 }
 
 pub mod object {
+    pub use crate::erasure::codec::shard_buf::{
+        ArenaBlock, RegisteredSlabArena, ShardArena, ShardBuf, install_shard_arena, shard_arena,
+    };
+    pub use crate::erasure::codec::shard_emit::ShardEmitter;
     pub use crate::object_api::{
         BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
         GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReadCondition, GetObjectReadMetadata, GetObjectReader,
+        GetObjectSink, GetObjectSinkSegmentFactory, GetObjectSinkSegments, GetObjectSinkStream, GetObjectSinkWriter,
         NamespaceLockFence, ObjectEncryptionResolver, ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions,
         PutObjReader, QuotaAdmission, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
         SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
@@ -632,8 +647,8 @@ pub mod rebalance {
 
 pub mod rio {
     pub use crate::io_support::rio::{
-        DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
-        boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
+        DecryptReader, DynReader, EncryptReader, EtagAlgorithm, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption,
+        WritePlan, boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
     };
 }
 

crates/ecstore/src/bucket/etag.rs

@@ -0,0 +1,150 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Per-bucket ETag algorithm.
+//!
+//! A bucket can derive the ETags of objects written without Content-MD5 from
+//! XXH3-128 instead of MD5, for clients that never treat the ETag as an MD5
+//! (an LLM KV cache). The choice is stored as a RustFS extension entry
+//! (`etag.json`) in the bucket metadata file and published to an in-memory
+//! registry whenever the metadata is installed, so the write path looks it up
+//! without I/O. Buckets without the entry keep MD5 ETags.
+
+use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{OnceLock, PoisonError, RwLock};
+
+pub const BUCKET_ETAG_ALGORITHM_MD5: &str = "md5";
+pub const BUCKET_ETAG_ALGORITHM_XXH3: &str = "xxh3";
+
+/// JSON payload stored under the `etag.json` bucket metadata entry. An absent
+/// entry, an empty payload or an invalid algorithm all mean MD5 ETags.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BucketEtagConfig {
+    /// `md5` | `xxh3` (case-insensitive, whitespace-tolerant).
+    #[serde(default)]
+    pub algorithm: String,
+}
+
+impl BucketEtagConfig {
+    pub fn new(algorithm: &str) -> Self {
+        Self {
+            algorithm: algorithm.to_string(),
+        }
+    }
+
+    /// Whether `algorithm` names a supported ETag algorithm.
+    pub fn is_valid_algorithm(algorithm: &str) -> bool {
+        matches!(
+            algorithm.trim().to_ascii_lowercase().as_str(),
+            BUCKET_ETAG_ALGORITHM_MD5 | BUCKET_ETAG_ALGORITHM_XXH3
+        )
+    }
+
+    /// The canonical (trimmed, lowercase) algorithm name, or `None` when the
+    /// config does not carry a valid one.
+    pub fn normalized_algorithm(&self) -> Option<String> {
+        let algorithm = self.algorithm.trim().to_ascii_lowercase();
+        if Self::is_valid_algorithm(&algorithm) {
+            Some(algorithm)
+        } else {
+            None
+        }
+    }
+
+    /// Whether this config selects XXH3 ETags.
+    pub fn uses_xxh3(&self) -> bool {
+        self.normalized_algorithm().as_deref() == Some(BUCKET_ETAG_ALGORITHM_XXH3)
+    }
+}
+
+/// Buckets whose ETags use XXH3, kept in step with the installed bucket
+/// metadata. `ANY` lets the write path skip the lock while no bucket opts in.
+static XXH3_BUCKETS: OnceLock<RwLock<HashSet<String>>> = OnceLock::new();
+static ANY: AtomicBool = AtomicBool::new(false);
+
+fn xxh3_buckets() -> &'static RwLock<HashSet<String>> {
+    XXH3_BUCKETS.get_or_init(|| RwLock::new(HashSet::new()))
+}
+
+/// Records whether `bucket` uses XXH3 ETags.
+pub fn set_bucket_xxh3_etag(bucket: &str, xxh3: bool) {
+    if bucket.is_empty() {
+        return;
+    }
+    let mut buckets = xxh3_buckets().write().unwrap_or_else(PoisonError::into_inner);
+    if xxh3 {
+        buckets.insert(bucket.to_string());
+    } else {
+        buckets.remove(bucket);
+    }
+    ANY.store(!buckets.is_empty(), Ordering::Release);
+}
+
+/// Whether objects written to `bucket` without Content-MD5 get XXH3 ETags.
+pub fn bucket_uses_xxh3_etag(bucket: &str) -> bool {
+    if !ANY.load(Ordering::Acquire) {
+        return false;
+    }
+    xxh3_buckets().read().unwrap_or_else(PoisonError::into_inner).contains(bucket)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn valid_algorithms_are_recognized() {
+        assert!(BucketEtagConfig::is_valid_algorithm("md5"));
+        assert!(BucketEtagConfig::is_valid_algorithm(" XXH3 "));
+        assert!(!BucketEtagConfig::is_valid_algorithm(""));
+        assert!(!BucketEtagConfig::is_valid_algorithm("sha256"));
+    }
+
+    #[test]
+    fn normalized_algorithm_canonicalizes_or_rejects() {
+        assert_eq!(BucketEtagConfig::new(" Xxh3 ").normalized_algorithm().as_deref(), Some("xxh3"));
+        assert_eq!(BucketEtagConfig::new("MD5").normalized_algorithm().as_deref(), Some("md5"));
+        assert_eq!(BucketEtagConfig::new("crc32").normalized_algorithm(), None);
+        assert!(BucketEtagConfig::new("xxh3").uses_xxh3());
+        assert!(!BucketEtagConfig::new("md5").uses_xxh3());
+        assert!(!BucketEtagConfig::default().uses_xxh3());
+    }
+
+    #[test]
+    fn json_round_trip() {
+        let config = BucketEtagConfig::new("xxh3");
+        let json = serde_json::to_vec(&config).expect("serialize");
+        assert_eq!(json, br#"{"algorithm":"xxh3"}"#);
+        assert_eq!(serde_json::from_slice::<BucketEtagConfig>(&json).expect("parse"), config);
+        assert_eq!(
+            serde_json::from_slice::<BucketEtagConfig>(b"{}").expect("parse"),
+            BucketEtagConfig::default()
+        );
+    }
+
+    #[test]
+    fn registry_tracks_xxh3_buckets() {
+        let bucket = "etag-registry-test-bucket";
+        assert!(!bucket_uses_xxh3_etag(bucket));
+        set_bucket_xxh3_etag(bucket, true);
+        assert!(bucket_uses_xxh3_etag(bucket));
+        assert!(!bucket_uses_xxh3_etag("etag-registry-test-other"));
+        set_bucket_xxh3_etag(bucket, false);
+        assert!(!bucket_uses_xxh3_etag(bucket));
+        set_bucket_xxh3_etag("", true);
+        assert!(!bucket_uses_xxh3_etag(""));
+    }
+}

crates/ecstore/src/bucket/lockless_cache.rs

@@ -0,0 +1,76 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Lockless cache buckets.
+//!
+//! A bucket named in `RUSTFS_LOCKLESS_CACHE_BUCKETS` holds a content-addressed
+//! cache (an LLM KV cache): a key always names the same bytes, and a lost or
+//! unreadable object only costs a recompute. PutObject, GetObject and HeadObject
+//! on such a bucket take no distributed locks and read no bucket metadata from
+//! disk, so concurrent writers into one bucket no longer meet on bucket-wide
+//! lock keys.
+//!
+//! What a listed bucket gives up, by design:
+//! - Writes of one key are not serialized. Racing commits of the same bytes can
+//!   leave the object without read quorum until it is rewritten or healed,
+//!   which a cache treats as a miss.
+//! - Reads are not fenced against a concurrent overwrite or delete.
+//! - Writes are not fenced against bucket deletion, Object Lock, quota,
+//!   versioning, replication or table-catalog changes made while they run. A
+//!   PUT falls back to the locked path whenever the cached bucket state shows
+//!   any of those features, but listed buckets must not enable them.
+//!
+//! The list is read once per process; RustFS system buckets are never lockless.
+
+use super::utils::is_meta_bucketname;
+use std::collections::HashSet;
+use std::sync::OnceLock;
+
+/// Buckets named in a comma-separated list, without blanks or system buckets.
+fn parse_lockless_cache_buckets(value: &str) -> HashSet<String> {
+    value
+        .split(',')
+        .map(str::trim)
+        .filter(|bucket| !bucket.is_empty() && !is_meta_bucketname(bucket))
+        .map(str::to_owned)
+        .collect()
+}
+
+/// Whether `bucket` is configured as a lockless cache bucket.
+pub fn is_lockless_cache_bucket(bucket: &str) -> bool {
+    static BUCKETS: OnceLock<HashSet<String>> = OnceLock::new();
+    BUCKETS
+        .get_or_init(|| {
+            rustfs_utils::get_env_opt_str(rustfs_config::ENV_LOCKLESS_CACHE_BUCKETS)
+                .map(|value| parse_lockless_cache_buckets(&value))
+                .unwrap_or_default()
+        })
+        .contains(bucket)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parse_keeps_named_user_buckets_only() {
+        let buckets = parse_lockless_cache_buckets(" kvcache-a,,kvcache-b , .rustfs.sys ,");
+        assert_eq!(
+            buckets,
+            HashSet::from(["kvcache-a".to_owned(), "kvcache-b".to_owned()]),
+            "blanks and system buckets must be dropped"
+        );
+        assert!(parse_lockless_cache_buckets("").is_empty());
+    }
+}

crates/ecstore/src/bucket/metadata.rs

@@ -390,6 +390,7 @@ pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
 pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
 pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json";
 pub const BUCKET_DURABILITY_CONFIG: &str = "durability.json";
+pub const BUCKET_ETAG_CONFIG: &str = "etag.json";
 pub const BUCKET_ON_DEMAND_MIGRATION_CONFIG: &str = "on-demand-migration.json";
 pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table";
 pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog";
@@ -581,6 +582,7 @@ pub struct BucketMetadata {
     pub table_bucket_config_json: Vec<u8>,
     pub durability_config_json: Vec<u8>,
     pub on_demand_migration_config_json: Vec<u8>,
+    pub etag_config_json: Vec<u8>,
 
     pub policy_config_updated_at: OffsetDateTime,
     pub object_lock_config_updated_at: OffsetDateTime,
@@ -603,6 +605,7 @@ pub struct BucketMetadata {
     pub table_bucket_config_updated_at: OffsetDateTime,
     pub durability_config_updated_at: OffsetDateTime,
     pub on_demand_migration_config_updated_at: OffsetDateTime,
+    pub etag_config_updated_at: OffsetDateTime,
 
     pub new_field_updated_at: OffsetDateTime,
 
@@ -657,6 +660,7 @@ impl Default for BucketMetadata {
             table_bucket_config_json: Default::default(),
             durability_config_json: Default::default(),
             on_demand_migration_config_json: Default::default(),
+            etag_config_json: Default::default(),
             policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH,
@@ -678,6 +682,7 @@ impl Default for BucketMetadata {
             table_bucket_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             durability_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             on_demand_migration_config_updated_at: OffsetDateTime::UNIX_EPOCH,
+            etag_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
             policy_config: Default::default(),
             notification_config: Default::default(),
@@ -804,6 +809,29 @@ impl BucketMetadata {
         }
     }
 
+    /// Parsed per-bucket ETag algorithm, if one is stored. Invalid payloads
+    /// keep MD5 ETags after logging a parse failure.
+    pub fn etag_config(&self) -> Option<super::etag::BucketEtagConfig> {
+        if self.etag_config_json.is_empty() {
+            return None;
+        }
+        match serde_json::from_slice(&self.etag_config_json) {
+            Ok(cfg) => Some(cfg),
+            Err(e) => {
+                tracing::warn!(
+                    event = "bucket_metadata_parse_failed",
+                    component = "ecstore",
+                    subsystem = "bucket_metadata",
+                    bucket = %self.name,
+                    config = "etag",
+                    error = %e,
+                    "Failed to parse bucket metadata config"
+                );
+                None
+            }
+        }
+    }
+
     /// Decode from msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
     pub fn decode_from<R: Read>(&mut self, rd: &mut R) -> Result<()> {
         let mut fields = rmp::decode::read_map_len(rd)?;
@@ -864,6 +892,7 @@ impl BucketMetadata {
                 "OnDemandMigrationConfigJSON" | "OnDemandMigrationConfigJson" => {
                     self.on_demand_migration_config_json = read_msgp_bin(rd)?
                 }
+                "EtagConfigJSON" | "EtagConfigJson" => self.etag_config_json = read_msgp_bin(rd)?,
                 "CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
                 "LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?,
                 "WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?,
@@ -874,6 +903,7 @@ impl BucketMetadata {
                 "TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
                 "DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
                 "OnDemandMigrationConfigUpdatedAt" => self.on_demand_migration_config_updated_at = read_msgp_time_value(rd)?,
+                "EtagConfigUpdatedAt" => self.etag_config_updated_at = read_msgp_time_value(rd)?,
                 "BucketCreationCommitted" => {
                     self.bucket_creation_committed = read_msgp_bool(rd)?;
                     self.bucket_creation_commit_record_present = true;
@@ -892,8 +922,8 @@ impl BucketMetadata {
     pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
         // Preserve the absence of a legacy commit record across config rewrites.
         let commit_record_present = self.bucket_creation_commit_record_present || self.bucket_creation_committed;
-        // Map size: MinIO fields (25) + RustFS extensions (21) + optional commit record.
-        let map_len: u32 = 46 + u32::from(commit_record_present);
+        // Map size: MinIO fields (25) + RustFS extensions (23) + optional commit record.
+        let map_len: u32 = 48 + u32::from(commit_record_present);
         rmp::encode::write_map_len(wr, map_len)?;
 
         // MinIO field order (same as Go struct)
@@ -954,6 +984,7 @@ impl BucketMetadata {
         write_bin_field(wr, "TableBucketConfigJSON", &self.table_bucket_config_json)?;
         write_bin_field(wr, "DurabilityConfigJSON", &self.durability_config_json)?;
         write_bin_field(wr, "OnDemandMigrationConfigJSON", &self.on_demand_migration_config_json)?;
+        write_bin_field(wr, "EtagConfigJSON", &self.etag_config_json)?;
         rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?;
         write_msgp_time(wr, self.cors_config_updated_at)?;
         rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?;
@@ -974,6 +1005,8 @@ impl BucketMetadata {
         write_msgp_time(wr, self.durability_config_updated_at)?;
         rmp::encode::write_str(wr, "OnDemandMigrationConfigUpdatedAt")?;
         write_msgp_time(wr, self.on_demand_migration_config_updated_at)?;
+        rmp::encode::write_str(wr, "EtagConfigUpdatedAt")?;
+        write_msgp_time(wr, self.etag_config_updated_at)?;
         if commit_record_present {
             rmp::encode::write_str(wr, "BucketCreationCommitted")?;
             rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
@@ -1082,6 +1115,9 @@ impl BucketMetadata {
         if self.on_demand_migration_config_updated_at == OffsetDateTime::UNIX_EPOCH {
             self.on_demand_migration_config_updated_at = self.created
         }
+        if self.etag_config_updated_at == OffsetDateTime::UNIX_EPOCH {
+            self.etag_config_updated_at = self.created
+        }
     }
 
     /// Replace one config payload and stamp its `*_config_updated_at` with the
@@ -1228,6 +1264,10 @@ impl BucketMetadata {
                 self.on_demand_migration_config_json = data;
                 self.on_demand_migration_config_updated_at = updated;
             }
+            BUCKET_ETAG_CONFIG => {
+                self.etag_config_json = data;
+                self.etag_config_updated_at = updated;
+            }
             _ => return Err(Error::other(format!("config file not found : {config_file}"))),
         }
 
@@ -2720,6 +2760,28 @@ mod test {
 
     /// HP-5b (rustfs/backlog#938): the durability override is a RustFS
     /// extension entry and must survive an encode/decode round trip.
+    #[test]
+    fn etag_config_round_trips_and_tracks_updates() {
+        let mut bm = BucketMetadata::new("etag-bucket");
+        assert!(bm.etag_config().is_none());
+        bm.update_config(BUCKET_ETAG_CONFIG, br#"{"algorithm":"xxh3"}"#.to_vec())
+            .expect("set etag config");
+        assert_ne!(bm.etag_config_updated_at, OffsetDateTime::UNIX_EPOCH);
+        assert!(bm.etag_config().expect("parsed").uses_xxh3());
+
+        let back = BucketMetadata::unmarshal(&bm.marshal_msg().expect("marshal")).expect("unmarshal");
+        assert_eq!(back.etag_config_json, bm.etag_config_json);
+        assert_eq!(back.etag_config_updated_at.unix_timestamp(), bm.etag_config_updated_at.unix_timestamp());
+        // The durability entry is independent of the ETag entry.
+        assert!(back.durability_config_json.is_empty());
+
+        bm.update_config(BUCKET_ETAG_CONFIG, Vec::new()).expect("clear etag config");
+        assert!(bm.etag_config().is_none());
+
+        bm.etag_config_json = b"not-json".to_vec();
+        assert!(bm.etag_config().is_none(), "garbage must keep MD5 ETags");
+    }
+
     #[test]
     fn durability_config_round_trips_and_tracks_updates() {
         let mut bm = BucketMetadata::new("durability-bucket");

crates/ecstore/src/bucket/metadata_sys.rs

@@ -440,6 +440,25 @@ fn clear_bucket_durability(bucket: &str) {
     crate::disk::local::bucket_durability::set(bucket, None);
 }
 
+/// Publish the bucket's ETag algorithm to the registry the write path consults.
+fn sync_bucket_etag(bucket: &str, bm: &BucketMetadata) {
+    let xxh3 = bm.etag_config().is_some_and(|cfg| cfg.uses_xxh3());
+    crate::bucket::etag::set_bucket_xxh3_etag(bucket, xxh3);
+}
+
+/// Publish every per-bucket override kept outside the metadata cache
+/// (durability tier, ETag algorithm) on each install path.
+fn sync_bucket_overrides(bucket: &str, bm: &BucketMetadata) {
+    sync_bucket_durability(bucket, bm);
+    sync_bucket_etag(bucket, bm);
+}
+
+/// Drop every per-bucket override when the bucket's metadata leaves the cache.
+fn clear_bucket_overrides(bucket: &str) {
+    clear_bucket_durability(bucket);
+    crate::bucket::etag::set_bucket_xxh3_etag(bucket, false);
+}
+
 /// Publish application-owned bytes on every cache install path.
 fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
     if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
@@ -1587,6 +1606,16 @@ pub async fn get_durability_config(
     Ok((bm.durability_config(), bm.durability_config_updated_at))
 }
 
+/// The bucket's ETag algorithm config with its update time. `Ok((None, ..))`
+/// means the bucket keeps MD5 ETags.
+pub async fn get_etag_config(bucket: &str) -> Result<(Option<crate::bucket::etag::BucketEtagConfig>, OffsetDateTime)> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    let (bm, _) = bucket_meta_sys.get_config(bucket).await?;
+    Ok((bm.etag_config(), bm.etag_config_updated_at))
+}
+
 /// The bucket's on-demand migration config with its update time, or
 /// `Ok(None)` when the bucket has none. Bytes are opaque to the metadata owner.
 pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
@@ -2451,7 +2480,7 @@ impl BucketMetadataSys {
                 self.missing_buckets.insert(bucket.to_string(), ()).await;
                 if removed {
                     BucketTargetSys::get().delete(bucket).await;
-                    clear_bucket_durability(bucket);
+                    clear_bucket_overrides(bucket);
                     clear_on_demand_migration(bucket);
                 }
             }
@@ -2489,7 +2518,7 @@ impl BucketMetadataSys {
                 self.fabricated_metadata.write().await.remove(bucket);
                 self.missing_buckets.invalidate(bucket).await;
                 sync_bucket_target_sys(bucket, &bm).await;
-                sync_bucket_durability(bucket, &bm);
+                sync_bucket_overrides(bucket, &bm);
                 sync_on_demand_migration(bucket, &bm);
             }
             MetadataLoadMode::Initial => {
@@ -2537,7 +2566,7 @@ impl BucketMetadataSys {
             self.missing_buckets.invalidate(bucket).await;
             if removed {
                 BucketTargetSys::get().delete(bucket).await;
-                clear_bucket_durability(bucket);
+                clear_bucket_overrides(bucket);
                 clear_on_demand_migration(bucket);
             }
             return Ok(());
@@ -2560,7 +2589,7 @@ impl BucketMetadataSys {
         self.fabricated_metadata.write().await.remove(bucket);
         self.missing_buckets.invalidate(bucket).await;
         sync_bucket_target_sys(bucket, &metadata).await;
-        sync_bucket_durability(bucket, &metadata);
+        sync_bucket_overrides(bucket, &metadata);
         sync_on_demand_migration(bucket, &metadata);
         Ok(())
     }
@@ -2590,7 +2619,7 @@ impl BucketMetadataSys {
             self.fabricated_metadata.write().await.remove(&bucket);
             self.missing_buckets.invalidate(&bucket).await;
             sync_bucket_target_sys(&bucket, &bm).await;
-            sync_bucket_durability(&bucket, &bm);
+            sync_bucket_overrides(&bucket, &bm);
             sync_on_demand_migration(&bucket, &bm);
         }
     }
@@ -2614,7 +2643,7 @@ impl BucketMetadataSys {
         super::config_parse_mode::forget_bucket_config_parse_state(bucket);
         if removed {
             BucketTargetSys::get().delete(bucket).await;
-            clear_bucket_durability(bucket);
+            clear_bucket_overrides(bucket);
             clear_on_demand_migration(bucket);
         }
         removed || removed_fabricated
@@ -2957,7 +2986,7 @@ impl BucketMetadataSys {
                 self.fabricated_metadata.write().await.remove(bucket);
                 self.missing_buckets.invalidate(bucket).await;
                 sync_bucket_target_sys(bucket, &bm).await;
-                sync_bucket_durability(bucket, &bm);
+                sync_bucket_overrides(bucket, &bm);
                 sync_on_demand_migration(bucket, &bm);
                 return Ok((bm, true));
             } else {
@@ -3408,7 +3437,7 @@ impl BucketMetadataSys {
         self.fabricated_metadata.write().await.remove(bucket);
         self.missing_buckets.invalidate(bucket).await;
         sync_bucket_target_sys(bucket, &metadata).await;
-        sync_bucket_durability(bucket, &metadata);
+        sync_bucket_overrides(bucket, &metadata);
         sync_on_demand_migration(bucket, &metadata);
         Ok(BucketMetadataAuthority::Authoritative(metadata))
     }
@@ -6686,6 +6715,41 @@ mod tests {
         target_sys.delete(bucket).await;
     }
 
+    /// Installing bucket metadata publishes the bucket's ETag algorithm to the
+    /// write-path registry; metadata without it, an invalid payload or
+    /// clearing the bucket's overrides withdraws XXH3.
+    #[test]
+    fn metadata_sync_publishes_and_clears_etag_algorithm() {
+        use crate::bucket::etag::bucket_uses_xxh3_etag;
+
+        let bucket = "metadata-sync-etag";
+
+        let mut bm = BucketMetadata::new(bucket);
+        bm.etag_config_json = br#"{"algorithm":"xxh3"}"#.to_vec();
+        sync_bucket_overrides(bucket, &bm);
+        assert!(bucket_uses_xxh3_etag(bucket));
+
+        sync_bucket_overrides(bucket, &BucketMetadata::new(bucket));
+        assert!(!bucket_uses_xxh3_etag(bucket));
+
+        let mut bm = BucketMetadata::new(bucket);
+        bm.etag_config_json = br#"{"algorithm":"md5"}"#.to_vec();
+        sync_bucket_overrides(bucket, &bm);
+        assert!(!bucket_uses_xxh3_etag(bucket));
+
+        let mut bm = BucketMetadata::new(bucket);
+        bm.etag_config_json = b"not-json".to_vec();
+        sync_bucket_overrides(bucket, &bm);
+        assert!(!bucket_uses_xxh3_etag(bucket));
+
+        let mut bm = BucketMetadata::new(bucket);
+        bm.etag_config_json = br#"{"algorithm":"XXH3"}"#.to_vec();
+        sync_bucket_overrides(bucket, &bm);
+        assert!(bucket_uses_xxh3_etag(bucket));
+        clear_bucket_overrides(bucket);
+        assert!(!bucket_uses_xxh3_etag(bucket));
+    }
+
     /// HP-5b (rustfs/backlog#938): installing bucket metadata publishes the
     /// durability override to the disk-layer registry, and clearing the
     /// config (or an invalid payload) withdraws it.

crates/ecstore/src/bucket/mod.rs

@@ -19,7 +19,9 @@ pub mod bucket_target_sys;
 pub mod config_parse_mode;
 pub mod durability;
 pub mod error;
+pub mod etag;
 pub mod lifecycle;
+pub mod lockless_cache;
 pub mod metadata;
 pub mod metadata_sys;
 #[cfg(test)]

crates/ecstore/src/bucket/quota/reservation.rs

@@ -826,12 +826,17 @@ pub(crate) async fn begin(
 ) -> Result<QuotaContext> {
     let snapshot_admission = opts.quota_admission;
     let data_movement = opts.data_movement;
-    if crate::bucket::utils::is_meta_bucketname(bucket) {
-        let meta_bucket_fast_path_started = rustfs_io_metrics::put_stage_timer();
-        rustfs_io_metrics::record_put_object_stage_duration_from(
-            rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
-            meta_bucket_fast_path_started,
-        );
+    // A lockless cache PUT is only taken when the bucket has no quota, so it
+    // reserves nothing and needs no metadata-transaction lock.
+    let lockless_cache = opts.is_lockless_cache_write() && snapshot_admission.is_none();
+    if lockless_cache || crate::bucket::utils::is_meta_bucketname(bucket) {
+        if !lockless_cache {
+            let meta_bucket_fast_path_started = rustfs_io_metrics::put_stage_timer();
+            rustfs_io_metrics::record_put_object_stage_duration_from(
+                rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
+                meta_bucket_fast_path_started,
+            );
+        }
         return Ok(QuotaContext {
             store: None,
             bucket: bucket.to_string(),

crates/ecstore/src/disk/local.rs

@@ -3873,6 +3873,33 @@ struct FileCacheReclaimReader {
     reclaim_len: usize,
     reclaim_on_drop: bool,
     reclaimed: bool,
+    /// Current read position, advanced by every successful poll.
+    pos: u64,
+    /// End of the ranged read (`reclaim_offset + reclaim_len`).
+    end: u64,
+    /// Page-cache read-ahead has been requested up to here.
+    advised_until: u64,
+    /// Read-ahead window in bytes; `0` disables.
+    read_ahead: u64,
+}
+
+/// Largest single read a ranged stream hands the blocking pool; one RDMA chunk.
+const STREAM_READ_MAX_BUF_SIZE: usize = 4 * 1024 * 1024;
+
+/// The next range to request page-cache read-ahead for, if less than half of
+/// the window is still queued in front of `pos`.
+#[cfg(any(target_os = "linux", test))]
+fn next_read_ahead_range(pos: u64, advised_until: u64, end: u64, window: u64) -> Option<(u64, u64)> {
+    if window == 0 || advised_until >= end {
+        return None;
+    }
+    let queued = advised_until.saturating_sub(pos);
+    if queued > window / 2 {
+        return None;
+    }
+    let start = advised_until.max(pos);
+    let stop = start.saturating_add(window).min(end);
+    (stop > start).then_some((start, stop))
 }
 
 struct StallTimeoutReader<R> {
@@ -4173,21 +4200,58 @@ async fn read_dir_entry_batch_with_walk_stall(
 }
 
 impl FileCacheReclaimReader {
-    fn new(inner: File, reclaim_offset: u64, reclaim_len: usize, reclaim_on_drop: bool) -> Self {
+    fn new(inner: File, reclaim_offset: u64, reclaim_len: usize, reclaim_on_drop: bool, read_ahead: usize) -> Self {
         #[cfg(target_os = "macos")]
         if reclaim_on_drop {
             let _ = set_fd_nocache(&inner);
         }
 
-        Self {
+        let mut reader = Self {
             inner,
             reclaim_offset,
             reclaim_len,
             reclaim_on_drop,
             reclaimed: false,
-        }
+            pos: reclaim_offset,
+            end: reclaim_offset.saturating_add(reclaim_len as u64),
+            advised_until: reclaim_offset,
+            read_ahead: read_ahead as u64,
+        };
+        reader.advise_read_ahead();
+        reader
     }
 
+    /// Queues page-cache read-ahead in front of the read position (Linux).
+    ///
+    /// `WILLNEED` allocates the pages and submits the reads before it
+    /// returns (a few milliseconds per 16 MiB), so it runs on the blocking
+    /// pool over a duplicate descriptor; the bookkeeping is optimistic.
+    #[cfg(target_os = "linux")]
+    fn advise_read_ahead(&mut self) {
+        use core::num::NonZeroU64;
+        use rustix::fs::{Advice, fadvise};
+        use std::os::fd::AsFd;
+
+        let Some((start, stop)) = next_read_ahead_range(self.pos, self.advised_until, self.end, self.read_ahead) else {
+            return;
+        };
+        let Some(len) = NonZeroU64::new(stop - start) else {
+            return;
+        };
+        let Ok(fd) = self.inner.as_fd().try_clone_to_owned() else {
+            self.read_ahead = 0;
+            return;
+        };
+        self.advised_until = stop;
+        tokio::task::spawn_blocking(move || {
+            // Best effort: a refused range only costs the read-ahead.
+            let _ = fadvise(&fd, start, Some(len), Advice::WillNeed);
+        });
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    fn advise_read_ahead(&mut self) {}
+
     #[cfg(target_os = "linux")]
     fn reclaim_file_cache(&mut self) -> std::io::Result<()> {
         if !self.reclaim_on_drop || self.reclaimed || self.reclaim_len == 0 {
@@ -4245,7 +4309,13 @@ impl AsyncRead for FileCacheReclaimReader {
         cx: &mut std::task::Context<'_>,
         buf: &mut ReadBuf<'_>,
     ) -> std::task::Poll<std::io::Result<()>> {
-        std::pin::Pin::new(&mut self.inner).poll_read(cx, buf)
+        let before = buf.filled().len();
+        let polled = std::pin::Pin::new(&mut self.inner).poll_read(cx, buf);
+        if let std::task::Poll::Ready(Ok(())) = &polled {
+            self.pos = self.pos.saturating_add((buf.filled().len() - before) as u64);
+            self.advise_read_ahead();
+        }
+        polled
     }
 }
 
@@ -5080,6 +5150,13 @@ cached_read_env! {
     );
 }
 
+cached_read_env! {
+    fn stream_read_ahead_bytes() -> usize = rustfs_utils::get_env_usize(
+        rustfs_config::ENV_OBJECT_STREAM_READ_AHEAD_BYTES,
+        rustfs_config::DEFAULT_OBJECT_STREAM_READ_AHEAD_BYTES,
+    );
+}
+
 fn should_reclaim_file_cache_after_read(length: usize) -> bool {
     if length == 0 {
         return false;
@@ -5950,9 +6027,10 @@ impl LocalIoBackend for StdBackend {
         if offset > 0 {
             f.seek(SeekFrom::Start(offset as u64)).await?;
         }
+        f.set_max_buf_size(STREAM_READ_MAX_BUF_SIZE);
 
         let reclaim_on_drop = should_reclaim_file_cache_after_read(length);
-        let reader = FileCacheReclaimReader::new(f, offset as u64, length, reclaim_on_drop);
+        let reader = FileCacheReclaimReader::new(f, offset as u64, length, reclaim_on_drop, stream_read_ahead_bytes());
         Ok(Box::new(StallTimeoutReader::new(reader, get_object_disk_read_timeout())))
     }
 
@@ -32889,7 +32967,7 @@ mod test {
         writer.shutdown().await.expect("writer should reclaim on shutdown");
 
         let file = File::open(&path).await.expect("reader file should open");
-        let mut reader = FileCacheReclaimReader::new(file, 0, 6, true);
+        let mut reader = FileCacheReclaimReader::new(file, 0, 6, true, 0);
         let mut body = Vec::new();
         reader.read_to_end(&mut body).await.expect("reader should forward reads");
         assert!(body.ends_with(b"abcdef"));
@@ -39297,3 +39375,28 @@ mod test {
         );
     }
 }
+
+#[cfg(test)]
+mod read_ahead_tests {
+    use super::next_read_ahead_range;
+
+    #[test]
+    fn read_ahead_queues_a_window_then_tops_up_past_the_half_mark() {
+        // A fresh stream queues one full window from the start offset.
+        assert_eq!(next_read_ahead_range(100, 100, 1_000, 200), Some((100, 300)));
+        // More than half the window still queued: nothing to do.
+        assert_eq!(next_read_ahead_range(150, 300, 1_000, 200), None);
+        // At the half mark the next window is queued from where advice stopped.
+        assert_eq!(next_read_ahead_range(200, 300, 1_000, 200), Some((300, 500)));
+        // The last window is clipped to the end of the range and then stops.
+        assert_eq!(next_read_ahead_range(850, 900, 1_000, 200), Some((900, 1_000)));
+        assert_eq!(next_read_ahead_range(950, 1_000, 1_000, 200), None);
+    }
+
+    #[test]
+    fn read_ahead_is_disabled_by_a_zero_window_and_never_lags_the_position() {
+        assert_eq!(next_read_ahead_range(0, 0, 1_000, 0), None);
+        // Advice that fell behind the read position restarts at the position.
+        assert_eq!(next_read_ahead_range(400, 100, 1_000, 200), Some((400, 600)));
+    }
+}

crates/ecstore/src/erasure/codec/bridge.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::erasure::codec::shard_buf::ShardBytes;
 use crate::erasure::codec::workspace::RustfsCodecDecodeWorkspace;
 use crate::erasure::coding::Erasure;
 use rustfs_erasure_codec::galois_8::ReedSolomon;
@@ -47,15 +48,19 @@ pub(crate) trait ErasureDecodeEngine: Send + Sync + 'static {
 
     fn prepare_workspace(&self, shard_len: usize) -> io::Result<Self::Workspace>;
 
-    fn reconstruct_into(&self, shards: &mut [Option<Vec<u8>>], workspace: &mut Self::Workspace) -> io::Result<&'static str>;
+    fn reconstruct_into<S: ShardBytes>(
+        &self,
+        shards: &mut [Option<S>],
+        workspace: &mut Self::Workspace,
+    ) -> io::Result<&'static str>;
 }
 
-fn data_shards_complete(shards: &[Option<Vec<u8>>], data_shards: usize) -> bool {
+fn data_shards_complete<S: ShardBytes>(shards: &[Option<S>], data_shards: usize) -> bool {
     shards.len() >= data_shards && shards.iter().take(data_shards).all(Option::is_some)
 }
 
-fn recover_empty_payload_data_shards(
-    shards: &mut [Option<Vec<u8>>],
+fn recover_empty_payload_data_shards<S: ShardBytes>(
+    shards: &mut [Option<S>],
     data_shards: usize,
     parity_shards: usize,
 ) -> io::Result<bool> {
@@ -81,7 +86,7 @@ fn recover_empty_payload_data_shards(
 
     for shard in shards.iter_mut().take(data_shards) {
         if shard.is_none() {
-            *shard = Some(Vec::new());
+            *shard = Some(S::from(Vec::new()));
         }
     }
 
@@ -153,7 +158,11 @@ impl ErasureDecodeEngine for LegacyEcDecodeEngine {
         Ok(LegacyDecodeWorkspace::new(shard_len))
     }
 
-    fn reconstruct_into(&self, shards: &mut [Option<Vec<u8>>], _workspace: &mut Self::Workspace) -> io::Result<&'static str> {
+    fn reconstruct_into<S: ShardBytes>(
+        &self,
+        shards: &mut [Option<S>],
+        _workspace: &mut Self::Workspace,
+    ) -> io::Result<&'static str> {
         if data_shards_complete(shards, self.erasure.data_shards) {
             return Ok(GET_RECONSTRUCT_OUTCOME_SKIP_DATA_COMPLETE);
         }
@@ -200,13 +209,18 @@ impl RustfsCodecDecodeEngine {
         Ok(Some(codec))
     }
 
-    fn needs_source_parity_verification(&self, shards: &[Option<Vec<u8>>]) -> bool {
+    fn needs_source_parity_verification<S: ShardBytes>(&self, shards: &[Option<S>]) -> bool {
         let missing_data_source = shards.iter().take(self.data_shards).any(|shard| shard.is_none());
         let available_shards = shards.iter().filter(|shard| shard.is_some()).count();
         missing_data_source && available_shards > self.data_shards
     }
 
-    fn verify_source_parity(&self, codec: &ReedSolomon, shards: &[Option<Vec<u8>>], needs_verification: bool) -> io::Result<()> {
+    fn verify_source_parity<S: ShardBytes>(
+        &self,
+        codec: &ReedSolomon,
+        shards: &[Option<S>],
+        needs_verification: bool,
+    ) -> io::Result<()> {
         if !needs_verification {
             return Ok(());
         }
@@ -272,7 +286,11 @@ impl ErasureDecodeEngine for RustfsCodecDecodeEngine {
         Ok(RustfsCodecDecodeWorkspace::new(shard_len))
     }
 
-    fn reconstruct_into(&self, shards: &mut [Option<Vec<u8>>], _workspace: &mut Self::Workspace) -> io::Result<&'static str> {
+    fn reconstruct_into<S: ShardBytes>(
+        &self,
+        shards: &mut [Option<S>],
+        _workspace: &mut Self::Workspace,
+    ) -> io::Result<&'static str> {
         if data_shards_complete(shards, self.data_shards) {
             return Ok(GET_RECONSTRUCT_OUTCOME_SKIP_DATA_COMPLETE);
         }
@@ -282,6 +300,12 @@ impl ErasureDecodeEngine for RustfsCodecDecodeEngine {
 
         if let Some(codec) = self.codec()? {
             let needs_source_parity_verification = self.needs_source_parity_verification(shards);
+            // The codec works on plain vectors: hand it the shard bytes (a
+            // move for heap buffers, a copy out of arena blocks) and put the
+            // results back. Reconstruction is the rare path, so the copy is
+            // acceptable.
+            let mut owned: Vec<Option<Vec<u8>>> = shards.iter_mut().map(|slot| slot.take().map(S::into_vec)).collect();
+            let vec_shards = owned.as_mut_slice();
             if needs_source_parity_verification {
                 // Rebuild missing parity together with missing data so the
                 // shard set is complete for `verify`. Rebuilt parity is
@@ -294,13 +318,16 @@ impl ErasureDecodeEngine for RustfsCodecDecodeEngine {
                 // example one missing data shard plus one missing parity
                 // shard) as an inconsistent-source failure.
                 codec
-                    .reconstruct_opt(shards)
+                    .reconstruct_opt(vec_shards)
                     .map_err(|err| io::Error::other(format!("RustFS codec reconstruct failed: {err:?}")))?;
             } else {
                 codec
-                    .reconstruct_data_opt(shards)
+                    .reconstruct_data_opt(vec_shards)
                     .map_err(|err| io::Error::other(format!("RustFS codec reconstruct failed: {err:?}")))?;
             }
+            for (slot, shard) in shards.iter_mut().zip(owned) {
+                *slot = shard.map(S::from);
+            }
             self.verify_source_parity(&codec, shards, needs_source_parity_verification)?;
         }
 
@@ -390,7 +417,11 @@ impl ErasureDecodeEngine for CodecStreamingDecodeEngine {
         }
     }
 
-    fn reconstruct_into(&self, shards: &mut [Option<Vec<u8>>], workspace: &mut Self::Workspace) -> io::Result<&'static str> {
+    fn reconstruct_into<S: ShardBytes>(
+        &self,
+        shards: &mut [Option<S>],
+        workspace: &mut Self::Workspace,
+    ) -> io::Result<&'static str> {
         match (self, workspace) {
             (Self::Legacy(engine), CodecStreamingDecodeWorkspace::Legacy(workspace)) => {
                 engine.reconstruct_into(shards, workspace)

crates/ecstore/src/erasure/codec/mod.rs

@@ -14,4 +14,6 @@
 
 pub(crate) mod bridge;
 pub(crate) mod buffer_pool;
+pub mod shard_buf;
+pub mod shard_emit;
 pub(crate) mod workspace;

crates/ecstore/src/erasure/codec/shard_buf.rs

@@ -0,0 +1,800 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Raw arena memory: the block handle owns a pointer into registered memory
+// (see `ArenaBlock::new` for the contract) and implements `BufMut` the way
+// `Vec<u8>` does.
+#![allow(unsafe_code)]
+
+//! Shard buffers for the erasure read path.
+//!
+//! A [`ShardBuf`] is the owned buffer one erasure shard is read into, verified
+//! in, reconstructed from, and emitted from. It is either a plain `Vec<u8>` or
+//! a block of a registered [`ShardArena`] (memory an RDMA device can DMA from
+//! directly). The decoder never cares which; a consumer that can RDMA-write
+//! straight from the buffer asks for its [`ShardBuf::arena_block`] and takes
+//! the buffer over instead of copying it.
+
+use bytes::BufMut;
+use bytes::buf::UninitSlice;
+use std::fmt;
+use std::ops::{Deref, DerefMut};
+use std::sync::{Arc, OnceLock};
+
+/// A registered memory arena that hands out fixed-size blocks.
+pub trait ShardArena: Send + Sync + 'static {
+    /// Size of the blocks this arena hands out.
+    fn block_len(&self) -> usize;
+    /// A free block, or `None` when the arena is exhausted (the caller then
+    /// falls back to the heap).
+    fn alloc(&self) -> Option<ArenaBlock>;
+    /// Returns a block previously handed out by [`Self::alloc`].
+    fn release(&self, block: &ArenaBlock);
+    /// Identifies the registration a block belongs to, for consumers that
+    /// match blocks against their own registered regions.
+    fn registration_id(&self) -> u64;
+}
+
+/// One block of a [`ShardArena`]: a raw pointer into the arena's registered
+/// memory, returned to the arena on drop.
+pub struct ArenaBlock {
+    arena: Arc<dyn ShardArena>,
+    ptr: *mut u8,
+    /// Byte offset of the block inside the arena's registration.
+    offset: usize,
+    len: usize,
+}
+
+// SAFETY: the block is exclusively owned memory of a `Send + Sync` arena.
+unsafe impl Send for ArenaBlock {}
+unsafe impl Sync for ArenaBlock {}
+
+impl ArenaBlock {
+    /// Builds a block handle. `ptr` must point at `len` writable bytes that
+    /// stay valid until the block is released and must not be aliased by
+    /// another live block.
+    ///
+    /// # Safety
+    ///
+    /// The caller guarantees the pointer contract above.
+    pub unsafe fn new(arena: Arc<dyn ShardArena>, ptr: *mut u8, offset: usize, len: usize) -> Self {
+        Self { arena, ptr, offset, len }
+    }
+
+    /// Offset of the block inside its arena's registration.
+    pub fn offset(&self) -> usize {
+        self.offset
+    }
+
+    /// Capacity of the block.
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len == 0
+    }
+
+    /// The arena's registration id.
+    pub fn registration_id(&self) -> u64 {
+        self.arena.registration_id()
+    }
+
+    fn as_slice(&self, len: usize) -> &[u8] {
+        debug_assert!(len <= self.len);
+        // SAFETY: `ptr` is valid for `self.len` bytes for the block's lifetime
+        // and the block is exclusively owned.
+        unsafe { std::slice::from_raw_parts(self.ptr, len) }
+    }
+
+    fn as_mut_slice(&mut self, len: usize) -> &mut [u8] {
+        debug_assert!(len <= self.len);
+        // SAFETY: as in `as_slice`, with `&mut self` for exclusivity.
+        unsafe { std::slice::from_raw_parts_mut(self.ptr, len) }
+    }
+}
+
+impl Drop for ArenaBlock {
+    fn drop(&mut self) {
+        self.arena.release(self);
+    }
+}
+
+impl fmt::Debug for ArenaBlock {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("ArenaBlock")
+            .field("offset", &self.offset)
+            .field("len", &self.len)
+            .finish()
+    }
+}
+
+/// A [`ShardArena`] over one contiguous registered region split into
+/// fixed-size blocks, with a free list. The region's owner is kept alive by
+/// `_owner` for as long as the arena (and therefore any block) exists.
+pub struct RegisteredSlabArena {
+    base: *mut u8,
+    len: usize,
+    block_len: usize,
+    registration_id: u64,
+    free: std::sync::Mutex<Vec<usize>>,
+    me: std::sync::Weak<RegisteredSlabArena>,
+    _owner: Box<dyn std::any::Any + Send + Sync>,
+}
+
+// SAFETY: the raw base pointer is only used to carve disjoint blocks whose
+// exclusivity is enforced by the free list; the region's owner is `Send + Sync`.
+unsafe impl Send for RegisteredSlabArena {}
+unsafe impl Sync for RegisteredSlabArena {}
+
+impl RegisteredSlabArena {
+    /// Builds an arena over `len` bytes at `base`, in blocks of `block_len`.
+    ///
+    /// # Safety
+    ///
+    /// `base` must point at `len` bytes of writable memory that stay valid and
+    /// unaliased for as long as `owner` is alive; the arena keeps `owner`.
+    pub unsafe fn new(
+        base: *mut u8,
+        len: usize,
+        block_len: usize,
+        registration_id: u64,
+        owner: Box<dyn std::any::Any + Send + Sync>,
+    ) -> Arc<Self> {
+        let blocks = len.checked_div(block_len).unwrap_or(0);
+        Arc::new_cyclic(|me| Self {
+            base,
+            len,
+            block_len,
+            registration_id,
+            free: std::sync::Mutex::new((0..blocks).rev().collect()),
+            me: me.clone(),
+            _owner: owner,
+        })
+    }
+
+    /// Blocks the arena was built with.
+    pub fn block_count(&self) -> usize {
+        self.len.checked_div(self.block_len).unwrap_or(0)
+    }
+
+    /// Blocks currently free.
+    pub fn free_blocks(&self) -> usize {
+        self.free.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).len()
+    }
+}
+
+impl ShardArena for RegisteredSlabArena {
+    fn block_len(&self) -> usize {
+        self.block_len
+    }
+
+    fn alloc(&self) -> Option<ArenaBlock> {
+        let index = self.free.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).pop()?;
+        let me = self.me.upgrade()?;
+        let offset = index * self.block_len;
+        // SAFETY: `offset + block_len <= len` by construction, the block is
+        // exclusively owned until released, and `me` keeps the owner alive.
+        Some(unsafe { ArenaBlock::new(me as Arc<dyn ShardArena>, self.base.add(offset), offset, self.block_len) })
+    }
+
+    fn release(&self, block: &ArenaBlock) {
+        let index = block.offset() / self.block_len;
+        self.free.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).push(index);
+    }
+
+    fn registration_id(&self) -> u64 {
+        self.registration_id
+    }
+}
+
+/// The byte container an erasure shard slot may hold: a plain `Vec<u8>` or a
+/// [`ShardBuf`]. The coder is generic over it so callers that never touch the
+/// arena keep using vectors.
+pub trait ShardBytes: AsRef<[u8]> + AsMut<[u8]> + From<Vec<u8>> + Send + Sync + 'static {
+    fn len(&self) -> usize {
+        self.as_ref().len()
+    }
+
+    fn is_empty(&self) -> bool {
+        self.as_ref().is_empty()
+    }
+
+    fn as_slice(&self) -> &[u8] {
+        self.as_ref()
+    }
+
+    fn as_mut_slice(&mut self) -> &mut [u8] {
+        self.as_mut()
+    }
+
+    /// The bytes as an owned vector (a move for a vector, a copy out of an
+    /// arena block).
+    fn into_vec(self) -> Vec<u8>;
+}
+
+impl ShardBytes for Vec<u8> {
+    fn into_vec(self) -> Vec<u8> {
+        self
+    }
+}
+
+impl ShardBytes for ShardBuf {
+    fn into_vec(self) -> Vec<u8> {
+        ShardBuf::into_vec(self)
+    }
+}
+
+/// A buffer a bitrot reader appends a shard frame to: `Vec<u8>` or [`ShardBuf`].
+pub trait ShardAppend: BufMut + Deref<Target = [u8]> + Default + Send + 'static {
+    fn reserve(&mut self, additional: usize);
+    fn truncate(&mut self, len: usize);
+    fn extend_from_slice(&mut self, bytes: &[u8]);
+    /// Whether the bytes live in a registered arena block.
+    fn lives_in_arena(&self) -> bool {
+        false
+    }
+}
+
+impl ShardAppend for Vec<u8> {
+    fn reserve(&mut self, additional: usize) {
+        Vec::reserve(self, additional)
+    }
+
+    fn truncate(&mut self, len: usize) {
+        Vec::truncate(self, len)
+    }
+
+    fn extend_from_slice(&mut self, bytes: &[u8]) {
+        Vec::extend_from_slice(self, bytes)
+    }
+}
+
+impl ShardAppend for ShardBuf {
+    fn reserve(&mut self, additional: usize) {
+        ShardBuf::reserve(self, additional)
+    }
+
+    fn lives_in_arena(&self) -> bool {
+        self.is_arena()
+    }
+
+    fn truncate(&mut self, len: usize) {
+        ShardBuf::truncate(self, len)
+    }
+
+    fn extend_from_slice(&mut self, bytes: &[u8]) {
+        ShardBuf::extend_from_slice(self, bytes)
+    }
+}
+
+static SHARD_ARENA: OnceLock<Arc<dyn ShardArena>> = OnceLock::new();
+
+/// Installs the process-wide shard arena; only the first installation wins.
+pub fn install_shard_arena(arena: Arc<dyn ShardArena>) -> bool {
+    SHARD_ARENA.set(arena).is_ok()
+}
+
+/// The installed shard arena, if any.
+pub fn shard_arena() -> Option<&'static Arc<dyn ShardArena>> {
+    SHARD_ARENA.get()
+}
+
+enum Storage {
+    Heap(Vec<u8>),
+    Arena { block: ArenaBlock, len: usize },
+}
+
+/// An owned shard buffer: heap or registered arena memory, with the part of
+/// the `Vec<u8>` surface the erasure read path uses.
+pub struct ShardBuf {
+    storage: Storage,
+}
+
+impl Default for ShardBuf {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ShardBuf {
+    pub const fn new() -> Self {
+        Self {
+            storage: Storage::Heap(Vec::new()),
+        }
+    }
+
+    /// An empty buffer with room for `capacity` bytes: an arena block when an
+    /// arena is installed and the block fits, otherwise heap memory.
+    pub fn with_capacity(capacity: usize) -> Self {
+        if let Some(arena) = shard_arena()
+            && capacity <= arena.block_len()
+            && let Some(block) = arena.alloc()
+        {
+            return Self {
+                storage: Storage::Arena { block, len: 0 },
+            };
+        }
+        Self::with_heap_capacity(capacity)
+    }
+
+    /// An empty heap buffer with room for `capacity` bytes.
+    pub fn with_heap_capacity(capacity: usize) -> Self {
+        Self {
+            storage: Storage::Heap(Vec::with_capacity(capacity)),
+        }
+    }
+
+    pub fn from_vec(vec: Vec<u8>) -> Self {
+        Self {
+            storage: Storage::Heap(vec),
+        }
+    }
+
+    /// Whether the bytes live in a registered arena block.
+    pub fn is_arena(&self) -> bool {
+        matches!(self.storage, Storage::Arena { .. })
+    }
+
+    /// The arena block behind the bytes, if any.
+    pub fn arena_block(&self) -> Option<&ArenaBlock> {
+        match &self.storage {
+            Storage::Arena { block, .. } => Some(block),
+            Storage::Heap(_) => None,
+        }
+    }
+
+    /// Takes the arena block out, leaving the buffer empty on the heap.
+    /// Returns `None` (and leaves the buffer untouched) for a heap buffer.
+    pub fn take_arena_block(&mut self) -> Option<(ArenaBlock, usize)> {
+        if !self.is_arena() {
+            return None;
+        }
+        match std::mem::replace(&mut self.storage, Storage::Heap(Vec::new())) {
+            Storage::Arena { block, len } => Some((block, len)),
+            Storage::Heap(_) => unreachable!("checked above"),
+        }
+    }
+
+    /// The bytes as an owned `Vec`, copying out of an arena block.
+    pub fn into_vec(self) -> Vec<u8> {
+        match self.storage {
+            Storage::Heap(vec) => vec,
+            Storage::Arena { block, len } => block.as_slice(len).to_vec(),
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        match &self.storage {
+            Storage::Heap(vec) => vec.len(),
+            Storage::Arena { len, .. } => *len,
+        }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    pub fn capacity(&self) -> usize {
+        match &self.storage {
+            Storage::Heap(vec) => vec.capacity(),
+            Storage::Arena { block, .. } => block.len(),
+        }
+    }
+
+    pub fn clear(&mut self) {
+        match &mut self.storage {
+            Storage::Heap(vec) => vec.clear(),
+            Storage::Arena { len, .. } => *len = 0,
+        }
+    }
+
+    pub fn truncate(&mut self, new_len: usize) {
+        match &mut self.storage {
+            Storage::Heap(vec) => vec.truncate(new_len),
+            Storage::Arena { len, .. } => {
+                if new_len < *len {
+                    *len = new_len;
+                }
+            }
+        }
+    }
+
+    /// Moves the bytes to the heap when the arena block cannot hold
+    /// `required` bytes.
+    fn ensure_capacity(&mut self, required: usize) {
+        if let Storage::Arena { block, len } = &mut self.storage
+            && required > block.len()
+        {
+            let mut vec = Vec::with_capacity(required);
+            vec.extend_from_slice(block.as_slice(*len));
+            self.storage = Storage::Heap(vec);
+        }
+    }
+
+    pub fn reserve(&mut self, additional: usize) {
+        let required = self.len().saturating_add(additional);
+        self.ensure_capacity(required);
+        if let Storage::Heap(vec) = &mut self.storage {
+            vec.reserve(additional);
+        }
+    }
+
+    pub fn reserve_exact(&mut self, additional: usize) {
+        let required = self.len().saturating_add(additional);
+        self.ensure_capacity(required);
+        if let Storage::Heap(vec) = &mut self.storage {
+            vec.reserve_exact(additional);
+        }
+    }
+
+    pub fn extend_from_slice(&mut self, other: &[u8]) {
+        let required = self.len().saturating_add(other.len());
+        self.ensure_capacity(required);
+        match &mut self.storage {
+            Storage::Heap(vec) => vec.extend_from_slice(other),
+            Storage::Arena { block, len } => {
+                let start = *len;
+                block.as_mut_slice(required)[start..].copy_from_slice(other);
+                *len = required;
+            }
+        }
+    }
+
+    pub fn resize(&mut self, new_len: usize, value: u8) {
+        self.ensure_capacity(new_len);
+        match &mut self.storage {
+            Storage::Heap(vec) => vec.resize(new_len, value),
+            Storage::Arena { block, len } => {
+                if new_len > *len {
+                    let start = *len;
+                    block.as_mut_slice(new_len)[start..].fill(value);
+                }
+                *len = new_len;
+            }
+        }
+    }
+
+    pub fn as_slice(&self) -> &[u8] {
+        match &self.storage {
+            Storage::Heap(vec) => vec.as_slice(),
+            Storage::Arena { block, len } => block.as_slice(*len),
+        }
+    }
+
+    pub fn as_mut_slice(&mut self) -> &mut [u8] {
+        match &mut self.storage {
+            Storage::Heap(vec) => vec.as_mut_slice(),
+            Storage::Arena { block, len } => block.as_mut_slice(*len),
+        }
+    }
+
+    /// Splits off the bytes from `at` into a new heap buffer.
+    pub fn split_off(&mut self, at: usize) -> ShardBuf {
+        match &mut self.storage {
+            Storage::Heap(vec) => ShardBuf::from_vec(vec.split_off(at)),
+            Storage::Arena { block, len } => {
+                let tail = block.as_slice(*len)[at..].to_vec();
+                *len = at;
+                ShardBuf::from_vec(tail)
+            }
+        }
+    }
+}
+
+impl Deref for ShardBuf {
+    type Target = [u8];
+
+    fn deref(&self) -> &[u8] {
+        self.as_slice()
+    }
+}
+
+impl DerefMut for ShardBuf {
+    fn deref_mut(&mut self) -> &mut [u8] {
+        self.as_mut_slice()
+    }
+}
+
+impl AsRef<[u8]> for ShardBuf {
+    fn as_ref(&self) -> &[u8] {
+        self.as_slice()
+    }
+}
+
+impl AsMut<[u8]> for ShardBuf {
+    fn as_mut(&mut self) -> &mut [u8] {
+        self.as_mut_slice()
+    }
+}
+
+impl From<Vec<u8>> for ShardBuf {
+    fn from(vec: Vec<u8>) -> Self {
+        Self::from_vec(vec)
+    }
+}
+
+impl From<ShardBuf> for Vec<u8> {
+    fn from(buf: ShardBuf) -> Self {
+        buf.into_vec()
+    }
+}
+
+impl FromIterator<u8> for ShardBuf {
+    fn from_iter<I: IntoIterator<Item = u8>>(iter: I) -> Self {
+        Self::from_vec(iter.into_iter().collect())
+    }
+}
+
+impl Extend<u8> for ShardBuf {
+    fn extend<I: IntoIterator<Item = u8>>(&mut self, iter: I) {
+        for byte in iter {
+            self.extend_from_slice(&[byte]);
+        }
+    }
+}
+
+impl<'a> Extend<&'a u8> for ShardBuf {
+    fn extend<I: IntoIterator<Item = &'a u8>>(&mut self, iter: I) {
+        for byte in iter {
+            self.extend_from_slice(&[*byte]);
+        }
+    }
+}
+
+impl Clone for ShardBuf {
+    /// Clones onto the heap; arena blocks are not duplicated.
+    fn clone(&self) -> Self {
+        Self::from_vec(self.as_slice().to_vec())
+    }
+}
+
+impl PartialEq for ShardBuf {
+    fn eq(&self, other: &Self) -> bool {
+        self.as_slice() == other.as_slice()
+    }
+}
+
+impl PartialEq<Vec<u8>> for ShardBuf {
+    fn eq(&self, other: &Vec<u8>) -> bool {
+        self.as_slice() == other.as_slice()
+    }
+}
+
+impl PartialEq<[u8]> for ShardBuf {
+    fn eq(&self, other: &[u8]) -> bool {
+        self.as_slice() == other
+    }
+}
+
+impl Eq for ShardBuf {}
+
+impl fmt::Debug for ShardBuf {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("ShardBuf")
+            .field("len", &self.len())
+            .field("capacity", &self.capacity())
+            .field("arena", &self.is_arena())
+            .finish()
+    }
+}
+
+// SAFETY: `chunk_mut` hands out the spare capacity and `advance_mut` only
+// moves the length within the initialized capacity, exactly like `Vec<u8>`;
+// readers write into the spare area before advancing.
+unsafe impl BufMut for ShardBuf {
+    fn remaining_mut(&self) -> usize {
+        match &self.storage {
+            Storage::Heap(vec) => vec.remaining_mut(),
+            Storage::Arena { block, len } => block.len() - *len,
+        }
+    }
+
+    unsafe fn advance_mut(&mut self, cnt: usize) {
+        match &mut self.storage {
+            // SAFETY: forwarded to `Vec<u8>`, which upholds the same contract.
+            Storage::Heap(vec) => unsafe { vec.advance_mut(cnt) },
+            Storage::Arena { block, len } => {
+                assert!(*len + cnt <= block.len(), "advance_mut past the arena block");
+                *len += cnt;
+            }
+        }
+    }
+
+    fn chunk_mut(&mut self) -> &mut UninitSlice {
+        match &mut self.storage {
+            Storage::Heap(vec) => vec.chunk_mut(),
+            Storage::Arena { block, len } => {
+                let spare = block.len() - *len;
+                // SAFETY: the spare area is valid, writable memory of the block.
+                unsafe { UninitSlice::from_raw_parts_mut(block.ptr.add(*len), spare) }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::Mutex;
+
+    #[test]
+    fn registered_slab_arena_hands_out_disjoint_blocks_and_takes_them_back() {
+        let mut slab = vec![0u8; 4 * 8];
+        let base = slab.as_mut_ptr();
+        // SAFETY: the slab outlives the arena (it is moved in as the owner below).
+        let arena = unsafe { RegisteredSlabArena::new(base, slab.len(), 8, 42, Box::new(slab)) };
+        assert_eq!(arena.block_count(), 4);
+        let dyn_arena: Arc<dyn ShardArena> = arena.clone();
+        let a = dyn_arena.alloc().unwrap();
+        let b = dyn_arena.alloc().unwrap();
+        assert_ne!(a.offset(), b.offset());
+        assert_eq!(a.registration_id(), 42);
+        assert_eq!(arena.free_blocks(), 2);
+        drop(a);
+        assert_eq!(arena.free_blocks(), 3);
+        let mut buf = ShardBuf {
+            storage: Storage::Arena { block: b, len: 0 },
+        };
+        buf.extend_from_slice(b"12345678");
+        assert_eq!(&*buf, b"12345678");
+        drop(buf);
+        assert_eq!(arena.free_blocks(), 4);
+    }
+
+    /// A test arena over a heap slab.
+    struct SlabArena {
+        slab: Mutex<Vec<u8>>,
+        block_len: usize,
+        free: Mutex<Vec<usize>>,
+        base: *mut u8,
+    }
+
+    unsafe impl Send for SlabArena {}
+    unsafe impl Sync for SlabArena {}
+
+    impl SlabArena {
+        fn new(blocks: usize, block_len: usize) -> Arc<Self> {
+            let mut slab = vec![0u8; blocks * block_len];
+            let base = slab.as_mut_ptr();
+            Arc::new(Self {
+                slab: Mutex::new(slab),
+                block_len,
+                free: Mutex::new((0..blocks).collect()),
+                base,
+            })
+        }
+
+        fn free_blocks(&self) -> usize {
+            self.free.lock().unwrap().len()
+        }
+    }
+
+    impl ShardArena for SlabArena {
+        fn block_len(&self) -> usize {
+            self.block_len
+        }
+
+        fn alloc(&self) -> Option<ArenaBlock> {
+            let index = self.free.lock().unwrap().pop()?;
+            let _keep = self.slab.lock().unwrap();
+            let ptr = unsafe { self.base.add(index * self.block_len) };
+            // SAFETY: the slab outlives every block (the arena is kept alive by
+            // the block's `Arc`) and the free list keeps blocks disjoint.
+            Some(unsafe { ArenaBlock::new(arena_ref(self), ptr, index * self.block_len, self.block_len) })
+        }
+
+        fn release(&self, block: &ArenaBlock) {
+            self.free.lock().unwrap().push(block.offset() / self.block_len);
+        }
+
+        fn registration_id(&self) -> u64 {
+            7
+        }
+    }
+
+    thread_local! {
+        static CURRENT: std::cell::RefCell<Option<Arc<SlabArena>>> = const { std::cell::RefCell::new(None) };
+    }
+
+    fn arena_ref(_arena: &SlabArena) -> Arc<dyn ShardArena> {
+        CURRENT.with(|current| current.borrow().clone().expect("arena set")) as Arc<dyn ShardArena>
+    }
+
+    fn arena_buf(arena: &Arc<SlabArena>, capacity: usize) -> ShardBuf {
+        CURRENT.with(|current| *current.borrow_mut() = Some(Arc::clone(arena)));
+        let block = arena.alloc().expect("free block");
+        assert!(capacity <= block.len());
+        ShardBuf {
+            storage: Storage::Arena { block, len: 0 },
+        }
+    }
+
+    #[test]
+    fn heap_buffer_behaves_like_a_vec() {
+        let mut buf = ShardBuf::with_heap_capacity(8);
+        buf.extend_from_slice(b"hello");
+        buf.resize(7, b'!');
+        assert_eq!(&*buf, b"hello!!");
+        buf.truncate(5);
+        assert_eq!(buf.len(), 5);
+        let tail = buf.split_off(2);
+        assert_eq!(&*buf, b"he");
+        assert_eq!(&*tail, b"llo");
+        buf.clear();
+        assert!(buf.is_empty());
+        assert!(!buf.is_arena());
+    }
+
+    #[test]
+    fn arena_buffer_fills_within_the_block_and_returns_it_on_drop() {
+        let arena = SlabArena::new(2, 16);
+        let mut buf = arena_buf(&arena, 16);
+        assert!(buf.is_arena());
+        assert_eq!(arena.free_blocks(), 1);
+        buf.extend_from_slice(b"0123456789");
+        buf.put_slice(b"abc");
+        assert_eq!(buf.len(), 13);
+        assert_eq!(&buf[10..], b"abc");
+        buf.as_mut_slice()[0] = b'X';
+        assert_eq!(buf[0], b'X');
+        buf.truncate(4);
+        assert_eq!(&*buf, b"X123");
+        buf.resize(6, b'-');
+        assert_eq!(&*buf, b"X123--");
+        assert_eq!(buf.arena_block().map(|block| block.offset()), Some(16));
+        drop(buf);
+        assert_eq!(arena.free_blocks(), 2);
+    }
+
+    #[test]
+    fn arena_buffer_moves_to_the_heap_when_it_outgrows_the_block() {
+        let arena = SlabArena::new(1, 8);
+        let mut buf = arena_buf(&arena, 8);
+        buf.extend_from_slice(b"12345678");
+        assert!(buf.is_arena());
+        buf.extend_from_slice(b"9");
+        assert!(!buf.is_arena());
+        assert_eq!(&*buf, b"123456789");
+        assert_eq!(arena.free_blocks(), 1, "the block went back to the arena");
+    }
+
+    #[test]
+    fn taking_the_block_leaves_an_empty_heap_buffer() {
+        let arena = SlabArena::new(1, 8);
+        let mut buf = arena_buf(&arena, 8);
+        buf.extend_from_slice(b"abcd");
+        let (block, len) = buf.take_arena_block().expect("arena buffer");
+        assert_eq!(len, 4);
+        assert_eq!(block.as_slice(len), b"abcd");
+        assert!(buf.is_empty() && !buf.is_arena());
+        assert!(buf.take_arena_block().is_none());
+        drop(block);
+        assert_eq!(arena.free_blocks(), 1);
+    }
+
+    #[test]
+    fn buf_mut_limit_reads_into_the_spare_area() {
+        let arena = SlabArena::new(1, 8);
+        let mut buf = arena_buf(&arena, 8);
+        buf.extend_from_slice(b"ab");
+        {
+            let mut limited = (&mut buf).limit(3);
+            limited.put_slice(b"cde");
+            assert_eq!(limited.remaining_mut(), 0);
+        }
+        assert_eq!(&*buf, b"abcde");
+        assert_eq!(buf.remaining_mut(), 3);
+    }
+}

crates/ecstore/src/erasure/codec/shard_emit.rs

@@ -0,0 +1,202 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Zero-copy emit: a consumer that can send bytes straight out of a shard
+//! buffer (an RDMA write from registered arena memory) registers a
+//! [`ShardEmitter`] for the decode task; the emit stage then hands it whole
+//! data shards instead of copying them into the byte stream.
+
+use super::shard_buf::ShardBuf;
+use std::cell::RefCell;
+use std::future::Future;
+use std::io;
+use std::ops::Range;
+use std::task::{Context, Poll};
+
+/// Takes ownership of data shards at the point where the decoder would write
+/// them into the byte stream. The bytes `range` of the shard are the next
+/// bytes of the stream, in order; the emitter keeps the buffer until it no
+/// longer needs the memory (for an RDMA write, until the DMA completes).
+pub trait ShardEmitter: Send {
+    /// Returns `Ready(Ok(true))` once the shard was taken (`shard` is then
+    /// `None`), `Ready(Ok(false))` when this shard cannot be taken (the
+    /// caller then writes the bytes through the stream instead), and
+    /// `Pending` while the emitter has no room yet.
+    fn poll_emit_shard(
+        &mut self,
+        cx: &mut Context<'_>,
+        shard: &mut Option<ShardBuf>,
+        range: Range<usize>,
+    ) -> Poll<io::Result<bool>>;
+
+    /// Called at a stripe boundary: an emitter that batches shards submits
+    /// what it holds. `Pending` while it waits for room.
+    fn poll_flush_shards(&mut self, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        Poll::Ready(Ok(()))
+    }
+}
+
+tokio::task_local! {
+    static SHARD_EMITTER: RefCell<Option<Box<dyn ShardEmitter>>>;
+}
+
+/// Runs `future` with `emitter` offered to the decode loop's emit stage.
+pub async fn with_shard_emitter<F: Future>(emitter: Box<dyn ShardEmitter>, future: F) -> F::Output {
+    SHARD_EMITTER.scope(RefCell::new(Some(emitter)), future).await
+}
+
+/// Whether the current task offers a shard emitter.
+pub fn shard_emitter_offered() -> bool {
+    SHARD_EMITTER.try_with(|cell| cell.borrow().is_some()).unwrap_or(false)
+}
+
+/// Offers `shard[range]` to the task's emitter. Resolves to `true` when the
+/// emitter took the shard, `false` when there is no emitter or it declined.
+pub async fn emit_shard(shard: &mut Option<ShardBuf>, range: Range<usize>) -> io::Result<bool> {
+    if !shard_emitter_offered() {
+        return Ok(false);
+    }
+    std::future::poll_fn(|cx| {
+        SHARD_EMITTER
+            .try_with(|cell| match cell.borrow_mut().as_mut() {
+                Some(emitter) => emitter.poll_emit_shard(cx, shard, range.clone()),
+                None => Poll::Ready(Ok(false)),
+            })
+            .unwrap_or(Poll::Ready(Ok(false)))
+    })
+    .await
+}
+
+/// Tells the task's emitter (if any) that a stripe's data shards are complete.
+pub async fn flush_shards() -> io::Result<()> {
+    if !shard_emitter_offered() {
+        return Ok(());
+    }
+    std::future::poll_fn(|cx| {
+        SHARD_EMITTER
+            .try_with(|cell| match cell.borrow_mut().as_mut() {
+                Some(emitter) => emitter.poll_flush_shards(cx),
+                None => Poll::Ready(Ok(())),
+            })
+            .unwrap_or(Poll::Ready(Ok(())))
+    })
+    .await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::{Arc, Mutex};
+
+    type Taken = Arc<Mutex<Vec<(usize, Range<usize>)>>>;
+
+    struct Recording {
+        taken: Taken,
+        accept: bool,
+    }
+
+    struct Flushing {
+        flushes: Arc<Mutex<usize>>,
+    }
+
+    impl ShardEmitter for Flushing {
+        fn poll_emit_shard(
+            &mut self,
+            _cx: &mut Context<'_>,
+            shard: &mut Option<ShardBuf>,
+            _range: Range<usize>,
+        ) -> Poll<io::Result<bool>> {
+            shard.take();
+            Poll::Ready(Ok(true))
+        }
+
+        fn poll_flush_shards(&mut self, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            *self.flushes.lock().unwrap() += 1;
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    impl ShardEmitter for Recording {
+        fn poll_emit_shard(
+            &mut self,
+            _cx: &mut Context<'_>,
+            shard: &mut Option<ShardBuf>,
+            range: Range<usize>,
+        ) -> Poll<io::Result<bool>> {
+            if !self.accept {
+                return Poll::Ready(Ok(false));
+            }
+            let taken = shard.take().expect("shard offered");
+            self.taken.lock().unwrap().push((taken.len(), range));
+            Poll::Ready(Ok(true))
+        }
+    }
+
+    #[tokio::test]
+    async fn emit_without_an_emitter_declines_and_keeps_the_shard() {
+        let mut shard = Some(ShardBuf::from(vec![1u8; 8]));
+        assert!(!emit_shard(&mut shard, 0..8).await.unwrap());
+        assert!(shard.is_some());
+        assert!(!shard_emitter_offered());
+    }
+
+    #[tokio::test]
+    async fn emitter_takes_the_shard_inside_the_scope_only() {
+        let taken = Arc::new(Mutex::new(Vec::new()));
+        let emitter = Box::new(Recording {
+            taken: Arc::clone(&taken),
+            accept: true,
+        });
+        with_shard_emitter(emitter, async {
+            assert!(shard_emitter_offered());
+            let mut shard = Some(ShardBuf::from(vec![7u8; 16]));
+            assert!(emit_shard(&mut shard, 4..16).await.unwrap());
+            assert!(shard.is_none());
+        })
+        .await;
+        assert_eq!(taken.lock().unwrap().as_slice(), &[(16, 4..16)]);
+        assert!(!shard_emitter_offered());
+    }
+
+    #[tokio::test]
+    async fn flush_reaches_the_emitter_only_inside_the_scope() {
+        assert!(flush_shards().await.is_ok());
+        let flushes = Arc::new(Mutex::new(0));
+        let emitter = Box::new(Flushing {
+            flushes: Arc::clone(&flushes),
+        });
+        with_shard_emitter(emitter, async {
+            let mut shard = Some(ShardBuf::from(vec![1u8; 4]));
+            assert!(emit_shard(&mut shard, 0..4).await.unwrap());
+            flush_shards().await.unwrap();
+            flush_shards().await.unwrap();
+        })
+        .await;
+        assert_eq!(*flushes.lock().unwrap(), 2);
+    }
+
+    #[tokio::test]
+    async fn declining_emitter_leaves_the_shard_to_the_stream() {
+        let emitter = Box::new(Recording {
+            taken: Arc::new(Mutex::new(Vec::new())),
+            accept: false,
+        });
+        with_shard_emitter(emitter, async {
+            let mut shard = Some(ShardBuf::from(vec![7u8; 16]));
+            assert!(!emit_shard(&mut shard, 0..16).await.unwrap());
+            assert!(shard.is_some());
+        })
+        .await;
+    }
+}

crates/ecstore/src/erasure/codec/workspace.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::erasure::codec::shard_buf::ShardBuf;
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(crate) struct RustfsCodecDecodeWorkspace {
     shard_len: usize,
@@ -30,23 +32,30 @@ impl RustfsCodecDecodeWorkspace {
     }
 }
 
+/// Buffers kept per shard slot. Depth-1 stripe prefetch reads stripe N+1 while
+/// stripe N is still being emitted, so a slot needs two generations in flight;
+/// with one, every prefetched read allocated (and page-faulted) a fresh shard
+/// buffer and the previous one was dropped on `put`, which at 4 MiB shards
+/// cost more than the overlap saved.
+const BUFFERS_PER_SLOT: usize = 2;
+
 #[derive(Debug, Default)]
 pub(crate) struct ShardBufferPool {
-    buffers: Vec<Option<Vec<u8>>>,
+    buffers: Vec<Vec<ShardBuf>>,
 }
 
 impl ShardBufferPool {
     #[inline]
     pub(crate) fn new(slot_count: usize) -> Self {
         let mut buffers = Vec::with_capacity(slot_count);
-        buffers.resize_with(slot_count, || None);
+        buffers.resize_with(slot_count, Vec::new);
         Self { buffers }
     }
 
     #[inline]
     pub(crate) fn ensure_slots(&mut self, slot_count: usize) {
         if self.buffers.len() < slot_count {
-            self.buffers.resize_with(slot_count, || None);
+            self.buffers.resize_with(slot_count, Vec::new);
         }
     }
 
@@ -61,9 +70,9 @@ impl ShardBufferPool {
     /// exists to reuse an allocation, and memsetting it gives that saving straight
     /// back.
     #[inline]
-    pub(crate) fn take(&mut self, index: usize, len: usize) -> Vec<u8> {
+    pub(crate) fn take(&mut self, index: usize, len: usize) -> ShardBuf {
         self.ensure_slots(index + 1);
-        let mut buf = self.buffers[index].take().unwrap_or_else(|| Vec::with_capacity(len));
+        let mut buf = self.buffers[index].pop().unwrap_or_else(|| ShardBuf::with_capacity(len));
         buf.clear();
         if buf.capacity() < len {
             buf.reserve_exact(len - buf.len());
@@ -72,21 +81,33 @@ impl ShardBufferPool {
     }
 
     #[inline]
-    pub(crate) fn put(&mut self, index: usize, buf: Vec<u8>) {
+    /// Returns `buf` to the slot; beyond `BUFFERS_PER_SLOT` the smallest
+    /// buffer is dropped so a slot never holds more than two generations.
+    pub(crate) fn put(&mut self, index: usize, buf: ShardBuf) {
         self.ensure_slots(index + 1);
-        self.buffers[index] = Some(buf);
+        let slot = &mut self.buffers[index];
+        slot.push(buf);
+        if slot.len() > BUFFERS_PER_SLOT {
+            let smallest = slot
+                .iter()
+                .enumerate()
+                .min_by_key(|(_, buf)| buf.capacity())
+                .map(|(i, _)| i)
+                .unwrap_or(0);
+            slot.swap_remove(smallest);
+        }
     }
 
     #[cfg(test)]
     pub(crate) fn stored_allocation(&self, index: usize) -> Option<(*const u8, usize)> {
         self.buffers
             .get(index)
-            .and_then(|buf| buf.as_ref().map(|buf| (buf.as_ptr(), buf.capacity())))
+            .and_then(|slot| slot.last().map(|buf| (buf.as_ptr(), buf.capacity())))
     }
 
     #[cfg(test)]
     fn stored_capacity(&self, index: usize) -> Option<usize> {
-        self.buffers.get(index).and_then(|buf| buf.as_ref().map(Vec::capacity))
+        self.buffers.get(index).and_then(|slot| slot.last().map(ShardBuf::capacity))
     }
 }
 
@@ -94,6 +115,25 @@ impl ShardBufferPool {
 mod tests {
     use super::*;
 
+    #[test]
+    fn slot_keeps_two_generations_for_prefetch() {
+        let mut pool = ShardBufferPool::new(1);
+        let first = pool.take(0, 16);
+        let second = pool.take(0, 16); // prefetch takes before the first is returned
+        let (first_ptr, second_ptr) = (first.as_ptr(), second.as_ptr());
+        pool.put(0, first);
+        pool.put(0, second);
+        let a = pool.take(0, 16);
+        let b = pool.take(0, 16);
+        let got = [a.as_ptr(), b.as_ptr()];
+        assert!(got.contains(&first_ptr) && got.contains(&second_ptr), "both generations are reused");
+        pool.put(0, a);
+        pool.put(0, b);
+        pool.put(0, ShardBuf::with_heap_capacity(64));
+        assert_eq!(pool.buffers[0].len(), BUFFERS_PER_SLOT, "a third buffer evicts one");
+        assert!(pool.buffers[0].iter().any(|buf| buf.capacity() >= 64), "the larger buffer is kept");
+    }
+
     /// `take` hands out capacity, never length: the caller appends every byte it
     /// will read back. Reusing a slot must keep the allocation and must not memset
     /// it (rustfs/backlog#1159).
@@ -132,7 +172,7 @@ mod tests {
         assert_eq!(workspace.shard_len(), 37);
 
         let mut pool = ShardBufferPool::new(1);
-        pool.put(0, Vec::with_capacity(2));
+        pool.put(0, ShardBuf::with_heap_capacity(2));
         let grown = pool.take(0, 8);
 
         assert_eq!(grown.len(), 0);

crates/ecstore/src/erasure/coding/bitrot.rs

@@ -14,6 +14,7 @@
 
 use crate::disk::error::Error as DiskError;
 use crate::erasure::codec::buffer_pool::{get_ec_buffer, return_ec_buffer};
+use crate::erasure::codec::shard_buf::ShardAppend;
 use pin_project_lite::pin_project;
 use rustfs_utils::{HashAlgorithm, LogThrottle};
 use std::future::poll_fn;
@@ -184,7 +185,8 @@ where
 
         let need = self.hash_algo.size() + want;
         self.read_scratch_block(need, want).await?;
-        let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need])?;
+        let verify = self.verify_scratch(need).await?;
+        let data = &self.buf[self.hash_algo.size()..need];
         if let Some(integrity) = &mut self.integrity {
             integrity.verify(data).await?;
         }
@@ -322,6 +324,143 @@ fn split_and_verify<'a>(hash_algo: &HashAlgorithm, skip_verify: bool, block: &'a
     Ok((data, verify))
 }
 
+/// Smallest frame verified off the decode task (see
+/// `RUSTFS_BITROT_VERIFY_OFFLOAD_MIN_BYTES`), read once.
+fn verify_offload_min_bytes() -> usize {
+    static MIN: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
+    *MIN.get_or_init(|| {
+        rustfs_utils::get_env_usize(
+            rustfs_config::ENV_BITROT_VERIFY_OFFLOAD_MIN_BYTES,
+            rustfs_config::DEFAULT_BITROT_VERIFY_OFFLOAD_MIN_BYTES,
+        )
+    })
+}
+
+fn should_offload_verify(skip_verify: bool, frame_len: usize) -> bool {
+    let min = verify_offload_min_bytes();
+    !skip_verify && min != 0 && frame_len >= min
+}
+
+fn join_error(error: tokio::task::JoinError) -> std::io::Error {
+    std::io::Error::other(error)
+}
+
+/// Verifies an owned frame (`hash || data`) on the blocking pool and returns
+/// the verification time; the caller still owns `frame` through the clone.
+/// Verifies an owned frame (`hash || data`) on the blocking pool and appends
+/// its data to `out` there, so the 4 MiB copy runs beside the hash on a
+/// blocking thread instead of serializing with the other shards of the
+/// stripe on the reader's task.
+async fn verify_and_append_off_thread<B: ShardAppend>(
+    hash_algo: &HashAlgorithm,
+    frame: bytes::Bytes,
+    hash_size: usize,
+    out: &mut B,
+) -> std::io::Result<Duration> {
+    let hash_algo = hash_algo.clone();
+    let mut taken = std::mem::take(out);
+    let (taken, result) = tokio::task::spawn_blocking(move || {
+        let result = split_and_verify(&hash_algo, false, &frame).map(|(_, verify)| verify);
+        if result.is_ok() {
+            taken.extend_from_slice(&frame[hash_size..]);
+        }
+        (taken, result)
+    })
+    .await
+    .map_err(join_error)?;
+    *out = taken;
+    result
+}
+
+/// Verifies a frame spread over `chunks` followed by `tail` (the chunk handoff
+/// path) against the hash in its first `hash_size` bytes.
+fn verify_chunked_frame(
+    hash_algo: &HashAlgorithm,
+    chunks: &[bytes::Bytes],
+    tail: &[u8],
+    hash_size: usize,
+    want: usize,
+) -> std::io::Result<Duration> {
+    let block_chunks = || {
+        chunks
+            .iter()
+            .map(|chunk| chunk.as_ref())
+            .chain((!tail.is_empty()).then_some(tail))
+    };
+    let verify_start = std::time::Instant::now();
+    let actual_hash = hash_algo.hash_encode_slices(block_chunks().scan(hash_size, |skip, chunk| {
+        let start = (*skip).min(chunk.len());
+        *skip -= start;
+        Some(&chunk[start..])
+    }));
+    let verify = verify_start.elapsed();
+    let mut hash_offset = 0;
+    let mut remaining = hash_size;
+    for chunk in block_chunks() {
+        let take = remaining.min(chunk.len());
+        if actual_hash.as_ref()[hash_offset..hash_offset + take] != chunk[..take] {
+            log_hash_mismatch(want);
+            return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, DiskError::FileCorrupt));
+        }
+        hash_offset += take;
+        remaining -= take;
+        if remaining == 0 {
+            break;
+        }
+    }
+    Ok(verify)
+}
+
+impl<R> BitrotReader<R> {
+    /// Verifies the frame held in `self.buf[..need]`, off-thread when it is large.
+    /// Verifies the scratch block and appends its data to `out`; both run on
+    /// the blocking pool when the block is large enough to offload.
+    async fn verify_scratch_appending<B: ShardAppend>(
+        &mut self,
+        need: usize,
+        hash_size: usize,
+        out: &mut B,
+    ) -> std::io::Result<Duration> {
+        if !should_offload_verify(self.skip_verify, need) {
+            let verify = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need]).map(|(_, verify)| verify)?;
+            out.extend_from_slice(&self.buf[hash_size..need]);
+            return Ok(verify);
+        }
+        let hash_algo = self.hash_algo.clone();
+        let buf = std::mem::take(&mut self.buf);
+        let mut taken = std::mem::take(out);
+        let (buf, taken, result) = tokio::task::spawn_blocking(move || {
+            let result = split_and_verify(&hash_algo, false, &buf[..need]).map(|(_, verify)| verify);
+            if result.is_ok() {
+                taken.extend_from_slice(&buf[hash_size..need]);
+            }
+            (buf, taken, result)
+        })
+        .await
+        .map_err(join_error)?;
+        self.buf = buf;
+        *out = taken;
+        result
+    }
+
+    async fn verify_scratch(&mut self, need: usize) -> std::io::Result<Duration> {
+        if !should_offload_verify(self.skip_verify, need) {
+            return split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need]).map(|(_, verify)| verify);
+        }
+        // Hand the scratch buffer to the blocking task and take it back after.
+        let hash_algo = self.hash_algo.clone();
+        let buf = std::mem::take(&mut self.buf);
+        let (buf, result) = tokio::task::spawn_blocking(move || {
+            let result = split_and_verify(&hash_algo, false, &buf[..need]).map(|(_, verify)| verify);
+            (buf, result)
+        })
+        .await
+        .map_err(join_error)?;
+        self.buf = buf;
+        result
+    }
+}
+
 impl<R> BitrotReader<R>
 where
     R: ShardSource,
@@ -337,9 +476,21 @@ where
     /// did not write is ever observable — the zeroing bought nothing.
     ///
     /// On return `out.len()` has grown by exactly the returned count.
-    pub async fn read_appending(&mut self, out: &mut Vec<u8>, want: usize) -> std::io::Result<usize> {
+    pub async fn read_appending<B: ShardAppend>(&mut self, out: &mut B, want: usize) -> std::io::Result<usize> {
         let start = out.len();
+        let arena_in = out.lives_in_arena();
         let count = self.read_appending_frame(out, want).await?;
+        if rustfs_io_metrics::get_stage_metrics_enabled() {
+            // Where the shard buffer lives before and after the frame read: a
+            // buffer that enters in the arena and leaves on the heap was moved
+            // by a reservation larger than its block.
+            let stage = match (arena_in, out.lives_in_arena()) {
+                (true, true) => "bitrot_arena_kept",
+                (true, false) => "bitrot_arena_lost",
+                (false, _) => "bitrot_heap_in",
+            };
+            rustfs_io_metrics::record_get_object_stage_duration("legacy_duplex", stage, 0.0);
+        }
         if let Some(integrity) = &mut self.integrity
             && let Err(error) = integrity.verify(&out[start..]).await
         {
@@ -349,12 +500,15 @@ where
         Ok(count)
     }
 
-    async fn read_appending_frame(&mut self, out: &mut Vec<u8>, want: usize) -> std::io::Result<usize> {
+    async fn read_appending_frame<B: ShardAppend>(&mut self, out: &mut B, want: usize) -> std::io::Result<usize> {
         use bytes::BufMut as _;
         use tokio::io::AsyncReadExt as _;
 
         self.begin_read(want)?;
         out.reserve(want);
+        if rustfs_io_metrics::get_stage_metrics_enabled() && !out.lives_in_arena() {
+            rustfs_io_metrics::record_get_object_stage_duration("legacy_duplex", "bitrot_heap_after_reserve", want as f64);
+        }
         let hash_size = self.hash_algo.size();
 
         // No-hash path: read straight into `out`'s spare capacity. `read_buf`
@@ -379,8 +533,16 @@ where
         let need = hash_size + want;
 
         if let Some(block) = self.inner.try_take_block(need) {
-            let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &block)?;
-            out.extend_from_slice(data);
+            if rustfs_io_metrics::get_stage_metrics_enabled() {
+                rustfs_io_metrics::record_get_object_stage_duration("legacy_duplex", "bitrot_branch_block", block.len() as f64);
+            }
+            let verify = if should_offload_verify(self.skip_verify, block.len()) {
+                verify_and_append_off_thread(&self.hash_algo, block, hash_size, out).await?
+            } else {
+                let verify = split_and_verify(&self.hash_algo, self.skip_verify, &block)?.1;
+                out.extend_from_slice(&block[hash_size..]);
+                verify
+            };
             self.last_verify_duration = verify;
             return Ok(want);
         }
@@ -455,50 +617,44 @@ where
         if handed_off {
             if self.chunks.len() == 1 && self.buf.is_empty() {
                 let block = &self.chunks[0];
-                let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, block)?;
-                out.extend_from_slice(data);
+                let verify = if should_offload_verify(self.skip_verify, block.len()) {
+                    verify_and_append_off_thread(&self.hash_algo, block.clone(), hash_size, out).await?
+                } else {
+                    let verify = split_and_verify(&self.hash_algo, self.skip_verify, block)?.1;
+                    out.extend_from_slice(&block[hash_size..]);
+                    verify
+                };
                 self.last_verify_duration = verify;
                 return Ok(want);
             }
 
-            let block_chunks = || {
-                self.chunks
-                    .iter()
-                    .map(|chunk| chunk.as_ref())
-                    .chain((!self.buf.is_empty()).then_some(self.buf.as_slice()))
-            };
+            if should_offload_verify(self.skip_verify, need) {
+                // Move the chunks, the tail and the output to the blocking task
+                // and take them back: the copy is as long as the hash, and the
+                // stripe's other shards are doing the same on their own threads.
+                let hash_algo = self.hash_algo.clone();
+                let chunks = std::mem::take(&mut self.chunks);
+                let tail = std::mem::take(&mut self.buf);
+                let mut taken = std::mem::take(out);
+                let (chunks, tail, taken, result) = tokio::task::spawn_blocking(move || {
+                    let result = verify_chunked_frame(&hash_algo, &chunks, &tail, hash_size, want);
+                    if result.is_ok() {
+                        append_chunked_data(&mut taken, &chunks, &tail, hash_size);
+                    }
+                    (chunks, tail, taken, result)
+                })
+                .await
+                .map_err(join_error)?;
+                self.chunks = chunks;
+                self.buf = tail;
+                *out = taken;
+                self.last_verify_duration = result?;
+                return Ok(want);
+            }
             if !self.skip_verify {
-                let verify_start = std::time::Instant::now();
-                let actual_hash = self
-                    .hash_algo
-                    .hash_encode_slices(block_chunks().scan(hash_size, |skip, chunk| {
-                        let start = (*skip).min(chunk.len());
-                        *skip -= start;
-                        Some(&chunk[start..])
-                    }));
-                let verify = verify_start.elapsed();
-                let mut hash_offset = 0;
-                let mut remaining = hash_size;
-                for chunk in block_chunks() {
-                    let take = remaining.min(chunk.len());
-                    if actual_hash.as_ref()[hash_offset..hash_offset + take] != chunk[..take] {
-                        log_hash_mismatch(want);
-                        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, DiskError::FileCorrupt));
-                    }
-                    hash_offset += take;
-                    remaining -= take;
-                    if remaining == 0 {
-                        break;
-                    }
-                }
-                self.last_verify_duration = verify;
-            }
-            let mut skip = hash_size;
-            for chunk in block_chunks() {
-                let start = skip.min(chunk.len());
-                skip -= start;
-                out.extend_from_slice(&chunk[start..]);
+                self.last_verify_duration = verify_chunked_frame(&self.hash_algo, &self.chunks, &self.buf, hash_size, want)?;
             }
+            append_chunked_data(out, &self.chunks, &self.buf, hash_size);
             return Ok(want);
         }
 
@@ -506,13 +662,27 @@ where
         // the sink differs (`extend_from_slice` into `out` instead of
         // `copy_from_slice` into a pre-zeroed buffer).
         self.read_scratch_block(need, want).await?;
-        let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need])?;
-        out.extend_from_slice(data);
+        let verify = self.verify_scratch_appending(need, hash_size, out).await?;
         self.last_verify_duration = verify;
         Ok(want)
     }
 }
 
+/// Appends the data of a chunked frame (`hash || data` spread over `chunks`
+/// and `tail`) to `out`.
+fn append_chunked_data<B: ShardAppend>(out: &mut B, chunks: &[bytes::Bytes], tail: &[u8], hash_size: usize) {
+    let mut skip = hash_size;
+    for chunk in chunks
+        .iter()
+        .map(|chunk| chunk.as_ref())
+        .chain((!tail.is_empty()).then_some(tail))
+    {
+        let start = skip.min(chunk.len());
+        skip -= start;
+        out.extend_from_slice(&chunk[start..]);
+    }
+}
+
 pin_project! {
     /// BitrotWriter writes (hash+data) blocks to an async writer.
     pub struct BitrotWriter<W> {

crates/ecstore/src/erasure/coding/decode.rs

@@ -22,6 +22,7 @@ use crate::diagnostics::get::{
 use crate::disk::disk_store::get_object_disk_read_timeout;
 use crate::disk::error::{Error, is_terminal_read_error};
 use crate::disk::error_reduce::reduce_errs;
+use crate::erasure::codec::shard_buf::ShardBuf;
 use crate::erasure::codec::workspace::ShardBufferPool;
 use crate::erasure::coding::{BitrotReader, Erasure};
 use crate::io_support::bitrot::DeferredReaderStripeHandle;
@@ -42,9 +43,9 @@ use tokio::io::AsyncWrite;
 use tokio::io::AsyncWriteExt;
 use tracing::{debug, error, warn};
 
-type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, bool)> + Send + 'a>>;
+type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<ShardBuf, Error>, bool)> + Send + 'a>>;
 type OwnedShardReadFuture<'a, R> =
-    Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, Option<BitrotReader<R>>, bool)> + Send + 'a>>;
+    Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<ShardBuf, Error>, Option<BitrotReader<R>>, bool)> + Send + 'a>>;
 pub(crate) type DeferredReaderReopener<R> = Arc<dyn Fn(usize) -> Option<BitrotReader<R>> + Send + Sync>;
 #[derive(Debug, Default)]
 pub(crate) struct DecodeOutcome {
@@ -330,20 +331,26 @@ async fn read_shard_result<R>(
     index: usize,
     read_cost: ShardReadCost,
     reader: &mut BitrotReader<R>,
-    recycled_buf: Option<Vec<u8>>,
+    recycled_buf: Option<ShardBuf>,
     shard_size: usize,
     data_shards: usize,
     read_timeout: Duration,
     metrics_path: Option<&'static str>,
-) -> (Result<Vec<u8>, Error>, bool)
+) -> (Result<ShardBuf, Error>, bool)
 where
     R: crate::erasure::coding::ShardSource,
 {
     let role = shard_role(index, data_shards);
     // Capacity, not length: `read_appending` writes every byte it returns, so
     // the buffer never needs zeroing first (rustfs/backlog#1159).
-    let mut buf = recycled_buf.unwrap_or_else(|| Vec::with_capacity(shard_size));
+    let mut buf = recycled_buf.unwrap_or_else(|| ShardBuf::with_capacity(shard_size));
     buf.clear();
+    if metrics_path.is_some() {
+        // Where the shard buffer lives, with the requested capacity as the
+        // recorded value so the summary shows it next to the arena block size.
+        let home = if buf.is_arena() { "shard_buf_arena" } else { "shard_buf_heap" };
+        rustfs_io_metrics::record_get_object_stage_duration("legacy_duplex", home, shard_size as f64);
+    }
     let read_start = metrics_path.map(|_| Instant::now());
     let read_result = if read_timeout.is_zero() {
         reader.read_appending(&mut buf, shard_size).await
@@ -416,7 +423,7 @@ fn read_shard<'a, R>(
     index: usize,
     read_cost: ShardReadCost,
     reader: &'a mut Option<BitrotReader<R>>,
-    recycled_buf: Option<Vec<u8>>,
+    recycled_buf: Option<ShardBuf>,
     shard_size: usize,
     data_shards: usize,
     read_timeout: Duration,
@@ -466,7 +473,7 @@ fn read_shard_owned<'a, R>(
     index: usize,
     read_cost: ShardReadCost,
     reader: Option<BitrotReader<R>>,
-    recycled_buf: Option<Vec<u8>>,
+    recycled_buf: Option<ShardBuf>,
     shard_size: usize,
     data_shards: usize,
     read_timeout: Duration,
@@ -867,14 +874,14 @@ where
 
 #[allow(clippy::too_many_arguments)]
 fn record_shard_read_result(
-    shards: &mut [Option<Vec<u8>>],
+    shards: &mut [Option<ShardBuf>],
     errs: &mut [Option<Error>],
     retire_readers: &mut ShardIndexes,
     success: &mut usize,
     successful_costs: &mut ShardReadCostCounts,
     i: usize,
     read_cost: ShardReadCost,
-    result: Result<Vec<u8>, Error>,
+    result: Result<ShardBuf, Error>,
     should_retire: bool,
 ) -> bool {
     match result {
@@ -932,7 +939,7 @@ fn shard_read_hedge_delay(read_timeout: Duration) -> Option<Duration> {
 /// layout from opening every remaining shard at once.  As a read completes the
 /// caller invokes this again, which refills one slot after a failure or a
 /// successful-but-insufficient parity result.
-fn demand_bound_parity_admission_limit(shards: &[Option<Vec<u8>>], active: &[bool], data_shards: usize) -> usize {
+fn demand_bound_parity_admission_limit(shards: &[Option<ShardBuf>], active: &[bool], data_shards: usize) -> usize {
     let missing_data = shards.iter().take(data_shards).filter(|shard| shard.is_none()).count();
     if missing_data == 0 {
         return 0;
@@ -2044,7 +2051,7 @@ where
         }
     }
 
-    pub fn recycle_shards(&mut self, shards: &mut [Option<Vec<u8>>]) {
+    pub fn recycle_shards(&mut self, shards: &mut [Option<ShardBuf>]) {
         for (i, reader) in self.readers.iter().enumerate() {
             if reader.is_some()
                 && let Some(buf) = shards.get_mut(i).and_then(Option::take)
@@ -2098,7 +2105,7 @@ where
 }
 
 /// Get the total length of data blocks
-fn get_data_block_len(shards: &[Option<Vec<u8>>], data_blocks: usize) -> usize {
+fn get_data_block_len(shards: &[Option<ShardBuf>], data_blocks: usize) -> usize {
     let mut size = 0;
     for shard in shards.iter().take(data_blocks).flatten() {
         size += shard.len();
@@ -2124,7 +2131,7 @@ where
 /// Write data blocks from encoded blocks to target, supporting offset and length
 async fn write_data_blocks<W>(
     writer: &mut W,
-    en_blocks: &[Option<Vec<u8>>],
+    en_blocks: &mut [Option<ShardBuf>],
     data_blocks: usize,
     mut offset: usize,
     length: usize,
@@ -2181,8 +2188,10 @@ where
     let mut total_written = 0;
     let mut write_left = length;
 
-    for block_op in &en_blocks[..data_blocks] {
-        let Some(block) = block_op else {
+    let emitter_offered = crate::erasure::codec::shard_emit::shard_emitter_offered();
+    let mut emitted_any = false;
+    for block_op in en_blocks[..data_blocks].iter_mut() {
+        let Some(block) = block_op.as_ref() else {
             let reason = GetObjectFailureReason::ShortRead;
             error!(
                 data_blocks,
@@ -2201,11 +2210,44 @@ where
             continue;
         }
 
-        let block_slice = &block[offset..];
+        let block_len = block.len();
+        let start = offset;
         offset = 0;
 
-        let write_len = write_left.min(block_slice.len());
+        let write_len = write_left.min(block_len - start);
         let write_stage_start = get_stage_timer_if_enabled(rustfs_io_metrics::get_stage_metrics_enabled());
+        // Zero-copy emit: a consumer that can send straight out of registered
+        // arena memory takes the whole shard; the bytes still go out in order.
+        // The three counters (zero duration, one per decision) show in the
+        // stage summary how many shards an emitter saw, how many lived in the
+        // arena and how many it took.
+        if emitter_offered && write_stage_start.is_some() {
+            rustfs_io_metrics::record_get_object_stage_duration("legacy_duplex", "emit_shard_offered", 0.0);
+            if block.is_arena() {
+                rustfs_io_metrics::record_get_object_stage_duration("legacy_duplex", "emit_shard_arena", 0.0);
+            }
+        }
+        if emitter_offered
+            && block.is_arena()
+            && crate::erasure::codec::shard_emit::emit_shard(block_op, start..start + write_len).await?
+        {
+            if write_stage_start.is_some() {
+                rustfs_io_metrics::record_get_object_stage_duration("legacy_duplex", "emit_shard_taken", 0.0);
+            }
+            if let Some(write_stage_start) = write_stage_start {
+                rustfs_io_metrics::record_get_object_duplex_backpressure_duration(write_stage_start.elapsed().as_secs_f64());
+            }
+            emitted_any = true;
+            total_written += write_len;
+            write_left -= write_len;
+            if write_left == 0 {
+                crate::erasure::codec::shard_emit::flush_shards().await?;
+                return Ok(total_written);
+            }
+            continue;
+        }
+        let block = block_op.as_ref().expect("the shard was not taken");
+        let block_slice = &block[start..];
         if let Err(e) = writer.write_all(&block_slice[..write_len]).await {
             if let Some(write_stage_start) = write_stage_start {
                 rustfs_io_metrics::record_get_object_duplex_backpressure_duration(write_stage_start.elapsed().as_secs_f64());
@@ -2243,6 +2285,9 @@ where
         write_left -= write_len;
 
         if write_left == 0 {
+            if emitted_any {
+                crate::erasure::codec::shard_emit::flush_shards().await?;
+            }
             return Ok(total_written);
         }
     }
@@ -2411,7 +2456,7 @@ impl Erasure {
     async fn emit_decoded_stripe<W>(
         &self,
         writer: &mut W,
-        shards: &mut [Option<Vec<u8>>],
+        shards: &mut [Option<ShardBuf>],
         errs: &[Option<Error>],
         block_offset: usize,
         block_length: usize,
@@ -3264,10 +3309,14 @@ mod tests {
 
     #[tokio::test]
     async fn test_write_data_blocks_writes_range_across_blocks() {
-        let blocks = vec![Some(vec![1, 2, 3, 4]), Some(vec![5, 6, 7]), Some(vec![8, 9])];
+        let mut blocks = vec![
+            Some(ShardBuf::from(vec![1, 2, 3, 4])),
+            Some(ShardBuf::from(vec![5, 6, 7])),
+            Some(ShardBuf::from(vec![8, 9])),
+        ];
         let mut out = Vec::new();
 
-        let written = write_data_blocks(&mut out, &blocks, 3, 2, 5).await.unwrap();
+        let written = write_data_blocks(&mut out, &mut blocks, 3, 2, 5).await.unwrap();
 
         assert_eq!(written, 5);
         assert_eq!(out, vec![3, 4, 5, 6, 7]);
@@ -3275,10 +3324,10 @@ mod tests {
 
     #[tokio::test]
     async fn test_write_data_blocks_rejects_short_data_after_offset() {
-        let blocks = vec![Some(vec![1, 2, 3, 4]), Some(vec![5, 6, 7])];
+        let mut blocks = vec![Some(ShardBuf::from(vec![1, 2, 3, 4])), Some(ShardBuf::from(vec![5, 6, 7]))];
         let mut out = Vec::new();
 
-        let err = write_data_blocks(&mut out, &blocks, 2, 3, 5).await.unwrap_err();
+        let err = write_data_blocks(&mut out, &mut blocks, 2, 3, 5).await.unwrap_err();
 
         assert_eq!(err.kind(), ErrorKind::UnexpectedEof);
         assert!(out.is_empty());
@@ -3286,10 +3335,10 @@ mod tests {
 
     #[tokio::test]
     async fn test_write_data_blocks_rejects_invalid_data_block_count() {
-        let blocks = vec![Some(vec![1, 2, 3, 4])];
+        let mut blocks = vec![Some(ShardBuf::from(vec![1, 2, 3, 4]))];
         let mut out = Vec::new();
 
-        let err = write_data_blocks(&mut out, &blocks, 2, 0, 1).await.unwrap_err();
+        let err = write_data_blocks(&mut out, &mut blocks, 2, 0, 1).await.unwrap_err();
 
         assert_eq!(err.kind(), ErrorKind::InvalidInput);
         assert!(out.is_empty());
@@ -3297,10 +3346,10 @@ mod tests {
 
     #[tokio::test]
     async fn test_write_data_blocks_rejects_offset_length_overflow() {
-        let blocks = vec![Some(vec![1, 2, 3, 4])];
+        let mut blocks = vec![Some(ShardBuf::from(vec![1, 2, 3, 4]))];
         let mut out = Vec::new();
 
-        let err = write_data_blocks(&mut out, &blocks, 1, usize::MAX, 1).await.unwrap_err();
+        let err = write_data_blocks(&mut out, &mut blocks, 1, usize::MAX, 1).await.unwrap_err();
 
         assert_eq!(err.kind(), ErrorKind::InvalidInput);
         assert!(out.is_empty());
@@ -3308,10 +3357,10 @@ mod tests {
 
     #[tokio::test]
     async fn test_write_data_blocks_rejects_missing_data_shard_even_when_total_bytes_are_available() {
-        let blocks = vec![None, Some(vec![1, 2, 3, 4])];
+        let mut blocks = vec![None, Some(ShardBuf::from(vec![1, 2, 3, 4]))];
         let mut out = Vec::new();
 
-        let err = write_data_blocks(&mut out, &blocks, 2, 0, 1).await.unwrap_err();
+        let err = write_data_blocks(&mut out, &mut blocks, 2, 0, 1).await.unwrap_err();
 
         assert_eq!(err.kind(), ErrorKind::UnexpectedEof);
         assert!(out.is_empty());
@@ -3319,10 +3368,10 @@ mod tests {
 
     #[tokio::test]
     async fn test_write_data_blocks_propagates_writer_emit_failure() {
-        let blocks = vec![Some(vec![1, 2, 3, 4])];
+        let mut blocks = vec![Some(ShardBuf::from(vec![1, 2, 3, 4]))];
         let mut writer = FailingEmitWriter;
 
-        let err = write_data_blocks(&mut writer, &blocks, 1, 0, 4)
+        let err = write_data_blocks(&mut writer, &mut blocks, 1, 0, 4)
             .await
             .expect_err("writer failure must fail the decoded emit path");
 
@@ -5468,7 +5517,11 @@ mod tests {
             erasure
                 .decode_data_with_reconstruction_verification(&mut shards)
                 .expect("recover the stripe after peer loss");
-            let actual: Vec<u8> = shards[..DATA_SHARDS].iter().flatten().flatten().copied().collect();
+            let actual: Vec<u8> = shards[..DATA_SHARDS]
+                .iter()
+                .flatten()
+                .flat_map(|shard| shard.iter().copied())
+                .collect();
             assert_eq!(actual, data[stripe * BLOCK_SIZE..(stripe + 1) * BLOCK_SIZE]);
         }
         assert_eq!(

crates/ecstore/src/erasure/coding/decode_reader.rs

@@ -1052,9 +1052,9 @@ mod tests {
             Ok(NoopDecodeWorkspace { shard_len })
         }
 
-        fn reconstruct_into(
+        fn reconstruct_into<S: crate::erasure::codec::shard_buf::ShardBytes>(
             &self,
-            _shards: &mut [Option<Vec<u8>>],
+            _shards: &mut [Option<S>],
             _workspace: &mut Self::Workspace,
         ) -> io::Result<&'static str> {
             Ok("noop_called")

crates/ecstore/src/erasure/coding/erasure.rs

@@ -18,6 +18,7 @@
 
 use crate::disk::background_memory::{BackgroundBuffer, Reservation};
 use crate::disk::{BackgroundMemoryBudget, current_resource_unit};
+use crate::erasure::codec::shard_buf::ShardBytes;
 use bytes::{Bytes, BytesMut};
 use reed_solomon_simd;
 use rustfs_erasure_codec::galois_8::ReedSolomon;
@@ -299,10 +300,16 @@ impl LegacyReedSolomonEncoder {
         Ok(())
     }
 
-    fn reconstruct_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+    fn reconstruct_data<S: ShardBytes>(&self, shards: &mut [Option<S>]) -> io::Result<()> {
         if recover_empty_payload_data_shards(shards, self.data_shards, self.parity_shards)? {
             return Ok(());
         }
+        // Every data shard present: nothing to rebuild. The conversions
+        // below would copy each shard out of its arena block and hand back
+        // heap buffers, which also defeats the zero-copy emit.
+        if shards.iter().take(self.data_shards).all(Option::is_some) {
+            return Ok(());
+        }
 
         let shard_len = shards
             .iter()
@@ -358,7 +365,7 @@ impl LegacyReedSolomonEncoder {
             if shard_opt.is_none() && i < self.data_shards {
                 for (restored_index, restored_data) in result.restored_original_iter() {
                     if restored_index == i {
-                        *shard_opt = Some(restored_data.to_vec());
+                        *shard_opt = Some(S::from(restored_data.to_vec()));
                         break;
                     }
                 }
@@ -380,7 +387,7 @@ impl LegacyReedSolomonEncoder {
         Ok(())
     }
 
-    fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+    fn reconstruct<S: ShardBytes>(&self, shards: &mut [Option<S>]) -> io::Result<()> {
         self.reconstruct_data(shards)?;
         self.encode_parity(shards)
     }
@@ -420,7 +427,7 @@ impl LegacyReedSolomonEncoder {
         Ok(true)
     }
 
-    fn encode_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+    fn encode_parity<S: ShardBytes>(&self, shards: &mut [Option<S>]) -> io::Result<()> {
         encode_parity_shards(shards, self.data_shards, self.parity_shards, |shards| self.encode(shards))
     }
 }
@@ -478,21 +485,35 @@ impl ReedSolomonEncoder {
     }
 
     /// Reconstruct missing data shards.
-    pub fn reconstruct_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+    pub fn reconstruct_data<S: ShardBytes>(&self, shards: &mut [Option<S>]) -> io::Result<()> {
         if recover_empty_payload_data_shards(shards, self.data_shards, self.parity_shards)? {
             return Ok(());
         }
+        // Every data shard present: nothing to rebuild. The conversions
+        // below would copy each shard out of its arena block and hand back
+        // heap buffers, which also defeats the zero-copy emit.
+        if shards.iter().take(self.data_shards).all(Option::is_some) {
+            return Ok(());
+        }
 
         if let Some(ref rs) = self.encoder {
-            rs.reconstruct_data(shards)
-                .map_err(|e| io::Error::other(format!("Reed-Solomon reconstruct failed: {e:?}")))
+            // The codec works on plain vectors: a move for vectors, a copy out
+            // of arena blocks (reconstruction is the rare path).
+            let mut owned: Vec<Option<Vec<u8>>> = shards.iter_mut().map(|slot| slot.take().map(S::into_vec)).collect();
+            let result = rs
+                .reconstruct_data(owned.as_mut_slice())
+                .map_err(|e| io::Error::other(format!("Reed-Solomon reconstruct failed: {e:?}")));
+            for (slot, shard) in shards.iter_mut().zip(owned) {
+                *slot = shard.map(S::from);
+            }
+            result
         } else {
             Ok(())
         }
     }
 
     /// Reconstruct missing data shards and regenerate parity shards.
-    pub fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+    pub fn reconstruct<S: ShardBytes>(&self, shards: &mut [Option<S>]) -> io::Result<()> {
         self.reconstruct_data(shards)?;
         self.encode_parity(shards)
     }
@@ -520,7 +541,7 @@ impl ReedSolomonEncoder {
         }
     }
 
-    fn encode_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+    fn encode_parity<S: ShardBytes>(&self, shards: &mut [Option<S>]) -> io::Result<()> {
         encode_parity_shards(shards, self.data_shards, self.parity_shards, |shards| self.encode(shards))
     }
 }
@@ -582,7 +603,12 @@ fn cached_legacy_reed_solomon_in(
     Ok(Arc::new(LegacyReedSolomonEncoder::new(data_shards, parity_shards)?))
 }
 
-fn encode_parity_shards<F>(shards: &mut [Option<Vec<u8>>], data_shards: usize, parity_shards: usize, encode: F) -> io::Result<()>
+fn encode_parity_shards<S: ShardBytes, F>(
+    shards: &mut [Option<S>],
+    data_shards: usize,
+    parity_shards: usize,
+    encode: F,
+) -> io::Result<()>
 where
     F: FnOnce(SmallVec<[&mut [u8]; 16]>) -> io::Result<()>,
 {
@@ -597,12 +623,12 @@ where
 
     let shard_len = shards
         .iter()
-        .find_map(|s| s.as_ref().map(Vec::len))
+        .find_map(|s| s.as_ref().map(ShardBytes::len))
         .ok_or_else(|| io::Error::other("No valid shards found for parity encoding"))?;
 
     for shard in shards.iter_mut().skip(data_shards) {
         if shard.is_none() {
-            *shard = Some(vec![0; shard_len]);
+            *shard = Some(S::from(vec![0; shard_len]));
         }
     }
 
@@ -640,8 +666,8 @@ where
     encode(shard_refs)
 }
 
-fn recover_empty_payload_data_shards(
-    shards: &mut [Option<Vec<u8>>],
+fn recover_empty_payload_data_shards<S: ShardBytes>(
+    shards: &mut [Option<S>],
     data_shards: usize,
     parity_shards: usize,
 ) -> io::Result<bool> {
@@ -667,7 +693,7 @@ fn recover_empty_payload_data_shards(
 
     for shard in shards.iter_mut().take(data_shards) {
         if shard.is_none() {
-            *shard = Some(Vec::new());
+            *shard = Some(S::from(Vec::new()));
         }
     }
     Ok(true)
@@ -978,7 +1004,7 @@ impl Erasure {
     /// # Returns
     /// Ok if reconstruction succeeds, error otherwise.
     #[hotpath::measure(impl_type = "Erasure")]
-    pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+    pub fn decode_data<S: ShardBytes>(&self, shards: &mut [Option<S>]) -> io::Result<()> {
         if self.parity_shards > 0 {
             if self.uses_legacy {
                 if let Some(encoder) = self.legacy_encoder.as_ref() {
@@ -998,7 +1024,7 @@ impl Erasure {
 
     /// Decode and reconstruct missing data shards, then regenerate parity shards.
     #[hotpath::measure(impl_type = "Erasure")]
-    pub fn decode_data_and_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+    pub fn decode_data_and_parity<S: ShardBytes>(&self, shards: &mut [Option<S>]) -> io::Result<()> {
         if self.parity_shards > 0 {
             if self.uses_legacy {
                 if let Some(encoder) = self.legacy_encoder.as_ref() {
@@ -1016,20 +1042,20 @@ impl Erasure {
         Ok(())
     }
 
-    pub(crate) fn decode_data_with_reconstruction_verification(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+    pub(crate) fn decode_data_with_reconstruction_verification<S: ShardBytes>(&self, shards: &mut [Option<S>]) -> io::Result<()> {
         self.decode_data_with_reconstruction_verification_policy(shards, false)
     }
 
-    pub(crate) fn decode_data_with_reconstruction_verification_for_lockstep(
+    pub(crate) fn decode_data_with_reconstruction_verification_for_lockstep<S: ShardBytes>(
         &self,
-        shards: &mut [Option<Vec<u8>>],
+        shards: &mut [Option<S>],
     ) -> io::Result<()> {
         self.decode_data_with_reconstruction_verification_policy(shards, true)
     }
 
-    fn decode_data_with_reconstruction_verification_policy(
+    fn decode_data_with_reconstruction_verification_policy<S: ShardBytes>(
         &self,
-        shards: &mut [Option<Vec<u8>>],
+        shards: &mut [Option<S>],
         require_surplus_source: bool,
     ) -> io::Result<()> {
         let missing_data_source = shards.iter().take(self.data_shards).any(|shard| shard.is_none());
@@ -1045,7 +1071,7 @@ impl Erasure {
                 .iter()
                 .enumerate()
                 .skip(self.data_shards)
-                .filter_map(|(index, shard)| shard.as_ref().map(|shard| (index, shard.clone())))
+                .filter_map(|(index, shard)| shard.as_ref().map(|shard| (index, shard.as_slice().to_vec())))
                 .collect::<Vec<_>>()
         } else {
             Vec::new()
@@ -1063,7 +1089,7 @@ impl Erasure {
                     "missing rebuilt parity shard after read verification",
                 ));
             };
-            if rebuilt != &source {
+            if rebuilt.as_slice() != source.as_slice() {
                 warn!(
                     shard_index = index,
                     data_shards = self.data_shards,
@@ -1077,7 +1103,7 @@ impl Erasure {
         Ok(())
     }
 
-    pub(crate) fn verify_data_and_parity(&self, shards: &[Option<Vec<u8>>]) -> io::Result<bool> {
+    pub(crate) fn verify_data_and_parity<S: ShardBytes>(&self, shards: &[Option<S>]) -> io::Result<bool> {
         let expected_shards = self.total_shard_count();
         if shards.len() != expected_shards {
             return Err(io::Error::other(format!(
@@ -1094,7 +1120,8 @@ impl Erasure {
         let mut shard_len = None;
         for (index, shard) in shards.iter().enumerate() {
             let shard = shard
-                .as_deref()
+                .as_ref()
+                .map(ShardBytes::as_slice)
                 .ok_or_else(|| io::Error::other(format!("missing shard {index} for data/parity verification")))?;
             if let Some(expected_len) = shard_len {
                 if shard.len() != expected_len {
@@ -1868,7 +1895,7 @@ mod tests {
         .expect_err("mismatched non-empty shards must be rejected");
         assert!(err.to_string().contains("inconsistent shard length"));
 
-        let mut no_present_empty_payload = vec![None, None, None];
+        let mut no_present_empty_payload: Vec<Option<Vec<u8>>> = vec![None, None, None];
         assert!(
             !recover_empty_payload_data_shards(&mut no_present_empty_payload, 2, 1)
                 .expect("all-missing empty payload marker should not be synthesized")

crates/ecstore/src/io_support/shard_integrity.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use crate::disk::{DiskAPI, DiskStore};
+use crate::erasure::codec::shard_buf::ShardBytes;
 use bytes::Bytes;
 use futures::{StreamExt, stream::FuturesUnordered};
 use rustfs_filemeta::FileInfo;
@@ -405,10 +406,10 @@ pub(crate) fn invalidate_verified_stripe_records(generation: uuid::Uuid) {
 }
 
 impl PartProofReader {
-    pub(crate) async fn reconstruction_proof(
+    pub(crate) async fn reconstruction_proof<S: ShardBytes>(
         &self,
         stripe: usize,
-        shards: &[Option<Vec<u8>>],
+        shards: &[Option<S>],
     ) -> io::Result<Option<ReconstructionProof>> {
         let mut missing = 0u16;
         for (index, shard) in shards.iter().take(usize::from(self.part.layout.data)).enumerate() {
@@ -620,12 +621,15 @@ pub(crate) struct ReconstructionProof {
 }
 
 impl ReconstructionProof {
-    pub(crate) fn verify(&self, shards: &[Option<Vec<u8>>]) -> io::Result<()> {
+    pub(crate) fn verify<S: ShardBytes>(&self, shards: &[Option<S>]) -> io::Result<()> {
         for index in 0..usize::from(self.part.layout.data) {
             if self.missing & (1 << index) == 0 {
                 continue;
             }
-            let payload = shards.get(index).and_then(Option::as_deref).ok_or_else(corrupt)?;
+            let payload = shards
+                .get(index)
+                .and_then(|shard| shard.as_ref().map(ShardBytes::as_slice))
+                .ok_or_else(corrupt)?;
             let actual = self.part.shard_digest(self.stripe, index, payload).map_err(|_| corrupt())?;
             if self.record.get(index * 32..(index + 1) * 32) != Some(actual.as_slice()) {
                 return Err(corrupt());

crates/ecstore/src/object_api/types.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::*;
+use crate::erasure::codec::shard_emit::ShardEmitter;
 use crate::storage_api_contracts::{
     list::VersionMarker,
     object::{
@@ -1040,10 +1041,192 @@ impl DeleteRecoveryWorkBudget {
     }
 }
 
+/// The byte-stream half of a sink writer.
+pub type GetObjectSinkStream = Box<dyn tokio::io::AsyncWrite + Send + Sync + Unpin + 'static>;
+
+/// The writer a [`GetObjectSink`] hands to the read pipeline: the byte stream
+/// the decoder writes into, plus an optional [`ShardEmitter`] that takes whole
+/// data shards out of registered arena memory instead (zero-copy emit). Both
+/// halves must share one byte position: bytes written through the stream and
+/// shards taken by the emitter arrive in request order.
+pub struct GetObjectSinkWriter {
+    stream: GetObjectSinkStream,
+    emitter: Option<Box<dyn ShardEmitter>>,
+}
+
+impl GetObjectSinkWriter {
+    pub fn new(stream: GetObjectSinkStream) -> Self {
+        Self { stream, emitter: None }
+    }
+
+    pub fn with_emitter(stream: GetObjectSinkStream, emitter: Box<dyn ShardEmitter>) -> Self {
+        Self {
+            stream,
+            emitter: Some(emitter),
+        }
+    }
+
+    /// Splits the writer into its stream and its emitter.
+    pub fn into_parts(self) -> (GetObjectSinkStream, Option<Box<dyn ShardEmitter>>) {
+        (self.stream, self.emitter)
+    }
+}
+
+impl std::fmt::Debug for GetObjectSinkWriter {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("GetObjectSinkWriter")
+            .field("emitter", &self.emitter.is_some())
+            .finish()
+    }
+}
+
+impl tokio::io::AsyncWrite for GetObjectSinkWriter {
+    fn poll_write(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &[u8],
+    ) -> std::task::Poll<std::io::Result<usize>> {
+        std::pin::Pin::new(&mut self.stream).poll_write(cx, buf)
+    }
+
+    fn poll_flush(mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
+        std::pin::Pin::new(&mut self.stream).poll_flush(cx)
+    }
+
+    fn poll_shutdown(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        std::pin::Pin::new(&mut self.stream).poll_shutdown(cx)
+    }
+}
+
+/// Builds the writer for one segment of a fanned-out GET: `(offset, len)` are
+/// relative to the start of the requested range, in bytes.
+pub type GetObjectSinkSegmentFactory = Arc<dyn Fn(u64, u64) -> GetObjectSinkWriter + Send + Sync + 'static>;
+
+/// A sink that can take the body as several concurrently produced segments
+/// (each written at its own offset) instead of one sequential stream.
+#[derive(Clone)]
+pub struct GetObjectSinkSegments {
+    pub factory: GetObjectSinkSegmentFactory,
+    /// Upper bound on concurrent segments; below 2 the sink is sequential.
+    pub max_segments: usize,
+    /// Smallest segment worth its own producer (rounded up to whole erasure
+    /// blocks); shorter requests stay sequential.
+    pub min_segment_bytes: u64,
+}
+
+impl std::fmt::Debug for GetObjectSinkSegments {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("GetObjectSinkSegments")
+            .field("max_segments", &self.max_segments)
+            .field("min_segment_bytes", &self.min_segment_bytes)
+            .finish()
+    }
+}
+
+#[derive(Default)]
+struct GetObjectSinkInner {
+    writer: Option<GetObjectSinkWriter>,
+    segments: Option<GetObjectSinkSegments>,
+    taken: bool,
+}
+
+/// A one-shot destination handed to the GET read pipeline through
+/// [`ObjectOptions::get_object_sink`]; the pipeline takes it exactly once,
+/// either as the sequential writer or (when offered and worthwhile) as a set
+/// of segment writers.
+#[derive(Clone, Default)]
+pub struct GetObjectSink {
+    inner: Arc<std::sync::Mutex<GetObjectSinkInner>>,
+}
+
+impl GetObjectSink {
+    pub fn new(writer: GetObjectSinkWriter) -> Self {
+        Self {
+            inner: Arc::new(std::sync::Mutex::new(GetObjectSinkInner {
+                writer: Some(writer),
+                segments: None,
+                taken: false,
+            })),
+        }
+    }
+
+    /// A sink that also offers segmented delivery.
+    pub fn with_segments(writer: GetObjectSinkWriter, segments: GetObjectSinkSegments) -> Self {
+        Self {
+            inner: Arc::new(std::sync::Mutex::new(GetObjectSinkInner {
+                writer: Some(writer),
+                segments: Some(segments),
+                taken: false,
+            })),
+        }
+    }
+
+    fn lock(&self) -> std::sync::MutexGuard<'_, GetObjectSinkInner> {
+        self.inner.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
+    }
+
+    /// Takes the sequential writer; `None` once the sink has been taken in
+    /// either form (or was never set).
+    pub fn take(&self) -> Option<GetObjectSinkWriter> {
+        let mut inner = self.lock();
+        if inner.taken {
+            return None;
+        }
+        let writer = inner.writer.take()?;
+        inner.taken = true;
+        inner.segments = None;
+        Some(writer)
+    }
+
+    /// The segment offer, without taking the sink.
+    pub fn segments(&self) -> Option<GetObjectSinkSegments> {
+        let inner = self.lock();
+        if inner.taken { None } else { inner.segments.clone() }
+    }
+
+    /// Takes the sink in segmented form; `None` if no segments were offered
+    /// or the sink was already taken.
+    pub fn take_segments(&self) -> Option<GetObjectSinkSegments> {
+        let mut inner = self.lock();
+        if inner.taken {
+            return None;
+        }
+        let segments = inner.segments.take()?;
+        inner.taken = true;
+        inner.writer = None;
+        Some(segments)
+    }
+
+    /// Whether the pipeline took the sink (in either form).
+    pub fn is_taken(&self) -> bool {
+        self.lock().taken
+    }
+}
+
+impl std::fmt::Debug for GetObjectSink {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let inner = self.lock();
+        f.debug_struct("GetObjectSink")
+            .field("taken", &inner.taken)
+            .field("segments", &inner.segments)
+            .finish()
+    }
+}
+
 #[derive(Default, Clone)]
 pub struct ObjectOptions {
     #[doc(hidden)]
     pub delete_recovery_work_budget: Option<Arc<DeleteRecoveryWorkBudget>>,
+    /// Caller-provided destination for a GET body: when set, the legacy
+    /// duplex read pipeline writes the decoded bytes straight into this sink
+    /// instead of a pipe the caller drains, and the returned reader only
+    /// carries the producer's completion. Paths that do not stream through the
+    /// pipeline (inline bodies, cache hits, the codec streaming reader) leave
+    /// the sink untaken, and the caller must then consume the reader as usual.
+    pub get_object_sink: Option<GetObjectSink>,
     // Use the maximum parity (N/2), used when saving server configuration files
     pub max_parity: bool,
     pub mod_time: Option<OffsetDateTime>,

crates/ecstore/src/object_api/write_commit_context.rs

@@ -93,6 +93,8 @@ pub struct WriteLockContext {
 enum WriteLockMode {
     Borrowed(Vec<WriteCommitGuard>),
     OfflineRecovery,
+    /// A PUT into a lockless cache bucket (`crate::bucket::lockless_cache`).
+    LocklessCache,
 }
 
 pub(crate) enum WriteCommitContext {
@@ -100,6 +102,7 @@ pub(crate) enum WriteCommitContext {
     Borrowed(Vec<WriteCommitGuard>),
     Publication,
     OfflineRecovery,
+    LocklessCache,
     #[cfg(test)]
     UnlockedFixture,
 }
@@ -126,6 +129,9 @@ impl WriteCommitContext {
                 Ok(Self::Borrowed(guards.clone()))
             }
             (true, Some(WriteLockMode::OfflineRecovery)) => Ok(Self::OfflineRecovery),
+            (true, Some(WriteLockMode::LocklessCache)) if crate::bucket::lockless_cache::is_lockless_cache_bucket(bucket) => {
+                Ok(Self::LocklessCache)
+            }
             #[cfg(test)]
             (true, None) if !opts.metadata_cache_safe => Ok(Self::UnlockedFixture),
             _ => Err(Error::InvalidArgument(
@@ -184,6 +190,35 @@ impl ObjectOptions {
         self.add_write_commit_guard(&WriteCommitGuard::from_guard(guard, bucket, object));
     }
 
+    /// Commit a PUT into `bucket` without a namespace owner when the server lists
+    /// it as a lockless cache bucket; returns whether the lockless path applies.
+    pub fn use_lockless_cache_write(&mut self, bucket: &str) -> bool {
+        if !crate::bucket::lockless_cache::is_lockless_cache_bucket(bucket) || self.write_lock_context.is_some() {
+            return false;
+        }
+        self.no_lock = true;
+        self.write_lock_context = Some(WriteLockContext {
+            mode: WriteLockMode::LocklessCache,
+        });
+        true
+    }
+
+    /// Return a lockless cache PUT to the locked path (for example once it is
+    /// known to need replication).
+    pub fn drop_lockless_cache_write(&mut self) {
+        if self.is_lockless_cache_write() {
+            self.no_lock = false;
+            self.write_lock_context = None;
+        }
+    }
+
+    pub fn is_lockless_cache_write(&self) -> bool {
+        matches!(
+            self.write_lock_context.as_ref().map(|context| &context.mode),
+            Some(WriteLockMode::LocklessCache)
+        )
+    }
+
     /// Only startup migration may publish without a running namespace owner.
     pub(crate) fn use_offline_recovery_write(&mut self) {
         self.no_lock = true;
@@ -193,3 +228,23 @@ impl ObjectOptions {
         self.write_completion = WriteCompletion::TailDrained;
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn unlisted_bucket_never_takes_the_lockless_write_path() {
+        let mut opts = ObjectOptions::default();
+        assert!(
+            !opts.use_lockless_cache_write("ordinary-bucket"),
+            "only buckets named in the server's lockless list may skip the namespace lock"
+        );
+        assert!(!opts.is_lockless_cache_write());
+        assert!(!opts.no_lock, "a refused lockless request must leave the locked path intact");
+        assert!(matches!(
+            WriteCommitContext::from_options(&opts, "ordinary-bucket", "object", false),
+            Ok(WriteCommitContext::Owned)
+        ));
+    }
+}

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -5622,7 +5622,9 @@ impl SetDisks {
         let part = ObjectPartInfo::unmarshal(&meta)?;
         let integrity = part.integrity.map(Arc::new);
 
+        let recover_started = rustfs_io_metrics::put_stage_timer();
         self.recover_part_transaction(dst_object, write_quorum).await?;
+        rustfs_io_metrics::record_put_object_stage_duration_from("rename_part_recover_before", recover_started);
 
         let src_bucket = Arc::new(src_bucket.to_string());
         let src_object = Arc::new(src_object.to_string());
@@ -5673,7 +5675,9 @@ impl SetDisks {
                 Some(Ok(()))
             }
         });
+        let prepare_started = rustfs_io_metrics::put_stage_timer();
         let prepare_results = join_all(prepare_tasks).await;
+        rustfs_io_metrics::record_put_object_stage_duration_from("rename_part_prepare", prepare_started);
         let prepare_errs = prepare_results
             .into_iter()
             .map(|result| match result {
@@ -5707,7 +5711,9 @@ impl SetDisks {
             })
         });
 
+        let rename_started = rustfs_io_metrics::put_stage_timer();
         let results = join_all(futures).await;
+        rustfs_io_metrics::record_put_object_stage_duration_from("rename_part_rename", rename_started);
         for result in results {
             match result? {
                 Ok(_) => {
@@ -5768,6 +5774,7 @@ impl SetDisks {
             .iter()
             .zip(prepared_disks.iter())
             .all(|(err, disk)| err.is_none() && disk.as_ref().is_some_and(|d| d.is_local()));
+        let settle_started = rustfs_io_metrics::put_stage_timer();
         let committed = if all_prepared_succeeded {
             let settles = prepared_disks.iter().map(|disk| {
                 let disk = disk.clone();
@@ -5794,6 +5801,14 @@ impl SetDisks {
         } else {
             self.recover_part_transaction(&dst_object, write_quorum).await?
         };
+        rustfs_io_metrics::record_put_object_stage_duration_from(
+            if all_prepared_succeeded {
+                "rename_part_settle"
+            } else {
+                "rename_part_recover_after"
+            },
+            settle_started,
+        );
         if !committed {
             let err = DiskError::ErasureWriteQuorum;
             if let Some(context) = quorum_context {

crates/ecstore/src/set_disk/mod.rs

@@ -700,7 +700,16 @@ fn adaptive_duplex_buffer_size(object_size: i64) -> usize {
         _ => 8 * MB,                        // > 256MB: 8MB
     };
     let object_cap = usize::try_from(object_size).ok().filter(|size| *size > 0).unwrap_or(target);
-    target.min(object_cap.max(64 * KB)).min(get_duplex_buffer_size())
+    let configured = get_duplex_buffer_size();
+    // An explicit `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE` above the default raises the
+    // pipe for large objects (so the decoder can run a whole erasure block ahead
+    // of a slow consumer); the default or a smaller value only caps it.
+    let target = if configured > rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE && object_size > 268_435_456 {
+        configured
+    } else {
+        target
+    };
+    target.min(object_cap.max(64 * KB)).min(configured)
 }
 
 // ============================================================================

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -2502,10 +2502,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         crate::hp_guard!("SetDisks::put_object_part");
         let upload_id_path = Self::validated_multipart_upload_path(bucket, object, upload_id, opts.data_movement)?;
 
-        let (fi, _) = match self
+        let upload_check_started = rustfs_io_metrics::put_stage_timer();
+        let upload_check = self
             .check_upload_id_exists_with_opts(bucket, object, upload_id, true, opts)
-            .await
-        {
+            .await;
+        rustfs_io_metrics::record_put_object_stage_duration_from("multipart_set_disk_upload_check", upload_check_started);
+        let (fi, _) = match upload_check {
             Ok(value) => value,
             Err(err @ Error::InvalidUploadID(..)) => {
                 remove_capped_multipart_staging_semaphore(&upload_id_path);
@@ -2706,6 +2708,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                 )))?;
             }
 
+            let integrity_started = rustfs_io_metrics::put_stage_timer();
             let part_integrity = if let Some(integrity) = integrity {
                 Some(
                     integrity
@@ -2718,6 +2721,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                         .await?,
                 )
             } else { None };
+            rustfs_io_metrics::record_put_object_stage_duration_from("multipart_set_disk_integrity_write", integrity_started);
             if shuffle_disks.iter().filter(|disk| disk.is_some()).count() < write_quorum {
                 return Err(Error::ErasureWriteQuorum);
             }
@@ -2790,6 +2794,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
 
             #[cfg(test)]
             pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockAcquire).await;
+            let commit_lock_started = rustfs_io_metrics::put_stage_timer();
             // Capped uploads reacquire the upload-wide write lock for the
             // final durable check and rename. Uncapped uploads retain the
             // concurrent encode path and only serialize the final same-part
@@ -2816,6 +2821,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             // Only guards owned by this operation may outlive its caller. Internal
             // data-movement and borrowed-lock callers retain their synchronous
             // lifetime; a fence by itself cannot transfer the outer lock lease.
+            rustfs_io_metrics::record_put_object_stage_duration_from("multipart_set_disk_commit_lock", commit_lock_started);
             let detach_commit_owner = _upload_commit_guard.is_some()
                 && opts.namespace_lock_fence.is_none()
                 && (opts.bucket_lifecycle_lock_fence.is_none() || opts.bucket_lifecycle_lock_owner.is_some());
@@ -2840,9 +2846,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                 let upload_id_path = commit_upload_id_path;
                 let opts = &commit_opts;
                 let result: Result<PartInfo> = async {
+                    let commit_check_started = rustfs_io_metrics::put_stage_timer();
                     let (commit_fi, _) = commit_set
                         .check_upload_id_exists_with_opts(bucket, object, upload_id, multipart_size_limit.is_some(), opts)
                         .await?;
+                    rustfs_io_metrics::record_put_object_stage_duration_from("multipart_set_disk_commit_upload_check", commit_check_started);
                     let commit_size_limit = multipart_size_limit_from_metadata(&commit_fi.metadata)?;
                     if commit_size_limit != multipart_size_limit {
                         return Err(Error::InvalidArgument(
@@ -2911,6 +2919,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                             achieved: 0,
                         });
                     }
+                    let rename_started = rustfs_io_metrics::put_stage_timer();
                     let rename_result = commit_set
                         .rename_part(
                             &shuffle_disks,
@@ -2929,6 +2938,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                             }),
                         )
                         .await;
+                    rustfs_io_metrics::record_put_object_stage_duration_from("multipart_set_disk_rename_part", rename_started);
                     drop(decommission_capacity_guard);
                     let _ = rename_result?;
                     #[cfg(test)]
@@ -3808,6 +3818,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             }
             rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE);
         }
+        // The upload's pinned ETag algorithm is upload state, not object metadata.
+        rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_ETAG_ALGORITHM);
 
         let mut checksum_type = rustfs_rio::ChecksumType::NONE;
 

crates/ecstore/src/set_disk/ops/object.rs

@@ -617,6 +617,7 @@ use crate::bucket::utils::is_meta_bucketname;
 use crate::bucket::versioning::VersioningApi;
 use crate::disk::DiskAPI;
 use crate::object_api::ScannerPublicationCommitScopeGuard;
+use crate::object_api::{GetObjectSinkSegments, GetObjectSinkWriter};
 use crate::set_disk::coding;
 use crate::set_disk::core::io_primitives::GetCodecStreamingReaderBuildOutcome;
 use crate::set_disk::mem;
@@ -3686,6 +3687,14 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
         let duplex_buffer_size = adaptive_duplex_buffer_size(object_info.size);
         let (rd, wd) = tokio::io::duplex(duplex_buffer_size);
         debug!(bucket, object, duplex_buffer_size, "Created duplex pipe for object data transfer");
+        // A caller-provided sink replaces the pipe's write end: the producer writes
+        // decoded bytes into it, the pipe's read end sees EOF at once, and the
+        // returned reader then only waits for the producer's terminal result.
+        // Only a plain body can bypass the pipe: compressed or encrypted objects
+        // are transformed by readers layered on the pipe's read end.
+        let sink_usable = !object_info.is_compressed() && !object_info.is_encrypted();
+        let caller_sink = opts.get_object_sink.as_ref().filter(|_| sink_usable).cloned();
+        let erasure_block_size = fi.erasure.block_size;
 
         let (producer_terminal_tx, producer_terminal_rx) = tokio::sync::oneshot::channel();
         let (mut reader, offset, length) =
@@ -3694,115 +3703,108 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
         // lookup on the streaming miss path (ODC-16).
         reader.body_source = body_source;
 
+        // A sink that offers segments takes the body as several concurrently
+        // produced, erasure-block-aligned ranges (each segment writer knows its
+        // offset), so one GET keeps several stripes in flight across cores.
+        let segment_ranges = caller_sink.as_ref().and_then(|sink| {
+            let segments = sink.segments()?;
+            let ranges =
+                plan_sink_segments(offset, length, erasure_block_size, segments.max_segments, segments.min_segment_bytes);
+            (ranges.len() >= 2).then_some(ranges)
+        });
+        let downstream = match (caller_sink, segment_ranges) {
+            (Some(sink), Some(ranges)) => match sink.take_segments() {
+                Some(segments) => {
+                    drop(wd);
+                    LegacyProducerDownstream::Segments(segments, ranges)
+                }
+                None => LegacyProducerDownstream::Writer(GetObjectSinkWriter::new(Box::new(wd))),
+            },
+            (Some(sink), None) => match sink.take() {
+                Some(writer) => {
+                    drop(wd);
+                    LegacyProducerDownstream::Writer(writer)
+                }
+                None => LegacyProducerDownstream::Writer(GetObjectSinkWriter::new(Box::new(wd))),
+            },
+            (None, _) => LegacyProducerDownstream::Writer(GetObjectSinkWriter::new(Box::new(wd))),
+        };
+
         // The producer is otherwise detached from the returned reader. Tie its
         // lifetime to the source stream so a cancelled copy (or an abandoned
         // GET) releases in-flight shard opens, response bodies, and the read
-        // lock immediately instead of waiting for a disk timeout.
-        let producer_cancellation = crate::set_disk::get_object_read_cancellation();
+        // lock immediately instead of waiting for a disk timeout. Segment
+        // producers always get a token so one failed segment stops the rest.
+        let mut producer_cancellation = crate::set_disk::get_object_read_cancellation();
+        if producer_cancellation.is_none() && matches!(downstream, LegacyProducerDownstream::Segments(..)) {
+            producer_cancellation = Some(CancellationToken::new());
+        }
         if let Some(cancellation) = producer_cancellation.as_ref() {
             reader.stream = Box::new(ProducerCancellationReader::new(reader.stream, cancellation.clone()));
         }
 
-        // let disks = disks.clone();
-        let bucket = bucket.to_owned();
-        let object = object.to_owned();
-        let set_index = self.set_index;
-        let pool_index = self.pool_index;
-        let skip_verify = opts.skip_verify_bitrot;
-        let suppress_read_repair = opts.suppress_read_repair;
         // The producer runs in a separate Tokio task, so carry the caller's
         // read policy across the task boundary explicitly. Tokio task-local
         // values are not inherited by spawned tasks.
-        let read_policy = crate::set_disk::get_object_read_policy();
-        let erasure_cache = Arc::clone(&self.erasure_cache);
         let (fi, files, disks) = snapshot.into_owned();
-        let producer = async move {
-            let _guard = read_lock_guard;
-            let mut writer = GetObjectDownstreamWriter::new(wd);
-            // Do not wrap the entire read+write pipeline in `disk_read_timeout`.
-            // `get_object_with_fileinfo` also waits on `writer`, so an outer timeout
-            // would incorrectly treat downstream backpressure as disk-read latency.
-            // Disk read timeouts must be enforced at the actual disk I/O operations.
-            let producer_result = tokio::select! {
-                biased;
-                result = crate::set_disk::with_get_object_read_policy(
-                    read_policy,
-                    Self::get_object_with_fileinfo(
-                        &bucket,
-                        &object,
-                        erasure_cache,
-                        offset,
-                        length,
-                        &mut writer,
-                        fi,
-                        files,
-                        &disks,
-                        set_index,
-                        pool_index,
-                        skip_verify,
-                        suppress_read_repair,
-                        false,
-                        false,
-                        GET_OBJECT_PATH_LEGACY_DUPLEX,
-                        object_class.as_str(),
-                        size_bucket,
-                    ),
-                ) => result,
-                _ = async {
-                    if let Some(cancellation) = producer_cancellation.as_ref() {
-                        cancellation.cancelled().await;
-                    } else {
-                        std::future::pending::<()>().await;
-                    }
-                } => Err(Error::OperationCanceled),
-            };
-            if let Err(e) = &producer_result
-                && !matches!(e, Error::OperationCanceled)
-            {
-                let reason = classify_storage_error(e);
-                if reason == GetObjectFailureReason::DownstreamClosed {
-                    debug!(
-                        event = EVENT_SET_DISK_WRITE,
-                        component = LOG_COMPONENT_ECSTORE,
-                        subsystem = LOG_SUBSYSTEM_SET_DISK,
-                        bucket,
-                        object,
-                        pool_index,
-                        set_index,
-                        offset,
-                        requested_length = length,
-                        state = "downstream_closed",
-                        stage = GET_STAGE_EMIT,
-                        reason = reason.as_str(),
-                        error = ?e,
-                        "Set disk object read pipeline stopped after downstream closed"
-                    );
-                } else {
-                    record_get_object_pipeline_failure(GET_STAGE_EMIT, reason);
-                    error!(
-                        event = EVENT_SET_DISK_WRITE,
-                        component = LOG_COMPONENT_ECSTORE,
-                        subsystem = LOG_SUBSYSTEM_SET_DISK,
-                        bucket,
-                        object,
-                        pool_index,
-                        set_index,
-                        offset,
-                        requested_length = length,
-                        skip_verify_bitrot = skip_verify,
-                        state = "read_pipeline_failed",
-                        stage = GET_STAGE_EMIT,
-                        reason = reason.as_str(),
-                        error = ?e,
-                        "Set disk object read pipeline failed"
-                    );
+        let context = Arc::new(LegacyProducerContext {
+            bucket: bucket.to_owned(),
+            object: object.to_owned(),
+            erasure_cache: Arc::clone(&self.erasure_cache),
+            fi,
+            files,
+            disks,
+            set_index: self.set_index,
+            pool_index: self.pool_index,
+            skip_verify: opts.skip_verify_bitrot,
+            suppress_read_repair: opts.suppress_read_repair,
+            read_policy: crate::set_disk::get_object_read_policy(),
+            object_class: object_class.as_str(),
+            size_bucket,
+            cancellation: producer_cancellation,
+        });
+        match downstream {
+            LegacyProducerDownstream::Writer(writer) => {
+                let producer = async move {
+                    let _guard = read_lock_guard;
+                    let result = run_legacy_producer_segment(context, offset, length, writer).await;
+                    let _ = producer_terminal_tx.send(result);
+                };
+                #[cfg(feature = "diagnostic-local-batch")]
+                let producer = crate::diagnostics::batch_read_diagnostics::with_captured_request_context(producer);
+                tokio::spawn(producer);
+            }
+            LegacyProducerDownstream::Segments(segments, ranges) => {
+                let read_lock_guard = Arc::new(read_lock_guard);
+                let mut handles = Vec::with_capacity(ranges.len());
+                for (segment_offset, segment_length) in ranges {
+                    let relative = u64::try_from(segment_offset - offset).unwrap_or(u64::MAX);
+                    let writer = (segments.factory)(relative, u64::try_from(segment_length).unwrap_or(0));
+                    let context = Arc::clone(&context);
+                    let read_lock_guard = Arc::clone(&read_lock_guard);
+                    let producer = async move {
+                        let _guard = read_lock_guard;
+                        run_legacy_producer_segment(context, segment_offset, segment_length, writer).await
+                    };
+                    #[cfg(feature = "diagnostic-local-batch")]
+                    let producer = crate::diagnostics::batch_read_diagnostics::with_captured_request_context(producer);
+                    handles.push(tokio::spawn(producer));
                 }
-            };
-            let _ = producer_terminal_tx.send(producer_result.map(|_| ()));
-        };
-        #[cfg(feature = "diagnostic-local-batch")]
-        let producer = crate::diagnostics::batch_read_diagnostics::with_captured_request_context(producer);
-        tokio::spawn(producer);
+                tokio::spawn(async move {
+                    let mut result = Ok(());
+                    for handle in handles {
+                        let segment_result = match handle.await {
+                            Ok(segment_result) => segment_result,
+                            Err(join) => Err(Error::other(format!("legacy GET segment producer task failed: {join}"))),
+                        };
+                        if result.is_ok() && segment_result.is_err() {
+                            result = segment_result;
+                        }
+                    }
+                    let _ = producer_terminal_tx.send(result);
+                });
+            }
+        }
 
         Ok(reader)
     }
@@ -3814,6 +3816,248 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
     }
 }
 
+/// Where a legacy duplex GET producer writes: the pipe or caller sink as one
+/// sequential writer, or the caller's segment writers for a fanned-out body.
+enum LegacyProducerDownstream {
+    Writer(GetObjectSinkWriter),
+    Segments(GetObjectSinkSegments, Vec<(usize, i64)>),
+}
+
+/// Everything a legacy duplex producer needs besides its range and writer;
+/// shared by the segment producers of one GET.
+struct LegacyProducerContext {
+    bucket: String,
+    object: String,
+    erasure_cache: Arc<super::super::ErasureCache>,
+    fi: FileInfo,
+    files: Vec<FileInfo>,
+    disks: Vec<Option<DiskStore>>,
+    set_index: usize,
+    pool_index: usize,
+    skip_verify: bool,
+    suppress_read_repair: bool,
+    read_policy: super::super::GetObjectReadPolicy,
+    object_class: &'static str,
+    size_bucket: &'static str,
+    cancellation: Option<CancellationToken>,
+}
+
+/// Runs one legacy duplex producer over `[offset, offset + length)` into
+/// `downstream`, with the caller's read policy and cancellation. A failing
+/// segment cancels its siblings through the shared token.
+async fn run_legacy_producer_segment(
+    context: Arc<LegacyProducerContext>,
+    offset: usize,
+    length: i64,
+    downstream: GetObjectSinkWriter,
+) -> Result<()> {
+    let (stream, emitter) = downstream.into_parts();
+    let mut writer = GetObjectDownstreamWriter::new(stream);
+    // Do not wrap the entire read+write pipeline in `disk_read_timeout`.
+    // `get_object_with_fileinfo` also waits on `writer`, so an outer timeout
+    // would incorrectly treat downstream backpressure as disk-read latency.
+    // Disk read timeouts must be enforced at the actual disk I/O operations.
+    // A sink that can take shards out of arena memory (zero-copy emit) is
+    // offered to the decode loop for the whole read.
+    let read = crate::set_disk::with_get_object_read_policy(
+        context.read_policy,
+        SetDisks::get_object_with_fileinfo(
+            &context.bucket,
+            &context.object,
+            Arc::clone(&context.erasure_cache),
+            offset,
+            length,
+            &mut writer,
+            context.fi.clone(),
+            context.files.clone(),
+            &context.disks,
+            context.set_index,
+            context.pool_index,
+            context.skip_verify,
+            context.suppress_read_repair,
+            false,
+            false,
+            GET_OBJECT_PATH_LEGACY_DUPLEX,
+            context.object_class,
+            context.size_bucket,
+        ),
+    );
+    let read = async move {
+        match emitter {
+            Some(emitter) => crate::erasure::codec::shard_emit::with_shard_emitter(emitter, read).await,
+            None => read.await,
+        }
+    };
+    let producer_result = tokio::select! {
+        biased;
+        result = read => result,
+        _ = async {
+            if let Some(cancellation) = context.cancellation.as_ref() {
+                cancellation.cancelled().await;
+            } else {
+                std::future::pending::<()>().await;
+            }
+        } => Err(Error::OperationCanceled),
+    };
+    if let Err(e) = &producer_result
+        && !matches!(e, Error::OperationCanceled)
+    {
+        if let Some(cancellation) = context.cancellation.as_ref() {
+            cancellation.cancel();
+        }
+        let reason = classify_storage_error(e);
+        if reason == GetObjectFailureReason::DownstreamClosed {
+            debug!(
+                event = EVENT_SET_DISK_WRITE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_SET_DISK,
+                bucket = context.bucket,
+                object = context.object,
+                pool_index = context.pool_index,
+                set_index = context.set_index,
+                offset,
+                requested_length = length,
+                state = "downstream_closed",
+                stage = GET_STAGE_EMIT,
+                reason = reason.as_str(),
+                error = ?e,
+                "Set disk object read pipeline stopped after downstream closed"
+            );
+        } else {
+            record_get_object_pipeline_failure(GET_STAGE_EMIT, reason);
+            error!(
+                event = EVENT_SET_DISK_WRITE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_SET_DISK,
+                bucket = context.bucket,
+                object = context.object,
+                pool_index = context.pool_index,
+                set_index = context.set_index,
+                offset,
+                requested_length = length,
+                skip_verify_bitrot = context.skip_verify,
+                state = "read_pipeline_failed",
+                stage = GET_STAGE_EMIT,
+                reason = reason.as_str(),
+                error = ?e,
+                "Set disk object read pipeline failed"
+            );
+        }
+    }
+    producer_result.map(|_| ())
+}
+
+/// Splits `[offset, offset + length)` into at most `max_segments` contiguous
+/// ranges cut only at erasure-block boundaries (multiples of `block_size`
+/// from the object start), so no stripe is read by two segments, and no
+/// segment shorter than `min_segment_bytes` rounded up to whole blocks.
+/// Returns a single range when the request does not span two such segments
+/// or segmentation is off.
+fn plan_sink_segments(
+    offset: usize,
+    length: i64,
+    block_size: usize,
+    max_segments: usize,
+    min_segment_bytes: u64,
+) -> Vec<(usize, i64)> {
+    let Ok(length_usize) = usize::try_from(length) else {
+        return vec![(offset, length)];
+    };
+    if max_segments < 2 || block_size == 0 || length_usize == 0 {
+        return vec![(offset, length)];
+    }
+    let end = offset.saturating_add(length_usize);
+    let first_block = offset / block_size + 1;
+    let last_block = end.div_ceil(block_size);
+    let pieces = last_block.saturating_sub(first_block) + 1;
+    let min_blocks = usize::try_from(min_segment_bytes.div_ceil(block_size as u64))
+        .unwrap_or(usize::MAX)
+        .max(1);
+    let segments = max_segments.min(pieces / min_blocks);
+    if segments < 2 {
+        return vec![(offset, length)];
+    }
+    let mut ranges = Vec::with_capacity(segments);
+    let mut start = offset;
+    for index in 0..segments {
+        let boundary_block = first_block + (pieces * (index + 1)) / segments - 1;
+        let stop = if index + 1 == segments {
+            end
+        } else {
+            (boundary_block * block_size).min(end)
+        };
+        if stop > start {
+            ranges.push((start, (stop - start) as i64));
+        }
+        start = stop;
+    }
+    ranges
+}
+
+#[cfg(test)]
+mod sink_segment_plan_tests {
+    use super::plan_sink_segments;
+
+    const BLOCK: usize = 48 << 20;
+
+    #[test]
+    fn single_block_or_segmentation_off_stays_whole() {
+        assert_eq!(plan_sink_segments(0, BLOCK as i64, BLOCK, 8, 0), vec![(0, BLOCK as i64)]);
+        assert_eq!(plan_sink_segments(10, 100, BLOCK, 8, 0), vec![(10, 100)]);
+        assert_eq!(plan_sink_segments(0, (4 * BLOCK) as i64, BLOCK, 1, 0), vec![(0, (4 * BLOCK) as i64)]);
+        assert_eq!(plan_sink_segments(0, (4 * BLOCK) as i64, 0, 8, 0), vec![(0, (4 * BLOCK) as i64)]);
+        assert_eq!(plan_sink_segments(0, -1, BLOCK, 8, 0), vec![(0, -1)]);
+    }
+
+    #[test]
+    fn whole_object_splits_on_block_boundaries_and_covers_everything() {
+        let total = 2usize << 30; // 2 GiB: 42 full blocks and a 32 MiB tail
+        let ranges = plan_sink_segments(0, total as i64, BLOCK, 8, 0);
+        assert_eq!(ranges.len(), 8);
+        let mut next = 0usize;
+        for (start, len) in &ranges {
+            assert_eq!(*start, next);
+            assert!(*len > 0);
+            next += *len as usize;
+            if next != total {
+                assert_eq!(next % BLOCK, 0, "segment ends on a block boundary");
+            }
+        }
+        assert_eq!(next, total);
+    }
+
+    #[test]
+    fn range_inside_object_cuts_only_at_block_boundaries() {
+        let offset = BLOCK + 1000;
+        let length = (3 * BLOCK) as i64;
+        let ranges = plan_sink_segments(offset, length, BLOCK, 8, 0);
+        assert_eq!(ranges.len(), 4);
+        assert_eq!(ranges[0], (offset, (BLOCK - 1000) as i64));
+        assert_eq!(ranges[1], (2 * BLOCK, BLOCK as i64));
+        assert_eq!(ranges[3], (4 * BLOCK, 1000));
+    }
+
+    #[test]
+    fn minimum_segment_size_limits_the_count_and_keeps_short_ranges_whole() {
+        // 11 blocks at 4 blocks per segment: two segments, not eight.
+        let ranges = plan_sink_segments(0, (11 * BLOCK) as i64, BLOCK, 8, (4 * BLOCK) as u64);
+        assert_eq!(ranges.len(), 2);
+        assert_eq!(ranges[0], (0, (5 * BLOCK) as i64));
+        assert_eq!(ranges[1], (5 * BLOCK, (6 * BLOCK) as i64));
+        // Fewer than two minimum-size segments: sequential.
+        assert_eq!(plan_sink_segments(0, (7 * BLOCK) as i64, BLOCK, 8, (4 * BLOCK) as u64).len(), 1);
+        // The minimum rounds up to whole blocks.
+        assert_eq!(plan_sink_segments(0, (4 * BLOCK) as i64, BLOCK, 8, (BLOCK + 1) as u64).len(), 2);
+    }
+
+    #[test]
+    fn segment_count_is_bounded_and_blocks_spread_evenly() {
+        let ranges = plan_sink_segments(0, (10 * BLOCK) as i64, BLOCK, 4, 0);
+        let lens: Vec<usize> = ranges.iter().map(|(_, len)| *len as usize / BLOCK).collect();
+        assert_eq!(lens, vec![2, 3, 2, 3]);
+    }
+}
+
 /// `ReplicationState::target_delete_marker_version_ids` is skipped by the
 /// positional `FileInfo` wire form, so a remote disk would otherwise receive a
 /// delete with an empty map and lose the exact per-target version. Copy it into

crates/ecstore/src/set_disk/read.rs

@@ -2772,7 +2772,12 @@ async fn setup_multipart_part_readers(
         parity_shards,
         BitrotReaderSetupMode::ReadQuorum,
         prefer_data_blocks_first,
-        None,
+        rustfs_io_metrics::get_stage_metrics_enabled().then_some(BitrotReaderStageMetrics {
+            path: GET_OBJECT_PATH_LEGACY_DUPLEX,
+            reader_construction_stage: GET_STAGE_READER_TASK_READER_CONSTRUCTION,
+            file_open_stage: GET_STAGE_READER_TASK_FILE_OPEN,
+            bitrot_reader_init_stage: GET_STAGE_READER_TASK_BITROT_READER_INIT,
+        }),
         Some(BitrotReaderSetupAttribution {
             path: metrics_path,
             object_class: metrics_object_class,

crates/ecstore/src/set_disk/shard_source.rs

@@ -16,6 +16,7 @@ use crate::diagnostics::get::{
     GET_SHARD_READ_COST_LOCAL, GET_SHARD_READ_COST_REMOTE, GET_SHARD_READ_COST_SAME_NODE, GET_SHARD_READ_COST_UNKNOWN,
 };
 use crate::disk::error::Error;
+use crate::erasure::codec::shard_buf::ShardBuf;
 use crate::layout::disks_layout::MAX_ERASURE_SET_DRIVE_COUNT;
 use smallvec::SmallVec;
 use std::sync::atomic::{AtomicU8, Ordering};
@@ -23,7 +24,7 @@ use std::sync::atomic::{AtomicU8, Ordering};
 /// Generic codec callers may exceed the production set limit; `SmallVec` then
 /// spills without changing slot semantics.
 pub(crate) const INLINE_SHARD_SLOTS: usize = MAX_ERASURE_SET_DRIVE_COUNT;
-pub(crate) type ShardBuffers = SmallVec<[Option<Vec<u8>>; INLINE_SHARD_SLOTS]>;
+pub(crate) type ShardBuffers = SmallVec<[Option<ShardBuf>; INLINE_SHARD_SLOTS]>;
 pub(crate) type ShardErrors = SmallVec<[Option<Error>; INLINE_SHARD_SLOTS]>;
 
 /// Repair evidence is independent of reconstruction quorum. Successful shards
@@ -92,7 +93,7 @@ pub(crate) struct StripeReadState {
 impl StripeReadState {
     #[cfg(test)]
     pub(crate) fn from_parts(shards: Vec<Option<Vec<u8>>>, errors: Vec<Option<Error>>, read_quorum: usize) -> Self {
-        let mut shards = SmallVec::from_vec(shards);
+        let mut shards: ShardBuffers = shards.into_iter().map(|shard| shard.map(ShardBuf::from)).collect();
         let mut errors = SmallVec::from_vec(errors);
         let slot_count = shards.len().max(errors.len());
         shards.resize_with(slot_count, || None);
@@ -170,7 +171,7 @@ impl StripeReadState {
     }
 
     #[cfg(test)]
-    pub(crate) fn scratch_storage(&self) -> (*const Option<Vec<u8>>, *const Option<Error>, bool, bool) {
+    pub(crate) fn scratch_storage(&self) -> (*const Option<ShardBuf>, *const Option<Error>, bool, bool) {
         (self.shards.as_ptr(), self.errors.as_ptr(), self.shards.spilled(), self.errors.spilled())
     }
 
@@ -209,7 +210,7 @@ mod tests {
 
     #[test]
     fn stripe_scratch_capacity_matches_the_production_set_limit() {
-        type OversizedShardBuffers = SmallVec<[Option<Vec<u8>>; 32]>;
+        type OversizedShardBuffers = SmallVec<[Option<ShardBuf>; 32]>;
         type OversizedShardErrors = SmallVec<[Option<Error>; 32]>;
 
         assert_eq!(INLINE_SHARD_SLOTS, MAX_ERASURE_SET_DRIVE_COUNT);
@@ -234,7 +235,11 @@ mod tests {
 
         assert!(!state.can_decode());
         let (shards, errors) = state.into_parts();
-        assert_eq!(shards.as_slice(), &[Some(vec![1, 2, 3]), None]);
+        let shards: Vec<Option<Vec<u8>>> = shards
+            .iter()
+            .map(|shard| shard.as_ref().map(|shard| shard.to_vec()))
+            .collect();
+        assert_eq!(shards, vec![Some(vec![1, 2, 3]), None]);
         assert_eq!(errors.as_slice(), &[None, Some(Error::FileCorrupt)]);
     }
 

crates/ecstore/src/store/mod.rs

@@ -411,7 +411,12 @@ async fn enqueue_transition_after_write(
             // still sees the object if that commit never reached this hook.
             if !oi.multipart_completion_replayed && should_enqueue_transition_immediately(&oi) {
                 enqueue_transition_immediate(&oi, src.clone()).await;
-                if let Ok(api) = metadata_sys::object_store_in(&store.ctx).await {
+                // Immediate expiry needs the bucket incarnation under the
+                // metadata-transaction lock; a lockless cache PUT leaves expiry
+                // to the scanner.
+                if !opts.is_lockless_cache_write()
+                    && let Ok(api) = metadata_sys::object_store_in(&store.ctx).await
+                {
                     enqueue_immediate_expiry(api, &oi, src, opts).await;
                 }
             }

crates/ecstore/src/store/object.rs

@@ -4830,7 +4830,11 @@ impl ECStore {
         if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() {
             opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?);
         }
-        if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
+        // A lockless cache bucket has no Object Lock, quota or versioning (the app
+        // layer checked the cached state), so the snapshot's lifecycle and
+        // metadata-transaction locks and its authoritative metadata read guard
+        // nothing it relies on.
+        if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) && !opts.is_lockless_cache_write() {
             if opts.object_lock_config_snapshot.is_none() && !opts.no_lock && !opts.data_movement {
                 // The request body is client-paced. Acquire configuration and
                 // lifecycle guards in SetDisks only once its staging has finished.

crates/filemeta/src/fileinfo.rs

@@ -34,6 +34,45 @@ use uuid::Uuid;
 pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
 pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
 
+/// Bucket holding RustFS's own metadata objects; they always use `BLOCK_SIZE_V2`.
+const SYSTEM_META_BUCKET_PREFIX: &str = ".rustfs.sys/";
+
+static CONFIGURED_ERASURE_SHARD_BYTES: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
+
+/// Shard size override from `RUSTFS_ERASURE_SHARD_BYTES` (`0` = none), read once.
+pub fn configured_erasure_shard_bytes() -> usize {
+    *CONFIGURED_ERASURE_SHARD_BYTES.get_or_init(|| {
+        let raw = rustfs_utils::get_env_usize(rustfs_config::ENV_ERASURE_SHARD_BYTES, rustfs_config::DEFAULT_ERASURE_SHARD_BYTES);
+        if raw == 0 {
+            return 0;
+        }
+        if !raw.is_multiple_of(4096) || raw > rustfs_config::MAX_ERASURE_SHARD_BYTES {
+            tracing::warn!(
+                setting = rustfs_config::ENV_ERASURE_SHARD_BYTES,
+                value = raw,
+                "ignoring erasure shard size: must be a multiple of 4096 and at most {} bytes",
+                rustfs_config::MAX_ERASURE_SHARD_BYTES
+            );
+            return 0;
+        }
+        raw
+    })
+}
+
+/// Erasure block for a new object: `data_blocks x` the configured shard size,
+/// or `BLOCK_SIZE_V2` when no shard size is configured or the object is system
+/// metadata.
+pub fn erasure_block_size_for(object: &str, data_blocks: usize) -> usize {
+    erasure_block_size_with(configured_erasure_shard_bytes(), object, data_blocks)
+}
+
+fn erasure_block_size_with(shard_bytes: usize, object: &str, data_blocks: usize) -> usize {
+    if shard_bytes == 0 || data_blocks == 0 || object.starts_with(SYSTEM_META_BUCKET_PREFIX) {
+        return BLOCK_SIZE_V2;
+    }
+    shard_bytes.saturating_mul(data_blocks)
+}
+
 pub(crate) const MAX_ERASURE_SHARDS: usize = 16;
 const MAX_FILEINFO_PARTS: usize = 10_000;
 const MAX_FILEINFO_CHECKSUMS: usize = 10_000;
@@ -804,7 +843,7 @@ impl FileInfo {
                 algorithm: String::from(ERASURE_ALGORITHM),
                 data_blocks,
                 parity_blocks,
-                block_size: BLOCK_SIZE_V2,
+                block_size: erasure_block_size_for(object, data_blocks),
                 distribution: indices,
                 ..Default::default()
             },
@@ -1588,6 +1627,22 @@ pub fn is_restored_object_on_disk(meta: &HashMap<String, String>) -> bool {
 
 #[cfg(test)]
 mod tests {
+    #[test]
+    fn configured_shard_bytes_derive_the_block_and_keep_shards_aligned() {
+        let shard = 4 * 1024 * 1024;
+        assert_eq!(super::erasure_block_size_with(shard, "bucket/object", 12), 48 * 1024 * 1024);
+        assert_eq!(calc_shard_size(super::erasure_block_size_with(shard, "bucket/object", 12), 12), shard);
+        assert_eq!(super::erasure_block_size_with(shard, "bucket/object", 14), 56 * 1024 * 1024);
+        assert_eq!(calc_shard_size(super::erasure_block_size_with(shard, "bucket/object", 14), 14), shard);
+    }
+
+    #[test]
+    fn unset_shard_bytes_and_system_metadata_keep_the_legacy_block() {
+        assert_eq!(super::erasure_block_size_with(0, "bucket/object", 12), BLOCK_SIZE_V2);
+        assert_eq!(super::erasure_block_size_with(4 * 1024 * 1024, ".rustfs.sys/pool.bin", 2), BLOCK_SIZE_V2);
+        assert_eq!(super::erasure_block_size_with(4 * 1024 * 1024, "bucket/object", 0), BLOCK_SIZE_V2);
+    }
+
     use super::*;
     use proptest::collection::{hash_map, vec};
     use proptest::prelude::*;

crates/io-metrics/src/lib.rs

@@ -2408,6 +2408,10 @@ pub fn record_put_object_stage_duration(stage: &'static str, duration_ms: f64) {
     if !put_stage_metrics_enabled() {
         return;
     }
+    if get_stage_local_summary_enabled() {
+        record_get_stage_local_summary("put", stage, "", "", duration_ms / 1000.0);
+        return;
+    }
     histogram!("rustfs_s3_put_object_stage_duration_ms", "stage" => stage).record(duration_ms);
 }
 
@@ -2646,6 +2650,10 @@ pub fn record_put_rename_quorum_wait_fanout(
 /// operations that are NOT part of the PUT object hot path.
 #[inline(always)]
 pub fn record_stage_duration(stage: &'static str, duration_ms: f64) {
+    if get_stage_local_summary_enabled() {
+        record_get_stage_local_summary("internal", stage, "", "", duration_ms / 1000.0);
+        return;
+    }
     if !metrics_enabled() {
         return;
     }

crates/object-data-cache/src/cache.rs

@@ -164,6 +164,12 @@ impl ObjectDataCache {
         self
     }
 
+    /// Largest body a fill can accept (`max_entry_bytes` clamped by the fill
+    /// ceiling); lets a producer avoid buffering a body that could never fit.
+    pub fn fill_size_ceiling_bytes(&self) -> u64 {
+        self.effective_size_ceiling()
+    }
+
     /// Effective size above which a body is planned `SkipTooLarge`:
     /// `max_entry_bytes` clamped by the fill ceiling when one is set.
     fn effective_size_ceiling(&self) -> u64 {
@@ -488,6 +494,11 @@ impl ObjectDataCache {
         matches!(self.config.mode, crate::config::ObjectDataCacheMode::FillMaterializeEnabled)
     }
 
+    /// Whether the configured mode accepts any fill (buffered or materialized).
+    pub fn fill_enabled(&self) -> bool {
+        self.config.fill_enabled()
+    }
+
     /// Publishes the cache-state gauge (and mirrors it into stats), debounced so
     /// it fires at most once per [`ENTRY_COUNT_PUBLISH_DEBOUNCE_MS`]. Moka's
     /// `entry_count`/`weighted_size` are approximations that only settle after

crates/rio-v2/src/lib.rs

@@ -23,6 +23,7 @@ pub use compress_reader::{CompressReader, DecompressReader};
 pub use encrypt_reader::{DecryptReader, EncryptReader, derive_part_key};
 pub use rustfs_rio::DEFAULT_ENCRYPTION_BLOCK_SIZE;
 pub use rustfs_rio::DynReader;
+pub use rustfs_rio::EtagAlgorithm;
 pub use rustfs_rio::EtagReader;
 pub use rustfs_rio::EtagResolvable;
 pub use rustfs_rio::HardLimitReader;

crates/rio/src/etag_reader.rs

@@ -14,15 +14,59 @@
 
 use crate::compress_index::{Index, TryGetIndex};
 use crate::{BadDigest, EtagResolvable, HashReaderDetector, HashReaderMut};
+use bytes::Bytes;
 use hmac::{Hmac, KeyInit, Mac};
 use pin_project_lite::pin_project;
 use rustfs_utils::FastMd5;
 use sha2::Sha256;
+use std::future::Future;
 use std::pin::Pin;
-use std::task::{Context, Poll};
+use std::task::{Context, Poll, ready};
 use tokio::io::{AsyncRead, ReadBuf};
+use tokio::sync::{mpsc, oneshot};
+use tokio_util::sync::PollSender;
 use tracing::error;
 
+/// Chunks queued for the offloaded MD5 before the reader waits for it.
+const MD5_OFFLOAD_QUEUE: usize = 4;
+
+/// The ETag MD5 running on a blocking-pool thread: the reader copies every
+/// chunk it returns into the queue and collects the digest at EOF.
+enum Md5Offload {
+    /// Requested; the hasher starts on the first read, inside the runtime.
+    Pending,
+    Running {
+        chunks: PollSender<Bytes>,
+        digest: oneshot::Receiver<[u8; 16]>,
+    },
+    /// The body ended; waiting for the hasher to finish the queued chunks.
+    Draining { digest: oneshot::Receiver<[u8; 16]> },
+}
+
+impl Md5Offload {
+    /// Starts the hasher, or `None` outside a Tokio runtime (hash inline then).
+    fn start() -> Option<Self> {
+        let handle = tokio::runtime::Handle::try_current().ok()?;
+        let (chunks, mut queue) = mpsc::channel::<Bytes>(MD5_OFFLOAD_QUEUE);
+        let (done, digest) = oneshot::channel();
+        handle.spawn_blocking(move || {
+            let mut md5 = FastMd5::new();
+            while let Some(chunk) = queue.blocking_recv() {
+                md5.update(&chunk);
+            }
+            let _ = done.send(md5.finalize());
+        });
+        Some(Self::Running {
+            chunks: PollSender::new(chunks),
+            digest,
+        })
+    }
+}
+
+fn md5_offload_failed() -> std::io::Error {
+    std::io::Error::other("offloaded ETag MD5 stopped before the body ended")
+}
+
 pin_project! {
     pub struct  EtagReader<R> {
         #[pin]
@@ -33,6 +77,8 @@ pin_project! {
         resolved_etag: Option<String>,
         encrypted_etag_mac: Option<Hmac<Sha256>>,
         delegate_inner_etag: bool,
+        md5_offload: Option<Md5Offload>,
+        xxh3: Option<Box<xxhash_rust::xxh3::Xxh3>>,
     }
 }
 
@@ -46,9 +92,35 @@ impl<R> EtagReader<R> {
             resolved_etag: None,
             encrypted_etag_mac: None,
             delegate_inner_etag: false,
+            md5_offload: None,
+            xxh3: None,
         }
     }
 
+    /// Derive the ETag from XXH3-128 instead of MD5, in the same 32-hex-digit
+    /// form. Only for a plain ETag the client does not check: with an
+    /// expected Content-MD5, an encrypted or a delegated ETag the reader
+    /// keeps MD5.
+    pub fn with_xxh3_etag(mut self, enable: bool) -> Self {
+        if enable && self.checksum.is_none() && self.encrypted_etag_mac.is_none() && !self.delegate_inner_etag {
+            self.xxh3 = Some(Box::new(xxhash_rust::xxh3::Xxh3::new()));
+            self.md5_offload = None;
+        }
+        self
+    }
+
+    /// Compute the MD5 on a blocking-pool thread while the body streams, so
+    /// it overlaps the consumer's work instead of running inline on every
+    /// read. Each returned chunk is copied once into the hasher's queue; EOF
+    /// is reported only after the digest is in, so a Content-MD5 mismatch
+    /// still fails the read. Plain ETags only: encrypted or delegated ETags
+    /// keep hashing inline.
+    pub fn with_md5_offload(mut self, offload: bool) -> Self {
+        self.md5_offload =
+            (offload && self.encrypted_etag_mac.is_none() && !self.delegate_inner_etag).then_some(Md5Offload::Pending);
+        self
+    }
+
     /// Protect the content fingerprint with the object's encryption material.
     /// Plaintext checksum verification remains the inner reader's responsibility.
     pub fn new_encrypted(inner: R, key: [u8; 32], nonce: [u8; 12]) -> std::io::Result<Self> {
@@ -78,8 +150,10 @@ impl<R> EtagReader<R> {
             return etag.clone();
         }
 
-        let digest = self.md5.finalize_bytes();
-        let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
+        let etag = match &self.xxh3 {
+            Some(xxh3) => hex_simd::encode_to_string(xxh3.digest128().to_be_bytes(), hex_simd::AsciiCase::Lower),
+            None => hex_simd::encode_to_string(self.md5.finalize_bytes(), hex_simd::AsciiCase::Lower),
+        };
         self.resolved_etag = Some(etag.clone());
         etag
     }
@@ -94,45 +168,94 @@ where
         if *this.finished {
             return Poll::Ready(Ok(()));
         }
+        if matches!(this.md5_offload, Some(Md5Offload::Pending)) {
+            *this.md5_offload = Md5Offload::start();
+        }
 
-        let orig_filled = buf.filled().len();
-        let poll = this.inner.as_mut().poll_read(cx, buf);
-        if let Poll::Ready(Ok(())) = &poll {
-            let filled = &buf.filled()[orig_filled..];
-            if !filled.is_empty() {
-                if !*this.delegate_inner_etag {
-                    this.md5.update(filled);
+        let digest = match this.md5_offload {
+            Some(Md5Offload::Running { chunks, .. }) => {
+                // Reserve the queue slot first, so a filled chunk always has room.
+                ready!(chunks.poll_reserve(cx)).map_err(|_| md5_offload_failed())?;
+                let orig_filled = buf.filled().len();
+                ready!(this.inner.as_mut().poll_read(cx, buf))?;
+                let filled = &buf.filled()[orig_filled..];
+                if !filled.is_empty() {
+                    chunks
+                        .send_item(Bytes::copy_from_slice(filled))
+                        .map_err(|_| md5_offload_failed())?;
+                    return Poll::Ready(Ok(()));
                 }
-            } else {
-                // EOF
-                *this.finished = true;
-                if let Some(checksum) = this.checksum {
-                    let etag = if let Some(etag) = this.resolved_etag.as_ref() {
-                        etag.clone()
-                    } else {
-                        let digest = this.md5.finalize_bytes();
-                        let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
-                        *this.resolved_etag = Some(etag.clone());
-                        etag
-                    };
-
-                    if *checksum != etag {
-                        error!("Checksum mismatch, expected={:?}, actual={:?}", checksum, etag);
-                        return Poll::Ready(Err(std::io::Error::new(
-                            std::io::ErrorKind::InvalidData,
-                            BadDigest {
-                                expected_md5: checksum.clone(),
-                                calculated_md5: etag,
-                            },
-                        )));
+                // EOF: close the queue and wait for the hasher.
+                let Some(Md5Offload::Running { digest, .. }) = this.md5_offload.take() else {
+                    unreachable!("matched the running offload above");
+                };
+                *this.md5_offload = Some(Md5Offload::Draining { digest });
+                let Some(Md5Offload::Draining { digest }) = this.md5_offload else {
+                    unreachable!("just set");
+                };
+                ready!(Pin::new(digest).poll(cx)).map_err(|_| md5_offload_failed())?
+            }
+            Some(Md5Offload::Draining { digest }) => ready!(Pin::new(digest).poll(cx)).map_err(|_| md5_offload_failed())?,
+            _ => {
+                let orig_filled = buf.filled().len();
+                let poll = this.inner.as_mut().poll_read(cx, buf);
+                if let Poll::Ready(Ok(())) = &poll {
+                    let filled = &buf.filled()[orig_filled..];
+                    if !filled.is_empty() {
+                        if let Some(xxh3) = this.xxh3.as_mut() {
+                            xxh3.update(filled);
+                        } else if !*this.delegate_inner_etag {
+                            this.md5.update(filled);
+                        }
+                        return poll;
+                    }
+                    // EOF
+                    *this.finished = true;
+                    if let Some(checksum) = this.checksum {
+                        let etag = if let Some(etag) = this.resolved_etag.as_ref() {
+                            etag.clone()
+                        } else {
+                            let digest = this.md5.finalize_bytes();
+                            let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
+                            *this.resolved_etag = Some(etag.clone());
+                            etag
+                        };
+                        if let Err(err) = check_digest(checksum, etag) {
+                            return Poll::Ready(Err(err));
+                        }
                     }
                 }
+                return poll;
             }
+        };
+
+        // The offloaded digest is in: the body ended.
+        *this.md5_offload = None;
+        *this.finished = true;
+        let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
+        *this.resolved_etag = Some(etag.clone());
+        if let Some(checksum) = this.checksum {
+            check_digest(checksum, etag)?;
         }
-        poll
+        Poll::Ready(Ok(()))
     }
 }
 
+/// Fails with `BadDigest` when the body's MD5 is not the client's Content-MD5.
+fn check_digest(expected: &str, etag: String) -> std::io::Result<()> {
+    if expected == etag {
+        return Ok(());
+    }
+    error!("Checksum mismatch, expected={:?}, actual={:?}", expected, etag);
+    Err(std::io::Error::new(
+        std::io::ErrorKind::InvalidData,
+        BadDigest {
+            expected_md5: expected.to_owned(),
+            calculated_md5: etag,
+        },
+    ))
+}
+
 impl<R> EtagResolvable for EtagReader<R>
 where
     R: EtagResolvable,
@@ -195,6 +318,91 @@ mod tests {
     use std::io::Cursor;
     use tokio::io::{AsyncReadExt, BufReader};
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn offloaded_md5_matches_inline_md5_across_many_reads() {
+        let mut data = vec![0u8; 3 * 1024 * 1024 + 517];
+        rand::rng().fill(&mut data[..]);
+        let expected = hex_simd::encode_to_string(Md5::digest(&data), hex_simd::AsciiCase::Lower);
+
+        // Small BufReader reads give the hasher many chunks to keep in order.
+        let reader = BufReader::with_capacity(4096, Cursor::new(data.clone()));
+        let mut etag_reader = EtagReader::new(reader, None).with_md5_offload(true);
+        let mut out = Vec::new();
+        etag_reader
+            .read_to_end(&mut out)
+            .await
+            .expect("offloaded read should succeed");
+
+        assert_eq!(out, data);
+        assert_eq!(etag_reader.try_resolve_etag(), Some(expected.clone()));
+        assert_eq!(etag_reader.get_etag(), expected);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn offloaded_md5_still_rejects_a_wrong_content_md5_at_eof() {
+        let data = vec![7u8; 256 * 1024];
+        let wrong = hex_simd::encode_to_string(Md5::digest(b"something else"), hex_simd::AsciiCase::Lower);
+        let reader = BufReader::with_capacity(8192, Cursor::new(data));
+        let mut etag_reader = EtagReader::new(reader, Some(wrong)).with_md5_offload(true);
+        let mut out = Vec::new();
+        let err = etag_reader
+            .read_to_end(&mut out)
+            .await
+            .expect_err("a wrong Content-MD5 must fail the read");
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+        assert!(
+            err.get_ref().and_then(|source| source.downcast_ref::<BadDigest>()).is_some(),
+            "the mismatch should surface as BadDigest"
+        );
+    }
+
+    #[tokio::test]
+    async fn xxh3_etag_is_the_xxh3_128_of_the_body() {
+        let mut data = vec![0u8; 1024 * 1024 + 33];
+        rand::rng().fill(&mut data[..]);
+        let expected = hex_simd::encode_to_string(xxhash_rust::xxh3::xxh3_128(&data).to_be_bytes(), hex_simd::AsciiCase::Lower);
+        let md5 = hex_simd::encode_to_string(Md5::digest(&data), hex_simd::AsciiCase::Lower);
+
+        let reader = BufReader::with_capacity(4096, Cursor::new(data.clone()));
+        let mut etag_reader = EtagReader::new(reader, None).with_xxh3_etag(true);
+        let mut out = Vec::new();
+        etag_reader.read_to_end(&mut out).await.expect("xxh3 read should succeed");
+
+        assert_eq!(out, data);
+        let etag = etag_reader.try_resolve_etag().expect("finished reader resolves its ETag");
+        assert_eq!(etag, expected);
+        assert_eq!(etag.len(), 32, "same width as an MD5 ETag");
+        assert_ne!(etag, md5);
+    }
+
+    #[tokio::test]
+    async fn xxh3_etag_keeps_md5_when_the_client_sent_content_md5() {
+        let data = b"payload with a Content-MD5".to_vec();
+        let md5 = hex_simd::encode_to_string(Md5::digest(&data), hex_simd::AsciiCase::Lower);
+        let mut etag_reader = EtagReader::new(Cursor::new(data), Some(md5.clone())).with_xxh3_etag(true);
+        assert!(etag_reader.xxh3.is_none());
+        let mut out = Vec::new();
+        etag_reader
+            .read_to_end(&mut out)
+            .await
+            .expect("matching Content-MD5 should pass");
+        assert_eq!(etag_reader.try_resolve_etag(), Some(md5));
+
+        let wrong = hex_simd::encode_to_string(Md5::digest(b"other"), hex_simd::AsciiCase::Lower);
+        let mut etag_reader = EtagReader::new(Cursor::new(b"payload".to_vec()), Some(wrong)).with_xxh3_etag(true);
+        let mut out = Vec::new();
+        assert!(etag_reader.read_to_end(&mut out).await.is_err(), "a wrong Content-MD5 must still fail");
+    }
+
+    #[tokio::test]
+    async fn md5_offload_is_ignored_for_encrypted_etags() {
+        let reader = Cursor::new(b"payload".to_vec());
+        let etag_reader = EtagReader::new_encrypted(reader, [1u8; 32], [2u8; 12])
+            .expect("encrypted reader")
+            .with_md5_offload(true);
+        assert!(etag_reader.md5_offload.is_none());
+    }
+
     #[tokio::test]
     async fn encrypted_etags_hide_plaintext_fingerprints_including_empty_objects() {
         for data in [b"".as_slice(), b"Salary 2026: 85000 EUR".as_slice()] {

crates/rio/src/hash_reader.rs

@@ -100,10 +100,48 @@ use std::io::Cursor;
 use std::io::Write;
 use std::mem;
 use std::pin::Pin;
+use std::sync::OnceLock;
 use std::task::{Context, Poll};
 use tokio::io::{AsyncRead, ReadBuf};
 use tracing::error;
 
+/// Whether a body of `size` bytes hashes its ETag MD5 off the request's task
+/// (`RUSTFS_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES`, read once).
+fn md5_offload_for(size: i64) -> bool {
+    static MIN_BYTES: OnceLock<u64> = OnceLock::new();
+    let min_bytes = *MIN_BYTES.get_or_init(|| {
+        rustfs_utils::get_env_u64(
+            rustfs_config::ENV_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES,
+            rustfs_config::DEFAULT_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES,
+        )
+    });
+    min_bytes > 0 && u64::try_from(size).is_ok_and(|size| size >= min_bytes)
+}
+
+/// Hash behind an object's ETag when the client sent no Content-MD5. A body
+/// with Content-MD5 always hashes with MD5 so the check stays exact.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub enum EtagAlgorithm {
+    /// The S3 convention.
+    #[default]
+    Md5,
+    /// XXH3-128 in the same 32-hex-digit form, for buckets whose clients never
+    /// treat the ETag as an MD5 (per-bucket `etag.json`).
+    Xxh3,
+}
+
+/// The ETag reader for a body of `size` bytes: XXH3 when requested and the
+/// client sent no Content-MD5, otherwise MD5, hashed off the request's task
+/// when the body is large.
+fn etag_reader<R>(inner: R, md5hex: &Option<String>, size: i64, algorithm: EtagAlgorithm) -> EtagReader<R> {
+    let reader = EtagReader::new(inner, md5hex.clone());
+    if md5hex.is_none() && algorithm == EtagAlgorithm::Xxh3 {
+        reader.with_xxh3_etag(true)
+    } else {
+        reader.with_md5_offload(md5_offload_for(size))
+    }
+}
+
 /// Trait for mutable operations on HashReader
 pub trait HashReaderMut {
     fn into_inner(self) -> DynReader;
@@ -163,12 +201,12 @@ impl HashReader {
         let inner = if size > 0 {
             let hard_limit_reader = HardLimitReader::new(inner, size);
             if !diskable_md5 {
-                boxed_reader(EtagReader::new(hard_limit_reader, md5hex.clone()))
+                boxed_reader(etag_reader(hard_limit_reader, &md5hex, size, EtagAlgorithm::Md5))
             } else {
                 boxed_reader(hard_limit_reader)
             }
         } else if size != Self::SIZE_PRESERVE_LAYER && !diskable_md5 {
-            boxed_reader(EtagReader::new(inner, md5hex.clone()))
+            boxed_reader(etag_reader(inner, &md5hex, size, EtagAlgorithm::Md5))
         } else {
             boxed_reader(inner)
         };
@@ -198,18 +236,35 @@ impl HashReader {
         sha256hex: Option<String>,
         diskable_md5: bool,
     ) -> std::io::Result<Self>
+    where
+        R: crate::ReadStream + 'static,
+    {
+        Self::from_stream_with_etag(inner, size, actual_size, md5hex, sha256hex, diskable_md5, EtagAlgorithm::Md5)
+    }
+
+    /// [`HashReader::from_stream`] with the hash behind the ETag of a body
+    /// that carries no Content-MD5.
+    pub fn from_stream_with_etag<R>(
+        inner: R,
+        size: i64,
+        actual_size: i64,
+        md5hex: Option<String>,
+        sha256hex: Option<String>,
+        diskable_md5: bool,
+        etag: EtagAlgorithm,
+    ) -> std::io::Result<Self>
     where
         R: crate::ReadStream + 'static,
     {
         let inner = WarpReader::new(inner);
         let inner = if size > 0 {
             if !diskable_md5 {
-                boxed_reader(EtagReader::new(HardLimitReader::new(inner, size), md5hex.clone()))
+                boxed_reader(etag_reader(HardLimitReader::new(inner, size), &md5hex, size, etag))
             } else {
                 boxed_reader(HardLimitReader::new(inner, size))
             }
         } else if size != Self::SIZE_PRESERVE_LAYER && !diskable_md5 {
-            boxed_reader(EtagReader::new(inner, md5hex.clone()))
+            boxed_reader(etag_reader(inner, &md5hex, size, etag))
         } else {
             boxed_reader(inner)
         };
@@ -306,11 +361,11 @@ impl HashReader {
                 inner = Box::new(hr);
 
                 if !diskable_md5 && !inner.is_hash_reader() {
-                    let er = EtagReader::new(inner, md5hex.clone());
+                    let er = etag_reader(inner, &md5hex, size, EtagAlgorithm::Md5);
                     inner = Box::new(er);
                 }
             } else if size != Self::SIZE_PRESERVE_LAYER && !diskable_md5 {
-                let er = EtagReader::new(inner, md5hex.clone());
+                let er = etag_reader(inner, &md5hex, size, EtagAlgorithm::Md5);
                 inner = Box::new(er);
             }
 
@@ -670,6 +725,40 @@ mod tests {
     use std::io::Cursor;
     use tokio::io::{AsyncReadExt, BufReader};
 
+    #[tokio::test]
+    async fn from_stream_with_etag_selects_the_etag_hash() {
+        use md5::{Digest, Md5};
+        let data: Vec<u8> = (0..200_000u32).map(|i| (i * 7) as u8).collect();
+        let size = data.len() as i64;
+        let md5 = hex_simd::encode_to_string(Md5::digest(&data), hex_simd::AsciiCase::Lower);
+        let xxh3 = hex_simd::encode_to_string(xxhash_rust::xxh3::xxh3_128(&data).to_be_bytes(), hex_simd::AsciiCase::Lower);
+
+        let etag_of = |etag: EtagAlgorithm, content_md5: Option<String>| {
+            let data = data.clone();
+            async move {
+                let mut reader = HashReader::from_stream_with_etag(
+                    BufReader::new(Cursor::new(data)),
+                    size,
+                    size,
+                    content_md5,
+                    None,
+                    false,
+                    etag,
+                )
+                .expect("reader");
+                let mut out = Vec::new();
+                reader.read_to_end(&mut out).await.map(|_| reader.try_resolve_etag())
+            }
+        };
+
+        assert_eq!(etag_of(EtagAlgorithm::Xxh3, None).await.expect("read"), Some(xxh3));
+        assert_eq!(etag_of(EtagAlgorithm::Md5, None).await.expect("read"), Some(md5.clone()));
+        // Content-MD5 always gets MD5, and a wrong one still fails.
+        assert_eq!(etag_of(EtagAlgorithm::Xxh3, Some(md5.clone())).await.expect("read"), Some(md5));
+        let wrong = hex_simd::encode_to_string(Md5::digest(b"other"), hex_simd::AsciiCase::Lower);
+        assert!(etag_of(EtagAlgorithm::Xxh3, Some(wrong)).await.is_err());
+    }
+
     #[tokio::test]
     async fn test_hashreader_wrapping_logic() {
         let data = b"hello world";

crates/utils/src/http/metadata_compat.rs

@@ -46,6 +46,10 @@ pub const SUFFIX_COMPRESSION_SIZE: &str = "compression-size";
 pub const SUFFIX_ACTUAL_SIZE: &str = "actual-size";
 /// Maximum logical object size for a capability-bound multipart upload.
 pub const SUFFIX_MAX_TOTAL_OBJECT_SIZE: &str = "max-total-object-size";
+/// ETag algorithm pinned for a multipart upload at its creation (`xxh3`
+/// when the bucket uses XXH3 ETags), so every part of one upload hashes the
+/// same way even if the bucket setting changes mid-upload.
+pub const SUFFIX_ETAG_ALGORITHM: &str = "etag-algorithm";
 pub const SUFFIX_ACTUAL_OBJECT_SIZE: &str = "actual-object-size";
 /// Used by replication; key stored with capital A
 pub const SUFFIX_ACTUAL_OBJECT_SIZE_CAP: &str = "Actual-Object-Size";

docs/README.md

@@ -16,6 +16,8 @@ collection:
 - [Read-only BTF declaration inspection](operations/btf-prototype-inspection.md) — bounded prototypes and scalar member layouts, explicit void handling, and independent native references.
 
 - [Internode HTTP streaming transport](operations/internode-http-transport.md) — protocol selection, pooled connection isolation, and distributed read validation.
+- [Erasure shard size](operations/erasure-shard-size.md) — `RUSTFS_ERASURE_SHARD_BYTES`: larger, aligned shards for new objects; rules and memory cost.
+- [Object data cache fill on PUT](operations/object-data-cache-fill-on-put.md) — `RUSTFS_OBJECT_DATA_CACHE_FILL_ON_PUT`: insert plain streaming PUT bodies so the first GET hits.
 
 - [Persisted config compatibility](operations/persisted-config-compatibility.md) — ignored MinIO settings, bounded diagnostics, alias retention, and safe resets.
 

docs/operations/erasure-shard-size.md

@@ -0,0 +1,40 @@
+# Erasure shard size (`RUSTFS_ERASURE_SHARD_BYTES`)
+
+RustFS writes every object as a sequence of erasure blocks; each block is split
+into `data` shards of `ceil(block / data)` bytes and `parity` shards of the
+same size. The historical block is 1 MiB (`BLOCK_SIZE_V2`), which on a 12+4
+layout gives 87,382-byte shards and about 1,000 stripes per GiB. Every stripe
+costs a round of shard reads, a quorum wait, a bitrot check and a reconstruct,
+so single-stream GET throughput is bound by per-stripe latency rather than by
+disks or the network.
+
+`RUSTFS_ERASURE_SHARD_BYTES` sets the shard size of newly written objects
+directly; the block becomes `data shards x shard bytes`, so shards stay exactly
+this large and aligned whatever the parity setting:
+
+| Layout | `RUSTFS_ERASURE_SHARD_BYTES=4194304` | Block |
+| --- | --- | --- |
+| 12+4 (EC:4) | 4 MiB shards | 48 MiB |
+| 14+2 (EC:2) | 4 MiB shards | 56 MiB |
+
+Rules:
+
+- `0` or unset keeps the 1 MiB block. The value must be a multiple of 4 KiB and
+  at most 64 MiB; anything else is ignored with a startup warning.
+- The block is recorded per object in `xl.meta`, so existing objects keep
+  their block and keep working. Re-upload objects that should use the new
+  layout. Set the same value on every node.
+- System metadata under `.rustfs.sys` always uses the 1 MiB block.
+- Memory per in-flight stripe grows with the block: about 64 MiB per GET with
+  4 MiB shards on 12+4, 128 MiB when a stripe is prefetched. Time to first byte
+  of a small ranged read inside a large object grows with the block as well.
+
+## Bitrot verification with large shards
+
+The shard reads of one stripe complete on the decode task, and each frame is
+bitrot-hashed as it lands, so with large shards the twelve hashes of a 12+4
+stripe used to run one after another on one core (about 10 ms per 48 MiB
+stripe). Frames of at least `RUSTFS_BITROT_VERIFY_OFFLOAD_MIN_BYTES`
+(default 2 MiB, `0` disables) are hashed on the blocking thread pool
+instead, so the shards of a stripe verify in parallel; smaller frames stay
+inline where a thread hop would cost more than the hash.

docs/operations/object-data-cache-fill-on-put.md

@@ -0,0 +1,28 @@
+# Object data cache: fill on PUT (`RUSTFS_OBJECT_DATA_CACHE_FILL_ON_PUT`)
+
+The object data cache (`RUSTFS_OBJECT_DATA_CACHE_*`) fills on GET: the first
+reader of an object pays the erasure read and the fill, later readers hit.
+For write-once data that is read right after it is written, such as KV-cache
+chunks pushed after prefill, the first read is the one that matters.
+
+With `RUSTFS_OBJECT_DATA_CACHE_FILL_ON_PUT=true` a PUT keeps a copy of its
+plaintext body while it streams to the erasure encoder and, once the object
+is committed, inserts it under the same key the GET path plans for that
+version (after the usual post-put invalidation). The first GET is then a hit.
+
+Only PUTs that meet all of the following are filled:
+
+- the cache is enabled in a fill mode (`fill_buffered_only` or
+  `fill_materialize_enabled`);
+- the body takes the plain streaming path: no compression, no ciphertext
+  passthrough, no SSE request headers (bucket-default encryption is caught by
+  the plan after the commit, and such objects are not inserted);
+- the object size is known and at most the cache's fill ceiling
+  (`RUSTFS_OBJECT_DATA_CACHE_MAX_ENTRY_BYTES`, clamped like a GET fill).
+
+Cost: one extra in-memory copy of the body for the duration of the PUT (the
+frames are released after the insert), and the insert itself. Nothing changes
+for PUTs that do not qualify. Default: off.
+
+Measured with 48 MiB chunks over cuObject RDMA GETs, the first GET after a
+PUT then comes from memory instead of the drives.

rustfs/src/admin/handlers/etag.rs

@@ -0,0 +1,275 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Per-bucket ETag algorithm admin handlers.
+//!
+//! A bucket can derive the ETags of objects written without Content-MD5 from
+//! XXH3-128 (`xxh3`) instead of MD5 (`md5`, the default), for clients that
+//! never treat the ETag as an MD5, such as an LLM KV cache. The choice is
+//! stored in the bucket metadata (`etag.json` entry) and consulted by the
+//! PutObject and UploadPart write paths; a multipart upload keeps the
+//! algorithm it was created with. See docs/operations/nixl-s3-over-rdma.md.
+
+use crate::admin::auth::validate_admin_request;
+use crate::admin::operation::{AdminRequest, AdminResponse, Operation};
+use crate::admin::router::{AdminOperation, S3Router};
+use crate::admin::runtime_sources::current_notification_system;
+use crate::admin::storage_api::bucket::etag::BucketEtagConfig;
+use crate::admin::storage_api::bucket::metadata::BUCKET_ETAG_CONFIG;
+use crate::admin::storage_api::bucket::metadata_sys;
+use crate::auth::{check_key_valid, get_session_token};
+use crate::server::ADMIN_PREFIX;
+use hyper::{Method, StatusCode};
+use matchit::Params;
+use rustfs_policy::policy::action::{Action, AdminAction};
+use rustfs_s3_types::{Body, S3Result, s3_error};
+use serde::{Deserialize, Serialize};
+use tracing::{info, warn};
+
+const LOG_COMPONENT_ADMIN: &str = "admin";
+const LOG_SUBSYSTEM_ETAG: &str = "bucket_etag";
+const EVENT_ADMIN_BUCKET_ETAG: &str = "admin_bucket_etag_state";
+
+#[derive(Debug, Deserialize)]
+struct SetBucketEtagRequest {
+    algorithm: String,
+}
+
+#[derive(Debug, Serialize)]
+struct BucketEtagResponse {
+    bucket: String,
+    /// The bucket's ETag algorithm, or `null` when the bucket has no setting
+    /// (MD5 ETags).
+    algorithm: Option<String>,
+}
+
+pub struct SetBucketEtagHandler;
+pub struct GetBucketEtagHandler;
+pub struct DeleteBucketEtagHandler;
+
+pub fn register_etag_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
+    r.insert(
+        Method::PUT,
+        format!("{}{}", ADMIN_PREFIX, "/v3/bucket-etag/{bucket}").as_str(),
+        AdminOperation(&SetBucketEtagHandler {}),
+    )?;
+
+    r.insert(
+        Method::GET,
+        format!("{}{}", ADMIN_PREFIX, "/v3/bucket-etag/{bucket}").as_str(),
+        AdminOperation(&GetBucketEtagHandler {}),
+    )?;
+
+    r.insert(
+        Method::DELETE,
+        format!("{}{}", ADMIN_PREFIX, "/v3/bucket-etag/{bucket}").as_str(),
+        AdminOperation(&DeleteBucketEtagHandler {}),
+    )?;
+
+    Ok(())
+}
+
+fn parse_set_bucket_etag_request(body: &[u8]) -> Result<SetBucketEtagRequest, rustfs_s3_types::S3Error> {
+    if body.is_empty() {
+        return Err(s3_error!(InvalidRequest, "request body is required, e.g. {{\"algorithm\":\"xxh3\"}}"));
+    }
+    serde_json::from_slice(body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))
+}
+
+/// Validates and canonicalizes the requested algorithm name.
+fn normalize_algorithm(algorithm: &str) -> Result<String, rustfs_s3_types::S3Error> {
+    let normalized = algorithm.trim().to_ascii_lowercase();
+    if BucketEtagConfig::is_valid_algorithm(&normalized) {
+        Ok(normalized)
+    } else {
+        Err(s3_error!(InvalidArgument, "invalid ETag algorithm {:?}: expected md5|xxh3", algorithm))
+    }
+}
+
+/// Kick peers to reload the bucket's metadata so the setting takes effect
+/// cluster-wide without waiting for the periodic refresh loop. Failures are
+/// logged, not surfaced: the refresh loop converges peers eventually.
+fn notify_peers_reload(bucket: String, operation: &'static str) {
+    tokio::spawn(async move {
+        if let Some(notification_sys) = current_notification_system()
+            && let Err(err) = notification_sys.load_bucket_metadata(&bucket).await
+        {
+            warn!(
+                event = EVENT_ADMIN_BUCKET_ETAG,
+                component = LOG_COMPONENT_ADMIN,
+                subsystem = LOG_SUBSYSTEM_ETAG,
+                bucket = %bucket,
+                error = %err,
+                "failed to notify peers after {operation}"
+            );
+        }
+    });
+}
+
+async fn authenticate_admin(req: &AdminRequest) -> S3Result<()> {
+    let Some(cred) = req.credentials() else {
+        return Err(s3_error!(InvalidRequest, "authentication required"));
+    };
+
+    let (cred, owner) =
+        check_key_valid(get_session_token(req.uri(), req.headers()).unwrap_or_default(), cred.access_key()).await?;
+
+    let remote_addr = req.policy_source_addr();
+    validate_admin_request(
+        req.headers(),
+        &cred,
+        owner,
+        false,
+        vec![Action::AdminAction(AdminAction::ConfigUpdateAdminAction)],
+        remote_addr,
+    )
+    .await?;
+
+    Ok(())
+}
+
+fn bucket_from_params(params: &Params<'_, '_>) -> S3Result<String> {
+    let bucket = params.get("bucket").unwrap_or("").to_string();
+    if bucket.is_empty() {
+        return Err(s3_error!(InvalidRequest, "bucket name is required"));
+    }
+    Ok(bucket)
+}
+
+fn etag_response(bucket: String, algorithm: Option<String>) -> S3Result<AdminResponse> {
+    let response = BucketEtagResponse { bucket, algorithm };
+    let json = serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
+    Ok(AdminResponse::new(StatusCode::OK, Body::from(json)))
+}
+
+#[async_trait::async_trait]
+impl Operation for SetBucketEtagHandler {
+    #[tracing::instrument(skip_all)]
+    async fn call(&self, mut req: AdminRequest, params: Params<'_, '_>) -> S3Result<AdminResponse> {
+        authenticate_admin(&req).await?;
+
+        let bucket = bucket_from_params(&params)?;
+        let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(&bucket)
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to capture bucket incarnation: {}", e))?;
+
+        let body = req
+            .take_body()
+            .store_all_limited(rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE)
+            .await
+            .map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?;
+
+        let request = parse_set_bucket_etag_request(&body)?;
+        let algorithm = normalize_algorithm(&request.algorithm)?;
+
+        let config = BucketEtagConfig::new(&algorithm);
+        let json = serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "failed to encode config: {}", e))?;
+
+        // System buckets are rejected by the metadata layer.
+        metadata_sys::update_if_incarnation(&bucket, BUCKET_ETAG_CONFIG, json, expected_incarnation_id)
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to set bucket ETag algorithm: {}", e))?;
+
+        info!(
+            event = EVENT_ADMIN_BUCKET_ETAG,
+            component = LOG_COMPONENT_ADMIN,
+            subsystem = LOG_SUBSYSTEM_ETAG,
+            bucket = %bucket,
+            algorithm = %algorithm,
+            "bucket ETag algorithm set"
+        );
+
+        notify_peers_reload(bucket.clone(), "set bucket ETag algorithm");
+
+        etag_response(bucket, Some(algorithm))
+    }
+}
+
+#[async_trait::async_trait]
+impl Operation for GetBucketEtagHandler {
+    #[tracing::instrument(skip_all)]
+    async fn call(&self, req: AdminRequest, params: Params<'_, '_>) -> S3Result<AdminResponse> {
+        authenticate_admin(&req).await?;
+
+        let bucket = bucket_from_params(&params)?;
+
+        let (config, _updated_at) = metadata_sys::get_etag_config(&bucket)
+            .await
+            .map_err(|e| s3_error!(NoSuchBucket, "failed to read bucket ETag algorithm: {}", e))?;
+
+        let algorithm = config.and_then(|c| c.normalized_algorithm());
+        etag_response(bucket, algorithm)
+    }
+}
+
+#[async_trait::async_trait]
+impl Operation for DeleteBucketEtagHandler {
+    #[tracing::instrument(skip_all)]
+    async fn call(&self, req: AdminRequest, params: Params<'_, '_>) -> S3Result<AdminResponse> {
+        authenticate_admin(&req).await?;
+
+        let bucket = bucket_from_params(&params)?;
+        let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(&bucket)
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to capture bucket incarnation: {}", e))?;
+
+        metadata_sys::delete_if_incarnation(&bucket, BUCKET_ETAG_CONFIG, expected_incarnation_id)
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to clear bucket ETag algorithm: {}", e))?;
+
+        info!(
+            event = EVENT_ADMIN_BUCKET_ETAG,
+            component = LOG_COMPONENT_ADMIN,
+            subsystem = LOG_SUBSYSTEM_ETAG,
+            bucket = %bucket,
+            "bucket ETag algorithm cleared"
+        );
+
+        notify_peers_reload(bucket.clone(), "clear bucket ETag algorithm");
+
+        etag_response(bucket, None)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parse_set_request_requires_body_and_valid_json() {
+        assert!(parse_set_bucket_etag_request(b"").is_err());
+        assert!(parse_set_bucket_etag_request(b"not-json").is_err());
+
+        let req = parse_set_bucket_etag_request(br#"{"algorithm":"xxh3"}"#).expect("parse");
+        assert_eq!(req.algorithm, "xxh3");
+    }
+
+    #[test]
+    fn normalize_algorithm_accepts_md5_and_xxh3_only() {
+        assert_eq!(normalize_algorithm("md5").unwrap(), "md5");
+        assert_eq!(normalize_algorithm(" XXH3 ").unwrap(), "xxh3");
+        assert!(normalize_algorithm("").is_err());
+        assert!(normalize_algorithm("sha256").is_err());
+    }
+
+    #[test]
+    fn response_serializes_no_setting_as_null() {
+        let json = serde_json::to_string(&BucketEtagResponse {
+            bucket: "b".to_string(),
+            algorithm: None,
+        })
+        .expect("serialize");
+        assert_eq!(json, r#"{"bucket":"b","algorithm":null}"#);
+    }
+}

rustfs/src/admin/handlers/mod.rs

@@ -24,6 +24,7 @@ pub mod cluster_snapshot;
 pub mod config_admin;
 pub mod diagnostics;
 pub mod durability;
+pub mod etag;
 pub mod event;
 pub mod extensions;
 pub mod gateway_key_inventory;

rustfs/src/admin/mod.rs

@@ -38,9 +38,9 @@ mod route_registration_test;
 
 use handlers::{
     account, audit, batch_job, bucket_meta, bucket_recovery, cluster_snapshot, config_admin, diagnostics,
-    durability as durability_handler, extensions, gateway_key_inventory, heal, health, idp_compat, ilm_transition,
-    inspect_archive, integrity, kms, mfa, module_switch, object_data_cache, object_zip_download, oidc, on_demand_migration,
-    plugins_catalog, plugins_instances, pools, profile_admin, quota as quota_handler, rebalance,
+    durability as durability_handler, etag as etag_handler, extensions, gateway_key_inventory, heal, health, idp_compat,
+    ilm_transition, inspect_archive, integrity, kms, mfa, module_switch, object_data_cache, object_zip_download, oidc,
+    on_demand_migration, plugins_catalog, plugins_instances, pools, profile_admin, quota as quota_handler, rebalance,
     replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, usage_prefix,
     user,
 };
@@ -83,6 +83,7 @@ fn register_admin_routes(r: &mut S3Router<AdminOperation>) -> std::io::Result<()
 
     quota_handler::register_quota_route(r)?;
     durability_handler::register_durability_route(r)?;
+    etag_handler::register_etag_route(r)?;
     bucket_recovery::register_bucket_recovery_route(r)?;
     on_demand_migration::register_on_demand_migration_route(r)?;
     bucket_meta::register_bucket_meta_route(r)?;

rustfs/src/admin/route_policy.rs

@@ -433,6 +433,24 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
         CONFIG_UPDATE,
         RouteRiskLevel::High,
     ),
+    admin(
+        HttpMethod::Put,
+        "/rustfs/admin/v3/bucket-etag/{bucket}",
+        CONFIG_UPDATE,
+        RouteRiskLevel::High,
+    ),
+    admin(
+        HttpMethod::Get,
+        "/rustfs/admin/v3/bucket-etag/{bucket}",
+        CONFIG_UPDATE,
+        RouteRiskLevel::Sensitive,
+    ),
+    admin(
+        HttpMethod::Delete,
+        "/rustfs/admin/v3/bucket-etag/{bucket}",
+        CONFIG_UPDATE,
+        RouteRiskLevel::High,
+    ),
     admin(
         HttpMethod::Put,
         "/rustfs/admin/v3/on-demand-migration/{bucket}",

rustfs/src/admin/route_registration_test.rs

@@ -311,6 +311,9 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
         admin_route_sample(Method::PUT, "/v3/bucket-durability/{bucket}", "/v3/bucket-durability/test-bucket"),
         admin_route_sample(Method::GET, "/v3/bucket-durability/{bucket}", "/v3/bucket-durability/test-bucket"),
         admin_route_sample(Method::DELETE, "/v3/bucket-durability/{bucket}", "/v3/bucket-durability/test-bucket"),
+        admin_route_sample(Method::PUT, "/v3/bucket-etag/{bucket}", "/v3/bucket-etag/test-bucket"),
+        admin_route_sample(Method::GET, "/v3/bucket-etag/{bucket}", "/v3/bucket-etag/test-bucket"),
+        admin_route_sample(Method::DELETE, "/v3/bucket-etag/{bucket}", "/v3/bucket-etag/test-bucket"),
         admin_route_sample(Method::PUT, "/v3/on-demand-migration/{bucket}", "/v3/on-demand-migration/test-bucket"),
         admin_route_sample(Method::GET, "/v3/on-demand-migration/{bucket}", "/v3/on-demand-migration/test-bucket"),
         admin_route_sample(Method::DELETE, "/v3/on-demand-migration/{bucket}", "/v3/on-demand-migration/test-bucket"),
@@ -1445,6 +1448,9 @@ fn test_register_routes_cover_representative_admin_paths() {
     assert_route(&router, Method::PUT, &admin_path("/v3/bucket-durability/test-bucket"));
     assert_route(&router, Method::GET, &admin_path("/v3/bucket-durability/test-bucket"));
     assert_route(&router, Method::DELETE, &admin_path("/v3/bucket-durability/test-bucket"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/bucket-etag/test-bucket"));
+    assert_route(&router, Method::GET, &admin_path("/v3/bucket-etag/test-bucket"));
+    assert_route(&router, Method::DELETE, &admin_path("/v3/bucket-etag/test-bucket"));
     assert_route(&router, Method::PUT, &admin_path("/v3/on-demand-migration/test-bucket"));
     assert_route(&router, Method::GET, &admin_path("/v3/on-demand-migration/test-bucket"));
     assert_route(&router, Method::DELETE, &admin_path("/v3/on-demand-migration/test-bucket"));

rustfs/src/admin/storage_api.rs

@@ -26,7 +26,7 @@ use time::OffsetDateTime;
 
 mod ecstore_bucket {
     pub(crate) use crate::storage::storage_api::ecstore_bucket::{
-        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, object_lock, quota, remote_s3_client,
+        bandwidth, bucket_target_sys, durability, etag, lifecycle, metadata, metadata_sys, object_lock, quota, remote_s3_client,
         replication, target, utils, versioning, versioning_sys,
     };
 }
@@ -304,6 +304,7 @@ pub(crate) mod metadata {
     pub(crate) const BUCKET_TARGETS_FILE: &str = super::ecstore_bucket::metadata::BUCKET_TARGETS_FILE;
     pub(crate) const BUCKET_VERSIONING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG;
     pub(crate) const BUCKET_DURABILITY_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_DURABILITY_CONFIG;
+    pub(crate) const BUCKET_ETAG_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_ETAG_CONFIG;
     pub(crate) const BUCKET_ON_DEMAND_MIGRATION_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG;
     pub(crate) const OBJECT_LOCK_CONFIG: &str = super::ecstore_bucket::metadata::OBJECT_LOCK_CONFIG;
 
@@ -318,6 +319,10 @@ pub(crate) mod durability {
     pub(crate) type BucketDurabilityConfig = super::ecstore_bucket::durability::BucketDurabilityConfig;
 }
 
+pub(crate) mod etag {
+    pub(crate) type BucketEtagConfig = super::ecstore_bucket::etag::BucketEtagConfig;
+}
+
 pub(crate) mod remote_s3_client {
     pub(crate) type PathStyle = super::ecstore_bucket::remote_s3_client::PathStyle;
     pub(crate) type RemoteCredentials = super::ecstore_bucket::remote_s3_client::RemoteCredentials;
@@ -512,6 +517,10 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::get_durability_config(bucket).await
     }
 
+    pub(crate) async fn get_etag_config(bucket: &str) -> Result<(Option<super::etag::BucketEtagConfig>, OffsetDateTime)> {
+        super::ecstore_bucket::metadata_sys::get_etag_config(bucket).await
+    }
+
     pub(crate) async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
         super::ecstore_bucket::metadata_sys::get_quota_config(bucket).await
     }
@@ -963,6 +972,7 @@ pub(crate) mod bucket {
     pub(crate) use super::bandwidth;
     pub(crate) use super::bucket_target_sys as target_sys;
     pub(crate) use super::durability;
+    pub(crate) use super::etag;
     #[cfg(test)]
     pub(crate) use super::lifecycle;
     pub(crate) use super::metadata;

rustfs/src/app/multipart_usecase.rs

@@ -21,6 +21,7 @@ use super::storage_api::multipart_usecase::access::{
 };
 use super::storage_api::multipart_usecase::bucket::quota::checker::QuotaChecker;
 use super::storage_api::multipart_usecase::bucket::{
+    etag::bucket_uses_xxh3_etag,
     lifecycle::{bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::enqueue_transition_immediate},
     metadata_sys,
     quota::QuotaOperation,
@@ -47,7 +48,9 @@ use super::storage_api::multipart_usecase::error::{
 use super::storage_api::multipart_usecase::helper::{OperationHelper, run_owned_mutation};
 #[cfg(test)]
 use super::storage_api::multipart_usecase::io::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
-use super::storage_api::multipart_usecase::io::{HashReader, WriteEncryption, WritePlan, compression_metadata_value};
+use super::storage_api::multipart_usecase::io::{
+    EtagAlgorithm, HashReader, WriteEncryption, WritePlan, compression_metadata_value,
+};
 use super::storage_api::multipart_usecase::object_utils::{to_s3s_etag, to_s3s_xml_etag};
 use super::storage_api::multipart_usecase::options::{
     copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization,
@@ -109,7 +112,7 @@ use rustfs_utils::CompressionAlgorithm;
 #[cfg(test)]
 use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, insert_header};
 use rustfs_utils::http::{
-    SUFFIX_MAX_TOTAL_OBJECT_SIZE, SUFFIX_PLAINTEXT_CHECKSUM, SUFFIX_REPLICATION_GENERATION,
+    SUFFIX_ETAG_ALGORITHM, SUFFIX_MAX_TOTAL_OBJECT_SIZE, SUFFIX_PLAINTEXT_CHECKSUM, SUFFIX_REPLICATION_GENERATION,
     SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP,
     SUFFIX_SOURCE_REPLICATION_REQUEST, contains_key_str, get_consistent_str, get_header, get_source_scheme,
     headers::{AMZ_CHECKSUM_TYPE, AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS},
@@ -166,6 +169,18 @@ fn multipart_part_numbers(parts: &[rustfs_filemeta::ObjectPartInfo]) -> Vec<usiz
     parts.iter().map(|part| part.number).collect()
 }
 
+/// ETag value pinned by CreateMultipartUpload for a bucket with XXH3 ETags.
+const ETAG_ALGORITHM_XXH3: &str = "xxh3";
+
+/// The ETag algorithm pinned for this multipart upload at its creation;
+/// uploads without the pin (MD5 buckets, older uploads) keep MD5.
+fn multipart_etag_algorithm(metadata: &HashMap<String, String>) -> EtagAlgorithm {
+    match get_consistent_str(metadata, SUFFIX_ETAG_ALGORITHM) {
+        Some(value) if value.eq_ignore_ascii_case(ETAG_ALGORITHM_XXH3) => EtagAlgorithm::Xxh3,
+        _ => EtagAlgorithm::Md5,
+    }
+}
+
 fn multipart_max_total_object_size(metadata: &HashMap<String, String>) -> S3Result<Option<u64>> {
     if !contains_key_str(metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE) {
         return Ok(None);
@@ -541,6 +556,24 @@ mod tests {
         }
     }
 
+    #[test]
+    fn multipart_etag_algorithm_reads_the_pinned_value() {
+        let mut metadata = HashMap::new();
+        assert_eq!(multipart_etag_algorithm(&metadata), EtagAlgorithm::Md5);
+
+        insert_str(&mut metadata, SUFFIX_ETAG_ALGORITHM, ETAG_ALGORITHM_XXH3.to_string());
+        assert_eq!(multipart_etag_algorithm(&metadata), EtagAlgorithm::Xxh3);
+        assert_eq!(
+            metadata.get("x-minio-internal-etag-algorithm").map(String::as_str),
+            Some("xxh3"),
+            "the pin is written under both internal prefixes"
+        );
+
+        // Conflicting copies fail closed to MD5.
+        metadata.insert("x-minio-internal-etag-algorithm".to_string(), "md5".to_string());
+        assert_eq!(multipart_etag_algorithm(&metadata), EtagAlgorithm::Md5);
+    }
+
     #[test]
     fn multipart_max_total_object_size_reads_compatible_internal_metadata() {
         let mut metadata = HashMap::new();

rustfs/src/app/multipart_usecase/create.rs

@@ -134,6 +134,9 @@ impl DefaultMultipartUsecase {
         if let Some(limit) = multipart_max_total_object_size {
             insert_str(&mut metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE, limit.to_string());
         }
+        if bucket_uses_xxh3_etag(&bucket) {
+            insert_str(&mut metadata, SUFFIX_ETAG_ALGORITHM, ETAG_ALGORITHM_XXH3.to_string());
+        }
 
         let has_explicit_object_lock_retention = object_lock_mode.is_some()
             || object_lock_retain_until_date.is_some()

rustfs/src/app/multipart_usecase/upload_part.rs

@@ -150,10 +150,12 @@ impl DefaultMultipartUsecase {
         let Some(store) = self.object_store() else {
             return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
         };
+        let multipart_info_started = rustfs_io_metrics::put_stage_timer();
         let fi = store
             .get_multipart_info(&bucket, &key, &upload_id, &opts)
             .await
             .map_err(multipart_upload_api_error)?;
+        rustfs_io_metrics::record_put_object_stage_duration_from("multipart_get_info", multipart_info_started);
         authorize_sse_kms_multipart_write(part_principal.as_ref(), &fi.user_defined).await?;
         let max_total_object_size = multipart_max_total_object_size(&fi.user_defined)?;
         let mut size = require_upload_part_size(size, max_total_object_size.is_some())?;
@@ -239,10 +241,19 @@ impl DefaultMultipartUsecase {
         let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query());
 
         let mut write_plan = WritePlan::new();
+        let etag_algorithm = multipart_etag_algorithm(&fi.user_defined);
         let mut reader = if is_disk_compressed {
             let algorithm = CompressionAlgorithm::default();
-            let mut hrd = HashReader::from_stream(body, size, actual_size, md5hex.take(), sha256hex.take(), false)
-                .map_err(ApiError::from)?;
+            let mut hrd = HashReader::from_stream_with_etag(
+                body,
+                size,
+                actual_size,
+                md5hex.take(),
+                sha256hex.take(),
+                false,
+                etag_algorithm,
+            )
+            .map_err(ApiError::from)?;
 
             if let Err(err) = hrd.add_checksum(&req.headers, trailer_source(req.trailing_headers.clone()), false) {
                 return Err(ApiError::from(err).into());
@@ -252,7 +263,8 @@ impl DefaultMultipartUsecase {
             write_plan = write_plan.with_compression(algorithm);
             hrd
         } else {
-            HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
+            HashReader::from_stream_with_etag(body, size, actual_size, md5hex, sha256hex, false, etag_algorithm)
+                .map_err(ApiError::from)?
         };
 
         if let Err(err) = reader.add_checksum(&req.headers, trailer_source(req.trailing_headers.clone()), size < 0) {

rustfs/src/app/object/get.rs

@@ -2829,9 +2829,15 @@ impl DefaultObjectUsecase {
             rs,
         } = validated;
 
-        let opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), part_number, headers)
+        let mut opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), part_number, headers)
             .await
             .map_err(ApiError::from)?;
+        if is_lockless_cache_bucket(&bucket) {
+            // Content-addressed cache objects never change once written, so the
+            // read needs neither the object read lock nor a fresh metadata quorum.
+            opts.no_lock = true;
+            opts.metadata_cache_safe = true;
+        }
 
         Ok(GetObjectRequestContext {
             version_id_for_event: version_id.unwrap_or_default(),

rustfs/src/app/object/head.rs

@@ -417,9 +417,14 @@ impl DefaultObjectUsecase {
             return result;
         }
 
-        let opts: ObjectOptions = get_opts(&bucket, &key, version_id, part_number, &req.headers)
+        let mut opts: ObjectOptions = get_opts(&bucket, &key, version_id, part_number, &req.headers)
             .await
             .map_err(ApiError::from)?;
+        if is_lockless_cache_bucket(&bucket) {
+            // See the GetObject path: cache objects never change once written.
+            opts.no_lock = true;
+            opts.metadata_cache_safe = true;
+        }
         let opts = internal_object_info_lookup_opts(opts);
 
         prepare_odm_read_generation(&store, &mut req, &bucket).await;

rustfs/src/app/object/mod.rs

@@ -26,6 +26,7 @@ use super::storage_api::object_usecase::access::{
     recursive_force_delete_has_authenticated_caller, replication_request_authorized, req_info_mut, req_info_ref,
     retained_table_data_plane_publication_guards, set_requested_object_lock_retain_until,
 };
+use super::storage_api::object_usecase::bucket::lockless_cache::is_lockless_cache_bucket;
 #[cfg(test)]
 use super::storage_api::object_usecase::bucket::quota::BucketQuota;
 use super::storage_api::object_usecase::bucket::quota::checker::QuotaChecker;
@@ -34,6 +35,7 @@ use super::storage_api::object_usecase::bucket::replication::ReplicationState;
 use super::storage_api::object_usecase::bucket::replication::replication_statuses_map;
 use super::storage_api::object_usecase::bucket::{
     VersioningConfigExt as _,
+    etag::bucket_uses_xxh3_etag,
     lifecycle::{
         bucket_lifecycle_audit::LcEventSrc,
         bucket_lifecycle_ops::{enqueue_transition_immediate, post_restore_opts},
@@ -84,7 +86,9 @@ use super::storage_api::object_usecase::head_prefix::{head_prefix_not_found_mess
 use super::storage_api::object_usecase::helper::{
     OperationHelper, build_event_resp_elements, run_owned_mutation, spawn_background_with_context,
 };
-use super::storage_api::object_usecase::io::{DynReader, HashReader, WritePlan, compression_metadata_value, wrap_reader};
+use super::storage_api::object_usecase::io::{
+    DynReader, EtagAlgorithm, HashReader, WritePlan, compression_metadata_value, wrap_reader,
+};
 #[cfg(test)]
 use super::storage_api::object_usecase::object_cache::GetObjectBodySource;
 #[cfg(test)]

rustfs/src/app/object/put.rs

@@ -790,6 +790,103 @@ pub(super) fn apply_put_request_object_lock_opts(
     Ok(())
 }
 
+/// Whether PUT bodies that fit the object data cache are inserted as they
+/// stream in (`RUSTFS_OBJECT_DATA_CACHE_FILL_ON_PUT`), read once.
+fn object_data_cache_fill_on_put_enabled() -> bool {
+    static ENABLED: OnceLock<bool> = OnceLock::new();
+    *ENABLED.get_or_init(|| {
+        rustfs_utils::get_env_bool(
+            rustfs_config::ENV_OBJECT_DATA_CACHE_FILL_ON_PUT,
+            rustfs_config::DEFAULT_OBJECT_DATA_CACHE_FILL_ON_PUT,
+        )
+    })
+}
+
+/// Frames of a streaming PUT body kept for a cache fill after the commit.
+type PutBodyCacheCollector = Arc<Mutex<Vec<Bytes>>>;
+
+/// Decides whether this PUT keeps a copy of its plaintext body for the cache:
+/// fill-on-put must be on, the cache must accept fills, and the body must be
+/// plain (no compression, no ciphertext, no SSE request) and fit the entry
+/// limit. Bucket-default encryption is caught after the commit by the plan.
+fn put_body_cache_collector(
+    adapter: &ObjectDataCacheAdapter,
+    size: i64,
+    should_compress: bool,
+    ciphertext_passthrough: bool,
+    headers: &HeaderMap,
+) -> Option<PutBodyCacheCollector> {
+    if should_compress
+        || ciphertext_passthrough
+        || has_put_sse_request_headers(headers)
+        || !put_body_may_fill_cache(adapter, size)
+    {
+        return None;
+    }
+    Some(Arc::new(Mutex::new(Vec::new())))
+}
+
+/// Whether fill-on-put could keep this PUT's body (see
+/// [`put_body_cache_collector`]); when it cannot, the body is consumed and
+/// dropped with the request.
+fn put_body_may_fill_cache(adapter: &ObjectDataCacheAdapter, size: i64) -> bool {
+    object_data_cache_fill_on_put_enabled()
+        && !adapter.is_disabled()
+        && adapter.fill_enabled()
+        && u64::try_from(size).is_ok_and(|size| size > 0 && size <= adapter.fill_size_ceiling_bytes())
+}
+
+/// Joins the collected frames into one body when they add up to `size` bytes.
+fn collected_put_body(collector: &PutBodyCacheCollector, size: i64) -> Option<Bytes> {
+    let frames = std::mem::take(&mut *collector.lock().unwrap_or_else(|poisoned| poisoned.into_inner()));
+    let total: usize = frames.iter().map(Bytes::len).sum();
+    if usize::try_from(size).ok() != Some(total) {
+        return None;
+    }
+    if frames.len() == 1 {
+        return frames.into_iter().next();
+    }
+    let mut body = BytesMut::with_capacity(total);
+    for frame in &frames {
+        body.extend_from_slice(frame);
+    }
+    Some(body.freeze())
+}
+
+/// Inserts the PUT body under the key the GET path would plan for the committed
+/// version, after the usual post-put invalidation.
+async fn fill_object_data_cache_after_put(
+    adapter: &ObjectDataCacheAdapter,
+    bucket: &str,
+    key: &str,
+    info: &ObjectInfo,
+    collector: &PutBodyCacheCollector,
+) {
+    let Some(body) = collected_put_body(collector, info.size) else {
+        debug!(
+            bucket,
+            key,
+            size = info.size,
+            "fill-on-put skipped: collected body does not match the committed size"
+        );
+        return;
+    };
+    let plan = build_get_object_body_cache_plan_for_revalidation(
+        adapter,
+        GetObjectBodyCacheRequest {
+            bucket,
+            key,
+            info,
+            response_content_length: info.size,
+            has_range: false,
+            part_number: None,
+            encryption_applied: false,
+        },
+    );
+    let result = fill_get_object_body_cache_from_buffered_body(adapter, &plan, &body).await;
+    debug!(bucket, key, size = info.size, ?result, "fill-on-put");
+}
+
 pub(super) fn is_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap) -> bool {
     input
         .server_side_encryption
@@ -1354,6 +1451,12 @@ impl DefaultObjectUsecase {
             preserve_etag,
             mut origin,
         } = write;
+        // Objects without Content-MD5 take the bucket's ETag algorithm.
+        let etag_algorithm = if bucket_uses_xxh3_etag(&bucket) {
+            EtagAlgorithm::Xxh3
+        } else {
+            EtagAlgorithm::Md5
+        };
         let PutObjectSseInput {
             server_side_encryption,
             ssekms_key_id,
@@ -1604,6 +1707,17 @@ impl DefaultObjectUsecase {
             object_lock.retain_until_date,
             &mut opts,
         )?;
+        // A lockless cache bucket commits without distributed locks. Every feature
+        // those locks protect sends the PUT down the locked path instead.
+        if matches!(origin, PutObjectOrigin::S3 { .. })
+            && matches!(object_lock_config_state, metadata_sys::ObjectLockConfigState::ConfirmedAbsent)
+            && !quota_enabled
+            && !opts.versioned
+            && !opts.version_suspended
+            && version_id.is_none()
+        {
+            opts.use_lockless_cache_write(&bucket);
+        }
         let eager_put_commit_cancellation =
             (use_zero_copy_eager_put_path || use_empty_or_small_eager_put_path).then(tokio_util::sync::CancellationToken::new);
         opts.put_object_cancellation = eager_put_commit_cancellation.clone();
@@ -1695,6 +1809,9 @@ impl DefaultObjectUsecase {
         // Additional-checksum (XXHash3/64/128, SHA-512) values to echo on the PutObject
         // response (#1256); captured at want_checksum set points before opts is moved.
         let mut put_extra_checksum_headers: Vec<(&'static str, String)> = Vec::new();
+        let cache_adapter = self.object_data_cache();
+        let fill_on_put_collector =
+            put_body_cache_collector(&cache_adapter, size, should_compress, ciphertext_passthrough, headers);
         let reader_prepare_stage_start = put_stage_metrics_enabled.then(Instant::now);
         let mut reader = if should_compress {
             let compressed_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
@@ -1707,7 +1824,8 @@ impl DefaultObjectUsecase {
             insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
 
             let mut hrd =
-                HashReader::from_stream(body, size, size, md5hex.take(), sha256hex.take(), false).map_err(ApiError::from)?;
+                HashReader::from_stream_with_etag(body, size, size, md5hex.take(), sha256hex.take(), false, etag_algorithm)
+                    .map_err(ApiError::from)?;
 
             if let Err(err) = hrd.add_checksum(headers, trailer_source(trailing_headers.clone()), false) {
                 return Err(ApiError::from(err).into());
@@ -1734,7 +1852,8 @@ impl DefaultObjectUsecase {
                 rustfs_io_metrics::record_zero_copy_write(body_size as usize, zero_copy_start.elapsed().as_secs_f64() * 1000.0);
                 let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
                 let reader =
-                    HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?;
+                    HashReader::from_stream_with_etag(eager_body, size, actual_size, md5hex, sha256hex, false, etag_algorithm)
+                        .map_err(ApiError::from)?;
                 rustfs_io_metrics::record_put_object_stage_duration_from(
                     "app_hash_reader_setup_zero_copy",
                     hash_reader_stage_start,
@@ -1756,8 +1875,16 @@ impl DefaultObjectUsecase {
                         eager_read_stage_start,
                     );
                     let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
-                    let reader = HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false)
-                        .map_err(ApiError::from)?;
+                    let reader = HashReader::from_stream_with_etag(
+                        eager_body,
+                        size,
+                        actual_size,
+                        md5hex,
+                        sha256hex,
+                        false,
+                        etag_algorithm,
+                    )
+                    .map_err(ApiError::from)?;
                     rustfs_io_metrics::record_put_object_stage_duration_from(
                         "app_hash_reader_setup_small_direct",
                         hash_reader_stage_start,
@@ -1778,8 +1905,16 @@ impl DefaultObjectUsecase {
                     );
                     let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
                     let eager_reader = PooledBufferReader::new(eager_body, body_size as usize);
-                    let reader = HashReader::from_stream(eager_reader, size, actual_size, md5hex, sha256hex, false)
-                        .map_err(ApiError::from)?;
+                    let reader = HashReader::from_stream_with_etag(
+                        eager_reader,
+                        size,
+                        actual_size,
+                        md5hex,
+                        sha256hex,
+                        false,
+                        etag_algorithm,
+                    )
+                    .map_err(ApiError::from)?;
                     rustfs_io_metrics::record_put_object_stage_duration_from(
                         "app_hash_reader_setup_small_pooled",
                         hash_reader_stage_start,
@@ -1788,12 +1923,22 @@ impl DefaultObjectUsecase {
                 }
             } else {
                 let streaming_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
+                let collector = fill_on_put_collector.clone();
                 let body = tokio::io::BufReader::with_capacity(
                     buffer_size,
-                    StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
+                    StreamReader::new(body.map(move |f| {
+                        let f = f.map_err(s3s_body_error_to_io);
+                        if let (Some(collector), Ok(frame)) = (&collector, &f) {
+                            collector
+                                .lock()
+                                .unwrap_or_else(|poisoned| poisoned.into_inner())
+                                .push(frame.clone());
+                        }
+                        f
+                    })),
                 );
-                let reader =
-                    HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?;
+                let reader = HashReader::from_stream_with_etag(body, size, actual_size, md5hex, sha256hex, false, etag_algorithm)
+                    .map_err(ApiError::from)?;
                 rustfs_io_metrics::record_put_object_stage_duration_from(
                     "app_reader_prepare_streaming",
                     streaming_reader_stage_start,
@@ -1923,6 +2068,7 @@ impl DefaultObjectUsecase {
         rustfs_io_metrics::record_put_object_stage_duration_from("app_replication_decision", replication_decision_stage_start);
 
         if dsc.replicate_any() {
+            opts.drop_lockless_cache_write();
             insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
             insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
             insert_str(
@@ -1932,7 +2078,6 @@ impl DefaultObjectUsecase {
             );
         }
 
-        let cache_adapter = self.object_data_cache();
         let cache_invalidate_before_stage_start = put_stage_metrics_enabled.then(Instant::now);
         let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
         rustfs_io_metrics::record_put_object_stage_duration_from(
@@ -2077,6 +2222,9 @@ impl DefaultObjectUsecase {
 
                 let cache_invalidate_stage_start = put_stage_metrics_enabled.then(Instant::now);
                 let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await;
+                if let Some(collector) = fill_on_put_collector.as_ref() {
+                    fill_object_data_cache_after_put(&cache_adapter, &bucket, &key, &obj_info, collector).await;
+                }
                 rustfs_io_metrics::record_put_object_stage_duration_from(
                     "app_post_store_data_cache_invalidate",
                     cache_invalidate_stage_start,
@@ -2242,6 +2390,18 @@ fn passthrough_compressed_actual_size(user_defined: &HashMap<String, String>) ->
 
 #[cfg(test)]
 mod tests {
+    #[test]
+    fn collected_put_body_joins_frames_only_when_they_match_the_size() {
+        let collector: super::PutBodyCacheCollector =
+            Arc::new(Mutex::new(vec![Bytes::from_static(b"hel"), Bytes::from_static(b"lo")]));
+        assert_eq!(super::collected_put_body(&collector, 5).as_deref(), Some(&b"hello"[..]));
+        let collector: super::PutBodyCacheCollector = Arc::new(Mutex::new(vec![Bytes::from_static(b"hello")]));
+        assert!(super::collected_put_body(&collector, 4).is_none(), "a size mismatch must not fill");
+        let collector: super::PutBodyCacheCollector = Arc::new(Mutex::new(vec![Bytes::from_static(b"hello")]));
+        assert_eq!(super::collected_put_body(&collector, 5).as_deref(), Some(&b"hello"[..]));
+        assert!(collector.lock().unwrap().is_empty(), "frames are released after the join");
+    }
+
     use super::*;
     use crate::app::storage_api::s3::dto::{GetObjectInput, HeadObjectInput, ObjectLockRetentionMode};
     use futures::StreamExt;

rustfs/src/app/object_data_cache/adapter.rs

@@ -110,6 +110,16 @@ impl ObjectDataCacheAdapter {
         self.cache.materialize_fill_enabled()
     }
 
+    /// Returns true when the adapter accepts any fill.
+    pub(crate) fn fill_enabled(&self) -> bool {
+        self.cache.fill_enabled()
+    }
+
+    /// Largest body a fill can accept.
+    pub(crate) fn fill_size_ceiling_bytes(&self) -> u64 {
+        self.cache.fill_size_ceiling_bytes()
+    }
+
     /// Builds an engine-level GET plan.
     pub(crate) fn plan_get(&self, request: ObjectDataCacheGetRequest<'_>) -> ObjectDataCacheGetPlan {
         self.cache.plan_get(request)

rustfs/src/app/storage_api.rs

@@ -343,6 +343,14 @@ pub(crate) mod bucket {
     };
     use std::sync::Arc;
 
+    pub(crate) mod etag {
+        pub(crate) use crate::storage::storage_api::ecstore_bucket::etag::bucket_uses_xxh3_etag;
+    }
+
+    pub(crate) mod lockless_cache {
+        pub(crate) use crate::storage::storage_api::ecstore_bucket::lockless_cache::is_lockless_cache_bucket;
+    }
+
     pub(crate) trait ObjectLockConfigExt {
         fn enabled(&self) -> bool;
     }
@@ -1627,7 +1635,7 @@ pub(crate) mod io {
     #[cfg(test)]
     pub(crate) use crate::storage::storage_api::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader};
     pub(crate) use crate::storage::storage_api::{
-        DynReader, HashReader, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader,
+        DynReader, EtagAlgorithm, HashReader, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader,
     };
 }
 

rustfs/src/startup_observability.rs

@@ -21,8 +21,6 @@ use crate::storage_api::startup::storage::{ECStore, init_compression_total_memor
 use tokio::io::AsyncWriteExt;
 use tokio_util::sync::CancellationToken;
 
-const GET_STAGE_LOCAL_SUMMARY_FILE: &str = "/var/log/rustfs/get-stage-summary.log";
-
 pub(crate) async fn init_observability_runtime(store: Arc<ECStore>, ctx: CancellationToken) {
     print_server_info();
     init_update_check();
@@ -51,7 +49,7 @@ fn configure_metric_gates(metrics_enabled: bool) {
         rustfs_utils::get_env_u64(rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE, 64).max(1);
     startup_runtime_sources::set_get_stage_local_summary_enabled(local_summary_enabled);
     startup_runtime_sources::set_get_stage_local_summary_sample_rate(sample_rate);
-    let put_stage_metrics_enabled = metrics_enabled
+    let put_stage_metrics_enabled = (metrics_enabled || local_summary_enabled)
         && rustfs_utils::get_env_bool(
             rustfs_config::observability::ENV_OBS_PUT_STAGE_METRICS_ENABLED,
             rustfs_config::DEFAULT_OBS_PUT_STAGE_METRICS_ENABLED,
@@ -64,6 +62,8 @@ fn configure_metric_gates(metrics_enabled: bool) {
 fn init_get_stage_local_summary_runtime(ctx: CancellationToken) {
     let interval_secs =
         rustfs_utils::get_env_u64(rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS, 15).max(1);
+    let summary_file = rustfs_utils::get_env_opt_str(rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_FILE)
+        .unwrap_or_else(|| rustfs_config::observability::DEFAULT_OBS_GET_STAGE_LOCAL_SUMMARY_FILE.to_owned());
     tokio::spawn(async move {
         let mut interval = tokio::time::interval(Duration::from_secs(interval_secs));
         interval.tick().await;
@@ -90,7 +90,7 @@ fn init_get_stage_local_summary_runtime(ctx: CancellationToken) {
                     let Ok(mut file) = tokio::fs::OpenOptions::new()
                         .create(true)
                         .append(true)
-                        .open(GET_STAGE_LOCAL_SUMMARY_FILE)
+                        .open(&summary_file)
                         .await
                     else {
                         break;
@@ -155,6 +155,54 @@ mod tests {
         startup_runtime_sources::set_get_stage_local_summary_sample_rate(previous_sample_rate);
     }
 
+    #[test]
+    #[serial_test::serial]
+    fn put_stage_local_summary_requires_explicit_opt_in() {
+        let previous_metrics = rustfs_io_metrics::metrics_enabled();
+        let previous_get_stages = rustfs_io_metrics::get_stage_metrics_enabled();
+        let previous_put_stages = rustfs_io_metrics::put_stage_metrics_enabled();
+        let previous_local_summary = rustfs_io_metrics::get_stage_local_summary_enabled();
+        let previous_sample_rate = rustfs_io_metrics::get_stage_local_summary_sample_rate();
+
+        temp_env::with_var(LOCAL_SUMMARY_ENV, Some("true"), || {
+            temp_env::with_var(LOCAL_SUMMARY_RATE_ENV, Some("1"), || {
+                temp_env::with_var(PUT_STAGE_ENV, None::<&str>, || {
+                    configure_metric_gates(false);
+                    assert!(!rustfs_io_metrics::put_stage_metrics_enabled());
+                });
+
+                temp_env::with_var(PUT_STAGE_ENV, Some("true"), || {
+                    configure_metric_gates(false);
+                    assert!(!rustfs_io_metrics::metrics_enabled());
+                    assert!(rustfs_io_metrics::put_stage_metrics_enabled());
+                    rustfs_io_metrics::take_get_stage_local_summary();
+
+                    rustfs_io_metrics::record_put_object_stage_duration("multipart_store_write", 2.0);
+                    rustfs_io_metrics::record_put_object_stage_duration("multipart_store_write", 4.0);
+                    rustfs_io_metrics::record_stage_duration("erasure_encode_write", 1.0);
+
+                    let rows = rustfs_io_metrics::take_get_stage_local_summary();
+                    let store_write = rows
+                        .iter()
+                        .find(|row| row.path == "put" && row.stage == "multipart_store_write")
+                        .expect("local summary should retain PUT stage observations");
+                    assert_eq!(store_write.sampled_count, 2);
+                    assert_eq!(store_write.duration_nanoseconds, 6_000_000);
+                    assert!(
+                        rows.iter()
+                            .any(|row| row.path == "internal" && row.stage == "erasure_encode_write")
+                    );
+                });
+            });
+        });
+
+        startup_runtime_sources::set_metrics_enabled(previous_metrics);
+        startup_runtime_sources::set_get_stage_metrics_enabled(previous_get_stages);
+        startup_runtime_sources::set_put_stage_metrics_enabled(previous_put_stages);
+        startup_runtime_sources::set_get_stage_local_summary_enabled(previous_local_summary);
+        startup_runtime_sources::set_get_stage_local_summary_sample_rate(previous_sample_rate);
+    }
+
     #[test]
     #[serial_test::serial]
     fn put_stage_metrics_require_explicit_opt_in() {

rustfs/src/storage/access.rs

@@ -33,6 +33,7 @@ use crate::storage::helper::DenialScope;
 use crate::storage::request_context::RequestContext;
 use crate::storage::storage_api::contract::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT;
 use crate::storage::storage_api::contract::namespace::NamespaceLocking as _;
+use crate::storage::storage_api::ecstore_bucket::lockless_cache::is_lockless_cache_bucket;
 use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot;
 use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
 use chacha20poly1305::{
@@ -2566,10 +2567,18 @@ async fn authorize_table_data_plane_if_needed<R: AuthorizationRequest + ?Sized>(
     let Some(admin_action) = table_data_plane_admin_action(action) else {
         return Ok(());
     };
-    if table_data_plane_publication_fence_required(req, action) {
+    // A lockless cache bucket skips the publication fence that holds off a
+    // concurrent table-bucket enablement; it still fences once the cached marker
+    // shows a table bucket.
+    let lockless_cache = is_lockless_cache_bucket(bucket);
+    if table_data_plane_publication_fence_required(req, action) && !lockless_cache {
         retain_table_bucket_publication_guard(req, bucket).await?;
     }
-    let table_bucket_enabled = table_bucket_enabled_for_data_plane(req, bucket).await?;
+    let mut table_bucket_enabled = table_bucket_enabled_for_data_plane(req, bucket).await?;
+    if lockless_cache && table_bucket_enabled && table_data_plane_publication_fence_required(req, action) {
+        retain_table_bucket_publication_guard(req, bucket).await?;
+        table_bucket_enabled = table_bucket_enabled_for_data_plane(req, bucket).await?;
+    }
     let Some(resource) = table_data_plane_resource_for_authorization(req, bucket, object, table_bucket_enabled).await? else {
         return Ok(());
     };
@@ -2621,10 +2630,18 @@ async fn deny_anonymous_table_data_plane_if_needed<R: AuthorizationRequest + ?Si
     if table_data_plane_admin_action(action).is_none() {
         return Ok(());
     }
-    if table_data_plane_publication_fence_required(req, action) {
+    // A lockless cache bucket skips the publication fence that holds off a
+    // concurrent table-bucket enablement; it still fences once the cached marker
+    // shows a table bucket.
+    let lockless_cache = is_lockless_cache_bucket(bucket);
+    if table_data_plane_publication_fence_required(req, action) && !lockless_cache {
         retain_table_bucket_publication_guard(req, bucket).await?;
     }
-    let table_bucket_enabled = table_bucket_enabled_for_data_plane(req, bucket).await?;
+    let mut table_bucket_enabled = table_bucket_enabled_for_data_plane(req, bucket).await?;
+    if lockless_cache && table_bucket_enabled && table_data_plane_publication_fence_required(req, action) {
+        retain_table_bucket_publication_guard(req, bucket).await?;
+        table_bucket_enabled = table_bucket_enabled_for_data_plane(req, bucket).await?;
+    }
     if table_data_plane_resource_for_authorization(req, bucket, object, table_bucket_enabled)
         .await?
         .is_some()

rustfs/src/storage/storage_api.rs

@@ -481,8 +481,8 @@ pub(crate) mod ecstore_bucket {
     #[cfg(test)]
     pub(crate) use rustfs_ecstore::api::bucket::lifecycle::tier_delete_journal::test_util::install_all_v6_fleet_capability_proof;
     pub(crate) use rustfs_ecstore::api::bucket::{
-        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys,
-        remote_s3_client, replication, tagging, target, utils,
+        bandwidth, bucket_target_sys, durability, etag, lifecycle, lockless_cache, metadata, metadata_sys, migration,
+        object_lock, policy_sys, remote_s3_client, replication, tagging, target, utils,
     };
     pub(crate) use rustfs_ecstore::api::bucket::{config_parse_mode, quota, versioning, versioning_sys};
 }
@@ -632,7 +632,7 @@ pub(crate) mod ecstore_rio {
     #[cfg(test)]
     pub(crate) use rustfs_ecstore::api::rio::{DecryptReader, EncryptReader, HardLimitReader, Reader, boxed_reader};
     pub(crate) use rustfs_ecstore::api::rio::{
-        DynReader, HashReader, ReadStream, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader,
+        DynReader, EtagAlgorithm, HashReader, ReadStream, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader,
     };
 }
 
@@ -815,6 +815,7 @@ pub(crate) type FileWriter = ecstore_disk::FileWriter;
 pub(crate) type SnapshotLeaseToken = ecstore_disk::SnapshotLeaseToken;
 pub(crate) type FS = super::ecfs::FS;
 pub(crate) type HashReader = ecstore_rio::HashReader;
+pub(crate) type EtagAlgorithm = ecstore_rio::EtagAlgorithm;
 pub(crate) type InstanceContext = ecstore_runtime::InstanceContext;
 pub(crate) type ServerContextSlot = crate::storage::runtime_sources::ServerContextSlot;
 pub(crate) type LocalPeerS3Client = ecstore_rpc::LocalPeerS3Client;