perf(get,heal): fix GET hot-path overhead and heal checkpoint scaling (#4237)

perf(get,heal): land verified fixes for backlog #800-#804

Five fixes from the GET-path performance audit and scanner/heal
completeness audit (rustfs/backlog#800..#804), each verified locally:

- backlog#800 (heal checkpoint O(N^2)): ResumeCheckpoint object sets are
  now HashSet (Vec::contains was O(n) per healed object; 1.5ms at n=1M
  vs 11ns measured), and per-object checkpoint/resume-state persistence
  is batched (1000 mutations / 5s) instead of rewriting the whole file
  per object. complete_page() prunes the sets at page boundaries so
  memory stays bounded; positions still persist unconditionally and
  legacy Vec-format checkpoints still deserialize.

- backlog#801 (DiskInfo.healing never set): erasure-set heal now writes
  a healing marker (.rustfs.sys/healing.bin) on the disks it rebuilds
  (endpoints plumbed via HealRequest/HealTask.heal_endpoints from the
  auto disk scanner) and clears it on success. LocalDisk::disk_info
  surfaces the marker, so scanner heal coordination, lock selection and
  admin/metrics healing counts see the rebuild.

- backlog#802 (cache probe after data read): new GetObjectBodyCacheHook
  in ecstore lets the app-layer object data cache serve the body inside
  get_object_reader, after metadata quorum resolution (etag known) but
  before the erasure shard read/decode. Previously a hit still paid the
  full disk read. Hook is None/no-op when the cache is disabled.

- backlog#803 (GET hot-path redundant work): ObjectInfo is cloned for
  event notification only when an event will actually be built (GET and
  HEAD paths; events are currently suppressed so the clone was pure
  waste); get_opts/put_opts/del_opts resolve bucket versioning with one
  metadata-sys lookup instead of two; skip_verify_bitrot and
  get_lock_acquire_timeout env reads are cached via OnceLock; the
  io-priority metric is no longer double-counted; GetObject input fields
  are cloned selectively instead of cloning the whole input.

- backlog#804 (disk permit starvation): the disk-read permit wait is now
  bounded (RUSTFS_OBJECT_DISK_PERMIT_WAIT_TIMEOUT, default 5s, 0 =
  previous unbounded behavior); on timeout the GET proceeds without a
  permit and the bypass is counted. DiskReadPermitReader also releases
  the permit at body EOF instead of holding it until the client drops
  the stream.

Verification: make pre-commit; cargo clippy -D warnings on the four
changed crates; full rustfs lib suite (2096 tests) green; rustfs-heal
lib suite green with new unit tests for checkpoint pruning/legacy
format/throttle, permit EOF release, and the cache hook (hit + SSE
skip). The heal_integration_test and one set_disk listing test fail
identically on unmodified main (pre-existing global-state ordering
flakes, verified via git stash A/B).

Co-authored-by: houseme <housemecn@gmail.com>

crates/ecstore/src/api/mod.rs

@@ -86,8 +86,8 @@ pub mod disk {
     pub use crate::disk::{
         BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
         CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
-        FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions,
-        RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
+        FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp,
+        ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
         validate_batch_read_version_item_count,
     };
     pub use crate::disk::{endpoint, error, error_reduce};
@@ -145,8 +145,8 @@ pub mod notification {
 
 pub mod object {
     pub use crate::object_api::{
-        BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, RangedDecompressReader,
-        StreamConsumer,
+        BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader,
+        RangedDecompressReader, StreamConsumer, register_get_object_body_cache_hook,
     };
 }
 

crates/ecstore/src/diagnostics/get.rs

@@ -22,6 +22,7 @@ pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE: &str = "codec_st
 pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE: &str = "codec_streaming_rustfs_engine";
 pub(crate) const GET_OBJECT_PATH_EMPTY: &str = "empty";
 pub(crate) const GET_OBJECT_PATH_DIRECT_MEMORY: &str = "direct_memory";
+pub(crate) const GET_OBJECT_PATH_BODY_CACHE: &str = "body_cache";
 pub(crate) const GET_OBJECT_PATH_INLINE_DIRECT: &str = "inline_direct";
 pub(crate) const GET_OBJECT_PATH_LEGACY_DUPLEX: &str = "legacy_duplex";
 pub(crate) const GET_OBJECT_PATH_REMOTE_TRANSITION: &str = "remote_transition";

crates/ecstore/src/disk/local.rs

@@ -864,6 +864,14 @@ impl LocalDisk {
                                 Vec::new()
                             }
                         };
+                        // An erasure-set heal drops a marker on the disks it is
+                        // rebuilding (see rustfs-heal); surface it so scanner
+                        // coordination, lock selection and admin/metrics see
+                        // the rebuild. Refreshed with this cache (~1s).
+                        let healing =
+                            tokio::fs::try_exists(root.join(super::RUSTFS_META_BUCKET).join(super::HEALING_MARKER_PATH))
+                                .await
+                                .unwrap_or(false);
                         let disk_info = DiskInfo {
                             total: info.total,
                             free: info.free,
@@ -876,13 +884,13 @@ impl LocalDisk {
                             root_disk: is_root_disk,
                             physical_device_ids,
                             id: disk_id,
+                            healing,
                             ..Default::default()
                         };
                         // if root {
                         //     return Err(Error::new(DiskError::DriveIsRoot));
                         // }
 
-                        // disk_info.healing =
                         Ok(disk_info)
                     }
                     Err(err) => Err(err.into()),

crates/ecstore/src/disk/mod.rs

@@ -33,6 +33,9 @@ pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp";
 pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash";
 pub const BUCKET_META_PREFIX: &str = "buckets";
 pub const FORMAT_CONFIG_FILE: &str = "format.json";
+/// Per-disk marker present while an erasure-set heal is rebuilding this disk.
+/// `LocalDisk::disk_info` reports `healing = true` while the file exists.
+pub const HEALING_MARKER_PATH: &str = "healing.bin";
 pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
 pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
 

crates/ecstore/src/object_api/body_cache_hook.rs

@@ -0,0 +1,49 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Hook that lets the application layer serve a GET body from its object data
+//! cache after metadata resolution but before the erasure data read.
+//!
+//! The cache itself lives above ecstore (it needs app-level config, metrics
+//! and invalidation), but the lookup must happen inside `get_object_reader`:
+//! probing earlier would require a second metadata fan-out, and probing later
+//! (after the reader is built) means a hit no longer saves any disk I/O.
+
+use crate::object_api::ObjectInfo;
+use bytes::Bytes;
+use std::sync::{Arc, OnceLock};
+
+/// Serves full-object GET bodies from a cache keyed by object identity.
+///
+/// Implementations must validate identity (etag/version/size) against the
+/// provided `ObjectInfo`, which reflects the just-resolved metadata quorum,
+/// and must return `None` for anything they cannot serve byte-identically
+/// (encrypted objects, remote/transitioned objects, size mismatches, ...).
+#[async_trait::async_trait]
+pub trait GetObjectBodyCacheHook: Send + Sync + 'static {
+    async fn lookup(&self, bucket: &str, object: &str, info: &ObjectInfo) -> Option<Bytes>;
+}
+
+static GET_OBJECT_BODY_CACHE_HOOK: OnceLock<Arc<dyn GetObjectBodyCacheHook>> = OnceLock::new();
+
+/// Register the process-wide GET body cache hook. First registration wins;
+/// later calls are ignored so tests and re-inits cannot swap the hook midway.
+pub fn register_get_object_body_cache_hook(hook: Arc<dyn GetObjectBodyCacheHook>) {
+    let _ = GET_OBJECT_BODY_CACHE_HOOK.set(hook);
+}
+
+/// The registered hook, if any.
+pub(crate) fn get_object_body_cache_hook() -> Option<&'static Arc<dyn GetObjectBodyCacheHook>> {
+    GET_OBJECT_BODY_CACHE_HOOK.get()
+}

crates/ecstore/src/object_api/mod.rs

@@ -52,8 +52,11 @@ use uuid::Uuid;
 pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
 pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
 
+mod body_cache_hook;
 mod readers;
 mod types;
 
+pub(crate) use body_cache_hook::get_object_body_cache_hook;
+pub use body_cache_hook::{GetObjectBodyCacheHook, register_get_object_body_cache_hook};
 pub use readers::*;
 pub use types::*;

crates/ecstore/src/set_disk/mod.rs

@@ -27,7 +27,7 @@ use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl};
 use crate::cluster::rpc::heal_bucket_local_on_disks;
 use crate::data_usage::record_compression_total_memory;
 use crate::diagnostics::get::{
-    GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_OBJECT_PATH_CODEC_STREAMING,
+    GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING,
     GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_DIRECT_MEMORY,
     GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_REMOTE_TRANSITION,
     GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE, GET_STAGE_LOCK_ACQUIRE,
@@ -51,6 +51,7 @@ use crate::error::{Error, Result, is_err_version_not_found};
 use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
 use crate::io_support::bitrot::{create_bitrot_reader, create_bitrot_reader_from_bytes, create_bitrot_writer};
 use crate::object_api::ObjectOptions;
+use crate::object_api::get_object_body_cache_hook;
 use crate::runtime::sources as runtime_sources;
 use crate::services::batch_processor::AsyncBatchProcessor;
 use crate::storage_api_contracts::{
@@ -453,11 +454,27 @@ mod write;
 
 /// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds)
 /// Defaults to 30 seconds if not set or invalid
+/// Lock acquisition timeout. Cached: this is consulted on every object
+/// lock acquisition and `std::env::var` takes a process-global lock. In test
+/// builds the env var is read directly so `temp_env` overrides take effect.
 pub fn get_lock_acquire_timeout() -> Duration {
-    Duration::from_secs(rustfs_utils::get_env_u64(
-        rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT,
-        rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT,
-    ))
+    #[cfg(test)]
+    {
+        Duration::from_secs(rustfs_utils::get_env_u64(
+            rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT,
+            rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT,
+        ))
+    }
+    #[cfg(not(test))]
+    {
+        static CACHED: OnceLock<Duration> = OnceLock::new();
+        *CACHED.get_or_init(|| {
+            Duration::from_secs(rustfs_utils::get_env_u64(
+                rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT,
+                rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT,
+            ))
+        })
+    }
 }
 
 pub fn is_object_lock_diag_enabled() -> bool {
@@ -2332,6 +2349,27 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
             ));
         }
 
+        // App-layer object data cache probe: metadata (etag/size) is resolved
+        // but no data shards have been read yet, so a hit skips the erasure
+        // read, bitrot verify and decode entirely. The hook validates object
+        // identity and rejects anything it cannot serve byte-identically.
+        if range.is_none()
+            && opts.part_number.is_none()
+            && let Some(hook) = get_object_body_cache_hook()
+            && let Some(body) = hook.lookup(bucket, object, &object_info).await
+        {
+            record_get_object_reader_path_observation(GET_OBJECT_PATH_BODY_CACHE, object_class, size_bucket);
+            let reader = GetObjectReader {
+                stream: Box::new(Cursor::new(body.clone())),
+                object_info,
+                buffered_body: Some(body),
+            };
+            if lock_optimization_enabled {
+                release_materialized_read_lock(bucket, object, read_lock_guard.take());
+            }
+            return Ok(reader);
+        }
+
         if is_get_small_object_direct_memory_eligible(&range, &object_info, &fi, opts) {
             let object_size = usize::try_from(object_info.size)
                 .map_err(|_| to_object_err(Error::other("direct-memory GET object size is invalid"), vec![bucket, object]))?;