feat(storage): harden internode data-path controls (#4224)

* fix(rio): propagate http writer shutdown errors

* fix(ecstore): unify remote lock rpc deadlines

* fix(storage): reject corrupt read multiple payloads

* feat(rio): add internode http tuning profiles

* feat(metrics): add internode baseline signals

* feat(ecstore): observe shard locality topology

* feat(ecstore): gate shard locality scheduling

* feat(ecstore): gate batch read version rpc

* feat(ecstore): observe batch processor adaptation

* feat(ecstore): gate batch processor observation

* docs: add get benchmark regression analysis

* docs: add issue 797 execution plan status

* fix(ecstore): require explicit batch rpc support

* fix(ecstore): honor documented batch read gate

* fix(ecstore): keep batch read gate stable per call

* chore: update workspace dependencies

* feat(ecstore): log batch read gate decisions

* feat(ecstore): count batch read gate decisions

* test(issue-797): add local internode A/B runner

* test(rio): fix tuning profile spelling fixture

* fix(protocols): adapt sftp channel open callbacks

* fix(metrics): wrap batch processor observation args

* chore(docs): keep issue notes local only

* fix(storage): address internode review feedback

* fix(storage): address internode data-path review findings

- Run the BatchReadVersion auto-mode unary fallback outside the batch
  RPC deadline so each read_version keeps its own per-op timeout and
  health accounting instead of racing the whole batch against one
  drive timeout.
- Cap adaptive batch-processor concurrency growth at a hard multiple
  of the configured baseline so sustained fast batches cannot ratchet
  past the configured limit.
- Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE,
  and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading
  the environment on hot paths.
- Skip shard read-cost collection in observe mode when stage metrics
  are disabled, and cache the local endpoint host list instead of
  rebuilding it on every read.
- Allow --warp-extra-args values starting with -- and drop the unused
  warp_hosts_csv helper in the issue-797 A/B runner.

* fix(storage): address internode data-path review findings

- Run the BatchReadVersion auto-mode unary fallback outside the batch
  RPC deadline so each read_version keeps its own per-op timeout and
  health accounting instead of racing the whole batch against one
  drive timeout.
- Cap adaptive batch-processor concurrency growth at a hard multiple
  of the configured baseline so sustained fast batches cannot ratchet
  past the configured limit.
- Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE,
  and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading
  the environment on hot paths.
- Skip shard read-cost collection in observe mode when stage metrics
  are disabled, and cache the local endpoint host list instead of
  rebuilding it on every read.
- Allow --warp-extra-args values starting with -- and drop the unused
  warp_hosts_csv helper in the issue-797 A/B runner.

Co-Authored-By: heihutu<heihutu@gmail.com>

* fix(storage): align buffer clamp test with media cap

* fix(ecstore): release optimized read locks before streaming

---------

Co-authored-by: Zhengchao An <anzhengchao@gmail.com>

crates/config/src/constants/internode.rs

@@ -32,6 +32,28 @@ pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3;
 pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS";
 pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 10;
 
+/// Profile selector for conservative internode HTTP data-plane client tuning.
+pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE";
+pub const DEFAULT_INTERNODE_HTTP_TUNING_PROFILE: &str = "legacy";
+
+/// Internode HTTP connection pool maximum idle connections per host.
+pub const ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST: &str = "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST";
+
+/// Internode HTTP connection pool idle timeout in seconds.
+pub const ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS";
+
+/// Internode HTTP/2 initial stream window size in bytes.
+pub const ENV_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE";
+
+/// Internode HTTP/2 initial connection window size in bytes.
+pub const ENV_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE";
+
+/// Whether internode HTTP/2 adaptive window sizing is enabled.
+pub const ENV_INTERNODE_HTTP2_ADAPTIVE_WINDOW: &str = "RUSTFS_INTERNODE_HTTP2_ADAPTIVE_WINDOW";
+
+/// Internode HTTP proxy mode: legacy, off, or system.
+pub const ENV_INTERNODE_HTTP_PROXY: &str = "RUSTFS_INTERNODE_HTTP_PROXY";
+
 /// Environment variable for selecting the internode data-plane transport backend.
 pub const ENV_RUSTFS_INTERNODE_DATA_TRANSPORT: &str = "RUSTFS_INTERNODE_DATA_TRANSPORT";
 pub const DEFAULT_INTERNODE_DATA_TRANSPORT: &str = "tcp-http";
@@ -53,6 +75,7 @@ mod tests {
         assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5);
         assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 3);
         assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 10);
+        assert_eq!(DEFAULT_INTERNODE_HTTP_TUNING_PROFILE, "legacy");
     }
 
     #[test]
@@ -68,6 +91,19 @@ mod tests {
             "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS"
         );
         assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS");
+        assert_eq!(ENV_INTERNODE_HTTP_TUNING_PROFILE, "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE");
+        assert_eq!(ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST, "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST");
+        assert_eq!(ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS, "RUSTFS_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS");
+        assert_eq!(
+            ENV_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE,
+            "RUSTFS_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE"
+        );
+        assert_eq!(
+            ENV_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE,
+            "RUSTFS_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE"
+        );
+        assert_eq!(ENV_INTERNODE_HTTP2_ADAPTIVE_WINDOW, "RUSTFS_INTERNODE_HTTP2_ADAPTIVE_WINDOW");
+        assert_eq!(ENV_INTERNODE_HTTP_PROXY, "RUSTFS_INTERNODE_HTTP_PROXY");
         assert_eq!(ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, "RUSTFS_INTERNODE_DATA_TRANSPORT");
         assert_eq!(DEFAULT_INTERNODE_DATA_TRANSPORT, "tcp-http");
         assert_eq!(INTERNODE_DATA_TRANSPORT_TCP, "tcp");

crates/config/src/constants/object.rs

@@ -191,8 +191,9 @@ pub const DEFAULT_OBJECT_IO_BUFFER_SIZE: usize = 128 * 1024;
 
 /// Environment variable to enable/disable lock optimization.
 ///
-/// When enabled, fully materialized reads may release read locks before the
-/// reader is returned. Streaming reads keep the lock until EOF or drop.
+/// When enabled, read locks may be released before the reader is returned.
+/// Disable this only when streaming readers must keep the object namespace
+/// locked until EOF or drop.
 ///
 /// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE`).
 pub const ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE: &str = "RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE";

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -21,8 +21,9 @@ use rustfs_lock::{LockClient, LockRequest};
 use rustfs_protos::{
     models::PingBodyBuilder,
     proto_gen::node_service::{
-        BatchGenerallyLockRequest, BatchGenerallyLockResponse, GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult,
-        PingRequest, PingResponse, node_service_server::NodeService,
+        BatchGenerallyLockRequest, BatchGenerallyLockResponse, BatchReadVersionRequest, BatchReadVersionResponse,
+        GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, PingRequest, PingResponse,
+        node_service_server::NodeService,
     },
 };
 use std::pin::Pin;
@@ -96,6 +97,13 @@ impl NodeService for MinimalLockNodeService {
         }))
     }
 
+    async fn batch_read_version(
+        &self,
+        _request: Request<BatchReadVersionRequest>,
+    ) -> Result<Response<BatchReadVersionResponse>, Status> {
+        Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
+    }
+
     async fn lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
         let request = request.into_inner();
         let args: LockRequest = match serde_json::from_str(&request.args) {

crates/ecstore/src/api/mod.rs

@@ -84,9 +84,11 @@ pub mod disk {
     pub use crate::disk::error_reduce::is_all_buckets_not_found;
     pub use crate::disk::local::ScanGuard;
     pub use crate::disk::{
-        BUCKET_META_PREFIX, CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
-        DiskStore, FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions,
+        BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
+        CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
+        FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions,
         RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
+        validate_batch_read_version_item_count,
     };
     pub use crate::disk::{endpoint, error, error_reduce};
     pub use bytes::Bytes;

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -20,8 +20,9 @@ use crate::cluster::rpc::internode_data_transport::{
 };
 use crate::disk::error::{Error, Result};
 use crate::disk::{
-    CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader,
-    FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
+    BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
+    DiskOption, FileInfoVersions, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
+    UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
     disk_store::{
         DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
         get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
@@ -30,6 +31,7 @@ use crate::disk::{
     },
     endpoint::Endpoint,
     health_state::{RuntimeDriveHealthState, get_drive_returning_probe_interval, record_drive_runtime_state},
+    validate_batch_read_version_item_count,
 };
 use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard};
 use crate::set_disk::DEFAULT_READ_BUFFER_SIZE;
@@ -40,11 +42,11 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use rustfs_protos::evict_failed_connection;
 use rustfs_protos::proto_gen::node_service::RenamePartRequest;
 use rustfs_protos::proto_gen::node_service::{
-    CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest,
-    DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest,
-    ReadMetadataRequest, ReadMultipleRequest, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
-    RenameFileRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
-    node_service_client::NodeServiceClient,
+    BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest,
+    DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
+    MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
+    ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest, StatVolumeRequest,
+    UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
 };
 use serde::{Serialize, de::DeserializeOwned};
 use std::{
@@ -63,7 +65,7 @@ use tokio::{
     time::timeout,
 };
 use tokio_util::sync::CancellationToken;
-use tonic::{Request, service::interceptor::InterceptedService, transport::Channel};
+use tonic::{Code, Request, service::interceptor::InterceptedService, transport::Channel};
 use tracing::{Instrument, debug, warn};
 use uuid::Uuid;
 
@@ -75,11 +77,78 @@ enum FailureHealthAction {
 
 const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2;
 const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20);
+const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
+const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC";
+const BATCH_METADATA_RPC_OFF: &str = "off";
+const BATCH_METADATA_RPC_AUTO: &str = "auto";
+const BATCH_METADATA_RPC_ON: &str = "on";
+const BATCH_READ_VERSION_GATE_ATTEMPT: &str = "attempt";
+const BATCH_READ_VERSION_GATE_OFF_UNARY: &str = "off_unary";
+const BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED: &str = "fallback_unimplemented";
+const BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK: &str = "unsupported_no_fallback";
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
 const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
 const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
 
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum BatchMetadataRpcMode {
+    Off,
+    Auto,
+    On,
+}
+
+impl BatchMetadataRpcMode {
+    fn as_str(self) -> &'static str {
+        match self {
+            Self::Off => BATCH_METADATA_RPC_OFF,
+            Self::Auto => BATCH_METADATA_RPC_AUTO,
+            Self::On => BATCH_METADATA_RPC_ON,
+        }
+    }
+
+    fn should_attempt(self) -> bool {
+        matches!(self, Self::Auto | Self::On)
+    }
+
+    fn should_fallback_on_unimplemented(self) -> bool {
+        matches!(self, Self::Auto)
+    }
+}
+
+fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode {
+    match raw.trim() {
+        value if value.eq_ignore_ascii_case(BATCH_METADATA_RPC_AUTO) => BatchMetadataRpcMode::Auto,
+        value if value.eq_ignore_ascii_case(BATCH_METADATA_RPC_ON) => BatchMetadataRpcMode::On,
+        value if value.eq_ignore_ascii_case(BATCH_METADATA_RPC_OFF) => BatchMetadataRpcMode::Off,
+        _ => BatchMetadataRpcMode::Off,
+    }
+}
+
+fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode {
+    rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)
+        .or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC))
+        .as_deref()
+        .map(parse_batch_metadata_rpc_mode)
+        .unwrap_or(BatchMetadataRpcMode::Off)
+}
+
+fn batch_metadata_rpc_mode() -> BatchMetadataRpcMode {
+    // The gate cannot change at runtime; parse it once instead of re-reading
+    // the environment on every batch RPC.
+    static MODE: std::sync::LazyLock<BatchMetadataRpcMode> = std::sync::LazyLock::new(batch_metadata_rpc_mode_from_env);
+    *MODE
+}
+
+fn record_batch_read_version_gate_decision(mode: BatchMetadataRpcMode, decision: &'static str) {
+    counter!(
+        "rustfs_remote_disk_batch_read_version_gate_total",
+        "mode" => mode.as_str(),
+        "decision" => decision
+    )
+    .increment(1);
+}
+
 async fn copy_stream_with_buffer<R, W>(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result<u64>
 where
     R: AsyncRead + Unpin,
@@ -769,6 +838,86 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str) -> Res
     serde_json::from_str(json).map_err(Error::from)
 }
 
+fn decode_read_multiple_response_items(response: ReadMultipleResponse, endpoint: &Endpoint) -> Result<Vec<ReadMultipleResp>> {
+    if !response.read_multiple_resps_bin.is_empty() {
+        if !response.read_multiple_resps.is_empty()
+            && response.read_multiple_resps.len() != response.read_multiple_resps_bin.len()
+        {
+            warn!(
+                event = EVENT_REMOTE_DISK_RPC,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
+                endpoint = %endpoint,
+                json_count = response.read_multiple_resps.len(),
+                msgpack_count = response.read_multiple_resps_bin.len(),
+                op = "read_multiple",
+                state = "response_count_mismatch",
+                "Remote disk ReadMultiple compatibility payload counts differ"
+            );
+        }
+
+        let mut read_multiple_resps = Vec::with_capacity(response.read_multiple_resps_bin.len());
+        for (index, buf) in response.read_multiple_resps_bin.iter().enumerate() {
+            let resp = decode_msgpack_or_json::<ReadMultipleResp>(buf, "").map_err(|err| {
+                Error::other(format!("decode ReadMultipleResp msgpack item {index} from {endpoint} failed: {err}"))
+            })?;
+            read_multiple_resps.push(resp);
+        }
+        return Ok(read_multiple_resps);
+    }
+
+    let mut read_multiple_resps = Vec::with_capacity(response.read_multiple_resps.len());
+    for (index, json_str) in response.read_multiple_resps.iter().enumerate() {
+        let resp = serde_json::from_str::<ReadMultipleResp>(json_str)
+            .map_err(|err| Error::other(format!("decode ReadMultipleResp json item {index} from {endpoint} failed: {err}")))?;
+        read_multiple_resps.push(resp);
+    }
+
+    Ok(read_multiple_resps)
+}
+
+fn decode_batch_read_version_response_items(
+    response: BatchReadVersionResponse,
+    endpoint: &Endpoint,
+) -> Result<Vec<BatchReadVersionResp>> {
+    if !response.batch_read_version_resps_bin.is_empty() {
+        if !response.batch_read_version_resps.is_empty()
+            && response.batch_read_version_resps.len() != response.batch_read_version_resps_bin.len()
+        {
+            warn!(
+                event = EVENT_REMOTE_DISK_RPC,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
+                endpoint = %endpoint,
+                json_count = response.batch_read_version_resps.len(),
+                msgpack_count = response.batch_read_version_resps_bin.len(),
+                op = "batch_read_version",
+                state = "response_count_mismatch",
+                "Remote disk BatchReadVersion compatibility payload counts differ"
+            );
+        }
+
+        let mut batch_read_version_resps = Vec::with_capacity(response.batch_read_version_resps_bin.len());
+        for (index, buf) in response.batch_read_version_resps_bin.iter().enumerate() {
+            let resp = decode_msgpack_or_json::<BatchReadVersionResp>(buf, "").map_err(|err| {
+                Error::other(format!("decode BatchReadVersionResp msgpack item {index} from {endpoint} failed: {err}"))
+            })?;
+            batch_read_version_resps.push(resp);
+        }
+        return Ok(batch_read_version_resps);
+    }
+
+    let mut batch_read_version_resps = Vec::with_capacity(response.batch_read_version_resps.len());
+    for (index, json_str) in response.batch_read_version_resps.iter().enumerate() {
+        let resp = serde_json::from_str::<BatchReadVersionResp>(json_str).map_err(|err| {
+            Error::other(format!("decode BatchReadVersionResp json item {index} from {endpoint} failed: {err}"))
+        })?;
+        batch_read_version_resps.push(resp);
+    }
+
+    Ok(batch_read_version_resps)
+}
+
 // TODO: all api need to handle errors
 #[async_trait::async_trait]
 impl DiskAPI for RemoteDisk {
@@ -1462,6 +1611,99 @@ impl DiskAPI for RemoteDisk {
         .await
     }
 
+    #[tracing::instrument(skip(self, req))]
+    async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
+        validate_batch_read_version_item_count(req.items.len())?;
+
+        let mode = batch_metadata_rpc_mode();
+        if !mode.should_attempt() {
+            record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_OFF_UNARY);
+            return batch_read_version_one_by_one(self, req).await;
+        }
+        record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_ATTEMPT);
+
+        debug!(
+            event = EVENT_REMOTE_DISK_RPC,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
+            endpoint = %self.endpoint,
+            item_count = req.items.len(),
+            batch_metadata_rpc_mode = mode.as_str(),
+            op = "batch_read_version",
+            state = "started",
+            "Remote disk RPC started"
+        );
+        let batch_read_version_req = serde_json::to_string(&req)?;
+        let batch_read_version_req_bin = encode_msgpack(&req)?;
+
+        let batch_result = self
+            .execute_with_timeout_for_op(
+                "batch_read_version",
+                move || async move {
+                    let disk = self.disk_ref().await;
+                    let mut client = self
+                        .get_client()
+                        .await
+                        .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
+                    let request = Request::new(BatchReadVersionRequest {
+                        disk,
+                        batch_read_version_req,
+                        batch_read_version_req_bin: batch_read_version_req_bin.into(),
+                    });
+
+                    let response = match client.batch_read_version(request).await {
+                        Ok(response) => response.into_inner(),
+                        Err(status) if status.code() == Code::Unimplemented => {
+                            if mode.should_fallback_on_unimplemented() {
+                                record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED);
+                                warn!(
+                                    event = EVENT_REMOTE_DISK_RPC,
+                                    component = LOG_COMPONENT_ECSTORE,
+                                    subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
+                                    endpoint = %self.endpoint,
+                                    batch_metadata_rpc_mode = mode.as_str(),
+                                    op = "batch_read_version",
+                                    state = "fallback_unimplemented",
+                                    "Remote disk BatchReadVersion unsupported; falling back to unary read_version"
+                                );
+                                return Ok(None);
+                            }
+
+                            record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK);
+                            warn!(
+                                event = EVENT_REMOTE_DISK_RPC,
+                                component = LOG_COMPONENT_ECSTORE,
+                                subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
+                                endpoint = %self.endpoint,
+                                batch_metadata_rpc_mode = mode.as_str(),
+                                op = "batch_read_version",
+                                state = "unsupported_no_fallback",
+                                "Remote disk BatchReadVersion unsupported and explicit batch RPC mode forbids fallback"
+                            );
+                            return Err(Error::from(status));
+                        }
+                        Err(status) => return Err(Error::from(status)),
+                    };
+
+                    if !response.success {
+                        return Err(response.error.unwrap_or_default().into());
+                    }
+
+                    decode_batch_read_version_response_items(response, &self.endpoint).map(Some)
+                },
+                get_max_timeout_duration(),
+            )
+            .await?;
+
+        match batch_result {
+            Some(batch_read_version_resps) => Ok(batch_read_version_resps),
+            // Run the unary fallback outside the batch RPC deadline so each
+            // read_version keeps its own per-op timeout and health accounting
+            // instead of racing the whole batch against one drive timeout.
+            None => batch_read_version_one_by_one(self, req).await,
+        }
+    }
+
     #[tracing::instrument(level = "debug", skip(self))]
     async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
         debug!(
@@ -2069,19 +2311,7 @@ impl DiskAPI for RemoteDisk {
                     return Err(response.error.unwrap_or_default().into());
                 }
 
-                let read_multiple_resps = if !response.read_multiple_resps_bin.is_empty() {
-                    response
-                        .read_multiple_resps_bin
-                        .into_iter()
-                        .filter_map(|buf| decode_msgpack_or_json::<ReadMultipleResp>(&buf, "").ok())
-                        .collect()
-                } else {
-                    response
-                        .read_multiple_resps
-                        .into_iter()
-                        .filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
-                        .collect()
-                };
+                let read_multiple_resps = decode_read_multiple_response_items(response, &self.endpoint)?;
 
                 Ok(read_multiple_resps)
             },
@@ -2413,6 +2643,196 @@ mod tests {
         }
     }
 
+    fn sample_read_multiple_resp(file: &str, data: &[u8]) -> ReadMultipleResp {
+        ReadMultipleResp {
+            bucket: "bucket".to_string(),
+            prefix: "prefix".to_string(),
+            file: file.to_string(),
+            exists: true,
+            data: data.to_vec(),
+            ..Default::default()
+        }
+    }
+
+    fn sample_remote_endpoint() -> Endpoint {
+        Endpoint {
+            url: url::Url::parse("http://server:9000/disk-a").expect("endpoint URL should parse"),
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        }
+    }
+
+    #[test]
+    fn read_multiple_response_decode_prefers_msgpack_payloads() {
+        let endpoint = sample_remote_endpoint();
+        let msgpack_resp = sample_read_multiple_resp("msgpack", b"binary");
+        let json_resp = sample_read_multiple_resp("json", b"fallback");
+        let response = ReadMultipleResponse {
+            success: true,
+            read_multiple_resps: vec![serde_json::to_string(&json_resp).expect("json fallback should encode")],
+            read_multiple_resps_bin: vec![encode_msgpack(&msgpack_resp).expect("msgpack response should encode").into()],
+            error: None,
+        };
+
+        let decoded = decode_read_multiple_response_items(response, &endpoint).expect("msgpack response should decode");
+
+        assert_eq!(decoded.len(), 1);
+        assert_eq!(decoded[0].file, "msgpack");
+        assert_eq!(decoded[0].data, b"binary");
+    }
+
+    #[test]
+    fn read_multiple_response_decode_falls_back_to_json_payloads() {
+        let endpoint = sample_remote_endpoint();
+        let json_resp = sample_read_multiple_resp("json", b"fallback");
+        let response = ReadMultipleResponse {
+            success: true,
+            read_multiple_resps: vec![serde_json::to_string(&json_resp).expect("json fallback should encode")],
+            read_multiple_resps_bin: Vec::new(),
+            error: None,
+        };
+
+        let decoded = decode_read_multiple_response_items(response, &endpoint).expect("json response should decode");
+
+        assert_eq!(decoded.len(), 1);
+        assert_eq!(decoded[0].file, "json");
+        assert_eq!(decoded[0].data, b"fallback");
+    }
+
+    #[test]
+    fn read_multiple_response_decode_reports_corrupt_msgpack_item() {
+        let endpoint = sample_remote_endpoint();
+        let response = ReadMultipleResponse {
+            success: true,
+            read_multiple_resps: Vec::new(),
+            read_multiple_resps_bin: vec![
+                encode_msgpack(&sample_read_multiple_resp("ok", b"data"))
+                    .expect("msgpack response should encode")
+                    .into(),
+                bytes::Bytes::from_static(b"not-msgpack"),
+            ],
+            error: None,
+        };
+
+        let err = decode_read_multiple_response_items(response, &endpoint).expect_err("corrupt msgpack item should fail");
+        let err = err.to_string();
+
+        assert!(err.contains("ReadMultipleResp msgpack item 1"), "unexpected error: {err}");
+        assert!(err.contains("server:9000"), "unexpected error: {err}");
+    }
+
+    fn sample_batch_read_version_resp(index: usize, path: &str, success: bool) -> BatchReadVersionResp {
+        BatchReadVersionResp {
+            index,
+            path: path.to_string(),
+            version_id: "version-a".to_string(),
+            success,
+            error: if success {
+                String::new()
+            } else {
+                "file version not found".to_string()
+            },
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn batch_read_version_response_decode_prefers_msgpack_payloads() {
+        let endpoint = sample_remote_endpoint();
+        let msgpack_resp = sample_batch_read_version_resp(7, "msgpack-object", true);
+        let json_resp = sample_batch_read_version_resp(1, "json-object", false);
+        let response = BatchReadVersionResponse {
+            success: true,
+            batch_read_version_resps: vec![serde_json::to_string(&json_resp).expect("json fallback should encode")],
+            batch_read_version_resps_bin: vec![encode_msgpack(&msgpack_resp).expect("msgpack response should encode").into()],
+            error: None,
+        };
+
+        let decoded = decode_batch_read_version_response_items(response, &endpoint).expect("msgpack response should decode");
+
+        assert_eq!(decoded.len(), 1);
+        assert_eq!(decoded[0].index, 7);
+        assert_eq!(decoded[0].path, "msgpack-object");
+        assert!(decoded[0].success);
+    }
+
+    #[test]
+    fn batch_read_version_response_decode_reports_corrupt_msgpack_item() {
+        let endpoint = sample_remote_endpoint();
+        let response = BatchReadVersionResponse {
+            success: true,
+            batch_read_version_resps: Vec::new(),
+            batch_read_version_resps_bin: vec![
+                encode_msgpack(&sample_batch_read_version_resp(0, "ok", true))
+                    .expect("msgpack response should encode")
+                    .into(),
+                bytes::Bytes::from_static(b"not-msgpack"),
+            ],
+            error: None,
+        };
+
+        let err = decode_batch_read_version_response_items(response, &endpoint)
+            .expect_err("corrupt msgpack item should fail")
+            .to_string();
+
+        assert!(err.contains("BatchReadVersionResp msgpack item 1"), "unexpected error: {err}");
+        assert!(err.contains("server:9000"), "unexpected error: {err}");
+    }
+
+    #[test]
+    fn batch_metadata_rpc_mode_defaults_to_off_and_parses_supported_values() {
+        assert_eq!(parse_batch_metadata_rpc_mode(""), BatchMetadataRpcMode::Off);
+        assert_eq!(parse_batch_metadata_rpc_mode("off"), BatchMetadataRpcMode::Off);
+        assert_eq!(parse_batch_metadata_rpc_mode("auto"), BatchMetadataRpcMode::Auto);
+        assert_eq!(parse_batch_metadata_rpc_mode("on"), BatchMetadataRpcMode::On);
+        assert_eq!(parse_batch_metadata_rpc_mode("unknown"), BatchMetadataRpcMode::Off);
+        assert_eq!(BatchMetadataRpcMode::Off.as_str(), "off");
+        assert_eq!(BatchMetadataRpcMode::Auto.as_str(), "auto");
+        assert_eq!(BatchMetadataRpcMode::On.as_str(), "on");
+        assert!(!BatchMetadataRpcMode::Off.should_attempt());
+        assert!(BatchMetadataRpcMode::Auto.should_attempt());
+        assert!(BatchMetadataRpcMode::On.should_attempt());
+        assert_eq!(BATCH_READ_VERSION_GATE_ATTEMPT, "attempt");
+        assert_eq!(BATCH_READ_VERSION_GATE_OFF_UNARY, "off_unary");
+        assert_eq!(BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED, "fallback_unimplemented");
+        assert_eq!(BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK, "unsupported_no_fallback");
+    }
+
+    #[test]
+    fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() {
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")),
+                (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
+            ],
+            || {
+                assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::Auto);
+            },
+        );
+    }
+
+    #[test]
+    fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() {
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>),
+                (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
+            ],
+            || {
+                assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On);
+            },
+        );
+    }
+
+    #[test]
+    fn batch_metadata_rpc_mode_only_auto_falls_back_on_unimplemented() {
+        assert!(!BatchMetadataRpcMode::Off.should_fallback_on_unimplemented());
+        assert!(BatchMetadataRpcMode::Auto.should_fallback_on_unimplemented());
+        assert!(!BatchMetadataRpcMode::On.should_fallback_on_unimplemented());
+    }
+
     #[test]
     fn rename_data_file_info_named_msgpack_is_smaller_than_json() {
         let file_info = sample_rename_data_file_info();

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -249,6 +249,22 @@ impl RemoteClient {
             }
         }
     }
+
+    fn unknown_lock_info(lock_id: &LockId) -> LockInfo {
+        LockInfo {
+            id: lock_id.clone(),
+            resource: lock_id.resource.clone(),
+            lock_type: LockType::Exclusive,
+            status: LockStatus::Acquired,
+            owner: "unknown".to_string(),
+            acquired_at: std::time::SystemTime::now(),
+            expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
+            last_refreshed: std::time::SystemTime::now(),
+            metadata: LockMetadata::default(),
+            priority: LockPriority::Normal,
+            wait_start_time: None,
+        }
+    }
 }
 
 #[async_trait]
@@ -338,11 +354,11 @@ impl LockClient for RemoteClient {
         let request_string = serde_json::to_string(&unlock_request)
             .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
         let mut client = self.get_client().await?;
+        let resource_summary = unlock_request.resource.to_string();
         let req = Request::new(GenerallyLockRequest { args: request_string });
-        let resp = client
-            .un_lock(req)
-            .await
-            .map_err(|e| LockError::internal(e.to_string()))?
+        let resp = self
+            .execute_rpc("release", &resource_summary, client.un_lock(req))
+            .await?
             .into_inner();
         if let Some(error_info) = resp.error_info {
             return Err(LockError::internal(error_info));
@@ -351,21 +367,25 @@ impl LockClient for RemoteClient {
     }
 
     async fn release_locks_batch(&self, lock_ids: &[LockId]) -> Result<Vec<bool>> {
+        if lock_ids.is_empty() {
+            return Ok(Vec::new());
+        }
+
+        let unlock_requests = lock_ids.iter().map(Self::create_unlock_request).collect::<Vec<_>>();
         let mut client = self.get_client().await?;
+        let resource_summary = Self::summarize_resources(&unlock_requests);
         let req = Request::new(BatchGenerallyLockRequest {
-            args: lock_ids
+            args: unlock_requests
                 .iter()
-                .map(|lock_id| {
-                    serde_json::to_string(&Self::create_unlock_request(lock_id))
-                        .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
+                .map(|request| {
+                    serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
                 })
                 .collect::<Result<Vec<_>>>()?,
         });
 
-        let resp = client
-            .un_lock_batch(req)
-            .await
-            .map_err(|e| LockError::internal(e.to_string()))?
+        let resp = self
+            .execute_rpc("release_batch", &resource_summary, client.un_lock_batch(req))
+            .await?
             .into_inner();
 
         Ok(lock_ids
@@ -379,14 +399,14 @@ impl LockClient for RemoteClient {
         info!("remote refresh for {}", lock_id);
         let refresh_request = Self::create_unlock_request(lock_id);
         let mut client = self.get_client().await?;
+        let resource_summary = refresh_request.resource.to_string();
         let req = Request::new(GenerallyLockRequest {
             args: serde_json::to_string(&refresh_request)
                 .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
         });
-        let resp = client
-            .refresh(req)
-            .await
-            .map_err(|e| LockError::internal(e.to_string()))?
+        let resp = self
+            .execute_rpc("refresh", &resource_summary, client.refresh(req))
+            .await?
             .into_inner();
         if let Some(error_info) = resp.error_info {
             return Err(LockError::internal(error_info));
@@ -398,14 +418,14 @@ impl LockClient for RemoteClient {
         info!("remote force_release for {}", lock_id);
         let force_request = Self::create_unlock_request(lock_id);
         let mut client = self.get_client().await?;
+        let resource_summary = force_request.resource.to_string();
         let req = Request::new(GenerallyLockRequest {
             args: serde_json::to_string(&force_request)
                 .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
         });
-        let resp = client
-            .force_un_lock(req)
-            .await
-            .map_err(|e| LockError::internal(e.to_string()))?
+        let resp = self
+            .execute_rpc("force_release", &resource_summary, client.force_un_lock(req))
+            .await?
             .into_inner();
         if let Some(error_info) = resp.error_info {
             return Err(LockError::internal(error_info));
@@ -419,6 +439,7 @@ impl LockClient for RemoteClient {
         // Since there's no direct status query in the gRPC service,
         // we attempt a non-blocking lock acquisition to check if the resource is available
         let status_request = Self::create_unlock_request(lock_id);
+        let resource_summary = status_request.resource.to_string();
         let mut client = self.get_client().await?;
 
         // Try to acquire a very short-lived lock to test availability
@@ -428,56 +449,27 @@ impl LockClient for RemoteClient {
         });
 
         // Try exclusive lock first with very short timeout
-        let resp = client.lock(req).await;
+        let resp = match self.execute_rpc("check_status", &resource_summary, client.lock(req)).await {
+            Ok(response) => response.into_inner(),
+            Err(_) => return Ok(Some(Self::unknown_lock_info(lock_id))),
+        };
 
-        match resp {
-            Ok(response) => {
-                let resp = response.into_inner();
-                if resp.success {
-                    // If we successfully acquired the lock, the resource was free
-                    // Immediately release it
-                    let release_req = Request::new(GenerallyLockRequest {
-                        args: serde_json::to_string(&status_request)
-                            .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
-                    });
-                    let _ = client.un_lock(release_req).await; // Best effort release
+        if resp.success {
+            // If we successfully acquired the lock, the resource was free.
+            // Immediately release it on a best-effort basis.
+            let release_req = Request::new(GenerallyLockRequest {
+                args: serde_json::to_string(&status_request)
+                    .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
+            });
+            let _ = self
+                .execute_rpc("check_status_release", &resource_summary, client.un_lock(release_req))
+                .await;
 
-                    // Return None since no one was holding the lock
-                    Ok(None)
-                } else {
-                    // Lock acquisition failed, meaning someone is holding it
-                    // We can't determine the exact details remotely, so return a generic status
-                    Ok(Some(LockInfo {
-                        id: lock_id.clone(),
-                        resource: lock_id.resource.clone(),
-                        lock_type: LockType::Exclusive, // We can't know the exact type
-                        status: LockStatus::Acquired,
-                        owner: "unknown".to_string(), // Remote client can't determine owner
-                        acquired_at: std::time::SystemTime::now(),
-                        expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
-                        last_refreshed: std::time::SystemTime::now(),
-                        metadata: LockMetadata::default(),
-                        priority: LockPriority::Normal,
-                        wait_start_time: None,
-                    }))
-                }
-            }
-            Err(_) => {
-                // Communication error or lock is held
-                Ok(Some(LockInfo {
-                    id: lock_id.clone(),
-                    resource: lock_id.resource.clone(),
-                    lock_type: LockType::Exclusive,
-                    status: LockStatus::Acquired,
-                    owner: "unknown".to_string(),
-                    acquired_at: std::time::SystemTime::now(),
-                    expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
-                    last_refreshed: std::time::SystemTime::now(),
-                    metadata: LockMetadata::default(),
-                    priority: LockPriority::Normal,
-                    wait_start_time: None,
-                }))
-            }
+            Ok(None)
+        } else {
+            // Lock acquisition failed, meaning someone is holding it.
+            // We can't determine the exact details remotely, so return a generic status.
+            Ok(Some(Self::unknown_lock_info(lock_id)))
         }
     }
 
@@ -558,6 +550,17 @@ mod tests {
         Some((addr, task))
     }
 
+    async fn closed_listener_addr() -> Option<String> {
+        let listener = match TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let addr = format!("http://{}", listener.local_addr().expect("listener local address should be available"));
+        drop(listener);
+        Some(addr)
+    }
+
     async fn cache_lazy_channel(addr: &str) {
         let channel = TonicEndpoint::from_shared(addr.to_string()).unwrap().connect_lazy();
         runtime_sources::cache_test_node_channel(addr.to_string(), channel).await;
@@ -664,6 +667,113 @@ mod tests {
         accept_task.abort();
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn test_remote_client_release_uses_rpc_timeout_and_evicts_connection() {
+        ensure_test_rpc_secret();
+        let Some((addr, accept_task)) = spawn_hanging_listener().await else {
+            return;
+        };
+        cache_lazy_channel(&addr).await;
+        assert!(runtime_sources::test_node_channel_is_cached(&addr).await);
+
+        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_RPC_TIMEOUT_MS, Some("50"))], async {
+            let client = RemoteClient::new(addr.clone());
+            let request = test_lock_request(Duration::from_millis(5));
+            let started_at = tokio::time::Instant::now();
+
+            let err = client.release(&request.lock_id).await.expect_err("release should time out");
+            let elapsed = started_at.elapsed();
+
+            assert!(
+                elapsed >= Duration::from_millis(40),
+                "remote release RPC should use configured transport timeout, got {elapsed:?}"
+            );
+            assert!(
+                elapsed < Duration::from_secs(1),
+                "test RPC timeout should keep the test fast, got {elapsed:?}"
+            );
+            assert!(matches!(err, LockError::Timeout { .. }), "expected remote release timeout, got {err:?}");
+            assert!(
+                !runtime_sources::test_node_channel_is_cached(&addr).await,
+                "release timeout should evict cached connection"
+            );
+        })
+        .await;
+
+        accept_task.abort();
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn test_remote_client_refresh_tonic_error_evicts_connection() {
+        ensure_test_rpc_secret();
+        let Some(addr) = closed_listener_addr().await else {
+            return;
+        };
+        cache_lazy_channel(&addr).await;
+        assert!(runtime_sources::test_node_channel_is_cached(&addr).await);
+
+        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_RPC_TIMEOUT_MS, Some("500"))], async {
+            let client = RemoteClient::new(addr.clone());
+            let request = test_lock_request(Duration::from_millis(5));
+
+            let err = client
+                .refresh(&request.lock_id)
+                .await
+                .expect_err("refresh should report tonic failure");
+
+            assert!(
+                err.to_string().contains("refresh RPC failed"),
+                "expected refresh RPC failure marker, got {err}"
+            );
+            assert!(
+                !runtime_sources::test_node_channel_is_cached(&addr).await,
+                "refresh tonic error should evict cached connection"
+            );
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn test_remote_client_check_status_timeout_evicts_connection_and_preserves_status_shape() {
+        ensure_test_rpc_secret();
+        let Some((addr, accept_task)) = spawn_hanging_listener().await else {
+            return;
+        };
+        cache_lazy_channel(&addr).await;
+        assert!(runtime_sources::test_node_channel_is_cached(&addr).await);
+
+        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_RPC_TIMEOUT_MS, Some("50"))], async {
+            let client = RemoteClient::new(addr.clone());
+            let request = test_lock_request(Duration::from_millis(5));
+            let started_at = tokio::time::Instant::now();
+
+            let status = client.check_status(&request.lock_id).await.unwrap();
+            let elapsed = started_at.elapsed();
+
+            assert!(
+                elapsed >= Duration::from_millis(40),
+                "remote check_status RPC should use configured transport timeout, got {elapsed:?}"
+            );
+            assert!(
+                elapsed < Duration::from_secs(1),
+                "test RPC timeout should keep the test fast, got {elapsed:?}"
+            );
+            let info = status.expect("communication failure should preserve unknown lock status shape");
+            assert_eq!(info.id, request.lock_id);
+            assert_eq!(info.owner, "unknown");
+            assert!(
+                !runtime_sources::test_node_channel_is_cached(&addr).await,
+                "check_status timeout should evict cached connection"
+            );
+        })
+        .await;
+
+        accept_task.abort();
+    }
+
     #[test]
     #[serial_test::serial]
     fn test_remote_client_rpc_timeout_honors_configured_deadline() {

crates/ecstore/src/disk/mod.rs

@@ -577,6 +577,9 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
         version_id: &str,
         opts: &ReadOptions,
     ) -> Result<FileInfo>;
+    async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
+        batch_read_version_one_by_one(self, req).await
+    }
     async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo>;
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes>;
     async fn rename_data(
@@ -637,6 +640,41 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     fn start_scan(&self) -> ScanGuard;
 }
 
+pub async fn batch_read_version_one_by_one<D>(disk: &D, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>>
+where
+    D: DiskAPI + ?Sized,
+{
+    validate_batch_read_version_item_count(req.items.len())?;
+
+    let mut responses = Vec::with_capacity(req.items.len());
+    for (index, item) in req.items.iter().enumerate() {
+        let response = match disk
+            .read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
+            .await
+        {
+            Ok(file_info) => BatchReadVersionResp {
+                index,
+                path: item.path.clone(),
+                version_id: item.version_id.clone(),
+                success: true,
+                file_info,
+                error: String::new(),
+            },
+            Err(err) => BatchReadVersionResp {
+                index,
+                path: item.path.clone(),
+                version_id: item.version_id.clone(),
+                success: false,
+                file_info: FileInfo::default(),
+                error: err.to_string(),
+            },
+        };
+        responses.push(response);
+    }
+
+    Ok(responses)
+}
+
 #[derive(Debug, Default, Serialize, Deserialize)]
 pub struct CheckPartsResp {
     pub results: Vec<usize>,
@@ -807,6 +845,41 @@ pub struct ReadMultipleResp {
     pub mod_time: Option<OffsetDateTime>,
 }
 
+pub const BATCH_READ_VERSION_MAX_ITEMS: usize = 128;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BatchReadVersionItem {
+    pub org_volume: String,
+    pub volume: String,
+    pub path: String,
+    pub version_id: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BatchReadVersionReq {
+    pub items: Vec<BatchReadVersionItem>,
+    pub opts: ReadOptions,
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct BatchReadVersionResp {
+    pub index: usize,
+    pub path: String,
+    pub version_id: String,
+    pub success: bool,
+    pub file_info: FileInfo,
+    pub error: String,
+}
+
+pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
+    if item_count > BATCH_READ_VERSION_MAX_ITEMS {
+        return Err(DiskError::other(format!(
+            "batch read version item count {item_count} exceeds limit {BATCH_READ_VERSION_MAX_ITEMS}"
+        )));
+    }
+    Ok(())
+}
+
 #[derive(Debug, Deserialize, Serialize)]
 pub struct VolumeInfo {
     pub name: String,

crates/ecstore/src/erasure/coding/decode.rs

@@ -40,18 +40,54 @@ use tracing::{error, warn};
 
 type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, bool)> + Send + 'a>>;
 
+const ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING: &str = "RUSTFS_SHARD_LOCALITY_SCHEDULING";
 const ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE: &str = "RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE";
-const DEFAULT_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE: bool = false;
+const SHARD_LOCALITY_SCHEDULING_OFF: &str = "off";
+const SHARD_LOCALITY_SCHEDULING_OBSERVE: &str = "observe";
+const SHARD_LOCALITY_SCHEDULING_ON: &str = "on";
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum ShardLocalitySchedulingMode {
+    Off,
+    Observe,
+    On,
+}
+
+impl ShardLocalitySchedulingMode {
+    fn is_on(self) -> bool {
+        matches!(self, ShardLocalitySchedulingMode::On)
+    }
+}
+
+fn parse_shard_locality_scheduling_mode(value: &str) -> ShardLocalitySchedulingMode {
+    match value.trim() {
+        value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_OFF) => ShardLocalitySchedulingMode::Off,
+        value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_OBSERVE) => ShardLocalitySchedulingMode::Observe,
+        value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_ON) => ShardLocalitySchedulingMode::On,
+        _ => ShardLocalitySchedulingMode::Off,
+    }
+}
+
+fn get_shard_locality_scheduling_mode() -> ShardLocalitySchedulingMode {
+    if let Some(value) = rustfs_utils::get_env_opt_str(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING) {
+        return parse_shard_locality_scheduling_mode(&value);
+    }
+
+    if rustfs_utils::get_env_bool(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, false) {
+        return ShardLocalitySchedulingMode::On;
+    }
+
+    ShardLocalitySchedulingMode::Off
+}
 
 fn get_shard_locality_preference_enabled() -> bool {
-    rustfs_utils::get_env_bool(
-        ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE,
-        DEFAULT_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE,
-    )
+    get_shard_locality_scheduling_mode().is_on()
 }
 
 pub(crate) fn should_collect_shard_read_costs() -> bool {
-    rustfs_io_metrics::get_stage_metrics_enabled() || get_shard_locality_preference_enabled()
+    // `observe` mode only feeds the stage-metrics histograms, so skip the
+    // cost-collection overhead when those metrics cannot be reported anyway.
+    rustfs_io_metrics::get_stage_metrics_enabled() || get_shard_locality_scheduling_mode().is_on()
 }
 
 /// Number of stripes to prefetch in the legacy decode path.
@@ -108,9 +144,10 @@ impl ShardReadCostCounts {
 }
 fn shard_read_launch_rank(cost: ShardReadCost) -> u8 {
     match cost {
-        ShardReadCost::Local | ShardReadCost::SameNode => 0,
-        ShardReadCost::Unknown => 1,
+        ShardReadCost::Local => 0,
+        ShardReadCost::SameNode => 1,
         ShardReadCost::Remote => 2,
+        ShardReadCost::Unknown => 3,
     }
 }
 
@@ -533,6 +570,11 @@ fn shard_read_hedge_delay(read_timeout: Duration) -> Option<Duration> {
     }
 }
 
+fn shard_locality_remote_avoid_potential(remote_scheduled: usize, low_cost_available: usize, data_shards: usize) -> usize {
+    let theoretical_remote_needed = data_shards.saturating_sub(low_cost_available);
+    remote_scheduled.saturating_sub(theoretical_remote_needed)
+}
+
 #[allow(clippy::too_many_arguments)]
 fn record_scheduled_read_cost(
     read_cost: ShardReadCost,
@@ -544,15 +586,13 @@ fn record_scheduled_read_cost(
     remote_scheduled: &mut usize,
     fallback_to_remote: &mut usize,
 ) {
-    if !locality_preference_enabled {
-        return;
-    }
-
     if read_cost.is_low_cost() {
-        *local_preferred += 1;
+        if locality_preference_enabled {
+            *local_preferred += 1;
+        }
     } else if read_cost.is_remote() {
         *remote_scheduled += 1;
-        if count_remote_as_fallback || low_cost_available < data_shards {
+        if locality_preference_enabled && (count_remote_as_fallback || low_cost_available < data_shards) {
             *fallback_to_remote += 1;
         }
     }
@@ -889,6 +929,13 @@ where
                         fallback_to_remote,
                     );
                 } else {
+                    let remote_avoid_potential =
+                        shard_locality_remote_avoid_potential(remote_scheduled, low_cost_available, self.data_shards);
+                    rustfs_io_metrics::record_get_object_shard_locality_observe_only(
+                        path,
+                        remote_scheduled,
+                        remote_avoid_potential,
+                    );
                     rustfs_io_metrics::record_get_object_shard_locality_policy_disabled(path);
                 }
             }
@@ -1598,14 +1645,138 @@ mod tests {
     }
 
     #[test]
+    #[serial_test::serial]
     fn test_shard_locality_preference_gate_defaults_disabled() {
-        temp_env::with_var(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>, || {
-            assert!(!get_shard_locality_preference_enabled());
-        });
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>),
+                (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
+            ],
+            || {
+                assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Off);
+                assert!(!get_shard_locality_preference_enabled());
+            },
+        );
     }
 
     #[test]
-    fn test_shard_read_launch_order_is_gated() {
+    #[serial_test::serial]
+    fn test_shard_locality_scheduling_mode_parses_supported_values() {
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("observe")),
+                (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
+            ],
+            || {
+                assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Observe);
+                assert!(!get_shard_locality_preference_enabled());
+            },
+        );
+
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
+                (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
+            ],
+            || {
+                assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::On);
+                assert!(get_shard_locality_preference_enabled());
+                assert!(should_collect_shard_read_costs());
+            },
+        );
+
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("unexpected")),
+                (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")),
+            ],
+            || {
+                assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Off);
+                assert!(!get_shard_locality_preference_enabled());
+            },
+        );
+
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>),
+                (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")),
+            ],
+            || {
+                assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::On);
+                assert!(get_shard_locality_preference_enabled());
+            },
+        );
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn test_shard_locality_scheduling_off_does_not_collect_without_metrics() {
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("off")),
+                (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
+            ],
+            || {
+                rustfs_io_metrics::set_get_stage_metrics_enabled(false);
+                assert!(!should_collect_shard_read_costs());
+            },
+        );
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn test_shard_locality_scheduling_observe_collects_only_with_stage_metrics() {
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("observe")),
+                (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
+            ],
+            || {
+                // Without stage metrics there is no reporting channel, so
+                // observe mode must not pay the cost-collection overhead.
+                rustfs_io_metrics::set_get_stage_metrics_enabled(false);
+                assert!(!should_collect_shard_read_costs());
+
+                rustfs_io_metrics::set_get_stage_metrics_enabled(true);
+                assert!(should_collect_shard_read_costs());
+                assert!(!get_shard_locality_preference_enabled());
+                let read_costs = [ShardReadCost::Remote, ShardReadCost::Local, ShardReadCost::SameNode];
+                assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), false), vec![0, 1, 2]);
+                rustfs_io_metrics::set_get_stage_metrics_enabled(false);
+            },
+        );
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn test_shard_locality_scheduling_on_enables_reordering() {
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
+                (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
+            ],
+            || {
+                assert!(get_shard_locality_preference_enabled());
+            },
+        );
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn test_shard_locality_legacy_preference_gate_still_enables_on() {
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>),
+                (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")),
+            ],
+            || {
+                assert!(get_shard_locality_preference_enabled());
+            },
+        );
+    }
+
+    #[test]
+    fn test_shard_locality_read_launch_order_is_gated() {
         let read_costs = [
             ShardReadCost::Remote,
             ShardReadCost::Local,
@@ -1615,7 +1786,14 @@ mod tests {
         ];
 
         assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), false), vec![0, 1, 2, 3, 4]);
-        assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), true), vec![1, 3, 2, 0, 4]);
+        assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), true), vec![1, 3, 0, 4, 2]);
+    }
+
+    #[test]
+    fn test_shard_locality_remote_avoid_potential_is_observe_only() {
+        assert_eq!(shard_locality_remote_avoid_potential(2, 4, 4), 2);
+        assert_eq!(shard_locality_remote_avoid_potential(2, 2, 4), 0);
+        assert_eq!(shard_locality_remote_avoid_potential(3, 3, 4), 2);
     }
 
     #[test]
@@ -1639,90 +1817,107 @@ mod tests {
     #[tokio::test]
     #[serial_test::serial]
     async fn test_parallel_reader_local_first_avoids_remote_when_local_quorum_exists() {
-        temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async {
-            const NUM_SHARDS: usize = 1;
-            const BLOCK_SIZE: usize = 64;
-            const DATA_SHARDS: usize = 4;
-            const PARITY_SHARDS: usize = 2;
-            const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
+        temp_env::async_with_vars(
+            [
+                (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
+                (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
+            ],
+            async {
+                const NUM_SHARDS: usize = 1;
+                const BLOCK_SIZE: usize = 64;
+                const DATA_SHARDS: usize = 4;
+                const PARITY_SHARDS: usize = 2;
+                const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
 
-            let hash_algo = HashAlgorithm::HighwayHash256;
-            let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[]).await;
-            let read_costs = vec![
-                ShardReadCost::Remote,
-                ShardReadCost::Remote,
-                ShardReadCost::Local,
-                ShardReadCost::SameNode,
-                ShardReadCost::Local,
-                ShardReadCost::SameNode,
-            ];
-            let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
-            let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
-                readers,
-                erasure,
-                0,
-                NUM_SHARDS * BLOCK_SIZE,
-                None,
-                read_costs,
-            );
+                let hash_algo = HashAlgorithm::HighwayHash256;
+                let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[]).await;
+                let read_costs = vec![
+                    ShardReadCost::Remote,
+                    ShardReadCost::Remote,
+                    ShardReadCost::Local,
+                    ShardReadCost::SameNode,
+                    ShardReadCost::Local,
+                    ShardReadCost::SameNode,
+                ];
+                let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
+                let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
+                    readers,
+                    erasure,
+                    0,
+                    NUM_SHARDS * BLOCK_SIZE,
+                    None,
+                    read_costs,
+                );
 
-            let (bufs, errs) = parallel_reader.read().await;
+                let (bufs, errs) = parallel_reader.read().await;
 
-            assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
-            assert!(bufs[0].is_none());
-            assert!(bufs[1].is_none());
-            for (index, buf) in bufs.iter().enumerate().take(DATA_SHARDS + PARITY_SHARDS).skip(2) {
-                assert_eq!(buf.as_deref(), Some(&[(index % 256) as u8; SHARD_SIZE][..]));
-            }
-            assert!(errs.iter().all(Option::is_none));
-        })
+                assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
+                assert!(bufs[0].is_none());
+                assert!(bufs[1].is_none());
+                for (index, buf) in bufs.iter().enumerate().take(DATA_SHARDS + PARITY_SHARDS).skip(2) {
+                    assert_eq!(buf.as_deref(), Some(&[(index % 256) as u8; SHARD_SIZE][..]));
+                }
+                assert!(errs.iter().all(Option::is_none));
+            },
+        )
         .await;
     }
 
     #[tokio::test]
     #[serial_test::serial]
     async fn test_parallel_reader_local_missing_falls_back_to_remote() {
-        temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async {
-            const NUM_SHARDS: usize = 1;
-            const BLOCK_SIZE: usize = 64;
-            const DATA_SHARDS: usize = 4;
-            const PARITY_SHARDS: usize = 2;
-            const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
+        temp_env::async_with_vars(
+            [
+                (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
+                (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
+            ],
+            async {
+                const NUM_SHARDS: usize = 1;
+                const BLOCK_SIZE: usize = 64;
+                const DATA_SHARDS: usize = 4;
+                const PARITY_SHARDS: usize = 2;
+                const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
 
-            let hash_algo = HashAlgorithm::HighwayHash256;
-            let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[1], &[]).await;
-            let read_costs = vec![
-                ShardReadCost::Local,
-                ShardReadCost::Local,
-                ShardReadCost::Local,
-                ShardReadCost::Local,
-                ShardReadCost::Remote,
-                ShardReadCost::Remote,
-            ];
-            let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
-            let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
-                readers,
-                erasure,
-                0,
-                NUM_SHARDS * BLOCK_SIZE,
-                None,
-                read_costs,
-            );
+                let hash_algo = HashAlgorithm::HighwayHash256;
+                let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[1], &[]).await;
+                let read_costs = vec![
+                    ShardReadCost::Local,
+                    ShardReadCost::Local,
+                    ShardReadCost::Local,
+                    ShardReadCost::Local,
+                    ShardReadCost::Remote,
+                    ShardReadCost::Remote,
+                ];
+                let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
+                let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
+                    readers,
+                    erasure,
+                    0,
+                    NUM_SHARDS * BLOCK_SIZE,
+                    None,
+                    read_costs,
+                );
 
-            let (bufs, errs) = parallel_reader.read().await;
+                let (bufs, errs) = parallel_reader.read().await;
 
-            assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
-            assert!(matches!(errs[1], Some(Error::FileNotFound)));
-            assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
-            assert!(bufs[5].is_none());
-        })
+                assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
+                assert!(matches!(errs[1], Some(Error::FileNotFound)));
+                assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
+                assert!(bufs[5].is_none());
+            },
+        )
         .await;
     }
 
     #[tokio::test]
     #[serial_test::serial]
     async fn test_parallel_reader_local_corrupt_falls_back_to_remote() {
-        temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async {
+        temp_env::async_with_vars(
+            [
+                (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
+                (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
+            ],
+            async {
             const NUM_SHARDS: usize = 1;
             const BLOCK_SIZE: usize = 64;
             const DATA_SHARDS: usize = 4;
@@ -1757,45 +1952,52 @@ mod tests {
             );
             assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
             assert!(bufs[5].is_none());
-        })
+            },
+        )
         .await;
     }
 
     #[tokio::test]
     #[serial_test::serial]
     async fn test_erasure_decode_local_first_preserves_output_order() {
-        temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async {
-            const DATA_SHARDS: usize = 4;
-            const PARITY_SHARDS: usize = 2;
-            const BLOCK_SIZE: usize = 64;
+        temp_env::async_with_vars(
+            [
+                (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
+                (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
+            ],
+            async {
+                const DATA_SHARDS: usize = 4;
+                const PARITY_SHARDS: usize = 2;
+                const BLOCK_SIZE: usize = 64;
 
-            let total_data: Vec<u8> = (0..BLOCK_SIZE as u32).map(|i| i as u8).collect();
-            let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
-            let shard_size = erasure.shard_size();
-            let hash_algo = HashAlgorithm::HighwayHash256;
-            let shard_bufs = encode_test_object(&erasure, &total_data, shard_size, &hash_algo).await;
-            let readers = shard_bufs
-                .iter()
-                .map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false)))
-                .collect();
-            let read_costs = vec![
-                ShardReadCost::Remote,
-                ShardReadCost::Remote,
-                ShardReadCost::Local,
-                ShardReadCost::Local,
-                ShardReadCost::SameNode,
-                ShardReadCost::SameNode,
-            ];
+                let total_data: Vec<u8> = (0..BLOCK_SIZE as u32).map(|i| i as u8).collect();
+                let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
+                let shard_size = erasure.shard_size();
+                let hash_algo = HashAlgorithm::HighwayHash256;
+                let shard_bufs = encode_test_object(&erasure, &total_data, shard_size, &hash_algo).await;
+                let readers = shard_bufs
+                    .iter()
+                    .map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false)))
+                    .collect();
+                let read_costs = vec![
+                    ShardReadCost::Remote,
+                    ShardReadCost::Remote,
+                    ShardReadCost::Local,
+                    ShardReadCost::Local,
+                    ShardReadCost::SameNode,
+                    ShardReadCost::SameNode,
+                ];
 
-            let mut output = Vec::new();
-            let (written, err) = erasure
-                .decode_with_read_costs(&mut output, readers, 0, total_data.len(), total_data.len(), read_costs)
-                .await;
+                let mut output = Vec::new();
+                let (written, err) = erasure
+                    .decode_with_read_costs(&mut output, readers, 0, total_data.len(), total_data.len(), read_costs)
+                    .await;
 
-            assert!(err.is_none(), "unexpected decode error: {err:?}");
-            assert_eq!(written, total_data.len());
-            assert_eq!(output, total_data);
-        })
+                assert!(err.is_none(), "unexpected decode error: {err:?}");
+                assert_eq!(written, total_data.len());
+                assert_eq!(output, total_data);
+            },
+        )
         .await;
     }
 

crates/ecstore/src/services/batch_processor.rs

@@ -19,17 +19,255 @@
 
 use crate::disk::error::{Error, Result};
 use std::future::Future;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
 use tokio::task::JoinSet;
 
+const BATCH_PROCESSOR_OPERATION_CUSTOM: &str = "custom";
+const BATCH_PROCESSOR_OPERATION_READ: &str = "read";
+const BATCH_PROCESSOR_OPERATION_WRITE: &str = "write";
+const BATCH_PROCESSOR_OPERATION_METADATA: &str = "metadata";
+const ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE: &str = "RUSTFS_BATCH_PROCESSOR_ADAPTIVE";
+const BATCH_PROCESSOR_ADAPTIVE_OFF: &str = "off";
+const BATCH_PROCESSOR_ADAPTIVE_OBSERVE: &str = "observe";
+const BATCH_PROCESSOR_ADAPTIVE_ON: &str = "on";
+const BATCH_SUGGESTION_REASON_IMPROVING: &str = "improving";
+const BATCH_SUGGESTION_REASON_DEGRADING: &str = "degrading";
+const BATCH_SUGGESTION_REASON_STABLE: &str = "stable";
+const BATCH_SUGGESTION_REASON_COOLDOWN: &str = "cooldown";
+const BATCH_OBSERVATION_IMPROVING_LATENCY: Duration = Duration::from_millis(50);
+const BATCH_OBSERVATION_DEGRADING_LATENCY: Duration = Duration::from_millis(250);
+const BATCH_OBSERVATION_COOLDOWN: Duration = Duration::from_secs(30);
+const BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR: usize = 4;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum BatchProcessorAdaptiveMode {
+    Off,
+    Observe,
+    On,
+}
+
+impl BatchProcessorAdaptiveMode {
+    fn should_observe(self) -> bool {
+        matches!(self, Self::Observe | Self::On)
+    }
+
+    fn should_apply(self) -> bool {
+        matches!(self, Self::On)
+    }
+}
+
+fn parse_batch_processor_adaptive_mode(raw: &str) -> BatchProcessorAdaptiveMode {
+    match raw.trim() {
+        value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_OBSERVE) => BatchProcessorAdaptiveMode::Observe,
+        value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_ON) => BatchProcessorAdaptiveMode::On,
+        value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_OFF) => BatchProcessorAdaptiveMode::Off,
+        _ => BatchProcessorAdaptiveMode::Off,
+    }
+}
+
+fn batch_processor_adaptive_mode_from_env() -> BatchProcessorAdaptiveMode {
+    rustfs_utils::get_env_opt_str(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE)
+        .as_deref()
+        .map(parse_batch_processor_adaptive_mode)
+        .unwrap_or(BatchProcessorAdaptiveMode::Off)
+}
+
+fn batch_processor_adaptive_mode() -> BatchProcessorAdaptiveMode {
+    // The gate cannot change at runtime; parse it once instead of re-reading
+    // the environment on every batch.
+    static MODE: std::sync::LazyLock<BatchProcessorAdaptiveMode> =
+        std::sync::LazyLock::new(batch_processor_adaptive_mode_from_env);
+    *MODE
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct BatchConcurrencySuggestion {
+    concurrency: usize,
+    reason: &'static str,
+}
+
+#[derive(Debug, Clone, Copy)]
+struct BatchObservation {
+    batch_size: usize,
+    success_count: usize,
+    error_count: usize,
+    timeout_count: usize,
+    max_queue_wait: Duration,
+    execution_latency: Duration,
+}
+
+#[derive(Debug)]
+struct BatchObservationState {
+    last_suggestion_at: Option<Instant>,
+    last_suggested_concurrency: usize,
+}
+
+impl BatchObservationState {
+    fn new(configured_concurrency: usize) -> Self {
+        Self {
+            last_suggestion_at: None,
+            last_suggested_concurrency: configured_concurrency,
+        }
+    }
+
+    fn suggest(
+        &mut self,
+        current_concurrency: usize,
+        configured_concurrency: usize,
+        observation: BatchObservation,
+        now: Instant,
+    ) -> BatchConcurrencySuggestion {
+        let raw = calculate_batch_concurrency_suggestion(current_concurrency, configured_concurrency, observation);
+        if raw.concurrency == current_concurrency {
+            self.last_suggested_concurrency = current_concurrency;
+            return raw;
+        }
+
+        if self
+            .last_suggestion_at
+            .is_some_and(|last_suggestion_at| now.duration_since(last_suggestion_at) < BATCH_OBSERVATION_COOLDOWN)
+        {
+            return BatchConcurrencySuggestion {
+                concurrency: self.last_suggested_concurrency,
+                reason: BATCH_SUGGESTION_REASON_COOLDOWN,
+            };
+        }
+
+        self.last_suggestion_at = Some(now);
+        self.last_suggested_concurrency = raw.concurrency;
+        raw
+    }
+}
+
+fn calculate_batch_concurrency_suggestion(
+    current_concurrency: usize,
+    configured_concurrency: usize,
+    observation: BatchObservation,
+) -> BatchConcurrencySuggestion {
+    let current_concurrency = current_concurrency.max(1);
+    let configured_concurrency = configured_concurrency.max(1);
+
+    if observation.batch_size == 0 {
+        return BatchConcurrencySuggestion {
+            concurrency: current_concurrency,
+            reason: BATCH_SUGGESTION_REASON_STABLE,
+        };
+    }
+
+    if observation.timeout_count > 0
+        || observation.error_count > 0
+        || (observation.batch_size >= current_concurrency && observation.execution_latency >= BATCH_OBSERVATION_DEGRADING_LATENCY)
+    {
+        return BatchConcurrencySuggestion {
+            concurrency: decrease_batch_concurrency(current_concurrency),
+            reason: BATCH_SUGGESTION_REASON_DEGRADING,
+        };
+    }
+
+    if observation.success_count == observation.batch_size
+        && observation.batch_size >= current_concurrency
+        && observation.execution_latency <= BATCH_OBSERVATION_IMPROVING_LATENCY
+    {
+        return BatchConcurrencySuggestion {
+            concurrency: increase_batch_concurrency(current_concurrency, configured_concurrency, observation.batch_size),
+            reason: BATCH_SUGGESTION_REASON_IMPROVING,
+        };
+    }
+
+    BatchConcurrencySuggestion {
+        concurrency: current_concurrency,
+        reason: BATCH_SUGGESTION_REASON_STABLE,
+    }
+}
+
+fn decrease_batch_concurrency(current_concurrency: usize) -> usize {
+    (current_concurrency.saturating_mul(3) / 4).max(1)
+}
+
+fn increase_batch_concurrency(current_concurrency: usize, configured_concurrency: usize, batch_size: usize) -> usize {
+    let step = (current_concurrency / 4).max(1);
+    // Suggestions build on the previous suggestion, so bound the ratchet at a
+    // hard multiple of the configured baseline instead of the current value.
+    let upper_bound = configured_concurrency
+        .saturating_mul(BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR)
+        .max(current_concurrency);
+    current_concurrency
+        .saturating_add(step)
+        .min(upper_bound)
+        .min(batch_size.max(current_concurrency))
+}
+
+fn is_timeout_error(err: &Error) -> bool {
+    matches!(err, Error::Timeout | Error::SourceStalled)
+        || matches!(err, Error::Io(io_err) if io_err.kind() == std::io::ErrorKind::TimedOut)
+}
+
 /// Batch processor that executes tasks concurrently with a semaphore
 pub struct AsyncBatchProcessor {
     max_concurrent: usize,
+    operation: &'static str,
+    observation_state: Mutex<BatchObservationState>,
 }
 
 impl AsyncBatchProcessor {
     pub fn new(max_concurrent: usize) -> Self {
-        Self { max_concurrent }
+        Self::new_with_operation(max_concurrent, BATCH_PROCESSOR_OPERATION_CUSTOM)
+    }
+
+    fn new_with_operation(max_concurrent: usize, operation: &'static str) -> Self {
+        let max_concurrent = max_concurrent.max(1);
+        Self {
+            max_concurrent,
+            operation,
+            observation_state: Mutex::new(BatchObservationState::new(max_concurrent)),
+        }
+    }
+
+    fn execution_concurrency(&self, mode: BatchProcessorAdaptiveMode) -> usize {
+        if !mode.should_apply() {
+            return self.max_concurrent;
+        }
+
+        self.observation_state
+            .lock()
+            .map(|state| state.last_suggested_concurrency.max(1))
+            .unwrap_or(self.max_concurrent)
+    }
+
+    fn observe_batch_with_mode(
+        &self,
+        mode: BatchProcessorAdaptiveMode,
+        execution_concurrency: usize,
+        observation: BatchObservation,
+    ) -> BatchConcurrencySuggestion {
+        let execution_concurrency = execution_concurrency.max(1);
+        if !mode.should_observe() {
+            return BatchConcurrencySuggestion {
+                concurrency: execution_concurrency,
+                reason: BATCH_SUGGESTION_REASON_STABLE,
+            };
+        }
+
+        let suggestion = match self.observation_state.lock() {
+            Ok(mut state) => state.suggest(execution_concurrency, self.max_concurrent, observation, Instant::now()),
+            Err(_) => calculate_batch_concurrency_suggestion(execution_concurrency, self.max_concurrent, observation),
+        };
+
+        rustfs_io_metrics::record_batch_processor_observation(rustfs_io_metrics::BatchProcessorObservation {
+            operation: self.operation,
+            batch_size: observation.batch_size,
+            configured_concurrency: execution_concurrency,
+            max_queue_wait_secs: observation.max_queue_wait.as_secs_f64(),
+            execution_latency_secs: observation.execution_latency.as_secs_f64(),
+            successes: observation.success_count,
+            errors: observation.error_count,
+            timeouts: observation.timeout_count,
+            suggested_concurrency: suggestion.concurrency,
+            suggestion_reason: suggestion.reason,
+        });
+
+        suggestion
     }
 
     /// Execute a batch of tasks concurrently with concurrency control
@@ -42,27 +280,35 @@ impl AsyncBatchProcessor {
             return Vec::new();
         }
 
-        let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent));
+        let batch_size = tasks.len();
+        let batch_started_at = Instant::now();
+        let adaptive_mode = batch_processor_adaptive_mode();
+        let execution_concurrency = self.execution_concurrency(adaptive_mode);
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency));
         let mut join_set = JoinSet::new();
-        let mut results = Vec::with_capacity(tasks.len());
-        for _ in 0..tasks.len() {
+        let mut results = Vec::with_capacity(batch_size);
+        for _ in 0..batch_size {
             results.push(Err(Error::other("Not completed")));
         }
 
         // Spawn all tasks with semaphore control
         for (i, task) in tasks.into_iter().enumerate() {
             let sem = semaphore.clone();
+            let queued_at = Instant::now();
             join_set.spawn(async move {
                 let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
+                let queue_wait = queued_at.elapsed();
                 let result = task.await;
-                Ok::<(usize, Result<T>), Error>((i, result))
+                Ok::<(usize, Result<T>, Duration), Error>((i, result, queue_wait))
             });
         }
 
         // Collect results
+        let mut max_queue_wait = Duration::ZERO;
         while let Some(join_result) = join_set.join_next().await {
             match join_result {
-                Ok(Ok((index, task_result))) => {
+                Ok(Ok((index, task_result, queue_wait))) => {
+                    max_queue_wait = max_queue_wait.max(queue_wait);
                     if index < results.len() {
                         results[index] = task_result;
                     }
@@ -78,6 +324,29 @@ impl AsyncBatchProcessor {
             }
         }
 
+        let mut success_count = 0;
+        let mut error_count = 0;
+        let mut timeout_count = 0;
+        for result in &results {
+            match result {
+                Ok(_) => success_count += 1,
+                Err(err) if is_timeout_error(err) => timeout_count += 1,
+                Err(_) => error_count += 1,
+            }
+        }
+        self.observe_batch_with_mode(
+            adaptive_mode,
+            execution_concurrency,
+            BatchObservation {
+                batch_size,
+                success_count,
+                error_count,
+                timeout_count,
+                max_queue_wait,
+                execution_latency: batch_started_at.elapsed(),
+            },
+        );
+
         results
     }
 
@@ -97,17 +366,29 @@ impl AsyncBatchProcessor {
             )));
         }
 
-        let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent));
+        let batch_size = tasks.len();
+        let batch_started_at = Instant::now();
+        let adaptive_mode = batch_processor_adaptive_mode();
+        let execution_concurrency = self.execution_concurrency(adaptive_mode);
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency));
         let mut join_set = JoinSet::new();
         let mut successes = Vec::new();
-        let mut pending_tasks = tasks.len();
+        let mut pending_tasks = batch_size;
         let mut first_error = None;
+        let mut error_count = 0;
+        let mut timeout_count = 0;
+        let mut max_queue_wait = Duration::ZERO;
 
         for task in tasks {
             let sem = semaphore.clone();
+            let queued_at = Instant::now();
             join_set.spawn(async move {
-                let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
-                task.await
+                let _permit = sem
+                    .acquire()
+                    .await
+                    .map_err(|_| (Error::other("Semaphore error"), queued_at.elapsed()))?;
+                let queue_wait = queued_at.elapsed();
+                task.await.map(|value| (value, queue_wait)).map_err(|err| (err, queue_wait))
             });
         }
 
@@ -115,18 +396,38 @@ impl AsyncBatchProcessor {
             pending_tasks = pending_tasks.saturating_sub(1);
 
             match join_result {
-                Ok(Ok(value)) => {
+                Ok(Ok((value, queue_wait))) => {
+                    max_queue_wait = max_queue_wait.max(queue_wait);
                     successes.push(value);
                     if successes.len() >= required_successes {
+                        self.observe_batch_with_mode(
+                            adaptive_mode,
+                            execution_concurrency,
+                            BatchObservation {
+                                batch_size,
+                                success_count: successes.len(),
+                                error_count,
+                                timeout_count,
+                                max_queue_wait,
+                                execution_latency: batch_started_at.elapsed(),
+                            },
+                        );
                         return Ok(successes);
                     }
                 }
-                Ok(Err(err)) => {
+                Ok(Err((err, queue_wait))) => {
+                    max_queue_wait = max_queue_wait.max(queue_wait);
+                    if is_timeout_error(&err) {
+                        timeout_count += 1;
+                    } else {
+                        error_count += 1;
+                    }
                     if first_error.is_none() {
                         first_error = Some(err);
                     }
                 }
                 Err(join_error) => {
+                    error_count += 1;
                     if first_error.is_none() {
                         first_error = Some(Error::other(format!("Task panicked in quorum batch processor: {join_error}")));
                     }
@@ -134,6 +435,18 @@ impl AsyncBatchProcessor {
             }
 
             if successes.len() + pending_tasks < required_successes {
+                self.observe_batch_with_mode(
+                    adaptive_mode,
+                    execution_concurrency,
+                    BatchObservation {
+                        batch_size,
+                        success_count: successes.len(),
+                        error_count,
+                        timeout_count,
+                        max_queue_wait,
+                        execution_latency: batch_started_at.elapsed(),
+                    },
+                );
                 return Err(first_error.unwrap_or_else(|| {
                     Error::other(format!(
                         "Insufficient successful results: got {}, needed {}",
@@ -144,6 +457,18 @@ impl AsyncBatchProcessor {
             }
         }
 
+        self.observe_batch_with_mode(
+            adaptive_mode,
+            execution_concurrency,
+            BatchObservation {
+                batch_size,
+                success_count: successes.len(),
+                error_count,
+                timeout_count,
+                max_queue_wait,
+                execution_latency: batch_started_at.elapsed(),
+            },
+        );
         Err(first_error.unwrap_or_else(|| {
             Error::other(format!(
                 "Insufficient successful results: got {}, needed {}",
@@ -164,9 +489,9 @@ pub struct GlobalBatchProcessors {
 impl GlobalBatchProcessors {
     pub fn new() -> Self {
         Self {
-            read_processor: AsyncBatchProcessor::new(16),     // Higher concurrency for reads
-            write_processor: AsyncBatchProcessor::new(8),     // Lower concurrency for writes
-            metadata_processor: AsyncBatchProcessor::new(12), // Medium concurrency for metadata
+            read_processor: AsyncBatchProcessor::new_with_operation(16, BATCH_PROCESSOR_OPERATION_READ),
+            write_processor: AsyncBatchProcessor::new_with_operation(8, BATCH_PROCESSOR_OPERATION_WRITE),
+            metadata_processor: AsyncBatchProcessor::new_with_operation(12, BATCH_PROCESSOR_OPERATION_METADATA),
         }
     }
 
@@ -321,4 +646,208 @@ mod tests {
         assert!(err.to_string().contains("first failure"));
         assert!(started.elapsed() < Duration::from_millis(120));
     }
+
+    #[tokio::test]
+    async fn test_batch_processor_observe_only_keeps_configured_concurrency() {
+        let processor = AsyncBatchProcessor::new(2);
+
+        let tasks: Vec<_> = (0..4)
+            .map(|i| async move {
+                tokio::time::sleep(Duration::from_millis(5)).await;
+                Ok::<i32, Error>(i)
+            })
+            .collect();
+
+        let results = processor.execute_batch(tasks).await;
+        assert_eq!(results.len(), 4);
+        assert_eq!(processor.max_concurrent, 2);
+    }
+
+    #[test]
+    fn batch_processor_adaptive_mode_defaults_to_off_and_parses_supported_values() {
+        assert_eq!(parse_batch_processor_adaptive_mode(""), BatchProcessorAdaptiveMode::Off);
+        assert_eq!(parse_batch_processor_adaptive_mode("off"), BatchProcessorAdaptiveMode::Off);
+        assert_eq!(parse_batch_processor_adaptive_mode("observe"), BatchProcessorAdaptiveMode::Observe);
+        assert_eq!(parse_batch_processor_adaptive_mode("on"), BatchProcessorAdaptiveMode::On);
+        assert_eq!(parse_batch_processor_adaptive_mode("unknown"), BatchProcessorAdaptiveMode::Off);
+        assert!(!BatchProcessorAdaptiveMode::Off.should_observe());
+        assert!(BatchProcessorAdaptiveMode::Observe.should_observe());
+        assert!(BatchProcessorAdaptiveMode::On.should_observe());
+        assert!(!BatchProcessorAdaptiveMode::Observe.should_apply());
+        assert!(BatchProcessorAdaptiveMode::On.should_apply());
+    }
+
+    #[test]
+    fn batch_processor_adaptive_mode_env_gate_is_parsed_from_env() {
+        temp_env::with_var(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE, None::<&str>, || {
+            assert_eq!(batch_processor_adaptive_mode_from_env(), BatchProcessorAdaptiveMode::Off);
+        });
+        temp_env::with_var(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE, Some("observe"), || {
+            assert_eq!(batch_processor_adaptive_mode_from_env(), BatchProcessorAdaptiveMode::Observe);
+        });
+    }
+
+    #[test]
+    fn batch_processor_observation_gate_keeps_default_off_stable() {
+        let processor = AsyncBatchProcessor::new(8);
+        let suggestion = processor.observe_batch_with_mode(
+            BatchProcessorAdaptiveMode::Off,
+            processor.execution_concurrency(BatchProcessorAdaptiveMode::Off),
+            BatchObservation {
+                batch_size: 16,
+                success_count: 16,
+                error_count: 0,
+                timeout_count: 0,
+                max_queue_wait: Duration::ZERO,
+                execution_latency: Duration::from_millis(20),
+            },
+        );
+
+        assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_STABLE);
+        assert_eq!(suggestion.concurrency, 8);
+    }
+
+    #[test]
+    fn batch_processor_observation_gate_observe_records_suggestion() {
+        let processor = AsyncBatchProcessor::new(8);
+        let suggestion = processor.observe_batch_with_mode(
+            BatchProcessorAdaptiveMode::Observe,
+            processor.execution_concurrency(BatchProcessorAdaptiveMode::Observe),
+            BatchObservation {
+                batch_size: 16,
+                success_count: 16,
+                error_count: 0,
+                timeout_count: 0,
+                max_queue_wait: Duration::ZERO,
+                execution_latency: Duration::from_millis(20),
+            },
+        );
+
+        assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
+        assert!(suggestion.concurrency > 8);
+        assert_eq!(processor.execution_concurrency(BatchProcessorAdaptiveMode::Observe), 8);
+    }
+
+    #[test]
+    fn batch_processor_adaptive_on_applies_last_suggestion_to_next_batch() {
+        let processor = AsyncBatchProcessor::new(8);
+        let suggestion = processor.observe_batch_with_mode(
+            BatchProcessorAdaptiveMode::On,
+            processor.execution_concurrency(BatchProcessorAdaptiveMode::On),
+            BatchObservation {
+                batch_size: 16,
+                success_count: 16,
+                error_count: 0,
+                timeout_count: 0,
+                max_queue_wait: Duration::ZERO,
+                execution_latency: Duration::from_millis(20),
+            },
+        );
+
+        assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
+        assert!(suggestion.concurrency > 8);
+        assert_eq!(processor.execution_concurrency(BatchProcessorAdaptiveMode::On), suggestion.concurrency);
+    }
+
+    #[test]
+    fn batch_processor_suggestion_increases_for_fast_successful_batches() {
+        let suggestion = calculate_batch_concurrency_suggestion(
+            8,
+            8,
+            BatchObservation {
+                batch_size: 16,
+                success_count: 16,
+                error_count: 0,
+                timeout_count: 0,
+                max_queue_wait: Duration::ZERO,
+                execution_latency: Duration::from_millis(20),
+            },
+        );
+
+        assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
+        assert!(suggestion.concurrency > 8);
+    }
+
+    #[test]
+    fn batch_processor_suggestion_decreases_for_timeout_batches() {
+        let suggestion = calculate_batch_concurrency_suggestion(
+            8,
+            8,
+            BatchObservation {
+                batch_size: 16,
+                success_count: 14,
+                error_count: 0,
+                timeout_count: 2,
+                max_queue_wait: Duration::ZERO,
+                execution_latency: Duration::from_millis(20),
+            },
+        );
+
+        assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_DEGRADING);
+        assert!(suggestion.concurrency < 8);
+        assert!(suggestion.concurrency >= 1);
+    }
+
+    #[test]
+    fn batch_processor_suggestion_clamps_to_batch_size() {
+        let suggestion = calculate_batch_concurrency_suggestion(
+            64,
+            64,
+            BatchObservation {
+                batch_size: 65,
+                success_count: 65,
+                error_count: 0,
+                timeout_count: 0,
+                max_queue_wait: Duration::ZERO,
+                execution_latency: Duration::from_millis(20),
+            },
+        );
+
+        assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
+        assert_eq!(suggestion.concurrency, 65);
+    }
+
+    #[test]
+    fn batch_processor_suggestion_cooldown_prevents_rapid_oscillation() {
+        let mut state = BatchObservationState::new(8);
+        let now = Instant::now();
+        let observation = BatchObservation {
+            batch_size: 16,
+            success_count: 16,
+            error_count: 0,
+            timeout_count: 0,
+            max_queue_wait: Duration::ZERO,
+            execution_latency: Duration::from_millis(20),
+        };
+
+        let first = state.suggest(8, 8, observation, now);
+        let second = state.suggest(8, 8, observation, now + Duration::from_secs(1));
+
+        assert_eq!(first.reason, BATCH_SUGGESTION_REASON_IMPROVING);
+        assert_eq!(second.reason, BATCH_SUGGESTION_REASON_COOLDOWN);
+        assert_eq!(second.concurrency, first.concurrency);
+    }
+
+    #[test]
+    fn batch_processor_suggestion_growth_is_capped_relative_to_configured_concurrency() {
+        const CONFIGURED: usize = 8;
+        let mut state = BatchObservationState::new(CONFIGURED);
+        let mut current = CONFIGURED;
+        let mut now = Instant::now();
+        let observation = BatchObservation {
+            batch_size: 1024,
+            success_count: 1024,
+            error_count: 0,
+            timeout_count: 0,
+            max_queue_wait: Duration::ZERO,
+            execution_latency: Duration::from_millis(20),
+        };
+
+        for _ in 0..64 {
+            current = state.suggest(current, CONFIGURED, observation, now).concurrency;
+            now += BATCH_OBSERVATION_COOLDOWN;
+        }
+
+        assert_eq!(current, CONFIGURED * BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR);
+    }
 }

crates/ecstore/src/set_disk/mod.rs

@@ -270,10 +270,19 @@ impl AsyncRead for SetDiskLockGuardedReader {
     }
 }
 
-fn attach_set_disk_read_lock_guard(mut reader: GetObjectReader, read_lock_guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
-    if let Some(guard) = read_lock_guard
-        && reader.buffered_body.is_none()
-    {
+fn finish_set_disk_read_lock(
+    mut reader: GetObjectReader,
+    read_lock_guard: Option<ObjectLockDiagGuard>,
+    lock_optimization_enabled: bool,
+    bucket: &str,
+    object: &str,
+) -> GetObjectReader {
+    if lock_optimization_enabled || reader.buffered_body.is_some() {
+        release_materialized_read_lock(bucket, object, read_lock_guard);
+        return reader;
+    }
+
+    if let Some(guard) = read_lock_guard {
         reader.stream = Box::new(SetDiskLockGuardedReader {
             inner: reader.stream,
             guard: Some(guard),
@@ -2314,7 +2323,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                 opts.part_number = Some(1);
             }
             let gr = get_transitioned_object_reader(bucket, object, &range, &h, &object_info, &opts).await?;
-            return Ok(attach_set_disk_read_lock_guard(gr, read_lock_guard.take()));
+            return Ok(finish_set_disk_read_lock(
+                gr,
+                read_lock_guard.take(),
+                lock_optimization_enabled,
+                bucket,
+                object,
+            ));
         }
 
         if is_get_small_object_direct_memory_eligible(&range, &object_info, &fi, opts) {
@@ -2418,7 +2433,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                         );
                         record_get_object_reader_path_observation(GET_OBJECT_PATH_CODEC_STREAMING, object_class, size_bucket);
                         let (reader, _offset, _length) = GetObjectReader::new(stream, range, &object_info, opts, &h).await?;
-                        return Ok(attach_set_disk_read_lock_guard(reader, read_lock_guard.take()));
+                        return Ok(finish_set_disk_read_lock(
+                            reader,
+                            read_lock_guard.take(),
+                            lock_optimization_enabled,
+                            bucket,
+                            object,
+                        ));
                     }
                     read::GetCodecStreamingReaderBuildOutcome::Fallback(reason) => {
                         record_get_codec_streaming_gate_decision(
@@ -2454,8 +2475,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
         let set_index = self.set_index;
         let pool_index = self.pool_index;
         let skip_verify = opts.skip_verify_bitrot;
-        // Move the read-lock guard into the task so it lives for the duration of the read.
-        // Fully materialized paths release it before returning; streaming paths keep it.
+        if lock_optimization_enabled {
+            release_materialized_read_lock(&bucket, &object, read_lock_guard.take());
+            debug!(bucket, object, "Lock optimization: released read lock before streaming read");
+        }
+
+        // When lock optimization is disabled, keep the read-lock guard in the
+        // task so it lives for the duration of the streaming read.
         tokio::spawn(async move {
             let _guard = read_lock_guard;
             let mut writer = wd;

crates/ecstore/src/set_disk/read.rs

@@ -595,14 +595,64 @@ fn resolve_read_part_from_responses(
     Err(DiskError::ErasureReadQuorum)
 }
 
-fn shard_read_cost_for_disk(disk: Option<&DiskStore>) -> ShardReadCost {
+fn shard_read_costs_for_disks(disks: &[Option<DiskStore>]) -> Vec<ShardReadCost> {
+    let local_endpoint_hosts = local_endpoint_hosts_for_shard_costs();
+    disks
+        .iter()
+        .map(|disk| shard_read_cost_for_disk(disk.as_ref(), local_endpoint_hosts))
+        .collect()
+}
+
+fn shard_read_cost_for_disk(disk: Option<&DiskStore>, local_endpoint_hosts: &[String]) -> ShardReadCost {
     match disk {
         Some(disk) if disk.is_local() => ShardReadCost::Local,
-        Some(_) => ShardReadCost::Remote,
+        Some(disk) => shard_read_cost_for_endpoint(false, &disk.host_name(), local_endpoint_hosts),
         None => ShardReadCost::Unknown,
     }
 }
 
+fn shard_read_cost_for_endpoint(is_local: bool, host_name: &str, local_endpoint_hosts: &[String]) -> ShardReadCost {
+    if is_local {
+        return ShardReadCost::Local;
+    }
+
+    if !host_name.is_empty() && local_endpoint_hosts.iter().any(|host| host == host_name) {
+        return ShardReadCost::SameNode;
+    }
+
+    ShardReadCost::Remote
+}
+
+fn local_endpoint_hosts_for_shard_costs() -> &'static [String] {
+    // Endpoint pools are immutable after startup, so build the host list once
+    // instead of walking every pool on each read. Do not cache the empty
+    // pre-startup answer: only memoize once the pools are published.
+    static LOCAL_ENDPOINT_HOSTS: std::sync::OnceLock<Vec<String>> = std::sync::OnceLock::new();
+
+    if let Some(hosts) = LOCAL_ENDPOINT_HOSTS.get() {
+        return hosts;
+    }
+
+    let Some(endpoint_pools) = runtime_sources::endpoint_pools() else {
+        return &[];
+    };
+
+    let mut hosts = Vec::new();
+    for pool in endpoint_pools.as_ref() {
+        for endpoint in pool.endpoints.as_ref() {
+            if !endpoint.is_local {
+                continue;
+            }
+
+            let host = endpoint.host_port();
+            if !host.is_empty() && !hosts.contains(&host) {
+                hosts.push(host);
+            }
+        }
+    }
+    LOCAL_ENDPOINT_HOSTS.get_or_init(|| hosts)
+}
+
 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
 struct ReadRepairHealCacheKey {
     bucket: String,
@@ -2813,12 +2863,7 @@ impl SetDisks {
             let use_mmap_read = object_mmap_read_enabled();
 
             let reader_setup_stage_start = Instant::now();
-            let read_costs = coding::decode::should_collect_shard_read_costs().then(|| {
-                disks
-                    .iter()
-                    .map(|disk| shard_read_cost_for_disk(disk.as_ref()))
-                    .collect::<Vec<_>>()
-            });
+            let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(&disks));
             let reader_setup = create_bitrot_readers_until_quorum_with_preference(
                 &files,
                 &disks,
@@ -3244,12 +3289,7 @@ impl SetDisks {
             bitrot_reader_init_stage: GET_STAGE_READER_TASK_BITROT_READER_INIT,
         });
         let reader_setup_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
-        let read_costs = coding::decode::should_collect_shard_read_costs().then(|| {
-            disks
-                .iter()
-                .map(|disk| shard_read_cost_for_disk(disk.as_ref()))
-                .collect::<Vec<_>>()
-        });
+        let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(disks));
         let reader_setup = create_bitrot_readers_until_quorum_with_preference(
             files,
             disks,
@@ -3835,6 +3875,17 @@ mod tests {
     const CODEC_STREAMING_TEST_BUCKET: &str = "bucket";
     const CODEC_STREAMING_TEST_OBJECT: &str = "object";
 
+    #[test]
+    fn shard_read_cost_for_endpoint_maps_topology_classes() {
+        let local_hosts = vec!["node-a:9000".to_string()];
+
+        assert_eq!(shard_read_cost_for_endpoint(true, "node-a:9000", &local_hosts), ShardReadCost::Local);
+        assert_eq!(shard_read_cost_for_endpoint(false, "node-a:9000", &local_hosts), ShardReadCost::SameNode);
+        assert_eq!(shard_read_cost_for_endpoint(false, "node-b:9000", &local_hosts), ShardReadCost::Remote);
+        assert_eq!(shard_read_cost_for_endpoint(false, "", &local_hosts), ShardReadCost::Remote);
+        assert_eq!(shard_read_cost_for_disk(None, &local_hosts), ShardReadCost::Unknown);
+    }
+
     fn metadata_fanout_test_fileinfo(object: &str) -> FileInfo {
         let mut fi = FileInfo::new(object, 2, 2);
         fi.volume = "bucket".to_string();

crates/ecstore/src/store/object.rs

@@ -15,7 +15,7 @@
 use super::*;
 use crate::set_disk::{
     get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
-    is_object_lock_diag_enabled,
+    is_lock_optimization_enabled, is_object_lock_diag_enabled,
 };
 use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
 use rustfs_io_metrics::{
@@ -442,7 +442,7 @@ impl ECStore {
     }
 
     fn attach_read_lock_guard(mut reader: GetObjectReader, guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
-        if reader.buffered_body.is_some() {
+        if is_lock_optimization_enabled() || reader.buffered_body.is_some() {
             return reader;
         }
 
@@ -1978,7 +1978,7 @@ mod tests {
 
     #[tokio::test]
     #[serial_test::serial]
-    async fn reader_lock_is_held_for_stream_when_optimization_is_enabled() {
+    async fn reader_lock_is_not_held_for_stream_when_optimization_is_enabled() {
         temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
             let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
             let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
@@ -2004,13 +2004,10 @@ mod tests {
 
             let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
 
-            lock.get_write_lock(key.clone(), "writer", Duration::from_millis(20))
-                .await
-                .expect_err("streaming reader should hold the read lock");
-            drop(reader);
             lock.get_write_lock(key, "writer", Duration::from_secs(1))
                 .await
-                .expect("dropping the reader should release the read lock");
+                .expect("lock optimization should release the read lock before returning the stream");
+            drop(reader);
         })
         .await;
     }

crates/io-metrics/src/internode_metrics.rs

@@ -33,6 +33,7 @@ const BACKEND_LABEL: &str = "backend";
 const CLASSIFICATION_LABEL: &str = "classification";
 const STAGE_LABEL: &str = "stage";
 const DOMINANT_ERROR_LABEL: &str = "dominant_error";
+const HTTP_VERSION_LABEL: &str = "http_version";
 const INTERNODE_OPERATION_SENT_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_sent_bytes_total";
 const INTERNODE_OPERATION_RECV_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_recv_bytes_total";
 const INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_outgoing_total";
@@ -42,6 +43,10 @@ const INTERNODE_OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_o
 const INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_classified_errors_total";
 const INTERNODE_OPERATION_RETRIES_TOTAL: &str = "rustfs_system_network_internode_operation_retries_total";
 const INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL: &str = "rustfs_system_network_internode_operation_retry_successes_total";
+const INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL: &str = "rustfs_system_network_internode_operation_http_versions_total";
+const INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL: &str = "rustfs_system_network_internode_operation_stall_timeouts_total";
+const INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL: &str =
+    "rustfs_system_network_internode_operation_write_shutdown_errors_total";
 const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -52,6 +57,7 @@ pub struct InternodeOperationMetricDescriptor {
 
 const OPERATION_BACKEND_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL];
 const OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL];
+const OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL];
 const QUORUM_FAILURE_LABELS: &[&str] = &[STAGE_LABEL, DOMINANT_ERROR_LABEL];
 
 pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &[
@@ -91,6 +97,18 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
         name: INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL,
         labels: OPERATION_BACKEND_CLASSIFICATION_LABELS,
     },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL,
+        labels: OPERATION_BACKEND_HTTP_VERSION_LABELS,
+    },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL,
+        labels: OPERATION_BACKEND_LABELS,
+    },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL,
+        labels: OPERATION_BACKEND_LABELS,
+    },
     InternodeOperationMetricDescriptor {
         name: ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
         labels: QUORUM_FAILURE_LABELS,
@@ -107,6 +125,9 @@ pub struct InternodeMetricsSnapshot {
     pub dial_errors_total: u64,
     pub dial_avg_time_nanos: u64,
     pub last_dial_unix_millis: u64,
+    pub operation_http_versions_total: u64,
+    pub operation_stall_timeouts_total: u64,
+    pub operation_write_shutdown_errors_total: u64,
 }
 
 #[derive(Debug, Default)]
@@ -120,6 +141,9 @@ pub struct InternodeMetrics {
     dial_total_time_nanos: AtomicU64,
     dial_samples_total: AtomicU64,
     last_dial_unix_millis: AtomicU64,
+    operation_http_versions_total: AtomicU64,
+    operation_stall_timeouts_total: AtomicU64,
+    operation_write_shutdown_errors_total: AtomicU64,
 }
 
 impl InternodeMetrics {
@@ -264,6 +288,33 @@ impl InternodeMetrics {
         .increment(1);
     }
 
+    pub fn record_http_version_for_operation_and_backend(
+        &self,
+        operation: &'static str,
+        backend: &'static str,
+        http_version: &'static str,
+    ) {
+        self.operation_http_versions_total.fetch_add(1, Ordering::Relaxed);
+        counter!(
+            INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL,
+            OPERATION_LABEL => operation,
+            BACKEND_LABEL => backend,
+            HTTP_VERSION_LABEL => http_version
+        )
+        .increment(1);
+    }
+
+    pub fn record_stall_timeout_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
+        self.operation_stall_timeouts_total.fetch_add(1, Ordering::Relaxed);
+        counter!(INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(1);
+    }
+
+    pub fn record_write_shutdown_error_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
+        self.operation_write_shutdown_errors_total.fetch_add(1, Ordering::Relaxed);
+        counter!(INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend)
+            .increment(1);
+    }
+
     pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) {
         counter!(
             ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
@@ -307,6 +358,9 @@ impl InternodeMetrics {
             dial_errors_total: self.dial_errors_total.load(Ordering::Relaxed),
             dial_avg_time_nanos,
             last_dial_unix_millis: self.last_dial_unix_millis.load(Ordering::Relaxed),
+            operation_http_versions_total: self.operation_http_versions_total.load(Ordering::Relaxed),
+            operation_stall_timeouts_total: self.operation_stall_timeouts_total.load(Ordering::Relaxed),
+            operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
         }
     }
 
@@ -321,6 +375,9 @@ impl InternodeMetrics {
         self.dial_total_time_nanos.store(0, Ordering::Relaxed);
         self.dial_samples_total.store(0, Ordering::Relaxed);
         self.last_dial_unix_millis.store(0, Ordering::Relaxed);
+        self.operation_http_versions_total.store(0, Ordering::Relaxed);
+        self.operation_stall_timeouts_total.store(0, Ordering::Relaxed);
+        self.operation_write_shutdown_errors_total.store(0, Ordering::Relaxed);
     }
 }
 
@@ -393,14 +450,21 @@ mod tests {
 
     #[test]
     fn operation_metric_descriptors_include_backend_and_operation_labels() {
-        assert_eq!(INTERNODE_OPERATION_METRICS.len(), 10);
+        assert_eq!(INTERNODE_OPERATION_METRICS.len(), 13);
         for metric in &INTERNODE_OPERATION_METRICS[..6] {
             assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]);
         }
         for metric in &INTERNODE_OPERATION_METRICS[6..9] {
             assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]);
         }
-        assert_eq!(INTERNODE_OPERATION_METRICS[9].labels, &[STAGE_LABEL, DOMINANT_ERROR_LABEL]);
+        assert_eq!(
+            INTERNODE_OPERATION_METRICS[9].labels,
+            &[OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL]
+        );
+        for metric in &INTERNODE_OPERATION_METRICS[10..12] {
+            assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]);
+        }
+        assert_eq!(INTERNODE_OPERATION_METRICS[12].labels, &[STAGE_LABEL, DOMINANT_ERROR_LABEL]);
     }
 
     #[test]
@@ -433,6 +497,18 @@ mod tests {
         );
         assert_eq!(
             INTERNODE_OPERATION_METRICS[9].name,
+            "rustfs_system_network_internode_operation_http_versions_total"
+        );
+        assert_eq!(
+            INTERNODE_OPERATION_METRICS[10].name,
+            "rustfs_system_network_internode_operation_stall_timeouts_total"
+        );
+        assert_eq!(
+            INTERNODE_OPERATION_METRICS[11].name,
+            "rustfs_system_network_internode_operation_write_shutdown_errors_total"
+        );
+        assert_eq!(
+            INTERNODE_OPERATION_METRICS[12].name,
             "rustfs_system_storage_erasure_write_quorum_failures_total"
         );
     }
@@ -456,6 +532,19 @@ mod tests {
             INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
             "connection_reset",
         );
+        metrics.record_http_version_for_operation_and_backend(
+            INTERNODE_OPERATION_PUT_FILE_STREAM,
+            INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
+            "http/1.1",
+        );
+        metrics.record_stall_timeout_for_operation_and_backend(
+            INTERNODE_OPERATION_READ_FILE_STREAM,
+            INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
+        );
+        metrics.record_write_shutdown_error_for_operation_and_backend(
+            INTERNODE_OPERATION_PUT_FILE_STREAM,
+            INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
+        );
         metrics.record_erasure_write_quorum_failure("write", "connection_reset");
 
         let snapshot = metrics.snapshot();
@@ -463,5 +552,8 @@ mod tests {
         assert_eq!(snapshot.recv_bytes_total, 0);
         assert_eq!(snapshot.outgoing_requests_total, 0);
         assert_eq!(snapshot.incoming_requests_total, 0);
+        assert_eq!(snapshot.operation_http_versions_total, 1);
+        assert_eq!(snapshot.operation_stall_timeouts_total, 1);
+        assert_eq!(snapshot.operation_write_shutdown_errors_total, 1);
     }
 }

crates/io-metrics/src/lib.rs

@@ -1098,6 +1098,19 @@ pub fn record_get_object_shard_locality_policy_disabled(path: &'static str) {
     }
     counter!("rustfs_io_get_object_shard_locality_policy_disabled_total", "path" => path).increment(1);
 }
+
+/// Record observe-only shard-locality potential while scheduling remains disabled.
+#[inline(always)]
+pub fn record_get_object_shard_locality_observe_only(path: &'static str, remote_scheduled: usize, remote_avoid_potential: usize) {
+    if !get_stage_metrics_enabled() {
+        return;
+    }
+    histogram!("rustfs_io_get_object_shard_remote_scheduled_observe_only", "path" => path)
+        .record(shard_read_fanout_to_f64(remote_scheduled));
+    histogram!("rustfs_io_get_object_shard_remote_avoid_potential", "path" => path)
+        .record(shard_read_fanout_to_f64(remote_avoid_potential));
+}
+
 /// Record per-stripe shard-read fanout shape for GetObject read-path attribution.
 #[inline(always)]
 pub fn record_get_object_shard_read_fanout(
@@ -1116,6 +1129,70 @@ pub fn record_get_object_shard_read_fanout(
     histogram!("rustfs_io_get_object_shard_read_failed", "path" => path).record(shard_read_fanout_to_f64(failed));
 }
 
+fn batch_processor_count_to_f64(value: usize) -> f64 {
+    u64::try_from(value).unwrap_or(u64::MAX) as f64
+}
+
+fn batch_processor_count_to_u64(value: usize) -> u64 {
+    u64::try_from(value).unwrap_or(u64::MAX)
+}
+
+/// Observe-only batch processor shape and adaptive-concurrency advice.
+#[derive(Clone, Copy, Debug)]
+pub struct BatchProcessorObservation {
+    pub operation: &'static str,
+    pub batch_size: usize,
+    pub configured_concurrency: usize,
+    pub max_queue_wait_secs: f64,
+    pub execution_latency_secs: f64,
+    pub successes: usize,
+    pub errors: usize,
+    pub timeouts: usize,
+    pub suggested_concurrency: usize,
+    pub suggestion_reason: &'static str,
+}
+
+/// Record observe-only batch processor shape and adaptive-concurrency advice.
+#[inline(always)]
+pub fn record_batch_processor_observation(observation: BatchProcessorObservation) {
+    if !get_stage_metrics_enabled() {
+        return;
+    }
+
+    histogram!("rustfs_ecstore_batch_processor_batch_size", "operation" => observation.operation)
+        .record(batch_processor_count_to_f64(observation.batch_size));
+    histogram!("rustfs_ecstore_batch_processor_configured_concurrency", "operation" => observation.operation)
+        .record(batch_processor_count_to_f64(observation.configured_concurrency));
+    histogram!("rustfs_ecstore_batch_processor_queue_wait_seconds", "operation" => observation.operation)
+        .record(observation.max_queue_wait_secs);
+    histogram!("rustfs_ecstore_batch_processor_execution_latency_seconds", "operation" => observation.operation)
+        .record(observation.execution_latency_secs);
+    counter!(
+        "rustfs_ecstore_batch_processor_results_total",
+        "operation" => observation.operation,
+        "outcome" => "success"
+    )
+    .increment(batch_processor_count_to_u64(observation.successes));
+    counter!(
+        "rustfs_ecstore_batch_processor_results_total",
+        "operation" => observation.operation,
+        "outcome" => "error"
+    )
+    .increment(batch_processor_count_to_u64(observation.errors));
+    counter!(
+        "rustfs_ecstore_batch_processor_results_total",
+        "operation" => observation.operation,
+        "outcome" => "timeout"
+    )
+    .increment(batch_processor_count_to_u64(observation.timeouts));
+    histogram!(
+        "rustfs_ecstore_batch_processor_suggested_concurrency",
+        "operation" => observation.operation,
+        "reason" => observation.suggestion_reason
+    )
+    .record(batch_processor_count_to_f64(observation.suggested_concurrency));
+}
+
 /// Record the bitrot reader setup scheduling strategy selected for a GET read.
 #[inline(always)]
 pub fn record_get_object_reader_setup_strategy(strategy: &'static str, mode: &'static str) {
@@ -2033,6 +2110,26 @@ mod tests {
         record_bytespool_return("xlarge", "dropped");
     }
 
+    #[test]
+    fn test_record_batch_processor_observation() {
+        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
+        set_get_stage_metrics_enabled(true);
+        record_batch_processor_observation(BatchProcessorObservation {
+            operation: "read",
+            batch_size: 16,
+            configured_concurrency: 8,
+            max_queue_wait_secs: 0.001,
+            execution_latency_secs: 0.025,
+            successes: 15,
+            errors: 1,
+            timeouts: 0,
+            suggested_concurrency: 10,
+            suggestion_reason: "improving",
+        });
+        assert!(get_stage_metrics_enabled());
+        set_get_stage_metrics_enabled(false);
+    }
+
     #[test]
     fn test_record_zero_copy_write() {
         record_zero_copy_write(1024, 10.5);
@@ -2094,6 +2191,7 @@ mod tests {
         record_get_object_pipeline_failure_for_path("codec_streaming", "decode", "read_quorum");
         record_get_object_shard_read_observation("codec_streaming", 0, "data", "local", "success", "none", 1024, 0.004, 0.001);
         record_get_object_shard_read_cost_summary("codec_streaming", 3, 1, 2, 0, 4, 4, 4, true);
+        record_get_object_shard_locality_observe_only("codec_streaming", 2, 1);
         record_get_object_reader_setup_strategy("data_blocks_first", "read_quorum");
         record_get_object_reader_setup_strategy_by_size(
             "codec_streaming",
@@ -2115,10 +2213,32 @@ mod tests {
             0,
             2,
         );
+        record_batch_processor_observation(BatchProcessorObservation {
+            operation: "read",
+            batch_size: 16,
+            configured_concurrency: 8,
+            max_queue_wait_secs: 0.001,
+            execution_latency_secs: 0.025,
+            successes: 15,
+            errors: 1,
+            timeouts: 0,
+            suggested_concurrency: 10,
+            suggestion_reason: "improving",
+        });
 
         assert!(0.005_f64.is_sign_positive());
     }
 
+    #[test]
+    fn test_get_object_shard_locality_observe_only_metrics_smoke() {
+        let remote_scheduled = 2;
+        let remote_avoid_potential = 1;
+
+        record_get_object_shard_locality_observe_only("codec_streaming", remote_scheduled, remote_avoid_potential);
+
+        assert!(remote_scheduled >= remote_avoid_potential);
+    }
+
     #[test]
     fn test_record_get_object_fill_metrics() {
         record_get_object_fill_queued("codec_streaming", "single_inflight", 1);

crates/protocols/src/sftp/server.rs

@@ -36,8 +36,8 @@ use super::wedge_watchdog;
 use crate::common::client::s3::StorageBackend;
 use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
 use russh::keys::{self, PrivateKey, PublicKeyBase64};
-use russh::server::{Auth, Msg, Session};
-use russh::{Channel, ChannelId, MethodKind, MethodSet, Pty, Sig};
+use russh::server::{Auth, ChannelOpenHandle, Msg, Session};
+use russh::{Channel, ChannelId, ChannelOpenFailure, MethodKind, MethodSet, Pty, Sig};
 use rustfs_config::{
     DEFAULT_SFTP_HOST_KEY_RELOAD_ENABLE, DEFAULT_SFTP_HOST_KEY_RELOAD_INTERVAL, ENV_SFTP_HOST_KEY_RELOAD_ENABLE,
     ENV_SFTP_HOST_KEY_RELOAD_INTERVAL,
@@ -1135,15 +1135,19 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
         }
     }
 
-    #[tracing::instrument(level = "debug", skip(self, channel, _session), fields(peer = %self.peer_addr))]
+    #[tracing::instrument(level = "debug", skip(self, channel, reply, _session), fields(peer = %self.peer_addr))]
     fn channel_open_session(
         &mut self,
         channel: Channel<Msg>,
+        reply: ChannelOpenHandle,
         _session: &mut Session,
-    ) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
+    ) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
         let id = channel.id();
         self.channels.insert(id, channel);
-        async { Ok(true) }
+        async move {
+            reply.accept().await;
+            Ok(())
+        }
     }
 
     #[tracing::instrument(level = "debug", skip(self, _session), fields(peer = %self.peer_addr, channel = ?channel))]
@@ -1404,13 +1408,13 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
         async { Ok(false) }
     }
 
-    // Channel-open rejections. russh 0.60 defaults all of these to
-    // Ok(false), but we override them explicitly with a warn log so
+    // Channel-open rejections. russh defaults to rejecting dropped channel-open
+    // handles, but we override them explicitly with a warn log so
     // (a) probe attempts are visible in operator logs and
     // (b) a future russh default flip cannot silently allow these
     //     channel types.
 
-    #[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))]
+    #[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))]
     fn channel_open_direct_tcpip(
         &mut self,
         _channel: Channel<Msg>,
@@ -1418,12 +1422,16 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
         port_to_connect: u32,
         _originator_address: &str,
         _originator_port: u32,
+        reply: ChannelOpenHandle,
         _session: &mut Session,
-    ) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
-        async { Ok(false) }
+    ) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
+        async move {
+            reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await;
+            Ok(())
+        }
     }
 
-    #[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))]
+    #[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))]
     fn channel_open_forwarded_tcpip(
         &mut self,
         _channel: Channel<Msg>,
@@ -1431,30 +1439,42 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
         port_to_connect: u32,
         _originator_address: &str,
         _originator_port: u32,
+        reply: ChannelOpenHandle,
         _session: &mut Session,
-    ) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
-        async { Ok(false) }
+    ) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
+        async move {
+            reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await;
+            Ok(())
+        }
     }
 
-    #[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr))]
+    #[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr))]
     fn channel_open_x11(
         &mut self,
         _channel: Channel<Msg>,
         _originator_address: &str,
         _originator_port: u32,
+        reply: ChannelOpenHandle,
         _session: &mut Session,
-    ) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
-        async { Ok(false) }
+    ) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
+        async move {
+            reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await;
+            Ok(())
+        }
     }
 
-    #[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr, socket = %socket_path))]
+    #[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr, socket = %socket_path))]
     fn channel_open_direct_streamlocal(
         &mut self,
         _channel: Channel<Msg>,
         socket_path: &str,
+        reply: ChannelOpenHandle,
         _session: &mut Session,
-    ) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
-        async { Ok(false) }
+    ) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
+        async move {
+            reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await;
+            Ok(())
+        }
     }
 }
 

crates/protos/src/generated/proto_gen/node_service.rs

@@ -531,6 +531,26 @@ pub struct ReadVersionResponse {
     pub file_info_bin: ::prost::bytes::Bytes,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct BatchReadVersionRequest {
+    #[prost(string, tag = "1")]
+    pub disk: ::prost::alloc::string::String,
+    #[prost(string, tag = "2")]
+    pub batch_read_version_req: ::prost::alloc::string::String,
+    #[prost(bytes = "bytes", tag = "3")]
+    pub batch_read_version_req_bin: ::prost::bytes::Bytes,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct BatchReadVersionResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(string, repeated, tag = "2")]
+    pub batch_read_version_resps: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+    #[prost(message, optional, tag = "3")]
+    pub error: ::core::option::Option<Error>,
+    #[prost(bytes = "bytes", repeated, tag = "4")]
+    pub batch_read_version_resps_bin: ::prost::alloc::vec::Vec<::prost::bytes::Bytes>,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ReadXlRequest {
     #[prost(string, tag = "1")]
     pub disk: ::prost::alloc::string::String,
@@ -1689,6 +1709,21 @@ pub mod node_service_client {
                 .insert(GrpcMethod::new("node_service.NodeService", "ReadVersion"));
             self.inner.unary(req, path, codec).await
         }
+        pub async fn batch_read_version(
+            &mut self,
+            request: impl tonic::IntoRequest<super::BatchReadVersionRequest>,
+        ) -> std::result::Result<tonic::Response<super::BatchReadVersionResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/BatchReadVersion");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "BatchReadVersion"));
+            self.inner.unary(req, path, codec).await
+        }
         pub async fn read_xl(
             &mut self,
             request: impl tonic::IntoRequest<super::ReadXlRequest>,
@@ -2610,6 +2645,10 @@ pub mod node_service_server {
             &self,
             request: tonic::Request<super::ReadVersionRequest>,
         ) -> std::result::Result<tonic::Response<super::ReadVersionResponse>, tonic::Status>;
+        async fn batch_read_version(
+            &self,
+            request: tonic::Request<super::BatchReadVersionRequest>,
+        ) -> std::result::Result<tonic::Response<super::BatchReadVersionResponse>, tonic::Status>;
         async fn read_xl(
             &self,
             request: tonic::Request<super::ReadXlRequest>,
@@ -3706,6 +3745,34 @@ pub mod node_service_server {
                     };
                     Box::pin(fut)
                 }
+                "/node_service.NodeService/BatchReadVersion" => {
+                    #[allow(non_camel_case_types)]
+                    struct BatchReadVersionSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::BatchReadVersionRequest> for BatchReadVersionSvc<T> {
+                        type Response = super::BatchReadVersionResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::BatchReadVersionRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::batch_read_version(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = BatchReadVersionSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 "/node_service.NodeService/ReadXL" => {
                     #[allow(non_camel_case_types)]
                     struct ReadXLSvc<T: NodeService>(pub Arc<T>);

crates/protos/src/node.proto

@@ -371,6 +371,19 @@ message ReadVersionResponse {
   bytes file_info_bin = 4;
 }
 
+message BatchReadVersionRequest {
+  string disk = 1;
+  string batch_read_version_req = 2;
+  bytes batch_read_version_req_bin = 3;
+}
+
+message BatchReadVersionResponse {
+  bool success = 1;
+  repeated string batch_read_version_resps = 2;
+  optional Error error = 3;
+  repeated bytes batch_read_version_resps_bin = 4;
+}
+
 message ReadXLRequest {
   string disk = 1;
   string volume = 2;
@@ -868,6 +881,7 @@ service NodeService {
   rpc ReadMetadata(ReadMetadataRequest) returns (ReadMetadataResponse) {};
   rpc WriteMetadata(WriteMetadataRequest) returns (WriteMetadataResponse) {};
   rpc ReadVersion(ReadVersionRequest) returns (ReadVersionResponse) {};
+  rpc BatchReadVersion(BatchReadVersionRequest) returns (BatchReadVersionResponse) {};
   rpc ReadXL(ReadXLRequest) returns (ReadXLResponse) {};
   rpc DeleteVersion(DeleteVersionRequest) returns (DeleteVersionResponse) {};
   rpc DeleteVersions(DeleteVersionsRequest) returns (DeleteVersionsResponse) {};

crates/rio/src/http_reader.rs

@@ -15,14 +15,14 @@
 use crate::{EtagResolvable, HashReaderDetector, HashReaderMut};
 use bytes::{Bytes, BytesMut};
 use futures::{Stream, TryStreamExt as _};
-use http::HeaderMap;
+use http::{HeaderMap, Version};
 use pin_project_lite::pin_project;
 use reqwest::{Certificate, Client, Identity, Method, RequestBuilder};
 use rustfs_io_metrics::internode_metrics::{
     INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
 };
 use rustfs_tls_runtime::load_cert_bundle_der_bytes;
-use rustfs_utils::get_env_opt_str;
+use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_opt_u64, get_env_opt_usize};
 use rustls_pki_types::pem::PemObject;
 use std::io::IoSlice;
 use std::io::{self, Error};
@@ -31,7 +31,7 @@ use std::ops::Not as _;
 use std::pin::Pin;
 use std::sync::LazyLock;
 use std::task::{Context, Poll};
-use std::time::Duration;
+use std::time::{Duration, Instant};
 use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
 use tokio::sync::{Mutex, mpsc};
 use tokio::time::{self, Sleep};
@@ -42,6 +42,11 @@ use tracing::{error, warn};
 const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
 const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream";
 const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir";
+const HTTP_VERSION_09_LABEL: &str = "http/0.9";
+const HTTP_VERSION_10_LABEL: &str = "http/1.0";
+const HTTP_VERSION_11_LABEL: &str = "http/1.1";
+const HTTP_VERSION_2_LABEL: &str = "h2";
+const HTTP_VERSION_UNKNOWN_LABEL: &str = "unknown";
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
 pub enum InternodeHttpErrorKind {
@@ -213,18 +218,192 @@ fn add_root_certificates_from_der(builder: reqwest::ClientBuilder, certs_der: &[
 struct CachedClients {
     generation: u64,
     client: Client,
-    local_client: Client,
+    no_proxy_client: Client,
 }
 
 static CLIENT_CACHE: LazyLock<Mutex<Option<CachedClients>>> = LazyLock::new(|| Mutex::new(None));
 
-async fn build_http_client(disable_proxy: bool, outbound_tls: &rustfs_tls_runtime::GlobalPublishedOutboundTlsState) -> Client {
+const INTERNODE_HTTP_PROFILE_LEGACY: &str = "legacy";
+const INTERNODE_HTTP_PROFILE_BALANCED: &str = "balanced";
+const INTERNODE_HTTP_PROFILE_THROUGHPUT: &str = "throughput";
+const INTERNODE_HTTP_PROXY_LEGACY: &str = "legacy";
+const INTERNODE_HTTP_PROXY_OFF: &str = "off";
+const INTERNODE_HTTP_PROXY_SYSTEM: &str = "system";
+const INTERNODE_HTTP2_WINDOW_MIN: u32 = 65_535;
+const INTERNODE_HTTP2_WINDOW_MAX: u32 = 64 * 1024 * 1024;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum InternodeHttpTuningProfile {
+    Legacy,
+    Balanced,
+    Throughput,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum InternodeHttpProxyMode {
+    Legacy,
+    NoProxy,
+    System,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+struct InternodeHttpClientTuning {
+    profile: InternodeHttpTuningProfile,
+    pool_max_idle_per_host: Option<usize>,
+    pool_idle_timeout_secs: Option<u64>,
+    http2_initial_stream_window_size: Option<u32>,
+    http2_initial_connection_window_size: Option<u32>,
+    http2_adaptive_window: bool,
+    proxy_mode: InternodeHttpProxyMode,
+}
+
+fn internode_http_client_tuning() -> InternodeHttpClientTuning {
+    // The tuning env vars cannot change at runtime; parse them once instead of
+    // re-reading the environment on every internode request.
+    static TUNING: LazyLock<InternodeHttpClientTuning> = LazyLock::new(InternodeHttpClientTuning::from_env);
+    *TUNING
+}
+
+impl InternodeHttpClientTuning {
+    fn from_env() -> Self {
+        let profile =
+            parse_internode_http_tuning_profile(get_env_opt_str(rustfs_config::ENV_INTERNODE_HTTP_TUNING_PROFILE).as_deref());
+        Self::from_values(
+            profile,
+            get_env_opt_usize(rustfs_config::ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST),
+            get_env_opt_u64(rustfs_config::ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS),
+            get_env_opt_u64(rustfs_config::ENV_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE),
+            get_env_opt_u64(rustfs_config::ENV_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE),
+            get_env_bool(
+                rustfs_config::ENV_INTERNODE_HTTP2_ADAPTIVE_WINDOW,
+                profile.default_http2_adaptive_window(),
+            ),
+            get_env_opt_str(rustfs_config::ENV_INTERNODE_HTTP_PROXY).as_deref(),
+        )
+    }
+
+    fn from_values(
+        profile: InternodeHttpTuningProfile,
+        pool_max_idle_per_host: Option<usize>,
+        pool_idle_timeout_secs: Option<u64>,
+        stream_window_size: Option<u64>,
+        connection_window_size: Option<u64>,
+        http2_adaptive_window: bool,
+        proxy_mode: Option<&str>,
+    ) -> Self {
+        Self {
+            profile,
+            pool_max_idle_per_host: pool_max_idle_per_host.or_else(|| profile.default_pool_max_idle_per_host()),
+            pool_idle_timeout_secs: pool_idle_timeout_secs.or_else(|| profile.default_pool_idle_timeout_secs()),
+            http2_initial_stream_window_size: clamp_http2_window(
+                stream_window_size.or_else(|| profile.default_stream_window_size()),
+            ),
+            http2_initial_connection_window_size: clamp_http2_window(
+                connection_window_size.or_else(|| profile.default_connection_window_size()),
+            ),
+            http2_adaptive_window,
+            proxy_mode: parse_internode_http_proxy_mode(proxy_mode, profile),
+        }
+    }
+}
+
+impl InternodeHttpTuningProfile {
+    fn default_pool_max_idle_per_host(self) -> Option<usize> {
+        match self {
+            Self::Legacy => None,
+            Self::Balanced => Some(64),
+            Self::Throughput => Some(256),
+        }
+    }
+
+    fn default_pool_idle_timeout_secs(self) -> Option<u64> {
+        match self {
+            Self::Legacy => None,
+            Self::Balanced => Some(120),
+            Self::Throughput => Some(300),
+        }
+    }
+
+    fn default_stream_window_size(self) -> Option<u64> {
+        match self {
+            Self::Legacy => None,
+            Self::Balanced => Some(1024 * 1024),
+            Self::Throughput => Some(4 * 1024 * 1024),
+        }
+    }
+
+    fn default_connection_window_size(self) -> Option<u64> {
+        match self {
+            Self::Legacy => None,
+            Self::Balanced => Some(4 * 1024 * 1024),
+            Self::Throughput => Some(16 * 1024 * 1024),
+        }
+    }
+
+    fn default_http2_adaptive_window(self) -> bool {
+        matches!(self, Self::Throughput)
+    }
+}
+
+fn parse_internode_http_tuning_profile(value: Option<&str>) -> InternodeHttpTuningProfile {
+    match value.map(|value| value.trim().to_ascii_lowercase()) {
+        Some(value) if value == INTERNODE_HTTP_PROFILE_LEGACY => InternodeHttpTuningProfile::Legacy,
+        Some(value) if value == INTERNODE_HTTP_PROFILE_BALANCED => InternodeHttpTuningProfile::Balanced,
+        Some(value) if value == INTERNODE_HTTP_PROFILE_THROUGHPUT => InternodeHttpTuningProfile::Throughput,
+        _ => InternodeHttpTuningProfile::Legacy,
+    }
+}
+
+fn parse_internode_http_proxy_mode(value: Option<&str>, profile: InternodeHttpTuningProfile) -> InternodeHttpProxyMode {
+    match value.map(|value| value.trim().to_ascii_lowercase()) {
+        Some(value) if value == INTERNODE_HTTP_PROXY_SYSTEM => InternodeHttpProxyMode::System,
+        Some(value) if value == INTERNODE_HTTP_PROXY_LEGACY => InternodeHttpProxyMode::Legacy,
+        Some(value) if matches!(value.as_str(), INTERNODE_HTTP_PROXY_OFF | "none" | "no_proxy" | "disabled") => {
+            InternodeHttpProxyMode::NoProxy
+        }
+        Some(_) | None if matches!(profile, InternodeHttpTuningProfile::Legacy) => InternodeHttpProxyMode::Legacy,
+        Some(_) | None => InternodeHttpProxyMode::NoProxy,
+    }
+}
+
+fn clamp_http2_window(value: Option<u64>) -> Option<u32> {
+    let value = value?;
+    let value = value.clamp(u64::from(INTERNODE_HTTP2_WINDOW_MIN), u64::from(INTERNODE_HTTP2_WINDOW_MAX));
+    u32::try_from(value).ok()
+}
+
+fn apply_http_client_tuning(mut builder: reqwest::ClientBuilder, tuning: InternodeHttpClientTuning) -> reqwest::ClientBuilder {
+    if let Some(pool_max_idle_per_host) = tuning.pool_max_idle_per_host {
+        builder = builder.pool_max_idle_per_host(pool_max_idle_per_host);
+    }
+    if let Some(pool_idle_timeout_secs) = tuning.pool_idle_timeout_secs {
+        builder = builder.pool_idle_timeout(std::time::Duration::from_secs(pool_idle_timeout_secs));
+    }
+    if tuning.http2_adaptive_window {
+        builder = builder.http2_adaptive_window(true);
+    } else {
+        if let Some(stream_window_size) = tuning.http2_initial_stream_window_size {
+            builder = builder.http2_initial_stream_window_size(stream_window_size);
+        }
+        if let Some(connection_window_size) = tuning.http2_initial_connection_window_size {
+            builder = builder.http2_initial_connection_window_size(connection_window_size);
+        }
+    }
+    builder
+}
+
+async fn build_http_client(
+    disable_proxy: bool,
+    tuning: InternodeHttpClientTuning,
+    outbound_tls: &rustfs_tls_runtime::GlobalPublishedOutboundTlsState,
+) -> Client {
     let mut builder = Client::builder()
         .connect_timeout(std::time::Duration::from_secs(5))
         .tcp_keepalive(std::time::Duration::from_secs(10))
         .http2_keep_alive_interval(std::time::Duration::from_secs(5))
         .http2_keep_alive_timeout(std::time::Duration::from_secs(3))
         .http2_keep_alive_while_idle(true);
+    builder = apply_http_client_tuning(builder, tuning);
 
     if disable_proxy {
         builder = builder.no_proxy();
@@ -292,10 +471,19 @@ fn should_bypass_proxy_for_url(url: &str) -> bool {
     host.eq_ignore_ascii_case("localhost") || host.parse::<IpAddr>().is_ok_and(|addr| addr.is_loopback())
 }
 
+fn should_disable_proxy_for_url(url: &str, tuning: InternodeHttpClientTuning) -> bool {
+    match tuning.proxy_mode {
+        InternodeHttpProxyMode::System => false,
+        InternodeHttpProxyMode::NoProxy => true,
+        InternodeHttpProxyMode::Legacy => should_bypass_proxy_for_url(url),
+    }
+}
+
 async fn get_http_client(url: &str) -> Client {
-    // Reuse HTTP connection pools while keeping loopback traffic away from
-    // system proxies so local RPC/tests do not leak to proxy listeners.
-    let disable_proxy = should_bypass_proxy_for_url(url);
+    let tuning = internode_http_client_tuning();
+    // Reuse HTTP connection pools while honoring the configured internode proxy
+    // policy. The legacy profile only bypasses loopback URLs to preserve defaults.
+    let disable_proxy = should_disable_proxy_for_url(url, tuning);
 
     // Fast path: check generation first (cheap atomic read) to avoid cloning
     // the full PEM + identity bytes when the TLS state hasn't changed.
@@ -305,7 +493,7 @@ async fn get_http_client(url: &str) -> Client {
     if let Some(cached) = guard.as_ref() {
         if cached.generation == generation {
             return if disable_proxy {
-                cached.local_client.clone()
+                cached.no_proxy_client.clone()
             } else {
                 cached.client.clone()
             };
@@ -317,16 +505,16 @@ async fn get_http_client(url: &str) -> Client {
     // Cache miss or stale generation — load full outbound TLS state.
     let outbound_tls = crate::http_runtime_sources::outbound_tls_state().await;
 
-    let client = build_http_client(false, &outbound_tls).await;
-    let local_client = build_http_client(true, &outbound_tls).await;
+    let client = build_http_client(false, tuning, &outbound_tls).await;
+    let no_proxy_client = build_http_client(true, tuning, &outbound_tls).await;
     let cached = CachedClients {
         generation,
         client,
-        local_client,
+        no_proxy_client,
     };
 
     let return_client = if disable_proxy {
-        cached.local_client.clone()
+        cached.no_proxy_client.clone()
     } else {
         cached.client.clone()
     };
@@ -419,6 +607,8 @@ pin_project! {
         internode_operation: Option<&'static str>,
         stall_timeout: Option<Duration>,
         stall_timer: Option<Pin<Box<Sleep>>>,
+        request_started: Instant,
+        duration_recorded: bool,
         #[pin]
         inner: StreamReader<Pin<Box<dyn Stream<Item=std::io::Result<Bytes>>+Send+Sync>>, Bytes>,
     }
@@ -467,13 +657,17 @@ impl HttpReader {
             request = request.body(body);
         }
 
+        let request_started = Instant::now();
         let resp = request.send().await.map_err(|e| {
+            record_internode_operation_duration(track_internode_metrics, internode_operation, request_started.elapsed());
             record_internode_error(track_internode_metrics, internode_operation);
             record_internode_classified_error(track_internode_metrics, internode_operation, classify_reqwest_error(&e));
             internode_reqwest_error(&method, &url, internode_operation, e)
         })?;
 
+        record_internode_http_version(track_internode_metrics, internode_operation, http_version_metric_label(resp.version()));
         if resp.status().is_success().not() {
+            record_internode_operation_duration(track_internode_metrics, internode_operation, request_started.elapsed());
             record_internode_error(track_internode_metrics, internode_operation);
             record_internode_classified_error(track_internode_metrics, internode_operation, classify_http_status(resp.status()));
             return Err(internode_status_error(&method, &url, internode_operation, resp.status()));
@@ -498,6 +692,8 @@ impl HttpReader {
             internode_operation,
             stall_timer: None,
             stall_timeout,
+            request_started,
+            duration_recorded: false,
         })
     }
     pub fn url(&self) -> &str {
@@ -521,6 +717,13 @@ impl AsyncRead for HttpReader {
                 let bytes_read = buf.filled().len().saturating_sub(filled_before);
                 if bytes_read > 0 {
                     record_internode_recv_bytes(*this.track_internode_metrics, *this.internode_operation, bytes_read);
+                } else {
+                    record_internode_operation_duration_once(
+                        *this.track_internode_metrics,
+                        *this.internode_operation,
+                        *this.request_started,
+                        this.duration_recorded,
+                    );
                 }
                 *this.stall_timer = None;
                 Poll::Ready(Ok(()))
@@ -531,6 +734,13 @@ impl AsyncRead for HttpReader {
                 };
                 let timer = this.stall_timer.get_or_insert_with(|| Box::pin(time::sleep(stall_timeout)));
                 if timer.as_mut().poll(cx).is_ready() {
+                    record_internode_operation_duration_once(
+                        *this.track_internode_metrics,
+                        *this.internode_operation,
+                        *this.request_started,
+                        this.duration_recorded,
+                    );
+                    record_internode_stall_timeout(*this.track_internode_metrics, *this.internode_operation);
                     record_internode_error(*this.track_internode_metrics, *this.internode_operation);
                     Poll::Ready(Err(Error::new(
                         io::ErrorKind::TimedOut,
@@ -540,7 +750,15 @@ impl AsyncRead for HttpReader {
                     Poll::Pending
                 }
             }
-            other => other,
+            Poll::Ready(Err(err)) => {
+                record_internode_operation_duration_once(
+                    *this.track_internode_metrics,
+                    *this.internode_operation,
+                    *this.request_started,
+                    this.duration_recorded,
+                );
+                Poll::Ready(Err(err))
+            }
         }
     }
 }
@@ -610,6 +828,8 @@ pin_project! {
         handle: tokio::task::JoinHandle<std::io::Result<()>>,
         pending_chunk: BytesMut,
         finish:bool,
+        track_internode_metrics: bool,
+        internode_operation: Option<&'static str>,
 
     }
 }
@@ -648,10 +868,17 @@ impl HttpWriter {
                 .body(body);
 
             // Hold the request until the shutdown signal is received
+            let request_started = Instant::now();
             let response = request.send().await;
 
             match response {
                 Ok(resp) => {
+                    record_internode_operation_duration(track_internode_metrics, internode_operation, request_started.elapsed());
+                    record_internode_http_version(
+                        track_internode_metrics,
+                        internode_operation,
+                        http_version_metric_label(resp.version()),
+                    );
                     // http_log!("[HttpWriter::spawn] got response: status={}", resp.status());
                     if !resp.status().is_success() {
                         record_internode_error(track_internode_metrics, internode_operation);
@@ -667,6 +894,7 @@ impl HttpWriter {
                     }
                 }
                 Err(e) => {
+                    record_internode_operation_duration(track_internode_metrics, internode_operation, request_started.elapsed());
                     record_internode_error(track_internode_metrics, internode_operation);
                     let classified = classify_reqwest_error(&e);
                     record_internode_classified_error(track_internode_metrics, internode_operation, classified);
@@ -691,6 +919,8 @@ impl HttpWriter {
             handle,
             pending_chunk: BytesMut::with_capacity(HTTP_WRITER_BUFFER_SIZE),
             finish: false,
+            track_internode_metrics,
+            internode_operation,
         })
     }
 
@@ -721,6 +951,20 @@ fn internode_rpc_operation(url: &str) -> Option<&'static str> {
     }
 }
 
+fn http_version_metric_label(version: Version) -> &'static str {
+    if version == Version::HTTP_09 {
+        HTTP_VERSION_09_LABEL
+    } else if version == Version::HTTP_10 {
+        HTTP_VERSION_10_LABEL
+    } else if version == Version::HTTP_11 {
+        HTTP_VERSION_11_LABEL
+    } else if version == Version::HTTP_2 {
+        HTTP_VERSION_2_LABEL
+    } else {
+        HTTP_VERSION_UNKNOWN_LABEL
+    }
+}
+
 fn record_internode_outgoing_request(track: bool, operation: Option<&'static str>) {
     if !track {
         return;
@@ -763,6 +1007,60 @@ fn record_internode_classified_error(track: bool, operation: Option<&'static str
     }
 }
 
+fn record_internode_operation_duration(track: bool, operation: Option<&'static str>, duration: Duration) {
+    if !track {
+        return;
+    }
+
+    if let Some(operation) = operation {
+        crate::http_runtime_sources::record_duration(operation, duration);
+    }
+}
+
+fn record_internode_operation_duration_once(
+    track: bool,
+    operation: Option<&'static str>,
+    request_started: Instant,
+    duration_recorded: &mut bool,
+) {
+    if *duration_recorded {
+        return;
+    }
+
+    *duration_recorded = true;
+    record_internode_operation_duration(track, operation, request_started.elapsed());
+}
+
+fn record_internode_http_version(track: bool, operation: Option<&'static str>, http_version: &'static str) {
+    if !track {
+        return;
+    }
+
+    if let Some(operation) = operation {
+        crate::http_runtime_sources::record_http_version(operation, http_version);
+    }
+}
+
+fn record_internode_stall_timeout(track: bool, operation: Option<&'static str>) {
+    if !track {
+        return;
+    }
+
+    if let Some(operation) = operation {
+        crate::http_runtime_sources::record_stall_timeout(operation);
+    }
+}
+
+fn record_internode_write_shutdown_error(track: bool, operation: Option<&'static str>) {
+    if !track {
+        return;
+    }
+
+    if let Some(operation) = operation {
+        crate::http_runtime_sources::record_write_shutdown_error(operation);
+    }
+}
+
 fn poll_send_error_to_io<T>(err: tokio_util::sync::PollSendError<T>, context: &str) -> io::Error {
     Error::other(format!("{context}: {err}"))
 }
@@ -772,6 +1070,13 @@ fn send_error_to_io<T>(err: tokio_util::sync::PollSendError<T>, context: &str) -
 }
 
 impl HttpWriter {
+    fn take_background_error(&mut self) -> io::Result<()> {
+        match self.err_rx.try_recv() {
+            Ok(err) => Err(err),
+            Err(tokio::sync::oneshot::error::TryRecvError::Empty | tokio::sync::oneshot::error::TryRecvError::Closed) => Ok(()),
+        }
+    }
+
     fn poll_send_pending_chunk(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
         if self.pending_chunk.is_empty() {
             return Poll::Ready(Ok(()));
@@ -799,11 +1104,10 @@ impl AsyncWrite for HttpWriter {
         //     self.method,
         //     buf.len()
         // );
-        if let Ok(e) = Pin::new(&mut self.err_rx).try_recv() {
-            return Poll::Ready(Err(e));
-        }
-
         let this = self.as_mut().get_mut();
+        if let Err(err) = this.take_background_error() {
+            return Poll::Ready(Err(err));
+        }
 
         if this.pending_chunk.len() >= HTTP_WRITER_BUFFER_SIZE {
             match this.poll_send_pending_chunk(cx) {
@@ -832,15 +1136,19 @@ impl AsyncWrite for HttpWriter {
     }
 
     fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
-        self.as_mut().get_mut().poll_send_pending_chunk(cx)
+        let this = self.as_mut().get_mut();
+        if let Err(err) = this.take_background_error() {
+            return Poll::Ready(Err(err));
+        }
+
+        this.poll_send_pending_chunk(cx)
     }
 
     fn poll_write_vectored(mut self: Pin<&mut Self>, cx: &mut Context<'_>, bufs: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
-        if let Ok(e) = Pin::new(&mut self.err_rx).try_recv() {
-            return Poll::Ready(Err(e));
-        }
-
         let this = self.as_mut().get_mut();
+        if let Err(err) = this.take_background_error() {
+            return Poll::Ready(Err(err));
+        }
 
         if this.pending_chunk.len() >= HTTP_WRITER_BUFFER_SIZE {
             match this.poll_send_pending_chunk(cx) {
@@ -883,9 +1191,17 @@ impl AsyncWrite for HttpWriter {
         // let url = self.url.clone();
         // let method = self.method.clone();
 
+        if let Err(err) = self.as_mut().get_mut().take_background_error() {
+            record_internode_write_shutdown_error(self.track_internode_metrics, self.internode_operation);
+            return Poll::Ready(Err(err));
+        }
+
         match self.as_mut().get_mut().poll_send_pending_chunk(cx) {
             Poll::Ready(Ok(())) => {}
-            Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
+            Poll::Ready(Err(err)) => {
+                record_internode_write_shutdown_error(self.track_internode_metrics, self.internode_operation);
+                return Poll::Ready(Err(err));
+            }
             Poll::Pending => return Poll::Pending,
         }
 
@@ -894,11 +1210,15 @@ impl AsyncWrite for HttpWriter {
             let this = self.as_mut().get_mut();
             match this.sender.poll_reserve(cx) {
                 Poll::Ready(Ok(())) => {
-                    this.sender
-                        .send_item(None)
-                        .map_err(|e| send_error_to_io(e, "HttpWriter shutdown error"))?;
+                    this.sender.send_item(None).map_err(|e| {
+                        record_internode_write_shutdown_error(this.track_internode_metrics, this.internode_operation);
+                        send_error_to_io(e, "HttpWriter shutdown error")
+                    })?;
+                }
+                Poll::Ready(Err(err)) => {
+                    record_internode_write_shutdown_error(this.track_internode_metrics, this.internode_operation);
+                    return Poll::Ready(Err(poll_send_error_to_io(err, "HttpWriter shutdown error")));
                 }
-                Poll::Ready(Err(err)) => return Poll::Ready(Err(poll_send_error_to_io(err, "HttpWriter shutdown error"))),
                 Poll::Pending => return Poll::Pending,
             }
             // http_log!(
@@ -911,15 +1231,20 @@ impl AsyncWrite for HttpWriter {
         }
         // Wait for the HTTP request to complete
         use futures::FutureExt;
-        match Pin::new(&mut self.get_mut().handle).poll_unpin(cx) {
-            Poll::Ready(Ok(_)) => {
+        match Pin::new(&mut self.as_mut().get_mut().handle).poll_unpin(cx) {
+            Poll::Ready(Ok(Ok(()))) => {
                 // http_log!(
                 //     "[HttpWriter::poll_shutdown] HTTP request finished successfully, url: {}, method: {:?}",
                 //     url,
                 //     method
                 // );
             }
+            Poll::Ready(Ok(Err(err))) => {
+                record_internode_write_shutdown_error(self.track_internode_metrics, self.internode_operation);
+                return Poll::Ready(Err(err));
+            }
             Poll::Ready(Err(e)) => {
+                record_internode_write_shutdown_error(self.track_internode_metrics, self.internode_operation);
                 // http_log!("[HttpWriter::poll_shutdown] HTTP request failed: {e}, url: {}, method: {:?}", url, method);
                 return Poll::Ready(Err(Error::other(format!("HTTP request failed: {e}"))));
             }
@@ -992,6 +1317,13 @@ mod tests {
         StatusCode::OK
     }
 
+    async fn reject_put(State(state): State<TestState>, body: Body) -> impl IntoResponse {
+        state.put_count.fetch_add(1, Ordering::SeqCst);
+        let bytes = body.collect().await.unwrap().to_bytes();
+        state.put_bodies.lock().await.push(bytes.to_vec());
+        StatusCode::INTERNAL_SERVER_ERROR
+    }
+
     async fn start_test_server(state: TestState) -> Option<(String, tokio::task::JoinHandle<()>)> {
         let listener = match TcpListener::bind("127.0.0.1:0").await {
             Ok(listener) => listener,
@@ -1001,6 +1333,7 @@ mod tests {
         let addr = listener.local_addr().expect("listener local address should be available");
         let app = Router::new()
             .route("/stream", get(get_stream).head(reject_head).put(accept_put))
+            .route("/reject-put", get(get_stream).put(reject_put))
             .route("/stall", get(get_stalling_stream))
             .route("/delayed-first", get(get_delayed_first_chunk))
             .with_state(state);
@@ -1033,6 +1366,14 @@ mod tests {
         );
     }
 
+    #[test]
+    fn http_version_metrics_labels_are_low_cardinality() {
+        assert_eq!(http_version_metric_label(Version::HTTP_09), HTTP_VERSION_09_LABEL);
+        assert_eq!(http_version_metric_label(Version::HTTP_10), HTTP_VERSION_10_LABEL);
+        assert_eq!(http_version_metric_label(Version::HTTP_11), HTTP_VERSION_11_LABEL);
+        assert_eq!(http_version_metric_label(Version::HTTP_2), HTTP_VERSION_2_LABEL);
+    }
+
     #[tokio::test]
     async fn http_reader_does_not_send_preflight_head() {
         let state = TestState::default();
@@ -1174,6 +1515,37 @@ mod tests {
         handle.abort();
     }
 
+    #[tokio::test]
+    async fn http_writer_shutdown_reports_http_status_error() {
+        let state = TestState::default();
+        let Some((base_url, handle)) = start_test_server(state.clone()).await else {
+            return;
+        };
+        let url = base_url.replace("/stream", "/reject-put");
+
+        let mut writer = HttpWriter::new(url, Method::PUT, HeaderMap::new()).await.unwrap();
+        writer.write_all(b"payload").await.unwrap();
+        let err = writer
+            .shutdown()
+            .await
+            .expect_err("shutdown should report the HTTP response failure");
+
+        let source = err
+            .get_ref()
+            .and_then(|source| source.downcast_ref::<InternodeHttpError>())
+            .expect("expected shutdown error to carry InternodeHttpError source");
+        assert_eq!(
+            source.kind(),
+            InternodeHttpErrorKind::HttpStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR)
+        );
+        assert_eq!(source.context().method(), "PUT");
+        assert!(source.context().target().contains("/reject-put"));
+        assert_eq!(state.put_count.load(Ordering::SeqCst), 1);
+        assert_eq!(state.put_bodies.lock().await.as_slice(), &[b"payload".to_vec()]);
+
+        handle.abort();
+    }
+
     #[tokio::test]
     async fn http_reader_request_error_includes_method_and_url() {
         let listener = match TcpListener::bind("127.0.0.1:0").await {
@@ -1267,4 +1639,126 @@ mod tests {
         assert!(!should_bypass_proxy_for_url("http://example.com/stream"));
         assert!(!should_bypass_proxy_for_url("not-a-url"));
     }
+
+    #[test]
+    fn internode_http_tuning_profile_parses_known_values_and_falls_back_to_legacy() {
+        assert_eq!(parse_internode_http_tuning_profile(None), InternodeHttpTuningProfile::Legacy);
+        assert_eq!(
+            parse_internode_http_tuning_profile(Some("balanced")),
+            InternodeHttpTuningProfile::Balanced
+        );
+        assert_eq!(
+            parse_internode_http_tuning_profile(Some(" Throughput ")),
+            InternodeHttpTuningProfile::Throughput
+        );
+        assert_eq!(
+            parse_internode_http_tuning_profile(Some("aggressive")),
+            InternodeHttpTuningProfile::Legacy
+        );
+    }
+
+    #[test]
+    fn legacy_internode_http_tuning_keeps_existing_defaults() {
+        let tuning = InternodeHttpClientTuning::from_values(
+            InternodeHttpTuningProfile::Legacy,
+            None,
+            None,
+            None,
+            None,
+            InternodeHttpTuningProfile::Legacy.default_http2_adaptive_window(),
+            None,
+        );
+
+        assert_eq!(tuning.pool_max_idle_per_host, None);
+        assert_eq!(tuning.pool_idle_timeout_secs, None);
+        assert_eq!(tuning.http2_initial_stream_window_size, None);
+        assert_eq!(tuning.http2_initial_connection_window_size, None);
+        assert!(!tuning.http2_adaptive_window);
+        assert_eq!(tuning.proxy_mode, InternodeHttpProxyMode::Legacy);
+    }
+
+    #[test]
+    fn balanced_and_throughput_profiles_apply_conservative_defaults() {
+        let balanced = InternodeHttpClientTuning::from_values(
+            InternodeHttpTuningProfile::Balanced,
+            None,
+            None,
+            None,
+            None,
+            InternodeHttpTuningProfile::Balanced.default_http2_adaptive_window(),
+            None,
+        );
+        let throughput = InternodeHttpClientTuning::from_values(
+            InternodeHttpTuningProfile::Throughput,
+            None,
+            None,
+            None,
+            None,
+            InternodeHttpTuningProfile::Throughput.default_http2_adaptive_window(),
+            None,
+        );
+
+        assert_eq!(balanced.pool_max_idle_per_host, Some(64));
+        assert_eq!(balanced.pool_idle_timeout_secs, Some(120));
+        assert_eq!(balanced.http2_initial_stream_window_size, Some(1024 * 1024));
+        assert_eq!(balanced.http2_initial_connection_window_size, Some(4 * 1024 * 1024));
+        assert!(!balanced.http2_adaptive_window);
+        assert_eq!(balanced.proxy_mode, InternodeHttpProxyMode::NoProxy);
+
+        assert_eq!(throughput.pool_max_idle_per_host, Some(256));
+        assert_eq!(throughput.pool_idle_timeout_secs, Some(300));
+        assert!(throughput.http2_adaptive_window);
+        assert_eq!(throughput.proxy_mode, InternodeHttpProxyMode::NoProxy);
+    }
+
+    #[test]
+    fn internode_http_tuning_overrides_are_clamped() {
+        let tuning = InternodeHttpClientTuning::from_values(
+            InternodeHttpTuningProfile::Balanced,
+            Some(8),
+            Some(30),
+            Some(1),
+            Some(u64::MAX),
+            false,
+            Some("system"),
+        );
+
+        assert_eq!(tuning.pool_max_idle_per_host, Some(8));
+        assert_eq!(tuning.pool_idle_timeout_secs, Some(30));
+        assert_eq!(tuning.http2_initial_stream_window_size, Some(INTERNODE_HTTP2_WINDOW_MIN));
+        assert_eq!(tuning.http2_initial_connection_window_size, Some(INTERNODE_HTTP2_WINDOW_MAX));
+        assert_eq!(tuning.proxy_mode, InternodeHttpProxyMode::System);
+    }
+
+    #[test]
+    fn internode_http_proxy_policy_matches_profile_and_overrides() {
+        let legacy =
+            InternodeHttpClientTuning::from_values(InternodeHttpTuningProfile::Legacy, None, None, None, None, false, None);
+        let balanced =
+            InternodeHttpClientTuning::from_values(InternodeHttpTuningProfile::Balanced, None, None, None, None, false, None);
+        let system_proxy = InternodeHttpClientTuning::from_values(
+            InternodeHttpTuningProfile::Throughput,
+            None,
+            None,
+            None,
+            None,
+            true,
+            Some("system"),
+        );
+        let no_proxy = InternodeHttpClientTuning::from_values(
+            InternodeHttpTuningProfile::Legacy,
+            None,
+            None,
+            None,
+            None,
+            false,
+            Some("off"),
+        );
+
+        assert!(should_disable_proxy_for_url("http://127.0.0.1:9000/stream", legacy));
+        assert!(!should_disable_proxy_for_url("http://192.168.1.10:9000/stream", legacy));
+        assert!(should_disable_proxy_for_url("http://192.168.1.10:9000/stream", balanced));
+        assert!(!should_disable_proxy_for_url("http://127.0.0.1:9000/stream", system_proxy));
+        assert!(should_disable_proxy_for_url("http://example.com/stream", no_proxy));
+    }
 }

crates/rio/src/http_runtime_sources.rs

@@ -17,6 +17,7 @@ use rustfs_tls_runtime::{
     GlobalPublishedOutboundTlsState, load_global_outbound_tls_generation, load_global_outbound_tls_state,
     record_tls_consumer_stale_generation,
 };
+use std::time::Duration;
 
 pub(crate) fn outbound_tls_generation() -> u64 {
     load_global_outbound_tls_generation().0
@@ -76,3 +77,28 @@ pub(crate) fn record_classified_error(operation: &'static str, classification: &
         classification,
     );
 }
+
+pub(crate) fn record_duration(operation: &'static str, duration: Duration) {
+    global_internode_metrics().record_duration_for_operation_and_backend(
+        operation,
+        INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
+        duration,
+    );
+}
+
+pub(crate) fn record_http_version(operation: &'static str, http_version: &'static str) {
+    global_internode_metrics().record_http_version_for_operation_and_backend(
+        operation,
+        INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
+        http_version,
+    );
+}
+
+pub(crate) fn record_stall_timeout(operation: &'static str) {
+    global_internode_metrics().record_stall_timeout_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP);
+}
+
+pub(crate) fn record_write_shutdown_error(operation: &'static str) {
+    global_internode_metrics()
+        .record_write_shutdown_error_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP);
+}