feat(storage): harden internode data-path controls (#4224)

* fix(rio): propagate http writer shutdown errors

* fix(ecstore): unify remote lock rpc deadlines

* fix(storage): reject corrupt read multiple payloads

* feat(rio): add internode http tuning profiles

* feat(metrics): add internode baseline signals

* feat(ecstore): observe shard locality topology

* feat(ecstore): gate shard locality scheduling

* feat(ecstore): gate batch read version rpc

* feat(ecstore): observe batch processor adaptation

* feat(ecstore): gate batch processor observation

* docs: add get benchmark regression analysis

* docs: add issue 797 execution plan status

* fix(ecstore): require explicit batch rpc support

* fix(ecstore): honor documented batch read gate

* fix(ecstore): keep batch read gate stable per call

* chore: update workspace dependencies

* feat(ecstore): log batch read gate decisions

* feat(ecstore): count batch read gate decisions

* test(issue-797): add local internode A/B runner

* test(rio): fix tuning profile spelling fixture

* fix(protocols): adapt sftp channel open callbacks

* fix(metrics): wrap batch processor observation args

* chore(docs): keep issue notes local only

* fix(storage): address internode review feedback

* fix(storage): address internode data-path review findings

- Run the BatchReadVersion auto-mode unary fallback outside the batch
  RPC deadline so each read_version keeps its own per-op timeout and
  health accounting instead of racing the whole batch against one
  drive timeout.
- Cap adaptive batch-processor concurrency growth at a hard multiple
  of the configured baseline so sustained fast batches cannot ratchet
  past the configured limit.
- Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE,
  and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading
  the environment on hot paths.
- Skip shard read-cost collection in observe mode when stage metrics
  are disabled, and cache the local endpoint host list instead of
  rebuilding it on every read.
- Allow --warp-extra-args values starting with -- and drop the unused
  warp_hosts_csv helper in the issue-797 A/B runner.

* fix(storage): address internode data-path review findings

- Run the BatchReadVersion auto-mode unary fallback outside the batch
  RPC deadline so each read_version keeps its own per-op timeout and
  health accounting instead of racing the whole batch against one
  drive timeout.
- Cap adaptive batch-processor concurrency growth at a hard multiple
  of the configured baseline so sustained fast batches cannot ratchet
  past the configured limit.
- Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE,
  and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading
  the environment on hot paths.
- Skip shard read-cost collection in observe mode when stage metrics
  are disabled, and cache the local endpoint host list instead of
  rebuilding it on every read.
- Allow --warp-extra-args values starting with -- and drop the unused
  warp_hosts_csv helper in the issue-797 A/B runner.

Co-Authored-By: heihutu<heihutu@gmail.com>

* fix(storage): align buffer clamp test with media cap

* fix(ecstore): release optimized read locks before streaming

---------

Co-authored-by: Zhengchao An <anzhengchao@gmail.com>

crates/config/src/constants/internode.rs

@@ -32,6 +32,28 @@ pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3;
 pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS";
 pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 10;
 
+/// Profile selector for conservative internode HTTP data-plane client tuning.
+pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE";
+pub const DEFAULT_INTERNODE_HTTP_TUNING_PROFILE: &str = "legacy";
+
+/// Internode HTTP connection pool maximum idle connections per host.
+pub const ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST: &str = "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST";
+
+/// Internode HTTP connection pool idle timeout in seconds.
+pub const ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS";
+
+/// Internode HTTP/2 initial stream window size in bytes.
+pub const ENV_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE";
+
+/// Internode HTTP/2 initial connection window size in bytes.
+pub const ENV_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE";
+
+/// Whether internode HTTP/2 adaptive window sizing is enabled.
+pub const ENV_INTERNODE_HTTP2_ADAPTIVE_WINDOW: &str = "RUSTFS_INTERNODE_HTTP2_ADAPTIVE_WINDOW";
+
+/// Internode HTTP proxy mode: legacy, off, or system.
+pub const ENV_INTERNODE_HTTP_PROXY: &str = "RUSTFS_INTERNODE_HTTP_PROXY";
+
 /// Environment variable for selecting the internode data-plane transport backend.
 pub const ENV_RUSTFS_INTERNODE_DATA_TRANSPORT: &str = "RUSTFS_INTERNODE_DATA_TRANSPORT";
 pub const DEFAULT_INTERNODE_DATA_TRANSPORT: &str = "tcp-http";
@@ -53,6 +75,7 @@ mod tests {
         assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5);
         assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 3);
         assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 10);
+        assert_eq!(DEFAULT_INTERNODE_HTTP_TUNING_PROFILE, "legacy");
     }
 
     #[test]
@@ -68,6 +91,19 @@ mod tests {
             "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS"
         );
         assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS");
+        assert_eq!(ENV_INTERNODE_HTTP_TUNING_PROFILE, "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE");
+        assert_eq!(ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST, "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST");
+        assert_eq!(ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS, "RUSTFS_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS");
+        assert_eq!(
+            ENV_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE,
+            "RUSTFS_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE"
+        );
+        assert_eq!(
+            ENV_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE,
+            "RUSTFS_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE"
+        );
+        assert_eq!(ENV_INTERNODE_HTTP2_ADAPTIVE_WINDOW, "RUSTFS_INTERNODE_HTTP2_ADAPTIVE_WINDOW");
+        assert_eq!(ENV_INTERNODE_HTTP_PROXY, "RUSTFS_INTERNODE_HTTP_PROXY");
         assert_eq!(ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, "RUSTFS_INTERNODE_DATA_TRANSPORT");
         assert_eq!(DEFAULT_INTERNODE_DATA_TRANSPORT, "tcp-http");
         assert_eq!(INTERNODE_DATA_TRANSPORT_TCP, "tcp");

crates/config/src/constants/object.rs

@@ -191,8 +191,9 @@ pub const DEFAULT_OBJECT_IO_BUFFER_SIZE: usize = 128 * 1024;
 
 /// Environment variable to enable/disable lock optimization.
 ///
-/// When enabled, fully materialized reads may release read locks before the
-/// reader is returned. Streaming reads keep the lock until EOF or drop.
+/// When enabled, read locks may be released before the reader is returned.
+/// Disable this only when streaming readers must keep the object namespace
+/// locked until EOF or drop.
 ///
 /// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE`).
 pub const ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE: &str = "RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE";