feat(storage): harden internode data-path controls (#4224)

* fix(rio): propagate http writer shutdown errors

* fix(ecstore): unify remote lock rpc deadlines

* fix(storage): reject corrupt read multiple payloads

* feat(rio): add internode http tuning profiles

* feat(metrics): add internode baseline signals

* feat(ecstore): observe shard locality topology

* feat(ecstore): gate shard locality scheduling

* feat(ecstore): gate batch read version rpc

* feat(ecstore): observe batch processor adaptation

* feat(ecstore): gate batch processor observation

* docs: add get benchmark regression analysis

* docs: add issue 797 execution plan status

* fix(ecstore): require explicit batch rpc support

* fix(ecstore): honor documented batch read gate

* fix(ecstore): keep batch read gate stable per call

* chore: update workspace dependencies

* feat(ecstore): log batch read gate decisions

* feat(ecstore): count batch read gate decisions

* test(issue-797): add local internode A/B runner

* test(rio): fix tuning profile spelling fixture

* fix(protocols): adapt sftp channel open callbacks

* fix(metrics): wrap batch processor observation args

* chore(docs): keep issue notes local only

* fix(storage): address internode review feedback

* fix(storage): address internode data-path review findings

- Run the BatchReadVersion auto-mode unary fallback outside the batch
  RPC deadline so each read_version keeps its own per-op timeout and
  health accounting instead of racing the whole batch against one
  drive timeout.
- Cap adaptive batch-processor concurrency growth at a hard multiple
  of the configured baseline so sustained fast batches cannot ratchet
  past the configured limit.
- Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE,
  and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading
  the environment on hot paths.
- Skip shard read-cost collection in observe mode when stage metrics
  are disabled, and cache the local endpoint host list instead of
  rebuilding it on every read.
- Allow --warp-extra-args values starting with -- and drop the unused
  warp_hosts_csv helper in the issue-797 A/B runner.

* fix(storage): address internode data-path review findings

- Run the BatchReadVersion auto-mode unary fallback outside the batch
  RPC deadline so each read_version keeps its own per-op timeout and
  health accounting instead of racing the whole batch against one
  drive timeout.
- Cap adaptive batch-processor concurrency growth at a hard multiple
  of the configured baseline so sustained fast batches cannot ratchet
  past the configured limit.
- Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE,
  and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading
  the environment on hot paths.
- Skip shard read-cost collection in observe mode when stage metrics
  are disabled, and cache the local endpoint host list instead of
  rebuilding it on every read.
- Allow --warp-extra-args values starting with -- and drop the unused
  warp_hosts_csv helper in the issue-797 A/B runner.

Co-Authored-By: heihutu<heihutu@gmail.com>

* fix(storage): align buffer clamp test with media cap

* fix(ecstore): release optimized read locks before streaming

---------

Co-authored-by: Zhengchao An <anzhengchao@gmail.com>

crates/ecstore/src/disk/mod.rs

@@ -577,6 +577,9 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
         version_id: &str,
         opts: &ReadOptions,
     ) -> Result<FileInfo>;
+    async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
+        batch_read_version_one_by_one(self, req).await
+    }
     async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo>;
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes>;
     async fn rename_data(
@@ -637,6 +640,41 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     fn start_scan(&self) -> ScanGuard;
 }
 
+pub async fn batch_read_version_one_by_one<D>(disk: &D, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>>
+where
+    D: DiskAPI + ?Sized,
+{
+    validate_batch_read_version_item_count(req.items.len())?;
+
+    let mut responses = Vec::with_capacity(req.items.len());
+    for (index, item) in req.items.iter().enumerate() {
+        let response = match disk
+            .read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
+            .await
+        {
+            Ok(file_info) => BatchReadVersionResp {
+                index,
+                path: item.path.clone(),
+                version_id: item.version_id.clone(),
+                success: true,
+                file_info,
+                error: String::new(),
+            },
+            Err(err) => BatchReadVersionResp {
+                index,
+                path: item.path.clone(),
+                version_id: item.version_id.clone(),
+                success: false,
+                file_info: FileInfo::default(),
+                error: err.to_string(),
+            },
+        };
+        responses.push(response);
+    }
+
+    Ok(responses)
+}
+
 #[derive(Debug, Default, Serialize, Deserialize)]
 pub struct CheckPartsResp {
     pub results: Vec<usize>,
@@ -807,6 +845,41 @@ pub struct ReadMultipleResp {
     pub mod_time: Option<OffsetDateTime>,
 }
 
+pub const BATCH_READ_VERSION_MAX_ITEMS: usize = 128;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BatchReadVersionItem {
+    pub org_volume: String,
+    pub volume: String,
+    pub path: String,
+    pub version_id: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BatchReadVersionReq {
+    pub items: Vec<BatchReadVersionItem>,
+    pub opts: ReadOptions,
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct BatchReadVersionResp {
+    pub index: usize,
+    pub path: String,
+    pub version_id: String,
+    pub success: bool,
+    pub file_info: FileInfo,
+    pub error: String,
+}
+
+pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
+    if item_count > BATCH_READ_VERSION_MAX_ITEMS {
+        return Err(DiskError::other(format!(
+            "batch read version item count {item_count} exceeds limit {BATCH_READ_VERSION_MAX_ITEMS}"
+        )));
+    }
+    Ok(())
+}
+
 #[derive(Debug, Deserialize, Serialize)]
 pub struct VolumeInfo {
     pub name: String,