feat(storage): harden internode data-path controls (#4224)

* fix(rio): propagate http writer shutdown errors * fix(ecstore): unify remote lock rpc deadlines * fix(storage): reject corrupt read multiple payloads * feat(rio): add internode http tuning profiles * feat(metrics): add internode baseline signals * feat(ecstore): observe shard locality topology * feat(ecstore): gate shard locality scheduling * feat(ecstore): gate batch read version rpc * feat(ecstore): observe batch processor adaptation * feat(ecstore): gate batch processor observation * docs: add get benchmark regression analysis * docs: add issue 797 execution plan status * fix(ecstore): require explicit batch rpc support * fix(ecstore): honor documented batch read gate * fix(ecstore): keep batch read gate stable per call * chore: update workspace dependencies * feat(ecstore): log batch read gate decisions * feat(ecstore): count batch read gate decisions * test(issue-797): add local internode A/B runner * test(rio): fix tuning profile spelling fixture * fix(protocols): adapt sftp channel open callbacks * fix(metrics): wrap batch processor observation args * chore(docs): keep issue notes local only * fix(storage): address internode review feedback * fix(storage): address internode data-path review findings - Run the BatchReadVersion auto-mode unary fallback outside the batch RPC deadline so each read_version keeps its own per-op timeout and health accounting instead of racing the whole batch against one drive timeout. - Cap adaptive batch-processor concurrency growth at a hard multiple of the configured baseline so sustained fast batches cannot ratchet past the configured limit. - Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE, and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading the environment on hot paths. - Skip shard read-cost collection in observe mode when stage metrics are disabled, and cache the local endpoint host list instead of rebuilding it on every read. - Allow --warp-extra-args values starting with -- and drop the unused warp_hosts_csv helper in the issue-797 A/B runner. * fix(storage): address internode data-path review findings - Run the BatchReadVersion auto-mode unary fallback outside the batch RPC deadline so each read_version keeps its own per-op timeout and health accounting instead of racing the whole batch against one drive timeout. - Cap adaptive batch-processor concurrency growth at a hard multiple of the configured baseline so sustained fast batches cannot ratchet past the configured limit. - Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE, and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading the environment on hot paths. - Skip shard read-cost collection in observe mode when stage metrics are disabled, and cache the local endpoint host list instead of rebuilding it on every read. - Allow --warp-extra-args values starting with -- and drop the unused warp_hosts_csv helper in the issue-797 A/B runner. Co-Authored-By: heihutu<heihutu@gmail.com> * fix(storage): align buffer clamp test with media cap * fix(ecstore): release optimized read locks before streaming --------- Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
2026-08-20 03:22:18 +00:00 · 2026-07-03 17:08:15 +08:00
parent a9ac5f578d
commit 25d80d7c60
26 changed files with 3627 additions and 483 deletions
@@ -531,6 +531,26 @@ pub struct ReadVersionResponse {
    pub file_info_bin: ::prost::bytes::Bytes,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct BatchReadVersionRequest {
+    #[prost(string, tag = "1")]
+    pub disk: ::prost::alloc::string::String,
+    #[prost(string, tag = "2")]
+    pub batch_read_version_req: ::prost::alloc::string::String,
+    #[prost(bytes = "bytes", tag = "3")]
+    pub batch_read_version_req_bin: ::prost::bytes::Bytes,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct BatchReadVersionResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(string, repeated, tag = "2")]
+    pub batch_read_version_resps: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+    #[prost(message, optional, tag = "3")]
+    pub error: ::core::option::Option<Error>,
+    #[prost(bytes = "bytes", repeated, tag = "4")]
+    pub batch_read_version_resps_bin: ::prost::alloc::vec::Vec<::prost::bytes::Bytes>,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ReadXlRequest {
    #[prost(string, tag = "1")]
    pub disk: ::prost::alloc::string::String,
@@ -1689,6 +1709,21 @@ pub mod node_service_client {
                .insert(GrpcMethod::new("node_service.NodeService", "ReadVersion"));
            self.inner.unary(req, path, codec).await
        }
+        pub async fn batch_read_version(
+            &mut self,
+            request: impl tonic::IntoRequest<super::BatchReadVersionRequest>,
+        ) -> std::result::Result<tonic::Response<super::BatchReadVersionResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/BatchReadVersion");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "BatchReadVersion"));
+            self.inner.unary(req, path, codec).await
+        }
        pub async fn read_xl(
            &mut self,
            request: impl tonic::IntoRequest<super::ReadXlRequest>,
@@ -2610,6 +2645,10 @@ pub mod node_service_server {
            &self,
            request: tonic::Request<super::ReadVersionRequest>,
        ) -> std::result::Result<tonic::Response<super::ReadVersionResponse>, tonic::Status>;
+        async fn batch_read_version(
+            &self,
+            request: tonic::Request<super::BatchReadVersionRequest>,
+        ) -> std::result::Result<tonic::Response<super::BatchReadVersionResponse>, tonic::Status>;
        async fn read_xl(
            &self,
            request: tonic::Request<super::ReadXlRequest>,
@@ -3706,6 +3745,34 @@ pub mod node_service_server {
                    };
                    Box::pin(fut)
                }
+                "/node_service.NodeService/BatchReadVersion" => {
+                    #[allow(non_camel_case_types)]
+                    struct BatchReadVersionSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::BatchReadVersionRequest> for BatchReadVersionSvc<T> {
+                        type Response = super::BatchReadVersionResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::BatchReadVersionRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::batch_read_version(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = BatchReadVersionSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                "/node_service.NodeService/ReadXL" => {
                    #[allow(non_camel_case_types)]
                    struct ReadXLSvc<T: NodeService>(pub Arc<T>);