refactor: centralize network client runtime sources (#3796)

This commit is contained in:
Zhengchao An
2026-06-23 22:35:51 +08:00
committed by GitHub
parent 726c26fa01
commit e59e1852ec
9 changed files with 210 additions and 75 deletions
+1
View File
@@ -35,6 +35,7 @@ pub mod constants;
pub mod credentials;
pub mod object_api_utils;
pub mod object_handlers_common;
pub(crate) mod runtime_sources;
pub mod signer_error;
pub mod transition_api;
pub mod utils;
@@ -0,0 +1,25 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, load_global_outbound_tls_state, record_tls_generation};
const ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER: &str = "ecstore_transition_client";
pub(crate) async fn transition_client_outbound_tls_state() -> GlobalPublishedOutboundTlsState {
load_global_outbound_tls_state().await
}
pub(crate) fn record_transition_client_tls_generation(generation: u64) {
record_tls_generation(ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER, generation);
}
+2 -3
View File
@@ -51,7 +51,6 @@ use md5::Md5;
use rand::{Rng, RngExt};
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
use rustfs_rio::HashReader;
use rustfs_tls_runtime::{load_global_outbound_tls_state, record_tls_generation};
use rustfs_utils::HashAlgorithm;
use rustfs_utils::{
net::get_endpoint_url,
@@ -192,8 +191,8 @@ where
async fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
with_rustls_init_guard(|| Ok(()))?;
let outbound_tls = load_global_outbound_tls_state().await;
record_tls_generation("ecstore_transition_client", outbound_tls.generation.0);
let outbound_tls = crate::client::runtime_sources::transition_client_outbound_tls_state().await;
crate::client::runtime_sources::record_transition_client_tls_generation(outbound_tls.generation.0);
let builder = if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() {
let mut reader = std::io::BufReader::new(root_ca_pem.as_slice());
let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader)
+1
View File
@@ -20,6 +20,7 @@ mod peer_rest_client;
mod peer_s3_client;
mod remote_disk;
mod remote_locker;
mod runtime_sources;
pub use client::{
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
+14 -58
View File
@@ -34,10 +34,6 @@ use bytes::Bytes;
use futures::lock::Mutex;
use metrics::counter;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
};
use rustfs_protos::evict_failed_connection;
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
use rustfs_protos::proto_gen::node_service::{
@@ -187,22 +183,14 @@ impl RemoteDisk {
if attempt > 1
&& let Some(classification) = last_retry_classification
{
global_internode_metrics().record_retry_success_for_operation_and_backend(
rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
classification,
);
crate::rpc::runtime_sources::record_remote_disk_open_write_retry_success(classification);
}
return Ok(writer);
}
Err(err) if attempt < REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS && Self::is_retryable_open_write_error(&err) => {
if let Some(classification) = err.internode_http_error_kind() {
let classification = classification.metric_label();
global_internode_metrics().record_retry_for_operation_and_backend(
rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
classification,
);
crate::rpc::runtime_sources::record_remote_disk_open_write_retry(classification);
last_retry_classification = Some(classification);
}
debug!(
@@ -2109,10 +2097,7 @@ impl DiskAPI for RemoteDisk {
let data_len = data.len();
let disk = self.disk_ref().await;
let mut client = self.get_client().await.map_err(|err| {
global_internode_metrics().record_error_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_WRITE_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
);
crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_error();
Error::other(format!("can not get client, err: {err}"))
})?;
let request = Request::new(WriteAllRequest {
@@ -2122,32 +2107,19 @@ impl DiskAPI for RemoteDisk {
data,
});
global_internode_metrics().record_outgoing_request_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_WRITE_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
);
crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
let response = match client.write_all(request).await {
Ok(response) => response.into_inner(),
Err(err) => {
global_internode_metrics().record_error_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_WRITE_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
);
crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_error();
return Err(err.into());
}
};
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_WRITE_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
data_len,
);
crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len);
if !response.success {
global_internode_metrics().record_error_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_WRITE_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
);
crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_error();
return Err(response.error.unwrap_or_default().into());
}
@@ -2176,10 +2148,7 @@ impl DiskAPI for RemoteDisk {
|| async {
let disk = self.disk_ref().await;
let mut client = self.get_client().await.map_err(|err| {
global_internode_metrics().record_error_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_READ_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
);
crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
Error::other(format!("can not get client, err: {err}"))
})?;
let request = Request::new(ReadAllRequest {
@@ -2188,34 +2157,21 @@ impl DiskAPI for RemoteDisk {
path: path.to_string(),
});
global_internode_metrics().record_outgoing_request_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_READ_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
);
crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_request();
let response = match client.read_all(request).await {
Ok(response) => response.into_inner(),
Err(err) => {
global_internode_metrics().record_error_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_READ_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
);
crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
return Err(err.into());
}
};
if !response.success {
global_internode_metrics().record_error_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_READ_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
);
crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
return Err(response.error.unwrap_or_default().into());
}
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_READ_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
response.data.len(),
);
crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_recv_bytes(response.data.len());
Ok(response.data)
},
get_max_timeout_duration(),
@@ -2972,7 +2928,7 @@ mod tests {
OpenWriteTestStep::Success,
]);
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
rustfs_io_metrics::internode_metrics::global_internode_metrics().reset_for_test();
crate::rpc::runtime_sources::reset_internode_metrics_for_test();
let _created = remote_disk
.create_file("orig-bucket", "bucket", "object/part.1", 4096)
@@ -2981,7 +2937,7 @@ mod tests {
let calls = transport.calls();
assert_eq!(calls.len(), 2, "create_file should retry exactly once");
let snapshot = rustfs_io_metrics::internode_metrics::global_internode_metrics().snapshot();
let snapshot = crate::rpc::runtime_sources::internode_metrics_snapshot_for_test();
assert_eq!(snapshot.outgoing_requests_total, 0);
}
+83
View File
@@ -0,0 +1,83 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
};
#[cfg(test)]
use rustfs_io_metrics::internode_metrics::InternodeMetricsSnapshot;
pub(crate) fn record_remote_disk_open_write_retry(classification: &'static str) {
global_internode_metrics().record_retry_for_operation_and_backend(
INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
classification,
);
}
pub(crate) fn record_remote_disk_open_write_retry_success(classification: &'static str) {
global_internode_metrics().record_retry_success_for_operation_and_backend(
INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
classification,
);
}
pub(crate) fn record_remote_disk_grpc_write_all_error() {
global_internode_metrics()
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_write_all_request() {
global_internode_metrics()
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_write_all_sent_bytes(bytes: usize) {
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_WRITE_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
bytes,
);
}
pub(crate) fn record_remote_disk_grpc_read_all_error() {
global_internode_metrics()
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_read_all_request() {
global_internode_metrics()
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_read_all_recv_bytes(bytes: usize) {
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_READ_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
bytes,
);
}
#[cfg(test)]
pub(crate) fn reset_internode_metrics_for_test() {
global_internode_metrics().reset_for_test();
}
#[cfg(test)]
pub(crate) fn internode_metrics_snapshot_for_test() -> InternodeMetricsSnapshot {
global_internode_metrics().snapshot()
}
+5 -6
View File
@@ -16,11 +16,10 @@
// scoped to that module so generated internals do not relax lints elsewhere.
#[allow(unsafe_code)]
mod generated;
mod runtime_sources;
use proto_gen::node_service::node_service_client::NodeServiceClient;
use rustfs_common::{GLOBAL_CONN_MAP, evict_connection_with_log_level};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_tls_runtime::{load_global_outbound_tls_state, record_tls_consumer_stale_generation};
use std::{
collections::HashMap,
error::Error,
@@ -134,7 +133,7 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
// Overall timeout for any RPC - fail fast on unresponsive peers
.timeout(rpc_timeout);
let outbound_tls = load_global_outbound_tls_state().await;
let outbound_tls = runtime_sources::outbound_tls_state().await;
let generation = outbound_tls.generation.0;
let mut stale_generation = false;
{
@@ -180,11 +179,11 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
let channel = match connector.connect().await {
Ok(channel) => {
global_internode_metrics().record_dial_result(dial_started_at.elapsed(), true);
runtime_sources::record_grpc_dial_result(dial_started_at.elapsed(), true);
channel
}
Err(err) => {
global_internode_metrics().record_dial_result(dial_started_at.elapsed(), false);
runtime_sources::record_grpc_dial_result(dial_started_at.elapsed(), false);
return Err(err.into());
}
};
@@ -199,7 +198,7 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
generation_cache.insert(addr.to_string(), generation);
}
if stale_generation {
record_tls_consumer_stale_generation("protos_grpc_channel");
runtime_sources::record_stale_grpc_channel_tls_generation();
}
debug!("Successfully created and cached gRPC channel to: {}", addr);
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, load_global_outbound_tls_state, record_tls_consumer_stale_generation};
use std::time::Duration;
const PROTOS_GRPC_CHANNEL_TLS_CONSUMER: &str = "protos_grpc_channel";
pub(crate) async fn outbound_tls_state() -> GlobalPublishedOutboundTlsState {
load_global_outbound_tls_state().await
}
pub(crate) fn record_stale_grpc_channel_tls_generation() {
record_tls_consumer_stale_generation(PROTOS_GRPC_CHANNEL_TLS_CONSUMER);
}
pub(crate) fn record_grpc_dial_result(duration: Duration, success: bool) {
global_internode_metrics().record_dial_result(duration, success);
}
+48 -8
View File
@@ -5,11 +5,11 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-rio-http-runtime-sources`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184`.
- Based on: latest `origin/main` after PR #3793 merged the API-182/API-183
stack; this branch batches RIO HTTP runtime source boundary cleanup on top of
that baseline.
- Branch: `overtrue/arch-network-client-runtime-sources`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185`.
- Based on: latest `origin/main` after PR #3795 merged API-184; this branch
batches network client runtime source boundary cleanup on top of that
baseline.
- PR type for this branch: `consumer-migration`
- Runtime behavior changes: none.
- Rust code changes: route replication pool, outbound TLS generation, runtime
@@ -17,8 +17,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
internode RPC metrics, IAM authorization/handler reads, notification
rules/event dispatch, admin OIDC/token-signing reads, IAM root credential
consumers, IAM OIDC config reads, scanner runtime-config reads, OBS metrics
runtime source reads, and RIO HTTP reader TLS/metrics runtime source reads
through AppContext-first or owner-crate resolver boundaries.
runtime source reads, RIO HTTP reader TLS/metrics runtime source reads, and
gRPC/transition network client TLS/metrics runtime source reads through
AppContext-first or owner-crate resolver boundaries.
- CI/script changes: lock completed owner and test/fuzz boundaries against
bare/glob imports, scattered raw ECStore facade subpaths, and startup
runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade
@@ -27,7 +28,7 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
and storage owner thin bridge regressions, plus app context and notify
event-bridge thin module regressions; accept the reviewed AppContext resolver
reverse dependencies in the layer baseline.
- Docs changes: record the API-136 through API-184 owner facade cleanup.
- Docs changes: record the API-136 through API-185 owner facade cleanup.
## Phase 0 Tasks
@@ -4667,6 +4668,23 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
risk scan, branch freshness check, pre-commit quality gate, and three-expert
review.
- [x] `API-185` Centralize network client runtime source reads.
- Do: add runtime-source boundaries for `rustfs-protos` gRPC channel TLS and
dial metrics, ECStore transition-client outbound TLS state and generation
reporting, and ECStore remote-disk gRPC/TCP HTTP internode metric recording.
- Acceptance: gRPC channel creation, transition-client TLS config, and remote
disk RPC/write-stream retry paths no longer import outbound TLS global
readers or internode metrics globals directly outside their owner runtime
source boundary modules.
- Must preserve: gRPC channel TLS generation cache invalidation, dial success
and error metrics, transition-client TLS material loading, TLS generation
gauges, remote-disk write-stream retry metrics, and gRPC read/write
request/error/byte counters.
- Verification: protos/ECStore compile coverage, focused unit tests,
formatting, migration guard, layer guard, diff hygiene, residual network
client runtime source scan, Rust risk scan, branch freshness check,
pre-commit quality gate, and three-expert review.
## Next PRs
1. `consumer-migration`: continue reducing direct global reads behind AppContext resolver boundaries.
@@ -4775,11 +4793,33 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
| Quality/architecture | pass | API-184 keeps RIO HTTP outbound TLS and internode metric globals behind a RIO-owned runtime-source module without widening public APIs. |
| Migration preservation | pass | HTTP client cache generation checks, TLS material loading, stale-generation reporting, and TCP/HTTP internode metrics keep existing behavior. |
| Testing/verification | pass | RIO compile/tests, formatting, residual RIO runtime source scan, targeted guard checks, and pre-commit passed for API-184. |
| Quality/architecture | pass | API-185 keeps protos and ECStore network client TLS/metrics globals behind owner runtime-source modules without widening public APIs. |
| Migration preservation | pass | gRPC TLS cache invalidation, transition-client TLS generation reporting, dial metrics, retry metrics, and read/write byte/error counters keep existing behavior. |
| Testing/verification | pass | Protos/ECStore compile/tests, formatting, residual network client runtime source scan, targeted guard checks, and pre-commit passed for API-185. |
## Verification Notes
Passed before push:
- Issue #660 API-185 current slice:
- `cargo check -p rustfs-protos -p rustfs-ecstore --tests`: passed.
- `cargo test -p rustfs-protos --lib`: passed.
- `cargo test -p rustfs-ecstore --lib remote_disk -- --test-threads=1`:
passed.
- `cargo fmt --all`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- Network client runtime source scan: passed; direct outbound TLS and
internode metrics global reads are isolated to the protos and ECStore
runtime source modules.
- Rust risk scan: passed; diff adds no new `expect`, `panic`, `todo`,
`unimplemented`, or `unsafe`.
- Branch freshness check: based on latest `origin/main` after PR #3795
merged API-184.
- `make pre-commit`: passed.
- Issue #660 API-184 current slice:
- `cargo check -p rustfs-rio --tests`: passed.
- `cargo test -p rustfs-rio --lib`: passed.