mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-24 13:16:28 +00:00
refactor: centralize network client runtime sources (#3796)
This commit is contained in:
@@ -35,6 +35,7 @@ pub mod constants;
|
|||||||
pub mod credentials;
|
pub mod credentials;
|
||||||
pub mod object_api_utils;
|
pub mod object_api_utils;
|
||||||
pub mod object_handlers_common;
|
pub mod object_handlers_common;
|
||||||
|
pub(crate) mod runtime_sources;
|
||||||
pub mod signer_error;
|
pub mod signer_error;
|
||||||
pub mod transition_api;
|
pub mod transition_api;
|
||||||
pub mod utils;
|
pub mod utils;
|
||||||
|
|||||||
@@ -0,0 +1,25 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, load_global_outbound_tls_state, record_tls_generation};
|
||||||
|
|
||||||
|
const ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER: &str = "ecstore_transition_client";
|
||||||
|
|
||||||
|
pub(crate) async fn transition_client_outbound_tls_state() -> GlobalPublishedOutboundTlsState {
|
||||||
|
load_global_outbound_tls_state().await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn record_transition_client_tls_generation(generation: u64) {
|
||||||
|
record_tls_generation(ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER, generation);
|
||||||
|
}
|
||||||
@@ -51,7 +51,6 @@ use md5::Md5;
|
|||||||
use rand::{Rng, RngExt};
|
use rand::{Rng, RngExt};
|
||||||
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
||||||
use rustfs_rio::HashReader;
|
use rustfs_rio::HashReader;
|
||||||
use rustfs_tls_runtime::{load_global_outbound_tls_state, record_tls_generation};
|
|
||||||
use rustfs_utils::HashAlgorithm;
|
use rustfs_utils::HashAlgorithm;
|
||||||
use rustfs_utils::{
|
use rustfs_utils::{
|
||||||
net::get_endpoint_url,
|
net::get_endpoint_url,
|
||||||
@@ -192,8 +191,8 @@ where
|
|||||||
async fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
|
async fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
|
||||||
with_rustls_init_guard(|| Ok(()))?;
|
with_rustls_init_guard(|| Ok(()))?;
|
||||||
|
|
||||||
let outbound_tls = load_global_outbound_tls_state().await;
|
let outbound_tls = crate::client::runtime_sources::transition_client_outbound_tls_state().await;
|
||||||
record_tls_generation("ecstore_transition_client", outbound_tls.generation.0);
|
crate::client::runtime_sources::record_transition_client_tls_generation(outbound_tls.generation.0);
|
||||||
let builder = if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() {
|
let builder = if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() {
|
||||||
let mut reader = std::io::BufReader::new(root_ca_pem.as_slice());
|
let mut reader = std::io::BufReader::new(root_ca_pem.as_slice());
|
||||||
let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader)
|
let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader)
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ mod peer_rest_client;
|
|||||||
mod peer_s3_client;
|
mod peer_s3_client;
|
||||||
mod remote_disk;
|
mod remote_disk;
|
||||||
mod remote_locker;
|
mod remote_locker;
|
||||||
|
mod runtime_sources;
|
||||||
|
|
||||||
pub use client::{
|
pub use client::{
|
||||||
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||||
|
|||||||
@@ -34,10 +34,6 @@ use bytes::Bytes;
|
|||||||
use futures::lock::Mutex;
|
use futures::lock::Mutex;
|
||||||
use metrics::counter;
|
use metrics::counter;
|
||||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||||
use rustfs_io_metrics::internode_metrics::{
|
|
||||||
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC,
|
|
||||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
|
|
||||||
};
|
|
||||||
use rustfs_protos::evict_failed_connection;
|
use rustfs_protos::evict_failed_connection;
|
||||||
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
|
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
|
||||||
use rustfs_protos::proto_gen::node_service::{
|
use rustfs_protos::proto_gen::node_service::{
|
||||||
@@ -187,22 +183,14 @@ impl RemoteDisk {
|
|||||||
if attempt > 1
|
if attempt > 1
|
||||||
&& let Some(classification) = last_retry_classification
|
&& let Some(classification) = last_retry_classification
|
||||||
{
|
{
|
||||||
global_internode_metrics().record_retry_success_for_operation_and_backend(
|
crate::rpc::runtime_sources::record_remote_disk_open_write_retry_success(classification);
|
||||||
rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM,
|
|
||||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
|
||||||
classification,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
return Ok(writer);
|
return Ok(writer);
|
||||||
}
|
}
|
||||||
Err(err) if attempt < REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS && Self::is_retryable_open_write_error(&err) => {
|
Err(err) if attempt < REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS && Self::is_retryable_open_write_error(&err) => {
|
||||||
if let Some(classification) = err.internode_http_error_kind() {
|
if let Some(classification) = err.internode_http_error_kind() {
|
||||||
let classification = classification.metric_label();
|
let classification = classification.metric_label();
|
||||||
global_internode_metrics().record_retry_for_operation_and_backend(
|
crate::rpc::runtime_sources::record_remote_disk_open_write_retry(classification);
|
||||||
rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM,
|
|
||||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
|
||||||
classification,
|
|
||||||
);
|
|
||||||
last_retry_classification = Some(classification);
|
last_retry_classification = Some(classification);
|
||||||
}
|
}
|
||||||
debug!(
|
debug!(
|
||||||
@@ -2109,10 +2097,7 @@ impl DiskAPI for RemoteDisk {
|
|||||||
let data_len = data.len();
|
let data_len = data.len();
|
||||||
let disk = self.disk_ref().await;
|
let disk = self.disk_ref().await;
|
||||||
let mut client = self.get_client().await.map_err(|err| {
|
let mut client = self.get_client().await.map_err(|err| {
|
||||||
global_internode_metrics().record_error_for_operation_and_backend(
|
crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_error();
|
||||||
INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
|
||||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
|
||||||
);
|
|
||||||
Error::other(format!("can not get client, err: {err}"))
|
Error::other(format!("can not get client, err: {err}"))
|
||||||
})?;
|
})?;
|
||||||
let request = Request::new(WriteAllRequest {
|
let request = Request::new(WriteAllRequest {
|
||||||
@@ -2122,32 +2107,19 @@ impl DiskAPI for RemoteDisk {
|
|||||||
data,
|
data,
|
||||||
});
|
});
|
||||||
|
|
||||||
global_internode_metrics().record_outgoing_request_for_operation_and_backend(
|
crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
|
||||||
INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
|
||||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
|
||||||
);
|
|
||||||
let response = match client.write_all(request).await {
|
let response = match client.write_all(request).await {
|
||||||
Ok(response) => response.into_inner(),
|
Ok(response) => response.into_inner(),
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
global_internode_metrics().record_error_for_operation_and_backend(
|
crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_error();
|
||||||
INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
|
||||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
|
||||||
);
|
|
||||||
return Err(err.into());
|
return Err(err.into());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
|
crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len);
|
||||||
INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
|
||||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
|
||||||
data_len,
|
|
||||||
);
|
|
||||||
|
|
||||||
if !response.success {
|
if !response.success {
|
||||||
global_internode_metrics().record_error_for_operation_and_backend(
|
crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_error();
|
||||||
INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
|
||||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
|
||||||
);
|
|
||||||
return Err(response.error.unwrap_or_default().into());
|
return Err(response.error.unwrap_or_default().into());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2176,10 +2148,7 @@ impl DiskAPI for RemoteDisk {
|
|||||||
|| async {
|
|| async {
|
||||||
let disk = self.disk_ref().await;
|
let disk = self.disk_ref().await;
|
||||||
let mut client = self.get_client().await.map_err(|err| {
|
let mut client = self.get_client().await.map_err(|err| {
|
||||||
global_internode_metrics().record_error_for_operation_and_backend(
|
crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
|
||||||
INTERNODE_OPERATION_GRPC_READ_ALL,
|
|
||||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
|
||||||
);
|
|
||||||
Error::other(format!("can not get client, err: {err}"))
|
Error::other(format!("can not get client, err: {err}"))
|
||||||
})?;
|
})?;
|
||||||
let request = Request::new(ReadAllRequest {
|
let request = Request::new(ReadAllRequest {
|
||||||
@@ -2188,34 +2157,21 @@ impl DiskAPI for RemoteDisk {
|
|||||||
path: path.to_string(),
|
path: path.to_string(),
|
||||||
});
|
});
|
||||||
|
|
||||||
global_internode_metrics().record_outgoing_request_for_operation_and_backend(
|
crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_request();
|
||||||
INTERNODE_OPERATION_GRPC_READ_ALL,
|
|
||||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
|
||||||
);
|
|
||||||
let response = match client.read_all(request).await {
|
let response = match client.read_all(request).await {
|
||||||
Ok(response) => response.into_inner(),
|
Ok(response) => response.into_inner(),
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
global_internode_metrics().record_error_for_operation_and_backend(
|
crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
|
||||||
INTERNODE_OPERATION_GRPC_READ_ALL,
|
|
||||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
|
||||||
);
|
|
||||||
return Err(err.into());
|
return Err(err.into());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if !response.success {
|
if !response.success {
|
||||||
global_internode_metrics().record_error_for_operation_and_backend(
|
crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
|
||||||
INTERNODE_OPERATION_GRPC_READ_ALL,
|
|
||||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
|
||||||
);
|
|
||||||
return Err(response.error.unwrap_or_default().into());
|
return Err(response.error.unwrap_or_default().into());
|
||||||
}
|
}
|
||||||
|
|
||||||
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
|
crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_recv_bytes(response.data.len());
|
||||||
INTERNODE_OPERATION_GRPC_READ_ALL,
|
|
||||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
|
||||||
response.data.len(),
|
|
||||||
);
|
|
||||||
Ok(response.data)
|
Ok(response.data)
|
||||||
},
|
},
|
||||||
get_max_timeout_duration(),
|
get_max_timeout_duration(),
|
||||||
@@ -2972,7 +2928,7 @@ mod tests {
|
|||||||
OpenWriteTestStep::Success,
|
OpenWriteTestStep::Success,
|
||||||
]);
|
]);
|
||||||
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
|
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
|
||||||
rustfs_io_metrics::internode_metrics::global_internode_metrics().reset_for_test();
|
crate::rpc::runtime_sources::reset_internode_metrics_for_test();
|
||||||
|
|
||||||
let _created = remote_disk
|
let _created = remote_disk
|
||||||
.create_file("orig-bucket", "bucket", "object/part.1", 4096)
|
.create_file("orig-bucket", "bucket", "object/part.1", 4096)
|
||||||
@@ -2981,7 +2937,7 @@ mod tests {
|
|||||||
|
|
||||||
let calls = transport.calls();
|
let calls = transport.calls();
|
||||||
assert_eq!(calls.len(), 2, "create_file should retry exactly once");
|
assert_eq!(calls.len(), 2, "create_file should retry exactly once");
|
||||||
let snapshot = rustfs_io_metrics::internode_metrics::global_internode_metrics().snapshot();
|
let snapshot = crate::rpc::runtime_sources::internode_metrics_snapshot_for_test();
|
||||||
assert_eq!(snapshot.outgoing_requests_total, 0);
|
assert_eq!(snapshot.outgoing_requests_total, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
use rustfs_io_metrics::internode_metrics::{
|
||||||
|
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM,
|
||||||
|
INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_io_metrics::internode_metrics::InternodeMetricsSnapshot;
|
||||||
|
|
||||||
|
pub(crate) fn record_remote_disk_open_write_retry(classification: &'static str) {
|
||||||
|
global_internode_metrics().record_retry_for_operation_and_backend(
|
||||||
|
INTERNODE_OPERATION_PUT_FILE_STREAM,
|
||||||
|
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||||
|
classification,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn record_remote_disk_open_write_retry_success(classification: &'static str) {
|
||||||
|
global_internode_metrics().record_retry_success_for_operation_and_backend(
|
||||||
|
INTERNODE_OPERATION_PUT_FILE_STREAM,
|
||||||
|
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||||
|
classification,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn record_remote_disk_grpc_write_all_error() {
|
||||||
|
global_internode_metrics()
|
||||||
|
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn record_remote_disk_grpc_write_all_request() {
|
||||||
|
global_internode_metrics()
|
||||||
|
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn record_remote_disk_grpc_write_all_sent_bytes(bytes: usize) {
|
||||||
|
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
|
||||||
|
INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||||
|
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||||
|
bytes,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn record_remote_disk_grpc_read_all_error() {
|
||||||
|
global_internode_metrics()
|
||||||
|
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn record_remote_disk_grpc_read_all_request() {
|
||||||
|
global_internode_metrics()
|
||||||
|
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn record_remote_disk_grpc_read_all_recv_bytes(bytes: usize) {
|
||||||
|
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
|
||||||
|
INTERNODE_OPERATION_GRPC_READ_ALL,
|
||||||
|
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||||
|
bytes,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn reset_internode_metrics_for_test() {
|
||||||
|
global_internode_metrics().reset_for_test();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn internode_metrics_snapshot_for_test() -> InternodeMetricsSnapshot {
|
||||||
|
global_internode_metrics().snapshot()
|
||||||
|
}
|
||||||
@@ -16,11 +16,10 @@
|
|||||||
// scoped to that module so generated internals do not relax lints elsewhere.
|
// scoped to that module so generated internals do not relax lints elsewhere.
|
||||||
#[allow(unsafe_code)]
|
#[allow(unsafe_code)]
|
||||||
mod generated;
|
mod generated;
|
||||||
|
mod runtime_sources;
|
||||||
|
|
||||||
use proto_gen::node_service::node_service_client::NodeServiceClient;
|
use proto_gen::node_service::node_service_client::NodeServiceClient;
|
||||||
use rustfs_common::{GLOBAL_CONN_MAP, evict_connection_with_log_level};
|
use rustfs_common::{GLOBAL_CONN_MAP, evict_connection_with_log_level};
|
||||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
|
||||||
use rustfs_tls_runtime::{load_global_outbound_tls_state, record_tls_consumer_stale_generation};
|
|
||||||
use std::{
|
use std::{
|
||||||
collections::HashMap,
|
collections::HashMap,
|
||||||
error::Error,
|
error::Error,
|
||||||
@@ -134,7 +133,7 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
|
|||||||
// Overall timeout for any RPC - fail fast on unresponsive peers
|
// Overall timeout for any RPC - fail fast on unresponsive peers
|
||||||
.timeout(rpc_timeout);
|
.timeout(rpc_timeout);
|
||||||
|
|
||||||
let outbound_tls = load_global_outbound_tls_state().await;
|
let outbound_tls = runtime_sources::outbound_tls_state().await;
|
||||||
let generation = outbound_tls.generation.0;
|
let generation = outbound_tls.generation.0;
|
||||||
let mut stale_generation = false;
|
let mut stale_generation = false;
|
||||||
{
|
{
|
||||||
@@ -180,11 +179,11 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
|
|||||||
|
|
||||||
let channel = match connector.connect().await {
|
let channel = match connector.connect().await {
|
||||||
Ok(channel) => {
|
Ok(channel) => {
|
||||||
global_internode_metrics().record_dial_result(dial_started_at.elapsed(), true);
|
runtime_sources::record_grpc_dial_result(dial_started_at.elapsed(), true);
|
||||||
channel
|
channel
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
global_internode_metrics().record_dial_result(dial_started_at.elapsed(), false);
|
runtime_sources::record_grpc_dial_result(dial_started_at.elapsed(), false);
|
||||||
return Err(err.into());
|
return Err(err.into());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -199,7 +198,7 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
|
|||||||
generation_cache.insert(addr.to_string(), generation);
|
generation_cache.insert(addr.to_string(), generation);
|
||||||
}
|
}
|
||||||
if stale_generation {
|
if stale_generation {
|
||||||
record_tls_consumer_stale_generation("protos_grpc_channel");
|
runtime_sources::record_stale_grpc_channel_tls_generation();
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!("Successfully created and cached gRPC channel to: {}", addr);
|
debug!("Successfully created and cached gRPC channel to: {}", addr);
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||||
|
use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, load_global_outbound_tls_state, record_tls_consumer_stale_generation};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
const PROTOS_GRPC_CHANNEL_TLS_CONSUMER: &str = "protos_grpc_channel";
|
||||||
|
|
||||||
|
pub(crate) async fn outbound_tls_state() -> GlobalPublishedOutboundTlsState {
|
||||||
|
load_global_outbound_tls_state().await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn record_stale_grpc_channel_tls_generation() {
|
||||||
|
record_tls_consumer_stale_generation(PROTOS_GRPC_CHANNEL_TLS_CONSUMER);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn record_grpc_dial_result(duration: Duration, success: bool) {
|
||||||
|
global_internode_metrics().record_dial_result(duration, success);
|
||||||
|
}
|
||||||
@@ -5,11 +5,11 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
## Current Context
|
## Current Context
|
||||||
|
|
||||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||||
- Branch: `overtrue/arch-rio-http-runtime-sources`
|
- Branch: `overtrue/arch-network-client-runtime-sources`
|
||||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184`.
|
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185`.
|
||||||
- Based on: latest `origin/main` after PR #3793 merged the API-182/API-183
|
- Based on: latest `origin/main` after PR #3795 merged API-184; this branch
|
||||||
stack; this branch batches RIO HTTP runtime source boundary cleanup on top of
|
batches network client runtime source boundary cleanup on top of that
|
||||||
that baseline.
|
baseline.
|
||||||
- PR type for this branch: `consumer-migration`
|
- PR type for this branch: `consumer-migration`
|
||||||
- Runtime behavior changes: none.
|
- Runtime behavior changes: none.
|
||||||
- Rust code changes: route replication pool, outbound TLS generation, runtime
|
- Rust code changes: route replication pool, outbound TLS generation, runtime
|
||||||
@@ -17,8 +17,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
internode RPC metrics, IAM authorization/handler reads, notification
|
internode RPC metrics, IAM authorization/handler reads, notification
|
||||||
rules/event dispatch, admin OIDC/token-signing reads, IAM root credential
|
rules/event dispatch, admin OIDC/token-signing reads, IAM root credential
|
||||||
consumers, IAM OIDC config reads, scanner runtime-config reads, OBS metrics
|
consumers, IAM OIDC config reads, scanner runtime-config reads, OBS metrics
|
||||||
runtime source reads, and RIO HTTP reader TLS/metrics runtime source reads
|
runtime source reads, RIO HTTP reader TLS/metrics runtime source reads, and
|
||||||
through AppContext-first or owner-crate resolver boundaries.
|
gRPC/transition network client TLS/metrics runtime source reads through
|
||||||
|
AppContext-first or owner-crate resolver boundaries.
|
||||||
- CI/script changes: lock completed owner and test/fuzz boundaries against
|
- CI/script changes: lock completed owner and test/fuzz boundaries against
|
||||||
bare/glob imports, scattered raw ECStore facade subpaths, and startup
|
bare/glob imports, scattered raw ECStore facade subpaths, and startup
|
||||||
runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade
|
runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade
|
||||||
@@ -27,7 +28,7 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
and storage owner thin bridge regressions, plus app context and notify
|
and storage owner thin bridge regressions, plus app context and notify
|
||||||
event-bridge thin module regressions; accept the reviewed AppContext resolver
|
event-bridge thin module regressions; accept the reviewed AppContext resolver
|
||||||
reverse dependencies in the layer baseline.
|
reverse dependencies in the layer baseline.
|
||||||
- Docs changes: record the API-136 through API-184 owner facade cleanup.
|
- Docs changes: record the API-136 through API-185 owner facade cleanup.
|
||||||
|
|
||||||
## Phase 0 Tasks
|
## Phase 0 Tasks
|
||||||
|
|
||||||
@@ -4667,6 +4668,23 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
risk scan, branch freshness check, pre-commit quality gate, and three-expert
|
risk scan, branch freshness check, pre-commit quality gate, and three-expert
|
||||||
review.
|
review.
|
||||||
|
|
||||||
|
- [x] `API-185` Centralize network client runtime source reads.
|
||||||
|
- Do: add runtime-source boundaries for `rustfs-protos` gRPC channel TLS and
|
||||||
|
dial metrics, ECStore transition-client outbound TLS state and generation
|
||||||
|
reporting, and ECStore remote-disk gRPC/TCP HTTP internode metric recording.
|
||||||
|
- Acceptance: gRPC channel creation, transition-client TLS config, and remote
|
||||||
|
disk RPC/write-stream retry paths no longer import outbound TLS global
|
||||||
|
readers or internode metrics globals directly outside their owner runtime
|
||||||
|
source boundary modules.
|
||||||
|
- Must preserve: gRPC channel TLS generation cache invalidation, dial success
|
||||||
|
and error metrics, transition-client TLS material loading, TLS generation
|
||||||
|
gauges, remote-disk write-stream retry metrics, and gRPC read/write
|
||||||
|
request/error/byte counters.
|
||||||
|
- Verification: protos/ECStore compile coverage, focused unit tests,
|
||||||
|
formatting, migration guard, layer guard, diff hygiene, residual network
|
||||||
|
client runtime source scan, Rust risk scan, branch freshness check,
|
||||||
|
pre-commit quality gate, and three-expert review.
|
||||||
|
|
||||||
## Next PRs
|
## Next PRs
|
||||||
|
|
||||||
1. `consumer-migration`: continue reducing direct global reads behind AppContext resolver boundaries.
|
1. `consumer-migration`: continue reducing direct global reads behind AppContext resolver boundaries.
|
||||||
@@ -4775,11 +4793,33 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
| Quality/architecture | pass | API-184 keeps RIO HTTP outbound TLS and internode metric globals behind a RIO-owned runtime-source module without widening public APIs. |
|
| Quality/architecture | pass | API-184 keeps RIO HTTP outbound TLS and internode metric globals behind a RIO-owned runtime-source module without widening public APIs. |
|
||||||
| Migration preservation | pass | HTTP client cache generation checks, TLS material loading, stale-generation reporting, and TCP/HTTP internode metrics keep existing behavior. |
|
| Migration preservation | pass | HTTP client cache generation checks, TLS material loading, stale-generation reporting, and TCP/HTTP internode metrics keep existing behavior. |
|
||||||
| Testing/verification | pass | RIO compile/tests, formatting, residual RIO runtime source scan, targeted guard checks, and pre-commit passed for API-184. |
|
| Testing/verification | pass | RIO compile/tests, formatting, residual RIO runtime source scan, targeted guard checks, and pre-commit passed for API-184. |
|
||||||
|
| Quality/architecture | pass | API-185 keeps protos and ECStore network client TLS/metrics globals behind owner runtime-source modules without widening public APIs. |
|
||||||
|
| Migration preservation | pass | gRPC TLS cache invalidation, transition-client TLS generation reporting, dial metrics, retry metrics, and read/write byte/error counters keep existing behavior. |
|
||||||
|
| Testing/verification | pass | Protos/ECStore compile/tests, formatting, residual network client runtime source scan, targeted guard checks, and pre-commit passed for API-185. |
|
||||||
|
|
||||||
## Verification Notes
|
## Verification Notes
|
||||||
|
|
||||||
Passed before push:
|
Passed before push:
|
||||||
|
|
||||||
|
- Issue #660 API-185 current slice:
|
||||||
|
- `cargo check -p rustfs-protos -p rustfs-ecstore --tests`: passed.
|
||||||
|
- `cargo test -p rustfs-protos --lib`: passed.
|
||||||
|
- `cargo test -p rustfs-ecstore --lib remote_disk -- --test-threads=1`:
|
||||||
|
passed.
|
||||||
|
- `cargo fmt --all`: passed.
|
||||||
|
- `cargo fmt --all --check`: passed.
|
||||||
|
- `git diff --check`: passed.
|
||||||
|
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||||
|
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||||
|
- Network client runtime source scan: passed; direct outbound TLS and
|
||||||
|
internode metrics global reads are isolated to the protos and ECStore
|
||||||
|
runtime source modules.
|
||||||
|
- Rust risk scan: passed; diff adds no new `expect`, `panic`, `todo`,
|
||||||
|
`unimplemented`, or `unsafe`.
|
||||||
|
- Branch freshness check: based on latest `origin/main` after PR #3795
|
||||||
|
merged API-184.
|
||||||
|
- `make pre-commit`: passed.
|
||||||
|
|
||||||
- Issue #660 API-184 current slice:
|
- Issue #660 API-184 current slice:
|
||||||
- `cargo check -p rustfs-rio --tests`: passed.
|
- `cargo check -p rustfs-rio --tests`: passed.
|
||||||
- `cargo test -p rustfs-rio --lib`: passed.
|
- `cargo test -p rustfs-rio --lib`: passed.
|
||||||
|
|||||||
Reference in New Issue
Block a user