gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-16 18:08:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(metrics): count internode RPC auth failures (#5777)

Browse Source 
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
2026-08-06 23:42:59 +08:00
committed by GitHub
parent da82fd995e
commit 5e7e25b7d1
3 changed files with 190 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/server/http.rs
+61
View File
@@ -53,6 +53,10 @@ use metrics::{counter, gauge, histogram};
use opentelemetry::global;
use opentelemetry::trace::TraceContextExt;
use rustfs_common::GlobalReadiness;
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_GRPC_OTHER, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics,
};
use rustfs_keystone::KeystoneAuthLayer;
#[cfg(feature = "swift")]
use rustfs_protocols::SwiftService;
@@ -1903,6 +1907,17 @@ fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
.map(|addr| addr.0.to_string())
.unwrap_or_else(|| "unknown".to_string());
let failure_reason = storage::tonic_rpc_auth_failure_reason(&e);
let operation = match rpc_method {
"ReadAll" => INTERNODE_OPERATION_GRPC_READ_ALL,
"ReadMultiple" => INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
"WriteAll" => INTERNODE_OPERATION_GRPC_WRITE_ALL,
_ => INTERNODE_OPERATION_GRPC_OTHER,
};
global_internode_metrics().record_rpc_auth_failure_for_operation_and_backend(
operation,
INTERNODE_TRANSPORT_BACKEND_GRPC,
failure_reason,
);
error!(
event = EVENT_RPC_SIGNATURE_VERIFICATION_FAILED,
component = LOG_COMPONENT_SERVER,
@@ -2025,7 +2040,10 @@ mod tests {
use http::Request as HttpRequest;
use http::{HeaderMap, StatusCode};
use http_body_util::{Empty, Full};
use metrics::with_local_recorder;
use metrics_util::debugging::DebuggingRecorder;
use opentelemetry::propagation::Extractor;
use std::collections::HashMap;
use std::convert::Infallible;
use std::future::Ready;
use std::sync::{Arc, Mutex};
@@ -2394,6 +2412,49 @@ mod tests {
rustfs_common::set_global_local_node_name(&previous_node_name).await;
}
#[tokio::test]
#[serial_test::serial]
async fn rpc_auth_rejection_records_failure_reason_metric() {
let _ = rustfs_credentials::set_global_rpc_secret("rpc-http-test-secret".to_string());
let previous_node_name = rustfs_common::get_global_local_node_name().await;
rustfs_common::set_global_local_node_name("127.0.0.1:9000").await;
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
with_local_recorder(&recorder, || {
let mut request = Request::new(());
request.extensions_mut().insert(RpcRequestTarget {
uri: "http://127.0.0.1:9000/node_service.NodeService/ReadAll"
.parse()
.expect("test RPC URI should parse"),
method: Method::POST,
});
let error = check_auth(request).expect_err("missing signature must be rejected");
assert_eq!(error.code(), tonic::Code::Unauthenticated);
assert_eq!(error.message(), "No valid auth token");
});
let entries: Vec<_> = snapshotter
.snapshot()
.into_vec()
.into_iter()
.filter(|(composite, _, _, _)| composite.key().name() == "rustfs_system_network_internode_rpc_auth_failures_total")
.collect();
assert_eq!(entries.len(), 1);
let labels: HashMap<_, _> = entries[0]
.0
.key()
.labels()
.map(|label| (label.key().to_string(), label.value().to_string()))
.collect();
assert_eq!(labels.get("operation").map(String::as_str), Some(INTERNODE_OPERATION_GRPC_READ_ALL));
assert_eq!(labels.get("backend").map(String::as_str), Some(INTERNODE_TRANSPORT_BACKEND_GRPC));
assert_eq!(labels.get("failure_reason").map(String::as_str), Some("missing_v1_signature"));
assert!(labels.get("server").is_some_and(|value| !value.is_empty()));
rustfs_common::set_global_local_node_name(&previous_node_name).await;
}
/// Rolling-upgrade compatibility anchor for <https://github.com/rustfs/backlog/issues/1327>:
/// a legacy-only peer (constant-target signature, no v2 headers) must keep authenticating
/// through the real production path (`check_auth` + `RpcRequestTarget` extension), and every
rustfs/src/storage/rpc/http_service.rs
+43 -2
View File
@@ -26,6 +26,7 @@ use crate::storage::storage_api::rpc_consumer::http_service::{
WALK_DIR_BODY_SHA256_QUERY,
};
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use crate::storage::storage_api::tonic_rpc_auth_failure_reason;
use bytes::{Bytes, BytesMut};
use futures_util::{Stream, StreamExt, TryStreamExt, stream};
use http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode, Uri};
@@ -472,11 +473,17 @@ fn verify_internode_rpc_signature(uri: &Uri, method: &Method, headers: &HeaderMa
verify_rpc_signature(&uri.to_string(), method, headers).map_err(|e| {
let message = format!("rpc signature verification failed: {e}");
let operation = internode_http_operation(uri.path());
runtime_sources::current_internode_metrics().record_rpc_auth_failure_for_operation_and_backend(
operation.unwrap_or(RPC_OPERATION_UNKNOWN),
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
tonic_rpc_auth_failure_reason(&e),
);
log_internode_rpc_response_failure!(
StatusCode::FORBIDDEN,
uri.path(),
method,
internode_http_operation(uri.path()),
operation,
"signature_verification_failed",
"rejected",
None,
@@ -1309,11 +1316,14 @@ mod tests {
use bytes::Bytes;
use http::{HeaderMap, HeaderValue, Method, StatusCode, Uri};
use http_body_util::BodyExt;
use metrics::with_local_recorder;
use metrics_util::debugging::DebuggingRecorder;
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_NS_SCANNER, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_OPERATION_WALK_DIR, global_internode_metrics,
INTERNODE_OPERATION_WALK_DIR, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
};
use sha2::Digest as _;
use std::collections::HashMap;
use tokio::io;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio_stream::StreamExt;
@@ -1384,6 +1394,37 @@ mod tests {
assert_eq!(response.status(), StatusCode::FORBIDDEN);
}
#[test]
fn rpc_get_request_auth_failure_records_failure_reason_metric() {
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let uri: Uri = READ_FILE_STREAM_PATH.parse().expect("uri");
let headers = HeaderMap::new();
with_local_recorder(&recorder, || {
let response = verify_internode_rpc_signature(&uri, &Method::GET, &headers).expect_err("response");
assert_eq!(response.status(), StatusCode::FORBIDDEN);
});
let entries: Vec<_> = snapshotter
.snapshot()
.into_vec()
.into_iter()
.filter(|(composite, _, _, _)| composite.key().name() == "rustfs_system_network_internode_rpc_auth_failures_total")
.collect();
assert_eq!(entries.len(), 1);
let labels: HashMap<_, _> = entries[0]
.0
.key()
.labels()
.map(|label| (label.key().to_string(), label.value().to_string()))
.collect();
assert_eq!(labels.get("operation").map(String::as_str), Some(INTERNODE_OPERATION_READ_FILE_STREAM));
assert_eq!(labels.get("backend").map(String::as_str), Some(INTERNODE_TRANSPORT_BACKEND_TCP_HTTP));
assert_eq!(labels.get("failure_reason").map(String::as_str), Some("missing_v1_signature"));
assert!(labels.get("server").is_some_and(|value| !value.is_empty()));
}
#[test]
fn put_file_stage_error_message_includes_stage_and_request_context() {
let query = PutFileQuery {