gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-07 22:03:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(internode): add transport observability (#3007)

Browse Source 
* docs: add internode data transport RFC

* feat: add internode operation metrics

* fix feedback

* fix(ci): fallback protoc token to github.token

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-05-19 15:16:05 +08:00
committed by GitHub
parent aa3f13c0d3
commit f695870626
7 changed files with 671 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/storage/rpc/disk.rs
+32 -14
View File
@@ -13,6 +13,9 @@
// limitations under the License.
use super::*;
use rustfs_common::internode_metrics::{
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, global_internode_metrics,
};
use serde::de::DeserializeOwned;
use std::io::Cursor;
@@ -929,18 +932,25 @@ impl NodeService {
pub(super) async fn handle_write_all(&self, request: Request<WriteAllRequest>) -> Result<Response<WriteAllResponse>, Status> {
let request = request.into_inner();
let data_len = request.data.len();
global_internode_metrics().record_incoming_request_for_operation(INTERNODE_OPERATION_GRPC_WRITE_ALL);
global_internode_metrics().record_recv_bytes_for_operation(INTERNODE_OPERATION_GRPC_WRITE_ALL, data_len);
if let Some(disk) = self.find_disk(&request.disk).await {
match disk.write_all(&request.volume, &request.path, request.data).await {
Ok(_) => Ok(Response::new(WriteAllResponse {
success: true,
error: None,
})),
Err(err) => Ok(Response::new(WriteAllResponse {
success: false,
error: Some(err.into()),
})),
Err(err) => {
global_internode_metrics().record_error_for_operation(INTERNODE_OPERATION_GRPC_WRITE_ALL);
Ok(Response::new(WriteAllResponse {
success: false,
error: Some(err.into()),
}))
}
}
} else {
global_internode_metrics().record_error_for_operation(INTERNODE_OPERATION_GRPC_WRITE_ALL);
Ok(Response::new(WriteAllResponse {
success: false,
error: Some(DiskError::other("can not find disk".to_string()).into()),
@@ -952,20 +962,28 @@ impl NodeService {
debug!("read all");
let request = request.into_inner();
global_internode_metrics().record_incoming_request_for_operation(INTERNODE_OPERATION_GRPC_READ_ALL);
if let Some(disk) = self.find_disk(&request.disk).await {
match disk.read_all(&request.volume, &request.path).await {
Ok(data) => Ok(Response::new(ReadAllResponse {
success: true,
data,
error: None,
})),
Err(err) => Ok(Response::new(ReadAllResponse {
success: false,
data: Bytes::new(),
error: Some(err.into()),
})),
Ok(data) => {
global_internode_metrics().record_sent_bytes_for_operation(INTERNODE_OPERATION_GRPC_READ_ALL, data.len());
Ok(Response::new(ReadAllResponse {
success: true,
data,
error: None,
}))
}
Err(err) => {
global_internode_metrics().record_error_for_operation(INTERNODE_OPERATION_GRPC_READ_ALL);
Ok(Response::new(ReadAllResponse {
success: false,
data: Bytes::new(),
error: Some(err.into()),
}))
}
}
} else {
global_internode_metrics().record_error_for_operation(INTERNODE_OPERATION_GRPC_READ_ALL);
Ok(Response::new(ReadAllResponse {
success: false,
data: Bytes::new(),
rustfs/src/storage/rpc/http_service.rs
+48 -13
View File
@@ -18,7 +18,10 @@ use futures_util::TryStreamExt;
use http::{HeaderMap, Method, Request, Response, StatusCode, Uri};
use http_body_util::{BodyExt, Limited};
use hyper::body::Incoming;
use rustfs_common::internode_metrics::global_internode_metrics;
use rustfs_common::internode_metrics::{
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
global_internode_metrics,
};
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
use rustfs_ecstore::disk::{DiskAPI, WalkDirOptions};
use rustfs_ecstore::rpc::verify_rpc_signature;
@@ -106,8 +109,9 @@ fn is_internode_rpc_path(path: &str) -> bool {
}
async fn handle_internode_rpc(req: Request<Incoming>) -> Response<Body> {
let operation = internode_http_operation(req.uri().path());
if let Err(response) = verify_internode_rpc_signature(req.uri(), req.method(), req.headers()) {
global_internode_metrics().record_error();
record_internode_rpc_error(operation);
return *response;
}
@@ -122,12 +126,28 @@ async fn handle_internode_rpc(req: Request<Incoming>) -> Response<Body> {
};
if !response.status().is_success() {
global_internode_metrics().record_error();
record_internode_rpc_error(operation);
}
response
}
fn internode_http_operation(path: &str) -> Option<&'static str> {
match path {
READ_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_READ_FILE_STREAM),
PUT_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_PUT_FILE_STREAM),
WALK_DIR_PATH => Some(INTERNODE_OPERATION_WALK_DIR),
_ => None,
}
}
fn record_internode_rpc_error(operation: Option<&'static str>) {
match operation {
Some(operation) => global_internode_metrics().record_error_for_operation(operation),
None => global_internode_metrics().record_error(),
}
}
fn verify_internode_rpc_signature(uri: &Uri, method: &Method, headers: &HeaderMap) -> Result<(), RpcErrorResponse> {
if method == Method::HEAD {
return Ok(());
@@ -163,8 +183,8 @@ async fn handle_read_file(req: Request<Incoming>) -> Response<Body> {
Err(e) => return response_with_status(StatusCode::INTERNAL_SERVER_ERROR, format!("read file err {e}")),
};
global_internode_metrics().record_incoming_request();
let stream = read_file_body_stream(file, query.length);
global_internode_metrics().record_incoming_request_for_operation(INTERNODE_OPERATION_READ_FILE_STREAM);
let stream = read_file_body_stream(file, query.length, INTERNODE_OPERATION_READ_FILE_STREAM);
Response::builder()
.status(StatusCode::OK)
@@ -172,13 +192,17 @@ async fn handle_read_file(req: Request<Incoming>) -> Response<Body> {
.expect("failed to build read file stream response")
}
fn read_file_body_stream<R>(reader: R, length: usize) -> Pin<Box<dyn futures::Stream<Item = io::Result<Bytes>> + Send + Sync>>
fn read_file_body_stream<R>(
reader: R,
length: usize,
operation: &'static str,
) -> Pin<Box<dyn futures::Stream<Item = io::Result<Bytes>> + Send + Sync>>
where
R: tokio::io::AsyncRead + Unpin + Send + Sync + 'static,
{
let metrics = global_internode_metrics().clone();
let stream = ReaderStream::with_capacity(reader, DEFAULT_READ_BUFFER_SIZE).map_ok(move |bytes| {
metrics.record_sent_bytes(bytes.len());
metrics.record_sent_bytes_for_operation(operation, bytes.len());
bytes
});
@@ -220,10 +244,10 @@ async fn handle_walk_dir(req: Request<Incoming>) -> Response<Body> {
}
});
global_internode_metrics().record_incoming_request();
global_internode_metrics().record_incoming_request_for_operation(INTERNODE_OPERATION_WALK_DIR);
let metrics = global_internode_metrics().clone();
let stream = ReaderStream::with_capacity(rd, DEFAULT_READ_BUFFER_SIZE).map_ok(move |bytes| {
metrics.record_sent_bytes(bytes.len());
metrics.record_sent_bytes_for_operation(INTERNODE_OPERATION_WALK_DIR, bytes.len());
bytes
});
@@ -260,8 +284,8 @@ async fn handle_put_file(req: Request<Incoming>) -> Response<Body> {
Err(e) => return response_with_status(StatusCode::INTERNAL_SERVER_ERROR, format!("write file err {e}")),
};
global_internode_metrics().record_incoming_request();
global_internode_metrics().record_recv_bytes(copied as usize);
global_internode_metrics().record_incoming_request_for_operation(INTERNODE_OPERATION_PUT_FILE_STREAM);
global_internode_metrics().record_recv_bytes_for_operation(INTERNODE_OPERATION_PUT_FILE_STREAM, copied as usize);
if let Err(e) = file.flush().await {
return response_with_status(StatusCode::INTERNAL_SERVER_ERROR, format!("write file err {e}"));
@@ -337,6 +361,17 @@ mod tests {
assert!(!is_internode_rpc_path("/rustfs/admin/v3/info"));
}
#[test]
fn internode_http_operation_maps_only_known_routes() {
assert_eq!(
internode_http_operation(READ_FILE_STREAM_PATH),
Some(INTERNODE_OPERATION_READ_FILE_STREAM)
);
assert_eq!(internode_http_operation(PUT_FILE_STREAM_PATH), Some(INTERNODE_OPERATION_PUT_FILE_STREAM));
assert_eq!(internode_http_operation(WALK_DIR_PATH), Some(INTERNODE_OPERATION_WALK_DIR));
assert_eq!(internode_http_operation("/rustfs/rpc/unknown"), None);
}
#[test]
fn rpc_head_signature_verification_is_skipped() {
let uri: Uri = READ_FILE_STREAM_PATH.parse().expect("uri");
@@ -377,7 +412,7 @@ mod tests {
writer.write_all(b"hello world").await.expect("write succeeds");
});
let mut stream = read_file_body_stream(reader, 0);
let mut stream = read_file_body_stream(reader, 0, INTERNODE_OPERATION_READ_FILE_STREAM);
let mut out = Vec::new();
while let Some(chunk) = stream.next().await {
out.extend_from_slice(&chunk.expect("chunk succeeds"));
@@ -393,7 +428,7 @@ mod tests {
writer.write_all(b"hello world").await.expect("write succeeds");
});
let mut stream = read_file_body_stream(reader, 5);
let mut stream = read_file_body_stream(reader, 5, INTERNODE_OPERATION_READ_FILE_STREAM);
let mut out = Vec::new();
while let Some(chunk) = stream.next().await {
out.extend_from_slice(&chunk.expect("chunk succeeds"));