gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-06 13:27:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(storage): optimize internode RPC transfer path (#2262)

Browse Source 
Co-authored-by: momoda693 <momoda693@gmail.com>
This commit is contained in:
weisd
2026-03-23 17:09:49 +08:00
committed by GitHub
parent 236142a682
commit 05dc131a49
43 changed files with 1846 additions and 566 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/mod.rs
-3
View File
@@ -16,7 +16,6 @@ mod auth;
pub mod console;
pub mod handlers;
pub mod router;
mod rpc;
pub mod utils;
#[cfg(test)]
@@ -28,7 +27,6 @@ use handlers::{
bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, sts, system, tier, user,
};
use router::{AdminOperation, S3Router};
use rpc::register_rpc_route;
use s3s::route::S3Route;
/// Create admin router
@@ -44,7 +42,6 @@ pub fn make_admin_route(console_enabled: bool) -> std::io::Result<impl S3Route>
health::register_health_route(&mut r)?;
sts::register_admin_auth_route(&mut r)?;
register_rpc_route(&mut r)?;
user::register_user_route(&mut r)?;
system::register_system_route(&mut r)?;
pools::register_pool_route(&mut r)?;
rustfs/src/admin/route_registration_test.rs
+6 -27
View File
@@ -15,7 +15,6 @@
use crate::admin::{
handlers::{bucket_meta, heal, health, kms, pools, profile_admin, quota, rebalance, replication, sts, system, tier, user},
router::{AdminOperation, S3Router},
rpc,
};
use crate::server::{ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH};
use hyper::Method;
@@ -54,8 +53,6 @@ fn test_register_routes_cover_representative_admin_paths() {
replication::register_replication_route(&mut router).expect("register replication route");
profile_admin::register_profiling_route(&mut router).expect("register profile route");
kms::register_kms_route(&mut router).expect("register kms route");
rpc::register_rpc_route(&mut router).expect("register rpc route");
assert_route(&router, Method::GET, HEALTH_PREFIX);
assert_route(&router, Method::HEAD, HEALTH_PREFIX);
assert_route(&router, Method::GET, HEALTH_READY_PATH);
@@ -117,8 +114,11 @@ fn test_register_routes_cover_representative_admin_paths() {
assert_route(&router, Method::POST, &admin_path("/v3/kms/keys"));
assert_route(&router, Method::GET, &admin_path("/v3/kms/keys"));
assert_route(&router, Method::GET, &admin_path("/v3/kms/keys/test-key"));
assert_route(&router, Method::GET, "/rustfs/rpc/read_file_stream");
assert_route(&router, Method::HEAD, "/rustfs/rpc/read_file_stream");
assert!(
!router.contains_route(Method::GET, "/rustfs/rpc/read_file_stream"),
"internode rpc routes should no longer be registered inside the admin router"
);
}
#[test]
@@ -160,9 +160,8 @@ fn test_admin_alias_paths_match_existing_admin_routes() {
}
#[test]
fn test_phase5_admin_info_and_rpc_read_file_contract() {
fn test_phase5_admin_info_contract() {
let system_src = include_str!("handlers/system.rs");
let rpc_src = include_str!("rpc.rs");
let server_info_impl_marker = "impl Operation for ServerInfoHandler";
let server_info_impl_start = system_src
@@ -175,24 +174,4 @@ fn test_phase5_admin_info_and_rpc_read_file_contract() {
&& server_info_impl_block.contains("execute_query_server_info(QueryServerInfoRequest { include_pools: true })"),
"admin server info path must be served through DefaultAdminUsecase::execute_query_server_info"
);
let register_route_marker = "pub fn register_rpc_route";
let register_route_start = rpc_src
.find(register_route_marker)
.expect("Expected register_rpc_route in rpc.rs");
let register_route_block = &rpc_src[register_route_start..];
let read_file_marker = "pub struct ReadFile {}";
let read_file_start = rpc_src.find(read_file_marker).expect("Expected ReadFile operation in rpc.rs");
let read_file_block = &rpc_src[read_file_start..];
assert!(
register_route_block.contains("format!(\"{}{}\", RPC_PREFIX, \"/read_file_stream\")"),
"rpc read_file_stream route path must remain registered with RPC_PREFIX"
);
assert!(
read_file_block.contains(".read_file_stream(&query.volume, &query.path, query.offset, query.length)"),
"rpc read_file_stream route must remain wired to disk.read_file_stream"
);
}
rustfs/src/admin/router.rs
+2 -18
View File
@@ -14,9 +14,7 @@
use crate::admin::console::{is_console_path, make_console_server};
use crate::admin::handlers::oidc::is_oidc_path;
use crate::server::{
ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX,
};
use crate::server::{ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH};
use hyper::HeaderMap;
use hyper::Method;
use hyper::StatusCode;
@@ -24,7 +22,6 @@ use hyper::Uri;
use hyper::http::Extensions;
use matchit::Params;
use matchit::Router;
use rustfs_ecstore::rpc::verify_rpc_signature;
use s3s::Body;
use s3s::S3Request;
use s3s::S3Response;
@@ -33,7 +30,6 @@ use s3s::header;
use s3s::route::S3Route;
use s3s::s3_error;
use tower::Service;
use tracing::error;
pub struct S3Router<T> {
router: Router<T>,
@@ -140,7 +136,7 @@ where
return true;
}
is_admin_path(path) || path.starts_with(RPC_PREFIX) || is_console_path(path)
is_admin_path(path) || is_console_path(path)
}
// check_access before call
@@ -168,18 +164,6 @@ where
return Ok(());
}
// Check RPC signature verification
if req.uri.path().starts_with(RPC_PREFIX) {
// Skip signature verification for HEAD requests (health checks)
if req.method != Method::HEAD {
verify_rpc_signature(&req.uri.to_string(), &req.method, &req.headers).map_err(|e| {
error!("RPC signature verification failed: {}", e);
s3_error!(AccessDenied, "{}", e)
})?;
}
return Ok(());
}
// Allow unauthenticated STS requests to POST / (AssumeRoleWithWebIdentity
// doesn't use SigV4 — the JWT token in the request body is the authentication).
// The handler dispatches on the Action parameter: AssumeRole will reject if
rustfs/src/admin/rpc.rs
-218
View File
@@ -1,218 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::server::RPC_PREFIX;
use futures::StreamExt;
use http::StatusCode;
use hyper::Method;
use matchit::Params;
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
use rustfs_ecstore::disk::DiskAPI;
use rustfs_ecstore::disk::WalkDirOptions;
use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
use rustfs_ecstore::store::find_local_disk;
use rustfs_utils::net::bytes_stream;
use s3s::Body;
use s3s::S3Request;
use s3s::S3Response;
use s3s::S3Result;
use s3s::dto::StreamingBlob;
use s3s::s3_error;
use serde_urlencoded::from_bytes;
use tokio::io::AsyncWriteExt;
use tokio_util::io::ReaderStream;
use tracing::warn;
pub fn register_rpc_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
Method::GET,
format!("{}{}", RPC_PREFIX, "/read_file_stream").as_str(),
AdminOperation(&ReadFile {}),
)?;
r.insert(
Method::HEAD,
format!("{}{}", RPC_PREFIX, "/read_file_stream").as_str(),
AdminOperation(&ReadFile {}),
)?;
r.insert(
Method::PUT,
format!("{}{}", RPC_PREFIX, "/put_file_stream").as_str(),
AdminOperation(&PutFile {}),
)?;
r.insert(
Method::GET,
format!("{}{}", RPC_PREFIX, "/walk_dir").as_str(),
AdminOperation(&WalkDir {}),
)?;
r.insert(
Method::HEAD,
format!("{}{}", RPC_PREFIX, "/walk_dir").as_str(),
AdminOperation(&WalkDir {}),
)?;
Ok(())
}
// /rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}"
#[derive(Debug, Default, serde::Deserialize)]
pub struct ReadFileQuery {
disk: String,
volume: String,
path: String,
offset: usize,
length: usize,
}
pub struct ReadFile {}
#[async_trait::async_trait]
impl Operation for ReadFile {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
if req.method == Method::HEAD {
return Ok(S3Response::new((StatusCode::OK, Body::empty())));
}
let query = {
if let Some(query) = req.uri.query() {
let input: ReadFileQuery =
from_bytes(query.as_bytes()).map_err(|e| s3_error!(InvalidArgument, "get query failed1 {:?}", e))?;
input
} else {
ReadFileQuery::default()
}
};
let Some(disk) = find_local_disk(&query.disk).await else {
return Err(s3_error!(InvalidArgument, "disk not found"));
};
let file = disk
.read_file_stream(&query.volume, &query.path, query.offset, query.length)
.await
.map_err(|e| s3_error!(InternalError, "read file err {}", e))?;
Ok(S3Response::new((
StatusCode::OK,
Body::from(StreamingBlob::wrap(bytes_stream(
ReaderStream::with_capacity(file, DEFAULT_READ_BUFFER_SIZE),
query.length,
))),
)))
}
}
#[derive(Debug, Default, serde::Deserialize)]
pub struct WalkDirQuery {
disk: String,
}
pub struct WalkDir {}
#[async_trait::async_trait]
impl Operation for WalkDir {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
if req.method == Method::HEAD {
return Ok(S3Response::new((StatusCode::OK, Body::empty())));
}
let query = {
if let Some(query) = req.uri.query() {
let input: WalkDirQuery =
from_bytes(query.as_bytes()).map_err(|e| s3_error!(InvalidArgument, "get query failed1 {:?}", e))?;
input
} else {
WalkDirQuery::default()
}
};
let mut input = req.input;
let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await {
Ok(b) => b,
Err(e) => {
warn!("get body failed, e: {:?}", e);
return Err(s3_error!(InvalidRequest, "RPC request body too large or failed to read"));
}
};
// let body_bytes = decrypt_data(input_cred.secret_key.expose().as_bytes(), &body)
// .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, format!("decrypt_data err {}", e)))?;
let args: WalkDirOptions =
serde_json::from_slice(&body).map_err(|e| s3_error!(InternalError, "unmarshal body err {}", e))?;
let Some(disk) = find_local_disk(&query.disk).await else {
return Err(s3_error!(InvalidArgument, "disk not found"));
};
let (rd, mut wd) = tokio::io::duplex(DEFAULT_READ_BUFFER_SIZE);
tokio::spawn(async move {
if let Err(e) = disk.walk_dir(args, &mut wd).await {
warn!("walk dir err {}", e);
}
});
let body = Body::from(StreamingBlob::wrap(ReaderStream::with_capacity(rd, DEFAULT_READ_BUFFER_SIZE)));
Ok(S3Response::new((StatusCode::OK, body)))
}
}
// /rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}"
#[derive(Debug, Default, serde::Deserialize)]
pub struct PutFileQuery {
disk: String,
volume: String,
path: String,
append: bool,
size: i64,
}
pub struct PutFile {}
#[async_trait::async_trait]
impl Operation for PutFile {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let query = {
if let Some(query) = req.uri.query() {
let input: PutFileQuery =
from_bytes(query.as_bytes()).map_err(|e| s3_error!(InvalidArgument, "get query failed1 {:?}", e))?;
input
} else {
PutFileQuery::default()
}
};
let Some(disk) = find_local_disk(&query.disk).await else {
return Err(s3_error!(InvalidArgument, "disk not found"));
};
let mut file = if query.append {
disk.append_file(&query.volume, &query.path)
.await
.map_err(|e| s3_error!(InternalError, "append file err {}", e))?
} else {
disk.create_file("", &query.volume, &query.path, query.size)
.await
.map_err(|e| s3_error!(InternalError, "read file err {}", e))?
};
let mut body = req.input;
while let Some(item) = body.next().await {
let bytes = item.map_err(|e| s3_error!(InternalError, "body stream err {}", e))?;
let result = file.write_all(&bytes).await;
result.map_err(|e| s3_error!(InternalError, "write file err {}", e))?;
}
Ok(S3Response::new((StatusCode::OK, Body::empty())))
}
}
rustfs/src/main.rs
+14 -7
View File
@@ -530,14 +530,17 @@ async fn run(config: config::Config) -> Result<()> {
"Background services configuration: scanner={}, heal={}", enable_scanner, enable_heal
);
// Initialize heal manager and scanner based on environment variables
// Scanner depends on the heal channel/manager, so scanner implies heal.
if enable_heal || enable_scanner {
let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone()));
init_heal_manager(heal_storage, None).await?;
}
if enable_scanner {
init_data_scanner(ctx.clone(), store.clone()).await;
} else {
}
if !enable_heal && !enable_scanner {
info!(target: "rustfs::main::run","Both scanner and heal are disabled, skipping AHM service initialization");
}
@@ -623,20 +626,24 @@ async fn handle_shutdown(
let enable_scanner = get_env_bool_with_aliases(ENV_SCANNER_ENABLED, &[ENV_SCANNER_ENABLED_DEPRECATED], true);
let enable_heal = get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true);
// Stop background services based on what was enabled
if enable_scanner || enable_heal {
// Stop background services based on what was enabled.
if enable_scanner {
info!(
target: "rustfs::main::handle_shutdown",
"Stopping background services (data scanner and auto heal)..."
"Stopping background services (data scanner)..."
);
shutdown_background_services();
}
if enable_heal || enable_scanner {
info!(
target: "rustfs::main::handle_shutdown",
"Stopping AHM services..."
);
shutdown_ahm_services();
} else {
}
if !enable_scanner && !enable_heal {
info!(
target: "rustfs::main::handle_shutdown",
"Background services were disabled, skipping AHM shutdown"
rustfs/src/server/http.rs
+2
View File
@@ -24,6 +24,7 @@ use crate::server::{
layer::{AdminChunkedContentLengthCompatLayer, ConditionalCorsLayer, ObjectAttributesEtagFixLayer, RedirectLayer},
};
use crate::storage;
use crate::storage::rpc::InternodeRpcService;
use crate::storage::tonic_service::make_server;
use bytes::Bytes;
use http::{HeaderMap, Method, Request as HttpRequest, Response};
@@ -592,6 +593,7 @@ fn process_connection(
let http_service = SwiftService::new(true, None, s3_service);
#[cfg(not(feature = "swift"))]
let http_service = s3_service;
let http_service = InternodeRpcService::new(http_service);
let service = hybrid(http_service, rpc_service);
rustfs/src/storage/rpc/disk.rs
+132 -30
View File
@@ -13,6 +13,26 @@
// limitations under the License.
use super::*;
use serde::de::DeserializeOwned;
use std::io::Cursor;
fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str, value_name: &str) -> std::result::Result<T, DiskError> {
if !binary.is_empty() {
let mut deserializer = rmp_serde::Deserializer::new(Cursor::new(binary));
return T::deserialize(&mut deserializer)
.map_err(|err| DiskError::other(format!("decode {value_name} msgpack failed: {err}")));
}
serde_json::from_str(json).map_err(|err| DiskError::other(format!("decode {value_name} failed: {err}")))
}
fn encode_msgpack<T: serde::Serialize>(value: &T, value_name: &str) -> std::result::Result<Vec<u8>, DiskError> {
let mut serializer = rmp_serde::Serializer::new(Vec::new());
value
.serialize(&mut serializer)
.map_err(|err| DiskError::other(format!("encode {value_name} msgpack failed: {err}")))?;
Ok(serializer.into_inner())
}
impl NodeService {
pub(super) async fn handle_disk_info(&self, request: Request<DiskInfoRequest>) -> Result<Response<DiskInfoResponse>, Status> {
@@ -86,32 +106,44 @@ impl NodeService {
) -> Result<Response<ReadMultipleResponse>, Status> {
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await {
let read_multiple_req = match serde_json::from_str::<ReadMultipleReq>(&request.read_multiple_req) {
let read_multiple_req = match decode_msgpack_or_json::<ReadMultipleReq>(
&request.read_multiple_req_bin,
&request.read_multiple_req,
"ReadMultipleReq",
) {
Ok(read_multiple_req) => read_multiple_req,
Err(err) => {
return Ok(Response::new(ReadMultipleResponse {
success: false,
read_multiple_resps: Vec::new(),
read_multiple_resps_bin: Vec::new(),
error: Some(DiskError::other(format!("decode ReadMultipleReq failed: {err}")).into()),
}));
}
};
match disk.read_multiple(read_multiple_req).await {
Ok(read_multiple_resps) => {
let read_multiple_resps = read_multiple_resps
let read_multiple_resps: Vec<String> = read_multiple_resps
.into_iter()
.filter_map(|read_multiple_resp| serde_json::to_string(&read_multiple_resp).ok())
.collect();
let read_multiple_resps_bin = read_multiple_resps
.iter()
.filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(json_str).ok())
.filter_map(|resp| encode_msgpack(&resp, "ReadMultipleResp").ok())
.collect();
Ok(Response::new(ReadMultipleResponse {
success: true,
read_multiple_resps,
read_multiple_resps_bin,
error: None,
}))
}
Err(err) => Ok(Response::new(ReadMultipleResponse {
success: false,
read_multiple_resps: Vec::new(),
read_multiple_resps_bin: Vec::new(),
error: Some(err.into()),
})),
}
@@ -119,6 +151,7 @@ impl NodeService {
Ok(Response::new(ReadMultipleResponse {
success: false,
read_multiple_resps: Vec::new(),
read_multiple_resps_bin: Vec::new(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
}))
}
@@ -239,21 +272,34 @@ impl NodeService {
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await {
match disk.read_xl(&request.volume, &request.path, request.read_data).await {
Ok(raw_file_info) => match serde_json::to_string(&raw_file_info) {
Ok(raw_file_info) => Ok(Response::new(ReadXlResponse {
success: true,
raw_file_info,
error: None,
})),
Err(err) => Ok(Response::new(ReadXlResponse {
success: false,
raw_file_info: String::new(),
error: Some(DiskError::other(format!("encode data failed: {err}")).into()),
})),
},
Ok(raw_file_info) => {
let raw_file_info_json = serde_json::to_string(&raw_file_info);
let raw_file_info_bin = encode_msgpack(&raw_file_info, "RawFileInfo");
match (raw_file_info_json, raw_file_info_bin) {
(Ok(raw_file_info), Ok(raw_file_info_bin)) => Ok(Response::new(ReadXlResponse {
success: true,
raw_file_info,
raw_file_info_bin,
error: None,
})),
(Err(err), _) => Ok(Response::new(ReadXlResponse {
success: false,
raw_file_info: String::new(),
raw_file_info_bin: Vec::new(),
error: Some(DiskError::other(format!("encode data failed: {err}")).into()),
})),
(_, Err(err)) => Ok(Response::new(ReadXlResponse {
success: false,
raw_file_info: String::new(),
raw_file_info_bin: Vec::new(),
error: Some(DiskError::other(format!("encode data failed: {err}")).into()),
})),
}
}
Err(err) => Ok(Response::new(ReadXlResponse {
success: false,
raw_file_info: String::new(),
raw_file_info_bin: Vec::new(),
error: Some(err.into()),
})),
}
@@ -261,6 +307,7 @@ impl NodeService {
Ok(Response::new(ReadXlResponse {
success: false,
raw_file_info: String::new(),
raw_file_info_bin: Vec::new(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
}))
}
@@ -272,12 +319,13 @@ impl NodeService {
) -> Result<Response<ReadVersionResponse>, Status> {
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await {
let opts = match serde_json::from_str::<ReadOptions>(&request.opts) {
let opts = match decode_msgpack_or_json::<ReadOptions>(&request.opts_bin, &request.opts, "ReadOptions") {
Ok(options) => options,
Err(err) => {
return Ok(Response::new(ReadVersionResponse {
success: false,
file_info: String::new(),
file_info_bin: Vec::new(),
error: Some(DiskError::other(format!("decode ReadOptions failed: {err}")).into()),
}));
}
@@ -286,21 +334,34 @@ impl NodeService {
.read_version("", &request.volume, &request.path, &request.version_id, &opts)
.await
{
Ok(file_info) => match serde_json::to_string(&file_info) {
Ok(file_info) => Ok(Response::new(ReadVersionResponse {
success: true,
file_info,
error: None,
})),
Err(err) => Ok(Response::new(ReadVersionResponse {
success: false,
file_info: String::new(),
error: Some(DiskError::other(format!("encode data failed: {err}")).into()),
})),
},
Ok(file_info) => {
let file_info_json = serde_json::to_string(&file_info);
let file_info_bin = encode_msgpack(&file_info, "FileInfo");
match (file_info_json, file_info_bin) {
(Ok(file_info), Ok(file_info_bin)) => Ok(Response::new(ReadVersionResponse {
success: true,
file_info,
file_info_bin,
error: None,
})),
(Err(err), _) => Ok(Response::new(ReadVersionResponse {
success: false,
file_info: String::new(),
file_info_bin: Vec::new(),
error: Some(DiskError::other(format!("encode data failed: {err}")).into()),
})),
(_, Err(err)) => Ok(Response::new(ReadVersionResponse {
success: false,
file_info: String::new(),
file_info_bin: Vec::new(),
error: Some(DiskError::other(format!("encode data failed: {err}")).into()),
})),
}
}
Err(err) => Ok(Response::new(ReadVersionResponse {
success: false,
file_info: String::new(),
file_info_bin: Vec::new(),
error: Some(err.into()),
})),
}
@@ -308,6 +369,7 @@ impl NodeService {
Ok(Response::new(ReadVersionResponse {
success: false,
file_info: String::new(),
file_info_bin: Vec::new(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
}))
}
@@ -319,7 +381,7 @@ impl NodeService {
) -> Result<Response<WriteMetadataResponse>, Status> {
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await {
let file_info = match serde_json::from_str::<FileInfo>(&request.file_info) {
let file_info = match decode_msgpack_or_json::<FileInfo>(&request.file_info_bin, &request.file_info, "FileInfo") {
Ok(file_info) => file_info,
Err(err) => {
return Ok(Response::new(WriteMetadataResponse {
@@ -352,7 +414,7 @@ impl NodeService {
) -> Result<Response<UpdateMetadataResponse>, Status> {
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await {
let file_info = match serde_json::from_str::<FileInfo>(&request.file_info) {
let file_info = match decode_msgpack_or_json::<FileInfo>(&request.file_info_bin, &request.file_info, "FileInfo") {
Ok(file_info) => file_info,
Err(err) => {
return Ok(Response::new(UpdateMetadataResponse {
@@ -361,7 +423,8 @@ impl NodeService {
}));
}
};
let opts = match serde_json::from_str::<UpdateMetadataOpts>(&request.opts) {
let opts = match decode_msgpack_or_json::<UpdateMetadataOpts>(&request.opts_bin, &request.opts, "UpdateMetadataOpts")
{
Ok(opts) => opts,
Err(err) => {
return Ok(Response::new(UpdateMetadataResponse {
@@ -910,3 +973,42 @@ impl NodeService {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde::{Deserialize, Serialize};
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
struct SamplePayload {
name: String,
count: u32,
}
#[test]
fn decode_msgpack_or_json_prefers_binary_payload() {
let payload = SamplePayload {
name: "rustfs".to_string(),
count: 3,
};
let binary = encode_msgpack(&payload, "SamplePayload").unwrap();
let decoded =
decode_msgpack_or_json::<SamplePayload>(&binary, r#"{"name":"ignored","count":1}"#, "SamplePayload").unwrap();
assert_eq!(decoded, payload);
}
#[test]
fn decode_msgpack_or_json_falls_back_to_json() {
let decoded = decode_msgpack_or_json::<SamplePayload>(&[], r#"{"name":"compat","count":7}"#, "SamplePayload").unwrap();
assert_eq!(
decoded,
SamplePayload {
name: "compat".to_string(),
count: 7,
}
);
}
}
rustfs/src/storage/rpc/http_service.rs
+404
View File
@@ -0,0 +1,404 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::server::RPC_PREFIX;
use bytes::{Bytes, BytesMut};
use futures_util::TryStreamExt;
use http::{HeaderMap, Method, Request, Response, StatusCode, Uri};
use http_body_util::{BodyExt, Limited};
use hyper::body::Incoming;
use rustfs_common::internode_metrics::global_internode_metrics;
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
use rustfs_ecstore::disk::{DiskAPI, WalkDirOptions};
use rustfs_ecstore::rpc::verify_rpc_signature;
use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
use rustfs_ecstore::store::find_local_disk_by_ref;
use rustfs_utils::net::bytes_stream;
use s3s::Body;
use s3s::dto::StreamingBlob;
use serde::de::DeserializeOwned;
use serde_urlencoded::from_bytes;
use std::future::Future;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{self, AsyncWriteExt};
use tokio_util::io::ReaderStream;
use tower::Service;
use tracing::warn;
type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
type RpcErrorResponse = Box<Response<Body>>;
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream";
const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir";
#[derive(Clone)]
pub struct InternodeRpcService<S> {
inner: S,
}
impl<S> InternodeRpcService<S> {
pub fn new(inner: S) -> Self {
Self { inner }
}
}
#[derive(Debug, Default, serde::Deserialize)]
struct ReadFileQuery {
disk: String,
volume: String,
path: String,
offset: usize,
length: usize,
}
#[derive(Debug, Default, serde::Deserialize)]
struct WalkDirQuery {
disk: String,
}
#[derive(Debug, Default, serde::Deserialize)]
struct PutFileQuery {
disk: String,
volume: String,
path: String,
append: bool,
size: i64,
}
impl<S> Service<Request<Incoming>> for InternodeRpcService<S>
where
S: Service<Request<Incoming>, Response = Response<Body>> + Clone + Send + 'static,
S::Future: Send + 'static,
S::Error: Into<BoxError> + Send + 'static,
{
type Response = Response<Body>;
type Error = S::Error;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, req: Request<Incoming>) -> Self::Future {
if !is_internode_rpc_path(req.uri().path()) {
let mut inner = self.inner.clone();
return Box::pin(async move { inner.call(req).await });
}
Box::pin(async move { Ok(handle_internode_rpc(req).await) })
}
}
fn is_internode_rpc_path(path: &str) -> bool {
path.starts_with(RPC_PREFIX)
}
async fn handle_internode_rpc(req: Request<Incoming>) -> Response<Body> {
if let Err(response) = verify_internode_rpc_signature(req.uri(), req.method(), req.headers()) {
global_internode_metrics().record_error();
return *response;
}
let method = req.method().clone();
let path = req.uri().path();
let response = match (method, path) {
(Method::GET, READ_FILE_STREAM_PATH) | (Method::HEAD, READ_FILE_STREAM_PATH) => handle_read_file(req).await,
(Method::GET, WALK_DIR_PATH) | (Method::HEAD, WALK_DIR_PATH) => handle_walk_dir(req).await,
(Method::PUT, PUT_FILE_STREAM_PATH) => handle_put_file(req).await,
_ => response_with_status(StatusCode::NOT_FOUND, "internode rpc route not found"),
};
if !response.status().is_success() {
global_internode_metrics().record_error();
}
response
}
fn verify_internode_rpc_signature(uri: &Uri, method: &Method, headers: &HeaderMap) -> Result<(), RpcErrorResponse> {
if method == Method::HEAD {
return Ok(());
}
verify_rpc_signature(&uri.to_string(), method, headers).map_err(|e| {
Box::new(response_with_status(
StatusCode::FORBIDDEN,
format!("rpc signature verification failed: {e}"),
))
})
}
async fn handle_read_file(req: Request<Incoming>) -> Response<Body> {
if req.method() == Method::HEAD {
return empty_ok();
}
let query = match parse_query::<ReadFileQuery>(&req) {
Ok(query) => query,
Err(response) => return *response,
};
let Some(disk) = find_local_disk_by_ref(&query.disk).await else {
return response_with_status(StatusCode::BAD_REQUEST, "disk not found");
};
let file = match disk
.read_file_stream(&query.volume, &query.path, query.offset, query.length)
.await
{
Ok(file) => file,
Err(e) => return response_with_status(StatusCode::INTERNAL_SERVER_ERROR, format!("read file err {e}")),
};
global_internode_metrics().record_incoming_request();
let stream = read_file_body_stream(file, query.length);
Response::builder()
.status(StatusCode::OK)
.body(Body::from(StreamingBlob::wrap(stream)))
.expect("failed to build read file stream response")
}
fn read_file_body_stream<R>(reader: R, length: usize) -> Pin<Box<dyn futures::Stream<Item = io::Result<Bytes>> + Send + Sync>>
where
R: tokio::io::AsyncRead + Unpin + Send + Sync + 'static,
{
let metrics = global_internode_metrics().clone();
let stream = ReaderStream::with_capacity(reader, DEFAULT_READ_BUFFER_SIZE).map_ok(move |bytes| {
metrics.record_sent_bytes(bytes.len());
bytes
});
if length == 0 {
Box::pin(stream)
} else {
Box::pin(bytes_stream(stream, length))
}
}
async fn handle_walk_dir(req: Request<Incoming>) -> Response<Body> {
if req.method() == Method::HEAD {
return empty_ok();
}
let query = match parse_query::<WalkDirQuery>(&req) {
Ok(query) => query,
Err(response) => return *response,
};
let Some(disk) = find_local_disk_by_ref(&query.disk).await else {
return response_with_status(StatusCode::BAD_REQUEST, "disk not found");
};
let body = match Limited::new(req.into_body(), MAX_ADMIN_REQUEST_BODY_SIZE).collect().await {
Ok(body) => body.to_bytes(),
Err(e) => return response_with_status(StatusCode::PAYLOAD_TOO_LARGE, format!("read body err {e}")),
};
let args: WalkDirOptions = match serde_json::from_slice(&body) {
Ok(args) => args,
Err(e) => return response_with_status(StatusCode::BAD_REQUEST, format!("unmarshal body err {e}")),
};
let (rd, mut wd) = tokio::io::duplex(DEFAULT_READ_BUFFER_SIZE);
tokio::spawn(async move {
if let Err(e) = disk.walk_dir(args, &mut wd).await {
warn!("walk dir err {}", e);
}
});
global_internode_metrics().record_incoming_request();
let metrics = global_internode_metrics().clone();
let stream = ReaderStream::with_capacity(rd, DEFAULT_READ_BUFFER_SIZE).map_ok(move |bytes| {
metrics.record_sent_bytes(bytes.len());
bytes
});
Response::builder()
.status(StatusCode::OK)
.body(Body::from(StreamingBlob::wrap(stream)))
.expect("failed to build walk dir response")
}
async fn handle_put_file(req: Request<Incoming>) -> Response<Body> {
let query = match parse_query::<PutFileQuery>(&req) {
Ok(query) => query,
Err(response) => return *response,
};
let Some(disk) = find_local_disk_by_ref(&query.disk).await else {
return response_with_status(StatusCode::BAD_REQUEST, "disk not found");
};
let mut file = if query.append {
match disk.append_file(&query.volume, &query.path).await {
Ok(file) => file,
Err(e) => return response_with_status(StatusCode::INTERNAL_SERVER_ERROR, format!("append file err {e}")),
}
} else {
match disk.create_file("", &query.volume, &query.path, query.size).await {
Ok(file) => file,
Err(e) => return response_with_status(StatusCode::INTERNAL_SERVER_ERROR, format!("create file err {e}")),
}
};
let copied = match write_body_chunks_to_writer(req.into_body().into_data_stream(), &mut file).await {
Ok(copied) => copied,
Err(e) => return response_with_status(StatusCode::INTERNAL_SERVER_ERROR, format!("write file err {e}")),
};
global_internode_metrics().record_incoming_request();
global_internode_metrics().record_recv_bytes(copied as usize);
if let Err(e) = file.flush().await {
return response_with_status(StatusCode::INTERNAL_SERVER_ERROR, format!("write file err {e}"));
}
empty_ok()
}
async fn write_body_chunks_to_writer<S, E, W>(body: S, writer: &mut W) -> io::Result<u64>
where
S: futures::TryStream<Ok = Bytes, Error = E> + Unpin,
E: Into<BoxError>,
W: tokio::io::AsyncWrite + Unpin,
{
let mut body = body;
let mut copied = 0_u64;
let mut pending = BytesMut::with_capacity(DEFAULT_READ_BUFFER_SIZE);
while let Some(bytes) = body.try_next().await.map_err(io::Error::other)? {
copied += bytes.len() as u64;
pending.extend_from_slice(&bytes);
if pending.len() >= DEFAULT_READ_BUFFER_SIZE {
writer.write_all(&pending).await?;
pending.clear();
}
}
if !pending.is_empty() {
writer.write_all(&pending).await?;
}
Ok(copied)
}
fn parse_query<T>(req: &Request<Incoming>) -> Result<T, RpcErrorResponse>
where
T: DeserializeOwned + Default,
{
match req.uri().query() {
Some(query) => from_bytes(query.as_bytes())
.map_err(|e| Box::new(response_with_status(StatusCode::BAD_REQUEST, format!("get query failed {e}")))),
None => Ok(T::default()),
}
}
fn empty_ok() -> Response<Body> {
Response::builder()
.status(StatusCode::OK)
.body(Body::empty())
.expect("failed to build empty ok response")
}
fn response_with_status(status: StatusCode, message: impl Into<String>) -> Response<Body> {
Response::builder()
.status(status)
.header(http::header::CONTENT_TYPE, "text/plain; charset=utf-8")
.body(Body::from(Bytes::from(message.into())))
.expect("failed to build rpc error response")
}
#[cfg(test)]
mod tests {
use super::*;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio_stream::StreamExt;
use tokio_stream::iter;
#[test]
fn internode_rpc_path_matches_rpc_prefix() {
assert!(is_internode_rpc_path("/rustfs/rpc/read_file_stream"));
assert!(is_internode_rpc_path("/rustfs/rpc/walk_dir"));
assert!(!is_internode_rpc_path("/rustfs/admin/v3/info"));
}
#[test]
fn rpc_head_signature_verification_is_skipped() {
let uri: Uri = READ_FILE_STREAM_PATH.parse().expect("uri");
let headers = HeaderMap::new();
assert!(verify_internode_rpc_signature(&uri, &Method::HEAD, &headers).is_ok());
}
#[test]
fn rpc_get_request_requires_signature() {
let uri: Uri = READ_FILE_STREAM_PATH.parse().expect("uri");
let headers = HeaderMap::new();
let response = verify_internode_rpc_signature(&uri, &Method::GET, &headers).expect_err("response");
assert_eq!(response.status(), StatusCode::FORBIDDEN);
}
#[tokio::test]
async fn write_body_chunks_to_writer_streams_all_chunks() {
let (mut reader, mut writer) = tokio::io::duplex(64);
let body = iter(vec![
Ok::<Bytes, io::Error>(Bytes::from_static(b"hello ")),
Ok(Bytes::from_static(b"world")),
]);
let copied = write_body_chunks_to_writer(body, &mut writer).await.expect("copy succeeds");
drop(writer);
let mut out = Vec::new();
reader.read_to_end(&mut out).await.expect("read succeeds");
assert_eq!(copied, 11);
assert_eq!(out, b"hello world");
}
#[tokio::test]
async fn read_file_body_stream_keeps_full_stream_when_length_is_zero() {
let (reader, mut writer) = tokio::io::duplex(64);
tokio::spawn(async move {
writer.write_all(b"hello world").await.expect("write succeeds");
});
let mut stream = read_file_body_stream(reader, 0);
let mut out = Vec::new();
while let Some(chunk) = stream.next().await {
out.extend_from_slice(&chunk.expect("chunk succeeds"));
}
assert_eq!(out, b"hello world");
}
#[tokio::test]
async fn read_file_body_stream_truncates_to_requested_length() {
let (reader, mut writer) = tokio::io::duplex(64);
tokio::spawn(async move {
writer.write_all(b"hello world").await.expect("write succeeds");
});
let mut stream = read_file_body_stream(reader, 5);
let mut out = Vec::new();
while let Some(chunk) = stream.next().await {
out.extend_from_slice(&chunk.expect("chunk succeeds"));
}
assert_eq!(out, b"hello");
}
}
rustfs/src/storage/rpc/mod.rs
+2
View File
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod http_service;
pub mod node_service;
pub use http_service::InternodeRpcService;
pub use node_service::{NodeService, make_server};
rustfs/src/storage/rpc/node_service.rs
+18 -6
View File
@@ -21,14 +21,14 @@ use rustfs_ecstore::{
admin_server_info::get_local_server_property,
bucket::{metadata::load_bucket_metadata, metadata_sys},
disk::{
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadOptions, UpdateMetadataOpts,
error::DiskError,
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
UpdateMetadataOpts, error::DiskError,
},
get_global_lock_client,
metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics},
new_object_layer_fn,
rpc::{LocalPeerS3Client, PeerS3Client},
store::{all_local_disk_path, find_local_disk},
store::{all_local_disk_path, find_local_disk_by_ref},
store_api::{BucketOptions, DeleteBucketOptions, MakeBucketOptions, StorageAPI},
};
use rustfs_filemeta::{FileInfo, MetacacheReader};
@@ -74,8 +74,8 @@ pub fn make_server() -> NodeService {
}
impl NodeService {
async fn find_disk(&self, disk_path: &String) -> Option<DiskStore> {
find_local_disk(disk_path).await
async fn find_disk(&self, disk_path: &str) -> Option<DiskStore> {
find_local_disk_by_ref(disk_path).await
}
async fn all_disk(&self) -> Vec<String> {
@@ -1353,6 +1353,8 @@ mod tests {
path: "test-path".to_string(),
file_info: "{}".to_string(),
opts: "{}".to_string(),
file_info_bin: Vec::new(),
opts_bin: Vec::new(),
});
let response = service.update_metadata(request).await;
@@ -1373,6 +1375,8 @@ mod tests {
path: "test-path".to_string(),
file_info: "invalid json".to_string(),
opts: "{}".to_string(),
file_info_bin: Vec::new(),
opts_bin: Vec::new(),
});
let response = service.update_metadata(request).await;
@@ -1393,6 +1397,8 @@ mod tests {
path: "test-path".to_string(),
file_info: "{}".to_string(),
opts: "invalid json".to_string(),
file_info_bin: Vec::new(),
opts_bin: Vec::new(),
});
let response = service.update_metadata(request).await;
@@ -1412,6 +1418,7 @@ mod tests {
volume: "test-volume".to_string(),
path: "test-path".to_string(),
file_info: "{}".to_string(),
file_info_bin: Vec::new(),
});
let response = service.write_metadata(request).await;
@@ -1431,6 +1438,7 @@ mod tests {
volume: "test-volume".to_string(),
path: "test-path".to_string(),
file_info: "invalid json".to_string(),
file_info_bin: Vec::new(),
});
let response = service.write_metadata(request).await;
@@ -1451,6 +1459,7 @@ mod tests {
path: "test-path".to_string(),
version_id: "version1".to_string(),
opts: "{}".to_string(),
opts_bin: Vec::new(),
});
let response = service.read_version(request).await;
@@ -1472,6 +1481,7 @@ mod tests {
path: "test-path".to_string(),
version_id: "version1".to_string(),
opts: "invalid json".to_string(),
opts_bin: Vec::new(),
});
let response = service.read_version(request).await;
@@ -1629,6 +1639,7 @@ mod tests {
let request = Request::new(ReadMultipleRequest {
disk: "invalid-disk-path".to_string(),
read_multiple_req: "{}".to_string(),
read_multiple_req_bin: Vec::new(),
});
let response = service.read_multiple(request).await;
@@ -1647,6 +1658,7 @@ mod tests {
let request = Request::new(ReadMultipleRequest {
disk: "invalid-disk-path".to_string(),
read_multiple_req: "invalid json".to_string(),
read_multiple_req_bin: Vec::new(),
});
let response = service.read_multiple(request).await;
@@ -2195,7 +2207,7 @@ mod tests {
#[tokio::test]
async fn test_find_disk_method() {
let service = create_test_node_service();
let disk = service.find_disk(&"non-existent-disk".to_string()).await;
let disk = service.find_disk("non-existent-disk").await;
// Should return None for non-existent disk
assert!(disk.is_none());
}