gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-19 02:56:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(internode): harden data transport baseline

Browse Source
This commit is contained in:
Henry Guo
2026-05-19 10:35:49 +00:00
parent 3b31f7e8fa
commit 5b8d6b8aa4
4 changed files with 273 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/rpc/internode_data_transport.rs
+194 -44
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::disk::error::Result;
use crate::disk::error::{Error, Result};
use crate::disk::{FileReader, FileWriter};
use crate::rpc::build_auth_headers;
use async_trait::async_trait;
@@ -20,10 +20,38 @@ use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
use rustfs_rio::{HttpReader, HttpWriter};
use std::sync::Arc;
use std::time::Duration;
use tracing::warn;
pub const ENV_RUSTFS_INTERNODE_DATA_TRANSPORT: &str = "RUSTFS_INTERNODE_DATA_TRANSPORT";
pub const INTERNODE_DATA_TRANSPORT_TCP_HTTP: &str = "tcp-http";
pub const INTERNODE_DATA_TRANSPORT_TCP: &str = "tcp";
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream";
const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir";
const CONTENT_TYPE_JSON: &str = "application/json";
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub struct InternodeDataTransportCapabilities {
pub stream_read: bool,
pub stream_write: bool,
pub walk_dir: bool,
pub registered_memory: bool,
pub scatter_gather: bool,
pub zero_copy_receive: bool,
}
impl InternodeDataTransportCapabilities {
pub const fn tcp_http() -> Self {
Self {
stream_read: true,
stream_write: true,
walk_dir: true,
registered_memory: false,
scatter_gather: false,
zero_copy_receive: false,
}
}
}
#[derive(Debug, Clone)]
pub struct ReadStreamRequest {
@@ -59,6 +87,7 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter>;
async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader>;
fn name(&self) -> &'static str;
fn capabilities(&self) -> InternodeDataTransportCapabilities;
}
#[derive(Debug, Default)]
@@ -67,41 +96,22 @@ pub struct TcpHttpInternodeDataTransport;
#[async_trait]
impl InternodeDataTransport for TcpHttpInternodeDataTransport {
async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader> {
let url = format!(
"{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
request.endpoint,
urlencoding::encode(&request.disk),
urlencoding::encode(&request.volume),
urlencoding::encode(&request.path),
request.offset,
request.length
);
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
let url = build_read_file_stream_url(&request);
let mut headers = json_headers();
build_auth_headers(&url, &Method::GET, &mut headers)?;
Ok(Box::new(HttpReader::new(url, Method::GET, headers, None).await?))
}
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
let url = format!(
"{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
request.endpoint,
urlencoding::encode(&request.disk),
urlencoding::encode(&request.volume),
urlencoding::encode(&request.path),
request.append,
request.size
);
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
let url = build_put_file_stream_url(&request);
let mut headers = json_headers();
build_auth_headers(&url, &Method::PUT, &mut headers)?;
Ok(Box::new(HttpWriter::new(url, Method::PUT, headers).await?))
}
async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader> {
let url = format!("{}/rustfs/rpc/walk_dir?disk={}", request.endpoint, urlencoding::encode(&request.disk));
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
let url = build_walk_dir_url(&request);
let mut headers = json_headers();
build_auth_headers(&url, &Method::GET, &mut headers)?;
Ok(Box::new(
HttpReader::new_with_stall_timeout(url, Method::GET, headers, Some(request.body), request.stall_timeout).await?,
@@ -111,23 +121,163 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
fn name(&self) -> &'static str {
INTERNODE_DATA_TRANSPORT_TCP_HTTP
}
}
pub fn build_internode_data_transport_from_env() -> Arc<dyn InternodeDataTransport> {
match std::env::var(ENV_RUSTFS_INTERNODE_DATA_TRANSPORT) {
Ok(transport) => {
if transport.eq_ignore_ascii_case(INTERNODE_DATA_TRANSPORT_TCP_HTTP) {
Arc::new(TcpHttpInternodeDataTransport)
} else {
warn!(
env = ENV_RUSTFS_INTERNODE_DATA_TRANSPORT,
requested = %transport,
fallback = INTERNODE_DATA_TRANSPORT_TCP_HTTP,
"unknown internode data transport, using default backend"
);
Arc::new(TcpHttpInternodeDataTransport)
}
}
Err(_) => Arc::new(TcpHttpInternodeDataTransport),
fn capabilities(&self) -> InternodeDataTransportCapabilities {
InternodeDataTransportCapabilities::tcp_http()
}
}
fn build_read_file_stream_url(request: &ReadStreamRequest) -> String {
format!(
"{}{}?disk={}&volume={}&path={}&offset={}&length={}",
request.endpoint,
READ_FILE_STREAM_PATH,
urlencoding::encode(&request.disk),
urlencoding::encode(&request.volume),
urlencoding::encode(&request.path),
request.offset,
request.length
)
}
fn build_put_file_stream_url(request: &WriteStreamRequest) -> String {
format!(
"{}{}?disk={}&volume={}&path={}&append={}&size={}",
request.endpoint,
PUT_FILE_STREAM_PATH,
urlencoding::encode(&request.disk),
urlencoding::encode(&request.volume),
urlencoding::encode(&request.path),
request.append,
request.size
)
}
fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String {
format!("{}{}?disk={}", request.endpoint, WALK_DIR_PATH, urlencoding::encode(&request.disk))
}
fn json_headers() -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static(CONTENT_TYPE_JSON));
headers
}
pub fn build_internode_data_transport(configured_transport: Option<&str>) -> Result<Arc<dyn InternodeDataTransport>> {
match configured_transport.map(str::trim).filter(|transport| !transport.is_empty()) {
None => Ok(Arc::new(TcpHttpInternodeDataTransport)),
Some(transport)
if transport.eq_ignore_ascii_case(INTERNODE_DATA_TRANSPORT_TCP_HTTP)
|| transport.eq_ignore_ascii_case(INTERNODE_DATA_TRANSPORT_TCP) =>
{
Ok(Arc::new(TcpHttpInternodeDataTransport))
}
Some(transport) => Err(Error::other(format!(
"unsupported internode data transport {transport:?}; supported values: {INTERNODE_DATA_TRANSPORT_TCP_HTTP}, {INTERNODE_DATA_TRANSPORT_TCP}"
))),
}
}
pub fn build_internode_data_transport_from_env() -> Result<Arc<dyn InternodeDataTransport>> {
build_internode_data_transport(std::env::var(ENV_RUSTFS_INTERNODE_DATA_TRANSPORT).ok().as_deref())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tcp_http_capabilities_are_behavior_preserving() {
let transport = TcpHttpInternodeDataTransport;
assert_eq!(transport.name(), INTERNODE_DATA_TRANSPORT_TCP_HTTP);
assert_eq!(
transport.capabilities(),
InternodeDataTransportCapabilities {
stream_read: true,
stream_write: true,
walk_dir: true,
registered_memory: false,
scatter_gather: false,
zero_copy_receive: false,
}
);
}
#[test]
fn read_file_stream_url_encodes_query_values() {
let url = build_read_file_stream_url(&ReadStreamRequest {
endpoint: "http://node1:9000".to_string(),
disk: "http://node1:9000/data/rustfs0".to_string(),
volume: ".rustfs.sys".to_string(),
path: "pool.bin/../part.1".to_string(),
offset: 7,
length: 11,
});
assert_eq!(
url,
"http://node1:9000/rustfs/rpc/read_file_stream?disk=http%3A%2F%2Fnode1%3A9000%2Fdata%2Frustfs0&volume=.rustfs.sys&path=pool.bin%2F..%2Fpart.1&offset=7&length=11"
);
}
#[test]
fn put_file_stream_url_encodes_query_values() {
let url = build_put_file_stream_url(&WriteStreamRequest {
endpoint: "http://node1:9000".to_string(),
disk: "http://node1:9000/data/rustfs0".to_string(),
volume: "bucket".to_string(),
path: "object/part.1".to_string(),
append: false,
size: 4096,
});
assert_eq!(
url,
"http://node1:9000/rustfs/rpc/put_file_stream?disk=http%3A%2F%2Fnode1%3A9000%2Fdata%2Frustfs0&volume=bucket&path=object%2Fpart.1&append=false&size=4096"
);
}
#[test]
fn walk_dir_url_encodes_disk_ref() {
let url = build_walk_dir_url(&WalkDirStreamRequest {
endpoint: "http://node1:9000".to_string(),
disk: "http://node1:9000/data/rustfs0".to_string(),
body: Vec::new(),
stall_timeout: None,
});
assert_eq!(
url,
"http://node1:9000/rustfs/rpc/walk_dir?disk=http%3A%2F%2Fnode1%3A9000%2Fdata%2Frustfs0"
);
}
#[test]
fn transport_config_defaults_to_tcp_http() {
let transport = build_internode_data_transport(None).unwrap();
assert_eq!(transport.name(), INTERNODE_DATA_TRANSPORT_TCP_HTTP);
}
#[test]
fn transport_config_accepts_tcp_aliases() {
for configured in [
INTERNODE_DATA_TRANSPORT_TCP_HTTP,
INTERNODE_DATA_TRANSPORT_TCP,
"TCP-HTTP",
"TCP",
] {
let transport = build_internode_data_transport(Some(configured)).unwrap();
assert_eq!(transport.name(), INTERNODE_DATA_TRANSPORT_TCP_HTTP);
}
}
#[test]
fn transport_config_rejects_unknown_backend() {
let err = build_internode_data_transport(Some("rdma")).expect_err("unknown backend should fail closed");
assert!(err.to_string().contains("unsupported internode data transport"));
}
}
crates/ecstore/src/rpc/remote_disk.rs
+1 -1
View File
@@ -129,7 +129,7 @@ impl RemoteDisk {
health_check: opt.health_check && env_health_check,
health: Arc::new(DiskHealthTracker::new()),
cancel_token: CancellationToken::new(),
data_transport: build_internode_data_transport_from_env(),
data_transport: build_internode_data_transport_from_env()?,
};
record_drive_runtime_state(ep, RuntimeDriveHealthState::Online);