From b42766f1d3b856a9a201f16735b041446cd3fd78 Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Fri, 22 May 2026 10:56:37 +0800 Subject: [PATCH] feat(internode): define transport capabilities (#3047) Co-authored-by: Henry Guo Co-authored-by: houseme --- INTERNODE_TRANSPORT_CAPABILITIES.md | 57 +++++++++++++++++ .../src/rpc/internode_data_transport.rs | 61 +++++++++++++------ 2 files changed, 100 insertions(+), 18 deletions(-) create mode 100644 INTERNODE_TRANSPORT_CAPABILITIES.md diff --git a/INTERNODE_TRANSPORT_CAPABILITIES.md b/INTERNODE_TRANSPORT_CAPABILITIES.md new file mode 100644 index 000000000..53a86ee18 --- /dev/null +++ b/INTERNODE_TRANSPORT_CAPABILITIES.md @@ -0,0 +1,57 @@ +# Internode Transport Capabilities + +Status: design note for backend-neutral capability reporting. This document +does not add an RDMA backend and does not require RDMA hardware or crates. + +## Purpose + +`InternodeDataTransportCapabilities` describes what a backend can honestly do +for RustFS internode data-plane transfers. The fields are intentionally neutral: +they can describe the current TCP/HTTP backend and a future high-speed backend +without naming a vendor stack or transport implementation. + +The capability report is descriptive. It does not select a backend, negotiate +with peers, or weaken object correctness semantics. + +## Capability Fields + +| Field | Meaning | +| --- | --- | +| `streaming_read` | The backend can open a remote disk reader for `read_file_stream`. | +| `streaming_write` | The backend can open a remote disk writer for `create_file` or `append_file`. | +| `streaming_walk_dir` | The backend can stream `walk_dir` responses. | +| `zero_copy_candidate` | The backend has an API shape that could avoid an extra user-space payload copy. This is not a promise that every transfer is zero-copy. | +| `registered_memory_required` | The backend requires pinned or registered buffers before payload transfer. | +| `ordered_delivery` | Bytes for each opened transfer are delivered in order. | +| `max_transfer_size` | Optional RustFS-level cap for a single transfer. `None` means no additional cap beyond the backend/protocol/runtime limits. | +| `fallback_supported` | The backend can participate in the behavior-preserving TCP fallback path. | + +## TCP/HTTP Backend + +The default TCP/HTTP backend reports only capabilities it actually provides: + +| Field | TCP/HTTP value | Reason | +| --- | --- | --- | +| `streaming_read` | `true` | `HttpReader` streams `/rustfs/rpc/read_file_stream` responses. | +| `streaming_write` | `true` | `HttpWriter` streams `/rustfs/rpc/put_file_stream` request bodies. | +| `streaming_walk_dir` | `true` | `HttpReader` streams `/rustfs/rpc/walk_dir` responses. | +| `zero_copy_candidate` | `false` | The current path exposes `AsyncRead`/`AsyncWrite` and HTTP body chunks; it copies through normal user-space buffers. | +| `registered_memory_required` | `false` | TCP/HTTP does not require RustFS-managed registered memory. | +| `ordered_delivery` | `true` | Each HTTP request body or response body is consumed as an ordered byte stream. | +| `max_transfer_size` | `None` | RustFS does not impose an extra per-transfer cap at the capability layer. | +| `fallback_supported` | `true` | TCP/HTTP is the behavior-preserving default and fallback path. | + +## Future Backend Fit + +A future high-speed backend can be described without changing the meaning of the +existing TCP report: + +| Capability shape | Interpretation | +| --- | --- | +| `zero_copy_candidate=true`, `registered_memory_required=true` | The backend can only use its fast path with buffers that satisfy registration or pinning rules. | +| `zero_copy_candidate=true`, `registered_memory_required=false` | The backend may expose owned chunks or another lower-copy path without requiring caller-registered memory. | +| `max_transfer_size=Some(n)` | The backend has a RustFS-visible transfer size ceiling and callers must split larger transfers or use fallback behavior. | +| `ordered_delivery=false` | The backend cannot be used behind the current stream API without an ordering or reassembly layer. | + +Unsupported or mismatched capabilities must not silently change quorum, +integrity verification, retry, or timeout semantics. diff --git a/crates/ecstore/src/rpc/internode_data_transport.rs b/crates/ecstore/src/rpc/internode_data_transport.rs index ff154101a..768884630 100644 --- a/crates/ecstore/src/rpc/internode_data_transport.rs +++ b/crates/ecstore/src/rpc/internode_data_transport.rs @@ -41,23 +41,35 @@ fn unsupported_transport_message(transport: &str) -> String { #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub struct InternodeDataTransportCapabilities { - pub stream_read: bool, - pub stream_write: bool, - pub walk_dir: bool, - pub registered_memory: bool, - pub scatter_gather: bool, - pub zero_copy_receive: bool, + /// Backend can open a streaming remote disk reader. + pub streaming_read: bool, + /// Backend can open a streaming remote disk writer. + pub streaming_write: bool, + /// Backend can stream walk-dir responses. + pub streaming_walk_dir: bool, + /// Backend has an API shape that could avoid an extra user-space copy. + pub zero_copy_candidate: bool, + /// Backend cannot transfer payloads unless buffers are registered or pinned. + pub registered_memory_required: bool, + /// Backend preserves in-order delivery for each opened transfer. + pub ordered_delivery: bool, + /// Largest payload the backend accepts for one transfer, or no RustFS-level cap. + pub max_transfer_size: Option, + /// Backend can participate in the behavior-preserving TCP fallback path. + pub fallback_supported: bool, } impl InternodeDataTransportCapabilities { pub const fn tcp_http() -> Self { Self { - stream_read: true, - stream_write: true, - walk_dir: true, - registered_memory: false, - scatter_gather: false, - zero_copy_receive: false, + streaming_read: true, + streaming_write: true, + streaming_walk_dir: true, + zero_copy_candidate: false, + registered_memory_required: false, + ordered_delivery: true, + max_transfer_size: None, + fallback_supported: true, } } } @@ -223,16 +235,29 @@ mod tests { assert_eq!( transport.capabilities(), InternodeDataTransportCapabilities { - stream_read: true, - stream_write: true, - walk_dir: true, - registered_memory: false, - scatter_gather: false, - zero_copy_receive: false, + streaming_read: true, + streaming_write: true, + streaming_walk_dir: true, + zero_copy_candidate: false, + registered_memory_required: false, + ordered_delivery: true, + max_transfer_size: None, + fallback_supported: true, } ); } + #[test] + fn tcp_http_capabilities_do_not_advertise_rdma_specific_features() { + let capabilities = TcpHttpInternodeDataTransport.capabilities(); + + assert!(!capabilities.zero_copy_candidate); + assert!(!capabilities.registered_memory_required); + assert!(capabilities.ordered_delivery); + assert_eq!(capabilities.max_transfer_size, None); + assert!(capabilities.fallback_supported); + } + #[test] fn read_file_stream_url_encodes_query_values() { let url = build_read_file_stream_url(&ReadStreamRequest {