From 2786a4734afa14659f142c87d0e9ec1c5670d92b Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Sat, 23 May 2026 00:27:27 +0800 Subject: [PATCH] docs(internode): align transport adapter scope (#3064) Co-authored-by: Henry Guo --- .../internode-transport-adapter-rfc.md | 142 ++++++++++------- .../transport-buffer-contract.md | 142 +++++++++++++++++ .../transport-buffer-lifecycle.md | 52 +++++-- .../transport-capabilities.md | 41 +++-- .../transport-fallback-and-selection.md | 144 ++++++++++++++++++ .../src/rpc/internode_data_transport.rs | 14 +- 6 files changed, 447 insertions(+), 88 deletions(-) rename INTERNODE_DATA_TRANSPORT_RFC.md => crates/ecstore/docs/internode-transport/internode-transport-adapter-rfc.md (85%) create mode 100644 crates/ecstore/docs/internode-transport/transport-buffer-contract.md rename INTERNODE_BUFFER_LIFECYCLE.md => crates/ecstore/docs/internode-transport/transport-buffer-lifecycle.md (85%) rename INTERNODE_TRANSPORT_CAPABILITIES.md => crates/ecstore/docs/internode-transport/transport-capabilities.md (69%) create mode 100644 crates/ecstore/docs/internode-transport/transport-fallback-and-selection.md diff --git a/INTERNODE_DATA_TRANSPORT_RFC.md b/crates/ecstore/docs/internode-transport/internode-transport-adapter-rfc.md similarity index 85% rename from INTERNODE_DATA_TRANSPORT_RFC.md rename to crates/ecstore/docs/internode-transport/internode-transport-adapter-rfc.md index 44c783b98..6c39b6822 100644 --- a/INTERNODE_DATA_TRANSPORT_RFC.md +++ b/crates/ecstore/docs/internode-transport/internode-transport-adapter-rfc.md @@ -1,23 +1,22 @@ -# RFC: Pluggable Internode Data Transport +# RFC: Internode Transport Adapter Boundary > Status: draft -> Last updated: 2026-05-21 -> Scope: internode data-path analysis, benchmark baseline, and transport boundary +> Last updated: 2026-05-22 +> Scope: OSS internode data-plane adapter analysis, benchmark baseline, and +> transport boundary ## Summary -RustFS does not currently include RDMA, RoCE, InfiniBand, DPU, BlueField/DOCA, -DPDK, SPDK, or SmartNIC offload support. The current distributed internode -paths use TCP-based HTTP/gRPC transports: +The current distributed internode paths use TCP-based HTTP/gRPC transports: - `tonic` gRPC `NodeService` for most control, metadata, lock, health, and peer operations. - HTTP streaming routes under `/rustfs/rpc/` for remote disk file streams. -RDMA/RoCE is still a plausible future optimization for large internode disk -data transfers, but it should not replace the whole internode RPC surface. -The correct first step is to isolate the data plane, establish a TCP baseline, -and introduce a pluggable transport boundary only around high-volume streams. +This document frames the existing work as an OSS `InternodeDataTransport` +adapter boundary. The adapter keeps RustFS data-plane logic separate from the +concrete transport backend while preserving the current TCP/HTTP behavior as +the default implementation. Current implementation status: @@ -33,16 +32,49 @@ Current implementation status: - `NodeService` gRPC remains the internode control plane and continues to carry metadata/control operations. +Related design notes in this directory: + +- `transport-capabilities.md` +- `transport-buffer-lifecycle.md` +- `transport-buffer-contract.md` +- `transport-fallback-and-selection.md` + +## Open-source Scope + +The OSS scope is: + +- define a clear `InternodeDataTransport` adapter boundary; +- keep `tcp-http` as the default backend; +- keep existing TCP/HTTP behavior unchanged; +- keep internode data-plane behavior observable through metrics and baseline + tooling; +- document buffer ownership, fallback, and capability expectations for + maintainable transport code; +- avoid hardware-specific dependencies or backend implementations. + +The OSS scope is not: + +- RDMA support; +- DPU support; +- DOCA support; +- BlueField support; +- RoCE/InfiniBand support; +- hardware benchmark planning; +- hardware-specific backend implementation. + +Hardware-specific transport experiments are outside the scope of this OSS +document. + ## Goals - Document the current internode control plane and data plane. -- Identify the existing transfer paths that could benefit from a future - high-throughput backend. +- Identify the existing transfer paths covered by the + `InternodeDataTransport` adapter and the paths that remain on gRPC. - Define the minimum benchmark baseline required before transport changes. - Sketch a pluggable transport boundary that preserves the current TCP/HTTP behavior as the default backend. -- Reserve explicit boundaries for future RDMA/RoCE/InfiniBand work without - committing RustFS to a specific vendor stack. +- Document backend-neutral capability, fallback, buffer ownership, and + observability expectations. ## Non-Goals @@ -50,8 +82,8 @@ Current implementation status: - Replace `tonic` gRPC for control-plane RPCs. - Redesign erasure coding, quorum handling, disk health tracking, or object correctness semantics. -- Require RDMA-capable hardware for default development, CI, or ordinary - RustFS deployments. +- Require specialized hardware for default development, CI, or ordinary RustFS + deployments. ## Current Internode Architecture @@ -92,9 +124,9 @@ classes of RPCs: - peer rest service: node health, metrics, IAM/policy reload, rebalance, profiling, events, and admin-style operations -The service layout is practical today, but it is too broad to become an RDMA -surface. A future high-throughput transport should target only disk data -streams and keep this gRPC service as the control plane. +The service layout is practical today, but it is too broad to become the +transport adapter surface. A pluggable data transport should target only disk +data streams and keep this gRPC service as the control plane. ## Control Plane vs Data Plane @@ -254,8 +286,8 @@ not yet isolate internode transport cost. ## Required TCP Baseline -Before adding any non-TCP backend, collect a baseline for the current -TCP/HTTP/gRPC implementation. +Before changing internode data transport behavior or comparing a non-default +backend, collect a baseline for the current TCP/HTTP/gRPC implementation. ### Topology @@ -281,7 +313,7 @@ Preferred: | Remote disk stream read | shard-sized ranges from `read_file_stream` | 1, 16, 64 | Isolated internode read path. | | Remote disk stream write | shard-sized writes through `put_file_stream` | 1, 16, 64 | Isolated internode write path. | | Healing / repair | missing disk or missing shard scenario | controlled | Rebuild throughput and read/write amplification. | -| Scanner walk | large bucket/object namespace | controlled | Metadata streaming pressure, not primary RDMA target. | +| Scanner walk | large bucket/object namespace | controlled | Metadata streaming pressure, not the primary object-byte transport path. | ### Measurements @@ -318,8 +350,8 @@ only below `RemoteDisk`, where remote disk byte streams are opened today. The first implementation should be a no-behavior-change TCP/HTTP backend that wraps the current `HttpReader`, `HttpWriter`, and `/rustfs/rpc/*` handlers. -Only after that wrapper is benchmarked should an experimental RDMA/RoCE backend -be considered. +Non-default backend work should not proceed until the default wrapper is +measured and adapter gaps are documented. ### Candidate boundary @@ -366,15 +398,16 @@ until measurements prove otherwise. ### Capability model -Avoid hard-coding RDMA assumptions into the generic interface. Use capabilities: +Avoid hard-coding transport-specific assumptions into the generic interface. +Use capabilities: - stream read - stream write - bounded range read - bidirectional streaming -- registered memory support -- scatter/gather support -- zero-copy receive into caller-owned buffers +- backend-specific buffer registration or staging requirements +- stable buffer ownership support +- copy-reduced receive into caller-owned or backend-owned buffers - authenticated out-of-band transfer - transport fallback support @@ -392,12 +425,12 @@ Fallback rules: `tcp-http` and the `tcp` alias. Empty and unset values use `tcp-http`. - Invalid configured values fail closed with an error that includes the env var name and invalid value. -- If a future experimental backend fails initialization, either fail fast with a +- If a future non-default backend fails initialization, either fail fast with a clear error or fall back to TCP only when the configured policy allows fallback. - Runtime fallback must preserve object correctness and quorum semantics. - Fallback events must be logged and counted in metrics. -- CI and local development must not require RDMA-capable hardware. +- CI and local development must not require specialized transport hardware. Suggested future configuration shape: @@ -442,40 +475,33 @@ Expected artifacts: - `internode_metric_deltas.csv` when `--metrics-url` is provided The baseline validates the default TCP/HTTP path only. It must not be used to -claim RDMA, RoCE, or InfiniBand support. +claim support or performance for any non-default transport backend. -## Future RDMA/RoCE/InfiniBand Boundary +## Non-default Backend Boundary -A future RDMA backend should be experimental and feature-gated. It should be -designed as an optional data-plane backend, not as a replacement for the gRPC -control plane. +A future non-default backend must be explicitly enabled and must not replace +`tcp-http` silently. It should be designed as an optional data-plane backend, +not as a replacement for the gRPC control plane. -A future non-TCP backend would need an explicit design for: +A future non-default backend would need an explicit design for: -- peer capability discovery over the existing gRPC control plane -- connection management and health mapping into existing disk fault handling -- memory registration lifecycle and registration cache -- buffer ownership, pinning, alignment, and lifetime rules -- scatter/gather behavior for erasure shards -- authentication and authorization for out-of-band data transfers -- encryption/TLS-equivalent story or a documented deployment boundary -- timeout, cancellation, retry, and fallback behavior -- metrics for registration cost, transfer latency, bytes, queue depth, retries, - fallback, and errors -- hardware and kernel compatibility matrix +- peer capability discovery over the existing gRPC control plane; +- connection management and health mapping into existing disk fault handling; +- backend-specific buffer lifecycle and any staging or registration cache; +- buffer ownership, alignment, and lifetime rules; +- stable buffer behavior for erasure shards; +- authentication and authorization for out-of-band data transfers; +- encryption or an equivalent documented security boundary; +- timeout, cancellation, retry, and fallback behavior; +- metrics for transfer latency, bytes, queue depth, retries, fallback, and + errors. `walk_dir`, metadata RPCs, locks, admin RPCs, and bucket coordination remain outside the current data-plane boundary. -## DPU, DOCA, DPDK, SPDK, and SmartNIC Notes +## Out of Scope -These technologies should not drive the first abstraction: - -- DPU/BlueField/DOCA may become relevant for TLS, checksum, compression, or - storage/network offload, but they are vendor- and deployment-specific. -- DPDK is a poor first fit because RustFS is currently an HTTP/S3 object store - and does not have a custom packet data plane. -- SPDK may be relevant only if RustFS adds a raw block or NVMe-oriented local - storage backend. The current disk model is filesystem-based. -- SmartNIC offload is outside the current boundary because this RFC does not - establish a CPU-offload bottleneck. +Hardware-specific transport experiments are outside the scope of this OSS +document. This RFC does not add a plugin system, split the adapter into a +separate crate, add accepted backend values, or implement a new transport +backend. diff --git a/crates/ecstore/docs/internode-transport/transport-buffer-contract.md b/crates/ecstore/docs/internode-transport/transport-buffer-contract.md new file mode 100644 index 000000000..321afeec8 --- /dev/null +++ b/crates/ecstore/docs/internode-transport/transport-buffer-contract.md @@ -0,0 +1,142 @@ +# Internode Transport Buffer Contract + +Status: design note only. This document defines a backend-neutral buffer +ownership and lifecycle contract for the `InternodeDataTransport` adapter. It +does not implement a new backend and does not change production behavior. + +## Open-source Scope + +The open-source RustFS path keeps `tcp-http` as the default internode data +transport. This document defines adapter contracts only: + +- no production RDMA, DPU, DOCA, BlueField, DPDK, SPDK, or hardware + acceleration backend is introduced; +- no hardware SDK, `libibverbs`, `rdma-core`, or vendor dependency is added; +- no new accepted production backend value is added; +- future external or separately maintained backends may implement the same + adapter boundary without changing RustFS core data-plane logic. + +Examples of possible future external backends include DOCA/BlueField, +RDMA/RoCE, or other DPU/NIC implementations. These are examples only and are +not implemented or scheduled by this design. + +## Current Adapter Surface + +The current data-plane surface is byte-stream based: + +| Current path | Current API shape | Current ownership | +| --- | --- | --- | +| Remote read stream | `InternodeDataTransport::open_read(...) -> FileReader` | Backend returns boxed `AsyncRead`; callers provide temporary `ReadBuf` storage per poll. | +| Remote write stream | `InternodeDataTransport::open_write(...) -> FileWriter` | Callers pass borrowed `&[u8]` slices into boxed `AsyncWrite`; the backend owns any async body staging. | +| Walk-dir stream | `InternodeDataTransport::open_walk_dir(...) -> FileReader` | Same boxed stream model as read, with a small serialized request body. | + +This API is correct for the current TCP/HTTP backend. A future non-default +backend may have stricter memory ownership, buffer lifetime, or completion +requirements, so the adapter contract needs to describe those boundaries +without assuming a specific implementation. + +## Buffer Ownership Model + +| Buffer role | Allocator | Lifetime owner | Backend-specific state | TCP/HTTP behavior | +| --- | --- | --- | --- | --- | +| Send buffer | Caller or RustFS-owned pool | Caller until the backend accepts the buffer; backend until completion if an owned-buffer API is used | Optional backend-managed buffer, staging buffer, or registration handle | Copy into the existing `AsyncWrite` path when the backend cannot use the buffer directly. | +| Receive buffer | Caller-provided storage or backend-owned receive pool | Backend while filling; caller after completion if ownership is returned | Optional backend-owned receive buffer or backend-specific registration | Copy from `AsyncRead` into caller storage as today. | +| Control metadata | RustFS caller | Caller/request object | Not buffer-managed by the data-plane backend | Serialize into HTTP/gRPC/control-plane messages. | +| Fallback staging | TCP/HTTP backend | TCP/HTTP backend | No backend-specific registration | Existing `HttpReader`/`HttpWriter` buffering semantics. | + +Backends with stricter memory requirements must not let callers mutate or reuse +a buffer while an async transfer is still in flight. A backend that cannot use a +caller buffer directly must either reject the transfer before payload movement +or copy through a clearly documented backend-owned staging buffer. + +Zero-copy is not guaranteed by this contract. Backends must document whether +their path is zero-copy, copy-reduced, or staging-buffer based, and must +document where copies occur. + +## Compatibility Contract + +The current stream API remains the OSS compatibility contract: + +```rust +#[async_trait::async_trait] +pub trait InternodeDataTransport { + async fn open_read(&self, request: ReadStreamRequest) -> Result; + async fn open_write(&self, request: WriteStreamRequest) -> Result; + async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result; +} +``` + +Future extensions for backend-managed buffers should be additive and +capability-gated. A possible shape is: + +```rust +pub struct TransferBuffer { + pub bytes: bytes::Bytes, + pub backend_state: Option, +} + +pub struct CompletedTransfer { + pub bytes: bytes::Bytes, + pub transfer_len: usize, +} + +#[async_trait::async_trait] +pub trait BackendBufferInternodeTransport: InternodeDataTransport { + async fn write_backend_buffer(&self, request: WriteStreamRequest, buffer: TransferBuffer) -> Result; + async fn read_backend_buffer(&self, request: ReadStreamRequest, buffer: TransferBuffer) -> Result; +} +``` + +The concrete type of `TransportBufferState` should stay backend-private. The +generic contract only needs to state whether the buffer is usable by the +backend, who owns it during transfer, and when ownership returns. + +This PR does not add this extension. It documents the boundary that a future +external backend may need. + +## Required Contract for Stricter Backends + +| Area | Required contract | +| --- | --- | +| Ownership | Define when caller-owned bytes become backend-owned and when ownership returns. | +| Completion | Signal completion before RustFS can reuse or mutate backend-managed memory. | +| Staging | Declare whether the backend copies through backend-owned staging buffers when direct use is unavailable. | +| Size limits | Expose any RustFS-visible `max_transfer_size`. | +| Ordering | Either provide ordered delivery or include a reassembly layer before exposing stream semantics. | +| Copy accounting | Document every known copy boundary and avoid claiming zero-copy unless the path proves it. | + +## Optional Optimizations + +| Area | Optional behavior | +| --- | --- | +| Buffer pooling | RustFS may keep reusable pools for send and receive buffers. | +| Backend state cache | A backend may cache backend-specific handles for long-lived buffers. | +| Scatter/gather | A backend may accept multiple shard slices without repacking when its completion model can report per-slice ownership safely. | +| Backend-owned receive | A backend may return owned receive chunks instead of filling caller-provided buffers. | + +These are optional optimizations, not requirements for the OSS TCP/HTTP path. + +## Current API Limitations + +| Current API | Limitation for stricter backends | +| --- | --- | +| `FileReader = Box` | `AsyncRead` exposes temporary caller `ReadBuf` storage, not a stable backend-managed receive buffer or explicit completion token. | +| `FileWriter = Box` | `AsyncWrite::poll_write` receives borrowed `&[u8]` that cannot outlive the poll, so async direct transfer requires copying or a different ownership API. | +| `HttpWriter` | The async HTTP body must own `Bytes`, so borrowed write buffers are copied into `BytesMut` or `Bytes`. | +| `write_body_chunks_to_writer` | Server-side HTTP body chunks are copied into `BytesMut` before local disk write. | +| Erasure encode output | Encoded shards are represented as `Vec` and written through `AsyncWrite`, not a completion-aware backend-buffer API. | +| Erasure decode input | Shard reads allocate `Vec` buffers before decode; no backend-owned receive pool is visible at the transport boundary. | + +These limitations do not block the current `tcp-http` backend. They describe +where a future external backend would need staging or an additive API. + +## External Backend Crate Compatibility + +`InternodeDataTransport` should remain implementable by future backends without +modifying RustFS core data-plane logic. In the short term, the trait and +`tcp-http` backend may remain inside `ecstore`. + +A future external or separately maintained backend could live in a separate +crate if the trait, request/response types, capability report, and error model +are public and stable enough. This PR does not perform a crate split, add +runtime loading, or introduce a plugin system. diff --git a/INTERNODE_BUFFER_LIFECYCLE.md b/crates/ecstore/docs/internode-transport/transport-buffer-lifecycle.md similarity index 85% rename from INTERNODE_BUFFER_LIFECYCLE.md rename to crates/ecstore/docs/internode-transport/transport-buffer-lifecycle.md index 17fd9c329..d42f6e309 100644 --- a/INTERNODE_BUFFER_LIFECYCLE.md +++ b/crates/ecstore/docs/internode-transport/transport-buffer-lifecycle.md @@ -1,8 +1,30 @@ # Internode Buffer Lifecycle and Copy Count Status: P1-D analysis only. This document records the current TCP/HTTP -internode data path and the ownership boundaries that matter before designing -RDMA/RoCE/InfiniBand backends. It does not claim RDMA support exists. +internode data path and the ownership boundaries that matter for the +backend-neutral `InternodeDataTransport` adapter. It does not implement a new +backend or change production behavior. + +## Open-source Scope + +The OSS scope is: + +- define buffer ownership and copy-count behavior for the current + `InternodeDataTransport` adapter; +- keep `tcp-http` as the default backend; +- keep existing TCP/HTTP behavior unchanged; +- document copy hotspots and ownership gaps for maintainable transport code; +- avoid hardware-specific dependencies or backend implementations. + +The OSS scope is not: + +- RDMA support; +- DPU support; +- DOCA support; +- BlueField support; +- RoCE/InfiniBand support; +- hardware benchmark planning; +- hardware-specific backend implementation. ## Scope @@ -74,30 +96,30 @@ through `ParallelReader` in `crates/ecstore/src/erasure_coding/decode.rs` and | 6 | `RemoteDisk::read_file_zero_copy` | Medium when used | Remote zero-copy reads buffer the whole stream into memory. The name does not mean zero-copy over the network. | | 7 | URL/query/header/JSON serialization | Low | Metadata copies are small and not on the large payload hot path. | -## RDMA-Blocking Ownership Issues +## Adapter Ownership Gaps 1. `FileReader` and `FileWriter` are boxed `AsyncRead`/`AsyncWrite` trait - objects. They expose borrowed buffers per poll, not registered memory - regions, memory keys, scatter/gather lists, or completion ownership. + objects. They expose borrowed buffers per poll, not stable backend-owned + regions, transfer handles, or explicit completion ownership. 2. `InternodeDataTransport` currently returns stream traits only. Its - capabilities advertise that TCP/HTTP has no registered memory, - scatter/gather, or zero-copy receive support, but there is no backend API to - pass registered buffers. + capabilities advertise that TCP/HTTP does not require backend-specific + buffer registration and is not a zero-copy candidate, but there is no + backend API to pass stable backend-managed buffers. 3. `HttpWriter` must own outgoing chunks because the async request body outlives - the caller's borrowed `&[u8]`. A zero-copy backend would need a different + the caller's borrowed `&[u8]`. A lower-copy backend would need a different lifetime contract or an owned buffer pool. 4. Server write handling normalizes all incoming body chunks into a new `BytesMut`. Avoiding that copy would require passing incoming `Bytes` or - registered receive buffers directly into the disk/bitrot writer contract. + backend-owned receive buffers directly into the disk/bitrot writer contract. 5. Erasure decode owns shard `Vec` buffers and write-back happens through - `AsyncWrite`. A zero-copy backend would need explicit ownership of shard + `AsyncWrite`. A lower-copy backend would need explicit ownership of shard buffers across decode, reconstruction, and network completion. 6. Erasure encode materializes `Vec` blocks before fanout. A - scatter/gather backend would need an encode output representation that can be - sent as stable slices without repacking. + backend that can send multiple stable slices would need an encode output + representation that can be transferred without repacking. 7. The HTTP auth and URL construction boundary is part of the current TCP/HTTP - backend. A non-HTTP backend will need equivalent peer authentication and disk - addressing without assuming URL query parameters. + backend. A non-HTTP backend would need equivalent peer authentication and + disk addressing without assuming URL query parameters. 8. Local disk zero-copy exists only for local reads via `read_file_zero_copy`. Remote disks deliberately fall back to network streaming and full-buffer collection for the zero-copy helper. diff --git a/INTERNODE_TRANSPORT_CAPABILITIES.md b/crates/ecstore/docs/internode-transport/transport-capabilities.md similarity index 69% rename from INTERNODE_TRANSPORT_CAPABILITIES.md rename to crates/ecstore/docs/internode-transport/transport-capabilities.md index 53a86ee18..1ff161f85 100644 --- a/INTERNODE_TRANSPORT_CAPABILITIES.md +++ b/crates/ecstore/docs/internode-transport/transport-capabilities.md @@ -1,14 +1,35 @@ # Internode Transport Capabilities Status: design note for backend-neutral capability reporting. This document -does not add an RDMA backend and does not require RDMA hardware or crates. +does not add a backend and does not require specialized hardware or transport +crates. + +## Open-source Scope + +The OSS scope is: + +- define honest capability reporting for the `InternodeDataTransport` adapter; +- keep `tcp-http` as the default backend; +- keep existing TCP/HTTP behavior unchanged; +- document the capability fields needed for maintainable transport code; +- avoid hardware-specific dependencies or backend implementations. + +The OSS scope is not: + +- RDMA support; +- DPU support; +- DOCA support; +- BlueField support; +- RoCE/InfiniBand support; +- hardware benchmark planning; +- hardware-specific backend implementation. ## Purpose `InternodeDataTransportCapabilities` describes what a backend can honestly do for RustFS internode data-plane transfers. The fields are intentionally neutral: -they can describe the current TCP/HTTP backend and a future high-speed backend -without naming a vendor stack or transport implementation. +they can describe the current TCP/HTTP backend and a future non-default backend +without naming a specific transport implementation. The capability report is descriptive. It does not select a backend, negotiate with peers, or weaken object correctness semantics. @@ -21,7 +42,7 @@ with peers, or weaken object correctness semantics. | `streaming_write` | The backend can open a remote disk writer for `create_file` or `append_file`. | | `streaming_walk_dir` | The backend can stream `walk_dir` responses. | | `zero_copy_candidate` | The backend has an API shape that could avoid an extra user-space payload copy. This is not a promise that every transfer is zero-copy. | -| `registered_memory_required` | The backend requires pinned or registered buffers before payload transfer. | +| `registered_memory_required` | The backend requires pinned, registered, staged, or otherwise backend-managed buffers before payload transfer. | | `ordered_delivery` | Bytes for each opened transfer are delivered in order. | | `max_transfer_size` | Optional RustFS-level cap for a single transfer. `None` means no additional cap beyond the backend/protocol/runtime limits. | | `fallback_supported` | The backend can participate in the behavior-preserving TCP fallback path. | @@ -36,20 +57,20 @@ The default TCP/HTTP backend reports only capabilities it actually provides: | `streaming_write` | `true` | `HttpWriter` streams `/rustfs/rpc/put_file_stream` request bodies. | | `streaming_walk_dir` | `true` | `HttpReader` streams `/rustfs/rpc/walk_dir` responses. | | `zero_copy_candidate` | `false` | The current path exposes `AsyncRead`/`AsyncWrite` and HTTP body chunks; it copies through normal user-space buffers. | -| `registered_memory_required` | `false` | TCP/HTTP does not require RustFS-managed registered memory. | +| `registered_memory_required` | `false` | TCP/HTTP does not require RustFS-managed pinned, registered, or backend-owned buffers. | | `ordered_delivery` | `true` | Each HTTP request body or response body is consumed as an ordered byte stream. | | `max_transfer_size` | `None` | RustFS does not impose an extra per-transfer cap at the capability layer. | | `fallback_supported` | `true` | TCP/HTTP is the behavior-preserving default and fallback path. | -## Future Backend Fit +## Non-default Backend Fit -A future high-speed backend can be described without changing the meaning of the -existing TCP report: +A future non-default backend can be described without changing the meaning of +the existing TCP report: | Capability shape | Interpretation | | --- | --- | -| `zero_copy_candidate=true`, `registered_memory_required=true` | The backend can only use its fast path with buffers that satisfy registration or pinning rules. | -| `zero_copy_candidate=true`, `registered_memory_required=false` | The backend may expose owned chunks or another lower-copy path without requiring caller-registered memory. | +| `zero_copy_candidate=true`, `registered_memory_required=true` | The backend can only use its lower-copy path with buffers that satisfy backend-specific ownership or registration rules. | +| `zero_copy_candidate=true`, `registered_memory_required=false` | The backend may expose owned chunks or another lower-copy path without requiring caller-managed registration. | | `max_transfer_size=Some(n)` | The backend has a RustFS-visible transfer size ceiling and callers must split larger transfers or use fallback behavior. | | `ordered_delivery=false` | The backend cannot be used behind the current stream API without an ordering or reassembly layer. | diff --git a/crates/ecstore/docs/internode-transport/transport-fallback-and-selection.md b/crates/ecstore/docs/internode-transport/transport-fallback-and-selection.md new file mode 100644 index 000000000..0123b4110 --- /dev/null +++ b/crates/ecstore/docs/internode-transport/transport-fallback-and-selection.md @@ -0,0 +1,144 @@ +# Internode Transport Fallback and Backend Selection Model + +Status: design note only. This document defines backend-neutral selection, +fallback, failure handling, negotiation, security, and observability rules for +the `InternodeDataTransport` adapter. It does not implement a new backend and +does not change production behavior. + +## Open-source Scope + +The open-source RustFS path keeps `tcp-http` as the default internode data +transport. This document defines adapter contracts only: + +- no production RDMA, DPU, DOCA, BlueField, DPDK, SPDK, or hardware + acceleration backend is introduced; +- no hardware SDK, `libibverbs`, `rdma-core`, or vendor dependency is added; +- no new accepted production backend value is added; +- future external or separately maintained backends may implement the same + adapter boundary without changing RustFS core data-plane logic. + +Examples of possible future external backends include DOCA/BlueField, +RDMA/RoCE, or other DPU/NIC implementations. These are examples only and are +not implemented or scheduled by this design. Hardware-specific backend plans +are out of scope for this document. + +## Static Backend Selection + +Static config is the first selection model. Existing accepted values remain: + +| Config value | Meaning | +| --- | --- | +| unset | Use default TCP/HTTP backend. | +| `tcp-http` | Use default TCP/HTTP backend. | +| `tcp` | Alias for `tcp-http`. | +| any unsupported value | Fail closed with a diagnostic naming `RUSTFS_INTERNODE_DATA_TRANSPORT` and the invalid value. | + +Unknown backend values must fail closed. Unsupported backend values must fail +closed. A future external backend must be explicitly enabled and must not +silently replace `tcp-http`. + +Backend selection must expose an observable backend identity for metrics, logs, +and benchmark interpretation. The default and fallback path remains `tcp-http`. + +## Fallback Contract + +Fallback must be explicit and observable. Silent fallback is not allowed for +benchmark or production interpretation because it hides which backend moved the +payload. + +| Condition | Default behavior | Explicit fallback behavior | Observability | +| --- | --- | --- | --- | +| Unsupported configured backend | Fail closed during transport construction. | Fall back only when a separately configured policy explicitly allows unsupported-backend fallback. | Error includes config key and invalid value; fallback event is counted when fallback is enabled. | +| Peer does not support selected backend | Fail before payload transfer. | Use TCP/HTTP only when both local policy and peer policy allow it. | Count peer mismatch and selected fallback backend. | +| Capability mismatch | Fail before payload transfer. | Use TCP/HTTP only if TCP satisfies the operation and policy allows fallback. | Record missing capability names or a low-cardinality reason. | +| Connection setup failure | Fail the operation. | Retry on TCP/HTTP only when fallback is allowed and no payload bytes have transferred. | Count setup failure, retry, fallback backend, and fallback result. | +| Partial transfer failure | Fail the operation and let existing object/quorum logic decide retry behavior. | Do not silently resume on another backend unless the transfer protocol can prove byte range, checksum, and idempotency boundaries. | Count partial failure with bytes completed. | +| Max transfer size exceeded | Fail before payload transfer or split at a higher layer. | Use TCP/HTTP if policy allows and TCP has no RustFS-level cap. | Record rejected size and selected backend. | +| Auth or encryption mismatch | Fail closed. | No fallback unless the fallback path satisfies the same or stronger security requirements. | Security failure metric and audit log entry. | + +Fallback settings should not be added until there is an implementation that +uses them. A backend must define failure behavior before production use. + +## Dynamic Negotiation Boundary + +Dynamic negotiation, if added, belongs on the existing gRPC control plane. Data +transfer must start only after both peers agree on: + +| Negotiated item | Required property | +| --- | --- | +| Backend name | Both peers know the backend and have it enabled. | +| Capability set | Required capabilities match the operation. | +| Max transfer size | The selected operation fits or is split before transfer starts. | +| Buffer rules | Both peers agree whether backend-managed or staged buffers are required. | +| Completion semantics | Both peers agree when a transfer is considered complete and when buffers may be reused. | +| Security mode | Authentication and encryption requirements are satisfied before any out-of-band transfer. | +| Fallback policy | Both peers agree whether TCP/HTTP fallback is allowed for this operation. | + +Negotiation must not silently downgrade security or bypass existing disk +health, quorum, timeout, and integrity semantics. + +## Failure Handling Requirements + +| Failure mode | Requirement | +| --- | --- | +| Invalid config | Fail closed with `RUSTFS_INTERNODE_DATA_TRANSPORT` and the invalid value. | +| Backend disabled | Fail closed with the selected backend name and the missing enablement condition. | +| Backend unavailable | Fail closed with an actionable diagnostic; do not silently use TCP/HTTP. | +| Peer mismatch | Fail before payload transfer unless explicit fallback is configured. | +| Connection failure | Fail the operation and record setup failure; fallback only if policy allows and no payload bytes moved. | +| Completion failure | Return an operation error and release backend-owned resources. | +| Timeout | Return an operation error and preserve existing disk health and quorum semantics. | +| Partial transfer | Do not silently resume on another backend without a safe byte-range/checksum/idempotency proof. | +| Unsupported operation | Return a clear unsupported-operation error. | + +## Security Requirements + +- Backend selection must preserve peer authentication. +- Fallback must not weaken encryption or authorization. +- Out-of-band data-plane transfers must still bind to the intended disk, + volume, path, request authority, and operation. +- Partial transfers must not bypass bitrot verification or erasure quorum + handling. +- Any future external backend must document whether it relies on the same + security boundary as TCP/HTTP or requires a separate deployment boundary. + +## Metrics and Observability Requirements + +Metrics and logs must use low-cardinality labels or metadata: + +- selected backend; +- requested backend; +- fallback backend, when used; +- operation name; +- success/failure; +- transferred bytes; +- setup failure count; +- partial transfer failure count; +- capability mismatch count; +- fallback decision count. + +Backends must not add high-cardinality labels such as object names, full paths, +full URLs, peer-specific dynamic strings, memory addresses, or backend-specific +buffer keys. + +## TCP/HTTP Compatibility + +The `tcp-http` backend remains the default and behavior-preserving path. It +uses ordinary byte streams, does not require backend-specific buffer +registration, and remains suitable as the fallback path when an explicit +fallback policy exists. + +A future non-default backend must not change the correctness semantics of +object writes, object reads, healing, bitrot verification, erasure quorum, +timeouts, or disk health handling. + +## External Backend Crate Compatibility + +`InternodeDataTransport` should remain implementable by future backends without +modifying RustFS core data-plane logic. In the short term, the trait and +`tcp-http` backend may remain inside `ecstore`. + +A future external or separately maintained backend could live in a separate +crate if the trait, request/response types, capability report, and error model +are public and stable enough. This PR does not perform a crate split, add +runtime loading, or introduce a plugin system. diff --git a/crates/ecstore/src/rpc/internode_data_transport.rs b/crates/ecstore/src/rpc/internode_data_transport.rs index 768884630..d1001af59 100644 --- a/crates/ecstore/src/rpc/internode_data_transport.rs +++ b/crates/ecstore/src/rpc/internode_data_transport.rs @@ -107,6 +107,9 @@ pub struct WalkDirStreamRequest { /// This boundary is limited to remote disk streams that can move large payloads. /// Internode metadata, lock, health, and administrative calls remain on the /// existing gRPC control plane. +/// +/// Buffer ownership, backend selection, and fallback expectations are documented +/// in `crates/ecstore/docs/internode-transport/`. #[async_trait] pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug { async fn open_read(&self, request: ReadStreamRequest) -> Result; @@ -248,7 +251,7 @@ mod tests { } #[test] - fn tcp_http_capabilities_do_not_advertise_rdma_specific_features() { + fn tcp_http_capabilities_do_not_advertise_stricter_backend_features() { let capabilities = TcpHttpInternodeDataTransport.capabilities(); assert!(!capabilities.zero_copy_candidate); @@ -337,18 +340,19 @@ mod tests { #[test] fn transport_config_rejects_unknown_backend() { - let err = build_internode_data_transport(Some("rdma")).expect_err("unknown backend should fail closed"); + let err = build_internode_data_transport(Some("unsupported-backend")).expect_err("unknown backend should fail closed"); assert!(err.to_string().contains(ENV_RUSTFS_INTERNODE_DATA_TRANSPORT)); - assert!(err.to_string().contains("rdma")); + assert!(err.to_string().contains("unsupported-backend")); } #[test] fn cached_transport_config_error_uses_raw_message() { - let err = build_internode_data_transport_result(Some("rdma")).expect_err("unknown backend should fail closed"); + let err = + build_internode_data_transport_result(Some("unsupported-backend")).expect_err("unknown backend should fail closed"); assert!(!err.starts_with("io error ")); assert!(err.contains(ENV_RUSTFS_INTERNODE_DATA_TRANSPORT)); - assert!(err.contains("rdma")); + assert!(err.contains("unsupported-backend")); } }