From 53b608c08935f190b3b3192544f416fc58e6af2b Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Sat, 23 May 2026 13:00:36 +0800 Subject: [PATCH] docs(internode): keep transport adapter OSS scoped (#3067) docs(internode): keep transport adapter oss scoped Co-authored-by: Henry Guo --- .../internode-transport-adapter-rfc.md | 70 ++++------- .../transport-buffer-contract.md | 119 +++++------------- .../transport-buffer-lifecycle.md | 13 +- .../transport-capabilities.md | 33 ++--- .../transport-fallback-and-selection.md | 53 ++++---- .../src/rpc/internode_data_transport.rs | 12 +- 6 files changed, 96 insertions(+), 204 deletions(-) diff --git a/crates/ecstore/docs/internode-transport/internode-transport-adapter-rfc.md b/crates/ecstore/docs/internode-transport/internode-transport-adapter-rfc.md index 6c39b6822..94544624a 100644 --- a/crates/ecstore/docs/internode-transport/internode-transport-adapter-rfc.md +++ b/crates/ecstore/docs/internode-transport/internode-transport-adapter-rfc.md @@ -50,20 +50,15 @@ The OSS scope is: tooling; - document buffer ownership, fallback, and capability expectations for maintainable transport code; -- avoid hardware-specific dependencies or backend implementations. +- avoid adding dependencies or backend implementations. The OSS scope is not: -- RDMA support; -- DPU support; -- DOCA support; -- BlueField support; -- RoCE/InfiniBand support; -- hardware benchmark planning; -- hardware-specific backend implementation. - -Hardware-specific transport experiments are outside the scope of this OSS -document. +- adding another transport backend; +- replacing the gRPC control plane; +- adding benchmark plans for another transport; +- adding runtime plugin loading; +- changing object correctness semantics. ## Goals @@ -78,12 +73,11 @@ document. ## Non-Goals -- Implement RDMA, RoCE, InfiniBand, DPU, DOCA, DPDK, SPDK, or SmartNIC support. +- Implement another transport backend. - Replace `tonic` gRPC for control-plane RPCs. - Redesign erasure coding, quorum handling, disk health tracking, or object correctness semantics. -- Require specialized hardware for default development, CI, or ordinary RustFS - deployments. +- Change default development, CI, or ordinary RustFS deployment requirements. ## Current Internode Architecture @@ -154,7 +148,7 @@ are the only reasonable first candidates for a pluggable transport. | P0 | `put_file_stream` | `RemoteDisk::create_file` and `RemoteDisk::append_file` | `handle_put_file` in `http_service.rs` | HTTP `PUT /rustfs/rpc/put_file_stream` with a streaming request body | Main remote disk write stream used by bitrot writers and erasure writes. | | P1 | `walk_dir` | `RemoteDisk::walk_dir` | `handle_walk_dir` in `http_service.rs` | HTTP `GET /rustfs/rpc/walk_dir` with a streamed metadata listing | Can be high-volume during scans/healing, but it is metadata-oriented rather than object byte data. | | P1 | `ReadAll` / `WriteAll` | `RemoteDisk::read_all` / `write_all` | gRPC unary disk handlers | gRPC unary `bytes` payload | Moves bytes today, but should be measured before treating it as a high-throughput data path. | -| P2 | proto `WriteStream` / `ReadAt` | currently not used | currently returns unimplemented | gRPC streaming definitions exist but are not implemented | Possible future API shape, not a current production path. | +| P2 | proto `WriteStream` / `ReadAt` | currently not used | currently returns unimplemented | gRPC streaming definitions exist but are not implemented | Declared proto shape, not a current production path. | ## P1 Data Path Inventory @@ -425,19 +419,9 @@ Fallback rules: `tcp-http` and the `tcp` alias. Empty and unset values use `tcp-http`. - Invalid configured values fail closed with an error that includes the env var name and invalid value. -- If a future non-default backend fails initialization, either fail fast with a - clear error or fall back to TCP only when the configured policy allows - fallback. +- Unsupported configured backends fail closed during transport construction. - Runtime fallback must preserve object correctness and quorum semantics. - Fallback events must be logged and counted in metrics. -- CI and local development must not require specialized transport hardware. - -Suggested future configuration shape: - -```text -RUSTFS_INTERNODE_DATA_TRANSPORT=tcp-http -RUSTFS_INTERNODE_DATA_TRANSPORT_FALLBACK=tcp -``` Do not add fallback settings until there is an implementation PR that uses them. @@ -475,33 +459,25 @@ Expected artifacts: - `internode_metric_deltas.csv` when `--metrics-url` is provided The baseline validates the default TCP/HTTP path only. It must not be used to -claim support or performance for any non-default transport backend. +claim support or performance for any other transport. -## Non-default Backend Boundary +## Adapter Constraints -A future non-default backend must be explicitly enabled and must not replace -`tcp-http` silently. It should be designed as an optional data-plane backend, -not as a replacement for the gRPC control plane. +The current adapter boundary has these constraints: -A future non-default backend would need an explicit design for: - -- peer capability discovery over the existing gRPC control plane; -- connection management and health mapping into existing disk fault handling; -- backend-specific buffer lifecycle and any staging or registration cache; -- buffer ownership, alignment, and lifetime rules; -- stable buffer behavior for erasure shards; -- authentication and authorization for out-of-band data transfers; -- encryption or an equivalent documented security boundary; -- timeout, cancellation, retry, and fallback behavior; -- metrics for transfer latency, bytes, queue depth, retries, fallback, and - errors. +- `tcp-http` is the default and only OSS backend. +- Backend selection is explicit and fail-closed. +- The gRPC control plane remains responsible for metadata, health, locks, and + coordination. +- Transport errors must preserve existing disk health, quorum, timeout, and + integrity semantics. +- Metrics must identify the selected backend and operation without high-cardinality + labels. `walk_dir`, metadata RPCs, locks, admin RPCs, and bucket coordination remain outside the current data-plane boundary. ## Out of Scope -Hardware-specific transport experiments are outside the scope of this OSS -document. This RFC does not add a plugin system, split the adapter into a -separate crate, add accepted backend values, or implement a new transport -backend. +This RFC does not add a plugin system, split the adapter into a separate crate, +add accepted backend values, or implement a new transport backend. diff --git a/crates/ecstore/docs/internode-transport/transport-buffer-contract.md b/crates/ecstore/docs/internode-transport/transport-buffer-contract.md index 321afeec8..dae422750 100644 --- a/crates/ecstore/docs/internode-transport/transport-buffer-contract.md +++ b/crates/ecstore/docs/internode-transport/transport-buffer-contract.md @@ -9,16 +9,11 @@ does not implement a new backend and does not change production behavior. The open-source RustFS path keeps `tcp-http` as the default internode data transport. This document defines adapter contracts only: -- no production RDMA, DPU, DOCA, BlueField, DPDK, SPDK, or hardware - acceleration backend is introduced; -- no hardware SDK, `libibverbs`, `rdma-core`, or vendor dependency is added; +- no additional production backend is introduced; +- no dependency is added; - no new accepted production backend value is added; -- future external or separately maintained backends may implement the same - adapter boundary without changing RustFS core data-plane logic. - -Examples of possible future external backends include DOCA/BlueField, -RDMA/RoCE, or other DPU/NIC implementations. These are examples only and are -not implemented or scheduled by this design. +- RustFS core data-plane logic remains independent of the concrete transport + implementation. ## Current Adapter Surface @@ -30,28 +25,24 @@ The current data-plane surface is byte-stream based: | Remote write stream | `InternodeDataTransport::open_write(...) -> FileWriter` | Callers pass borrowed `&[u8]` slices into boxed `AsyncWrite`; the backend owns any async body staging. | | Walk-dir stream | `InternodeDataTransport::open_walk_dir(...) -> FileReader` | Same boxed stream model as read, with a small serialized request body. | -This API is correct for the current TCP/HTTP backend. A future non-default -backend may have stricter memory ownership, buffer lifetime, or completion -requirements, so the adapter contract needs to describe those boundaries -without assuming a specific implementation. +This API is correct for the current TCP/HTTP backend. The adapter contract +describes current ownership boundaries without assuming implementation details +outside `TcpHttpInternodeDataTransport`. ## Buffer Ownership Model -| Buffer role | Allocator | Lifetime owner | Backend-specific state | TCP/HTTP behavior | +| Buffer role | Allocator | Lifetime owner | Transport state | TCP/HTTP behavior | | --- | --- | --- | --- | --- | -| Send buffer | Caller or RustFS-owned pool | Caller until the backend accepts the buffer; backend until completion if an owned-buffer API is used | Optional backend-managed buffer, staging buffer, or registration handle | Copy into the existing `AsyncWrite` path when the backend cannot use the buffer directly. | -| Receive buffer | Caller-provided storage or backend-owned receive pool | Backend while filling; caller after completion if ownership is returned | Optional backend-owned receive buffer or backend-specific registration | Copy from `AsyncRead` into caller storage as today. | +| Send buffer | Caller or RustFS-owned pool | Caller until the writer copies or accepts bytes for the HTTP body | Existing HTTP body staging | Copy into the existing `AsyncWrite` path when the writer cannot use the borrowed slice directly. | +| Receive buffer | Caller-provided storage | Reader while filling; caller after `poll_read` returns | Existing HTTP response body chunks | Copy from `AsyncRead` into caller storage as today. | | Control metadata | RustFS caller | Caller/request object | Not buffer-managed by the data-plane backend | Serialize into HTTP/gRPC/control-plane messages. | -| Fallback staging | TCP/HTTP backend | TCP/HTTP backend | No backend-specific registration | Existing `HttpReader`/`HttpWriter` buffering semantics. | +| Fallback staging | TCP/HTTP backend | TCP/HTTP backend | Existing `HttpReader`/`HttpWriter` buffers | Existing buffering semantics. | -Backends with stricter memory requirements must not let callers mutate or reuse -a buffer while an async transfer is still in flight. A backend that cannot use a -caller buffer directly must either reject the transfer before payload movement -or copy through a clearly documented backend-owned staging buffer. +The current writer must not retain borrowed caller slices beyond the write call. +When bytes must outlive the call, they are copied into owned HTTP body chunks. -Zero-copy is not guaranteed by this contract. Backends must document whether -their path is zero-copy, copy-reduced, or staging-buffer based, and must -document where copies occur. +This contract does not claim zero-copy behavior. The current TCP/HTTP path +documents where copies occur. ## Compatibility Contract @@ -66,77 +57,35 @@ pub trait InternodeDataTransport { } ``` -Future extensions for backend-managed buffers should be additive and -capability-gated. A possible shape is: - -```rust -pub struct TransferBuffer { - pub bytes: bytes::Bytes, - pub backend_state: Option, -} - -pub struct CompletedTransfer { - pub bytes: bytes::Bytes, - pub transfer_len: usize, -} - -#[async_trait::async_trait] -pub trait BackendBufferInternodeTransport: InternodeDataTransport { - async fn write_backend_buffer(&self, request: WriteStreamRequest, buffer: TransferBuffer) -> Result; - async fn read_backend_buffer(&self, request: ReadStreamRequest, buffer: TransferBuffer) -> Result; -} -``` - -The concrete type of `TransportBufferState` should stay backend-private. The -generic contract only needs to state whether the buffer is usable by the -backend, who owns it during transfer, and when ownership returns. - -This PR does not add this extension. It documents the boundary that a future -external backend may need. - -## Required Contract for Stricter Backends +## Current Adapter Contract | Area | Required contract | | --- | --- | -| Ownership | Define when caller-owned bytes become backend-owned and when ownership returns. | -| Completion | Signal completion before RustFS can reuse or mutate backend-managed memory. | -| Staging | Declare whether the backend copies through backend-owned staging buffers when direct use is unavailable. | -| Size limits | Expose any RustFS-visible `max_transfer_size`. | -| Ordering | Either provide ordered delivery or include a reassembly layer before exposing stream semantics. | -| Copy accounting | Document every known copy boundary and avoid claiming zero-copy unless the path proves it. | - -## Optional Optimizations - -| Area | Optional behavior | -| --- | --- | -| Buffer pooling | RustFS may keep reusable pools for send and receive buffers. | -| Backend state cache | A backend may cache backend-specific handles for long-lived buffers. | -| Scatter/gather | A backend may accept multiple shard slices without repacking when its completion model can report per-slice ownership safely. | -| Backend-owned receive | A backend may return owned receive chunks instead of filling caller-provided buffers. | - -These are optional optimizations, not requirements for the OSS TCP/HTTP path. +| Ownership | Define when caller-owned bytes are copied or accepted by the transport. | +| Completion | Return from stream operations only when bytes are accepted or an error is reported. | +| Staging | Keep staging behavior inside the TCP/HTTP implementation. | +| Size limits | Report any RustFS-visible `max_transfer_size`; TCP/HTTP currently reports none. | +| Ordering | Preserve ordered byte-stream semantics. | +| Copy accounting | Document known copy boundaries and avoid unmeasured zero-copy claims. | ## Current API Limitations -| Current API | Limitation for stricter backends | +| Current API | Current limitation | | --- | --- | -| `FileReader = Box` | `AsyncRead` exposes temporary caller `ReadBuf` storage, not a stable backend-managed receive buffer or explicit completion token. | -| `FileWriter = Box` | `AsyncWrite::poll_write` receives borrowed `&[u8]` that cannot outlive the poll, so async direct transfer requires copying or a different ownership API. | +| `FileReader = Box` | `AsyncRead` exposes temporary caller `ReadBuf` storage. | +| `FileWriter = Box` | `AsyncWrite::poll_write` receives borrowed `&[u8]` that cannot outlive the poll. | | `HttpWriter` | The async HTTP body must own `Bytes`, so borrowed write buffers are copied into `BytesMut` or `Bytes`. | | `write_body_chunks_to_writer` | Server-side HTTP body chunks are copied into `BytesMut` before local disk write. | -| Erasure encode output | Encoded shards are represented as `Vec` and written through `AsyncWrite`, not a completion-aware backend-buffer API. | -| Erasure decode input | Shard reads allocate `Vec` buffers before decode; no backend-owned receive pool is visible at the transport boundary. | +| Erasure encode output | Encoded shards are represented as `Vec` and written through `AsyncWrite`. | +| Erasure decode input | Shard reads allocate `Vec` buffers before decode. | -These limitations do not block the current `tcp-http` backend. They describe -where a future external backend would need staging or an additive API. +These limitations do not block the current `tcp-http` backend. -## External Backend Crate Compatibility +## Adapter Stability -`InternodeDataTransport` should remain implementable by future backends without -modifying RustFS core data-plane logic. In the short term, the trait and -`tcp-http` backend may remain inside `ecstore`. +`InternodeDataTransport` should keep RustFS core data-plane logic separate from +the concrete transport implementation. The trait and `tcp-http` backend remain +inside `ecstore`. -A future external or separately maintained backend could live in a separate -crate if the trait, request/response types, capability report, and error model -are public and stable enough. This PR does not perform a crate split, add -runtime loading, or introduce a plugin system. +This PR does not perform a crate split, add runtime loading, introduce a plugin +system, add a backend value, or implement a new transport backend. diff --git a/crates/ecstore/docs/internode-transport/transport-buffer-lifecycle.md b/crates/ecstore/docs/internode-transport/transport-buffer-lifecycle.md index d42f6e309..d32c74d36 100644 --- a/crates/ecstore/docs/internode-transport/transport-buffer-lifecycle.md +++ b/crates/ecstore/docs/internode-transport/transport-buffer-lifecycle.md @@ -14,17 +14,14 @@ The OSS scope is: - keep `tcp-http` as the default backend; - keep existing TCP/HTTP behavior unchanged; - document copy hotspots and ownership gaps for maintainable transport code; -- avoid hardware-specific dependencies or backend implementations. +- avoid adding dependencies or backend implementations. The OSS scope is not: -- RDMA support; -- DPU support; -- DOCA support; -- BlueField support; -- RoCE/InfiniBand support; -- hardware benchmark planning; -- hardware-specific backend implementation. +- adding another transport backend; +- replacing the current TCP/HTTP path; +- adding benchmark plans for another transport; +- changing object correctness semantics. ## Scope diff --git a/crates/ecstore/docs/internode-transport/transport-capabilities.md b/crates/ecstore/docs/internode-transport/transport-capabilities.md index 1ff161f85..4489a40bd 100644 --- a/crates/ecstore/docs/internode-transport/transport-capabilities.md +++ b/crates/ecstore/docs/internode-transport/transport-capabilities.md @@ -1,8 +1,7 @@ # Internode Transport Capabilities Status: design note for backend-neutral capability reporting. This document -does not add a backend and does not require specialized hardware or transport -crates. +does not add a backend or transport crate. ## Open-source Scope @@ -12,24 +11,20 @@ The OSS scope is: - keep `tcp-http` as the default backend; - keep existing TCP/HTTP behavior unchanged; - document the capability fields needed for maintainable transport code; -- avoid hardware-specific dependencies or backend implementations. +- avoid adding dependencies or backend implementations. The OSS scope is not: -- RDMA support; -- DPU support; -- DOCA support; -- BlueField support; -- RoCE/InfiniBand support; -- hardware benchmark planning; -- hardware-specific backend implementation. +- adding another transport backend; +- replacing the current TCP/HTTP path; +- adding benchmark plans for another transport; +- changing object correctness semantics. ## Purpose `InternodeDataTransportCapabilities` describes what a backend can honestly do -for RustFS internode data-plane transfers. The fields are intentionally neutral: -they can describe the current TCP/HTTP backend and a future non-default backend -without naming a specific transport implementation. +for RustFS internode data-plane transfers. The fields describe observable +adapter behavior without naming a specific transport implementation. The capability report is descriptive. It does not select a backend, negotiate with peers, or weaken object correctness semantics. @@ -41,8 +36,6 @@ with peers, or weaken object correctness semantics. | `streaming_read` | The backend can open a remote disk reader for `read_file_stream`. | | `streaming_write` | The backend can open a remote disk writer for `create_file` or `append_file`. | | `streaming_walk_dir` | The backend can stream `walk_dir` responses. | -| `zero_copy_candidate` | The backend has an API shape that could avoid an extra user-space payload copy. This is not a promise that every transfer is zero-copy. | -| `registered_memory_required` | The backend requires pinned, registered, staged, or otherwise backend-managed buffers before payload transfer. | | `ordered_delivery` | Bytes for each opened transfer are delivered in order. | | `max_transfer_size` | Optional RustFS-level cap for a single transfer. `None` means no additional cap beyond the backend/protocol/runtime limits. | | `fallback_supported` | The backend can participate in the behavior-preserving TCP fallback path. | @@ -56,21 +49,17 @@ The default TCP/HTTP backend reports only capabilities it actually provides: | `streaming_read` | `true` | `HttpReader` streams `/rustfs/rpc/read_file_stream` responses. | | `streaming_write` | `true` | `HttpWriter` streams `/rustfs/rpc/put_file_stream` request bodies. | | `streaming_walk_dir` | `true` | `HttpReader` streams `/rustfs/rpc/walk_dir` responses. | -| `zero_copy_candidate` | `false` | The current path exposes `AsyncRead`/`AsyncWrite` and HTTP body chunks; it copies through normal user-space buffers. | -| `registered_memory_required` | `false` | TCP/HTTP does not require RustFS-managed pinned, registered, or backend-owned buffers. | | `ordered_delivery` | `true` | Each HTTP request body or response body is consumed as an ordered byte stream. | | `max_transfer_size` | `None` | RustFS does not impose an extra per-transfer cap at the capability layer. | | `fallback_supported` | `true` | TCP/HTTP is the behavior-preserving default and fallback path. | -## Non-default Backend Fit +## Capability Compatibility -A future non-default backend can be described without changing the meaning of -the existing TCP report: +Any new capability field should describe observable RustFS behavior without +assuming a specific transport implementation: | Capability shape | Interpretation | | --- | --- | -| `zero_copy_candidate=true`, `registered_memory_required=true` | The backend can only use its lower-copy path with buffers that satisfy backend-specific ownership or registration rules. | -| `zero_copy_candidate=true`, `registered_memory_required=false` | The backend may expose owned chunks or another lower-copy path without requiring caller-managed registration. | | `max_transfer_size=Some(n)` | The backend has a RustFS-visible transfer size ceiling and callers must split larger transfers or use fallback behavior. | | `ordered_delivery=false` | The backend cannot be used behind the current stream API without an ordering or reassembly layer. | diff --git a/crates/ecstore/docs/internode-transport/transport-fallback-and-selection.md b/crates/ecstore/docs/internode-transport/transport-fallback-and-selection.md index 0123b4110..cb686f985 100644 --- a/crates/ecstore/docs/internode-transport/transport-fallback-and-selection.md +++ b/crates/ecstore/docs/internode-transport/transport-fallback-and-selection.md @@ -10,17 +10,11 @@ does not change production behavior. The open-source RustFS path keeps `tcp-http` as the default internode data transport. This document defines adapter contracts only: -- no production RDMA, DPU, DOCA, BlueField, DPDK, SPDK, or hardware - acceleration backend is introduced; -- no hardware SDK, `libibverbs`, `rdma-core`, or vendor dependency is added; +- no additional production backend is introduced; +- no dependency is added; - no new accepted production backend value is added; -- future external or separately maintained backends may implement the same - adapter boundary without changing RustFS core data-plane logic. - -Examples of possible future external backends include DOCA/BlueField, -RDMA/RoCE, or other DPU/NIC implementations. These are examples only and are -not implemented or scheduled by this design. Hardware-specific backend plans -are out of scope for this document. +- RustFS core data-plane logic remains independent of the concrete transport + implementation. ## Static Backend Selection @@ -34,7 +28,7 @@ Static config is the first selection model. Existing accepted values remain: | any unsupported value | Fail closed with a diagnostic naming `RUSTFS_INTERNODE_DATA_TRANSPORT` and the invalid value. | Unknown backend values must fail closed. Unsupported backend values must fail -closed. A future external backend must be explicitly enabled and must not +closed. Any additional backend value must be explicitly added and must not silently replace `tcp-http`. Backend selection must expose an observable backend identity for metrics, logs, @@ -50,7 +44,7 @@ payload. | --- | --- | --- | --- | | Unsupported configured backend | Fail closed during transport construction. | Fall back only when a separately configured policy explicitly allows unsupported-backend fallback. | Error includes config key and invalid value; fallback event is counted when fallback is enabled. | | Peer does not support selected backend | Fail before payload transfer. | Use TCP/HTTP only when both local policy and peer policy allow it. | Count peer mismatch and selected fallback backend. | -| Capability mismatch | Fail before payload transfer. | Use TCP/HTTP only if TCP satisfies the operation and policy allows fallback. | Record missing capability names or a low-cardinality reason. | +| Capability mismatch | Fail before payload transfer. | Use TCP/HTTP only if it satisfies the operation and policy allows fallback. | Record missing capability names or a low-cardinality reason. | | Connection setup failure | Fail the operation. | Retry on TCP/HTTP only when fallback is allowed and no payload bytes have transferred. | Count setup failure, retry, fallback backend, and fallback result. | | Partial transfer failure | Fail the operation and let existing object/quorum logic decide retry behavior. | Do not silently resume on another backend unless the transfer protocol can prove byte range, checksum, and idempotency boundaries. | Count partial failure with bytes completed. | | Max transfer size exceeded | Fail before payload transfer or split at a higher layer. | Use TCP/HTTP if policy allows and TCP has no RustFS-level cap. | Record rejected size and selected backend. | @@ -61,15 +55,16 @@ uses them. A backend must define failure behavior before production use. ## Dynamic Negotiation Boundary -Dynamic negotiation, if added, belongs on the existing gRPC control plane. Data -transfer must start only after both peers agree on: +Dynamic negotiation is not implemented by this PR. If it is added later, it +belongs on the existing gRPC control plane. Data transfer must start only after +both peers agree on: | Negotiated item | Required property | | --- | --- | | Backend name | Both peers know the backend and have it enabled. | | Capability set | Required capabilities match the operation. | | Max transfer size | The selected operation fits or is split before transfer starts. | -| Buffer rules | Both peers agree whether backend-managed or staged buffers are required. | +| Buffer rules | Both peers agree on staging and ownership rules. | | Completion semantics | Both peers agree when a transfer is considered complete and when buffers may be reused. | | Security mode | Authentication and encryption requirements are satisfied before any out-of-band transfer. | | Fallback policy | Both peers agree whether TCP/HTTP fallback is allowed for this operation. | @@ -95,12 +90,10 @@ health, quorum, timeout, and integrity semantics. - Backend selection must preserve peer authentication. - Fallback must not weaken encryption or authorization. -- Out-of-band data-plane transfers must still bind to the intended disk, - volume, path, request authority, and operation. - Partial transfers must not bypass bitrot verification or erasure quorum handling. -- Any future external backend must document whether it relies on the same - security boundary as TCP/HTTP or requires a separate deployment boundary. +- Any adapter implementation must preserve the same request authority, disk, + volume, path, and operation binding as the current TCP/HTTP path. ## Metrics and Observability Requirements @@ -117,9 +110,9 @@ Metrics and logs must use low-cardinality labels or metadata: - capability mismatch count; - fallback decision count. -Backends must not add high-cardinality labels such as object names, full paths, -full URLs, peer-specific dynamic strings, memory addresses, or backend-specific -buffer keys. +Adapter implementations must not add high-cardinality labels such as object +names, full paths, full URLs, peer-specific dynamic strings, memory addresses, +or buffer identifiers. ## TCP/HTTP Compatibility @@ -128,17 +121,15 @@ uses ordinary byte streams, does not require backend-specific buffer registration, and remains suitable as the fallback path when an explicit fallback policy exists. -A future non-default backend must not change the correctness semantics of +Any adapter implementation must not change the correctness semantics of object writes, object reads, healing, bitrot verification, erasure quorum, timeouts, or disk health handling. -## External Backend Crate Compatibility +## Adapter Stability -`InternodeDataTransport` should remain implementable by future backends without -modifying RustFS core data-plane logic. In the short term, the trait and -`tcp-http` backend may remain inside `ecstore`. +`InternodeDataTransport` should keep RustFS core data-plane logic separate from +the concrete transport implementation. The trait and `tcp-http` backend remain +inside `ecstore`. -A future external or separately maintained backend could live in a separate -crate if the trait, request/response types, capability report, and error model -are public and stable enough. This PR does not perform a crate split, add -runtime loading, or introduce a plugin system. +This PR does not perform a crate split, add runtime loading, introduce a plugin +system, add a backend value, or implement a new transport backend. diff --git a/crates/ecstore/src/rpc/internode_data_transport.rs b/crates/ecstore/src/rpc/internode_data_transport.rs index d1001af59..bc61d89f8 100644 --- a/crates/ecstore/src/rpc/internode_data_transport.rs +++ b/crates/ecstore/src/rpc/internode_data_transport.rs @@ -47,10 +47,6 @@ pub struct InternodeDataTransportCapabilities { pub streaming_write: bool, /// Backend can stream walk-dir responses. pub streaming_walk_dir: bool, - /// Backend has an API shape that could avoid an extra user-space copy. - pub zero_copy_candidate: bool, - /// Backend cannot transfer payloads unless buffers are registered or pinned. - pub registered_memory_required: bool, /// Backend preserves in-order delivery for each opened transfer. pub ordered_delivery: bool, /// Largest payload the backend accepts for one transfer, or no RustFS-level cap. @@ -65,8 +61,6 @@ impl InternodeDataTransportCapabilities { streaming_read: true, streaming_write: true, streaming_walk_dir: true, - zero_copy_candidate: false, - registered_memory_required: false, ordered_delivery: true, max_transfer_size: None, fallback_supported: true, @@ -241,8 +235,6 @@ mod tests { streaming_read: true, streaming_write: true, streaming_walk_dir: true, - zero_copy_candidate: false, - registered_memory_required: false, ordered_delivery: true, max_transfer_size: None, fallback_supported: true, @@ -251,11 +243,9 @@ mod tests { } #[test] - fn tcp_http_capabilities_do_not_advertise_stricter_backend_features() { + fn tcp_http_capabilities_are_conservative() { let capabilities = TcpHttpInternodeDataTransport.capabilities(); - assert!(!capabilities.zero_copy_candidate); - assert!(!capabilities.registered_memory_required); assert!(capabilities.ordered_delivery); assert_eq!(capabilities.max_transfer_size, None); assert!(capabilities.fallback_supported);