feat(rpc): add replay-scoped internode authentication (#5455)

rustfs/src/server/http.rs

@@ -37,7 +37,8 @@ use crate::storage_api::server::http as storage;
 use crate::storage_api::server::http::rpc::InternodeRpcService;
 use crate::storage_api::server::http::tonic_service::make_server;
 use crate::storage_api::server::http::{
-    ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, verify_tonic_rpc_signature,
+    ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge,
+    tonic_boot_epoch_response_headers, verify_tonic_rpc_signature_with_bootstrap,
 };
 use bytes::Bytes;
 use http::{HeaderMap, Method, Request as HttpRequest, Response, Uri};
@@ -152,13 +153,17 @@ impl<S> RpcRequestPathService<S> {
     }
 }
 
-impl<S, B> Service<HttpRequest<B>> for RpcRequestPathService<S>
+impl<S, B, ResBody> Service<HttpRequest<B>> for RpcRequestPathService<S>
 where
-    S: Service<HttpRequest<B>>,
+    S: Service<HttpRequest<B>, Response = Response<ResBody>>,
+    S::Error: Send + 'static,
+    S::Future: Send + 'static,
+    B: Send + 'static,
+    ResBody: Send + 'static,
 {
-    type Response = S::Response;
+    type Response = Response<ResBody>;
     type Error = S::Error;
-    type Future = S::Future;
+    type Future = Pin<Box<dyn Future<Output = std::result::Result<Self::Response, Self::Error>> + Send>>;
 
     fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
         self.inner.poll_ready(cx)
@@ -170,7 +175,22 @@ where
             method: req.method().clone(),
         };
         req.extensions_mut().insert(target);
-        self.inner.call(req)
+        let response_headers = tonic_boot_epoch_challenge(req.headers())
+            .ok()
+            .flatten()
+            .and_then(|challenge| {
+                storage::try_current_local_node_name()
+                    .and_then(|node| normalize_tonic_rpc_audience(&node).ok())
+                    .and_then(|audience| tonic_boot_epoch_response_headers(&audience, challenge).ok())
+            });
+        let future = self.inner.call(req);
+        Box::pin(async move {
+            let mut response = future.await?;
+            if let Some(headers) = response_headers {
+                response.headers_mut().extend(headers);
+            }
+            Ok(response)
+        })
     }
 }
 
@@ -1862,7 +1882,15 @@ fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
         .filter(|method| !method.is_empty() && !method.contains('/'))
         .ok_or_else(|| Status::unauthenticated("Invalid RPC request path"))?;
     debug_assert!(!rpc_method.is_empty());
-    verify_tonic_rpc_signature(&audience, target.uri.path(), req.metadata().as_ref()).map_err(|e| {
+    let allow_replay_scope_bootstrap = target.uri.path() == "/node_service.NodeService/Ping"
+        && tonic_boot_epoch_challenge(req.metadata().as_ref()).is_ok_and(|challenge| challenge.is_some());
+    verify_tonic_rpc_signature_with_bootstrap(
+        &audience,
+        target.uri.path(),
+        req.metadata().as_ref(),
+        allow_replay_scope_bootstrap,
+    )
+    .map_err(|e| {
         error!(
             event = EVENT_RPC_SIGNATURE_VERIFICATION_FAILED,
             component = LOG_COMPONENT_SERVER,

rustfs/src/storage/storage_api.rs

@@ -495,8 +495,9 @@ pub(crate) mod ecstore_rpc {
     pub(crate) use rustfs_ecstore::api::rpc::{
         LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
         SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience,
-        sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest,
-        verify_tonic_mutation_body_digest, verify_tonic_rpc_signature,
+        sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
+        verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
+        verify_tonic_rpc_signature_with_bootstrap,
     };
     #[cfg(test)]
     pub(crate) use rustfs_ecstore::api::rpc::{
@@ -1605,8 +1606,21 @@ pub(crate) fn sign_ns_scanner_capability(challenge: uuid::Uuid, server_epoch: uu
     ecstore_rpc::sign_ns_scanner_capability(challenge, server_epoch)
 }
 
-pub(crate) fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &http::HeaderMap) -> std::io::Result<()> {
-    ecstore_rpc::verify_tonic_rpc_signature(audience, path, headers)
+pub(crate) fn verify_tonic_rpc_signature_with_bootstrap(
+    audience: &str,
+    path: &str,
+    headers: &http::HeaderMap,
+    allow_replay_scope_bootstrap: bool,
+) -> std::io::Result<()> {
+    ecstore_rpc::verify_tonic_rpc_signature_with_bootstrap(audience, path, headers, allow_replay_scope_bootstrap)
+}
+
+pub(crate) fn tonic_boot_epoch_challenge(headers: &http::HeaderMap) -> std::io::Result<Option<uuid::Uuid>> {
+    ecstore_rpc::tonic_boot_epoch_challenge(headers)
+}
+
+pub(crate) fn tonic_boot_epoch_response_headers(audience: &str, challenge: uuid::Uuid) -> std::io::Result<http::HeaderMap> {
+    ecstore_rpc::tonic_boot_epoch_response_headers(audience, challenge)
 }
 
 pub(crate) fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {

rustfs/src/storage_api.rs

@@ -97,7 +97,8 @@ pub(crate) mod server {
 
     pub(crate) mod http {
         pub(crate) use crate::storage::storage_api::{
-            ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, verify_tonic_rpc_signature,
+            ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge,
+            tonic_boot_epoch_response_headers, verify_tonic_rpc_signature_with_bootstrap,
         };
 
         pub(crate) fn try_current_local_node_name() -> Option<String> {