mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
fix(rpc): bind internode auth to exact targets (#4988)
This commit is contained in:
@@ -54,6 +54,15 @@ pub async fn get_global_local_node_name() -> String {
|
||||
GLOBAL_LOCAL_NODE_NAME.read().await.clone()
|
||||
}
|
||||
|
||||
/// Read the local node name without waiting for initialization or a writer.
|
||||
pub fn try_get_global_local_node_name() -> Option<String> {
|
||||
GLOBAL_LOCAL_NODE_NAME
|
||||
.try_read()
|
||||
.ok()
|
||||
.map(|name| name.clone())
|
||||
.filter(|name| !name.is_empty())
|
||||
}
|
||||
|
||||
/// Set the global RustFS initialization time to the current UTC time.
|
||||
pub async fn set_global_init_time_now() {
|
||||
let now = Utc::now();
|
||||
|
||||
@@ -382,8 +382,10 @@ pub mod rio {
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, verify_rpc_signature,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
|
||||
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -12,10 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::{TONIC_RPC_PREFIX, gen_signature_headers};
|
||||
use crate::cluster::rpc::http_auth::RPC_CONTENT_SHA256_HEADER;
|
||||
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
|
||||
use crate::disk::error::{DiskError, Error as DiskErrorType};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use http::Method;
|
||||
use http::Uri;
|
||||
use rustfs_protos::{
|
||||
ChannelClass, create_new_channel, get_channel_for_class, proto_gen::node_service::node_service_client::NodeServiceClient,
|
||||
};
|
||||
@@ -47,6 +48,7 @@ pub async fn node_service_time_out_client_for_class(
|
||||
interceptor: TonicInterceptor,
|
||||
class: ChannelClass,
|
||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match class {
|
||||
ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
|
||||
Some(channel) => {
|
||||
@@ -111,11 +113,25 @@ pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TonicSignatureInterceptor;
|
||||
pub struct TonicSignatureInterceptor {
|
||||
audience: Option<String>,
|
||||
}
|
||||
|
||||
impl tonic::service::Interceptor for TonicSignatureInterceptor {
|
||||
fn call(&mut self, mut req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
|
||||
let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET)
|
||||
let method = req
|
||||
.extensions()
|
||||
.get::<tonic::GrpcMethod<'_>>()
|
||||
.ok_or_else(|| tonic::Status::unauthenticated("Missing gRPC method metadata"))?;
|
||||
let audience = self
|
||||
.audience
|
||||
.as_deref()
|
||||
.ok_or_else(|| tonic::Status::unauthenticated("Missing gRPC audience"))?;
|
||||
let content_sha256 = req
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let headers = gen_tonic_signature_headers(audience, method.service(), method.method(), content_sha256)
|
||||
.map_err(|_| tonic::Status::unauthenticated("No valid auth token"))?;
|
||||
req.metadata_mut().as_mut().extend(headers);
|
||||
inject_trace_context_into_metadata(req.metadata_mut());
|
||||
@@ -125,7 +141,7 @@ impl tonic::service::Interceptor for TonicSignatureInterceptor {
|
||||
}
|
||||
|
||||
pub fn gen_tonic_signature_interceptor() -> TonicSignatureInterceptor {
|
||||
TonicSignatureInterceptor
|
||||
TonicSignatureInterceptor { audience: None }
|
||||
}
|
||||
|
||||
pub struct NoOpInterceptor;
|
||||
@@ -141,6 +157,22 @@ pub enum TonicInterceptor {
|
||||
NoOp(NoOpInterceptor),
|
||||
}
|
||||
|
||||
impl TonicInterceptor {
|
||||
fn with_rpc_audience(mut self, addr: &str) -> std::io::Result<Self> {
|
||||
if let Self::Signature(interceptor) = &mut self {
|
||||
let uri = addr
|
||||
.parse::<Uri>()
|
||||
.map_err(|_| std::io::Error::other("Invalid gRPC peer URI"))?;
|
||||
let audience = uri
|
||||
.authority()
|
||||
.map(|authority| normalize_tonic_rpc_audience(authority.as_str()))
|
||||
.ok_or_else(|| std::io::Error::other("Missing gRPC peer authority"))?;
|
||||
interceptor.audience = Some(audience?);
|
||||
}
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl tonic::service::Interceptor for TonicInterceptor {
|
||||
fn call(&mut self, req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
|
||||
match self {
|
||||
@@ -165,6 +197,20 @@ mod tests {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
}
|
||||
|
||||
fn test_request() -> tonic::Request<()> {
|
||||
let mut request = tonic::Request::new(());
|
||||
request
|
||||
.extensions_mut()
|
||||
.insert(tonic::GrpcMethod::new("node_service.NodeService", "Ping"));
|
||||
request
|
||||
}
|
||||
|
||||
fn test_interceptor() -> TonicSignatureInterceptor {
|
||||
TonicSignatureInterceptor {
|
||||
audience: Some("node-a:9000".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_trace_parent<F>(trace_id_hex: &str, f: F)
|
||||
where
|
||||
F: FnOnce(),
|
||||
@@ -192,20 +238,55 @@ mod tests {
|
||||
#[test]
|
||||
fn test_signature_interceptor_keeps_auth_headers() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut interceptor = TonicSignatureInterceptor;
|
||||
let req = tonic::Request::new(());
|
||||
let mut interceptor = test_interceptor();
|
||||
let req = test_request();
|
||||
|
||||
let req = interceptor.call(req).expect("interceptor call should succeed");
|
||||
|
||||
assert!(req.metadata().contains_key("x-rustfs-signature"));
|
||||
assert!(req.metadata().contains_key("x-rustfs-timestamp"));
|
||||
assert!(req.metadata().contains_key("x-rustfs-rpc-signature-v2"));
|
||||
assert!(req.metadata().contains_key("x-rustfs-rpc-nonce"));
|
||||
assert!(
|
||||
crate::cluster::rpc::verify_tonic_rpc_signature(
|
||||
"node-a:9000",
|
||||
"/node_service.NodeService/Ping",
|
||||
req.metadata().as_ref(),
|
||||
)
|
||||
.is_ok(),
|
||||
"interceptor signature should bind the configured peer audience and generated method"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_signature_interceptor_binds_audience_from_peer_uri() {
|
||||
let interceptor = TonicInterceptor::Signature(gen_tonic_signature_interceptor())
|
||||
.with_rpc_audience("http://node-a:9000")
|
||||
.expect("peer URI should provide an audience");
|
||||
let TonicInterceptor::Signature(interceptor) = interceptor else {
|
||||
panic!("signature interceptor variant should be preserved");
|
||||
};
|
||||
|
||||
assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_signature_interceptor_requires_generated_method_metadata() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut interceptor = test_interceptor();
|
||||
let error = interceptor
|
||||
.call(tonic::Request::new(()))
|
||||
.expect_err("requests without an exact generated method must fail closed");
|
||||
|
||||
assert_eq!(error.code(), tonic::Code::Unauthenticated);
|
||||
assert_eq!(error.message(), "Missing gRPC method metadata");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_signature_interceptor_may_inject_request_id() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut interceptor = TonicSignatureInterceptor;
|
||||
let req = tonic::Request::new(());
|
||||
let mut interceptor = test_interceptor();
|
||||
let req = test_request();
|
||||
|
||||
let span = tracing::info_span!("grpc-rpc-test-span");
|
||||
let _guard = span.enter();
|
||||
@@ -219,8 +300,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_signature_interceptor_injects_traceparent_metadata() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut interceptor = TonicSignatureInterceptor;
|
||||
let req = tonic::Request::new(());
|
||||
let mut interceptor = test_interceptor();
|
||||
let req = test_request();
|
||||
|
||||
with_trace_parent("4bf92f3577b34da6a3ce929d0e0e4736", || {
|
||||
let req = interceptor.call(req).expect("interceptor call should succeed");
|
||||
|
||||
@@ -19,8 +19,9 @@
|
||||
//! GHSA-r5qv-rc46-hv8q (internode RPC authentication must fail closed, fixed in
|
||||
//! rustfs/rustfs#4402) is anchored by the `ghsa_r5qv_*` tests in the module
|
||||
//! below, plus the broader negative-signature suite. The advisory class is: a
|
||||
//! node must never accept an RPC whose auth is missing, malformed, replayed, or
|
||||
//! signed with the default/empty shared secret. See
|
||||
//! node must never accept an RPC whose auth is missing, malformed, or signed
|
||||
//! with the default/empty shared secret. Body-bound v2 requests additionally
|
||||
//! receive process-local replay protection. See
|
||||
//! `docs/testing/security-regressions.md` for the full advisory -> test map.
|
||||
//!
|
||||
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
|
||||
@@ -29,23 +30,89 @@ use crate::cluster::rpc::context_propagation::{inject_request_id_into_http_heade
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose;
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use http::uri::Authority;
|
||||
use http::{HeaderMap, HeaderValue, Method, Uri};
|
||||
#[cfg(test)]
|
||||
use rustfs_credentials::{DEFAULT_SECRET_KEY, RPC_SECRET_REQUIRED_MESSAGE};
|
||||
use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token};
|
||||
use sha2::Digest as _;
|
||||
use sha2::Sha256;
|
||||
use std::sync::Once;
|
||||
use std::collections::{HashSet, VecDeque};
|
||||
use std::sync::{LazyLock, Mutex, Once};
|
||||
use std::time::{Duration, Instant};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::error;
|
||||
use uuid::Uuid;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||
const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
|
||||
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
|
||||
const RPC_NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
|
||||
pub(crate) const RPC_CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
|
||||
const RPC_AUTH_VERSION_V2: &str = "2";
|
||||
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
|
||||
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
|
||||
const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
|
||||
const MAX_REPLAY_PROTECTED_NONCES: usize = 65_536;
|
||||
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
|
||||
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
|
||||
|
||||
#[derive(Default)]
|
||||
struct RpcNonceCache {
|
||||
nonces: HashSet<Uuid>,
|
||||
expirations: VecDeque<(Instant, i64, Uuid)>,
|
||||
max_wall_time: i64,
|
||||
}
|
||||
|
||||
impl RpcNonceCache {
|
||||
fn remove_expired(&mut self, now: Instant, wall_time: i64) {
|
||||
while matches!(
|
||||
self.expirations.front(),
|
||||
Some((expires_at, valid_until, _)) if *expires_at < now && *valid_until < wall_time
|
||||
) {
|
||||
let Some((_, _, nonce)) = self.expirations.pop_front() else {
|
||||
break;
|
||||
};
|
||||
self.nonces.remove(&nonce);
|
||||
}
|
||||
}
|
||||
|
||||
fn check_and_record(
|
||||
&mut self,
|
||||
nonce: Uuid,
|
||||
signed_at: i64,
|
||||
now: Instant,
|
||||
wall_time: i64,
|
||||
expires_at: Instant,
|
||||
capacity: usize,
|
||||
) -> std::io::Result<()> {
|
||||
self.max_wall_time = self.max_wall_time.max(wall_time);
|
||||
if self.max_wall_time.saturating_sub(signed_at) > SIGNATURE_VALID_DURATION {
|
||||
return Err(std::io::Error::other("RPC request timestamp expired after clock regression"));
|
||||
}
|
||||
self.remove_expired(now, self.max_wall_time);
|
||||
if self.nonces.contains(&nonce) {
|
||||
return Err(std::io::Error::other("RPC request replay detected"));
|
||||
}
|
||||
if self.nonces.len() >= capacity {
|
||||
return Err(std::io::Error::other("RPC replay cache capacity exceeded"));
|
||||
}
|
||||
self.nonces.insert(nonce);
|
||||
self.expirations
|
||||
.push_back((expires_at, signed_at.saturating_add(SIGNATURE_VALID_DURATION), nonce));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// This cache is a process-local wire-replay defense only. Mutation handlers
|
||||
// still need a stable operation ID and coordinator-owned idempotency across
|
||||
// retries, node failover, and restart.
|
||||
static LOCAL_RPC_NONCE_CACHE: LazyLock<Mutex<RpcNonceCache>> = LazyLock::new(|| Mutex::new(RpcNonceCache::default()));
|
||||
|
||||
/// Get the shared secret for HMAC signing
|
||||
#[cfg(test)]
|
||||
fn resolve_shared_secret(env_secret: Option<&str>, global_secret: Option<&str>) -> std::io::Result<String> {
|
||||
@@ -109,6 +176,96 @@ fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, si
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct SignatureV2Scope<'a> {
|
||||
audience: &'a str,
|
||||
service: &'a str,
|
||||
rpc_method: &'a str,
|
||||
timestamp: &'a str,
|
||||
nonce: &'a str,
|
||||
content_sha256: &'a str,
|
||||
}
|
||||
|
||||
fn update_signature_v2(mac: &mut HmacSha256, scope: SignatureV2Scope<'_>) {
|
||||
for part in [
|
||||
b"rustfs-rpc-auth-v2|".as_slice(),
|
||||
scope.audience.as_bytes(),
|
||||
b"|/",
|
||||
scope.service.as_bytes(),
|
||||
b"/",
|
||||
scope.rpc_method.as_bytes(),
|
||||
b"|POST|",
|
||||
scope.timestamp.as_bytes(),
|
||||
b"|",
|
||||
scope.nonce.as_bytes(),
|
||||
b"|",
|
||||
scope.content_sha256.as_bytes(),
|
||||
] {
|
||||
mac.update(part);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_signature_v2(secret: &str, scope: SignatureV2Scope<'_>) -> std::io::Result<String> {
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_signature_v2(&mut mac, scope);
|
||||
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &str) -> bool {
|
||||
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
|
||||
return false;
|
||||
};
|
||||
update_signature_v2(&mut mac, scope);
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
|
||||
fn valid_content_sha256(value: &str) -> bool {
|
||||
value == UNSIGNED_PAYLOAD
|
||||
|| (value.len() == 64
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)))
|
||||
}
|
||||
|
||||
fn header_value(value: &str, name: &str) -> std::io::Result<HeaderValue> {
|
||||
HeaderValue::from_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name} header value")))
|
||||
}
|
||||
|
||||
pub fn normalize_tonic_rpc_audience(value: &str) -> std::io::Result<String> {
|
||||
let authority = value
|
||||
.parse::<Authority>()
|
||||
.map_err(|_| std::io::Error::other("Invalid gRPC peer authority"))?;
|
||||
Ok(authority.as_str().to_ascii_lowercase())
|
||||
}
|
||||
|
||||
fn check_timestamp(timestamp: i64) -> std::io::Result<()> {
|
||||
let current_time = OffsetDateTime::now_utc().unix_timestamp();
|
||||
if current_time.saturating_sub(timestamp) > SIGNATURE_VALID_DURATION
|
||||
|| timestamp.saturating_sub(current_time) > SIGNATURE_VALID_DURATION
|
||||
{
|
||||
return Err(std::io::Error::other("Request timestamp expired"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_and_record_nonce(nonce: Uuid, signed_at: i64) -> std::io::Result<()> {
|
||||
let wall_time = OffsetDateTime::now_utc().unix_timestamp();
|
||||
let mut cache = LOCAL_RPC_NONCE_CACHE
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("RPC replay cache unavailable"))?;
|
||||
// Take the monotonic timestamp after acquiring the lock so expiration
|
||||
// entries remain ordered by the same serialization point as insertion.
|
||||
let now = Instant::now();
|
||||
let expires_at = now
|
||||
.checked_add(REPLAY_CACHE_RETENTION)
|
||||
.ok_or_else(|| std::io::Error::other("RPC replay expiry overflow"))?;
|
||||
cache.check_and_record(nonce, signed_at, now, wall_time, expires_at, MAX_REPLAY_PROTECTED_NONCES)
|
||||
}
|
||||
|
||||
/// Build headers with authentication signature
|
||||
pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) -> std::io::Result<()> {
|
||||
let auth_headers = gen_signature_headers(url, method)?;
|
||||
@@ -135,6 +292,184 @@ pub fn gen_signature_headers(url: &str, method: &Method) -> std::io::Result<Head
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Generate rolling-upgrade-safe gRPC auth metadata.
|
||||
///
|
||||
/// The legacy signature remains present for old servers. New servers prefer the
|
||||
/// v2 signature and bind it to the destination authority and exact generated
|
||||
/// gRPC method. A versioned canonical mutation payload can opt into the
|
||||
/// additional body-digest capability.
|
||||
pub fn gen_tonic_signature_headers(
|
||||
audience: &str,
|
||||
service: &str,
|
||||
rpc_method: &str,
|
||||
content_sha256: Option<&str>,
|
||||
) -> std::io::Result<HeaderMap> {
|
||||
if audience.is_empty() || service.is_empty() || rpc_method.is_empty() || service.contains('/') || rpc_method.contains('/') {
|
||||
return Err(std::io::Error::other("Invalid RPC v2 signing scope"));
|
||||
}
|
||||
let content_sha256 = content_sha256.unwrap_or(UNSIGNED_PAYLOAD);
|
||||
if !valid_content_sha256(content_sha256) {
|
||||
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
|
||||
}
|
||||
|
||||
let secret = get_shared_secret()?;
|
||||
let timestamp = OffsetDateTime::now_utc().unix_timestamp();
|
||||
let timestamp_header = timestamp.to_string();
|
||||
let body_nonce = (content_sha256 != UNSIGNED_PAYLOAD).then(|| Uuid::new_v4().to_string());
|
||||
let nonce = body_nonce.as_deref().unwrap_or(UNSIGNED_PAYLOAD_NONCE);
|
||||
let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp);
|
||||
let signature_v2 = generate_signature_v2(
|
||||
&secret,
|
||||
SignatureV2Scope {
|
||||
audience,
|
||||
service,
|
||||
rpc_method,
|
||||
timestamp: ×tamp_header,
|
||||
nonce,
|
||||
content_sha256,
|
||||
},
|
||||
)?;
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(SIGNATURE_HEADER, header_value(&legacy_signature, SIGNATURE_HEADER)?);
|
||||
headers.insert(TIMESTAMP_HEADER, header_value(×tamp_header, TIMESTAMP_HEADER)?);
|
||||
headers.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
|
||||
headers.insert(RPC_SIGNATURE_V2_HEADER, header_value(&signature_v2, RPC_SIGNATURE_V2_HEADER)?);
|
||||
headers.insert(RPC_NONCE_HEADER, header_value(nonce, RPC_NONCE_HEADER)?);
|
||||
headers.insert(RPC_CONTENT_SHA256_HEADER, header_value(content_sha256, RPC_CONTENT_SHA256_HEADER)?);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Bind a mutation to a versioned, deterministic canonical payload.
|
||||
///
|
||||
/// Do not pass a protobuf re-encoding here: unknown fields and map ordering are
|
||||
/// not a stable mixed-version contract.
|
||||
pub fn set_tonic_canonical_body_digest<T>(request: &mut tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
let digest = hex_simd::encode_to_string(Sha256::digest(canonical_body), hex_simd::AsciiCase::Lower);
|
||||
request
|
||||
.metadata_mut()
|
||||
.as_mut()
|
||||
.insert(RPC_CONTENT_SHA256_HEADER, header_value(&digest, RPC_CONTENT_SHA256_HEADER)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
let version = request
|
||||
.metadata()
|
||||
.get(RPC_AUTH_VERSION_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
if version != Some(RPC_AUTH_VERSION_V2) {
|
||||
return Err(std::io::Error::other("RPC mutation requires v2 authentication"));
|
||||
}
|
||||
let expected = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
|
||||
if expected == UNSIGNED_PAYLOAD || !valid_content_sha256(expected) {
|
||||
return Err(std::io::Error::other("RPC body is not bound to the signature"));
|
||||
}
|
||||
let actual = hex_simd::encode_to_string(Sha256::digest(canonical_body), hex_simd::AsciiCase::Lower);
|
||||
if actual != expected {
|
||||
return Err(std::io::Error::other("RPC content SHA-256 mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
|
||||
[
|
||||
RPC_AUTH_VERSION_HEADER,
|
||||
RPC_SIGNATURE_V2_HEADER,
|
||||
RPC_NONCE_HEADER,
|
||||
RPC_CONTENT_SHA256_HEADER,
|
||||
]
|
||||
.iter()
|
||||
.any(|name| headers.contains_key(*name))
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
|
||||
if !has_v2_auth_headers(headers) {
|
||||
// RUSTFS_COMPAT_TODO(heal-rpc-auth-v2): accept old peers during rolling upgrades. Remove after the minimum
|
||||
// supported RustFS peer version sends v2 authentication on every internode gRPC request.
|
||||
return verify_rpc_signature(TONIC_RPC_PREFIX, &Method::GET, headers);
|
||||
}
|
||||
|
||||
let path = path
|
||||
.strip_prefix('/')
|
||||
.ok_or_else(|| std::io::Error::other("Invalid RPC request path"))?;
|
||||
let (service, rpc_method) = path
|
||||
.split_once('/')
|
||||
.filter(|(service, rpc_method)| !service.is_empty() && !rpc_method.is_empty() && !rpc_method.contains('/'))
|
||||
.ok_or_else(|| std::io::Error::other("Invalid RPC request path"))?;
|
||||
if audience.is_empty() {
|
||||
return Err(std::io::Error::other("Missing RPC audience"));
|
||||
}
|
||||
|
||||
let version = headers
|
||||
.get(RPC_AUTH_VERSION_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC auth version"))?;
|
||||
if version != RPC_AUTH_VERSION_V2 {
|
||||
return Err(std::io::Error::other("Unsupported RPC auth version"));
|
||||
}
|
||||
let signature = headers
|
||||
.get(RPC_SIGNATURE_V2_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC v2 signature"))?;
|
||||
let timestamp_header = headers
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
|
||||
let timestamp = timestamp_header
|
||||
.parse::<i64>()
|
||||
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||
check_timestamp(timestamp)?;
|
||||
let nonce = headers
|
||||
.get(RPC_NONCE_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC nonce"))?;
|
||||
let content_sha256 = headers
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
|
||||
if !valid_content_sha256(content_sha256) {
|
||||
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
|
||||
}
|
||||
let parsed_nonce = if content_sha256 == UNSIGNED_PAYLOAD {
|
||||
if nonce != UNSIGNED_PAYLOAD_NONCE {
|
||||
return Err(std::io::Error::other("Invalid unsigned RPC nonce"));
|
||||
}
|
||||
None
|
||||
} else {
|
||||
let parsed_nonce = Uuid::parse_str(nonce).map_err(|_| std::io::Error::other("Invalid RPC nonce"))?;
|
||||
if parsed_nonce.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid RPC nonce"));
|
||||
}
|
||||
Some(parsed_nonce)
|
||||
};
|
||||
|
||||
let secret = get_shared_secret()?;
|
||||
if !verify_signature_v2(
|
||||
&secret,
|
||||
SignatureV2Scope {
|
||||
audience,
|
||||
service,
|
||||
rpc_method,
|
||||
timestamp: timestamp_header,
|
||||
nonce,
|
||||
content_sha256,
|
||||
},
|
||||
signature,
|
||||
) {
|
||||
return Err(std::io::Error::other("Invalid RPC v2 signature"));
|
||||
}
|
||||
if let Some(nonce) = parsed_nonce {
|
||||
check_and_record_nonce(nonce, timestamp)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify the request signature for RPC requests
|
||||
pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) -> std::io::Result<()> {
|
||||
// Get signature from header
|
||||
@@ -153,14 +488,7 @@ pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) ->
|
||||
.parse()
|
||||
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||
|
||||
// Check timestamp validity (prevent replay attacks)
|
||||
let current_time = OffsetDateTime::now_utc().unix_timestamp();
|
||||
|
||||
if current_time.saturating_sub(timestamp) > SIGNATURE_VALID_DURATION
|
||||
|| timestamp.saturating_sub(current_time) > SIGNATURE_VALID_DURATION
|
||||
{
|
||||
return Err(std::io::Error::other("Request timestamp expired"));
|
||||
}
|
||||
check_timestamp(timestamp)?;
|
||||
|
||||
// Verify signature with constant-time HMAC comparison.
|
||||
let secret = get_shared_secret()?;
|
||||
@@ -707,4 +1035,141 @@ mod tests {
|
||||
assert!(result.is_ok(), "Round-trip test failed for {method} {url}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_v2_signature_is_bound_to_exact_method() {
|
||||
ensure_test_rpc_secret();
|
||||
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("tonic auth headers should build");
|
||||
|
||||
assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers).is_ok());
|
||||
let error = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
|
||||
.expect_err("signature replayed to a different method must fail");
|
||||
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_v2_signature_is_bound_to_destination_audience() {
|
||||
ensure_test_rpc_secret();
|
||||
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("tonic auth headers should build");
|
||||
|
||||
let error = verify_tonic_rpc_signature("node-b:9000", "/node_service.NodeService/Ping", &headers)
|
||||
.expect_err("signature replayed to a different node must fail");
|
||||
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("tonic auth headers should build");
|
||||
headers.insert(RPC_SIGNATURE_V2_HEADER, HeaderValue::from_static("invalid"));
|
||||
|
||||
assert!(
|
||||
verify_rpc_signature(TONIC_RPC_PREFIX, &Method::GET, &headers).is_ok(),
|
||||
"the compatibility signature should remain valid for old servers"
|
||||
);
|
||||
let error = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers)
|
||||
.expect_err("new servers must not downgrade malformed v2 auth");
|
||||
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_tonic_signature_remains_accepted_during_rolling_upgrade() {
|
||||
ensure_test_rpc_secret();
|
||||
let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build");
|
||||
|
||||
assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_bound_tonic_request_rejects_replay_and_body_tampering() {
|
||||
ensure_test_rpc_secret();
|
||||
let body = b"heal-control-request";
|
||||
let mut request = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut request, body).expect("canonical body digest should be attached");
|
||||
let content_sha256 = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "HealControl", content_sha256)
|
||||
.expect("body-bound auth headers should build");
|
||||
request.metadata_mut().as_mut().extend(headers.clone());
|
||||
|
||||
assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/HealControl", &headers).is_ok());
|
||||
let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/HealControl", &headers)
|
||||
.expect_err("reusing a body-bound nonce must fail");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
assert!(verify_tonic_canonical_body_digest(&request, body).is_ok());
|
||||
let tampered = verify_tonic_canonical_body_digest(&request, b"different-body")
|
||||
.expect_err("a different canonical request body must fail");
|
||||
assert_eq!(tampered.to_string(), "RPC content SHA-256 mismatch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_v2_metadata_fails_closed() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build");
|
||||
headers.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
|
||||
|
||||
let error = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers)
|
||||
.expect_err("partial v2 metadata must not fall back to legacy auth");
|
||||
assert_eq!(error.to_string(), "Missing RPC v2 signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_mutation_digest_rejects_legacy_only_auth() {
|
||||
let mut request = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut request, b"heal-control-v1\0start").expect("canonical digest should be attached");
|
||||
|
||||
let error = verify_tonic_canonical_body_digest(&request, b"heal-control-v1\0start")
|
||||
.expect_err("mutation body verification must also require v2 auth");
|
||||
assert_eq!(error.to_string(), "RPC mutation requires v2 authentication");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_expires_by_monotonic_deadline_and_fails_closed_at_capacity() {
|
||||
let now = Instant::now();
|
||||
let expiry = now.checked_add(REPLAY_CACHE_RETENTION).expect("test expiry should fit");
|
||||
let after_expiry = expiry.checked_add(Duration::from_secs(1)).expect("test expiry should fit");
|
||||
let nonce_a = Uuid::new_v4();
|
||||
let nonce_b = Uuid::new_v4();
|
||||
let mut cache = RpcNonceCache::default();
|
||||
|
||||
cache
|
||||
.check_and_record(nonce_a, 100, now, 100, expiry, 1)
|
||||
.expect("first nonce should be recorded");
|
||||
let capacity = cache
|
||||
.check_and_record(nonce_b, 100, now, 100, expiry, 1)
|
||||
.expect_err("a full replay cache must fail closed");
|
||||
assert_eq!(capacity.to_string(), "RPC replay cache capacity exceeded");
|
||||
cache
|
||||
.check_and_record(nonce_b, 702, after_expiry, 702, after_expiry, 1)
|
||||
.expect("expired nonce should release capacity");
|
||||
assert!(!cache.nonces.contains(&nonce_a));
|
||||
assert!(cache.nonces.contains(&nonce_b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_rejects_replay_after_wall_clock_regression() {
|
||||
let now = Instant::now();
|
||||
let expiry = now.checked_add(REPLAY_CACHE_RETENTION).expect("test expiry should fit");
|
||||
let after_expiry = expiry.checked_add(Duration::from_secs(1)).expect("test expiry should fit");
|
||||
let nonce = Uuid::new_v4();
|
||||
let mut cache = RpcNonceCache::default();
|
||||
|
||||
cache
|
||||
.check_and_record(nonce, 1_000, now, 1_000, expiry, 2)
|
||||
.expect("first nonce should be recorded");
|
||||
let replay = cache
|
||||
.check_and_record(nonce, 1_000, after_expiry, 900, after_expiry, 2)
|
||||
.expect_err("wall clock regression must not make an old signature reusable");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
|
||||
let stale = cache
|
||||
.check_and_record(Uuid::new_v4(), 600, after_expiry, 900, after_expiry, 2)
|
||||
.expect_err("the monotonic wall-clock high-water mark must fail closed");
|
||||
assert_eq!(stale.to_string(), "RPC request timestamp expired after clock regression");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,7 +28,10 @@ pub(crate) use background_monitor::spawn_background_monitor;
|
||||
pub use client::{
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
};
|
||||
pub use http_auth::{TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, verify_rpc_signature};
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
pub use internode_data_transport::build_internode_data_transport_from_env;
|
||||
|
||||
@@ -176,12 +176,28 @@ pub fn init_lock_clients(endpoint_pools: EndpointServerPools) {
|
||||
}
|
||||
}
|
||||
|
||||
fn endpoint_rpc_authority(endpoint: &Endpoint) -> Option<String> {
|
||||
let host = endpoint.url.host_str()?;
|
||||
let host = if host.contains(':') && !host.starts_with('[') {
|
||||
format!("[{host}]")
|
||||
} else {
|
||||
host.to_string()
|
||||
};
|
||||
Some(match endpoint.url.port() {
|
||||
Some(port) => format!("{host}:{port}"),
|
||||
None => host,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn init_local_peer(endpoint_pools: &EndpointServerPools, host: &String, port: &String) {
|
||||
let mut peer_set = Vec::new();
|
||||
endpoint_pools.as_ref().iter().for_each(|endpoints| {
|
||||
endpoints.endpoints.as_ref().iter().for_each(|endpoint| {
|
||||
if endpoint.get_type() == EndpointType::Url && endpoint.is_local && endpoint.url.has_host() {
|
||||
peer_set.push(endpoint.url.host_str().unwrap().to_string());
|
||||
if endpoint.get_type() == EndpointType::Url
|
||||
&& endpoint.is_local
|
||||
&& let Some(authority) = endpoint_rpc_authority(endpoint)
|
||||
{
|
||||
peer_set.push(authority);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -234,6 +250,37 @@ mod tests {
|
||||
}])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_rpc_authority_preserves_port_and_ipv6_brackets() {
|
||||
let endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");
|
||||
assert_eq!(endpoint_rpc_authority(&endpoint).as_deref(), Some("127.0.0.1:9001"));
|
||||
|
||||
let endpoint = Endpoint::try_from("https://[::1]:9002/d1").expect("IPv6 URL endpoint");
|
||||
assert_eq!(endpoint_rpc_authority(&endpoint).as_deref(), Some("[::1]:9002"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn init_local_peer_publishes_complete_rpc_authority() {
|
||||
let previous = rustfs_common::get_global_local_node_name().await;
|
||||
let mut endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");
|
||||
endpoint.is_local = true;
|
||||
let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 1,
|
||||
endpoints: Endpoints::from(vec![endpoint]),
|
||||
cmd_line: "rpc-authority-test".to_string(),
|
||||
platform: "test".to_string(),
|
||||
}]);
|
||||
|
||||
let host = String::new();
|
||||
let port = "9000".to_string();
|
||||
init_local_peer(&endpoint_pools, &host, &port).await;
|
||||
assert_eq!(rustfs_common::try_get_global_local_node_name().as_deref(), Some("127.0.0.1:9001"));
|
||||
rustfs_common::set_global_local_node_name(&previous).await;
|
||||
}
|
||||
|
||||
// Phase 5 follow-up (backlog#1052): registering local disks through the
|
||||
// ctx-explicit entry writes the passed context's registry only — the
|
||||
// process bootstrap context (and any other instance) stays clean, so a
|
||||
|
||||
@@ -13,6 +13,7 @@ for later deletion.
|
||||
## Open Items
|
||||
|
||||
- `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability.
|
||||
- `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request.
|
||||
|
||||
## Review Checklist
|
||||
|
||||
|
||||
+152
-5
@@ -36,9 +36,11 @@ use crate::storage_api::server::http as storage;
|
||||
use crate::storage_api::server::http::request_context::{RequestContext, extract_request_id_from_headers};
|
||||
use crate::storage_api::server::http::rpc::InternodeRpcService;
|
||||
use crate::storage_api::server::http::tonic_service::make_server;
|
||||
use crate::storage_api::server::http::{ServerContextSlot, TONIC_RPC_PREFIX, verify_rpc_signature};
|
||||
use crate::storage_api::server::http::{
|
||||
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, verify_tonic_rpc_signature,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, Method, Request as HttpRequest, Response};
|
||||
use http::{HeaderMap, Method, Request as HttpRequest, Response, Uri};
|
||||
use hyper::body::Incoming;
|
||||
use hyper_util::{
|
||||
rt::{TokioExecutor, TokioIo, TokioTimer},
|
||||
@@ -125,6 +127,45 @@ const EVENT_GRPC_TRACE_CONTEXT_PROPAGATION_FAILED: &str = "grpc_trace_context_pr
|
||||
|
||||
static ACTIVE_HTTP_REQUESTS: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct RpcRequestTarget {
|
||||
uri: Uri,
|
||||
method: Method,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RpcRequestPathService<S> {
|
||||
inner: S,
|
||||
}
|
||||
|
||||
impl<S> RpcRequestPathService<S> {
|
||||
fn new(inner: S) -> Self {
|
||||
Self { inner }
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, B> Service<HttpRequest<B>> for RpcRequestPathService<S>
|
||||
where
|
||||
S: Service<HttpRequest<B>>,
|
||||
{
|
||||
type Response = S::Response;
|
||||
type Error = S::Error;
|
||||
type Future = S::Future;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
self.inner.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, mut req: HttpRequest<B>) -> Self::Future {
|
||||
let target = RpcRequestTarget {
|
||||
uri: req.uri().clone(),
|
||||
method: req.method().clone(),
|
||||
};
|
||||
req.extensions_mut().insert(target);
|
||||
self.inner.call(req)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn request_method_label(method: &Method) -> &'static str {
|
||||
match method.as_str() {
|
||||
@@ -1260,12 +1301,12 @@ fn process_connection(
|
||||
// service in the auth interceptor (which returns an `InterceptedService` without those
|
||||
// methods).
|
||||
let rpc_max_message_size = rustfs_protos::internode_rpc_max_message_size();
|
||||
let rpc_service = InterceptedService::new(
|
||||
let rpc_service = RpcRequestPathService::new(InterceptedService::new(
|
||||
NodeServiceServer::new(make_server())
|
||||
.max_decoding_message_size(rpc_max_message_size)
|
||||
.max_encoding_message_size(rpc_max_message_size),
|
||||
check_auth,
|
||||
);
|
||||
));
|
||||
|
||||
#[cfg(feature = "swift")]
|
||||
let http_service = SwiftService::new(true, None, s3_service);
|
||||
@@ -1787,7 +1828,26 @@ fn handle_connection_error(peer_addr: Option<&str>, err: &(dyn std::error::Error
|
||||
|
||||
#[allow(clippy::result_large_err)]
|
||||
fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
|
||||
verify_rpc_signature(TONIC_RPC_PREFIX, &Method::GET, req.metadata().as_ref()).map_err(|e| {
|
||||
let local_node_name =
|
||||
storage::try_current_local_node_name().ok_or_else(|| Status::unavailable("RPC identity unavailable"))?;
|
||||
let audience =
|
||||
normalize_tonic_rpc_audience(&local_node_name).map_err(|_| Status::unavailable("Invalid local RPC identity"))?;
|
||||
let target = req
|
||||
.extensions()
|
||||
.get::<RpcRequestTarget>()
|
||||
.ok_or_else(|| Status::unauthenticated("Missing RPC request target"))?;
|
||||
if target.method != Method::POST {
|
||||
return Err(Status::unauthenticated("Invalid RPC request method"));
|
||||
}
|
||||
let rpc_method = target
|
||||
.uri
|
||||
.path()
|
||||
.strip_prefix(TONIC_RPC_PREFIX)
|
||||
.and_then(|suffix| suffix.strip_prefix('/'))
|
||||
.filter(|method| !method.is_empty() && !method.contains('/'))
|
||||
.ok_or_else(|| Status::unauthenticated("Invalid RPC request path"))?;
|
||||
debug_assert!(!rpc_method.is_empty());
|
||||
verify_tonic_rpc_signature(&audience, target.uri.path(), req.metadata().as_ref()).map_err(|e| {
|
||||
error!(
|
||||
event = EVENT_RPC_SIGNATURE_VERIFICATION_FAILED,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
@@ -2125,6 +2185,93 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct RpcPathObserver;
|
||||
|
||||
impl<ReqBody> Service<HttpRequest<ReqBody>> for RpcPathObserver {
|
||||
type Response = Response<Empty<Bytes>>;
|
||||
type Error = Infallible;
|
||||
type Future = Ready<std::result::Result<Response<Empty<Bytes>>, Infallible>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, req: HttpRequest<ReqBody>) -> Self::Future {
|
||||
let mut response = Response::new(Empty::new());
|
||||
if let Some(target) = req.extensions().get::<RpcRequestTarget>() {
|
||||
response.extensions_mut().insert(target.clone());
|
||||
}
|
||||
std::future::ready(Ok(response))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rpc_request_path_service_preserves_exact_http_target() {
|
||||
let expected_path = "/node_service.NodeService/BackgroundHealStatus";
|
||||
let request = HttpRequest::builder()
|
||||
.method(Method::POST)
|
||||
.uri(format!("http://node-a:9000{expected_path}"))
|
||||
.body(())
|
||||
.expect("request");
|
||||
let mut service = RpcRequestPathService::new(RpcPathObserver);
|
||||
|
||||
let response = futures::executor::block_on(service.call(request)).expect("response");
|
||||
let captured = response
|
||||
.extensions()
|
||||
.get::<RpcRequestTarget>()
|
||||
.expect("inner gRPC service should receive the exact request URI");
|
||||
assert_eq!(captured.uri.path(), expected_path);
|
||||
assert_eq!(captured.uri.authority().map(|authority| authority.as_str()), Some("node-a:9000"));
|
||||
assert_eq!(captured.method, Method::POST);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rpc_auth_binds_post_method_authority_and_exact_path() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("rpc-http-test-secret".to_string());
|
||||
let previous_node_name = rustfs_common::get_global_local_node_name().await;
|
||||
rustfs_common::set_global_local_node_name("127.0.0.1:9000").await;
|
||||
let headers =
|
||||
storage::gen_tonic_signature_headers("127.0.0.1:9000", "node_service.NodeService", "BackgroundHealStatus", None)
|
||||
.expect("v2 auth headers should build");
|
||||
let uri: Uri = "http://127.0.0.1:9000/node_service.NodeService/BackgroundHealStatus"
|
||||
.parse()
|
||||
.expect("test RPC URI should parse");
|
||||
|
||||
let mut request = Request::new(());
|
||||
request.metadata_mut().as_mut().extend(headers.clone());
|
||||
request.extensions_mut().insert(RpcRequestTarget {
|
||||
uri: uri.clone(),
|
||||
method: Method::POST,
|
||||
});
|
||||
assert!(check_auth(request).is_ok(), "matching trusted node audience should authenticate");
|
||||
|
||||
rustfs_common::set_global_local_node_name("127.0.0.1:9001").await;
|
||||
let mut replay_to_other_node = Request::new(());
|
||||
replay_to_other_node.metadata_mut().as_mut().extend(headers.clone());
|
||||
replay_to_other_node.extensions_mut().insert(RpcRequestTarget {
|
||||
uri: uri.clone(),
|
||||
method: Method::POST,
|
||||
});
|
||||
assert!(
|
||||
check_auth(replay_to_other_node).is_err(),
|
||||
"same-host request must not replay across node ports"
|
||||
);
|
||||
|
||||
rustfs_common::set_global_local_node_name("127.0.0.1:9000").await;
|
||||
let mut get_request = Request::new(());
|
||||
get_request.metadata_mut().as_mut().extend(headers);
|
||||
get_request.extensions_mut().insert(RpcRequestTarget {
|
||||
uri,
|
||||
method: Method::GET,
|
||||
});
|
||||
let error = check_auth(get_request).expect_err("wire GET must not reuse a POST gRPC signature");
|
||||
assert_eq!(error.code(), tonic::Code::Unauthenticated);
|
||||
assert_eq!(error.message(), "Invalid RPC request method");
|
||||
rustfs_common::set_global_local_node_name(&previous_node_name).await;
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct MarkerService {
|
||||
name: &'static str,
|
||||
|
||||
@@ -48,6 +48,10 @@ pub(crate) async fn current_local_node_name() -> String {
|
||||
root_runtime_sources::current_local_node_name().await.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(crate) fn try_current_local_node_name() -> Option<String> {
|
||||
rustfs_common::try_get_global_local_node_name()
|
||||
}
|
||||
|
||||
pub(crate) fn current_action_credentials() -> Option<Credentials> {
|
||||
root_runtime_sources::current_action_credentials()
|
||||
}
|
||||
|
||||
@@ -462,9 +462,12 @@ pub(crate) mod ecstore_rio {
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_rpc {
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::gen_tonic_signature_headers;
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, verify_rpc_signature,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, verify_rpc_signature,
|
||||
verify_tonic_rpc_signature,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -518,6 +521,18 @@ pub(crate) const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = ecstore_rpc::SERVICE_SIGNA
|
||||
pub(crate) const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = ecstore_rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC;
|
||||
pub(crate) const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET;
|
||||
pub(crate) const TONIC_RPC_PREFIX: &str = ecstore_rpc::TONIC_RPC_PREFIX;
|
||||
|
||||
pub(crate) fn normalize_tonic_rpc_audience(value: &str) -> std::io::Result<String> {
|
||||
ecstore_rpc::normalize_tonic_rpc_audience(value)
|
||||
}
|
||||
|
||||
pub(crate) fn try_current_local_node_name() -> Option<String> {
|
||||
crate::storage::runtime_sources::try_current_local_node_name()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use ecstore_rpc::gen_tonic_signature_headers;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) const STORAGE_CLASS_SUB_SYS: &str = ecstore_config::com::STORAGE_CLASS_SUB_SYS;
|
||||
|
||||
@@ -1384,6 +1399,10 @@ pub(crate) fn verify_rpc_signature(url: &str, method: &http::Method, headers: &h
|
||||
ecstore_rpc::verify_rpc_signature(url, method, headers)
|
||||
}
|
||||
|
||||
pub(crate) fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &http::HeaderMap) -> std::io::Result<()> {
|
||||
ecstore_rpc::verify_tonic_rpc_signature(audience, path, headers)
|
||||
}
|
||||
|
||||
pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag {
|
||||
ecstore_client::object_api_utils::to_s3s_etag(etag)
|
||||
}
|
||||
|
||||
@@ -93,7 +93,16 @@ pub(crate) mod server {
|
||||
}
|
||||
|
||||
pub(crate) mod http {
|
||||
pub(crate) use crate::storage::storage_api::{ServerContextSlot, TONIC_RPC_PREFIX, verify_rpc_signature};
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, verify_tonic_rpc_signature,
|
||||
};
|
||||
|
||||
pub(crate) fn try_current_local_node_name() -> Option<String> {
|
||||
crate::storage::storage_api::try_current_local_node_name()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::gen_tonic_signature_headers;
|
||||
|
||||
pub(crate) mod ecfs {
|
||||
pub(crate) type FS = crate::storage::storage_api::FS;
|
||||
|
||||
Reference in New Issue
Block a user