fix(rpc): bind internode auth to exact targets (#4988)

This commit is contained in:
cxymds
2026-07-19 21:52:57 +08:00
committed by GitHub
parent b0c6c4cbce
commit 133499c2d5
11 changed files with 821 additions and 34 deletions
+9
View File
@@ -54,6 +54,15 @@ pub async fn get_global_local_node_name() -> String {
GLOBAL_LOCAL_NODE_NAME.read().await.clone()
}
/// Read the local node name without waiting for initialization or a writer.
pub fn try_get_global_local_node_name() -> Option<String> {
GLOBAL_LOCAL_NODE_NAME
.try_read()
.ok()
.map(|name| name.clone())
.filter(|name| !name.is_empty())
}
/// Set the global RustFS initialization time to the current UTC time.
pub async fn set_global_init_time_now() {
let now = Utc::now();
+4 -2
View File
@@ -382,8 +382,10 @@ pub mod rio {
pub mod rpc {
pub use crate::cluster::rpc::{
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_tonic_signature_interceptor,
node_service_time_out_client, node_service_time_out_client_no_auth, verify_rpc_signature,
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature,
};
}
+92 -11
View File
@@ -12,10 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::cluster::rpc::{TONIC_RPC_PREFIX, gen_signature_headers};
use crate::cluster::rpc::http_auth::RPC_CONTENT_SHA256_HEADER;
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
use crate::disk::error::{DiskError, Error as DiskErrorType};
use crate::runtime::sources as runtime_sources;
use http::Method;
use http::Uri;
use rustfs_protos::{
ChannelClass, create_new_channel, get_channel_for_class, proto_gen::node_service::node_service_client::NodeServiceClient,
};
@@ -47,6 +48,7 @@ pub async fn node_service_time_out_client_for_class(
interceptor: TonicInterceptor,
class: ChannelClass,
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match class {
ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
Some(channel) => {
@@ -111,11 +113,25 @@ pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
}
}
pub struct TonicSignatureInterceptor;
pub struct TonicSignatureInterceptor {
audience: Option<String>,
}
impl tonic::service::Interceptor for TonicSignatureInterceptor {
fn call(&mut self, mut req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET)
let method = req
.extensions()
.get::<tonic::GrpcMethod<'_>>()
.ok_or_else(|| tonic::Status::unauthenticated("Missing gRPC method metadata"))?;
let audience = self
.audience
.as_deref()
.ok_or_else(|| tonic::Status::unauthenticated("Missing gRPC audience"))?;
let content_sha256 = req
.metadata()
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok());
let headers = gen_tonic_signature_headers(audience, method.service(), method.method(), content_sha256)
.map_err(|_| tonic::Status::unauthenticated("No valid auth token"))?;
req.metadata_mut().as_mut().extend(headers);
inject_trace_context_into_metadata(req.metadata_mut());
@@ -125,7 +141,7 @@ impl tonic::service::Interceptor for TonicSignatureInterceptor {
}
pub fn gen_tonic_signature_interceptor() -> TonicSignatureInterceptor {
TonicSignatureInterceptor
TonicSignatureInterceptor { audience: None }
}
pub struct NoOpInterceptor;
@@ -141,6 +157,22 @@ pub enum TonicInterceptor {
NoOp(NoOpInterceptor),
}
impl TonicInterceptor {
fn with_rpc_audience(mut self, addr: &str) -> std::io::Result<Self> {
if let Self::Signature(interceptor) = &mut self {
let uri = addr
.parse::<Uri>()
.map_err(|_| std::io::Error::other("Invalid gRPC peer URI"))?;
let audience = uri
.authority()
.map(|authority| normalize_tonic_rpc_audience(authority.as_str()))
.ok_or_else(|| std::io::Error::other("Missing gRPC peer authority"))?;
interceptor.audience = Some(audience?);
}
Ok(self)
}
}
impl tonic::service::Interceptor for TonicInterceptor {
fn call(&mut self, req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
match self {
@@ -165,6 +197,20 @@ mod tests {
runtime_sources::ensure_test_rpc_secret();
}
fn test_request() -> tonic::Request<()> {
let mut request = tonic::Request::new(());
request
.extensions_mut()
.insert(tonic::GrpcMethod::new("node_service.NodeService", "Ping"));
request
}
fn test_interceptor() -> TonicSignatureInterceptor {
TonicSignatureInterceptor {
audience: Some("node-a:9000".to_string()),
}
}
fn with_trace_parent<F>(trace_id_hex: &str, f: F)
where
F: FnOnce(),
@@ -192,20 +238,55 @@ mod tests {
#[test]
fn test_signature_interceptor_keeps_auth_headers() {
ensure_test_rpc_secret();
let mut interceptor = TonicSignatureInterceptor;
let req = tonic::Request::new(());
let mut interceptor = test_interceptor();
let req = test_request();
let req = interceptor.call(req).expect("interceptor call should succeed");
assert!(req.metadata().contains_key("x-rustfs-signature"));
assert!(req.metadata().contains_key("x-rustfs-timestamp"));
assert!(req.metadata().contains_key("x-rustfs-rpc-signature-v2"));
assert!(req.metadata().contains_key("x-rustfs-rpc-nonce"));
assert!(
crate::cluster::rpc::verify_tonic_rpc_signature(
"node-a:9000",
"/node_service.NodeService/Ping",
req.metadata().as_ref(),
)
.is_ok(),
"interceptor signature should bind the configured peer audience and generated method"
);
}
#[test]
fn test_signature_interceptor_binds_audience_from_peer_uri() {
let interceptor = TonicInterceptor::Signature(gen_tonic_signature_interceptor())
.with_rpc_audience("http://node-a:9000")
.expect("peer URI should provide an audience");
let TonicInterceptor::Signature(interceptor) = interceptor else {
panic!("signature interceptor variant should be preserved");
};
assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
}
#[test]
fn test_signature_interceptor_requires_generated_method_metadata() {
ensure_test_rpc_secret();
let mut interceptor = test_interceptor();
let error = interceptor
.call(tonic::Request::new(()))
.expect_err("requests without an exact generated method must fail closed");
assert_eq!(error.code(), tonic::Code::Unauthenticated);
assert_eq!(error.message(), "Missing gRPC method metadata");
}
#[test]
fn test_signature_interceptor_may_inject_request_id() {
ensure_test_rpc_secret();
let mut interceptor = TonicSignatureInterceptor;
let req = tonic::Request::new(());
let mut interceptor = test_interceptor();
let req = test_request();
let span = tracing::info_span!("grpc-rpc-test-span");
let _guard = span.enter();
@@ -219,8 +300,8 @@ mod tests {
#[test]
fn test_signature_interceptor_injects_traceparent_metadata() {
ensure_test_rpc_secret();
let mut interceptor = TonicSignatureInterceptor;
let req = tonic::Request::new(());
let mut interceptor = test_interceptor();
let req = test_request();
with_trace_parent("4bf92f3577b34da6a3ce929d0e0e4736", || {
let req = interceptor.call(req).expect("interceptor call should succeed");
+476 -11
View File
@@ -19,8 +19,9 @@
//! GHSA-r5qv-rc46-hv8q (internode RPC authentication must fail closed, fixed in
//! rustfs/rustfs#4402) is anchored by the `ghsa_r5qv_*` tests in the module
//! below, plus the broader negative-signature suite. The advisory class is: a
//! node must never accept an RPC whose auth is missing, malformed, replayed, or
//! signed with the default/empty shared secret. See
//! node must never accept an RPC whose auth is missing, malformed, or signed
//! with the default/empty shared secret. Body-bound v2 requests additionally
//! receive process-local replay protection. See
//! `docs/testing/security-regressions.md` for the full advisory -> test map.
//!
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
@@ -29,23 +30,89 @@ use crate::cluster::rpc::context_propagation::{inject_request_id_into_http_heade
use base64::Engine as _;
use base64::engine::general_purpose;
use hmac::{Hmac, KeyInit, Mac};
use http::uri::Authority;
use http::{HeaderMap, HeaderValue, Method, Uri};
#[cfg(test)]
use rustfs_credentials::{DEFAULT_SECRET_KEY, RPC_SECRET_REQUIRED_MESSAGE};
use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token};
use sha2::Digest as _;
use sha2::Sha256;
use std::sync::Once;
use std::collections::{HashSet, VecDeque};
use std::sync::{LazyLock, Mutex, Once};
use std::time::{Duration, Instant};
use time::OffsetDateTime;
use tracing::error;
use uuid::Uuid;
type HmacSha256 = Hmac<Sha256>;
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
const RPC_NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
pub(crate) const RPC_CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
const RPC_AUTH_VERSION_V2: &str = "2";
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
const MAX_REPLAY_PROTECTED_NONCES: usize = 65_536;
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
#[derive(Default)]
struct RpcNonceCache {
nonces: HashSet<Uuid>,
expirations: VecDeque<(Instant, i64, Uuid)>,
max_wall_time: i64,
}
impl RpcNonceCache {
fn remove_expired(&mut self, now: Instant, wall_time: i64) {
while matches!(
self.expirations.front(),
Some((expires_at, valid_until, _)) if *expires_at < now && *valid_until < wall_time
) {
let Some((_, _, nonce)) = self.expirations.pop_front() else {
break;
};
self.nonces.remove(&nonce);
}
}
fn check_and_record(
&mut self,
nonce: Uuid,
signed_at: i64,
now: Instant,
wall_time: i64,
expires_at: Instant,
capacity: usize,
) -> std::io::Result<()> {
self.max_wall_time = self.max_wall_time.max(wall_time);
if self.max_wall_time.saturating_sub(signed_at) > SIGNATURE_VALID_DURATION {
return Err(std::io::Error::other("RPC request timestamp expired after clock regression"));
}
self.remove_expired(now, self.max_wall_time);
if self.nonces.contains(&nonce) {
return Err(std::io::Error::other("RPC request replay detected"));
}
if self.nonces.len() >= capacity {
return Err(std::io::Error::other("RPC replay cache capacity exceeded"));
}
self.nonces.insert(nonce);
self.expirations
.push_back((expires_at, signed_at.saturating_add(SIGNATURE_VALID_DURATION), nonce));
Ok(())
}
}
// This cache is a process-local wire-replay defense only. Mutation handlers
// still need a stable operation ID and coordinator-owned idempotency across
// retries, node failover, and restart.
static LOCAL_RPC_NONCE_CACHE: LazyLock<Mutex<RpcNonceCache>> = LazyLock::new(|| Mutex::new(RpcNonceCache::default()));
/// Get the shared secret for HMAC signing
#[cfg(test)]
fn resolve_shared_secret(env_secret: Option<&str>, global_secret: Option<&str>) -> std::io::Result<String> {
@@ -109,6 +176,96 @@ fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, si
mac.verify_slice(&signature).is_ok()
}
#[derive(Clone, Copy)]
struct SignatureV2Scope<'a> {
audience: &'a str,
service: &'a str,
rpc_method: &'a str,
timestamp: &'a str,
nonce: &'a str,
content_sha256: &'a str,
}
fn update_signature_v2(mac: &mut HmacSha256, scope: SignatureV2Scope<'_>) {
for part in [
b"rustfs-rpc-auth-v2|".as_slice(),
scope.audience.as_bytes(),
b"|/",
scope.service.as_bytes(),
b"/",
scope.rpc_method.as_bytes(),
b"|POST|",
scope.timestamp.as_bytes(),
b"|",
scope.nonce.as_bytes(),
b"|",
scope.content_sha256.as_bytes(),
] {
mac.update(part);
}
}
fn generate_signature_v2(secret: &str, scope: SignatureV2Scope<'_>) -> std::io::Result<String> {
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_signature_v2(&mut mac, scope);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
}
fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &str) -> bool {
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
return false;
};
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
return false;
};
update_signature_v2(&mut mac, scope);
mac.verify_slice(&signature).is_ok()
}
fn valid_content_sha256(value: &str) -> bool {
value == UNSIGNED_PAYLOAD
|| (value.len() == 64
&& value
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)))
}
fn header_value(value: &str, name: &str) -> std::io::Result<HeaderValue> {
HeaderValue::from_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name} header value")))
}
pub fn normalize_tonic_rpc_audience(value: &str) -> std::io::Result<String> {
let authority = value
.parse::<Authority>()
.map_err(|_| std::io::Error::other("Invalid gRPC peer authority"))?;
Ok(authority.as_str().to_ascii_lowercase())
}
fn check_timestamp(timestamp: i64) -> std::io::Result<()> {
let current_time = OffsetDateTime::now_utc().unix_timestamp();
if current_time.saturating_sub(timestamp) > SIGNATURE_VALID_DURATION
|| timestamp.saturating_sub(current_time) > SIGNATURE_VALID_DURATION
{
return Err(std::io::Error::other("Request timestamp expired"));
}
Ok(())
}
fn check_and_record_nonce(nonce: Uuid, signed_at: i64) -> std::io::Result<()> {
let wall_time = OffsetDateTime::now_utc().unix_timestamp();
let mut cache = LOCAL_RPC_NONCE_CACHE
.lock()
.map_err(|_| std::io::Error::other("RPC replay cache unavailable"))?;
// Take the monotonic timestamp after acquiring the lock so expiration
// entries remain ordered by the same serialization point as insertion.
let now = Instant::now();
let expires_at = now
.checked_add(REPLAY_CACHE_RETENTION)
.ok_or_else(|| std::io::Error::other("RPC replay expiry overflow"))?;
cache.check_and_record(nonce, signed_at, now, wall_time, expires_at, MAX_REPLAY_PROTECTED_NONCES)
}
/// Build headers with authentication signature
pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) -> std::io::Result<()> {
let auth_headers = gen_signature_headers(url, method)?;
@@ -135,6 +292,184 @@ pub fn gen_signature_headers(url: &str, method: &Method) -> std::io::Result<Head
Ok(headers)
}
/// Generate rolling-upgrade-safe gRPC auth metadata.
///
/// The legacy signature remains present for old servers. New servers prefer the
/// v2 signature and bind it to the destination authority and exact generated
/// gRPC method. A versioned canonical mutation payload can opt into the
/// additional body-digest capability.
pub fn gen_tonic_signature_headers(
audience: &str,
service: &str,
rpc_method: &str,
content_sha256: Option<&str>,
) -> std::io::Result<HeaderMap> {
if audience.is_empty() || service.is_empty() || rpc_method.is_empty() || service.contains('/') || rpc_method.contains('/') {
return Err(std::io::Error::other("Invalid RPC v2 signing scope"));
}
let content_sha256 = content_sha256.unwrap_or(UNSIGNED_PAYLOAD);
if !valid_content_sha256(content_sha256) {
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
}
let secret = get_shared_secret()?;
let timestamp = OffsetDateTime::now_utc().unix_timestamp();
let timestamp_header = timestamp.to_string();
let body_nonce = (content_sha256 != UNSIGNED_PAYLOAD).then(|| Uuid::new_v4().to_string());
let nonce = body_nonce.as_deref().unwrap_or(UNSIGNED_PAYLOAD_NONCE);
let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp);
let signature_v2 = generate_signature_v2(
&secret,
SignatureV2Scope {
audience,
service,
rpc_method,
timestamp: &timestamp_header,
nonce,
content_sha256,
},
)?;
let mut headers = HeaderMap::new();
headers.insert(SIGNATURE_HEADER, header_value(&legacy_signature, SIGNATURE_HEADER)?);
headers.insert(TIMESTAMP_HEADER, header_value(&timestamp_header, TIMESTAMP_HEADER)?);
headers.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
headers.insert(RPC_SIGNATURE_V2_HEADER, header_value(&signature_v2, RPC_SIGNATURE_V2_HEADER)?);
headers.insert(RPC_NONCE_HEADER, header_value(nonce, RPC_NONCE_HEADER)?);
headers.insert(RPC_CONTENT_SHA256_HEADER, header_value(content_sha256, RPC_CONTENT_SHA256_HEADER)?);
Ok(headers)
}
/// Bind a mutation to a versioned, deterministic canonical payload.
///
/// Do not pass a protobuf re-encoding here: unknown fields and map ordering are
/// not a stable mixed-version contract.
pub fn set_tonic_canonical_body_digest<T>(request: &mut tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
let digest = hex_simd::encode_to_string(Sha256::digest(canonical_body), hex_simd::AsciiCase::Lower);
request
.metadata_mut()
.as_mut()
.insert(RPC_CONTENT_SHA256_HEADER, header_value(&digest, RPC_CONTENT_SHA256_HEADER)?);
Ok(())
}
pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
let version = request
.metadata()
.get(RPC_AUTH_VERSION_HEADER)
.and_then(|value| value.to_str().ok());
if version != Some(RPC_AUTH_VERSION_V2) {
return Err(std::io::Error::other("RPC mutation requires v2 authentication"));
}
let expected = request
.metadata()
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
if expected == UNSIGNED_PAYLOAD || !valid_content_sha256(expected) {
return Err(std::io::Error::other("RPC body is not bound to the signature"));
}
let actual = hex_simd::encode_to_string(Sha256::digest(canonical_body), hex_simd::AsciiCase::Lower);
if actual != expected {
return Err(std::io::Error::other("RPC content SHA-256 mismatch"));
}
Ok(())
}
fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
[
RPC_AUTH_VERSION_HEADER,
RPC_SIGNATURE_V2_HEADER,
RPC_NONCE_HEADER,
RPC_CONTENT_SHA256_HEADER,
]
.iter()
.any(|name| headers.contains_key(*name))
}
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
if !has_v2_auth_headers(headers) {
// RUSTFS_COMPAT_TODO(heal-rpc-auth-v2): accept old peers during rolling upgrades. Remove after the minimum
// supported RustFS peer version sends v2 authentication on every internode gRPC request.
return verify_rpc_signature(TONIC_RPC_PREFIX, &Method::GET, headers);
}
let path = path
.strip_prefix('/')
.ok_or_else(|| std::io::Error::other("Invalid RPC request path"))?;
let (service, rpc_method) = path
.split_once('/')
.filter(|(service, rpc_method)| !service.is_empty() && !rpc_method.is_empty() && !rpc_method.contains('/'))
.ok_or_else(|| std::io::Error::other("Invalid RPC request path"))?;
if audience.is_empty() {
return Err(std::io::Error::other("Missing RPC audience"));
}
let version = headers
.get(RPC_AUTH_VERSION_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC auth version"))?;
if version != RPC_AUTH_VERSION_V2 {
return Err(std::io::Error::other("Unsupported RPC auth version"));
}
let signature = headers
.get(RPC_SIGNATURE_V2_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC v2 signature"))?;
let timestamp_header = headers
.get(TIMESTAMP_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
let timestamp = timestamp_header
.parse::<i64>()
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
check_timestamp(timestamp)?;
let nonce = headers
.get(RPC_NONCE_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC nonce"))?;
let content_sha256 = headers
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
if !valid_content_sha256(content_sha256) {
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
}
let parsed_nonce = if content_sha256 == UNSIGNED_PAYLOAD {
if nonce != UNSIGNED_PAYLOAD_NONCE {
return Err(std::io::Error::other("Invalid unsigned RPC nonce"));
}
None
} else {
let parsed_nonce = Uuid::parse_str(nonce).map_err(|_| std::io::Error::other("Invalid RPC nonce"))?;
if parsed_nonce.is_nil() {
return Err(std::io::Error::other("Invalid RPC nonce"));
}
Some(parsed_nonce)
};
let secret = get_shared_secret()?;
if !verify_signature_v2(
&secret,
SignatureV2Scope {
audience,
service,
rpc_method,
timestamp: timestamp_header,
nonce,
content_sha256,
},
signature,
) {
return Err(std::io::Error::other("Invalid RPC v2 signature"));
}
if let Some(nonce) = parsed_nonce {
check_and_record_nonce(nonce, timestamp)?;
}
Ok(())
}
/// Verify the request signature for RPC requests
pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) -> std::io::Result<()> {
// Get signature from header
@@ -153,14 +488,7 @@ pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) ->
.parse()
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
// Check timestamp validity (prevent replay attacks)
let current_time = OffsetDateTime::now_utc().unix_timestamp();
if current_time.saturating_sub(timestamp) > SIGNATURE_VALID_DURATION
|| timestamp.saturating_sub(current_time) > SIGNATURE_VALID_DURATION
{
return Err(std::io::Error::other("Request timestamp expired"));
}
check_timestamp(timestamp)?;
// Verify signature with constant-time HMAC comparison.
let secret = get_shared_secret()?;
@@ -707,4 +1035,141 @@ mod tests {
assert!(result.is_ok(), "Round-trip test failed for {method} {url}");
}
}
#[test]
fn tonic_v2_signature_is_bound_to_exact_method() {
ensure_test_rpc_secret();
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
.expect("tonic auth headers should build");
assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers).is_ok());
let error = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
.expect_err("signature replayed to a different method must fail");
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
}
#[test]
fn tonic_v2_signature_is_bound_to_destination_audience() {
ensure_test_rpc_secret();
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
.expect("tonic auth headers should build");
let error = verify_tonic_rpc_signature("node-b:9000", "/node_service.NodeService/Ping", &headers)
.expect_err("signature replayed to a different node must fail");
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
}
#[test]
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
ensure_test_rpc_secret();
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
.expect("tonic auth headers should build");
headers.insert(RPC_SIGNATURE_V2_HEADER, HeaderValue::from_static("invalid"));
assert!(
verify_rpc_signature(TONIC_RPC_PREFIX, &Method::GET, &headers).is_ok(),
"the compatibility signature should remain valid for old servers"
);
let error = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers)
.expect_err("new servers must not downgrade malformed v2 auth");
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
}
#[test]
fn legacy_tonic_signature_remains_accepted_during_rolling_upgrade() {
ensure_test_rpc_secret();
let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build");
assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers).is_ok());
}
#[test]
fn body_bound_tonic_request_rejects_replay_and_body_tampering() {
ensure_test_rpc_secret();
let body = b"heal-control-request";
let mut request = tonic::Request::new(());
set_tonic_canonical_body_digest(&mut request, body).expect("canonical body digest should be attached");
let content_sha256 = request
.metadata()
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok());
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "HealControl", content_sha256)
.expect("body-bound auth headers should build");
request.metadata_mut().as_mut().extend(headers.clone());
assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/HealControl", &headers).is_ok());
let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/HealControl", &headers)
.expect_err("reusing a body-bound nonce must fail");
assert_eq!(replay.to_string(), "RPC request replay detected");
assert!(verify_tonic_canonical_body_digest(&request, body).is_ok());
let tampered = verify_tonic_canonical_body_digest(&request, b"different-body")
.expect_err("a different canonical request body must fail");
assert_eq!(tampered.to_string(), "RPC content SHA-256 mismatch");
}
#[test]
fn partial_v2_metadata_fails_closed() {
ensure_test_rpc_secret();
let mut headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build");
headers.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
let error = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers)
.expect_err("partial v2 metadata must not fall back to legacy auth");
assert_eq!(error.to_string(), "Missing RPC v2 signature");
}
#[test]
fn canonical_mutation_digest_rejects_legacy_only_auth() {
let mut request = tonic::Request::new(());
set_tonic_canonical_body_digest(&mut request, b"heal-control-v1\0start").expect("canonical digest should be attached");
let error = verify_tonic_canonical_body_digest(&request, b"heal-control-v1\0start")
.expect_err("mutation body verification must also require v2 auth");
assert_eq!(error.to_string(), "RPC mutation requires v2 authentication");
}
#[test]
fn nonce_cache_expires_by_monotonic_deadline_and_fails_closed_at_capacity() {
let now = Instant::now();
let expiry = now.checked_add(REPLAY_CACHE_RETENTION).expect("test expiry should fit");
let after_expiry = expiry.checked_add(Duration::from_secs(1)).expect("test expiry should fit");
let nonce_a = Uuid::new_v4();
let nonce_b = Uuid::new_v4();
let mut cache = RpcNonceCache::default();
cache
.check_and_record(nonce_a, 100, now, 100, expiry, 1)
.expect("first nonce should be recorded");
let capacity = cache
.check_and_record(nonce_b, 100, now, 100, expiry, 1)
.expect_err("a full replay cache must fail closed");
assert_eq!(capacity.to_string(), "RPC replay cache capacity exceeded");
cache
.check_and_record(nonce_b, 702, after_expiry, 702, after_expiry, 1)
.expect("expired nonce should release capacity");
assert!(!cache.nonces.contains(&nonce_a));
assert!(cache.nonces.contains(&nonce_b));
}
#[test]
fn nonce_cache_rejects_replay_after_wall_clock_regression() {
let now = Instant::now();
let expiry = now.checked_add(REPLAY_CACHE_RETENTION).expect("test expiry should fit");
let after_expiry = expiry.checked_add(Duration::from_secs(1)).expect("test expiry should fit");
let nonce = Uuid::new_v4();
let mut cache = RpcNonceCache::default();
cache
.check_and_record(nonce, 1_000, now, 1_000, expiry, 2)
.expect("first nonce should be recorded");
let replay = cache
.check_and_record(nonce, 1_000, after_expiry, 900, after_expiry, 2)
.expect_err("wall clock regression must not make an old signature reusable");
assert_eq!(replay.to_string(), "RPC request replay detected");
let stale = cache
.check_and_record(Uuid::new_v4(), 600, after_expiry, 900, after_expiry, 2)
.expect_err("the monotonic wall-clock high-water mark must fail closed");
assert_eq!(stale.to_string(), "RPC request timestamp expired after clock regression");
}
}
+4 -1
View File
@@ -28,7 +28,10 @@ pub(crate) use background_monitor::spawn_background_monitor;
pub use client::{
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
};
pub use http_auth::{TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, verify_rpc_signature};
pub use http_auth::{
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature,
};
#[cfg(test)]
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
pub use internode_data_transport::build_internode_data_transport_from_env;
+49 -2
View File
@@ -176,12 +176,28 @@ pub fn init_lock_clients(endpoint_pools: EndpointServerPools) {
}
}
fn endpoint_rpc_authority(endpoint: &Endpoint) -> Option<String> {
let host = endpoint.url.host_str()?;
let host = if host.contains(':') && !host.starts_with('[') {
format!("[{host}]")
} else {
host.to_string()
};
Some(match endpoint.url.port() {
Some(port) => format!("{host}:{port}"),
None => host,
})
}
pub(super) async fn init_local_peer(endpoint_pools: &EndpointServerPools, host: &String, port: &String) {
let mut peer_set = Vec::new();
endpoint_pools.as_ref().iter().for_each(|endpoints| {
endpoints.endpoints.as_ref().iter().for_each(|endpoint| {
if endpoint.get_type() == EndpointType::Url && endpoint.is_local && endpoint.url.has_host() {
peer_set.push(endpoint.url.host_str().unwrap().to_string());
if endpoint.get_type() == EndpointType::Url
&& endpoint.is_local
&& let Some(authority) = endpoint_rpc_authority(endpoint)
{
peer_set.push(authority);
}
});
});
@@ -234,6 +250,37 @@ mod tests {
}])
}
#[test]
fn endpoint_rpc_authority_preserves_port_and_ipv6_brackets() {
let endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");
assert_eq!(endpoint_rpc_authority(&endpoint).as_deref(), Some("127.0.0.1:9001"));
let endpoint = Endpoint::try_from("https://[::1]:9002/d1").expect("IPv6 URL endpoint");
assert_eq!(endpoint_rpc_authority(&endpoint).as_deref(), Some("[::1]:9002"));
}
#[tokio::test]
#[serial_test::serial]
async fn init_local_peer_publishes_complete_rpc_authority() {
let previous = rustfs_common::get_global_local_node_name().await;
let mut endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");
endpoint.is_local = true;
let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 1,
endpoints: Endpoints::from(vec![endpoint]),
cmd_line: "rpc-authority-test".to_string(),
platform: "test".to_string(),
}]);
let host = String::new();
let port = "9000".to_string();
init_local_peer(&endpoint_pools, &host, &port).await;
assert_eq!(rustfs_common::try_get_global_local_node_name().as_deref(), Some("127.0.0.1:9001"));
rustfs_common::set_global_local_node_name(&previous).await;
}
// Phase 5 follow-up (backlog#1052): registering local disks through the
// ctx-explicit entry writes the passed context's registry only — the
// process bootstrap context (and any other instance) stays clean, so a