mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 15:37:02 +00:00
fix(rpc): bind internode auth to exact targets (#4988)
This commit is contained in:
@@ -382,8 +382,10 @@ pub mod rio {
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, verify_rpc_signature,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
|
||||
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -12,10 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::{TONIC_RPC_PREFIX, gen_signature_headers};
|
||||
use crate::cluster::rpc::http_auth::RPC_CONTENT_SHA256_HEADER;
|
||||
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
|
||||
use crate::disk::error::{DiskError, Error as DiskErrorType};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use http::Method;
|
||||
use http::Uri;
|
||||
use rustfs_protos::{
|
||||
ChannelClass, create_new_channel, get_channel_for_class, proto_gen::node_service::node_service_client::NodeServiceClient,
|
||||
};
|
||||
@@ -47,6 +48,7 @@ pub async fn node_service_time_out_client_for_class(
|
||||
interceptor: TonicInterceptor,
|
||||
class: ChannelClass,
|
||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match class {
|
||||
ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
|
||||
Some(channel) => {
|
||||
@@ -111,11 +113,25 @@ pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TonicSignatureInterceptor;
|
||||
pub struct TonicSignatureInterceptor {
|
||||
audience: Option<String>,
|
||||
}
|
||||
|
||||
impl tonic::service::Interceptor for TonicSignatureInterceptor {
|
||||
fn call(&mut self, mut req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
|
||||
let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET)
|
||||
let method = req
|
||||
.extensions()
|
||||
.get::<tonic::GrpcMethod<'_>>()
|
||||
.ok_or_else(|| tonic::Status::unauthenticated("Missing gRPC method metadata"))?;
|
||||
let audience = self
|
||||
.audience
|
||||
.as_deref()
|
||||
.ok_or_else(|| tonic::Status::unauthenticated("Missing gRPC audience"))?;
|
||||
let content_sha256 = req
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let headers = gen_tonic_signature_headers(audience, method.service(), method.method(), content_sha256)
|
||||
.map_err(|_| tonic::Status::unauthenticated("No valid auth token"))?;
|
||||
req.metadata_mut().as_mut().extend(headers);
|
||||
inject_trace_context_into_metadata(req.metadata_mut());
|
||||
@@ -125,7 +141,7 @@ impl tonic::service::Interceptor for TonicSignatureInterceptor {
|
||||
}
|
||||
|
||||
pub fn gen_tonic_signature_interceptor() -> TonicSignatureInterceptor {
|
||||
TonicSignatureInterceptor
|
||||
TonicSignatureInterceptor { audience: None }
|
||||
}
|
||||
|
||||
pub struct NoOpInterceptor;
|
||||
@@ -141,6 +157,22 @@ pub enum TonicInterceptor {
|
||||
NoOp(NoOpInterceptor),
|
||||
}
|
||||
|
||||
impl TonicInterceptor {
|
||||
fn with_rpc_audience(mut self, addr: &str) -> std::io::Result<Self> {
|
||||
if let Self::Signature(interceptor) = &mut self {
|
||||
let uri = addr
|
||||
.parse::<Uri>()
|
||||
.map_err(|_| std::io::Error::other("Invalid gRPC peer URI"))?;
|
||||
let audience = uri
|
||||
.authority()
|
||||
.map(|authority| normalize_tonic_rpc_audience(authority.as_str()))
|
||||
.ok_or_else(|| std::io::Error::other("Missing gRPC peer authority"))?;
|
||||
interceptor.audience = Some(audience?);
|
||||
}
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl tonic::service::Interceptor for TonicInterceptor {
|
||||
fn call(&mut self, req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
|
||||
match self {
|
||||
@@ -165,6 +197,20 @@ mod tests {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
}
|
||||
|
||||
fn test_request() -> tonic::Request<()> {
|
||||
let mut request = tonic::Request::new(());
|
||||
request
|
||||
.extensions_mut()
|
||||
.insert(tonic::GrpcMethod::new("node_service.NodeService", "Ping"));
|
||||
request
|
||||
}
|
||||
|
||||
fn test_interceptor() -> TonicSignatureInterceptor {
|
||||
TonicSignatureInterceptor {
|
||||
audience: Some("node-a:9000".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_trace_parent<F>(trace_id_hex: &str, f: F)
|
||||
where
|
||||
F: FnOnce(),
|
||||
@@ -192,20 +238,55 @@ mod tests {
|
||||
#[test]
|
||||
fn test_signature_interceptor_keeps_auth_headers() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut interceptor = TonicSignatureInterceptor;
|
||||
let req = tonic::Request::new(());
|
||||
let mut interceptor = test_interceptor();
|
||||
let req = test_request();
|
||||
|
||||
let req = interceptor.call(req).expect("interceptor call should succeed");
|
||||
|
||||
assert!(req.metadata().contains_key("x-rustfs-signature"));
|
||||
assert!(req.metadata().contains_key("x-rustfs-timestamp"));
|
||||
assert!(req.metadata().contains_key("x-rustfs-rpc-signature-v2"));
|
||||
assert!(req.metadata().contains_key("x-rustfs-rpc-nonce"));
|
||||
assert!(
|
||||
crate::cluster::rpc::verify_tonic_rpc_signature(
|
||||
"node-a:9000",
|
||||
"/node_service.NodeService/Ping",
|
||||
req.metadata().as_ref(),
|
||||
)
|
||||
.is_ok(),
|
||||
"interceptor signature should bind the configured peer audience and generated method"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_signature_interceptor_binds_audience_from_peer_uri() {
|
||||
let interceptor = TonicInterceptor::Signature(gen_tonic_signature_interceptor())
|
||||
.with_rpc_audience("http://node-a:9000")
|
||||
.expect("peer URI should provide an audience");
|
||||
let TonicInterceptor::Signature(interceptor) = interceptor else {
|
||||
panic!("signature interceptor variant should be preserved");
|
||||
};
|
||||
|
||||
assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_signature_interceptor_requires_generated_method_metadata() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut interceptor = test_interceptor();
|
||||
let error = interceptor
|
||||
.call(tonic::Request::new(()))
|
||||
.expect_err("requests without an exact generated method must fail closed");
|
||||
|
||||
assert_eq!(error.code(), tonic::Code::Unauthenticated);
|
||||
assert_eq!(error.message(), "Missing gRPC method metadata");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_signature_interceptor_may_inject_request_id() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut interceptor = TonicSignatureInterceptor;
|
||||
let req = tonic::Request::new(());
|
||||
let mut interceptor = test_interceptor();
|
||||
let req = test_request();
|
||||
|
||||
let span = tracing::info_span!("grpc-rpc-test-span");
|
||||
let _guard = span.enter();
|
||||
@@ -219,8 +300,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_signature_interceptor_injects_traceparent_metadata() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut interceptor = TonicSignatureInterceptor;
|
||||
let req = tonic::Request::new(());
|
||||
let mut interceptor = test_interceptor();
|
||||
let req = test_request();
|
||||
|
||||
with_trace_parent("4bf92f3577b34da6a3ce929d0e0e4736", || {
|
||||
let req = interceptor.call(req).expect("interceptor call should succeed");
|
||||
|
||||
@@ -19,8 +19,9 @@
|
||||
//! GHSA-r5qv-rc46-hv8q (internode RPC authentication must fail closed, fixed in
|
||||
//! rustfs/rustfs#4402) is anchored by the `ghsa_r5qv_*` tests in the module
|
||||
//! below, plus the broader negative-signature suite. The advisory class is: a
|
||||
//! node must never accept an RPC whose auth is missing, malformed, replayed, or
|
||||
//! signed with the default/empty shared secret. See
|
||||
//! node must never accept an RPC whose auth is missing, malformed, or signed
|
||||
//! with the default/empty shared secret. Body-bound v2 requests additionally
|
||||
//! receive process-local replay protection. See
|
||||
//! `docs/testing/security-regressions.md` for the full advisory -> test map.
|
||||
//!
|
||||
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
|
||||
@@ -29,23 +30,89 @@ use crate::cluster::rpc::context_propagation::{inject_request_id_into_http_heade
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose;
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use http::uri::Authority;
|
||||
use http::{HeaderMap, HeaderValue, Method, Uri};
|
||||
#[cfg(test)]
|
||||
use rustfs_credentials::{DEFAULT_SECRET_KEY, RPC_SECRET_REQUIRED_MESSAGE};
|
||||
use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token};
|
||||
use sha2::Digest as _;
|
||||
use sha2::Sha256;
|
||||
use std::sync::Once;
|
||||
use std::collections::{HashSet, VecDeque};
|
||||
use std::sync::{LazyLock, Mutex, Once};
|
||||
use std::time::{Duration, Instant};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::error;
|
||||
use uuid::Uuid;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||
const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
|
||||
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
|
||||
const RPC_NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
|
||||
pub(crate) const RPC_CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
|
||||
const RPC_AUTH_VERSION_V2: &str = "2";
|
||||
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
|
||||
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
|
||||
const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
|
||||
const MAX_REPLAY_PROTECTED_NONCES: usize = 65_536;
|
||||
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
|
||||
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
|
||||
|
||||
#[derive(Default)]
|
||||
struct RpcNonceCache {
|
||||
nonces: HashSet<Uuid>,
|
||||
expirations: VecDeque<(Instant, i64, Uuid)>,
|
||||
max_wall_time: i64,
|
||||
}
|
||||
|
||||
impl RpcNonceCache {
|
||||
fn remove_expired(&mut self, now: Instant, wall_time: i64) {
|
||||
while matches!(
|
||||
self.expirations.front(),
|
||||
Some((expires_at, valid_until, _)) if *expires_at < now && *valid_until < wall_time
|
||||
) {
|
||||
let Some((_, _, nonce)) = self.expirations.pop_front() else {
|
||||
break;
|
||||
};
|
||||
self.nonces.remove(&nonce);
|
||||
}
|
||||
}
|
||||
|
||||
fn check_and_record(
|
||||
&mut self,
|
||||
nonce: Uuid,
|
||||
signed_at: i64,
|
||||
now: Instant,
|
||||
wall_time: i64,
|
||||
expires_at: Instant,
|
||||
capacity: usize,
|
||||
) -> std::io::Result<()> {
|
||||
self.max_wall_time = self.max_wall_time.max(wall_time);
|
||||
if self.max_wall_time.saturating_sub(signed_at) > SIGNATURE_VALID_DURATION {
|
||||
return Err(std::io::Error::other("RPC request timestamp expired after clock regression"));
|
||||
}
|
||||
self.remove_expired(now, self.max_wall_time);
|
||||
if self.nonces.contains(&nonce) {
|
||||
return Err(std::io::Error::other("RPC request replay detected"));
|
||||
}
|
||||
if self.nonces.len() >= capacity {
|
||||
return Err(std::io::Error::other("RPC replay cache capacity exceeded"));
|
||||
}
|
||||
self.nonces.insert(nonce);
|
||||
self.expirations
|
||||
.push_back((expires_at, signed_at.saturating_add(SIGNATURE_VALID_DURATION), nonce));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// This cache is a process-local wire-replay defense only. Mutation handlers
|
||||
// still need a stable operation ID and coordinator-owned idempotency across
|
||||
// retries, node failover, and restart.
|
||||
static LOCAL_RPC_NONCE_CACHE: LazyLock<Mutex<RpcNonceCache>> = LazyLock::new(|| Mutex::new(RpcNonceCache::default()));
|
||||
|
||||
/// Get the shared secret for HMAC signing
|
||||
#[cfg(test)]
|
||||
fn resolve_shared_secret(env_secret: Option<&str>, global_secret: Option<&str>) -> std::io::Result<String> {
|
||||
@@ -109,6 +176,96 @@ fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, si
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct SignatureV2Scope<'a> {
|
||||
audience: &'a str,
|
||||
service: &'a str,
|
||||
rpc_method: &'a str,
|
||||
timestamp: &'a str,
|
||||
nonce: &'a str,
|
||||
content_sha256: &'a str,
|
||||
}
|
||||
|
||||
fn update_signature_v2(mac: &mut HmacSha256, scope: SignatureV2Scope<'_>) {
|
||||
for part in [
|
||||
b"rustfs-rpc-auth-v2|".as_slice(),
|
||||
scope.audience.as_bytes(),
|
||||
b"|/",
|
||||
scope.service.as_bytes(),
|
||||
b"/",
|
||||
scope.rpc_method.as_bytes(),
|
||||
b"|POST|",
|
||||
scope.timestamp.as_bytes(),
|
||||
b"|",
|
||||
scope.nonce.as_bytes(),
|
||||
b"|",
|
||||
scope.content_sha256.as_bytes(),
|
||||
] {
|
||||
mac.update(part);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_signature_v2(secret: &str, scope: SignatureV2Scope<'_>) -> std::io::Result<String> {
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_signature_v2(&mut mac, scope);
|
||||
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &str) -> bool {
|
||||
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
|
||||
return false;
|
||||
};
|
||||
update_signature_v2(&mut mac, scope);
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
|
||||
fn valid_content_sha256(value: &str) -> bool {
|
||||
value == UNSIGNED_PAYLOAD
|
||||
|| (value.len() == 64
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)))
|
||||
}
|
||||
|
||||
fn header_value(value: &str, name: &str) -> std::io::Result<HeaderValue> {
|
||||
HeaderValue::from_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name} header value")))
|
||||
}
|
||||
|
||||
pub fn normalize_tonic_rpc_audience(value: &str) -> std::io::Result<String> {
|
||||
let authority = value
|
||||
.parse::<Authority>()
|
||||
.map_err(|_| std::io::Error::other("Invalid gRPC peer authority"))?;
|
||||
Ok(authority.as_str().to_ascii_lowercase())
|
||||
}
|
||||
|
||||
fn check_timestamp(timestamp: i64) -> std::io::Result<()> {
|
||||
let current_time = OffsetDateTime::now_utc().unix_timestamp();
|
||||
if current_time.saturating_sub(timestamp) > SIGNATURE_VALID_DURATION
|
||||
|| timestamp.saturating_sub(current_time) > SIGNATURE_VALID_DURATION
|
||||
{
|
||||
return Err(std::io::Error::other("Request timestamp expired"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_and_record_nonce(nonce: Uuid, signed_at: i64) -> std::io::Result<()> {
|
||||
let wall_time = OffsetDateTime::now_utc().unix_timestamp();
|
||||
let mut cache = LOCAL_RPC_NONCE_CACHE
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("RPC replay cache unavailable"))?;
|
||||
// Take the monotonic timestamp after acquiring the lock so expiration
|
||||
// entries remain ordered by the same serialization point as insertion.
|
||||
let now = Instant::now();
|
||||
let expires_at = now
|
||||
.checked_add(REPLAY_CACHE_RETENTION)
|
||||
.ok_or_else(|| std::io::Error::other("RPC replay expiry overflow"))?;
|
||||
cache.check_and_record(nonce, signed_at, now, wall_time, expires_at, MAX_REPLAY_PROTECTED_NONCES)
|
||||
}
|
||||
|
||||
/// Build headers with authentication signature
|
||||
pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) -> std::io::Result<()> {
|
||||
let auth_headers = gen_signature_headers(url, method)?;
|
||||
@@ -135,6 +292,184 @@ pub fn gen_signature_headers(url: &str, method: &Method) -> std::io::Result<Head
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Generate rolling-upgrade-safe gRPC auth metadata.
|
||||
///
|
||||
/// The legacy signature remains present for old servers. New servers prefer the
|
||||
/// v2 signature and bind it to the destination authority and exact generated
|
||||
/// gRPC method. A versioned canonical mutation payload can opt into the
|
||||
/// additional body-digest capability.
|
||||
pub fn gen_tonic_signature_headers(
|
||||
audience: &str,
|
||||
service: &str,
|
||||
rpc_method: &str,
|
||||
content_sha256: Option<&str>,
|
||||
) -> std::io::Result<HeaderMap> {
|
||||
if audience.is_empty() || service.is_empty() || rpc_method.is_empty() || service.contains('/') || rpc_method.contains('/') {
|
||||
return Err(std::io::Error::other("Invalid RPC v2 signing scope"));
|
||||
}
|
||||
let content_sha256 = content_sha256.unwrap_or(UNSIGNED_PAYLOAD);
|
||||
if !valid_content_sha256(content_sha256) {
|
||||
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
|
||||
}
|
||||
|
||||
let secret = get_shared_secret()?;
|
||||
let timestamp = OffsetDateTime::now_utc().unix_timestamp();
|
||||
let timestamp_header = timestamp.to_string();
|
||||
let body_nonce = (content_sha256 != UNSIGNED_PAYLOAD).then(|| Uuid::new_v4().to_string());
|
||||
let nonce = body_nonce.as_deref().unwrap_or(UNSIGNED_PAYLOAD_NONCE);
|
||||
let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp);
|
||||
let signature_v2 = generate_signature_v2(
|
||||
&secret,
|
||||
SignatureV2Scope {
|
||||
audience,
|
||||
service,
|
||||
rpc_method,
|
||||
timestamp: ×tamp_header,
|
||||
nonce,
|
||||
content_sha256,
|
||||
},
|
||||
)?;
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(SIGNATURE_HEADER, header_value(&legacy_signature, SIGNATURE_HEADER)?);
|
||||
headers.insert(TIMESTAMP_HEADER, header_value(×tamp_header, TIMESTAMP_HEADER)?);
|
||||
headers.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
|
||||
headers.insert(RPC_SIGNATURE_V2_HEADER, header_value(&signature_v2, RPC_SIGNATURE_V2_HEADER)?);
|
||||
headers.insert(RPC_NONCE_HEADER, header_value(nonce, RPC_NONCE_HEADER)?);
|
||||
headers.insert(RPC_CONTENT_SHA256_HEADER, header_value(content_sha256, RPC_CONTENT_SHA256_HEADER)?);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Bind a mutation to a versioned, deterministic canonical payload.
|
||||
///
|
||||
/// Do not pass a protobuf re-encoding here: unknown fields and map ordering are
|
||||
/// not a stable mixed-version contract.
|
||||
pub fn set_tonic_canonical_body_digest<T>(request: &mut tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
let digest = hex_simd::encode_to_string(Sha256::digest(canonical_body), hex_simd::AsciiCase::Lower);
|
||||
request
|
||||
.metadata_mut()
|
||||
.as_mut()
|
||||
.insert(RPC_CONTENT_SHA256_HEADER, header_value(&digest, RPC_CONTENT_SHA256_HEADER)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
let version = request
|
||||
.metadata()
|
||||
.get(RPC_AUTH_VERSION_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
if version != Some(RPC_AUTH_VERSION_V2) {
|
||||
return Err(std::io::Error::other("RPC mutation requires v2 authentication"));
|
||||
}
|
||||
let expected = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
|
||||
if expected == UNSIGNED_PAYLOAD || !valid_content_sha256(expected) {
|
||||
return Err(std::io::Error::other("RPC body is not bound to the signature"));
|
||||
}
|
||||
let actual = hex_simd::encode_to_string(Sha256::digest(canonical_body), hex_simd::AsciiCase::Lower);
|
||||
if actual != expected {
|
||||
return Err(std::io::Error::other("RPC content SHA-256 mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
|
||||
[
|
||||
RPC_AUTH_VERSION_HEADER,
|
||||
RPC_SIGNATURE_V2_HEADER,
|
||||
RPC_NONCE_HEADER,
|
||||
RPC_CONTENT_SHA256_HEADER,
|
||||
]
|
||||
.iter()
|
||||
.any(|name| headers.contains_key(*name))
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
|
||||
if !has_v2_auth_headers(headers) {
|
||||
// RUSTFS_COMPAT_TODO(heal-rpc-auth-v2): accept old peers during rolling upgrades. Remove after the minimum
|
||||
// supported RustFS peer version sends v2 authentication on every internode gRPC request.
|
||||
return verify_rpc_signature(TONIC_RPC_PREFIX, &Method::GET, headers);
|
||||
}
|
||||
|
||||
let path = path
|
||||
.strip_prefix('/')
|
||||
.ok_or_else(|| std::io::Error::other("Invalid RPC request path"))?;
|
||||
let (service, rpc_method) = path
|
||||
.split_once('/')
|
||||
.filter(|(service, rpc_method)| !service.is_empty() && !rpc_method.is_empty() && !rpc_method.contains('/'))
|
||||
.ok_or_else(|| std::io::Error::other("Invalid RPC request path"))?;
|
||||
if audience.is_empty() {
|
||||
return Err(std::io::Error::other("Missing RPC audience"));
|
||||
}
|
||||
|
||||
let version = headers
|
||||
.get(RPC_AUTH_VERSION_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC auth version"))?;
|
||||
if version != RPC_AUTH_VERSION_V2 {
|
||||
return Err(std::io::Error::other("Unsupported RPC auth version"));
|
||||
}
|
||||
let signature = headers
|
||||
.get(RPC_SIGNATURE_V2_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC v2 signature"))?;
|
||||
let timestamp_header = headers
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
|
||||
let timestamp = timestamp_header
|
||||
.parse::<i64>()
|
||||
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||
check_timestamp(timestamp)?;
|
||||
let nonce = headers
|
||||
.get(RPC_NONCE_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC nonce"))?;
|
||||
let content_sha256 = headers
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
|
||||
if !valid_content_sha256(content_sha256) {
|
||||
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
|
||||
}
|
||||
let parsed_nonce = if content_sha256 == UNSIGNED_PAYLOAD {
|
||||
if nonce != UNSIGNED_PAYLOAD_NONCE {
|
||||
return Err(std::io::Error::other("Invalid unsigned RPC nonce"));
|
||||
}
|
||||
None
|
||||
} else {
|
||||
let parsed_nonce = Uuid::parse_str(nonce).map_err(|_| std::io::Error::other("Invalid RPC nonce"))?;
|
||||
if parsed_nonce.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid RPC nonce"));
|
||||
}
|
||||
Some(parsed_nonce)
|
||||
};
|
||||
|
||||
let secret = get_shared_secret()?;
|
||||
if !verify_signature_v2(
|
||||
&secret,
|
||||
SignatureV2Scope {
|
||||
audience,
|
||||
service,
|
||||
rpc_method,
|
||||
timestamp: timestamp_header,
|
||||
nonce,
|
||||
content_sha256,
|
||||
},
|
||||
signature,
|
||||
) {
|
||||
return Err(std::io::Error::other("Invalid RPC v2 signature"));
|
||||
}
|
||||
if let Some(nonce) = parsed_nonce {
|
||||
check_and_record_nonce(nonce, timestamp)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify the request signature for RPC requests
|
||||
pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) -> std::io::Result<()> {
|
||||
// Get signature from header
|
||||
@@ -153,14 +488,7 @@ pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) ->
|
||||
.parse()
|
||||
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||
|
||||
// Check timestamp validity (prevent replay attacks)
|
||||
let current_time = OffsetDateTime::now_utc().unix_timestamp();
|
||||
|
||||
if current_time.saturating_sub(timestamp) > SIGNATURE_VALID_DURATION
|
||||
|| timestamp.saturating_sub(current_time) > SIGNATURE_VALID_DURATION
|
||||
{
|
||||
return Err(std::io::Error::other("Request timestamp expired"));
|
||||
}
|
||||
check_timestamp(timestamp)?;
|
||||
|
||||
// Verify signature with constant-time HMAC comparison.
|
||||
let secret = get_shared_secret()?;
|
||||
@@ -707,4 +1035,141 @@ mod tests {
|
||||
assert!(result.is_ok(), "Round-trip test failed for {method} {url}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_v2_signature_is_bound_to_exact_method() {
|
||||
ensure_test_rpc_secret();
|
||||
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("tonic auth headers should build");
|
||||
|
||||
assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers).is_ok());
|
||||
let error = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
|
||||
.expect_err("signature replayed to a different method must fail");
|
||||
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_v2_signature_is_bound_to_destination_audience() {
|
||||
ensure_test_rpc_secret();
|
||||
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("tonic auth headers should build");
|
||||
|
||||
let error = verify_tonic_rpc_signature("node-b:9000", "/node_service.NodeService/Ping", &headers)
|
||||
.expect_err("signature replayed to a different node must fail");
|
||||
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("tonic auth headers should build");
|
||||
headers.insert(RPC_SIGNATURE_V2_HEADER, HeaderValue::from_static("invalid"));
|
||||
|
||||
assert!(
|
||||
verify_rpc_signature(TONIC_RPC_PREFIX, &Method::GET, &headers).is_ok(),
|
||||
"the compatibility signature should remain valid for old servers"
|
||||
);
|
||||
let error = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers)
|
||||
.expect_err("new servers must not downgrade malformed v2 auth");
|
||||
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_tonic_signature_remains_accepted_during_rolling_upgrade() {
|
||||
ensure_test_rpc_secret();
|
||||
let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build");
|
||||
|
||||
assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_bound_tonic_request_rejects_replay_and_body_tampering() {
|
||||
ensure_test_rpc_secret();
|
||||
let body = b"heal-control-request";
|
||||
let mut request = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut request, body).expect("canonical body digest should be attached");
|
||||
let content_sha256 = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "HealControl", content_sha256)
|
||||
.expect("body-bound auth headers should build");
|
||||
request.metadata_mut().as_mut().extend(headers.clone());
|
||||
|
||||
assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/HealControl", &headers).is_ok());
|
||||
let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/HealControl", &headers)
|
||||
.expect_err("reusing a body-bound nonce must fail");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
assert!(verify_tonic_canonical_body_digest(&request, body).is_ok());
|
||||
let tampered = verify_tonic_canonical_body_digest(&request, b"different-body")
|
||||
.expect_err("a different canonical request body must fail");
|
||||
assert_eq!(tampered.to_string(), "RPC content SHA-256 mismatch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_v2_metadata_fails_closed() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build");
|
||||
headers.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
|
||||
|
||||
let error = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers)
|
||||
.expect_err("partial v2 metadata must not fall back to legacy auth");
|
||||
assert_eq!(error.to_string(), "Missing RPC v2 signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_mutation_digest_rejects_legacy_only_auth() {
|
||||
let mut request = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut request, b"heal-control-v1\0start").expect("canonical digest should be attached");
|
||||
|
||||
let error = verify_tonic_canonical_body_digest(&request, b"heal-control-v1\0start")
|
||||
.expect_err("mutation body verification must also require v2 auth");
|
||||
assert_eq!(error.to_string(), "RPC mutation requires v2 authentication");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_expires_by_monotonic_deadline_and_fails_closed_at_capacity() {
|
||||
let now = Instant::now();
|
||||
let expiry = now.checked_add(REPLAY_CACHE_RETENTION).expect("test expiry should fit");
|
||||
let after_expiry = expiry.checked_add(Duration::from_secs(1)).expect("test expiry should fit");
|
||||
let nonce_a = Uuid::new_v4();
|
||||
let nonce_b = Uuid::new_v4();
|
||||
let mut cache = RpcNonceCache::default();
|
||||
|
||||
cache
|
||||
.check_and_record(nonce_a, 100, now, 100, expiry, 1)
|
||||
.expect("first nonce should be recorded");
|
||||
let capacity = cache
|
||||
.check_and_record(nonce_b, 100, now, 100, expiry, 1)
|
||||
.expect_err("a full replay cache must fail closed");
|
||||
assert_eq!(capacity.to_string(), "RPC replay cache capacity exceeded");
|
||||
cache
|
||||
.check_and_record(nonce_b, 702, after_expiry, 702, after_expiry, 1)
|
||||
.expect("expired nonce should release capacity");
|
||||
assert!(!cache.nonces.contains(&nonce_a));
|
||||
assert!(cache.nonces.contains(&nonce_b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_rejects_replay_after_wall_clock_regression() {
|
||||
let now = Instant::now();
|
||||
let expiry = now.checked_add(REPLAY_CACHE_RETENTION).expect("test expiry should fit");
|
||||
let after_expiry = expiry.checked_add(Duration::from_secs(1)).expect("test expiry should fit");
|
||||
let nonce = Uuid::new_v4();
|
||||
let mut cache = RpcNonceCache::default();
|
||||
|
||||
cache
|
||||
.check_and_record(nonce, 1_000, now, 1_000, expiry, 2)
|
||||
.expect("first nonce should be recorded");
|
||||
let replay = cache
|
||||
.check_and_record(nonce, 1_000, after_expiry, 900, after_expiry, 2)
|
||||
.expect_err("wall clock regression must not make an old signature reusable");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
|
||||
let stale = cache
|
||||
.check_and_record(Uuid::new_v4(), 600, after_expiry, 900, after_expiry, 2)
|
||||
.expect_err("the monotonic wall-clock high-water mark must fail closed");
|
||||
assert_eq!(stale.to_string(), "RPC request timestamp expired after clock regression");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,7 +28,10 @@ pub(crate) use background_monitor::spawn_background_monitor;
|
||||
pub use client::{
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
};
|
||||
pub use http_auth::{TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, verify_rpc_signature};
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
pub use internode_data_transport::build_internode_data_transport_from_env;
|
||||
|
||||
@@ -176,12 +176,28 @@ pub fn init_lock_clients(endpoint_pools: EndpointServerPools) {
|
||||
}
|
||||
}
|
||||
|
||||
fn endpoint_rpc_authority(endpoint: &Endpoint) -> Option<String> {
|
||||
let host = endpoint.url.host_str()?;
|
||||
let host = if host.contains(':') && !host.starts_with('[') {
|
||||
format!("[{host}]")
|
||||
} else {
|
||||
host.to_string()
|
||||
};
|
||||
Some(match endpoint.url.port() {
|
||||
Some(port) => format!("{host}:{port}"),
|
||||
None => host,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn init_local_peer(endpoint_pools: &EndpointServerPools, host: &String, port: &String) {
|
||||
let mut peer_set = Vec::new();
|
||||
endpoint_pools.as_ref().iter().for_each(|endpoints| {
|
||||
endpoints.endpoints.as_ref().iter().for_each(|endpoint| {
|
||||
if endpoint.get_type() == EndpointType::Url && endpoint.is_local && endpoint.url.has_host() {
|
||||
peer_set.push(endpoint.url.host_str().unwrap().to_string());
|
||||
if endpoint.get_type() == EndpointType::Url
|
||||
&& endpoint.is_local
|
||||
&& let Some(authority) = endpoint_rpc_authority(endpoint)
|
||||
{
|
||||
peer_set.push(authority);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -234,6 +250,37 @@ mod tests {
|
||||
}])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_rpc_authority_preserves_port_and_ipv6_brackets() {
|
||||
let endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");
|
||||
assert_eq!(endpoint_rpc_authority(&endpoint).as_deref(), Some("127.0.0.1:9001"));
|
||||
|
||||
let endpoint = Endpoint::try_from("https://[::1]:9002/d1").expect("IPv6 URL endpoint");
|
||||
assert_eq!(endpoint_rpc_authority(&endpoint).as_deref(), Some("[::1]:9002"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn init_local_peer_publishes_complete_rpc_authority() {
|
||||
let previous = rustfs_common::get_global_local_node_name().await;
|
||||
let mut endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");
|
||||
endpoint.is_local = true;
|
||||
let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 1,
|
||||
endpoints: Endpoints::from(vec![endpoint]),
|
||||
cmd_line: "rpc-authority-test".to_string(),
|
||||
platform: "test".to_string(),
|
||||
}]);
|
||||
|
||||
let host = String::new();
|
||||
let port = "9000".to_string();
|
||||
init_local_peer(&endpoint_pools, &host, &port).await;
|
||||
assert_eq!(rustfs_common::try_get_global_local_node_name().as_deref(), Some("127.0.0.1:9001"));
|
||||
rustfs_common::set_global_local_node_name(&previous).await;
|
||||
}
|
||||
|
||||
// Phase 5 follow-up (backlog#1052): registering local disks through the
|
||||
// ctx-explicit entry writes the passed context's registry only — the
|
||||
// process bootstrap context (and any other instance) stays clean, so a
|
||||
|
||||
Reference in New Issue
Block a user