diff --git a/crates/config/src/constants/internode.rs b/crates/config/src/constants/internode.rs index 8814b8b3e..aecbb10cf 100644 --- a/crates/config/src/constants/internode.rs +++ b/crates/config/src/constants/internode.rs @@ -118,6 +118,24 @@ pub const DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED: bool = false; const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY); const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED); +/// Require target-bound v2 signatures on every internode gRPC request, rejecting the legacy +/// constant-target fallback instead of accepting it (). +/// +/// Defaults to `false` (fail-open): a request without any v2 auth headers keeps authenticating +/// through the legacy signature, so legacy-only peers survive rolling upgrades with byte-for-byte +/// the pre-gate acceptance behavior. This is a rollout lever, not a wire-format change: it may only +/// be enabled **after** the v1-fallback counter +/// (`rustfs_system_network_internode_signature_v1_fallback_total`) has read zero across a release +/// window fleet-wide, confirming every peer already sends v2 authentication on every internode gRPC +/// request. Single-env rollback. Requests that do carry v2 headers are unaffected by this switch: +/// they are always verified as v2 with no downgrade, strict or not. +pub const ENV_INTERNODE_RPC_SIGNATURE_STRICT: &str = "RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT"; +pub const DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT: bool = false; + +// Compile-time invariant: fail-open by default so legacy-only peers keep authenticating during +// rolling upgrades until the fleet-wide v1-fallback counter reads zero. +const _: () = assert!(!DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT); + /// Consecutive-failure threshold after which an internode peer is marked offline (grpc-optimization /// P3 observability). /// @@ -288,6 +306,12 @@ mod tests { ); } + #[test] + fn internode_signature_strict_env_name_is_stable() { + // The fail-open default invariant is asserted at compile time next to the definition. + assert_eq!(ENV_INTERNODE_RPC_SIGNATURE_STRICT, "RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT"); + } + #[test] fn internode_offline_failure_threshold_defaults_and_env_name() { assert_eq!(DEFAULT_INTERNODE_OFFLINE_FAILURE_THRESHOLD, 3); diff --git a/crates/ecstore/src/cluster/rpc/http_auth.rs b/crates/ecstore/src/cluster/rpc/http_auth.rs index 1b1802795..1c387160e 100644 --- a/crates/ecstore/src/cluster/rpc/http_auth.rs +++ b/crates/ecstore/src/cluster/rpc/http_auth.rs @@ -35,6 +35,8 @@ use http::{HeaderMap, HeaderValue, Method, Uri}; #[cfg(test)] use rustfs_credentials::{DEFAULT_SECRET_KEY, RPC_SECRET_REQUIRED_MESSAGE}; use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token}; +use rustfs_io_metrics::internode_metrics::global_internode_metrics; +use rustfs_utils::get_env_bool; use sha2::Digest as _; use sha2::Sha256; use std::collections::{HashSet, VecDeque}; @@ -60,6 +62,12 @@ const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601); const MAX_REPLAY_PROTECTED_NONCES: usize = 65_536; pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService"; +static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock = LazyLock::new(|| { + get_env_bool( + rustfs_config::ENV_INTERNODE_RPC_SIGNATURE_STRICT, + rustfs_config::DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT, + ) +}); static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new(); #[derive(Default)] @@ -412,12 +420,40 @@ fn has_v2_auth_headers(headers: &HeaderMap) -> bool { .any(|name| headers.contains_key(*name)) } +/// Whether the server requires target-bound v2 authentication on every internode gRPC request, +/// rejecting the legacy constant-target fallback instead of accepting it. Default-off rollout +/// lever gated on the v1-fallback counter reading zero fleet-wide; see +/// [`rustfs_config::ENV_INTERNODE_RPC_SIGNATURE_STRICT`] and +/// . +fn internode_rpc_signature_strict() -> bool { + *INTERNODE_RPC_SIGNATURE_STRICT +} + /// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata. pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> { + verify_tonic_rpc_signature_with_strictness(audience, path, headers, internode_rpc_signature_strict()) +} + +/// [`verify_tonic_rpc_signature`] with the strict gate injected as a parameter, so both rollout +/// postures are unit-testable without racing on process-global environment variables. +fn verify_tonic_rpc_signature_with_strictness( + audience: &str, + path: &str, + headers: &HeaderMap, + strict: bool, +) -> std::io::Result<()> { if !has_v2_auth_headers(headers) { // RUSTFS_COMPAT_TODO(heal-rpc-auth-v2): accept old peers during rolling upgrades. Remove after the minimum // supported RustFS peer version sends v2 authentication on every internode gRPC request. - return verify_rpc_signature(TONIC_RPC_PREFIX, &Method::GET, headers); + if strict { + return Err(std::io::Error::other("RPC v2 authentication required")); + } + verify_rpc_signature(TONIC_RPC_PREFIX, &Method::GET, headers)?; + // Count only ACCEPTED legacy-only requests: this counter is the convergence gate that must + // read zero fleet-wide across a release window before + // `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT` may be enabled. + global_internode_metrics().record_signature_v1_fallback(); + return Ok(()); } let path = path @@ -1111,7 +1147,10 @@ mod tests { assert_eq!(error.to_string(), "Invalid RPC v2 signature"); } + // The `rpc_v1_fallback_counter` serial group covers every test that drives (or asserts on) the + // process-global v1-fallback counter, so exact-delta assertions cannot race with each other. #[test] + #[serial_test::serial(rpc_v1_fallback_counter)] fn legacy_tonic_signature_remains_accepted_during_rolling_upgrade() { ensure_test_rpc_secret(); let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build"); @@ -1119,6 +1158,87 @@ mod tests { assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers).is_ok()); } + #[test] + #[serial_test::serial(rpc_v1_fallback_counter)] + fn accepted_legacy_fallback_increments_v1_fallback_counter() { + ensure_test_rpc_secret(); + let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build"); + let before = global_internode_metrics().snapshot().signature_v1_fallback_total; + + assert!( + verify_tonic_rpc_signature_with_strictness("node-a:9000", "/node_service.NodeService/Ping", &headers, false).is_ok(), + "a legacy-only peer must keep authenticating while the strict gate is off" + ); + + let after = global_internode_metrics().snapshot().signature_v1_fallback_total; + assert_eq!( + after, + before + 1, + "an accepted legacy-only request must increment the v1 fallback counter exactly once" + ); + } + + #[test] + #[serial_test::serial(rpc_v1_fallback_counter)] + fn rejected_legacy_fallback_does_not_count_as_v1_fallback() { + ensure_test_rpc_secret(); + // Legacy-shaped headers with a forged signature: the fallback path runs but must reject, + // and a rejected request is not a rollout-convergence signal. + let mut headers = HeaderMap::new(); + let now = OffsetDateTime::now_utc().unix_timestamp(); + headers.insert(SIGNATURE_HEADER, HeaderValue::from_static("not-a-real-signature")); + headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&now.to_string()).unwrap()); + let before = global_internode_metrics().snapshot().signature_v1_fallback_total; + + assert!( + verify_tonic_rpc_signature_with_strictness("node-a:9000", "/node_service.NodeService/Ping", &headers, false).is_err(), + "a forged legacy signature must still be rejected" + ); + + let after = global_internode_metrics().snapshot().signature_v1_fallback_total; + assert_eq!(after, before, "a rejected legacy request must not count as an accepted fallback"); + } + + #[test] + #[serial_test::serial(rpc_v1_fallback_counter)] + fn strict_gate_rejects_legacy_only_auth_but_keeps_v2() { + ensure_test_rpc_secret(); + let legacy = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build"); + let before = global_internode_metrics().snapshot().signature_v1_fallback_total; + let error = verify_tonic_rpc_signature_with_strictness("node-a:9000", "/node_service.NodeService/Ping", &legacy, true) + .expect_err("strict mode must reject legacy-only authentication"); + assert_eq!(error.to_string(), "RPC v2 authentication required"); + + let v2 = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None) + .expect("tonic auth headers should build"); + assert!( + verify_tonic_rpc_signature_with_strictness("node-a:9000", "/node_service.NodeService/Ping", &v2, true).is_ok(), + "strict mode must keep accepting v2-authenticated peers" + ); + let after = global_internode_metrics().snapshot().signature_v1_fallback_total; + assert_eq!(after, before, "neither a strict rejection nor a v2 acceptance is a legacy fallback"); + } + + #[test] + #[serial_test::serial(rpc_v1_fallback_counter)] + fn strict_gate_default_posture_is_fail_open_legacy_accept() { + ensure_test_rpc_secret(); + // The public entry point resolves strictness from the environment, whose compile-time + // default is pinned to false in `rustfs_config`. A legacy-only peer therefore keeps + // authenticating through the default build with no configuration at all. + let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build"); + assert!( + verify_tonic_rpc_signature_with_strictness( + "node-a:9000", + "/node_service.NodeService/Ping", + &headers, + rustfs_config::DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT, + ) + .is_ok(), + "the default strict posture must accept legacy-only peers" + ); + } + #[test] fn body_bound_tonic_request_rejects_replay_and_body_tampering() { ensure_test_rpc_secret(); diff --git a/crates/io-metrics/src/internode_metrics.rs b/crates/io-metrics/src/internode_metrics.rs index d620e17b5..2c0985a56 100644 --- a/crates/io-metrics/src/internode_metrics.rs +++ b/crates/io-metrics/src/internode_metrics.rs @@ -59,6 +59,7 @@ const INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL: &str = const INTERNODE_OPERATION_PAYLOAD_BYTES: &str = "rustfs_system_network_internode_operation_payload_bytes"; const INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL: &str = "rustfs_system_network_internode_operation_large_payloads_total"; const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_fallback_total"; +const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total"; const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -148,6 +149,7 @@ pub struct InternodeMetricsSnapshot { pub operation_http_versions_total: u64, pub operation_stall_timeouts_total: u64, pub operation_write_shutdown_errors_total: u64, + pub signature_v1_fallback_total: u64, } #[derive(Debug, Default)] @@ -164,6 +166,7 @@ pub struct InternodeMetrics { operation_http_versions_total: AtomicU64, operation_stall_timeouts_total: AtomicU64, operation_write_shutdown_errors_total: AtomicU64, + signature_v1_fallback_total: AtomicU64, } impl InternodeMetrics { @@ -360,6 +363,17 @@ impl InternodeMetrics { counter!(INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL, DIRECTION_LABEL => direction, MESSAGE_LABEL => message).increment(1); } + /// Count an internode gRPC request that was accepted through the legacy constant-target + /// signature because it carried no v2 auth headers (rolling-upgrade fallback, see + /// ). Only accepted requests count: rejected + /// requests never authenticated, so they are not a rollout signal. This counter must read zero + /// across a release window fleet-wide before `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT` may be + /// enabled; after the strict flip the legacy fallback path is closed and the counter stays flat. + pub fn record_signature_v1_fallback(&self) { + self.signature_v1_fallback_total.fetch_add(1, Ordering::Relaxed); + counter!(INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL).increment(1); + } + pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) { counter!( ERASURE_WRITE_QUORUM_FAILURES_TOTAL, @@ -406,6 +420,7 @@ impl InternodeMetrics { operation_http_versions_total: self.operation_http_versions_total.load(Ordering::Relaxed), operation_stall_timeouts_total: self.operation_stall_timeouts_total.load(Ordering::Relaxed), operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed), + signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed), } } @@ -423,6 +438,7 @@ impl InternodeMetrics { self.operation_http_versions_total.store(0, Ordering::Relaxed); self.operation_stall_timeouts_total.store(0, Ordering::Relaxed); self.operation_write_shutdown_errors_total.store(0, Ordering::Relaxed); + self.signature_v1_fallback_total.store(0, Ordering::Relaxed); } } @@ -727,6 +743,10 @@ mod tests { ); assert_eq!(INTERNODE_MSGPACK_DIRECTION_REQUEST, "request"); assert_eq!(INTERNODE_MSGPACK_DIRECTION_RESPONSE, "response"); + assert_eq!( + INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL, + "rustfs_system_network_internode_signature_v1_fallback_total" + ); } #[test] @@ -737,6 +757,20 @@ mod tests { metrics.record_msgpack_json_fallback(INTERNODE_MSGPACK_DIRECTION_RESPONSE, "RawFileInfo"); } + #[test] + fn signature_v1_fallback_counter_updates_snapshot_and_resets() { + // Instance-local metrics keep this independent of the process-global registry. + let metrics = InternodeMetrics::default(); + assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0); + + metrics.record_signature_v1_fallback(); + metrics.record_signature_v1_fallback(); + assert_eq!(metrics.snapshot().signature_v1_fallback_total, 2); + + metrics.reset_for_test(); + assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0); + } + #[test] fn cluster_peer_flips_offline_after_threshold_and_back_online() { // Unique addr keeps this independent of the process-global registry / other tests. diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index d072c2c9e..895f43378 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -2368,6 +2368,48 @@ mod tests { rustfs_common::set_global_local_node_name(&previous_node_name).await; } + /// Rolling-upgrade compatibility anchor for : + /// a legacy-only peer (constant-target signature, no v2 headers) must keep authenticating + /// through the real production path (`check_auth` + `RpcRequestTarget` extension), and every + /// such acceptance must increment the v1-fallback convergence counter exactly once. That + /// counter reading zero fleet-wide is the precondition for ever enabling + /// `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT`. + #[tokio::test] + #[serial_test::serial] + async fn rpc_auth_accepts_legacy_only_peer_and_counts_v1_fallback() { + use rustfs_io_metrics::internode_metrics::global_internode_metrics; + + let _ = rustfs_credentials::set_global_rpc_secret("rpc-http-test-secret".to_string()); + let previous_node_name = rustfs_common::get_global_local_node_name().await; + rustfs_common::set_global_local_node_name("127.0.0.1:9000").await; + + // Exactly what an old peer sends on the gRPC plane: the legacy constant-target signature + // and timestamp headers, nothing else. + let legacy_headers = storage::gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy headers should build"); + let mut request = Request::new(()); + request.metadata_mut().as_mut().extend(legacy_headers); + request.extensions_mut().insert(RpcRequestTarget { + uri: "http://127.0.0.1:9000/node_service.NodeService/Ping" + .parse() + .expect("test RPC URI should parse"), + method: Method::POST, + }); + + let before = global_internode_metrics().snapshot().signature_v1_fallback_total; + assert!( + check_auth(request).is_ok(), + "a legacy-only peer must keep authenticating during rolling upgrades" + ); + let after = global_internode_metrics().snapshot().signature_v1_fallback_total; + assert_eq!( + after, + before + 1, + "an accepted legacy-only request must increment signature_v1_fallback_total exactly once" + ); + + rustfs_common::set_global_local_node_name(&previous_node_name).await; + } + #[tokio::test] #[serial_test::serial] async fn peer_rest_heal_control_uses_production_auth_and_keeps_validation_errors_online() { diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 91fb0430e..da959a502 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -488,7 +488,7 @@ pub(crate) mod ecstore_rpc { }; #[cfg(test)] pub(crate) use rustfs_ecstore::api::rpc::{ - gen_tonic_signature_headers, set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, + gen_signature_headers, gen_tonic_signature_headers, set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, }; } @@ -552,6 +552,8 @@ pub(crate) fn try_current_local_node_name() -> Option { crate::storage::runtime_sources::try_current_local_node_name() } +#[cfg(test)] +pub(crate) use ecstore_rpc::gen_signature_headers; #[cfg(test)] pub(crate) use ecstore_rpc::gen_tonic_signature_headers; pub(crate) use ecstore_rpc::sign_tonic_rpc_response_proof; diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index fa2bed37f..13c4e638a 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -106,7 +106,8 @@ pub(crate) mod server { #[cfg(test)] pub(crate) use crate::storage::storage_api::{ - Endpoint, EndpointServerPools, Endpoints, PeerRestClient, PoolEndpoints, gen_tonic_signature_headers, + Endpoint, EndpointServerPools, Endpoints, PeerRestClient, PoolEndpoints, gen_signature_headers, + gen_tonic_signature_headers, }; pub(crate) mod ecfs {