feat(rpc): add server-side internode v2 signature verification (fail-open, method-path binding groundwork) (#5160)

* feat(rpc): gate internode legacy signature fallback behind a convergence counter and strict env

Add the backlog#1327 Plan-A rollout infrastructure on top of the already-merged v2 target-bound internode gRPC authentication: a rustfs_system_network_internode_signature_v1_fallback_total counter that increments only when a request without any v2 auth headers is accepted through the legacy constant-target signature, and a RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT env (default false, compile-time asserted) that, when enabled later, closes the legacy fallback path. Default behavior is fail-open and byte-identical for legacy-only peers; requests carrying v2 headers are verified as v2 with no downgrade exactly as before.

Refs https://github.com/rustfs/backlog/issues/1327

* ci: fix internode auth test lint failures

* perf(ecstore): cache internode sig strict env

---------

Co-authored-by: houseme <housemecn@gmail.com>

crates/config/src/constants/internode.rs

@@ -118,6 +118,24 @@ pub const DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED: bool = false;
 const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY);
 const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED);
 
+/// Require target-bound v2 signatures on every internode gRPC request, rejecting the legacy
+/// constant-target fallback instead of accepting it (<https://github.com/rustfs/backlog/issues/1327>).
+///
+/// Defaults to `false` (fail-open): a request without any v2 auth headers keeps authenticating
+/// through the legacy signature, so legacy-only peers survive rolling upgrades with byte-for-byte
+/// the pre-gate acceptance behavior. This is a rollout lever, not a wire-format change: it may only
+/// be enabled **after** the v1-fallback counter
+/// (`rustfs_system_network_internode_signature_v1_fallback_total`) has read zero across a release
+/// window fleet-wide, confirming every peer already sends v2 authentication on every internode gRPC
+/// request. Single-env rollback. Requests that do carry v2 headers are unaffected by this switch:
+/// they are always verified as v2 with no downgrade, strict or not.
+pub const ENV_INTERNODE_RPC_SIGNATURE_STRICT: &str = "RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT";
+pub const DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT: bool = false;
+
+// Compile-time invariant: fail-open by default so legacy-only peers keep authenticating during
+// rolling upgrades until the fleet-wide v1-fallback counter reads zero.
+const _: () = assert!(!DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT);
+
 /// Consecutive-failure threshold after which an internode peer is marked offline (grpc-optimization
 /// P3 observability).
 ///
@@ -288,6 +306,12 @@ mod tests {
         );
     }
 
+    #[test]
+    fn internode_signature_strict_env_name_is_stable() {
+        // The fail-open default invariant is asserted at compile time next to the definition.
+        assert_eq!(ENV_INTERNODE_RPC_SIGNATURE_STRICT, "RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT");
+    }
+
     #[test]
     fn internode_offline_failure_threshold_defaults_and_env_name() {
         assert_eq!(DEFAULT_INTERNODE_OFFLINE_FAILURE_THRESHOLD, 3);

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -35,6 +35,8 @@ use http::{HeaderMap, HeaderValue, Method, Uri};
 #[cfg(test)]
 use rustfs_credentials::{DEFAULT_SECRET_KEY, RPC_SECRET_REQUIRED_MESSAGE};
 use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token};
+use rustfs_io_metrics::internode_metrics::global_internode_metrics;
+use rustfs_utils::get_env_bool;
 use sha2::Digest as _;
 use sha2::Sha256;
 use std::collections::{HashSet, VecDeque};
@@ -60,6 +62,12 @@ const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
 const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
 const MAX_REPLAY_PROTECTED_NONCES: usize = 65_536;
 pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
+static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
+    get_env_bool(
+        rustfs_config::ENV_INTERNODE_RPC_SIGNATURE_STRICT,
+        rustfs_config::DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT,
+    )
+});
 static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
 
 #[derive(Default)]
@@ -412,12 +420,40 @@ fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
     .any(|name| headers.contains_key(*name))
 }
 
+/// Whether the server requires target-bound v2 authentication on every internode gRPC request,
+/// rejecting the legacy constant-target fallback instead of accepting it. Default-off rollout
+/// lever gated on the v1-fallback counter reading zero fleet-wide; see
+/// [`rustfs_config::ENV_INTERNODE_RPC_SIGNATURE_STRICT`] and
+/// <https://github.com/rustfs/backlog/issues/1327>.
+fn internode_rpc_signature_strict() -> bool {
+    *INTERNODE_RPC_SIGNATURE_STRICT
+}
+
 /// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
 pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
+    verify_tonic_rpc_signature_with_strictness(audience, path, headers, internode_rpc_signature_strict())
+}
+
+/// [`verify_tonic_rpc_signature`] with the strict gate injected as a parameter, so both rollout
+/// postures are unit-testable without racing on process-global environment variables.
+fn verify_tonic_rpc_signature_with_strictness(
+    audience: &str,
+    path: &str,
+    headers: &HeaderMap,
+    strict: bool,
+) -> std::io::Result<()> {
     if !has_v2_auth_headers(headers) {
         // RUSTFS_COMPAT_TODO(heal-rpc-auth-v2): accept old peers during rolling upgrades. Remove after the minimum
         // supported RustFS peer version sends v2 authentication on every internode gRPC request.
-        return verify_rpc_signature(TONIC_RPC_PREFIX, &Method::GET, headers);
+        if strict {
+            return Err(std::io::Error::other("RPC v2 authentication required"));
+        }
+        verify_rpc_signature(TONIC_RPC_PREFIX, &Method::GET, headers)?;
+        // Count only ACCEPTED legacy-only requests: this counter is the convergence gate that must
+        // read zero fleet-wide across a release window before
+        // `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT` may be enabled.
+        global_internode_metrics().record_signature_v1_fallback();
+        return Ok(());
     }
 
     let path = path
@@ -1111,7 +1147,10 @@ mod tests {
         assert_eq!(error.to_string(), "Invalid RPC v2 signature");
     }
 
+    // The `rpc_v1_fallback_counter` serial group covers every test that drives (or asserts on) the
+    // process-global v1-fallback counter, so exact-delta assertions cannot race with each other.
     #[test]
+    #[serial_test::serial(rpc_v1_fallback_counter)]
     fn legacy_tonic_signature_remains_accepted_during_rolling_upgrade() {
         ensure_test_rpc_secret();
         let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build");
@@ -1119,6 +1158,87 @@ mod tests {
         assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/Ping", &headers).is_ok());
     }
 
+    #[test]
+    #[serial_test::serial(rpc_v1_fallback_counter)]
+    fn accepted_legacy_fallback_increments_v1_fallback_counter() {
+        ensure_test_rpc_secret();
+        let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build");
+        let before = global_internode_metrics().snapshot().signature_v1_fallback_total;
+
+        assert!(
+            verify_tonic_rpc_signature_with_strictness("node-a:9000", "/node_service.NodeService/Ping", &headers, false).is_ok(),
+            "a legacy-only peer must keep authenticating while the strict gate is off"
+        );
+
+        let after = global_internode_metrics().snapshot().signature_v1_fallback_total;
+        assert_eq!(
+            after,
+            before + 1,
+            "an accepted legacy-only request must increment the v1 fallback counter exactly once"
+        );
+    }
+
+    #[test]
+    #[serial_test::serial(rpc_v1_fallback_counter)]
+    fn rejected_legacy_fallback_does_not_count_as_v1_fallback() {
+        ensure_test_rpc_secret();
+        // Legacy-shaped headers with a forged signature: the fallback path runs but must reject,
+        // and a rejected request is not a rollout-convergence signal.
+        let mut headers = HeaderMap::new();
+        let now = OffsetDateTime::now_utc().unix_timestamp();
+        headers.insert(SIGNATURE_HEADER, HeaderValue::from_static("not-a-real-signature"));
+        headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&now.to_string()).unwrap());
+        let before = global_internode_metrics().snapshot().signature_v1_fallback_total;
+
+        assert!(
+            verify_tonic_rpc_signature_with_strictness("node-a:9000", "/node_service.NodeService/Ping", &headers, false).is_err(),
+            "a forged legacy signature must still be rejected"
+        );
+
+        let after = global_internode_metrics().snapshot().signature_v1_fallback_total;
+        assert_eq!(after, before, "a rejected legacy request must not count as an accepted fallback");
+    }
+
+    #[test]
+    #[serial_test::serial(rpc_v1_fallback_counter)]
+    fn strict_gate_rejects_legacy_only_auth_but_keeps_v2() {
+        ensure_test_rpc_secret();
+        let legacy = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build");
+        let before = global_internode_metrics().snapshot().signature_v1_fallback_total;
+        let error = verify_tonic_rpc_signature_with_strictness("node-a:9000", "/node_service.NodeService/Ping", &legacy, true)
+            .expect_err("strict mode must reject legacy-only authentication");
+        assert_eq!(error.to_string(), "RPC v2 authentication required");
+
+        let v2 = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
+            .expect("tonic auth headers should build");
+        assert!(
+            verify_tonic_rpc_signature_with_strictness("node-a:9000", "/node_service.NodeService/Ping", &v2, true).is_ok(),
+            "strict mode must keep accepting v2-authenticated peers"
+        );
+        let after = global_internode_metrics().snapshot().signature_v1_fallback_total;
+        assert_eq!(after, before, "neither a strict rejection nor a v2 acceptance is a legacy fallback");
+    }
+
+    #[test]
+    #[serial_test::serial(rpc_v1_fallback_counter)]
+    fn strict_gate_default_posture_is_fail_open_legacy_accept() {
+        ensure_test_rpc_secret();
+        // The public entry point resolves strictness from the environment, whose compile-time
+        // default is pinned to false in `rustfs_config`. A legacy-only peer therefore keeps
+        // authenticating through the default build with no configuration at all.
+        let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy auth headers should build");
+        assert!(
+            verify_tonic_rpc_signature_with_strictness(
+                "node-a:9000",
+                "/node_service.NodeService/Ping",
+                &headers,
+                rustfs_config::DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT,
+            )
+            .is_ok(),
+            "the default strict posture must accept legacy-only peers"
+        );
+    }
+
     #[test]
     fn body_bound_tonic_request_rejects_replay_and_body_tampering() {
         ensure_test_rpc_secret();

crates/io-metrics/src/internode_metrics.rs

@@ -59,6 +59,7 @@ const INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL: &str =
 const INTERNODE_OPERATION_PAYLOAD_BYTES: &str = "rustfs_system_network_internode_operation_payload_bytes";
 const INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL: &str = "rustfs_system_network_internode_operation_large_payloads_total";
 const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_fallback_total";
+const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
 const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -148,6 +149,7 @@ pub struct InternodeMetricsSnapshot {
     pub operation_http_versions_total: u64,
     pub operation_stall_timeouts_total: u64,
     pub operation_write_shutdown_errors_total: u64,
+    pub signature_v1_fallback_total: u64,
 }
 
 #[derive(Debug, Default)]
@@ -164,6 +166,7 @@ pub struct InternodeMetrics {
     operation_http_versions_total: AtomicU64,
     operation_stall_timeouts_total: AtomicU64,
     operation_write_shutdown_errors_total: AtomicU64,
+    signature_v1_fallback_total: AtomicU64,
 }
 
 impl InternodeMetrics {
@@ -360,6 +363,17 @@ impl InternodeMetrics {
         counter!(INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL, DIRECTION_LABEL => direction, MESSAGE_LABEL => message).increment(1);
     }
 
+    /// Count an internode gRPC request that was accepted through the legacy constant-target
+    /// signature because it carried no v2 auth headers (rolling-upgrade fallback, see
+    /// <https://github.com/rustfs/backlog/issues/1327>). Only accepted requests count: rejected
+    /// requests never authenticated, so they are not a rollout signal. This counter must read zero
+    /// across a release window fleet-wide before `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT` may be
+    /// enabled; after the strict flip the legacy fallback path is closed and the counter stays flat.
+    pub fn record_signature_v1_fallback(&self) {
+        self.signature_v1_fallback_total.fetch_add(1, Ordering::Relaxed);
+        counter!(INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL).increment(1);
+    }
+
     pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) {
         counter!(
             ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
@@ -406,6 +420,7 @@ impl InternodeMetrics {
             operation_http_versions_total: self.operation_http_versions_total.load(Ordering::Relaxed),
             operation_stall_timeouts_total: self.operation_stall_timeouts_total.load(Ordering::Relaxed),
             operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
+            signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
         }
     }
 
@@ -423,6 +438,7 @@ impl InternodeMetrics {
         self.operation_http_versions_total.store(0, Ordering::Relaxed);
         self.operation_stall_timeouts_total.store(0, Ordering::Relaxed);
         self.operation_write_shutdown_errors_total.store(0, Ordering::Relaxed);
+        self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
     }
 }
 
@@ -727,6 +743,10 @@ mod tests {
         );
         assert_eq!(INTERNODE_MSGPACK_DIRECTION_REQUEST, "request");
         assert_eq!(INTERNODE_MSGPACK_DIRECTION_RESPONSE, "response");
+        assert_eq!(
+            INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL,
+            "rustfs_system_network_internode_signature_v1_fallback_total"
+        );
     }
 
     #[test]
@@ -737,6 +757,20 @@ mod tests {
         metrics.record_msgpack_json_fallback(INTERNODE_MSGPACK_DIRECTION_RESPONSE, "RawFileInfo");
     }
 
+    #[test]
+    fn signature_v1_fallback_counter_updates_snapshot_and_resets() {
+        // Instance-local metrics keep this independent of the process-global registry.
+        let metrics = InternodeMetrics::default();
+        assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0);
+
+        metrics.record_signature_v1_fallback();
+        metrics.record_signature_v1_fallback();
+        assert_eq!(metrics.snapshot().signature_v1_fallback_total, 2);
+
+        metrics.reset_for_test();
+        assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0);
+    }
+
     #[test]
     fn cluster_peer_flips_offline_after_threshold_and_back_online() {
         // Unique addr keeps this independent of the process-global registry / other tests.

rustfs/src/server/http.rs

@@ -2368,6 +2368,48 @@ mod tests {
         rustfs_common::set_global_local_node_name(&previous_node_name).await;
     }
 
+    /// Rolling-upgrade compatibility anchor for <https://github.com/rustfs/backlog/issues/1327>:
+    /// a legacy-only peer (constant-target signature, no v2 headers) must keep authenticating
+    /// through the real production path (`check_auth` + `RpcRequestTarget` extension), and every
+    /// such acceptance must increment the v1-fallback convergence counter exactly once. That
+    /// counter reading zero fleet-wide is the precondition for ever enabling
+    /// `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT`.
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn rpc_auth_accepts_legacy_only_peer_and_counts_v1_fallback() {
+        use rustfs_io_metrics::internode_metrics::global_internode_metrics;
+
+        let _ = rustfs_credentials::set_global_rpc_secret("rpc-http-test-secret".to_string());
+        let previous_node_name = rustfs_common::get_global_local_node_name().await;
+        rustfs_common::set_global_local_node_name("127.0.0.1:9000").await;
+
+        // Exactly what an old peer sends on the gRPC plane: the legacy constant-target signature
+        // and timestamp headers, nothing else.
+        let legacy_headers = storage::gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET).expect("legacy headers should build");
+        let mut request = Request::new(());
+        request.metadata_mut().as_mut().extend(legacy_headers);
+        request.extensions_mut().insert(RpcRequestTarget {
+            uri: "http://127.0.0.1:9000/node_service.NodeService/Ping"
+                .parse()
+                .expect("test RPC URI should parse"),
+            method: Method::POST,
+        });
+
+        let before = global_internode_metrics().snapshot().signature_v1_fallback_total;
+        assert!(
+            check_auth(request).is_ok(),
+            "a legacy-only peer must keep authenticating during rolling upgrades"
+        );
+        let after = global_internode_metrics().snapshot().signature_v1_fallback_total;
+        assert_eq!(
+            after,
+            before + 1,
+            "an accepted legacy-only request must increment signature_v1_fallback_total exactly once"
+        );
+
+        rustfs_common::set_global_local_node_name(&previous_node_name).await;
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn peer_rest_heal_control_uses_production_auth_and_keeps_validation_errors_online() {

rustfs/src/storage/storage_api.rs

@@ -488,7 +488,7 @@ pub(crate) mod ecstore_rpc {
     };
     #[cfg(test)]
     pub(crate) use rustfs_ecstore::api::rpc::{
-        gen_tonic_signature_headers, set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof,
+        gen_signature_headers, gen_tonic_signature_headers, set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof,
     };
 }
 
@@ -552,6 +552,8 @@ pub(crate) fn try_current_local_node_name() -> Option<String> {
     crate::storage::runtime_sources::try_current_local_node_name()
 }
 
+#[cfg(test)]
+pub(crate) use ecstore_rpc::gen_signature_headers;
 #[cfg(test)]
 pub(crate) use ecstore_rpc::gen_tonic_signature_headers;
 pub(crate) use ecstore_rpc::sign_tonic_rpc_response_proof;

rustfs/src/storage_api.rs

@@ -106,7 +106,8 @@ pub(crate) mod server {
 
         #[cfg(test)]
         pub(crate) use crate::storage::storage_api::{
-            Endpoint, EndpointServerPools, Endpoints, PeerRestClient, PoolEndpoints, gen_tonic_signature_headers,
+            Endpoint, EndpointServerPools, Endpoints, PeerRestClient, PoolEndpoints, gen_signature_headers,
+            gen_tonic_signature_headers,
         };
 
         pub(crate) mod ecfs {