feat(rpc): add server-side internode v2 signature verification (fail-open, method-path binding groundwork) (#5160)

* feat(rpc): gate internode legacy signature fallback behind a convergence counter and strict env

Add the backlog#1327 Plan-A rollout infrastructure on top of the already-merged v2 target-bound internode gRPC authentication: a rustfs_system_network_internode_signature_v1_fallback_total counter that increments only when a request without any v2 auth headers is accepted through the legacy constant-target signature, and a RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT env (default false, compile-time asserted) that, when enabled later, closes the legacy fallback path. Default behavior is fail-open and byte-identical for legacy-only peers; requests carrying v2 headers are verified as v2 with no downgrade exactly as before.

Refs https://github.com/rustfs/backlog/issues/1327

* ci: fix internode auth test lint failures

* perf(ecstore): cache internode sig strict env

---------

Co-authored-by: houseme <housemecn@gmail.com>

crates/io-metrics/src/internode_metrics.rs

@@ -59,6 +59,7 @@ const INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL: &str =
 const INTERNODE_OPERATION_PAYLOAD_BYTES: &str = "rustfs_system_network_internode_operation_payload_bytes";
 const INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL: &str = "rustfs_system_network_internode_operation_large_payloads_total";
 const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_fallback_total";
+const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
 const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -148,6 +149,7 @@ pub struct InternodeMetricsSnapshot {
     pub operation_http_versions_total: u64,
     pub operation_stall_timeouts_total: u64,
     pub operation_write_shutdown_errors_total: u64,
+    pub signature_v1_fallback_total: u64,
 }
 
 #[derive(Debug, Default)]
@@ -164,6 +166,7 @@ pub struct InternodeMetrics {
     operation_http_versions_total: AtomicU64,
     operation_stall_timeouts_total: AtomicU64,
     operation_write_shutdown_errors_total: AtomicU64,
+    signature_v1_fallback_total: AtomicU64,
 }
 
 impl InternodeMetrics {
@@ -360,6 +363,17 @@ impl InternodeMetrics {
         counter!(INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL, DIRECTION_LABEL => direction, MESSAGE_LABEL => message).increment(1);
     }
 
+    /// Count an internode gRPC request that was accepted through the legacy constant-target
+    /// signature because it carried no v2 auth headers (rolling-upgrade fallback, see
+    /// <https://github.com/rustfs/backlog/issues/1327>). Only accepted requests count: rejected
+    /// requests never authenticated, so they are not a rollout signal. This counter must read zero
+    /// across a release window fleet-wide before `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT` may be
+    /// enabled; after the strict flip the legacy fallback path is closed and the counter stays flat.
+    pub fn record_signature_v1_fallback(&self) {
+        self.signature_v1_fallback_total.fetch_add(1, Ordering::Relaxed);
+        counter!(INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL).increment(1);
+    }
+
     pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) {
         counter!(
             ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
@@ -406,6 +420,7 @@ impl InternodeMetrics {
             operation_http_versions_total: self.operation_http_versions_total.load(Ordering::Relaxed),
             operation_stall_timeouts_total: self.operation_stall_timeouts_total.load(Ordering::Relaxed),
             operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
+            signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
         }
     }
 
@@ -423,6 +438,7 @@ impl InternodeMetrics {
         self.operation_http_versions_total.store(0, Ordering::Relaxed);
         self.operation_stall_timeouts_total.store(0, Ordering::Relaxed);
         self.operation_write_shutdown_errors_total.store(0, Ordering::Relaxed);
+        self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
     }
 }
 
@@ -727,6 +743,10 @@ mod tests {
         );
         assert_eq!(INTERNODE_MSGPACK_DIRECTION_REQUEST, "request");
         assert_eq!(INTERNODE_MSGPACK_DIRECTION_RESPONSE, "response");
+        assert_eq!(
+            INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL,
+            "rustfs_system_network_internode_signature_v1_fallback_total"
+        );
     }
 
     #[test]
@@ -737,6 +757,20 @@ mod tests {
         metrics.record_msgpack_json_fallback(INTERNODE_MSGPACK_DIRECTION_RESPONSE, "RawFileInfo");
     }
 
+    #[test]
+    fn signature_v1_fallback_counter_updates_snapshot_and_resets() {
+        // Instance-local metrics keep this independent of the process-global registry.
+        let metrics = InternodeMetrics::default();
+        assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0);
+
+        metrics.record_signature_v1_fallback();
+        metrics.record_signature_v1_fallback();
+        assert_eq!(metrics.snapshot().signature_v1_fallback_total, 2);
+
+        metrics.reset_for_test();
+        assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0);
+    }
+
     #[test]
     fn cluster_peer_flips_offline_after_threshold_and_back_online() {
         // Unique addr keeps this independent of the process-global registry / other tests.