feat(kms): converge KMS configuration across nodes after a runtime change (#5551)

crates/ecstore/src/api/mod.rs

@@ -419,15 +419,15 @@ pub mod rio {
 
 pub mod rpc {
     pub use crate::cluster::rpc::{
-        AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
-        PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
-        ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
-        gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
-        node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
-        sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
-        verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
-        verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
-        verify_tonic_rpc_signature_with_bootstrap,
+        AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
+        PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
+        ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
+        gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
+        node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
+        set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
+        tonic_boot_epoch_response_headers, verify_rpc_signature, verify_tonic_boot_epoch_response,
+        verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
+        verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
     };
 }
 

crates/ecstore/src/cluster/rpc/mod.rs

@@ -44,7 +44,7 @@ pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
 pub use internode_data_transport::build_internode_data_transport_from_env;
 pub(crate) use peer_rest_client::TierConfigReloadOutcome;
 pub use peer_rest_client::{
-    PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
+    KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
     SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
 };
 pub(crate) use peer_s3_client::heal_bucket_local_on_disks;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -45,9 +45,9 @@ use rustfs_protos::proto_gen::node_service::{
     HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
     LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
     LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ScannerActivityRequest,
-    ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest,
-    StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
-    TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
+    ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
+    StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
+    TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
     tier_mutation_control_service_client::TierMutationControlServiceClient,
 };
 pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
@@ -71,6 +71,12 @@ use uuid::Uuid;
 
 pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
 pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
+/// Dynamic config subsystem for the cluster-persisted KMS configuration.
+///
+/// KMS configuration lives in its own cluster object rather than in the server
+/// config document, so it is not a `ServerConfig` subsystem; it only shares the
+/// reload signal transport.
+pub const KMS_SIGNAL_SUBSYSTEM: &str = "kms";
 const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
 const HEAL_CONTROL_FINGERPRINT_MAX_SIZE: usize = 256;
 const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
@@ -99,8 +105,13 @@ fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<
 }
 
 fn validate_signal_service_protocol(sig: u64, sub_sys: &str, protocol_version: u32) -> Result<()> {
+    // The version stays pinned to DYNAMIC_CONFIG_PROTOCOL_VERSION rather than
+    // being bumped per subsystem: the comparison is shared, so raising it would
+    // retire peers that already converge scanner and heal config correctly.
+    // Subsystems added after a peer was built are rejected by that peer's own
+    // subsystem allow-list, which surfaces as an explicit failed signal.
     if sig == SERVICE_SIGNAL_RELOAD_DYNAMIC
-        && matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS)
+        && matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS | KMS_SIGNAL_SUBSYSTEM)
         && protocol_version < rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION
     {
         return Err(Error::other(format!("peer does not support dynamic {sub_sys} config convergence")));
@@ -1500,6 +1511,22 @@ impl PeerRestClient {
     }
 
     pub async fn signal_service(&self, sig: u64, sub_sys: &str, dry_run: bool, _exec_at: SystemTime) -> Result<()> {
+        self.signal_service_checked(sig, sub_sys, dry_run).await.map(|_| ())
+    }
+
+    /// Report the KMS configuration fingerprint the peer is currently running.
+    ///
+    /// Sent as a dry-run reload signal so the peer answers without swapping its
+    /// own configuration. `None` means the peer has no KMS configuration. The
+    /// fingerprint is advisory and feeds cluster status reporting only, so the
+    /// response is not proof-signed.
+    pub async fn kms_config_fingerprint(&self) -> Result<Option<String>> {
+        self.signal_service_checked(SERVICE_SIGNAL_RELOAD_DYNAMIC, KMS_SIGNAL_SUBSYSTEM, true)
+            .await
+            .map(|response| response.config_fingerprint)
+    }
+
+    async fn signal_service_checked(&self, sig: u64, sub_sys: &str, dry_run: bool) -> Result<SignalServiceResponse> {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
@@ -1520,7 +1547,7 @@ impl PeerRestClient {
                     return Err(Error::other(""));
                 }
                 validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
-                Ok(())
+                Ok(response)
             }
             .await,
         )
@@ -2347,6 +2374,19 @@ mod tests {
             .expect("full refresh compatibility is guarded by its scanner preflight");
     }
 
+    #[test]
+    fn dynamic_kms_config_requires_versioned_peer_acknowledgement() {
+        let err = validate_signal_service_protocol(SERVICE_SIGNAL_RELOAD_DYNAMIC, KMS_SIGNAL_SUBSYSTEM, 0)
+            .expect_err("an unversioned peer must not claim KMS config convergence");
+        assert!(err.to_string().contains("does not support dynamic"));
+        validate_signal_service_protocol(
+            SERVICE_SIGNAL_RELOAD_DYNAMIC,
+            KMS_SIGNAL_SUBSYSTEM,
+            rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION,
+        )
+        .expect("a current peer should support dynamic KMS config");
+    }
+
     #[test]
     fn peer_rest_client_marks_network_like_errors() {
         assert!(PeerRestClient::is_network_like_error(&Error::other("transport error")));

crates/ecstore/src/services/notification_sys.rs

@@ -337,6 +337,15 @@ pub struct NotificationPeerErr {
     pub err: Option<Error>,
 }
 
+/// One peer's answer to a KMS configuration fingerprint probe.
+pub struct PeerKmsConfigFingerprint {
+    pub host: String,
+    /// `None` when the peer has no KMS configuration of its own, or could not
+    /// be asked at all, in which case `err` carries the reason.
+    pub fingerprint: Option<String>,
+    pub err: Option<Error>,
+}
+
 fn notification_peer_result<T>(host: String, result: Result<T>) -> NotificationPeerErr {
     NotificationPeerErr { host, err: result.err() }
 }
@@ -518,6 +527,47 @@ impl NotificationSys {
         self.signal_dynamic_config(sub_sys, false).await
     }
 
+    /// Ask every peer to re-read the cluster-persisted KMS configuration.
+    ///
+    /// Best-effort by contract: the caller has already switched locally, so a
+    /// peer that fails is reported rather than rolled back. Peers built before
+    /// the KMS subsystem existed reject the signal with an explicit error.
+    pub async fn reload_kms_config(&self) -> Vec<NotificationPeerErr> {
+        self.reload_dynamic_config(crate::cluster::rpc::KMS_SIGNAL_SUBSYSTEM).await
+    }
+
+    /// Collect the KMS configuration fingerprint each peer is running.
+    ///
+    /// A peer whose build predates the KMS subsystem rejects the probe, so it
+    /// is reported as an error rather than silently agreeing with this node.
+    pub async fn kms_config_fingerprints(&self) -> Vec<PeerKmsConfigFingerprint> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            futures.push(async move {
+                let Some(client) = client else {
+                    return PeerKmsConfigFingerprint {
+                        host: String::new(),
+                        fingerprint: None,
+                        err: Some(Error::other("peer is not reachable")),
+                    };
+                };
+                match client.kms_config_fingerprint().await {
+                    Ok(fingerprint) => PeerKmsConfigFingerprint {
+                        host: client.host.to_string(),
+                        fingerprint,
+                        err: None,
+                    },
+                    Err(e) => PeerKmsConfigFingerprint {
+                        host: client.host.to_string(),
+                        fingerprint: None,
+                        err: Some(e),
+                    },
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
     pub async fn refresh_config_snapshot(&self) -> Vec<NotificationPeerErr> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for client in self.peer_clients.iter() {

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1171,6 +1171,12 @@ pub struct SignalServiceResponse {
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
     #[prost(uint32, tag = "3")]
     pub protocol_version: u32,
+    /// Redacted fingerprint of the KMS configuration this node is running.
+    /// Only set for the "kms" dynamic config subsystem, and left unset when the
+    /// responder has no KMS configuration. Peers built before that subsystem
+    /// reject the signal outright rather than answering without this field.
+    #[prost(string, optional, tag = "4")]
+    pub config_fingerprint: ::core::option::Option<::prost::alloc::string::String>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ScannerActivityRequest {

crates/protos/src/node.proto

@@ -817,6 +817,11 @@ message SignalServiceResponse {
   bool success = 1;
   optional string error_info = 2;
   uint32 protocol_version = 3;
+  // Redacted fingerprint of the KMS configuration this node is running.
+  // Only set for the "kms" dynamic config subsystem, and left unset when the
+  // responder has no KMS configuration. Peers built before that subsystem
+  // reject the signal outright rather than answering without this field.
+  optional string config_fingerprint = 4;
 }
 
 message ScannerActivityRequest {