Fix KMS configuration synchronization across cluster nodes (#855)

* Initial plan

* Add KMS configuration persistence to cluster storage

Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

* Apply code formatting to KMS configuration changes

Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

* add comment

* fix fmt

* fix

* Fix overlapping dependabot cargo configurations

Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

* improve code for comment and replace  `Once_Cell` to `std::sync::OnceLock`

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: loverustfs <155562731+loverustfs@users.noreply.github.com>

crates/kms/src/cache.rs

@@ -34,6 +34,13 @@ pub struct KmsCache {
 
 impl KmsCache {
     /// Create a new KMS cache with the specified capacity
+    ///
+    /// # Arguments
+    /// * `capacity` - Maximum number of entries in the cache
+    ///
+    /// # Returns
+    /// A new instance of `KmsCache`
+    ///
     pub fn new(capacity: u64) -> Self {
         Self {
             key_metadata_cache: Cache::builder()
@@ -48,22 +55,47 @@ impl KmsCache {
     }
 
     /// Get key metadata from cache
+    ///
+    /// # Arguments
+    /// * `key_id` - The ID of the key to retrieve metadata for
+    ///
+    /// # Returns
+    /// An `Option` containing the `KeyMetadata` if found, or `None` if not found
+    ///
     pub async fn get_key_metadata(&self, key_id: &str) -> Option<KeyMetadata> {
         self.key_metadata_cache.get(key_id).await
     }
 
     /// Put key metadata into cache
+    ///
+    /// # Arguments
+    /// * `key_id` - The ID of the key to store metadata for
+    /// * `metadata` - The `KeyMetadata` to store in the cache
+    ///
     pub async fn put_key_metadata(&mut self, key_id: &str, metadata: &KeyMetadata) {
         self.key_metadata_cache.insert(key_id.to_string(), metadata.clone()).await;
         self.key_metadata_cache.run_pending_tasks().await;
     }
 
     /// Get data key from cache
+    ///
+    /// # Arguments
+    /// * `key_id` - The ID of the key to retrieve the data key for
+    ///
+    /// # Returns
+    /// An `Option` containing the `CachedDataKey` if found, or `None` if not found
+    ///
     pub async fn get_data_key(&self, key_id: &str) -> Option<CachedDataKey> {
         self.data_key_cache.get(key_id).await
     }
 
     /// Put data key into cache
+    ///
+    /// # Arguments
+    /// * `key_id` - The ID of the key to store the data key for
+    /// * `plaintext` - The plaintext data key bytes
+    /// * `ciphertext` - The ciphertext data key bytes
+    ///
     pub async fn put_data_key(&mut self, key_id: &str, plaintext: &[u8], ciphertext: &[u8]) {
         let cached_key = CachedDataKey {
             plaintext: plaintext.to_vec(),
@@ -75,11 +107,19 @@ impl KmsCache {
     }
 
     /// Remove key metadata from cache
+    ///
+    /// # Arguments
+    /// * `key_id` - The ID of the key to remove metadata for
+    ///
     pub async fn remove_key_metadata(&mut self, key_id: &str) {
         self.key_metadata_cache.remove(key_id).await;
     }
 
     /// Remove data key from cache
+    ///
+    /// # Arguments
+    /// * `key_id` - The ID of the key to remove the data key for
+    ///
     pub async fn remove_data_key(&mut self, key_id: &str) {
         self.data_key_cache.remove(key_id).await;
     }
@@ -95,6 +135,10 @@ impl KmsCache {
     }
 
     /// Get cache statistics (hit count, miss count)
+    ///
+    /// # Returns
+    /// A tuple containing total entries and total misses
+    ///
     pub fn stats(&self) -> (u64, u64) {
         let metadata_stats = (
             self.key_metadata_cache.entry_count(),