gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-10 07:06:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(rio): rio_v2 is compatible with minio for storing data. (#3115)

Browse Source 
* Set up a compatibility layer for replacing old Rio components with new ones.

* fix(rio). compress range

* feat(rio). Add the experimental feature rio_v2 to support minio data at the binary level.

* feat(rio_v2): add sse-c test

* test compression component

* simple fix

* fix minlz encode

* fix metadata

* fix kms key cache error

* Update launch.json

* ci: set nix crate download user agent

* fix: gate obs pyroscope backend

* ignore minio test

* fix encrypt check

* fix

* fix

* fix

* Update object_usecase.rs

* Update ci.yml

* fix

* ci add rio-v2 test

* fix

* ci fix

* fix

* Reconstructed into a more reasonable compatibility mode

* fix

* fix

---------

Signed-off-by: houseme <housemecn@gmail.com>
Signed-off-by: 唐小鸭 <tangtang1251@qq.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
This commit is contained in:
唐小鸭
2026-06-08 19:59:14 +08:00
committed by GitHub
parent 9504dff595
commit f7724d223b
47 changed files with 8742 additions and 682 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/kms/src/cache.rs
+8 -88
View File
@@ -14,22 +14,13 @@
//! Caching layer for KMS operations to improve performance
use crate::types::{KeyMetadata, KeySpec};
use crate::types::KeyMetadata;
use moka::future::Cache;
use std::time::Duration;
/// Cached data key entry
#[derive(Clone, Debug)]
pub struct CachedDataKey {
pub plaintext: Vec<u8>,
pub ciphertext: Vec<u8>,
pub key_spec: KeySpec,
}
/// KMS cache for storing frequently accessed keys and metadata
pub struct KmsCache {
key_metadata_cache: Cache<String, KeyMetadata>,
data_key_cache: Cache<String, CachedDataKey>,
}
impl KmsCache {
@@ -44,13 +35,9 @@ impl KmsCache {
pub fn new(capacity: u64) -> Self {
Self {
key_metadata_cache: Cache::builder()
.max_capacity(capacity / 2)
.max_capacity(capacity)
.time_to_live(Duration::from_secs(300)) // 5 minutes default TTL
.build(),
data_key_cache: Cache::builder()
.max_capacity(capacity / 2)
.time_to_live(Duration::from_secs(60)) // 1 minute for data keys (shorter for security)
.build(),
}
}
@@ -77,35 +64,6 @@ impl KmsCache {
self.key_metadata_cache.run_pending_tasks().await;
}
/// Get data key from cache
///
/// # Arguments
/// * `key_id` - The ID of the key to retrieve the data key for
///
/// # Returns
/// An `Option` containing the `CachedDataKey` if found, or `None` if not found
///
pub async fn get_data_key(&self, key_id: &str) -> Option<CachedDataKey> {
self.data_key_cache.get(key_id).await
}
/// Put data key into cache
///
/// # Arguments
/// * `key_id` - The ID of the key to store the data key for
/// * `plaintext` - The plaintext data key bytes
/// * `ciphertext` - The ciphertext data key bytes
///
pub async fn put_data_key(&mut self, key_id: &str, plaintext: &[u8], ciphertext: &[u8]) {
let cached_key = CachedDataKey {
plaintext: plaintext.to_vec(),
ciphertext: ciphertext.to_vec(),
key_spec: KeySpec::Aes256, // Default to AES-256
};
self.data_key_cache.insert(key_id.to_string(), cached_key).await;
self.data_key_cache.run_pending_tasks().await;
}
/// Remove key metadata from cache
///
/// # Arguments
@@ -115,23 +73,12 @@ impl KmsCache {
self.key_metadata_cache.remove(key_id).await;
}
/// Remove data key from cache
///
/// # Arguments
/// * `key_id` - The ID of the key to remove the data key for
///
pub async fn remove_data_key(&mut self, key_id: &str) {
self.data_key_cache.remove(key_id).await;
}
/// Clear all cached entries
pub async fn clear(&mut self) {
self.key_metadata_cache.invalidate_all();
self.data_key_cache.invalidate_all();
// Wait for invalidation to complete
self.key_metadata_cache.run_pending_tasks().await;
self.data_key_cache.run_pending_tasks().await;
}
/// Get cache statistics (hit count, miss count)
@@ -140,13 +87,10 @@ impl KmsCache {
/// A tuple containing total entries and total misses
///
pub fn stats(&self) -> (u64, u64) {
let metadata_stats = (
(
self.key_metadata_cache.entry_count(),
0u64, // moka doesn't provide miss count directly
);
let data_key_stats = (self.data_key_cache.entry_count(), 0u64);
(metadata_stats.0 + data_key_stats.0, metadata_stats.1 + data_key_stats.1)
)
}
}
@@ -160,37 +104,30 @@ mod tests {
#[derive(Debug, Clone)]
struct CacheInfo {
key_metadata_count: u64,
data_key_count: u64,
}
impl CacheInfo {
fn total_entries(&self) -> u64 {
self.key_metadata_count + self.data_key_count
self.key_metadata_count
}
}
impl KmsCache {
fn with_ttl_for_tests(capacity: u64, metadata_ttl: Duration, data_key_ttl: Duration) -> Self {
fn with_ttl_for_tests(capacity: u64, metadata_ttl: Duration) -> Self {
Self {
key_metadata_cache: Cache::builder().max_capacity(capacity / 2).time_to_live(metadata_ttl).build(),
data_key_cache: Cache::builder().max_capacity(capacity / 2).time_to_live(data_key_ttl).build(),
key_metadata_cache: Cache::builder().max_capacity(capacity).time_to_live(metadata_ttl).build(),
}
}
fn info_for_tests(&self) -> CacheInfo {
CacheInfo {
key_metadata_count: self.key_metadata_cache.entry_count(),
data_key_count: self.data_key_cache.entry_count(),
}
}
fn contains_key_metadata_for_tests(&self, key_id: &str) -> bool {
self.key_metadata_cache.contains_key(key_id)
}
fn contains_data_key_for_tests(&self, key_id: &str) -> bool {
self.data_key_cache.contains_key(key_id)
}
}
#[tokio::test]
@@ -216,23 +153,10 @@ mod tests {
assert!(retrieved.is_some());
assert_eq!(retrieved.expect("metadata should be cached").key_id, "test-key-1");
// Test data key caching
let plaintext = vec![1, 2, 3, 4];
let ciphertext = vec![5, 6, 7, 8];
cache.put_data_key("test-key-1", &plaintext, &ciphertext).await;
let cached_data_key = cache.get_data_key("test-key-1").await;
assert!(cached_data_key.is_some());
let cached_data_key = cached_data_key.expect("data key should be cached");
assert_eq!(cached_data_key.plaintext, plaintext);
assert_eq!(cached_data_key.ciphertext, ciphertext);
assert_eq!(cached_data_key.key_spec, KeySpec::Aes256);
// Test cache info
let info = cache.info_for_tests();
assert_eq!(info.key_metadata_count, 1);
assert_eq!(info.data_key_count, 1);
assert_eq!(info.total_entries(), 2);
assert_eq!(info.total_entries(), 1);
// Test cache clearing
cache.clear().await;
@@ -245,7 +169,6 @@ mod tests {
let mut cache = KmsCache::with_ttl_for_tests(
100,
Duration::from_millis(100), // Short TTL for testing
Duration::from_millis(50),
);
let metadata = KeyMetadata {
@@ -277,7 +200,6 @@ mod tests {
let mut cache = KmsCache::new(100);
assert!(!cache.contains_key_metadata_for_tests("nonexistent"));
assert!(!cache.contains_data_key_for_tests("nonexistent"));
let metadata = KeyMetadata {
key_id: "contains-test".to_string(),
@@ -292,9 +214,7 @@ mod tests {
};
cache.put_key_metadata("contains-test", &metadata).await;
cache.put_data_key("contains-test", &[1, 2, 3], &[4, 5, 6]).await;
assert!(cache.contains_key_metadata_for_tests("contains-test"));
assert!(cache.contains_data_key_for_tests("contains-test"));
}
}
crates/kms/src/lib.rs
+8
View File
@@ -33,6 +33,14 @@
//! - **Data Encryption Keys (DEK)**: Generated per object, encrypted by master keys
//! - **Object Data**: Encrypted using DEKs with AES-256-GCM or ChaCha20-Poly1305
//!
//! ## Caching Discipline
//!
//! KMS may cache stable master-key metadata, but it must not cache or reuse generated
//! data encryption keys by master key id alone. A generated DEK and its encrypted
//! ciphertext can be bound to the object encryption context, such as the bucket and
//! object path. Reusing it for another object can break context validation and would
//! also violate the expected per-object DEK model for SSE-S3 and SSE-KMS.
//!
//! ## Example
//!
//! ```rust,no_run
crates/kms/src/manager.rs
+60 -27
View File
@@ -71,32 +71,7 @@ impl KmsManager {
/// Generate a data encryption key
pub async fn generate_data_key(&self, request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> {
// Check cache first if enabled
if self.config.enable_cache {
let cache = self.cache.read().await;
if let Some(cached_key) = cache.get_data_key(&request.key_id).await
&& cached_key.key_spec == request.key_spec
{
return Ok(GenerateDataKeyResponse {
key_id: request.key_id.clone(),
plaintext_key: cached_key.plaintext.clone(),
ciphertext_blob: cached_key.ciphertext,
});
}
}
// Generate new data key from backend
let response = self.backend.generate_data_key(request).await?;
// Cache the data key if enabled
if self.config.enable_cache {
let mut cache = self.cache.write().await;
cache
.put_data_key(&response.key_id, &response.plaintext_key, &response.ciphertext_blob)
.await;
}
Ok(response)
self.backend.generate_data_key(request).await
}
/// Describe a key
@@ -156,7 +131,6 @@ impl KmsManager {
if self.config.enable_cache {
let mut cache = self.cache.write().await;
cache.remove_key_metadata(&response.key_id).await;
cache.remove_data_key(&response.key_id).await;
}
Ok(response)
@@ -186,6 +160,7 @@ mod tests {
use super::*;
use crate::backends::local::LocalKmsBackend;
use crate::types::{KeySpec, KeyState, KeyUsage};
use std::collections::HashMap;
use tempfile::tempdir;
#[tokio::test]
@@ -237,4 +212,62 @@ mod tests {
let health = manager.health_check().await.expect("Health check failed");
assert!(health);
}
#[tokio::test]
async fn generate_data_key_does_not_reuse_context_bound_ciphertext() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let config = KmsConfig::local(temp_dir.path().to_path_buf());
let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
let manager = KmsManager::new(backend, config);
let create_response = manager
.create_key(CreateKeyRequest {
key_usage: KeyUsage::EncryptDecrypt,
description: Some("Context-bound data key test".to_string()),
..Default::default()
})
.await
.expect("Failed to create key");
let first_context = HashMap::from([
("bucket".to_string(), "sse-smoke".to_string()),
("object".to_string(), "first.bin".to_string()),
]);
let second_context = HashMap::from([
("bucket".to_string(), "sse-smoke".to_string()),
("object".to_string(), "second.bin".to_string()),
]);
let first = manager
.generate_data_key(GenerateDataKeyRequest {
key_id: create_response.key_id.clone(),
key_spec: KeySpec::Aes256,
encryption_context: first_context.clone(),
})
.await
.expect("Failed to generate first data key");
let second = manager
.generate_data_key(GenerateDataKeyRequest {
key_id: create_response.key_id.clone(),
key_spec: KeySpec::Aes256,
encryption_context: second_context.clone(),
})
.await
.expect("Failed to generate second data key");
assert_ne!(
first.ciphertext_blob, second.ciphertext_blob,
"data keys must not be cached only by KMS key id because ciphertext is bound to object context"
);
manager
.decrypt(DecryptRequest {
ciphertext: second.ciphertext_blob,
encryption_context: second_context,
grant_tokens: Vec::new(),
})
.await
.expect("second data key should decrypt with its own context");
}
}
crates/kms/src/service.rs
+59 -8
View File
@@ -49,6 +49,26 @@ pub struct ObjectEncryptionService {
kms_manager: KmsManager,
}
fn canonical_bucket_path(bucket: &str, object_key: &str) -> String {
let bucket = bucket.trim_matches('/');
let object_key = object_key.trim_matches('/');
if object_key.is_empty() {
bucket.to_string()
} else if bucket.is_empty() {
object_key.to_string()
} else {
format!("{bucket}/{object_key}")
}
}
fn request_encryption_context(context: &ObjectEncryptionContext) -> HashMap<String, String> {
let mut enc_context = context.encryption_context.clone();
enc_context
.entry(context.bucket.clone())
.or_insert_with(|| canonical_bucket_path(&context.bucket, &context.object_key));
enc_context
}
const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id";
/// Result of object encryption
@@ -178,15 +198,10 @@ impl ObjectEncryptionService {
.or_else(|| self.kms_manager.get_default_key_id().map(|s| s.as_str()))
.ok_or_else(|| KmsError::configuration_error("No KMS key ID specified and no default configured"))?;
// Build encryption context
let mut enc_context = context.encryption_context.clone();
enc_context.insert("bucket".to_string(), context.bucket.clone());
enc_context.insert("object_key".to_string(), context.object_key.clone());
let request = GenerateDataKeyRequest {
key_id: actual_key_id.to_string(),
key_spec: KeySpec::Aes256,
encryption_context: enc_context,
encryption_context: request_encryption_context(context),
};
let data_key_response = self.kms_manager.generate_data_key(request).await?;
@@ -216,10 +231,10 @@ impl ObjectEncryptionService {
/// # Returns
/// DataKey with decrypted key
///
pub async fn decrypt_data_key(&self, encrypted_key: &[u8], _context: &ObjectEncryptionContext) -> Result<DataKey> {
pub async fn decrypt_data_key(&self, encrypted_key: &[u8], context: &ObjectEncryptionContext) -> Result<DataKey> {
let decrypt_request = DecryptRequest {
ciphertext: encrypted_key.to_vec(),
encryption_context: HashMap::new(),
encryption_context: request_encryption_context(context),
grant_tokens: Vec::new(),
};
@@ -864,4 +879,40 @@ mod tests {
.is_err()
);
}
#[tokio::test]
async fn test_decrypt_data_key_uses_object_encryption_context() {
let (service, _temp_dir) = create_test_service().await;
service
.create_key(CreateKeyRequest {
key_name: Some("test-key".to_string()),
key_usage: KeyUsage::EncryptDecrypt,
description: None,
policy: None,
tags: HashMap::new(),
origin: None,
})
.await
.expect("test key should be created");
let create_context = ObjectEncryptionContext::new("bucket".to_string(), "dir/object".to_string())
.with_encryption_context("tenant".to_string(), "alpha".to_string());
let kms_key = Some("test-key".to_string());
let (_data_key, encrypted_key) = service
.create_data_key(&kms_key, &create_context)
.await
.expect("create data key should succeed");
let wrong_context = ObjectEncryptionContext::new("bucket".to_string(), "dir/object".to_string())
.with_encryption_context("tenant".to_string(), "beta".to_string());
assert!(
service.decrypt_data_key(&encrypted_key, &wrong_context).await.is_err(),
"decrypt should reject mismatched KMS context"
);
let decrypted = service
.decrypt_data_key(&encrypted_key, &create_context)
.await
.expect("decrypt should accept matching KMS context");
assert_ne!(decrypted.plaintext_key, [0u8; 32]);
}
}