Refactor: refactor SSE layer and KMS subsystem (#1703)

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
唐小鸭
2026-02-04 16:10:33 +08:00
committed by GitHub
parent 4d19b069c3
commit 7a42af922d
25 changed files with 3919 additions and 975 deletions
+2
View File
@@ -126,6 +126,8 @@ urlencoding = { workspace = true }
uuid = { workspace = true }
zip = { workspace = true }
libc = { workspace = true }
rand = { workspace = true }
aes-gcm = { workspace = true }
# Observability and Metrics
metrics = { workspace = true }
+2 -2
View File
@@ -209,7 +209,7 @@ pub(crate) async fn init_kms_system(opt: &config::Opt) -> std::io::Result<()> {
rustfs_kms::config::KmsConfig {
backend: rustfs_kms::config::KmsBackend::Vault,
backend_config: rustfs_kms::config::BackendConfig::Vault(rustfs_kms::config::VaultConfig {
backend_config: rustfs_kms::config::BackendConfig::Vault(Box::new(rustfs_kms::config::VaultConfig {
address: vault_address.clone(),
auth_method: rustfs_kms::config::VaultAuthMethod::Token {
token: vault_token.clone(),
@@ -219,7 +219,7 @@ pub(crate) async fn init_kms_system(opt: &config::Opt) -> std::io::Result<()> {
kv_mount: "secret".to_string(),
key_path_prefix: "rustfs/kms/keys".to_string(),
tls: None,
}),
})),
default_key_id: opt.kms_default_key_id.clone(),
timeout: std::time::Duration::from_secs(30),
retry_attempts: 3,
+326 -516
View File
File diff suppressed because it is too large Load Diff
+2 -180
View File
@@ -17,7 +17,7 @@ use crate::config::workload_profiles::{
};
use crate::error::ApiError;
use crate::server::cors;
use crate::storage::ecfs::{InMemoryAsyncReader, ListObjectUnorderedQuery};
use crate::storage::ecfs::ListObjectUnorderedQuery;
use axum::body::Body;
use http::{HeaderMap, HeaderValue, StatusCode};
use metrics::counter;
@@ -28,9 +28,6 @@ use rustfs_ecstore::bucket::replication::ReplicationConfigurationExt;
use rustfs_ecstore::error::StorageError;
use rustfs_ecstore::store_api::{BucketOptions, ObjectInfo, ObjectToDelete};
use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
use rustfs_filemeta::ObjectPartInfo;
use rustfs_kms::{EncryptionMetadata, ObjectEncryptionContext, get_global_encryption_service};
use rustfs_rio::{DecryptReader, Reader, WarpReader};
use rustfs_targets::EventName;
use rustfs_targets::arn::{TargetID, TargetIDError};
use rustfs_utils::http::{
@@ -40,7 +37,7 @@ use rustfs_utils::http::{
use s3s::dto::{
Delimiter, LambdaFunctionConfiguration, NotificationConfigurationFilter, ObjectLockConfiguration, ObjectLockEnabled,
ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, ObjectLockRetentionMode, QueueConfiguration,
ServerSideEncryption, TopicConfiguration,
TopicConfiguration,
};
use s3s::{S3Error, S3ErrorCode, S3Response, S3Result};
use serde_urlencoded::from_bytes;
@@ -50,7 +47,6 @@ use std::sync::Arc;
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
use time::{format_description::FormatItem, macros::format_description};
use tokio::io::AsyncRead;
use tracing::{debug, warn};
pub const RFC1123: &[FormatItem<'_>] =
@@ -326,180 +322,6 @@ pub(crate) fn get_buffer_size_opt_in(file_size: i64) -> usize {
buffer_size
}
pub(crate) async fn create_managed_encryption_material(
bucket: &str,
key: &str,
algorithm: &ServerSideEncryption,
kms_key_id: Option<String>,
original_size: i64,
) -> Result<crate::storage::ecfs::ManagedEncryptionMaterial, ApiError> {
let Some(service) = get_global_encryption_service().await else {
return Err(ApiError::from(StorageError::other("KMS encryption service is not initialized")));
};
if !is_managed_sse(algorithm) {
return Err(ApiError::from(StorageError::other(format!(
"Unsupported server-side encryption algorithm: {}",
algorithm.as_str()
))));
}
let algorithm_str = algorithm.as_str();
let mut context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
if original_size >= 0 {
context = context.with_size(original_size as u64);
}
let mut kms_key_candidate = kms_key_id;
if kms_key_candidate.is_none() {
kms_key_candidate = service.get_default_key_id().cloned();
}
let kms_key_to_use = kms_key_candidate
.clone()
.ok_or_else(|| ApiError::from(StorageError::other("No KMS key available for managed server-side encryption")))?;
let (data_key, encrypted_data_key) = service
.create_data_key(&kms_key_candidate, &context)
.await
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to create data key: {e}"))))?;
let metadata = EncryptionMetadata {
algorithm: algorithm_str.to_string(),
key_id: kms_key_to_use.clone(),
key_version: 1,
iv: data_key.nonce.to_vec(),
tag: None,
encryption_context: context.encryption_context.clone(),
encrypted_at: jiff::Zoned::now(),
original_size: if original_size >= 0 { original_size as u64 } else { 0 },
encrypted_data_key,
};
let mut headers = service.metadata_to_headers(&metadata);
headers.insert("x-rustfs-encryption-original-size".to_string(), metadata.original_size.to_string());
Ok(crate::storage::ecfs::ManagedEncryptionMaterial {
data_key,
headers,
kms_key_id: kms_key_to_use,
})
}
pub(crate) async fn decrypt_managed_encryption_key(
bucket: &str,
key: &str,
metadata: &HashMap<String, String>,
) -> Result<Option<([u8; 32], [u8; 12], Option<i64>)>, ApiError> {
if !metadata.contains_key("x-rustfs-encryption-key") {
return Ok(None);
}
let Some(service) = get_global_encryption_service().await else {
return Err(ApiError::from(StorageError::other("KMS encryption service is not initialized")));
};
let parsed = service
.headers_to_metadata(metadata)
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to parse encryption metadata: {e}"))))?;
if parsed.iv.len() != 12 {
return Err(ApiError::from(StorageError::other("Invalid encryption nonce length; expected 12 bytes")));
}
let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
let data_key = service
.decrypt_data_key(&parsed.encrypted_data_key, &context)
.await
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to decrypt data key: {e}"))))?;
let key_bytes = data_key.plaintext_key;
let mut nonce = [0u8; 12];
nonce.copy_from_slice(&parsed.iv[..12]);
let original_size = metadata
.get("x-rustfs-encryption-original-size")
.and_then(|s| s.parse::<i64>().ok());
Ok(Some((key_bytes, nonce, original_size)))
}
pub(crate) fn derive_part_nonce(base: [u8; 12], part_number: usize) -> [u8; 12] {
let mut nonce = base;
let current = u32::from_be_bytes([nonce[8], nonce[9], nonce[10], nonce[11]]);
let incremented = current.wrapping_add(part_number as u32);
nonce[8..12].copy_from_slice(&incremented.to_be_bytes());
nonce
}
pub(crate) async fn decrypt_multipart_managed_stream(
mut encrypted_stream: Box<dyn AsyncRead + Unpin + Send + Sync>,
parts: &[ObjectPartInfo],
key_bytes: [u8; 32],
base_nonce: [u8; 12],
) -> Result<(Box<dyn Reader>, i64), StorageError> {
let total_plain_capacity: usize = parts.iter().map(|part| part.actual_size.max(0) as usize).sum();
let mut plaintext = Vec::with_capacity(total_plain_capacity);
for part in parts {
if part.size == 0 {
continue;
}
let mut encrypted_part = vec![0u8; part.size];
tokio::io::AsyncReadExt::read_exact(&mut encrypted_stream, &mut encrypted_part)
.await
.map_err(|e| StorageError::other(format!("failed to read encrypted multipart segment {}: {}", part.number, e)))?;
let part_nonce = derive_part_nonce(base_nonce, part.number);
let cursor = std::io::Cursor::new(encrypted_part);
let mut decrypt_reader = DecryptReader::new(WarpReader::new(cursor), key_bytes, part_nonce);
tokio::io::AsyncReadExt::read_to_end(&mut decrypt_reader, &mut plaintext)
.await
.map_err(|e| StorageError::other(format!("failed to decrypt multipart segment {}: {}", part.number, e)))?;
}
let total_plain_size = plaintext.len() as i64;
let reader = Box::new(WarpReader::new(InMemoryAsyncReader::new(plaintext))) as Box<dyn Reader>;
Ok((reader, total_plain_size))
}
pub(crate) fn strip_managed_encryption_metadata(metadata: &mut HashMap<String, String>) {
const KEYS: [&str; 7] = [
"x-amz-server-side-encryption",
"x-amz-server-side-encryption-aws-kms-key-id",
"x-rustfs-encryption-iv",
"x-rustfs-encryption-tag",
"x-rustfs-encryption-key",
"x-rustfs-encryption-context",
"x-rustfs-encryption-original-size",
];
for key in KEYS.iter() {
metadata.remove(*key);
}
}
/// Check if the given server-side encryption algorithm is a managed SSE type
///
/// This function checks if the provided ServerSideEncryption algorithm
/// corresponds to a managed server-side encryption method, specifically
/// "AES256" or "aws:kms".
///
/// # Arguments
/// * `algorithm` - A reference to the ServerSideEncryption enum to check.
///
/// # Returns
/// * `true` if the algorithm is "AES256" or "aws:kms", otherwise `false`.
///
pub(crate) fn is_managed_sse(algorithm: &ServerSideEncryption) -> bool {
matches!(algorithm.as_str(), "AES256" | "aws:kms")
}
/// Validate object key for control characters and log special characters
///
/// This function:
+3
View File
@@ -26,5 +26,8 @@ mod ecfs_extend;
#[cfg(test)]
mod ecfs_test;
pub(crate) mod head_prefix;
mod sse;
#[cfg(test)]
mod sse_test;
pub(crate) use ecfs_extend::*;
File diff suppressed because it is too large Load Diff
+250
View File
@@ -0,0 +1,250 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(test)]
mod tests {
use crate::storage::sse::SseDekProvider;
use crate::storage::sse::TestSseDekProvider;
use rustfs_rio::{DecryptReader, EncryptReader, WarpReader};
use std::io::Cursor;
use tokio::io::AsyncReadExt;
/// Test EncryptReader encryption and DecryptReader decryption integration without KMS
/// This test verifies the complete encryption/decryption flow:
/// 1. Create TestSseDekProvider with a test master key
/// 2. Generate data encryption key (DEK) using the provider
/// 3. Encrypt data using EncryptReader
/// 4. Decrypt data using DecryptReader
/// 5. Verify decrypted data matches original plaintext
#[tokio::test]
async fn test_encrypt_reader_decrypt_reader_integration_without_kms() {
// Step 1: Create TestSseDekProvider with test master key
let provider = TestSseDekProvider::new_with_key([0x42u8; 32]);
// Step 2: Generate a data encryption key
let bucket = "test-bucket";
let key = "test-key";
let kms_key_id = "default"; // Key ID is ignored in test provider
let (data_key, _encrypted_dek) = provider
.generate_sse_dek(bucket, key, kms_key_id)
.await
.expect("Failed to generate DEK");
// Verify data key properties
assert_eq!(data_key.plaintext_key.len(), 32);
assert_eq!(data_key.nonce.len(), 12);
// Step 3: Prepare test data
let plaintext = b"Hello, World! This is a test message for encryption and decryption.";
println!("Original plaintext: {:?}", String::from_utf8_lossy(plaintext));
println!("Plaintext length: {} bytes", plaintext.len());
// Step 4: Encrypt using EncryptReader (wrap Cursor with WarpReader)
let plaintext_reader = WarpReader::new(Cursor::new(plaintext.to_vec()));
let mut encrypt_reader = EncryptReader::new(plaintext_reader, data_key.plaintext_key, data_key.nonce);
// Read encrypted data
let mut encrypted_data = Vec::new();
encrypt_reader
.read_to_end(&mut encrypted_data)
.await
.expect("Failed to read encrypted data");
println!("Encrypted data length: {} bytes", encrypted_data.len());
println!(
"First 16 bytes of encrypted data: {:02x?}",
&encrypted_data[..16.min(encrypted_data.len())]
);
// Verify encrypted data is different from plaintext
assert_ne!(
&encrypted_data[..plaintext.len().min(encrypted_data.len())],
plaintext,
"Encrypted data should be different from plaintext"
);
// Step 5: Decrypt using DecryptReader (wrap Cursor with WarpReader)
let encrypted_reader = WarpReader::new(Cursor::new(encrypted_data));
let mut decrypt_reader = DecryptReader::new(encrypted_reader, data_key.plaintext_key, data_key.nonce);
// Read decrypted data
let mut decrypted_data = Vec::new();
decrypt_reader
.read_to_end(&mut decrypted_data)
.await
.expect("Failed to read decrypted data");
println!("Decrypted data: {:?}", String::from_utf8_lossy(&decrypted_data));
println!("Decrypted length: {} bytes", decrypted_data.len());
// Step 6: Verify decrypted data matches original plaintext
assert_eq!(decrypted_data, plaintext, "Decrypted data should match original plaintext");
println!("✅ EncryptReader/DecryptReader integration test passed!");
}
/// Test EncryptReader with large data (10MB) without KMS
#[tokio::test]
async fn test_encrypt_reader_large_data_without_kms() {
// Create TestSseDekProvider with test master key
let provider = TestSseDekProvider::new_with_key([0x42u8; 32]);
let bucket = "test-bucket";
let key = "test-key-large";
let kms_key_id = "default";
let (data_key, _encrypted_dek) = provider
.generate_sse_dek(bucket, key, kms_key_id)
.await
.expect("Failed to generate DEK");
// Create 1MB of test data
let plaintext_size = 1024 * 1024 * 10;
let plaintext: Vec<u8> = (0..plaintext_size).map(|i| (i % 256) as u8).collect();
println!("Testing with {} bytes of data", plaintext.len());
// Encrypt (wrap with WarpReader)
let plaintext_reader = WarpReader::new(Cursor::new(plaintext.clone()));
let mut encrypt_reader = EncryptReader::new(plaintext_reader, data_key.plaintext_key, data_key.nonce);
let mut encrypted_data = Vec::new();
encrypt_reader
.read_to_end(&mut encrypted_data)
.await
.expect("Failed to encrypt large data");
println!("Encrypted {} bytes to {} bytes", plaintext.len(), encrypted_data.len());
// Decrypt (wrap with WarpReader)
let encrypted_reader = WarpReader::new(Cursor::new(encrypted_data));
let mut decrypt_reader = DecryptReader::new(encrypted_reader, data_key.plaintext_key, data_key.nonce);
let mut decrypted_data = Vec::new();
decrypt_reader
.read_to_end(&mut decrypted_data)
.await
.expect("Failed to decrypt large data");
// Verify
assert_eq!(decrypted_data.len(), plaintext.len(), "Decrypted size should match original");
assert_eq!(decrypted_data, plaintext, "Decrypted data should match original plaintext");
println!("✅ Large data encryption/decryption test passed!");
}
/// Test EncryptReader with different nonces produce different ciphertexts
#[tokio::test]
async fn test_encrypt_reader_different_nonces_produce_different_ciphertext() {
// Create TestSseDekProvider with test master key
let provider = TestSseDekProvider::new_with_key([0x42u8; 32]);
let bucket = "test-bucket";
let key = "test-key";
let kms_key_id = "default";
// Generate two different keys (with different nonces)
let (data_key1, _) = provider
.generate_sse_dek(bucket, key, kms_key_id)
.await
.expect("Failed to generate DEK 1");
let (data_key2, _) = provider
.generate_sse_dek(bucket, key, kms_key_id)
.await
.expect("Failed to generate DEK 2");
// Verify nonces are different
assert_ne!(data_key1.nonce, data_key2.nonce, "Different keys should have different nonces");
// Same plaintext
let plaintext = b"Same plaintext";
// Encrypt with first key (wrap with WarpReader)
let reader1 = WarpReader::new(Cursor::new(plaintext.to_vec()));
let mut encrypt_reader1 = EncryptReader::new(reader1, data_key1.plaintext_key, data_key1.nonce);
let mut encrypted1 = Vec::new();
encrypt_reader1.read_to_end(&mut encrypted1).await.unwrap();
// Encrypt with second key (wrap with WarpReader)
let reader2 = WarpReader::new(Cursor::new(plaintext.to_vec()));
let mut encrypt_reader2 = EncryptReader::new(reader2, data_key2.plaintext_key, data_key2.nonce);
let mut encrypted2 = Vec::new();
encrypt_reader2.read_to_end(&mut encrypted2).await.unwrap();
// Verify ciphertexts are different (due to different nonces/keys)
assert_ne!(
encrypted1, encrypted2,
"Same plaintext with different nonces should produce different ciphertext"
);
println!("✅ Different nonces produce different ciphertext - test passed!");
}
/// Test EncryptReader with decrypted DEK (simulating full cycle)
#[tokio::test]
async fn test_encrypt_reader_with_decrypted_dek() {
// Create TestSseDekProvider with test master key
let provider = TestSseDekProvider::new_with_key([0x42u8; 32]);
let bucket = "test-bucket";
let key = "test-key";
let kms_key_id = "default";
// Step 1: Generate DEK and get encrypted DEK
let (data_key, encrypted_dek) = provider
.generate_sse_dek(bucket, key, kms_key_id)
.await
.expect("Failed to generate DEK");
let original_plaintext_key = data_key.plaintext_key;
let original_nonce = data_key.nonce;
// Step 2: Later, decrypt the DEK (simulating GET operation)
let decrypted_plaintext_key = provider
.decrypt_sse_dek(&encrypted_dek, kms_key_id)
.await
.expect("Failed to decrypt DEK");
// Step 3: Verify decrypted key matches original
assert_eq!(
decrypted_plaintext_key, original_plaintext_key,
"Decrypted DEK should match original plaintext key"
);
// Step 4: Use decrypted key to encrypt/decrypt data
let plaintext = b"Test data with decrypted DEK";
// Encrypt with original key (wrap with WarpReader)
let reader = WarpReader::new(Cursor::new(plaintext.to_vec()));
let mut encrypt_reader = EncryptReader::new(reader, original_plaintext_key, original_nonce);
let mut encrypted_data = Vec::new();
encrypt_reader.read_to_end(&mut encrypted_data).await.unwrap();
// Decrypt with recovered key (simulating GET operation) (wrap with WarpReader)
let reader = WarpReader::new(Cursor::new(encrypted_data));
let mut decrypt_reader = DecryptReader::new(
reader,
decrypted_plaintext_key,
original_nonce, // In real scenario, read from metadata
);
let mut decrypted_data = Vec::new();
decrypt_reader.read_to_end(&mut decrypted_data).await.unwrap();
// Step 5: Verify
assert_eq!(decrypted_data, plaintext, "Data decrypted with recovered key should match original");
println!("✅ Full cycle (generate -> encrypt DEK -> decrypt DEK -> decrypt data) test passed!");
}
}