refactor(app): add application layer module entry (#1907)

2026-08-19 11:06:17 +00:00 · 2026-02-22 22:15:37 +08:00
parent 4a6e81d427
commit 4211652991
15 changed files with 3009 additions and 1044 deletions
@@ -17,7 +17,7 @@ use crate::config::workload_profiles::{
 };
 use crate::error::ApiError;
 use crate::server::cors;
-use crate::storage::ecfs::ListObjectUnorderedQuery;
+use crate::storage::ecfs::{InMemoryAsyncReader, ListObjectUnorderedQuery};
 use http::{HeaderMap, HeaderValue, StatusCode};
 use metrics::counter;
 use rustfs_ecstore::bucket::metadata_sys;
@@ -27,6 +27,9 @@ use rustfs_ecstore::bucket::replication::ReplicationConfigurationExt;
 use rustfs_ecstore::error::StorageError;
 use rustfs_ecstore::store_api::{BucketOptions, ObjectInfo, ObjectToDelete};
 use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
+use rustfs_filemeta::ObjectPartInfo;
+use rustfs_kms::{EncryptionMetadata, ObjectEncryptionContext, get_global_encryption_service};
+use rustfs_rio::{DecryptReader, Reader, WarpReader};
 use rustfs_targets::EventName;
 use rustfs_targets::arn::{TargetID, TargetIDError};
 use rustfs_utils::http::{
@@ -36,7 +39,7 @@ use rustfs_utils::http::{
 use s3s::dto::{
    Delimiter, LambdaFunctionConfiguration, NotificationConfigurationFilter, ObjectLockConfiguration, ObjectLockEnabled,
    ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, ObjectLockRetentionMode, QueueConfiguration,
-    TopicConfiguration,
+    ServerSideEncryption, TopicConfiguration,
 };
 use s3s::{S3Error, S3ErrorCode, S3Response, S3Result};
 use serde_urlencoded::from_bytes;
@@ -46,6 +49,7 @@ use std::sync::Arc;
 use time::OffsetDateTime;
 use time::format_description::well_known::Rfc3339;
 use time::{format_description::FormatItem, macros::format_description};
+use tokio::io::AsyncRead;
 use tracing::{debug, warn};

 pub const RFC1123: &[FormatItem<'_>] =
@@ -63,6 +67,7 @@ pub const RFC1123: &[FormatItem<'_>] =
 /// # Arguments
 /// * `object_lock_config` - Optional bucket Object Lock configuration. If None, no retention is applied.
 /// * `metadata` - Mutable reference to object metadata HashMap. Retention headers are inserted here.
+#[allow(dead_code)]
 pub(crate) fn apply_lock_retention(object_lock_config: Option<ObjectLockConfiguration>, metadata: &mut HashMap<String, String>) {
    if metadata.contains_key(AMZ_OBJECT_LOCK_MODE_LOWER) || metadata.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER) {
        return;
@@ -187,6 +192,168 @@ pub(crate) fn get_buffer_size_opt_in(file_size: i64) -> usize {
    buffer_size
 }

+pub(crate) async fn create_managed_encryption_material(
+    bucket: &str,
+    key: &str,
+    algorithm: &ServerSideEncryption,
+    kms_key_id: Option<String>,
+    original_size: i64,
+) -> Result<crate::storage::ecfs::ManagedEncryptionMaterial, ApiError> {
+    let Some(service) = get_global_encryption_service().await else {
+        return Err(ApiError::from(StorageError::other("KMS encryption service is not initialized")));
+    };
+
+    if !is_managed_sse(algorithm) {
+        return Err(ApiError::from(StorageError::other(format!(
+            "Unsupported server-side encryption algorithm: {}",
+            algorithm.as_str()
+        ))));
+    }
+
+    let algorithm_str = algorithm.as_str();
+
+    let mut context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
+    if original_size >= 0 {
+        context = context.with_size(original_size as u64);
+    }
+
+    let mut kms_key_candidate = kms_key_id;
+    if kms_key_candidate.is_none() {
+        kms_key_candidate = service.get_default_key_id().cloned();
+    }
+
+    let kms_key_to_use = kms_key_candidate
+        .clone()
+        .ok_or_else(|| ApiError::from(StorageError::other("No KMS key available for managed server-side encryption")))?;
+
+    let (data_key, encrypted_data_key) = service
+        .create_data_key(&kms_key_candidate, &context)
+        .await
+        .map_err(|e| ApiError::from(StorageError::other(format!("Failed to create data key: {e}"))))?;
+
+    let metadata = EncryptionMetadata {
+        algorithm: algorithm_str.to_string(),
+        key_id: kms_key_to_use.clone(),
+        key_version: 1,
+        iv: data_key.nonce.to_vec(),
+        tag: None,
+        encryption_context: context.encryption_context.clone(),
+        encrypted_at: jiff::Zoned::now(),
+        original_size: if original_size >= 0 { original_size as u64 } else { 0 },
+        encrypted_data_key,
+    };
+
+    let mut headers = service.metadata_to_headers(&metadata);
+    headers.insert("x-rustfs-encryption-original-size".to_string(), metadata.original_size.to_string());
+
+    Ok(crate::storage::ecfs::ManagedEncryptionMaterial {
+        data_key,
+        headers,
+        kms_key_id: kms_key_to_use,
+    })
+}
+
+pub(crate) async fn decrypt_managed_encryption_key(
+    bucket: &str,
+    key: &str,
+    metadata: &HashMap<String, String>,
+) -> Result<Option<([u8; 32], [u8; 12], Option<i64>)>, ApiError> {
+    if !metadata.contains_key("x-rustfs-encryption-key") {
+        return Ok(None);
+    }
+
+    let Some(service) = get_global_encryption_service().await else {
+        return Err(ApiError::from(StorageError::other("KMS encryption service is not initialized")));
+    };
+
+    let parsed = service
+        .headers_to_metadata(metadata)
+        .map_err(|e| ApiError::from(StorageError::other(format!("Failed to parse encryption metadata: {e}"))))?;
+
+    if parsed.iv.len() != 12 {
+        return Err(ApiError::from(StorageError::other("Invalid encryption nonce length; expected 12 bytes")));
+    }
+
+    let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
+    let data_key = service
+        .decrypt_data_key(&parsed.encrypted_data_key, &context)
+        .await
+        .map_err(|e| ApiError::from(StorageError::other(format!("Failed to decrypt data key: {e}"))))?;
+
+    let key_bytes = data_key.plaintext_key;
+    let mut nonce = [0u8; 12];
+    nonce.copy_from_slice(&parsed.iv[..12]);
+
+    let original_size = metadata
+        .get("x-rustfs-encryption-original-size")
+        .and_then(|s| s.parse::<i64>().ok());
+
+    Ok(Some((key_bytes, nonce, original_size)))
+}
+
+pub(crate) fn derive_part_nonce(base: [u8; 12], part_number: usize) -> [u8; 12] {
+    let mut nonce = base;
+    let current = u32::from_be_bytes([nonce[8], nonce[9], nonce[10], nonce[11]]);
+    let incremented = current.wrapping_add(part_number as u32);
+    nonce[8..12].copy_from_slice(&incremented.to_be_bytes());
+    nonce
+}
+
+pub(crate) async fn decrypt_multipart_managed_stream(
+    mut encrypted_stream: Box<dyn AsyncRead + Unpin + Send + Sync>,
+    parts: &[ObjectPartInfo],
+    key_bytes: [u8; 32],
+    base_nonce: [u8; 12],
+) -> Result<(Box<dyn Reader>, i64), StorageError> {
+    let total_plain_capacity: usize = parts.iter().map(|part| part.actual_size.max(0) as usize).sum();
+
+    let mut plaintext = Vec::with_capacity(total_plain_capacity);
+
+    for part in parts {
+        if part.size == 0 {
+            continue;
+        }
+
+        let mut encrypted_part = vec![0u8; part.size];
+        tokio::io::AsyncReadExt::read_exact(&mut encrypted_stream, &mut encrypted_part)
+            .await
+            .map_err(|e| StorageError::other(format!("failed to read encrypted multipart segment {}: {}", part.number, e)))?;
+
+        let part_nonce = derive_part_nonce(base_nonce, part.number);
+        let cursor = std::io::Cursor::new(encrypted_part);
+        let mut decrypt_reader = DecryptReader::new(WarpReader::new(cursor), key_bytes, part_nonce);
+
+        tokio::io::AsyncReadExt::read_to_end(&mut decrypt_reader, &mut plaintext)
+            .await
+            .map_err(|e| StorageError::other(format!("failed to decrypt multipart segment {}: {}", part.number, e)))?;
+    }
+
+    let total_plain_size = plaintext.len() as i64;
+    let reader = Box::new(WarpReader::new(InMemoryAsyncReader::new(plaintext))) as Box<dyn Reader>;
+
+    Ok((reader, total_plain_size))
+}
+
+pub(crate) fn strip_managed_encryption_metadata(metadata: &mut HashMap<String, String>) {
+    const KEYS: [&str; 7] = [
+        "x-amz-server-side-encryption",
+        "x-amz-server-side-encryption-aws-kms-key-id",
+        "x-rustfs-encryption-iv",
+        "x-rustfs-encryption-tag",
+        "x-rustfs-encryption-key",
+        "x-rustfs-encryption-context",
+        "x-rustfs-encryption-original-size",
+    ];
+
+    for key in KEYS.iter() {
+        metadata.remove(*key);
+    }
+}
+
+pub(crate) fn is_managed_sse(algorithm: &ServerSideEncryption) -> bool {
+    matches!(algorithm.as_str(), "AES256" | "aws:kms")
+}
+
 /// Validate object key for control characters and log special characters
 ///
 /// This function: