Files
rustfs/crates/e2e_test/src/kms/bucket_default_encryption_test.rs
T
anthonymartin e26b869259 fix(ecstore): preserve checksums through write transforms (#5765)
* fix: preserve checksums through write transforms

* test(e2e): cover SSE-KMS multipart CRC32

---------

Co-authored-by: Anthony Martin <949506+anthonymartin@users.noreply.github.com>
2026-08-06 17:02:08 +08:00

640 lines
23 KiB
Rust

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Bucket Default Encryption Configuration Integration Tests
//!
//! This test suite verifies that bucket-level default encryption configuration is properly integrated with:
//! 1. put_object operations
//! 2. create_multipart_upload operations
//! 3. KMS service integration
use super::common::LocalKMSTestEnvironment;
use crate::common::{TEST_BUCKET, init_logging};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart, ServerSideEncryption,
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
};
use rustfs_rio::{Checksum, ChecksumType};
use serial_test::serial;
use tracing::{debug, info, warn};
/// Test 1: When bucket is configured with default SSE-S3 encryption, put_object should automatically apply encryption
#[tokio::test]
#[serial]
async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Testing bucket default SSE-S3 encryption impact on put_object");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
// Step 1: Set bucket default encryption to SSE-S3
info!("Setting bucket default encryption configuration");
let encryption_config = ServerSideEncryptionConfiguration::builder()
.rules(
ServerSideEncryptionRule::builder()
.apply_server_side_encryption_by_default(
ServerSideEncryptionByDefault::builder()
.sse_algorithm(ServerSideEncryption::Aes256)
.build()
.unwrap(),
)
.build(),
)
.build()
.unwrap();
s3_client
.put_bucket_encryption()
.bucket(TEST_BUCKET)
.server_side_encryption_configuration(encryption_config)
.send()
.await
.expect("Failed to set bucket encryption");
info!("Bucket default encryption configuration set successfully");
// Verify bucket encryption configuration
let get_encryption_response = s3_client
.get_bucket_encryption()
.bucket(TEST_BUCKET)
.send()
.await
.expect("Failed to get bucket encryption");
debug!(
"Bucket encryption configuration: {:?}",
get_encryption_response.server_side_encryption_configuration()
);
// Step 2: put_object without specifying encryption parameters should automatically use bucket default encryption
info!("Uploading file (without specifying encryption parameters, should use bucket default encryption)");
let test_data = b"test-bucket-default-sse-s3-data";
let test_key = "test-bucket-default-sse-s3.txt";
let put_response = s3_client
.put_object()
.bucket(TEST_BUCKET)
.key(test_key)
.body(test_data.to_vec().into())
// Note: No server_side_encryption specified here, should use bucket default
.send()
.await
.expect("Failed to put object");
debug!(
"PUT response: ETag={:?}, SSE={:?}",
put_response.e_tag(),
put_response.server_side_encryption()
);
// Verify: Response should contain SSE-S3 encryption information
assert_eq!(
put_response.server_side_encryption(),
Some(&ServerSideEncryption::Aes256),
"put_object response should contain bucket default SSE-S3 encryption information"
);
// Step 3: Download file and verify encryption status
info!("Downloading file and verifying encryption status");
let get_response = s3_client
.get_object()
.bucket(TEST_BUCKET)
.key(test_key)
.send()
.await
.expect("Failed to get object");
debug!("GET response: SSE={:?}", get_response.server_side_encryption());
// Verify: GET response should contain encryption information
assert_eq!(
get_response.server_side_encryption(),
Some(&ServerSideEncryption::Aes256),
"get_object response should contain SSE-S3 encryption information"
);
// Verify data integrity
let downloaded_data = get_response
.body
.collect()
.await
.expect("Failed to collect body")
.into_bytes();
assert_eq!(&downloaded_data[..], test_data, "Downloaded data should match original data");
// Step 4: Explicitly specifying encryption parameters should override bucket default
info!("Uploading file (explicitly specifying no encryption, should override bucket default)");
let _test_key_2 = "test-explicit-override.txt";
// Note: This test might temporarily fail because current implementation might not support explicit override
// But this is the target behavior we want to implement
warn!("Test for explicitly overriding bucket default encryption is temporarily skipped, this is a feature to be implemented");
// TODO: Add test for explicit override when implemented
info!("Test passed: bucket default SSE-S3 encryption correctly applied to put_object");
Ok(())
}
/// Test 2: When bucket is configured with default SSE-KMS encryption, put_object should automatically apply encryption and use the specified KMS key
#[tokio::test]
#[serial]
async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Testing bucket default SSE-KMS encryption impact on put_object");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
// Step 1: Set bucket default encryption to SSE-KMS with specified KMS key
info!("Setting bucket default encryption configuration to SSE-KMS");
let encryption_config = ServerSideEncryptionConfiguration::builder()
.rules(
ServerSideEncryptionRule::builder()
.apply_server_side_encryption_by_default(
ServerSideEncryptionByDefault::builder()
.sse_algorithm(ServerSideEncryption::AwsKms)
.kms_master_key_id(&default_key_id)
.build()
.unwrap(),
)
.build(),
)
.build()
.unwrap();
s3_client
.put_bucket_encryption()
.bucket(TEST_BUCKET)
.server_side_encryption_configuration(encryption_config)
.send()
.await
.expect("Failed to set bucket SSE-KMS encryption");
info!("Bucket default SSE-KMS encryption configuration set successfully");
// Step 2: put_object without specifying encryption parameters should automatically use bucket default SSE-KMS
info!("Uploading file (without specifying encryption parameters, should use bucket default SSE-KMS)");
let test_data = b"test-bucket-default-sse-kms-data";
let test_key = "test-bucket-default-sse-kms.txt";
let put_response = s3_client
.put_object()
.bucket(TEST_BUCKET)
.key(test_key)
.body(test_data.to_vec().into())
// Note: No encryption parameters specified here, should use bucket default SSE-KMS
.send()
.await
.expect("Failed to put object with bucket default SSE-KMS");
debug!(
"PUT response: ETag={:?}, SSE={:?}, KMS_Key={:?}",
put_response.e_tag(),
put_response.server_side_encryption(),
put_response.ssekms_key_id()
);
// Verify: Response should contain SSE-KMS encryption information
assert_eq!(
put_response.server_side_encryption(),
Some(&ServerSideEncryption::AwsKms),
"put_object response should contain bucket default SSE-KMS encryption information"
);
assert_eq!(
put_response.ssekms_key_id().unwrap(),
&default_key_id,
"put_object response should contain correct KMS key ID"
);
// Step 3: Download file and verify encryption status
info!("Downloading file and verifying encryption status");
let get_response = s3_client
.get_object()
.bucket(TEST_BUCKET)
.key(test_key)
.send()
.await
.expect("Failed to get object");
debug!(
"GET response: SSE={:?}, KMS_Key={:?}",
get_response.server_side_encryption(),
get_response.ssekms_key_id()
);
// Verify: GET response should contain encryption information
assert_eq!(
get_response.server_side_encryption(),
Some(&ServerSideEncryption::AwsKms),
"get_object response should contain SSE-KMS encryption information"
);
assert_eq!(
get_response.ssekms_key_id().unwrap(),
&default_key_id,
"get_object response should contain correct KMS key ID"
);
// Verify data integrity
let downloaded_data = get_response
.body
.collect()
.await
.expect("Failed to collect body")
.into_bytes();
assert_eq!(&downloaded_data[..], test_data, "Downloaded data should match original data");
// Cleanup is handled automatically when the test environment is dropped
info!("Test passed: bucket default SSE-KMS encryption correctly applied to put_object");
Ok(())
}
/// Test 3: When bucket is configured with default encryption, create_multipart_upload should inherit the configuration
#[tokio::test]
#[serial]
async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Testing bucket default encryption impact on create_multipart_upload");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
// Step 1: Set bucket default encryption to SSE-KMS
info!("Setting bucket default encryption configuration to SSE-KMS");
let encryption_config = ServerSideEncryptionConfiguration::builder()
.rules(
ServerSideEncryptionRule::builder()
.apply_server_side_encryption_by_default(
ServerSideEncryptionByDefault::builder()
.sse_algorithm(ServerSideEncryption::AwsKms)
.kms_master_key_id(&default_key_id)
.build()
.unwrap(),
)
.build(),
)
.build()
.unwrap();
s3_client
.put_bucket_encryption()
.bucket(TEST_BUCKET)
.server_side_encryption_configuration(encryption_config)
.send()
.await
.expect("Failed to set bucket encryption");
// Step 2: Declare CRC32 without specifying encryption parameters. The AWS SDK
// calculates each UploadPart checksum and sends it as a flexible checksum.
info!("Creating CRC32 multipart upload that should use bucket default encryption");
let test_key = "test-multipart-bucket-default-crc32.bin";
let create_multipart_response = s3_client
.create_multipart_upload()
.bucket(TEST_BUCKET)
.key(test_key)
.checksum_algorithm(ChecksumAlgorithm::Crc32)
.send()
.await
.expect("Failed to create multipart upload");
let upload_id = create_multipart_response.upload_id().unwrap();
debug!(
"CreateMultipartUpload response: UploadId={}, SSE={:?}, KMS_Key={:?}",
upload_id,
create_multipart_response.server_side_encryption(),
create_multipart_response.ssekms_key_id()
);
// Verify: create_multipart_upload response should contain bucket default encryption configuration
assert_eq!(
create_multipart_response.server_side_encryption(),
Some(&ServerSideEncryption::AwsKms),
"create_multipart_upload response should contain bucket default SSE-KMS encryption information"
);
assert_eq!(
create_multipart_response.ssekms_key_id().unwrap(),
&default_key_id,
"create_multipart_upload response should contain correct KMS key ID"
);
// Step 3: Upload two parts. The first is exactly the S3 minimum size so this
// follows the same managed SSE-KMS multipart path as issue #5756.
const PART_SIZE: usize = 5 * 1024 * 1024;
let part1: Vec<u8> = (0..PART_SIZE).map(|i| (i % 251) as u8).collect();
let part2: Vec<u8> = (0..1024 * 1024).map(|i| ((i + 17) % 251) as u8).collect();
let expected_body: Vec<u8> = part1.iter().chain(&part2).copied().collect();
let upload_part = |part_number: i32, body: Vec<u8>| {
s3_client
.upload_part()
.bucket(TEST_BUCKET)
.key(test_key)
.upload_id(upload_id)
.part_number(part_number)
.checksum_algorithm(ChecksumAlgorithm::Crc32)
.body(ByteStream::from(body))
.send()
};
let expected_part1_crc32 = Checksum::new_from_data(ChecksumType::CRC32, &part1)
.expect("calculate part 1 CRC32")
.encoded;
let upload1 = upload_part(1, part1).await.expect("Failed to upload part 1 with CRC32");
assert_eq!(
upload1.checksum_crc32(),
Some(expected_part1_crc32.as_str()),
"UploadPart must return the CRC32 calculated over plaintext"
);
let expected_part2_crc32 = Checksum::new_from_data(ChecksumType::CRC32, &part2)
.expect("calculate part 2 CRC32")
.encoded;
let upload2 = upload_part(2, part2).await.expect("Failed to upload part 2 with CRC32");
assert_eq!(
upload2.checksum_crc32(),
Some(expected_part2_crc32.as_str()),
"UploadPart must return the CRC32 calculated over plaintext"
);
// Complete multipart upload
let completed_upload = CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.e_tag(upload1.e_tag().expect("No ETag for part 1"))
.checksum_crc32(upload1.checksum_crc32().expect("No CRC32 for part 1"))
.build(),
)
.parts(
CompletedPart::builder()
.part_number(2)
.e_tag(upload2.e_tag().expect("No ETag for part 2"))
.checksum_crc32(upload2.checksum_crc32().expect("No CRC32 for part 2"))
.build(),
)
.build();
let complete_multipart_response = s3_client
.complete_multipart_upload()
.bucket(TEST_BUCKET)
.key(test_key)
.upload_id(upload_id)
.multipart_upload(completed_upload)
.send()
.await
.expect("Failed to complete multipart upload");
debug!(
"CompleteMultipartUpload response: ETag={:?}, SSE={:?}, KMS_Key={:?}",
complete_multipart_response.e_tag(),
complete_multipart_response.server_side_encryption(),
complete_multipart_response.ssekms_key_id()
);
assert_eq!(
complete_multipart_response.server_side_encryption(),
Some(&ServerSideEncryption::AwsKms),
"complete_multipart_upload response should contain SSE-KMS encryption information"
);
// Step 4: Download file and verify encryption status
info!("Downloading file and verifying encryption status");
let get_response = s3_client
.get_object()
.bucket(TEST_BUCKET)
.key(test_key)
.checksum_mode(ChecksumMode::Enabled)
.send()
.await
.expect("Failed to get object");
// Verify: Final object should be properly encrypted
assert_eq!(
get_response.server_side_encryption(),
Some(&ServerSideEncryption::AwsKms),
"Final object should contain SSE-KMS encryption information"
);
if let Some(completed_crc32) = complete_multipart_response.checksum_crc32() {
assert_eq!(
get_response.checksum_crc32(),
Some(completed_crc32),
"GetObject should return the persisted composite CRC32 when completion reports it"
);
}
// Verify data integrity
let downloaded_data = get_response
.body
.collect()
.await
.expect("Failed to collect body")
.into_bytes();
assert_eq!(
downloaded_data.as_ref(),
expected_body.as_slice(),
"Downloaded data should match the uploaded multipart body"
);
// Cleanup is handled automatically when the test environment is dropped
info!("Test passed: bucket default encryption correctly applied to multipart upload");
Ok(())
}
/// Test 4: Explicitly specified encryption parameters in requests should override bucket default configuration
#[tokio::test]
#[serial]
async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Testing explicitly specified encryption parameters override bucket default configuration");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
// Step 1: Set bucket default encryption to SSE-S3
info!("Setting bucket default encryption configuration to SSE-S3");
let encryption_config = ServerSideEncryptionConfiguration::builder()
.rules(
ServerSideEncryptionRule::builder()
.apply_server_side_encryption_by_default(
ServerSideEncryptionByDefault::builder()
.sse_algorithm(ServerSideEncryption::Aes256)
.build()
.unwrap(),
)
.build(),
)
.build()
.unwrap();
s3_client
.put_bucket_encryption()
.bucket(TEST_BUCKET)
.server_side_encryption_configuration(encryption_config)
.send()
.await
.expect("Failed to set bucket encryption");
// Step 2: Explicitly specify SSE-KMS encryption (should override bucket default SSE-S3)
info!("Uploading file (explicitly specifying SSE-KMS, should override bucket default SSE-S3)");
let test_data = b"test-explicit-override-data";
let test_key = "test-explicit-override.txt";
let put_response = s3_client
.put_object()
.bucket(TEST_BUCKET)
.key(test_key)
.body(test_data.to_vec().into())
// Explicitly specify SSE-KMS, should override bucket default SSE-S3
.server_side_encryption(ServerSideEncryption::AwsKms)
.ssekms_key_id(&default_key_id)
.send()
.await
.expect("Failed to put object with explicit SSE-KMS");
debug!(
"PUT response: SSE={:?}, KMS_Key={:?}",
put_response.server_side_encryption(),
put_response.ssekms_key_id()
);
// Verify: Should use explicitly specified SSE-KMS, not bucket default SSE-S3
assert_eq!(
put_response.server_side_encryption(),
Some(&ServerSideEncryption::AwsKms),
"Explicitly specified SSE-KMS should override bucket default SSE-S3"
);
assert_eq!(
put_response.ssekms_key_id().unwrap(),
&default_key_id,
"Should use explicitly specified KMS key ID"
);
// Verify GET response
let get_response = s3_client
.get_object()
.bucket(TEST_BUCKET)
.key(test_key)
.send()
.await
.expect("Failed to get object");
assert_eq!(
get_response.server_side_encryption(),
Some(&ServerSideEncryption::AwsKms),
"GET response should reflect the actually used SSE-KMS encryption"
);
// Cleanup is handled automatically when the test environment is dropped
info!("Test passed: explicitly specified encryption parameters correctly override bucket default configuration");
Ok(())
}
/// Test 5: Setting SSE-KMS without a specific key ID should auto-populate the
/// default KMS key ID so that GetBucketEncryption returns it (issue #3039).
#[tokio::test]
#[serial]
async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Testing SSE-KMS without explicit key ID populates default key");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
// Set bucket encryption to SSE-KMS WITHOUT specifying a key ID
info!("Setting bucket default encryption to SSE-KMS without key ID");
let encryption_config = ServerSideEncryptionConfiguration::builder()
.rules(
ServerSideEncryptionRule::builder()
.apply_server_side_encryption_by_default(
ServerSideEncryptionByDefault::builder()
.sse_algorithm(ServerSideEncryption::AwsKms)
.kms_master_key_id("")
.build()
.unwrap(),
)
.build(),
)
.build()
.unwrap();
s3_client
.put_bucket_encryption()
.bucket(TEST_BUCKET)
.server_side_encryption_configuration(encryption_config)
.send()
.await
.expect("Failed to set bucket SSE-KMS encryption without key ID");
// GetBucketEncryption should return the default KMS key ID
info!("Verifying GetBucketEncryption returns default KMS key ID");
let get_response = s3_client
.get_bucket_encryption()
.bucket(TEST_BUCKET)
.send()
.await
.expect("Failed to get bucket encryption");
let config = get_response.server_side_encryption_configuration();
let rule = config
.and_then(|c| c.rules().first())
.expect("Should have at least one encryption rule");
let by_default = rule
.apply_server_side_encryption_by_default()
.expect("Should have ApplyServerSideEncryptionByDefault");
assert_eq!(by_default.sse_algorithm(), &ServerSideEncryption::AwsKms, "Algorithm should be aws:kms");
assert!(
by_default.kms_master_key_id().is_some(),
"KMS key ID should be populated with the default key (was None)"
);
assert_eq!(
by_default.kms_master_key_id().unwrap(),
&default_key_id,
"KMS key ID should match the configured default key"
);
info!("Test passed: SSE-KMS without key ID correctly populates default key '{}'", default_key_id);
Ok(())
}