improve code for metrics and switch tokio-tar to astral-tokio-tar (#705)

* improve code for metrics and switch tokio-tar to astral-tokio-tar

* remove log

* fix
This commit is contained in:
houseme
2025-10-24 13:07:56 +08:00
committed by GitHub
parent b47765b4c0
commit c5264f9703
10 changed files with 297 additions and 325 deletions
@@ -33,7 +33,7 @@ use tracing::info;
#[serial]
async fn test_comprehensive_kms_full_workflow() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("🏁 开始KMS全功能综合测试");
info!("🏁 Start the KMS full-featured synthesis test");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -43,25 +43,25 @@ async fn test_comprehensive_kms_full_workflow() -> Result<(), Box<dyn std::error
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
// Phase 1: Test all single encryption types
info!("📋 阶段1: 测试所有单文件加密类型");
info!("📋 Phase 1: Test all single-file encryption types");
test_sse_s3_encryption(&s3_client, TEST_BUCKET).await?;
test_sse_kms_encryption(&s3_client, TEST_BUCKET).await?;
test_sse_c_encryption(&s3_client, TEST_BUCKET).await?;
// Phase 2: Test KMS key management APIs
info!("📋 阶段2: 测试KMS密钥管理API");
info!("📋 Phase 2: Test the KMS Key Management API");
test_kms_key_management(&kms_env.base_env.url, &kms_env.base_env.access_key, &kms_env.base_env.secret_key).await?;
// Phase 3: Test all multipart encryption types
info!("📋 阶段3: 测试所有分片上传加密类型");
info!("📋 Phase 3: Test all shard upload encryption types");
test_all_multipart_encryption_types(&s3_client, TEST_BUCKET, "comprehensive-multipart-test").await?;
// Phase 4: Mixed workload test
info!("📋 阶段4: 混合工作负载测试");
info!("📋 Phase 4: Mixed workload testing");
test_mixed_encryption_workload(&s3_client, TEST_BUCKET).await?;
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
info!("✅ KMS全功能综合测试通过");
info!("✅ KMS fully functional comprehensive test passed");
Ok(())
}
@@ -70,7 +70,7 @@ async fn test_mixed_encryption_workload(
s3_client: &aws_sdk_s3::Client,
bucket: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
info!("🔄 测试混合加密工作负载");
info!("🔄 Test hybrid crypto workloads");
// Test configuration: different sizes and encryption types
let test_configs = vec![
@@ -89,11 +89,11 @@ async fn test_mixed_encryption_workload(
];
for (i, config) in test_configs.iter().enumerate() {
info!("🔄 执行混合测试 {}/{}: {:?}", i + 1, test_configs.len(), config.encryption_type);
info!("🔄 Perform hybrid testing {}/{}: {:?}", i + 1, test_configs.len(), config.encryption_type);
test_multipart_upload_with_config(s3_client, bucket, config).await?;
}
info!("混合加密工作负载测试通过");
info!("Hybrid cryptographic workload tests pass");
Ok(())
}
@@ -102,7 +102,7 @@ async fn test_mixed_encryption_workload(
#[serial]
async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("💪 开始KMS压力测试");
info!("💪 Start the KMS stress test");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -120,7 +120,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro
for config in stress_configs {
info!(
"💪 执行压力测试: {:?}, 总大小: {}MB",
"💪 Perform stress test: {:?}, Total size: {}MB",
config.encryption_type,
config.total_size() / (1024 * 1024)
);
@@ -128,7 +128,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro
}
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
info!("✅ KMS压力测试通过");
info!("✅ KMS stress test passed");
Ok(())
}
@@ -137,7 +137,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro
#[serial]
async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("🔐 开始加密密钥隔离综合测试");
info!("🔐 Begin the comprehensive test of encryption key isolation");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -173,14 +173,14 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
);
// Upload with different keys
info!("🔐 上传文件用密钥1");
info!("🔐 Key 1 for uploading files");
test_multipart_upload_with_config(&s3_client, TEST_BUCKET, &config1).await?;
info!("🔐 上传文件用密钥2");
info!("🔐 Key 2 for uploading files");
test_multipart_upload_with_config(&s3_client, TEST_BUCKET, &config2).await?;
// Verify that files cannot be read with wrong keys
info!("🔒 验证密钥隔离");
info!("🔒 Verify key isolation");
let wrong_key = "11111111111111111111111111111111";
let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
let wrong_key_md5 = format!("{:x}", md5::compute(wrong_key));
@@ -196,11 +196,11 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
.send()
.await;
assert!(wrong_read_result.is_err(), "应该无法用错误密钥读取加密文件");
info!("确认密钥隔离正常工作");
assert!(wrong_read_result.is_err(), "The encrypted file should not be readable with the wrong key");
info!("Confirm that key isolation is working correctly");
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
info!("加密密钥隔离综合测试通过");
info!("Encryption key isolation comprehensive test passed");
Ok(())
}
@@ -209,7 +209,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
#[serial]
async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("开始并发加密操作综合测试");
info!("Started comprehensive testing of concurrent encryption operations");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -228,7 +228,7 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e
];
// Execute uploads concurrently
info!("开始并发上传");
info!("Start concurrent uploads");
let mut tasks = Vec::new();
for config in concurrent_configs {
let client = s3_client.clone();
@@ -243,10 +243,10 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e
task.await??;
}
info!("所有并发操作完成");
info!("All concurrent operations are completed");
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
info!("并发加密操作综合测试通过");
info!("The comprehensive test of concurrent encryption operation has passed");
Ok(())
}
@@ -255,7 +255,7 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e
#[serial]
async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("📊 开始KMS性能基准测试");
info!("📊 Start KMS performance benchmarking");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -278,7 +278,7 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e
];
for (size_name, config) in perf_configs {
info!("📊 测试{}文件性能 ({}MB)", size_name, config.total_size() / (1024 * 1024));
info!("📊 Test {} file performance ({}MB)", size_name, config.total_size() / (1024 * 1024));
let start_time = std::time::Instant::now();
test_multipart_upload_with_config(&s3_client, TEST_BUCKET, &config).await?;
@@ -286,7 +286,7 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e
let throughput_mbps = (config.total_size() as f64 / (1024.0 * 1024.0)) / duration.as_secs_f64();
info!(
"📊 {}文件测试完成: {:.2}秒, 吞吐量: {:.2} MB/s",
"📊 {} file test completed: {:.2} seconds, throughput: {:.2} MB/s",
size_name,
duration.as_secs_f64(),
throughput_mbps
@@ -294,6 +294,6 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e
}
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
info!("✅ KMS性能基准测试通过");
info!("✅ KMS performance benchmark passed");
Ok(())
}
@@ -25,6 +25,7 @@ use super::common::LocalKMSTestEnvironment;
use crate::common::{TEST_BUCKET, init_logging};
use aws_sdk_s3::types::ServerSideEncryption;
use base64::Engine;
use md5::compute;
use serial_test::serial;
use std::sync::Arc;
use tokio::sync::Semaphore;
@@ -71,7 +72,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
info!("📤 Testing SSE-C with zero-byte file");
let test_key = "01234567890123456789012345678901";
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let test_key_md5 = format!("{:x}", md5::compute(test_key));
let test_key_md5 = format!("{:x}", compute(test_key));
let object_key_c = "zero-byte-sse-c";
let _put_response_c = s3_client
@@ -165,7 +166,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
info!("📤 Testing SSE-C with single-byte file");
let test_key = "01234567890123456789012345678901";
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let test_key_md5 = format!("{:x}", md5::compute(test_key));
let test_key_md5 = format!("{:x}", compute(test_key));
let object_key_c = "single-byte-sse-c";
s3_client
@@ -293,7 +294,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
info!("🔍 Testing invalid SSE-C key length");
let invalid_short_key = "short"; // Too short
let invalid_key_b64 = base64::engine::general_purpose::STANDARD.encode(invalid_short_key);
let invalid_key_md5 = format!("{:x}", md5::compute(invalid_short_key));
let invalid_key_md5 = format!("{:x}", compute(invalid_short_key));
let invalid_key_result = s3_client
.put_object()
@@ -333,7 +334,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
info!("🔍 Testing access to SSE-C object without key");
// First upload a valid SSE-C object
let valid_key_md5 = format!("{:x}", md5::compute(valid_key));
let valid_key_md5 = format!("{:x}", compute(valid_key));
s3_client
.put_object()
.bucket(TEST_BUCKET)
@@ -420,7 +421,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
// SSE-C
let key = format!("testkey{i:026}"); // 32-byte key
let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key);
let key_md5 = format!("{:x}", md5::compute(&key));
let key_md5 = format!("{:x}", compute(&key));
client
.put_object()
@@ -492,8 +493,8 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
let key1_b64 = base64::engine::general_purpose::STANDARD.encode(key1);
let key2_b64 = base64::engine::general_purpose::STANDARD.encode(key2);
let key1_md5 = format!("{:x}", md5::compute(key1));
let key2_md5 = format!("{:x}", md5::compute(key2));
let key1_md5 = format!("{:x}", compute(key1));
let key2_md5 = format!("{:x}", compute(key2));
// Upload same data with different keys
s3_client
+3 -2
View File
@@ -19,6 +19,7 @@
//! multipart upload behaviour.
use crate::common::{TEST_BUCKET, init_logging};
use md5::compute;
use serial_test::serial;
use tokio::time::{Duration, sleep};
use tracing::{error, info};
@@ -132,8 +133,8 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
let key2 = "98765432109876543210987654321098";
let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
let key1_md5 = format!("{:x}", md5::compute(key1));
let key2_md5 = format!("{:x}", md5::compute(key2));
let key1_md5 = format!("{:x}", compute(key1));
let key2_md5 = format!("{:x}", compute(key2));
let data1 = b"Vault data encrypted with key 1";
let data2 = b"Vault data encrypted with key 2";
@@ -13,25 +13,25 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! 分片上传加密功能的分步测试用例
//! Step-by-step test cases for sharded upload encryption
//!
//! 这个测试套件将验证分片上传加密功能的每一个步骤:
//! 1. 测试基础的单分片加密(验证加密基础逻辑)
//! 2. 测试多分片上传(验证分片拼接逻辑)
//! 3. 测试加密元数据的保存和读取
//! 4. 测试完整的分片上传加密流程
//! This test suite will validate every step of the sharded upload encryption feature:
//! 1. Test the underlying single-shard encryption (validate the encryption underlying logic)
//! 2. Test multi-shard uploads (verify shard stitching logic)
//! 3. Test the saving and reading of encrypted metadata
//! 4. Test the complete sharded upload encryption process
use super::common::LocalKMSTestEnvironment;
use crate::common::{TEST_BUCKET, init_logging};
use serial_test::serial;
use tracing::{debug, info};
/// 步骤1:测试基础单文件加密功能(确保SSE-S3在非分片场景下正常工作)
/// Step 1: Test the basic single-file encryption function (ensure that SSE-S3 works properly in non-sharded scenarios)
#[tokio::test]
#[serial]
async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("🧪 步骤1:测试基础单文件加密功能");
info!("🧪 Step 1: Test the basic single-file encryption function");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -40,11 +40,11 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
// 测试小文件加密(应该会内联存储)
// Test small file encryption (should be stored inline)
let test_data = b"Hello, this is a small test file for SSE-S3!";
let object_key = "test-single-file-encrypted";
info!("📤 上传小文件({}字节),启用SSE-S3加密", test_data.len());
info!("📤 Upload a small file ({} bytes) with SSE-S3 encryption enabled", test_data.len());
let put_response = s3_client
.put_object()
.bucket(TEST_BUCKET)
@@ -54,41 +54,41 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er
.send()
.await?;
debug!("PUT响应ETag: {:?}", put_response.e_tag());
debug!("PUT响应SSE: {:?}", put_response.server_side_encryption());
debug!("PUT responds to ETags: {:?}", put_response.e_tag());
debug!("PUT responds to SSE: {:?}", put_response.server_side_encryption());
// 验证PUT响应包含正确的加密头
// Verify that the PUT response contains the correct cipher header
assert_eq!(
put_response.server_side_encryption(),
Some(&aws_sdk_s3::types::ServerSideEncryption::Aes256)
);
info!("📥 下载文件并验证加密状态");
info!("📥 Download the file and verify the encryption status");
let get_response = s3_client.get_object().bucket(TEST_BUCKET).key(object_key).send().await?;
debug!("GET响应SSE: {:?}", get_response.server_side_encryption());
debug!("GET responds to SSE: {:?}", get_response.server_side_encryption());
// 验证GET响应包含正确的加密头
// Verify that the GET response contains the correct cipher header
assert_eq!(
get_response.server_side_encryption(),
Some(&aws_sdk_s3::types::ServerSideEncryption::Aes256)
);
// 验证数据完整性
// Verify data integrity
let downloaded_data = get_response.body.collect().await?.into_bytes();
assert_eq!(&downloaded_data[..], test_data);
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
info!("步骤1通过:基础单文件加密功能正常");
info!("Step 1: The basic single file encryption function is normal");
Ok(())
}
/// 步骤2:测试不加密的分片上传(确保分片上传基础功能正常)
/// Step 2: Test the unencrypted shard upload (make sure the shard upload base is working properly)
#[tokio::test]
#[serial]
async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("🧪 步骤2:测试不加密的分片上传");
info!("🧪 Step 2: Test unencrypted shard uploads");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -102,12 +102,16 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
let total_parts = 2;
let total_size = part_size * total_parts;
// 生成测试数据(有明显的模式便于验证)
// Generate test data (with obvious patterns for easy verification)
let test_data: Vec<u8> = (0..total_size).map(|i| (i % 256) as u8).collect();
info!("🚀 开始分片上传(无加密):{} parts,每个 {}MB", total_parts, part_size / (1024 * 1024));
info!(
"🚀 Start sharded upload (unencrypted): {} parts, {}MB each",
total_parts,
part_size / (1024 * 1024)
);
// 步骤1:创建分片上传
// Step 1: Create a sharded upload
let create_multipart_output = s3_client
.create_multipart_upload()
.bucket(TEST_BUCKET)
@@ -116,16 +120,16 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
.await?;
let upload_id = create_multipart_output.upload_id().unwrap();
info!("📋 创建分片上传,ID: {}", upload_id);
info!("📋 Create a shard upload with ID: {}", upload_id);
// 步骤2:上传各个分片
// Step 2: Upload individual shards
let mut completed_parts = Vec::new();
for part_number in 1..=total_parts {
let start = (part_number - 1) * part_size;
let end = std::cmp::min(start + part_size, total_size);
let part_data = &test_data[start..end];
info!("📤 上传分片 {} ({} bytes)", part_number, part_data.len());
info!("📤 Upload the shard {} ({} bytes)", part_number, part_data.len());
let upload_part_output = s3_client
.upload_part()
@@ -145,15 +149,15 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
.build(),
);
debug!("分片 {} 上传完成,ETag: {}", part_number, etag);
debug!("Fragment {} upload complete,ETag: {}", part_number, etag);
}
// 步骤3:完成分片上传
// Step 3: Complete the shard upload
let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
.set_parts(Some(completed_parts))
.build();
info!("🔗 完成分片上传");
info!("🔗 Complete the shard upload");
let complete_output = s3_client
.complete_multipart_upload()
.bucket(TEST_BUCKET)
@@ -163,10 +167,10 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
.send()
.await?;
debug!("完成分片上传,ETag: {:?}", complete_output.e_tag());
debug!("Complete the shard upload,ETag: {:?}", complete_output.e_tag());
// 步骤4:下载并验证
info!("📥 下载文件并验证数据完整性");
// Step 4: Download and verify
info!("📥 Download the file and verify data integrity");
let get_response = s3_client.get_object().bucket(TEST_BUCKET).key(object_key).send().await?;
let downloaded_data = get_response.body.collect().await?.into_bytes();
@@ -174,16 +178,16 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
assert_eq!(&downloaded_data[..], &test_data[..]);
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
info!("步骤2通过:不加密的分片上传功能正常");
info!("Step 2: Unencrypted shard upload functions normally");
Ok(())
}
/// 步骤3:测试分片上传 + SSE-S3加密(重点测试)
/// Step 3: Test Shard Upload + SSE-S3 Encryption (Focus Test)
#[tokio::test]
#[serial]
async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("🧪 步骤3:测试分片上传 + SSE-S3加密");
info!("🧪 Step 3: Test Shard Upload + SSE-S3 Encryption");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -201,12 +205,12 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
let test_data: Vec<u8> = (0..total_size).map(|i| ((i / 1000) % 256) as u8).collect();
info!(
"🔐 开始分片上传(SSE-S3加密):{} parts,每个 {}MB",
"🔐 Start sharded upload (SSE-S3 encryption): {} parts, {}MB each",
total_parts,
part_size / (1024 * 1024)
);
// 步骤1:创建分片上传并启用SSE-S3
// Step 1: Create a shard upload and enable SSE-S3
let create_multipart_output = s3_client
.create_multipart_upload()
.bucket(TEST_BUCKET)
@@ -216,24 +220,24 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
.await?;
let upload_id = create_multipart_output.upload_id().unwrap();
info!("📋 创建加密分片上传,ID: {}", upload_id);
info!("📋 Create an encrypted shard upload with ID: {}", upload_id);
// 验证CreateMultipartUpload响应(如果有SSE头的话)
// Verify the CreateMultipartUpload response (if there is an SSE header)
if let Some(sse) = create_multipart_output.server_side_encryption() {
debug!("CreateMultipartUpload包含SSE响应: {:?}", sse);
debug!("CreateMultipartUpload Contains SSE responses: {:?}", sse);
assert_eq!(sse, &aws_sdk_s3::types::ServerSideEncryption::Aes256);
} else {
debug!("CreateMultipartUpload不包含SSE响应头(某些实现中正常)");
debug!("CreateMultipartUpload does not contain SSE response headers (normal in some implementations)");
}
// 步骤2:上传各个分片
// Step 2: Upload individual shards
let mut completed_parts = Vec::new();
for part_number in 1..=total_parts {
let start = (part_number - 1) * part_size;
let end = std::cmp::min(start + part_size, total_size);
let part_data = &test_data[start..end];
info!("🔐 上传加密分片 {} ({} bytes)", part_number, part_data.len());
info!("🔐 Upload encrypted shards {} ({} bytes)", part_number, part_data.len());
let upload_part_output = s3_client
.upload_part()
@@ -253,15 +257,15 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
.build(),
);
debug!("加密分片 {} 上传完成,ETag: {}", part_number, etag);
debug!("Encrypted shard {} upload complete,ETag: {}", part_number, etag);
}
// 步骤3:完成分片上传
// Step 3: Complete the shard upload
let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
.set_parts(Some(completed_parts))
.build();
info!("🔗 完成加密分片上传");
info!("🔗 Complete the encrypted shard upload");
let complete_output = s3_client
.complete_multipart_upload()
.bucket(TEST_BUCKET)
@@ -273,20 +277,20 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
debug!("完成加密分片上传,ETag: {:?}", complete_output.e_tag());
// 步骤4HEAD请求检查元数据
// 步骤 4HEAD 请求检查元数据
info!("📋 检查对象元数据");
let head_response = s3_client.head_object().bucket(TEST_BUCKET).key(object_key).send().await?;
debug!("HEAD响应 SSE: {:?}", head_response.server_side_encryption());
debug!("HEAD响应 元数据: {:?}", head_response.metadata());
debug!("HEAD 响应 SSE: {:?}", head_response.server_side_encryption());
debug!("HEAD 响应 元数据{:?}", head_response.metadata());
// 步骤5GET请求下载并验证
// 步骤 5GET 请求下载并验证
info!("📥 下载加密文件并验证");
let get_response = s3_client.get_object().bucket(TEST_BUCKET).key(object_key).send().await?;
debug!("GET响应 SSE: {:?}", get_response.server_side_encryption());
debug!("GET 响应 SSE: {:?}", get_response.server_side_encryption());
// 🎯 关键验证:GET响应必须包含SSE-S3加密头
// 🎯 关键验证:GET 响应必须包含 SSE-S3 加密头
assert_eq!(
get_response.server_side_encryption(),
Some(&aws_sdk_s3::types::ServerSideEncryption::Aes256)
@@ -298,16 +302,16 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
assert_eq!(&downloaded_data[..], &test_data[..]);
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
info!("✅ 步骤3通过:分片上传 + SSE-S3加密功能正常");
info!("✅ 步骤 3 通过:分片上传 + SSE-S3 加密功能正常");
Ok(())
}
/// 步骤4:测试更大的分片上传(测试流式加密)
/// 步骤 4:测试更大的分片上传(测试流式加密)
#[tokio::test]
#[serial]
async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("🧪 步骤4:测试大文件分片上传加密");
info!("🧪 步骤 4:测试大文件分片上传加密");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -317,8 +321,8 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
let object_key = "test-large-multipart-encrypted";
let part_size = 6 * 1024 * 1024; // 6MB per part (大于1MB加密块大小)
let total_parts = 3; // 总共18MB
let part_size = 6 * 1024 * 1024; // 6MB per part (大于 1MB 加密块大小)
let total_parts = 3; // 总共 18MB
let total_size = part_size * total_parts;
info!(
@@ -337,7 +341,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
})
.collect();
info!("🔐 开始大文件分片上传(SSE-S3加密)");
info!("🔐 开始大文件分片上传(SSE-S3 加密)");
// 创建分片上传
let create_multipart_output = s3_client
@@ -419,21 +423,21 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
// 逐字节验证数据(对于大文件更严格)
for (i, (&actual, &expected)) in downloaded_data.iter().zip(test_data.iter()).enumerate() {
if actual != expected {
panic!("大文件数据在第{i}字节不匹配: 实际={actual}, 期待={expected}");
panic!("大文件数据在第{i}字节不匹配实际={actual}, 期待={expected}");
}
}
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
info!("✅ 步骤4通过:大文件分片上传加密功能正常");
info!("✅ 步骤 4 通过:大文件分片上传加密功能正常");
Ok(())
}
/// 步骤5:测试所有加密类型的分片上传
/// 步骤 5:测试所有加密类型的分片上传
#[tokio::test]
#[serial]
async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("🧪 步骤5:测试所有加密类型的分片上传");
info!("🧪 步骤 5:测试所有加密类型的分片上传");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -446,7 +450,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
let total_parts = 2;
let total_size = part_size * total_parts;
// 测试SSE-KMS
// 测试 SSE-KMS
info!("🔐 测试 SSE-KMS 分片上传");
test_multipart_encryption_type(
&s3_client,
@@ -459,7 +463,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
)
.await?;
// 测试SSE-C
// 测试 SSE-C
info!("🔐 测试 SSE-C 分片上传");
test_multipart_encryption_type(
&s3_client,
@@ -473,7 +477,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
.await?;
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
info!("✅ 步骤5通过:所有加密类型的分片上传功能正常");
info!("✅ 步骤 5 通过:所有加密类型的分片上传功能正常");
Ok(())
}
@@ -496,7 +500,7 @@ async fn test_multipart_encryption_type(
// 生成测试数据
let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 7) % 256) as u8).collect();
// 准备SSE-C所需的密钥(如果需要)
// 准备 SSE-C 所需的密钥(如果需要)
let (sse_c_key, sse_c_md5) = if matches!(encryption_type, EncryptionType::SSEC) {
let key = "01234567890123456789012345678901";
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key);
@@ -537,7 +541,7 @@ async fn test_multipart_encryption_type(
.part_number(part_number as i32)
.body(aws_sdk_s3::primitives::ByteStream::from(part_data.to_vec()));
// SSE-C需要在每个UploadPart请求中包含密钥
// SSE-C 需要在每个 UploadPart 请求中包含密钥
if matches!(encryption_type, EncryptionType::SSEC) {
upload_request = upload_request
.sse_customer_algorithm("AES256")
@@ -574,7 +578,7 @@ async fn test_multipart_encryption_type(
// 下载并验证
let mut get_request = s3_client.get_object().bucket(bucket).key(object_key);
// SSE-C需要在GET请求中包含密钥
// SSE-C 需要在 GET 请求中包含密钥
if matches!(encryption_type, EncryptionType::SSEC) {
get_request = get_request
.sse_customer_algorithm("AES256")
+1 -1
View File
@@ -37,7 +37,7 @@ async-compression = { workspace = true, features = [
] }
tokio = { workspace = true, features = ["full"] }
tokio-stream = { workspace = true }
tokio-tar = { workspace = true }
astral-tokio-tar = { workspace = true }
[lints]
+2 -2
View File
@@ -12,11 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZlibDecoder, ZstdDecoder};
use async_compression::tokio::write::{BzEncoder, GzipEncoder, XzEncoder, ZlibEncoder, ZstdEncoder};
// use async_zip::tokio::read::seek::ZipFileReader;
// use async_zip::tokio::write::ZipFileWriter;
// use async_zip::{Compression, ZipEntryBuilder};
use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZlibDecoder, ZstdDecoder};
use async_compression::tokio::write::{BzEncoder, GzipEncoder, XzEncoder, ZlibEncoder, ZstdEncoder};
use std::path::Path;
use tokio::fs::File;
use tokio::io::{self, AsyncRead, AsyncWrite, AsyncWriteExt, BufReader, BufWriter};