mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 03:46:37 +00:00
improve code for metrics and switch tokio-tar to astral-tokio-tar (#705)
* improve code for metrics and switch tokio-tar to astral-tokio-tar * remove log * fix
This commit is contained in:
@@ -33,7 +33,7 @@ use tracing::info;
|
||||
#[serial]
|
||||
async fn test_comprehensive_kms_full_workflow() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("🏁 开始KMS全功能综合测试");
|
||||
info!("🏁 Start the KMS full-featured synthesis test");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
@@ -43,25 +43,25 @@ async fn test_comprehensive_kms_full_workflow() -> Result<(), Box<dyn std::error
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
// Phase 1: Test all single encryption types
|
||||
info!("📋 阶段1: 测试所有单文件加密类型");
|
||||
info!("📋 Phase 1: Test all single-file encryption types");
|
||||
test_sse_s3_encryption(&s3_client, TEST_BUCKET).await?;
|
||||
test_sse_kms_encryption(&s3_client, TEST_BUCKET).await?;
|
||||
test_sse_c_encryption(&s3_client, TEST_BUCKET).await?;
|
||||
|
||||
// Phase 2: Test KMS key management APIs
|
||||
info!("📋 阶段2: 测试KMS密钥管理API");
|
||||
info!("📋 Phase 2: Test the KMS Key Management API");
|
||||
test_kms_key_management(&kms_env.base_env.url, &kms_env.base_env.access_key, &kms_env.base_env.secret_key).await?;
|
||||
|
||||
// Phase 3: Test all multipart encryption types
|
||||
info!("📋 阶段3: 测试所有分片上传加密类型");
|
||||
info!("📋 Phase 3: Test all shard upload encryption types");
|
||||
test_all_multipart_encryption_types(&s3_client, TEST_BUCKET, "comprehensive-multipart-test").await?;
|
||||
|
||||
// Phase 4: Mixed workload test
|
||||
info!("📋 阶段4: 混合工作负载测试");
|
||||
info!("📋 Phase 4: Mixed workload testing");
|
||||
test_mixed_encryption_workload(&s3_client, TEST_BUCKET).await?;
|
||||
|
||||
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
|
||||
info!("✅ KMS全功能综合测试通过");
|
||||
info!("✅ KMS fully functional comprehensive test passed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@ async fn test_mixed_encryption_workload(
|
||||
s3_client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
info!("🔄 测试混合加密工作负载");
|
||||
info!("🔄 Test hybrid crypto workloads");
|
||||
|
||||
// Test configuration: different sizes and encryption types
|
||||
let test_configs = vec![
|
||||
@@ -89,11 +89,11 @@ async fn test_mixed_encryption_workload(
|
||||
];
|
||||
|
||||
for (i, config) in test_configs.iter().enumerate() {
|
||||
info!("🔄 执行混合测试 {}/{}: {:?}", i + 1, test_configs.len(), config.encryption_type);
|
||||
info!("🔄 Perform hybrid testing {}/{}: {:?}", i + 1, test_configs.len(), config.encryption_type);
|
||||
test_multipart_upload_with_config(s3_client, bucket, config).await?;
|
||||
}
|
||||
|
||||
info!("✅ 混合加密工作负载测试通过");
|
||||
info!("✅ Hybrid cryptographic workload tests pass");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -102,7 +102,7 @@ async fn test_mixed_encryption_workload(
|
||||
#[serial]
|
||||
async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("💪 开始KMS压力测试");
|
||||
info!("💪 Start the KMS stress test");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
@@ -120,7 +120,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro
|
||||
|
||||
for config in stress_configs {
|
||||
info!(
|
||||
"💪 执行压力测试: {:?}, 总大小: {}MB",
|
||||
"💪 Perform stress test: {:?}, Total size: {}MB",
|
||||
config.encryption_type,
|
||||
config.total_size() / (1024 * 1024)
|
||||
);
|
||||
@@ -128,7 +128,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro
|
||||
}
|
||||
|
||||
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
|
||||
info!("✅ KMS压力测试通过");
|
||||
info!("✅ KMS stress test passed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -137,7 +137,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro
|
||||
#[serial]
|
||||
async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("🔐 开始加密密钥隔离综合测试");
|
||||
info!("🔐 Begin the comprehensive test of encryption key isolation");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
@@ -173,14 +173,14 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
|
||||
);
|
||||
|
||||
// Upload with different keys
|
||||
info!("🔐 上传文件用密钥1");
|
||||
info!("🔐 Key 1 for uploading files");
|
||||
test_multipart_upload_with_config(&s3_client, TEST_BUCKET, &config1).await?;
|
||||
|
||||
info!("🔐 上传文件用密钥2");
|
||||
info!("🔐 Key 2 for uploading files");
|
||||
test_multipart_upload_with_config(&s3_client, TEST_BUCKET, &config2).await?;
|
||||
|
||||
// Verify that files cannot be read with wrong keys
|
||||
info!("🔒 验证密钥隔离");
|
||||
info!("🔒 Verify key isolation");
|
||||
let wrong_key = "11111111111111111111111111111111";
|
||||
let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
|
||||
let wrong_key_md5 = format!("{:x}", md5::compute(wrong_key));
|
||||
@@ -196,11 +196,11 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
|
||||
.send()
|
||||
.await;
|
||||
|
||||
assert!(wrong_read_result.is_err(), "应该无法用错误密钥读取加密文件");
|
||||
info!("✅ 确认密钥隔离正常工作");
|
||||
assert!(wrong_read_result.is_err(), "The encrypted file should not be readable with the wrong key");
|
||||
info!("✅ Confirm that key isolation is working correctly");
|
||||
|
||||
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
|
||||
info!("✅ 加密密钥隔离综合测试通过");
|
||||
info!("✅ Encryption key isolation comprehensive test passed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -209,7 +209,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
|
||||
#[serial]
|
||||
async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("⚡ 开始并发加密操作综合测试");
|
||||
info!("⚡ Started comprehensive testing of concurrent encryption operations");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
@@ -228,7 +228,7 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e
|
||||
];
|
||||
|
||||
// Execute uploads concurrently
|
||||
info!("⚡ 开始并发上传");
|
||||
info!("⚡ Start concurrent uploads");
|
||||
let mut tasks = Vec::new();
|
||||
for config in concurrent_configs {
|
||||
let client = s3_client.clone();
|
||||
@@ -243,10 +243,10 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e
|
||||
task.await??;
|
||||
}
|
||||
|
||||
info!("✅ 所有并发操作完成");
|
||||
info!("✅ All concurrent operations are completed");
|
||||
|
||||
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
|
||||
info!("✅ 并发加密操作综合测试通过");
|
||||
info!("✅ The comprehensive test of concurrent encryption operation has passed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -255,7 +255,7 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e
|
||||
#[serial]
|
||||
async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("📊 开始KMS性能基准测试");
|
||||
info!("📊 Start KMS performance benchmarking");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
@@ -278,7 +278,7 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e
|
||||
];
|
||||
|
||||
for (size_name, config) in perf_configs {
|
||||
info!("📊 测试{}文件性能 ({}MB)", size_name, config.total_size() / (1024 * 1024));
|
||||
info!("📊 Test {} file performance ({}MB)", size_name, config.total_size() / (1024 * 1024));
|
||||
|
||||
let start_time = std::time::Instant::now();
|
||||
test_multipart_upload_with_config(&s3_client, TEST_BUCKET, &config).await?;
|
||||
@@ -286,7 +286,7 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e
|
||||
|
||||
let throughput_mbps = (config.total_size() as f64 / (1024.0 * 1024.0)) / duration.as_secs_f64();
|
||||
info!(
|
||||
"📊 {}文件测试完成: {:.2}秒, 吞吐量: {:.2} MB/s",
|
||||
"📊 {} file test completed: {:.2} seconds, throughput: {:.2} MB/s",
|
||||
size_name,
|
||||
duration.as_secs_f64(),
|
||||
throughput_mbps
|
||||
@@ -294,6 +294,6 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e
|
||||
}
|
||||
|
||||
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
|
||||
info!("✅ KMS性能基准测试通过");
|
||||
info!("✅ KMS performance benchmark passed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ use super::common::LocalKMSTestEnvironment;
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64::Engine;
|
||||
use md5::compute;
|
||||
use serial_test::serial;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Semaphore;
|
||||
@@ -71,7 +72,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
|
||||
info!("📤 Testing SSE-C with zero-byte file");
|
||||
let test_key = "01234567890123456789012345678901";
|
||||
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
|
||||
let test_key_md5 = format!("{:x}", md5::compute(test_key));
|
||||
let test_key_md5 = format!("{:x}", compute(test_key));
|
||||
let object_key_c = "zero-byte-sse-c";
|
||||
|
||||
let _put_response_c = s3_client
|
||||
@@ -165,7 +166,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
|
||||
info!("📤 Testing SSE-C with single-byte file");
|
||||
let test_key = "01234567890123456789012345678901";
|
||||
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
|
||||
let test_key_md5 = format!("{:x}", md5::compute(test_key));
|
||||
let test_key_md5 = format!("{:x}", compute(test_key));
|
||||
let object_key_c = "single-byte-sse-c";
|
||||
|
||||
s3_client
|
||||
@@ -293,7 +294,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
|
||||
info!("🔍 Testing invalid SSE-C key length");
|
||||
let invalid_short_key = "short"; // Too short
|
||||
let invalid_key_b64 = base64::engine::general_purpose::STANDARD.encode(invalid_short_key);
|
||||
let invalid_key_md5 = format!("{:x}", md5::compute(invalid_short_key));
|
||||
let invalid_key_md5 = format!("{:x}", compute(invalid_short_key));
|
||||
|
||||
let invalid_key_result = s3_client
|
||||
.put_object()
|
||||
@@ -333,7 +334,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
|
||||
info!("🔍 Testing access to SSE-C object without key");
|
||||
|
||||
// First upload a valid SSE-C object
|
||||
let valid_key_md5 = format!("{:x}", md5::compute(valid_key));
|
||||
let valid_key_md5 = format!("{:x}", compute(valid_key));
|
||||
s3_client
|
||||
.put_object()
|
||||
.bucket(TEST_BUCKET)
|
||||
@@ -420,7 +421,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
|
||||
// SSE-C
|
||||
let key = format!("testkey{i:026}"); // 32-byte key
|
||||
let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key);
|
||||
let key_md5 = format!("{:x}", md5::compute(&key));
|
||||
let key_md5 = format!("{:x}", compute(&key));
|
||||
|
||||
client
|
||||
.put_object()
|
||||
@@ -492,8 +493,8 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
|
||||
|
||||
let key1_b64 = base64::engine::general_purpose::STANDARD.encode(key1);
|
||||
let key2_b64 = base64::engine::general_purpose::STANDARD.encode(key2);
|
||||
let key1_md5 = format!("{:x}", md5::compute(key1));
|
||||
let key2_md5 = format!("{:x}", md5::compute(key2));
|
||||
let key1_md5 = format!("{:x}", compute(key1));
|
||||
let key2_md5 = format!("{:x}", compute(key2));
|
||||
|
||||
// Upload same data with different keys
|
||||
s3_client
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
//! multipart upload behaviour.
|
||||
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use md5::compute;
|
||||
use serial_test::serial;
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::{error, info};
|
||||
@@ -132,8 +133,8 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
|
||||
let key2 = "98765432109876543210987654321098";
|
||||
let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
|
||||
let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
|
||||
let key1_md5 = format!("{:x}", md5::compute(key1));
|
||||
let key2_md5 = format!("{:x}", md5::compute(key2));
|
||||
let key1_md5 = format!("{:x}", compute(key1));
|
||||
let key2_md5 = format!("{:x}", compute(key2));
|
||||
|
||||
let data1 = b"Vault data encrypted with key 1";
|
||||
let data2 = b"Vault data encrypted with key 2";
|
||||
|
||||
@@ -13,25 +13,25 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! 分片上传加密功能的分步测试用例
|
||||
//! Step-by-step test cases for sharded upload encryption
|
||||
//!
|
||||
//! 这个测试套件将验证分片上传加密功能的每一个步骤:
|
||||
//! 1. 测试基础的单分片加密(验证加密基础逻辑)
|
||||
//! 2. 测试多分片上传(验证分片拼接逻辑)
|
||||
//! 3. 测试加密元数据的保存和读取
|
||||
//! 4. 测试完整的分片上传加密流程
|
||||
//! This test suite will validate every step of the sharded upload encryption feature:
|
||||
//! 1. Test the underlying single-shard encryption (validate the encryption underlying logic)
|
||||
//! 2. Test multi-shard uploads (verify shard stitching logic)
|
||||
//! 3. Test the saving and reading of encrypted metadata
|
||||
//! 4. Test the complete sharded upload encryption process
|
||||
|
||||
use super::common::LocalKMSTestEnvironment;
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use serial_test::serial;
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// 步骤1:测试基础单文件加密功能(确保SSE-S3在非分片场景下正常工作)
|
||||
/// Step 1: Test the basic single-file encryption function (ensure that SSE-S3 works properly in non-sharded scenarios)
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("🧪 步骤1:测试基础单文件加密功能");
|
||||
info!("🧪 Step 1: Test the basic single-file encryption function");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
@@ -40,11 +40,11 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
// 测试小文件加密(应该会内联存储)
|
||||
// Test small file encryption (should be stored inline)
|
||||
let test_data = b"Hello, this is a small test file for SSE-S3!";
|
||||
let object_key = "test-single-file-encrypted";
|
||||
|
||||
info!("📤 上传小文件({}字节),启用SSE-S3加密", test_data.len());
|
||||
info!("📤 Upload a small file ({} bytes) with SSE-S3 encryption enabled", test_data.len());
|
||||
let put_response = s3_client
|
||||
.put_object()
|
||||
.bucket(TEST_BUCKET)
|
||||
@@ -54,41 +54,41 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
debug!("PUT响应ETag: {:?}", put_response.e_tag());
|
||||
debug!("PUT响应SSE: {:?}", put_response.server_side_encryption());
|
||||
debug!("PUT responds to ETags: {:?}", put_response.e_tag());
|
||||
debug!("PUT responds to SSE: {:?}", put_response.server_side_encryption());
|
||||
|
||||
// 验证PUT响应包含正确的加密头
|
||||
// Verify that the PUT response contains the correct cipher header
|
||||
assert_eq!(
|
||||
put_response.server_side_encryption(),
|
||||
Some(&aws_sdk_s3::types::ServerSideEncryption::Aes256)
|
||||
);
|
||||
|
||||
info!("📥 下载文件并验证加密状态");
|
||||
info!("📥 Download the file and verify the encryption status");
|
||||
let get_response = s3_client.get_object().bucket(TEST_BUCKET).key(object_key).send().await?;
|
||||
|
||||
debug!("GET响应SSE: {:?}", get_response.server_side_encryption());
|
||||
debug!("GET responds to SSE: {:?}", get_response.server_side_encryption());
|
||||
|
||||
// 验证GET响应包含正确的加密头
|
||||
// Verify that the GET response contains the correct cipher header
|
||||
assert_eq!(
|
||||
get_response.server_side_encryption(),
|
||||
Some(&aws_sdk_s3::types::ServerSideEncryption::Aes256)
|
||||
);
|
||||
|
||||
// 验证数据完整性
|
||||
// Verify data integrity
|
||||
let downloaded_data = get_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(&downloaded_data[..], test_data);
|
||||
|
||||
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
|
||||
info!("✅ 步骤1通过:基础单文件加密功能正常");
|
||||
info!("✅ Step 1: The basic single file encryption function is normal");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 步骤2:测试不加密的分片上传(确保分片上传基础功能正常)
|
||||
/// Step 2: Test the unencrypted shard upload (make sure the shard upload base is working properly)
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("🧪 步骤2:测试不加密的分片上传");
|
||||
info!("🧪 Step 2: Test unencrypted shard uploads");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
@@ -102,12 +102,16 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
|
||||
let total_parts = 2;
|
||||
let total_size = part_size * total_parts;
|
||||
|
||||
// 生成测试数据(有明显的模式便于验证)
|
||||
// Generate test data (with obvious patterns for easy verification)
|
||||
let test_data: Vec<u8> = (0..total_size).map(|i| (i % 256) as u8).collect();
|
||||
|
||||
info!("🚀 开始分片上传(无加密):{} parts,每个 {}MB", total_parts, part_size / (1024 * 1024));
|
||||
info!(
|
||||
"🚀 Start sharded upload (unencrypted): {} parts, {}MB each",
|
||||
total_parts,
|
||||
part_size / (1024 * 1024)
|
||||
);
|
||||
|
||||
// 步骤1:创建分片上传
|
||||
// Step 1: Create a sharded upload
|
||||
let create_multipart_output = s3_client
|
||||
.create_multipart_upload()
|
||||
.bucket(TEST_BUCKET)
|
||||
@@ -116,16 +120,16 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
|
||||
.await?;
|
||||
|
||||
let upload_id = create_multipart_output.upload_id().unwrap();
|
||||
info!("📋 创建分片上传,ID: {}", upload_id);
|
||||
info!("📋 Create a shard upload with ID: {}", upload_id);
|
||||
|
||||
// 步骤2:上传各个分片
|
||||
// Step 2: Upload individual shards
|
||||
let mut completed_parts = Vec::new();
|
||||
for part_number in 1..=total_parts {
|
||||
let start = (part_number - 1) * part_size;
|
||||
let end = std::cmp::min(start + part_size, total_size);
|
||||
let part_data = &test_data[start..end];
|
||||
|
||||
info!("📤 上传分片 {} ({} bytes)", part_number, part_data.len());
|
||||
info!("📤 Upload the shard {} ({} bytes)", part_number, part_data.len());
|
||||
|
||||
let upload_part_output = s3_client
|
||||
.upload_part()
|
||||
@@ -145,15 +149,15 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
|
||||
.build(),
|
||||
);
|
||||
|
||||
debug!("分片 {} 上传完成,ETag: {}", part_number, etag);
|
||||
debug!("Fragment {} upload complete,ETag: {}", part_number, etag);
|
||||
}
|
||||
|
||||
// 步骤3:完成分片上传
|
||||
// Step 3: Complete the shard upload
|
||||
let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
|
||||
.set_parts(Some(completed_parts))
|
||||
.build();
|
||||
|
||||
info!("🔗 完成分片上传");
|
||||
info!("🔗 Complete the shard upload");
|
||||
let complete_output = s3_client
|
||||
.complete_multipart_upload()
|
||||
.bucket(TEST_BUCKET)
|
||||
@@ -163,10 +167,10 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
debug!("完成分片上传,ETag: {:?}", complete_output.e_tag());
|
||||
debug!("Complete the shard upload,ETag: {:?}", complete_output.e_tag());
|
||||
|
||||
// 步骤4:下载并验证
|
||||
info!("📥 下载文件并验证数据完整性");
|
||||
// Step 4: Download and verify
|
||||
info!("📥 Download the file and verify data integrity");
|
||||
let get_response = s3_client.get_object().bucket(TEST_BUCKET).key(object_key).send().await?;
|
||||
|
||||
let downloaded_data = get_response.body.collect().await?.into_bytes();
|
||||
@@ -174,16 +178,16 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
|
||||
assert_eq!(&downloaded_data[..], &test_data[..]);
|
||||
|
||||
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
|
||||
info!("✅ 步骤2通过:不加密的分片上传功能正常");
|
||||
info!("✅ Step 2: Unencrypted shard upload functions normally");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 步骤3:测试分片上传 + SSE-S3加密(重点测试)
|
||||
/// Step 3: Test Shard Upload + SSE-S3 Encryption (Focus Test)
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("🧪 步骤3:测试分片上传 + SSE-S3加密");
|
||||
info!("🧪 Step 3: Test Shard Upload + SSE-S3 Encryption");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
@@ -201,12 +205,12 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
|
||||
let test_data: Vec<u8> = (0..total_size).map(|i| ((i / 1000) % 256) as u8).collect();
|
||||
|
||||
info!(
|
||||
"🔐 开始分片上传(SSE-S3加密):{} parts,每个 {}MB",
|
||||
"🔐 Start sharded upload (SSE-S3 encryption): {} parts, {}MB each",
|
||||
total_parts,
|
||||
part_size / (1024 * 1024)
|
||||
);
|
||||
|
||||
// 步骤1:创建分片上传并启用SSE-S3
|
||||
// Step 1: Create a shard upload and enable SSE-S3
|
||||
let create_multipart_output = s3_client
|
||||
.create_multipart_upload()
|
||||
.bucket(TEST_BUCKET)
|
||||
@@ -216,24 +220,24 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
|
||||
.await?;
|
||||
|
||||
let upload_id = create_multipart_output.upload_id().unwrap();
|
||||
info!("📋 创建加密分片上传,ID: {}", upload_id);
|
||||
info!("📋 Create an encrypted shard upload with ID: {}", upload_id);
|
||||
|
||||
// 验证CreateMultipartUpload响应(如果有SSE头的话)
|
||||
// Verify the CreateMultipartUpload response (if there is an SSE header)
|
||||
if let Some(sse) = create_multipart_output.server_side_encryption() {
|
||||
debug!("CreateMultipartUpload包含SSE响应: {:?}", sse);
|
||||
debug!("CreateMultipartUpload Contains SSE responses: {:?}", sse);
|
||||
assert_eq!(sse, &aws_sdk_s3::types::ServerSideEncryption::Aes256);
|
||||
} else {
|
||||
debug!("CreateMultipartUpload不包含SSE响应头(某些实现中正常)");
|
||||
debug!("CreateMultipartUpload does not contain SSE response headers (normal in some implementations)");
|
||||
}
|
||||
|
||||
// 步骤2:上传各个分片
|
||||
// Step 2: Upload individual shards
|
||||
let mut completed_parts = Vec::new();
|
||||
for part_number in 1..=total_parts {
|
||||
let start = (part_number - 1) * part_size;
|
||||
let end = std::cmp::min(start + part_size, total_size);
|
||||
let part_data = &test_data[start..end];
|
||||
|
||||
info!("🔐 上传加密分片 {} ({} bytes)", part_number, part_data.len());
|
||||
info!("🔐 Upload encrypted shards {} ({} bytes)", part_number, part_data.len());
|
||||
|
||||
let upload_part_output = s3_client
|
||||
.upload_part()
|
||||
@@ -253,15 +257,15 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
|
||||
.build(),
|
||||
);
|
||||
|
||||
debug!("加密分片 {} 上传完成,ETag: {}", part_number, etag);
|
||||
debug!("Encrypted shard {} upload complete,ETag: {}", part_number, etag);
|
||||
}
|
||||
|
||||
// 步骤3:完成分片上传
|
||||
// Step 3: Complete the shard upload
|
||||
let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
|
||||
.set_parts(Some(completed_parts))
|
||||
.build();
|
||||
|
||||
info!("🔗 完成加密分片上传");
|
||||
info!("🔗 Complete the encrypted shard upload");
|
||||
let complete_output = s3_client
|
||||
.complete_multipart_upload()
|
||||
.bucket(TEST_BUCKET)
|
||||
@@ -273,20 +277,20 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
|
||||
|
||||
debug!("完成加密分片上传,ETag: {:?}", complete_output.e_tag());
|
||||
|
||||
// 步骤4:HEAD请求检查元数据
|
||||
// 步骤 4:HEAD 请求检查元数据
|
||||
info!("📋 检查对象元数据");
|
||||
let head_response = s3_client.head_object().bucket(TEST_BUCKET).key(object_key).send().await?;
|
||||
|
||||
debug!("HEAD响应 SSE: {:?}", head_response.server_side_encryption());
|
||||
debug!("HEAD响应 元数据: {:?}", head_response.metadata());
|
||||
debug!("HEAD 响应 SSE: {:?}", head_response.server_side_encryption());
|
||||
debug!("HEAD 响应 元数据:{:?}", head_response.metadata());
|
||||
|
||||
// 步骤5:GET请求下载并验证
|
||||
// 步骤 5:GET 请求下载并验证
|
||||
info!("📥 下载加密文件并验证");
|
||||
let get_response = s3_client.get_object().bucket(TEST_BUCKET).key(object_key).send().await?;
|
||||
|
||||
debug!("GET响应 SSE: {:?}", get_response.server_side_encryption());
|
||||
debug!("GET 响应 SSE: {:?}", get_response.server_side_encryption());
|
||||
|
||||
// 🎯 关键验证:GET响应必须包含SSE-S3加密头
|
||||
// 🎯 关键验证:GET 响应必须包含 SSE-S3 加密头
|
||||
assert_eq!(
|
||||
get_response.server_side_encryption(),
|
||||
Some(&aws_sdk_s3::types::ServerSideEncryption::Aes256)
|
||||
@@ -298,16 +302,16 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
|
||||
assert_eq!(&downloaded_data[..], &test_data[..]);
|
||||
|
||||
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
|
||||
info!("✅ 步骤3通过:分片上传 + SSE-S3加密功能正常");
|
||||
info!("✅ 步骤 3 通过:分片上传 + SSE-S3 加密功能正常");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 步骤4:测试更大的分片上传(测试流式加密)
|
||||
/// 步骤 4:测试更大的分片上传(测试流式加密)
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("🧪 步骤4:测试大文件分片上传加密");
|
||||
info!("🧪 步骤 4:测试大文件分片上传加密");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
@@ -317,8 +321,8 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
let object_key = "test-large-multipart-encrypted";
|
||||
let part_size = 6 * 1024 * 1024; // 6MB per part (大于1MB加密块大小)
|
||||
let total_parts = 3; // 总共18MB
|
||||
let part_size = 6 * 1024 * 1024; // 6MB per part (大于 1MB 加密块大小)
|
||||
let total_parts = 3; // 总共 18MB
|
||||
let total_size = part_size * total_parts;
|
||||
|
||||
info!(
|
||||
@@ -337,7 +341,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
|
||||
})
|
||||
.collect();
|
||||
|
||||
info!("🔐 开始大文件分片上传(SSE-S3加密)");
|
||||
info!("🔐 开始大文件分片上传(SSE-S3 加密)");
|
||||
|
||||
// 创建分片上传
|
||||
let create_multipart_output = s3_client
|
||||
@@ -419,21 +423,21 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
|
||||
// 逐字节验证数据(对于大文件更严格)
|
||||
for (i, (&actual, &expected)) in downloaded_data.iter().zip(test_data.iter()).enumerate() {
|
||||
if actual != expected {
|
||||
panic!("大文件数据在第{i}字节不匹配: 实际={actual}, 期待={expected}");
|
||||
panic!("大文件数据在第{i}字节不匹配:实际={actual}, 期待={expected}");
|
||||
}
|
||||
}
|
||||
|
||||
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
|
||||
info!("✅ 步骤4通过:大文件分片上传加密功能正常");
|
||||
info!("✅ 步骤 4 通过:大文件分片上传加密功能正常");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 步骤5:测试所有加密类型的分片上传
|
||||
/// 步骤 5:测试所有加密类型的分片上传
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("🧪 步骤5:测试所有加密类型的分片上传");
|
||||
info!("🧪 步骤 5:测试所有加密类型的分片上传");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
@@ -446,7 +450,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
|
||||
let total_parts = 2;
|
||||
let total_size = part_size * total_parts;
|
||||
|
||||
// 测试SSE-KMS
|
||||
// 测试 SSE-KMS
|
||||
info!("🔐 测试 SSE-KMS 分片上传");
|
||||
test_multipart_encryption_type(
|
||||
&s3_client,
|
||||
@@ -459,7 +463,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
|
||||
)
|
||||
.await?;
|
||||
|
||||
// 测试SSE-C
|
||||
// 测试 SSE-C
|
||||
info!("🔐 测试 SSE-C 分片上传");
|
||||
test_multipart_encryption_type(
|
||||
&s3_client,
|
||||
@@ -473,7 +477,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
|
||||
.await?;
|
||||
|
||||
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
|
||||
info!("✅ 步骤5通过:所有加密类型的分片上传功能正常");
|
||||
info!("✅ 步骤 5 通过:所有加密类型的分片上传功能正常");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -496,7 +500,7 @@ async fn test_multipart_encryption_type(
|
||||
// 生成测试数据
|
||||
let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 7) % 256) as u8).collect();
|
||||
|
||||
// 准备SSE-C所需的密钥(如果需要)
|
||||
// 准备 SSE-C 所需的密钥(如果需要)
|
||||
let (sse_c_key, sse_c_md5) = if matches!(encryption_type, EncryptionType::SSEC) {
|
||||
let key = "01234567890123456789012345678901";
|
||||
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key);
|
||||
@@ -537,7 +541,7 @@ async fn test_multipart_encryption_type(
|
||||
.part_number(part_number as i32)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(part_data.to_vec()));
|
||||
|
||||
// SSE-C需要在每个UploadPart请求中包含密钥
|
||||
// SSE-C 需要在每个 UploadPart 请求中包含密钥
|
||||
if matches!(encryption_type, EncryptionType::SSEC) {
|
||||
upload_request = upload_request
|
||||
.sse_customer_algorithm("AES256")
|
||||
@@ -574,7 +578,7 @@ async fn test_multipart_encryption_type(
|
||||
// 下载并验证
|
||||
let mut get_request = s3_client.get_object().bucket(bucket).key(object_key);
|
||||
|
||||
// SSE-C需要在GET请求中包含密钥
|
||||
// SSE-C 需要在 GET 请求中包含密钥
|
||||
if matches!(encryption_type, EncryptionType::SSEC) {
|
||||
get_request = get_request
|
||||
.sse_customer_algorithm("AES256")
|
||||
|
||||
@@ -37,7 +37,7 @@ async-compression = { workspace = true, features = [
|
||||
] }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
tokio-stream = { workspace = true }
|
||||
tokio-tar = { workspace = true }
|
||||
astral-tokio-tar = { workspace = true }
|
||||
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -12,11 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZlibDecoder, ZstdDecoder};
|
||||
use async_compression::tokio::write::{BzEncoder, GzipEncoder, XzEncoder, ZlibEncoder, ZstdEncoder};
|
||||
// use async_zip::tokio::read::seek::ZipFileReader;
|
||||
// use async_zip::tokio::write::ZipFileWriter;
|
||||
// use async_zip::{Compression, ZipEntryBuilder};
|
||||
use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZlibDecoder, ZstdDecoder};
|
||||
use async_compression::tokio::write::{BzEncoder, GzipEncoder, XzEncoder, ZlibEncoder, ZstdEncoder};
|
||||
use std::path::Path;
|
||||
use tokio::fs::File;
|
||||
use tokio::io::{self, AsyncRead, AsyncWrite, AsyncWriteExt, BufReader, BufWriter};
|
||||
|
||||
Reference in New Issue
Block a user