gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 13:16:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: Reimplement bucket replication system with enhanced architecture (#590)

Browse Source 
* feat:refactor replication

* use aws sdk for replication client

* refactor/replication

* merge main

* fix lifecycle test
This commit is contained in:
weisd
2025-09-26 14:27:53 +08:00
committed by GitHub
parent 9b029d18b2
commit 90f21a9102
91 changed files with 10532 additions and 4917 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/common.rs
+5 -5
View File
@@ -94,7 +94,7 @@ fn build_rustfs_binary() {
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
panic!("Failed to build RustFS binary. Error: {}", stderr);
panic!("Failed to build RustFS binary. Error: {stderr}");
}
info!("✅ RustFS binary built successfully");
@@ -134,8 +134,8 @@ impl RustFSTestEnvironment {
// Use a unique port for each test environment
let port = Self::find_available_port().await?;
let address = format!("127.0.0.1:{}", port);
let url = format!("http://{}", address);
let address = format!("127.0.0.1:{port}");
let url = format!("http://{address}");
Ok(Self {
temp_dir,
@@ -152,7 +152,7 @@ impl RustFSTestEnvironment {
let temp_dir = format!("/tmp/rustfs_e2e_test_{}", Uuid::new_v4());
fs::create_dir_all(&temp_dir).await?;
let url = format!("http://{}", address);
let url = format!("http://{address}");
Ok(Self {
temp_dir,
@@ -327,7 +327,7 @@ pub async fn execute_awscurl(
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(format!("awscurl failed: {}", stderr).into());
return Err(format!("awscurl failed: {stderr}").into());
}
let response = String::from_utf8_lossy(&output.stdout).to_string();
crates/e2e_test/src/kms/common.rs
+18 -24
View File
@@ -59,7 +59,7 @@ pub async fn configure_kms(
access_key: &str,
secret_key: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/kms/configure", base_url);
let url = format!("{base_url}/rustfs/admin/v3/kms/configure");
awscurl_post(&url, config_json, access_key, secret_key).await?;
info!("KMS configured successfully");
Ok(())
@@ -71,7 +71,7 @@ pub async fn start_kms(
access_key: &str,
secret_key: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/kms/start", base_url);
let url = format!("{base_url}/rustfs/admin/v3/kms/start");
awscurl_post(&url, "{}", access_key, secret_key).await?;
info!("KMS started successfully");
Ok(())
@@ -83,7 +83,7 @@ pub async fn get_kms_status(
access_key: &str,
secret_key: &str,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/kms/status", base_url);
let url = format!("{base_url}/rustfs/admin/v3/kms/status");
let status = awscurl_get(&url, access_key, secret_key).await?;
info!("KMS status retrieved: {}", status);
Ok(status)
@@ -101,7 +101,7 @@ pub async fn create_default_key(
})
.to_string();
let url = format!("{}/rustfs/admin/v3/kms/keys", base_url);
let url = format!("{base_url}/rustfs/admin/v3/kms/keys");
let response = awscurl_post(&url, &create_key_body, access_key, secret_key).await?;
// Parse response to get the actual key ID
@@ -141,7 +141,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
});
// Write the key to file with the specified ID as JSON
let key_path = format!("{}/{}.key", key_dir, key_id);
let key_path = format!("{key_dir}/{key_id}.key");
let content = serde_json::to_vec_pretty(&stored_key)?;
fs::write(&key_path, &content).await?;
@@ -281,13 +281,8 @@ pub async fn test_kms_key_management(
})
.to_string();
let create_response = awscurl_post(
&format!("{}/rustfs/admin/v3/kms/keys", base_url),
&create_key_body,
access_key,
secret_key,
)
.await?;
let create_response =
awscurl_post(&format!("{base_url}/rustfs/admin/v3/kms/keys"), &create_key_body, access_key, secret_key).await?;
let create_result: serde_json::Value = serde_json::from_str(&create_response)?;
let key_id = create_result["key_id"]
@@ -296,8 +291,7 @@ pub async fn test_kms_key_management(
info!("Created key with ID: {}", key_id);
// Test DescribeKey
let describe_response =
awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys/{}", base_url, key_id), access_key, secret_key).await?;
let describe_response = awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?;
info!("DescribeKey response: {}", describe_response);
let describe_result: serde_json::Value = serde_json::from_str(&describe_response)?;
@@ -306,7 +300,7 @@ pub async fn test_kms_key_management(
info!("Successfully described key: {}", key_id);
// Test ListKeys
let list_response = awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys", base_url), access_key, secret_key).await?;
let list_response = awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys"), access_key, secret_key).await?;
let list_result: serde_json::Value = serde_json::from_str(&list_response)?;
let keys = list_result["keys"]
@@ -412,7 +406,7 @@ impl VaultTestEnvironment {
let port_check = TcpStream::connect(VAULT_ADDRESS).await.is_ok();
if port_check {
// Additional check by making a health request
if let Ok(response) = reqwest::get(&format!("{}/v1/sys/health", VAULT_URL)).await {
if let Ok(response) = reqwest::get(&format!("{VAULT_URL}/v1/sys/health")).await {
if response.status().is_success() {
info!("Vault server is ready after {} seconds", i);
return Ok(());
@@ -438,7 +432,7 @@ impl VaultTestEnvironment {
// Enable transit secrets engine
let enable_response = client
.post(format!("{}/v1/sys/mounts/{}", VAULT_URL, VAULT_TRANSIT_PATH))
.post(format!("{VAULT_URL}/v1/sys/mounts/{VAULT_TRANSIT_PATH}"))
.header("X-Vault-Token", VAULT_TOKEN)
.json(&serde_json::json!({
"type": "transit"
@@ -448,14 +442,14 @@ impl VaultTestEnvironment {
if !enable_response.status().is_success() && enable_response.status() != 400 {
let error_text = enable_response.text().await?;
return Err(format!("Failed to enable transit engine: {}", error_text).into());
return Err(format!("Failed to enable transit engine: {error_text}").into());
}
info!("Creating Vault encryption key");
// Create encryption key
let key_response = client
.post(format!("{}/v1/{}/keys/{}", VAULT_URL, VAULT_TRANSIT_PATH, VAULT_KEY_NAME))
.post(format!("{VAULT_URL}/v1/{VAULT_TRANSIT_PATH}/keys/{VAULT_KEY_NAME}"))
.header("X-Vault-Token", VAULT_TOKEN)
.json(&serde_json::json!({
"type": "aes256-gcm96"
@@ -465,7 +459,7 @@ impl VaultTestEnvironment {
if !key_response.status().is_success() && key_response.status() != 400 {
let error_text = key_response.text().await?;
return Err(format!("Failed to create encryption key: {}", error_text).into());
return Err(format!("Failed to create encryption key: {error_text}").into());
}
info!("Vault transit engine setup completed");
@@ -713,10 +707,10 @@ pub async fn test_all_multipart_encryption_types(
// Test configurations for all encryption types
let test_configs = vec![
MultipartTestConfig::new(format!("{}-no-encryption", base_object_key), part_size, total_parts, EncryptionType::None),
MultipartTestConfig::new(format!("{}-sse-s3", base_object_key), part_size, total_parts, EncryptionType::SSES3),
MultipartTestConfig::new(format!("{}-sse-kms", base_object_key), part_size, total_parts, EncryptionType::SSEKMS),
MultipartTestConfig::new(format!("{}-sse-c", base_object_key), part_size, total_parts, create_sse_c_config()),
MultipartTestConfig::new(format!("{base_object_key}-no-encryption"), part_size, total_parts, EncryptionType::None),
MultipartTestConfig::new(format!("{base_object_key}-sse-s3"), part_size, total_parts, EncryptionType::SSES3),
MultipartTestConfig::new(format!("{base_object_key}-sse-kms"), part_size, total_parts, EncryptionType::SSEKMS),
MultipartTestConfig::new(format!("{base_object_key}-sse-c"), part_size, total_parts, create_sse_c_config()),
];
// Run tests for each encryption type
crates/e2e_test/src/kms/encryption_metadata_test.rs
+4 -8
View File
@@ -33,11 +33,10 @@ fn assert_encryption_metadata(metadata: &HashMap<String, String>, expected_size:
"x-rustfs-encryption-context",
"x-rustfs-encryption-original-size",
] {
assert!(metadata.contains_key(key), "expected managed encryption metadata '{}' to be present", key);
assert!(metadata.contains_key(key), "expected managed encryption metadata '{key}' to be present");
assert!(
!metadata.get(key).unwrap().is_empty(),
"managed encryption metadata '{}' should not be empty",
key
"managed encryption metadata '{key}' should not be empty"
);
}
@@ -84,10 +83,7 @@ fn assert_storage_encrypted(storage_root: &std::path::Path, bucket: &str, key: &
assert!(
scanned > 0,
"Failed to locate stored data files for bucket '{}' and key '{}' under {:?}",
bucket,
key,
storage_root
"Failed to locate stored data files for bucket '{bucket}' and key '{key}' under {storage_root:?}"
);
assert!(plaintext_path.is_none(), "Plaintext detected on disk at {:?}", plaintext_path.unwrap());
}
@@ -220,7 +216,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
assert_encryption_metadata(source_metadata, payload.len());
let dest_key = "metadata-sse-kms-object-copy";
let copy_source = format!("{}/{}", TEST_BUCKET, source_key);
let copy_source = format!("{TEST_BUCKET}/{source_key}");
s3_client
.copy_object()
crates/e2e_test/src/kms/kms_edge_cases_test.rs
+4 -6
View File
@@ -389,8 +389,8 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
let task = tokio::spawn(async move {
let _permit = sem.acquire().await.unwrap();
let test_data = format!("Concurrent test data {}", i).into_bytes();
let object_key = format!("concurrent-test-{}", i);
let test_data = format!("Concurrent test data {i}").into_bytes();
let object_key = format!("concurrent-test-{i}");
// Alternate between different encryption types
let result = match i % 3 {
@@ -418,7 +418,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
}
2 => {
// SSE-C
let key = format!("testkey{:026}", i); // 32-byte key
let key = format!("testkey{i:026}"); // 32-byte key
let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key);
let key_md5 = format!("{:x}", md5::compute(&key));
@@ -459,9 +459,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
assert!(
successful_uploads >= num_concurrent - 1,
"Most concurrent uploads should succeed (got {}/{})",
successful_uploads,
num_concurrent
"Most concurrent uploads should succeed (got {successful_uploads}/{num_concurrent})"
);
info!("✅ Successfully completed {}/{} concurrent uploads", successful_uploads, num_concurrent);
crates/e2e_test/src/kms/kms_fault_recovery_test.rs
+3 -3
View File
@@ -152,7 +152,7 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box<dyn std::error::Error
// Corrupt the default key file
info!("🔧 Corrupting default key file");
let key_file_path = format!("{}/{}.key", kms_env.kms_keys_dir, default_key_id);
let backup_key_path = format!("{}.backup", key_file_path);
let backup_key_path = format!("{key_file_path}.backup");
// Backup the original key file
fs::copy(&key_file_path, &backup_key_path)?;
@@ -417,8 +417,8 @@ async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error
for i in 0..10 {
let client = s3_client.clone();
let test_data = format!("Rapid test data {}", i).into_bytes();
let object_key = format!("rapid-test-{}", i);
let test_data = format!("Rapid test data {i}").into_bytes();
let object_key = format!("rapid-test-{i}");
let task = tokio::spawn(async move {
let result = client
crates/e2e_test/src/kms/kms_local_test.rs
+1 -1
View File
@@ -740,7 +740,7 @@ async fn test_large_multipart_upload(
// Verify data integrity
for (i, (&actual, &expected)) in downloaded_data.iter().zip(test_data.iter()).enumerate() {
if actual != expected {
panic!("Data mismatch at byte {}: got {}, expected {}", i, actual, expected);
panic!("Data mismatch at byte {i}: got {actual}, expected {expected}");
}
}
crates/e2e_test/src/kms/kms_vault_test.rs
+9 -13
View File
@@ -321,13 +321,9 @@ async fn test_vault_kms_key_crud(
})
.to_string();
let create_response = crate::common::awscurl_post(
&format!("{}/rustfs/admin/v3/kms/keys", base_url),
&create_key_body,
access_key,
secret_key,
)
.await?;
let create_response =
crate::common::awscurl_post(&format!("{base_url}/rustfs/admin/v3/kms/keys"), &create_key_body, access_key, secret_key)
.await?;
let create_result: serde_json::Value = serde_json::from_str(&create_response)?;
let key_id = create_result["key_id"]
@@ -337,7 +333,7 @@ async fn test_vault_kms_key_crud(
// Read
let describe_response =
crate::common::awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys/{}", base_url, key_id), access_key, secret_key).await?;
crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?;
let describe_result: serde_json::Value = serde_json::from_str(&describe_response)?;
assert_eq!(describe_result["key_metadata"]["key_id"], key_id);
@@ -380,7 +376,7 @@ async fn test_vault_kms_key_crud(
// Read
let list_response =
crate::common::awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys", base_url), access_key, secret_key).await?;
crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys"), access_key, secret_key).await?;
let list_result: serde_json::Value = serde_json::from_str(&list_response)?;
let keys = list_result["keys"]
@@ -407,7 +403,7 @@ async fn test_vault_kms_key_crud(
// Delete
let delete_response = crate::common::execute_awscurl(
&format!("{}/rustfs/admin/v3/kms/keys/delete?keyId={}", base_url, key_id),
&format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}"),
"DELETE",
None,
access_key,
@@ -422,7 +418,7 @@ async fn test_vault_kms_key_crud(
// Verify key state after deletion
let describe_deleted_response =
crate::common::awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys/{}", base_url, key_id), access_key, secret_key).await?;
crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?;
let describe_result: serde_json::Value = serde_json::from_str(&describe_deleted_response)?;
let key_state = describe_result["key_metadata"]["key_state"]
@@ -439,7 +435,7 @@ async fn test_vault_kms_key_crud(
// Force Delete - Force immediate deletion for PendingDeletion key
let force_delete_response = crate::common::execute_awscurl(
&format!("{}/rustfs/admin/v3/kms/keys/delete?keyId={}&force_immediate=true", base_url, key_id),
&format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&force_immediate=true"),
"DELETE",
None,
access_key,
@@ -454,7 +450,7 @@ async fn test_vault_kms_key_crud(
// Verify key no longer exists after force deletion (should return error)
let describe_force_deleted_result =
crate::common::awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys/{}", base_url, key_id), access_key, secret_key).await;
crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await;
// After force deletion, key should not be found (GET should fail)
assert!(describe_force_deleted_result.is_err(), "Force deleted key should not be found");
crates/e2e_test/src/kms/multipart_encryption_test.rs
+1 -1
View File
@@ -419,7 +419,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
// 逐字节验证数据（对于大文件更严格）
for (i, (&actual, &expected)) in downloaded_data.iter().zip(test_data.iter()).enumerate() {
if actual != expected {
panic!("大文件数据在第{}字节不匹配: 实际={}, 期待={}", i, actual, expected);
panic!("大文件数据在第{i}字节不匹配: 实际={actual}, 期待={expected}");
}
}
crates/e2e_test/src/kms/test_runner.rs
+2 -2
View File
@@ -476,7 +476,7 @@ async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Sen
let failed_count = results.iter().filter(|r| !r.success).count();
if failed_count > 0 {
return Err(format!("Critical test suite failed: {} tests failed", failed_count).into());
return Err(format!("Critical test suite failed: {failed_count} tests failed").into());
}
info!("✅ 所有关键测试通过");
@@ -498,7 +498,7 @@ async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send +
// Allow up to 10% failure rate for non-critical tests
if success_rate < 90.0 {
return Err(format!("Test suite success rate too low: {:.1}%", success_rate).into());
return Err(format!("Test suite success rate too low: {success_rate:.1}%").into());
}
info!("✅ 完整测试套件通过");