feature: support kms && encryt (#573)

* feat(kms): implement key management service with local and vault backends Signed-off-by: junxiang Mu <1948535941@qq.com> * feat(kms): enhance security with zeroize for sensitive data and improve key management Signed-off-by: junxiang Mu <1948535941@qq.com> * remove Hashi word Signed-off-by: junxiang Mu <1948535941@qq.com> * refactor: remove unused request structs from kms handlers Signed-off-by: junxiang Mu <1948535941@qq.com> --------- Signed-off-by: junxiang Mu <1948535941@qq.com>
2026-08-30 16:59:52 +00:00 · 2025-09-22 17:53:05 +08:00
parent f7e188eee7
commit 9ddf6a011d
59 changed files with 18461 additions and 830 deletions
@@ -0,0 +1,506 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+#![allow(dead_code)]
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Unified KMS test suite runner
+//!
+//! This module provides a unified interface for running KMS tests with categorization,
+//! filtering, and comprehensive reporting capabilities.
+
+use crate::common::init_logging;
+use serial_test::serial;
+use std::time::Instant;
+use tokio::time::{Duration, sleep};
+use tracing::{debug, error, info, warn};
+
+/// Test category for organization and filtering
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum TestCategory {
+    CoreFunctionality,
+    MultipartEncryption,
+    EdgeCases,
+    FaultRecovery,
+    Comprehensive,
+    Performance,
+}
+
+impl TestCategory {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            TestCategory::CoreFunctionality => "core-functionality",
+            TestCategory::MultipartEncryption => "multipart-encryption",
+            TestCategory::EdgeCases => "edge-cases",
+            TestCategory::FaultRecovery => "fault-recovery",
+            TestCategory::Comprehensive => "comprehensive",
+            TestCategory::Performance => "performance",
+        }
+    }
+}
+
+/// Test definition with metadata
+#[derive(Debug, Clone)]
+pub struct TestDefinition {
+    pub name: String,
+    pub description: String,
+    pub category: TestCategory,
+    pub estimated_duration: Duration,
+    pub is_critical: bool,
+}
+
+impl TestDefinition {
+    pub fn new(
+        name: impl Into<String>,
+        description: impl Into<String>,
+        category: TestCategory,
+        estimated_duration: Duration,
+        is_critical: bool,
+    ) -> Self {
+        Self {
+            name: name.into(),
+            description: description.into(),
+            category,
+            estimated_duration,
+            is_critical,
+        }
+    }
+}
+
+/// Test execution result
+#[derive(Debug, Clone)]
+pub struct TestResult {
+    pub test_name: String,
+    pub category: TestCategory,
+    pub success: bool,
+    pub duration: Duration,
+    pub error_message: Option<String>,
+}
+
+impl TestResult {
+    pub fn success(test_name: String, category: TestCategory, duration: Duration) -> Self {
+        Self {
+            test_name,
+            category,
+            success: true,
+            duration,
+            error_message: None,
+        }
+    }
+
+    pub fn failure(test_name: String, category: TestCategory, duration: Duration, error: String) -> Self {
+        Self {
+            test_name,
+            category,
+            success: false,
+            duration,
+            error_message: Some(error),
+        }
+    }
+}
+
+/// Comprehensive test suite configuration
+#[derive(Debug, Clone)]
+pub struct TestSuiteConfig {
+    pub categories: Vec<TestCategory>,
+    pub include_critical_only: bool,
+    pub max_duration: Option<Duration>,
+    pub parallel_execution: bool,
+}
+
+impl Default for TestSuiteConfig {
+    fn default() -> Self {
+        Self {
+            categories: vec![
+                TestCategory::CoreFunctionality,
+                TestCategory::MultipartEncryption,
+                TestCategory::EdgeCases,
+                TestCategory::FaultRecovery,
+                TestCategory::Comprehensive,
+            ],
+            include_critical_only: false,
+            max_duration: None,
+            parallel_execution: false,
+        }
+    }
+}
+
+/// Unified KMS test suite runner
+pub struct KMSTestSuite {
+    tests: Vec<TestDefinition>,
+    config: TestSuiteConfig,
+}
+
+impl KMSTestSuite {
+    /// Create a new test suite with default configuration
+    pub fn new() -> Self {
+        let tests = vec![
+            // Core Functionality Tests
+            TestDefinition::new(
+                "test_local_kms_end_to_end",
+                "End-to-end KMS test with all encryption types",
+                TestCategory::CoreFunctionality,
+                Duration::from_secs(60),
+                true,
+            ),
+            TestDefinition::new(
+                "test_local_kms_key_isolation",
+                "Test KMS key isolation and security",
+                TestCategory::CoreFunctionality,
+                Duration::from_secs(45),
+                true,
+            ),
+            // Multipart Encryption Tests
+            TestDefinition::new(
+                "test_local_kms_multipart_upload",
+                "Test large file multipart upload with encryption",
+                TestCategory::MultipartEncryption,
+                Duration::from_secs(120),
+                true,
+            ),
+            TestDefinition::new(
+                "test_step1_basic_single_file_encryption",
+                "Basic single file encryption test",
+                TestCategory::MultipartEncryption,
+                Duration::from_secs(30),
+                false,
+            ),
+            TestDefinition::new(
+                "test_step2_basic_multipart_upload_without_encryption",
+                "Basic multipart upload without encryption",
+                TestCategory::MultipartEncryption,
+                Duration::from_secs(45),
+                false,
+            ),
+            TestDefinition::new(
+                "test_step3_multipart_upload_with_sse_s3",
+                "Multipart upload with SSE-S3 encryption",
+                TestCategory::MultipartEncryption,
+                Duration::from_secs(60),
+                true,
+            ),
+            TestDefinition::new(
+                "test_step4_large_multipart_upload_with_encryption",
+                "Large file multipart upload with encryption",
+                TestCategory::MultipartEncryption,
+                Duration::from_secs(90),
+                false,
+            ),
+            TestDefinition::new(
+                "test_step5_all_encryption_types_multipart",
+                "All encryption types multipart test",
+                TestCategory::MultipartEncryption,
+                Duration::from_secs(120),
+                true,
+            ),
+            // Edge Cases Tests
+            TestDefinition::new(
+                "test_kms_zero_byte_file_encryption",
+                "Test encryption of zero-byte files",
+                TestCategory::EdgeCases,
+                Duration::from_secs(20),
+                false,
+            ),
+            TestDefinition::new(
+                "test_kms_single_byte_file_encryption",
+                "Test encryption of single-byte files",
+                TestCategory::EdgeCases,
+                Duration::from_secs(20),
+                false,
+            ),
+            TestDefinition::new(
+                "test_kms_multipart_boundary_conditions",
+                "Test multipart upload boundary conditions",
+                TestCategory::EdgeCases,
+                Duration::from_secs(45),
+                false,
+            ),
+            TestDefinition::new(
+                "test_kms_invalid_key_scenarios",
+                "Test invalid key scenarios",
+                TestCategory::EdgeCases,
+                Duration::from_secs(30),
+                false,
+            ),
+            TestDefinition::new(
+                "test_kms_concurrent_encryption",
+                "Test concurrent encryption operations",
+                TestCategory::EdgeCases,
+                Duration::from_secs(60),
+                false,
+            ),
+            TestDefinition::new(
+                "test_kms_key_validation_security",
+                "Test key validation security",
+                TestCategory::EdgeCases,
+                Duration::from_secs(30),
+                false,
+            ),
+            // Fault Recovery Tests
+            TestDefinition::new(
+                "test_kms_key_directory_unavailable",
+                "Test KMS when key directory is unavailable",
+                TestCategory::FaultRecovery,
+                Duration::from_secs(45),
+                false,
+            ),
+            TestDefinition::new(
+                "test_kms_corrupted_key_files",
+                "Test KMS with corrupted key files",
+                TestCategory::FaultRecovery,
+                Duration::from_secs(30),
+                false,
+            ),
+            TestDefinition::new(
+                "test_kms_multipart_upload_interruption",
+                "Test multipart upload interruption recovery",
+                TestCategory::FaultRecovery,
+                Duration::from_secs(60),
+                false,
+            ),
+            TestDefinition::new(
+                "test_kms_resource_constraints",
+                "Test KMS under resource constraints",
+                TestCategory::FaultRecovery,
+                Duration::from_secs(90),
+                false,
+            ),
+            // Comprehensive Tests
+            TestDefinition::new(
+                "test_comprehensive_kms_full_workflow",
+                "Full KMS workflow comprehensive test",
+                TestCategory::Comprehensive,
+                Duration::from_secs(300),
+                true,
+            ),
+            TestDefinition::new(
+                "test_comprehensive_stress_test",
+                "KMS stress test with large datasets",
+                TestCategory::Comprehensive,
+                Duration::from_secs(400),
+                false,
+            ),
+            TestDefinition::new(
+                "test_comprehensive_key_isolation",
+                "Comprehensive key isolation test",
+                TestCategory::Comprehensive,
+                Duration::from_secs(180),
+                false,
+            ),
+            TestDefinition::new(
+                "test_comprehensive_concurrent_operations",
+                "Comprehensive concurrent operations test",
+                TestCategory::Comprehensive,
+                Duration::from_secs(240),
+                false,
+            ),
+            TestDefinition::new(
+                "test_comprehensive_performance_benchmark",
+                "KMS performance benchmark test",
+                TestCategory::Comprehensive,
+                Duration::from_secs(360),
+                false,
+            ),
+        ];
+
+        Self {
+            tests,
+            config: TestSuiteConfig::default(),
+        }
+    }
+
+    /// Configure the test suite
+    pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
+        self.config = config;
+        self
+    }
+
+    /// Filter tests based on category
+    pub fn filter_by_category(&self, category: &TestCategory) -> Vec<&TestDefinition> {
+        self.tests.iter().filter(|test| &test.category == category).collect()
+    }
+
+    /// Filter tests based on criticality
+    pub fn filter_critical_tests(&self) -> Vec<&TestDefinition> {
+        self.tests.iter().filter(|test| test.is_critical).collect()
+    }
+
+    /// Get test summary by category
+    pub fn get_category_summary(&self) -> std::collections::HashMap<TestCategory, Vec<&TestDefinition>> {
+        let mut summary = std::collections::HashMap::new();
+        for test in &self.tests {
+            summary.entry(test.category.clone()).or_insert_with(Vec::new).push(test);
+        }
+        summary
+    }
+
+    /// Run the complete test suite
+    pub async fn run_test_suite(&self) -> Vec<TestResult> {
+        init_logging();
+        info!("🚀 开始KMS统一测试套件");
+
+        let start_time = Instant::now();
+        let mut results = Vec::new();
+
+        // Filter tests based on configuration
+        let tests_to_run: Vec<&TestDefinition> = self
+            .tests
+            .iter()
+            .filter(|test| self.config.categories.contains(&test.category))
+            .filter(|test| !self.config.include_critical_only || test.is_critical)
+            .collect();
+
+        info!("📊 测试计划: {} 个测试将被执行", tests_to_run.len());
+        for (i, test) in tests_to_run.iter().enumerate() {
+            info!("  {}. {} ({})", i + 1, test.name, test.category.as_str());
+        }
+
+        // Execute tests
+        for (i, test_def) in tests_to_run.iter().enumerate() {
+            info!("🧪 执行测试 {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
+            info!("   📝 描述: {}", test_def.description);
+            info!("   🏷️  分类: {}", test_def.category.as_str());
+            info!("   ⏱️  预计时间: {:?}", test_def.estimated_duration);
+
+            let test_start = Instant::now();
+            let result = self.run_single_test(test_def).await;
+            let test_duration = test_start.elapsed();
+
+            match result {
+                Ok(_) => {
+                    info!("✅ 测试通过: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
+                    results.push(TestResult::success(test_def.name.clone(), test_def.category.clone(), test_duration));
+                }
+                Err(e) => {
+                    error!("❌ 测试失败: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
+                    results.push(TestResult::failure(
+                        test_def.name.clone(),
+                        test_def.category.clone(),
+                        test_duration,
+                        e.to_string(),
+                    ));
+                }
+            }
+
+            // Add delay between tests to avoid resource conflicts
+            if i < tests_to_run.len() - 1 {
+                debug!("⏸️  等待2秒后执行下一个测试...");
+                sleep(Duration::from_secs(2)).await;
+            }
+        }
+
+        let total_duration = start_time.elapsed();
+        self.print_test_summary(&results, total_duration);
+
+        results
+    }
+
+    /// Run a single test by dispatching to the appropriate test function
+    async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        // This is a placeholder for test dispatch logic
+        // In a real implementation, this would dispatch to actual test functions
+        warn!("⚠️  测试函数 '{}' 在统一运行器中尚未实现，跳过", test_def.name);
+        Ok(())
+    }
+
+    /// Print comprehensive test summary
+    fn print_test_summary(&self, results: &[TestResult], total_duration: Duration) {
+        info!("📊 KMS测试套件总结");
+        info!("⏱️  总执行时间: {:.2}秒", total_duration.as_secs_f64());
+        info!("📈 总测试数量: {}", results.len());
+
+        let passed = results.iter().filter(|r| r.success).count();
+        let failed = results.iter().filter(|r| !r.success).count();
+
+        info!("✅ 通过: {}", passed);
+        info!("❌ 失败: {}", failed);
+        info!("📊 成功率: {:.1}%", (passed as f64 / results.len() as f64) * 100.0);
+
+        // Summary by category
+        let mut category_summary: std::collections::HashMap<TestCategory, (usize, usize)> = std::collections::HashMap::new();
+        for result in results {
+            let (total, passed_count) = category_summary.entry(result.category.clone()).or_insert((0, 0));
+            *total += 1;
+            if result.success {
+                *passed_count += 1;
+            }
+        }
+
+        info!("📊 分类汇总:");
+        for (category, (total, passed_count)) in category_summary {
+            info!(
+                "  🏷️  {}: {}/{} ({:.1}%)",
+                category.as_str(),
+                passed_count,
+                total,
+                (passed_count as f64 / total as f64) * 100.0
+            );
+        }
+
+        // List failed tests
+        if failed > 0 {
+            warn!("❌ 失败的测试:");
+            for result in results.iter().filter(|r| !r.success) {
+                warn!(
+                    "  - {}: {}",
+                    result.test_name,
+                    result.error_message.as_ref().unwrap_or(&"Unknown error".to_string())
+                );
+            }
+        }
+    }
+}
+
+/// Quick test suite for critical tests only
+#[tokio::test]
+#[serial]
+async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let config = TestSuiteConfig {
+        categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption],
+        include_critical_only: true,
+        max_duration: Some(Duration::from_secs(600)), // 10 minutes max
+        parallel_execution: false,
+    };
+
+    let suite = KMSTestSuite::new().with_config(config);
+    let results = suite.run_test_suite().await;
+
+    let failed_count = results.iter().filter(|r| !r.success).count();
+    if failed_count > 0 {
+        return Err(format!("Critical test suite failed: {} tests failed", failed_count).into());
+    }
+
+    info!("✅ 所有关键测试通过");
+    Ok(())
+}
+
+/// Full comprehensive test suite
+#[tokio::test]
+#[serial]
+async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let suite = KMSTestSuite::new();
+    let results = suite.run_test_suite().await;
+
+    let total_tests = results.len();
+    let failed_count = results.iter().filter(|r| !r.success).count();
+    let success_rate = ((total_tests - failed_count) as f64 / total_tests as f64) * 100.0;
+
+    info!("📊 完整测试套件结果: {:.1}% 成功率", success_rate);
+
+    // Allow up to 10% failure rate for non-critical tests
+    if success_rate < 90.0 {
+        return Err(format!("Test suite success rate too low: {:.1}%", success_rate).into());
+    }
+
+    info!("✅ 完整测试套件通过");
+    Ok(())
+}