Refactor: Introduce content checksums and improve multipart/object metadata handling (#671)

* feat:  adapt to s3s typed etag support

* refactor: move replication struct to rustfs_filemeta, fix filemeta transition bug

* add head_object checksum, filter object metadata output

* fix multipart checksum

* fix multipart checksum

* add content md5,sha256 check

* fix test

* fix cargo

---------

Co-authored-by: overtrue <anzhengchao@gmail.com>

crates/audit/tests/config_parsing_test.rs

@@ -81,8 +81,8 @@ fn test_config_section_names() {
 fn test_environment_variable_parsing() {
     // Test environment variable prefix patterns
     let env_prefix = "RUSTFS_";
-    let audit_webhook_prefix = format!("{}AUDIT_WEBHOOK_", env_prefix);
-    let audit_mqtt_prefix = format!("{}AUDIT_MQTT_", env_prefix);
+    let audit_webhook_prefix = format!("{env_prefix}AUDIT_WEBHOOK_");
+    let audit_mqtt_prefix = format!("{env_prefix}AUDIT_MQTT_");
 
     assert_eq!(audit_webhook_prefix, "RUSTFS_AUDIT_WEBHOOK_");
     assert_eq!(audit_mqtt_prefix, "RUSTFS_AUDIT_MQTT_");
@@ -141,13 +141,13 @@ fn test_duration_parsing_formats() {
         let result = parse_duration_test(input);
         match (result, expected_seconds) {
             (Some(duration), Some(expected)) => {
-                assert_eq!(duration.as_secs(), expected, "Failed for input: {}", input);
+                assert_eq!(duration.as_secs(), expected, "Failed for input: {input}");
             }
             (None, None) => {
                 // Both None, test passes
             }
             _ => {
-                panic!("Mismatch for input: {}, got: {:?}, expected: {:?}", input, result, expected_seconds);
+                panic!("Mismatch for input: {input}, got: {result:?}, expected: {expected_seconds:?}");
             }
         }
     }
@@ -188,13 +188,13 @@ fn test_url_validation() {
 
     for url_str in valid_urls {
         let result = Url::parse(url_str);
-        assert!(result.is_ok(), "Valid URL should parse: {}", url_str);
+        assert!(result.is_ok(), "Valid URL should parse: {url_str}");
     }
 
     for url_str in &invalid_urls[..3] {
         // Skip the ftp one as it's technically valid
         let result = Url::parse(url_str);
-        assert!(result.is_err(), "Invalid URL should not parse: {}", url_str);
+        assert!(result.is_err(), "Invalid URL should not parse: {url_str}");
     }
 }
 
@@ -214,6 +214,6 @@ fn test_qos_parsing() {
             0..=2 => Some(q),
             _ => None,
         });
-        assert_eq!(result, expected, "Failed for QoS input: {}", input);
+        assert_eq!(result, expected, "Failed for QoS input: {input}");
     }
 }

crates/audit/tests/integration_test.rs

@@ -57,7 +57,7 @@ async fn test_config_parsing_webhook() {
         }
         Err(e) => {
             // Other errors might indicate parsing issues
-            println!("Unexpected error: {}", e);
+            println!("Unexpected error: {e}");
         }
         Ok(_) => {
             // Unexpected success in test environment without server storage
@@ -103,6 +103,6 @@ fn test_enable_value_parsing() {
 
     for (input, expected) in test_cases {
         let result = matches!(input.to_lowercase().as_str(), "1" | "on" | "true" | "yes");
-        assert_eq!(result, expected, "Failed for input: {}", input);
+        assert_eq!(result, expected, "Failed for input: {input}");
     }
 }

crates/audit/tests/performance_test.rs

@@ -32,10 +32,10 @@ async fn test_audit_system_startup_performance() {
     let _result = timeout(Duration::from_secs(5), system.start(config)).await;
     let elapsed = start.elapsed();
 
-    println!("Audit system startup took: {:?}", elapsed);
+    println!("Audit system startup took: {elapsed:?}");
 
     // Should complete within 5 seconds
-    assert!(elapsed < Duration::from_secs(5), "Startup took too long: {:?}", elapsed);
+    assert!(elapsed < Duration::from_secs(5), "Startup took too long: {elapsed:?}");
 
     // Clean up
     let _ = system.close().await;
@@ -54,8 +54,8 @@ async fn test_concurrent_target_creation() {
     for i in 1..=5 {
         let mut kvs = rustfs_ecstore::config::KVS::new();
         kvs.insert("enable".to_string(), "on".to_string());
-        kvs.insert("endpoint".to_string(), format!("http://localhost:302{}/webhook", i));
-        webhook_section.insert(format!("instance_{}", i), kvs);
+        kvs.insert("endpoint".to_string(), format!("http://localhost:302{i}/webhook"));
+        webhook_section.insert(format!("instance_{i}"), kvs);
     }
 
     config.0.insert("audit_webhook".to_string(), webhook_section);
@@ -66,10 +66,10 @@ async fn test_concurrent_target_creation() {
     let result = registry.create_targets_from_config(&config).await;
     let elapsed = start.elapsed();
 
-    println!("Concurrent target creation took: {:?}", elapsed);
+    println!("Concurrent target creation took: {elapsed:?}");
 
     // Should complete quickly even with multiple targets
-    assert!(elapsed < Duration::from_secs(10), "Target creation took too long: {:?}", elapsed);
+    assert!(elapsed < Duration::from_secs(10), "Target creation took too long: {elapsed:?}");
 
     // Verify it fails with expected error (server not initialized)
     match result {
@@ -77,7 +77,7 @@ async fn test_concurrent_target_creation() {
             // Expected in test environment
         }
         Err(e) => {
-            println!("Unexpected error during concurrent creation: {}", e);
+            println!("Unexpected error during concurrent creation: {e}");
         }
         Ok(_) => {
             println!("Unexpected success in test environment");
@@ -93,7 +93,7 @@ async fn test_audit_log_dispatch_performance() {
     let config = rustfs_ecstore::config::Config(HashMap::new());
     let start_result = system.start(config).await;
     if start_result.is_err() {
-        println!("AuditSystem failed to start: {:?}", start_result);
+        println!("AuditSystem failed to start: {start_result:?}");
         return; // 或 assert!(false, "AuditSystem failed to start");
     }
 
@@ -104,14 +104,14 @@ async fn test_audit_log_dispatch_performance() {
     let id = 1;
 
     let mut req_header = HashMap::new();
-    req_header.insert("authorization".to_string(), format!("Bearer test-token-{}", id));
+    req_header.insert("authorization".to_string(), format!("Bearer test-token-{id}"));
     req_header.insert("content-type".to_string(), "application/octet-stream".to_string());
 
     let mut resp_header = HashMap::new();
     resp_header.insert("x-response".to_string(), "ok".to_string());
 
     let mut tags = HashMap::new();
-    tags.insert(format!("tag-{}", id), json!("sample"));
+    tags.insert(format!("tag-{id}"), json!("sample"));
 
     let mut req_query = HashMap::new();
     req_query.insert("id".to_string(), id.to_string());
@@ -119,7 +119,7 @@ async fn test_audit_log_dispatch_performance() {
     let api_details = ApiDetails {
         name: Some("PutObject".to_string()),
         bucket: Some("test-bucket".to_string()),
-        object: Some(format!("test-object-{}", id)),
+        object: Some(format!("test-object-{id}")),
         status: Some("success".to_string()),
         status_code: Some(200),
         input_bytes: Some(1024),
@@ -134,7 +134,7 @@ async fn test_audit_log_dispatch_performance() {
     // Create sample audit log entry
     let audit_entry = AuditEntry {
         version: "1".to_string(),
-        deployment_id: Some(format!("test-deployment-{}", id)),
+        deployment_id: Some(format!("test-deployment-{id}")),
         site_name: Some("test-site".to_string()),
         time: Utc::now(),
         event: EventName::ObjectCreatedPut,
@@ -142,9 +142,9 @@ async fn test_audit_log_dispatch_performance() {
         trigger: "api".to_string(),
         api: api_details,
         remote_host: Some("127.0.0.1".to_string()),
-        request_id: Some(format!("test-request-{}", id)),
+        request_id: Some(format!("test-request-{id}")),
         user_agent: Some("test-agent".to_string()),
-        req_path: Some(format!("/test-bucket/test-object-{}", id)),
+        req_path: Some(format!("/test-bucket/test-object-{id}")),
         req_host: Some("test-host".to_string()),
         req_node: Some("node-1".to_string()),
         req_claims: None,
@@ -152,8 +152,8 @@ async fn test_audit_log_dispatch_performance() {
         req_header: Some(req_header),
         resp_header: Some(resp_header),
         tags: Some(tags),
-        access_key: Some(format!("AKIA{}", id)),
-        parent_user: Some(format!("parent-{}", id)),
+        access_key: Some(format!("AKIA{id}")),
+        parent_user: Some(format!("parent-{id}")),
         error: None,
     };
 
@@ -163,10 +163,10 @@ async fn test_audit_log_dispatch_performance() {
     let result = system.dispatch(Arc::new(audit_entry)).await;
     let elapsed = start.elapsed();
 
-    println!("Audit log dispatch took: {:?}", elapsed);
+    println!("Audit log dispatch took: {elapsed:?}");
 
     // Should be very fast (sub-millisecond for no targets)
-    assert!(elapsed < Duration::from_millis(100), "Dispatch took too long: {:?}", elapsed);
+    assert!(elapsed < Duration::from_millis(100), "Dispatch took too long: {elapsed:?}");
 
     // Should succeed even with no targets
     assert!(result.is_ok(), "Dispatch should succeed with no targets");
@@ -226,10 +226,10 @@ fn test_event_name_mask_performance() {
     }
 
     let elapsed = start.elapsed();
-    println!("Event mask calculation (5000 ops) took: {:?}", elapsed);
+    println!("Event mask calculation (5000 ops) took: {elapsed:?}");
 
     // Should be very fast
-    assert!(elapsed < Duration::from_millis(100), "Mask calculation too slow: {:?}", elapsed);
+    assert!(elapsed < Duration::from_millis(100), "Mask calculation too slow: {elapsed:?}");
 }
 
 #[test]
@@ -254,10 +254,10 @@ fn test_event_name_expansion_performance() {
     }
 
     let elapsed = start.elapsed();
-    println!("Event expansion (4000 ops) took: {:?}", elapsed);
+    println!("Event expansion (4000 ops) took: {elapsed:?}");
 
     // Should be very fast
-    assert!(elapsed < Duration::from_millis(100), "Expansion too slow: {:?}", elapsed);
+    assert!(elapsed < Duration::from_millis(100), "Expansion too slow: {elapsed:?}");
 }
 
 #[tokio::test]
@@ -274,10 +274,10 @@ async fn test_registry_operations_performance() {
     }
 
     let elapsed = start.elapsed();
-    println!("Registry operations (2000 ops) took: {:?}", elapsed);
+    println!("Registry operations (2000 ops) took: {elapsed:?}");
 
     // Should be very fast for empty registry
-    assert!(elapsed < Duration::from_millis(100), "Registry ops too slow: {:?}", elapsed);
+    assert!(elapsed < Duration::from_millis(100), "Registry ops too slow: {elapsed:?}");
 }
 
 // Performance requirements validation
@@ -294,7 +294,7 @@ fn test_performance_requirements() {
     // Simulate processing 3000 events worth of operations
     for i in 0..3000 {
         // Simulate event name parsing and processing
-        let _event_id = format!("s3:ObjectCreated:Put_{}", i);
+        let _event_id = format!("s3:ObjectCreated:Put_{i}");
         let _timestamp = chrono::Utc::now().to_rfc3339();
 
         // Simulate basic audit entry creation overhead
@@ -305,16 +305,16 @@ fn test_performance_requirements() {
     let elapsed = start.elapsed();
     let eps = 3000.0 / elapsed.as_secs_f64();
 
-    println!("Simulated 3000 events in {:?} ({:.0} EPS)", elapsed, eps);
+    println!("Simulated 3000 events in {elapsed:?} ({eps:.0} EPS)");
 
     // Our core processing should easily handle 3k EPS worth of CPU overhead
     // The actual EPS limit will be determined by network I/O to targets
-    assert!(eps > 10000.0, "Core processing too slow for 3k EPS target: {} EPS", eps);
+    assert!(eps > 10000.0, "Core processing too slow for 3k EPS target: {eps} EPS");
 
     // P99 latency requirement: < 30ms
     // For core processing, we should be much faster than this
     let avg_latency = elapsed / 3000;
-    println!("Average processing latency: {:?}", avg_latency);
+    println!("Average processing latency: {avg_latency:?}");
 
-    assert!(avg_latency < Duration::from_millis(1), "Processing latency too high: {:?}", avg_latency);
+    assert!(avg_latency < Duration::from_millis(1), "Processing latency too high: {avg_latency:?}");
 }

crates/audit/tests/system_integration_test.rs

@@ -52,7 +52,7 @@ async fn test_complete_audit_system_lifecycle() {
             assert_eq!(system.get_state().await, system::AuditSystemState::Running);
         }
         Err(e) => {
-            panic!("Unexpected error: {}", e);
+            panic!("Unexpected error: {e}");
         }
     }
 
@@ -103,7 +103,7 @@ async fn test_audit_log_dispatch_with_no_targets() {
             // Also acceptable since system not running
         }
         Err(e) => {
-            panic!("Unexpected error: {}", e);
+            panic!("Unexpected error: {e}");
         }
     }
 }
@@ -172,7 +172,7 @@ async fn test_config_parsing_with_multiple_instances() {
             // Expected - parsing worked but save failed
         }
         Err(e) => {
-            println!("Config parsing error: {}", e);
+            println!("Config parsing error: {e}");
             // Other errors might indicate parsing issues, but not necessarily failures
         }
         Ok(_) => {
@@ -261,7 +261,7 @@ async fn test_concurrent_operations() {
         let (i, state, is_running) = task.await.expect("Task should complete");
         assert_eq!(state, system::AuditSystemState::Stopped);
         assert!(!is_running);
-        println!("Task {} completed successfully", i);
+        println!("Task {i} completed successfully");
     }
 }
 
@@ -295,8 +295,8 @@ async fn test_performance_under_load() {
     }
 
     let elapsed = start.elapsed();
-    println!("100 concurrent dispatches took: {:?}", elapsed);
-    println!("Successes: {}, Errors: {}", success_count, error_count);
+    println!("100 concurrent dispatches took: {elapsed:?}");
+    println!("Successes: {success_count}, Errors: {error_count}");
 
     // Should complete reasonably quickly
     assert!(elapsed < Duration::from_secs(5), "Concurrent operations took too long");
@@ -318,14 +318,14 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
     use std::collections::HashMap;
 
     let mut req_header = HashMap::new();
-    req_header.insert("authorization".to_string(), format!("Bearer test-token-{}", id));
+    req_header.insert("authorization".to_string(), format!("Bearer test-token-{id}"));
     req_header.insert("content-type".to_string(), "application/octet-stream".to_string());
 
     let mut resp_header = HashMap::new();
     resp_header.insert("x-response".to_string(), "ok".to_string());
 
     let mut tags = HashMap::new();
-    tags.insert(format!("tag-{}", id), json!("sample"));
+    tags.insert(format!("tag-{id}"), json!("sample"));
 
     let mut req_query = HashMap::new();
     req_query.insert("id".to_string(), id.to_string());
@@ -333,7 +333,7 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
     let api_details = ApiDetails {
         name: Some("PutObject".to_string()),
         bucket: Some("test-bucket".to_string()),
-        object: Some(format!("test-object-{}", id)),
+        object: Some(format!("test-object-{id}")),
         status: Some("success".to_string()),
         status_code: Some(200),
         input_bytes: Some(1024),
@@ -348,7 +348,7 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
 
     AuditEntry {
         version: "1".to_string(),
-        deployment_id: Some(format!("test-deployment-{}", id)),
+        deployment_id: Some(format!("test-deployment-{id}")),
         site_name: Some("test-site".to_string()),
         time: Utc::now(),
         event: EventName::ObjectCreatedPut,
@@ -356,9 +356,9 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
         trigger: "api".to_string(),
         api: api_details,
         remote_host: Some("127.0.0.1".to_string()),
-        request_id: Some(format!("test-request-{}", id)),
+        request_id: Some(format!("test-request-{id}")),
         user_agent: Some("test-agent".to_string()),
-        req_path: Some(format!("/test-bucket/test-object-{}", id)),
+        req_path: Some(format!("/test-bucket/test-object-{id}")),
         req_host: Some("test-host".to_string()),
         req_node: Some("node-1".to_string()),
         req_claims: None,
@@ -366,8 +366,8 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
         req_header: Some(req_header),
         resp_header: Some(resp_header),
         tags: Some(tags),
-        access_key: Some(format!("AKIA{}", id)),
-        parent_user: Some(format!("parent-{}", id)),
+        access_key: Some(format!("AKIA{id}")),
+        parent_user: Some(format!("parent-{id}")),
         error: None,
     }
 }