test(e2e_test): add automated cluster environment for conditional PUT race test (#1673)

Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: loverustfs <hello@rustfs.com>
2026-08-28 16:07:05 +00:00 · 2026-02-06 15:01:59 +08:00
parent 6eb90e7df9
commit a2b88a79ec
3 changed files with 517 additions and 0 deletions
@@ -0,0 +1,241 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::RustFSTestClusterEnvironment;
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::SdkError;
+use bytes::Bytes;
+use serial_test::serial;
+use std::sync::Arc;
+use tokio::sync::Barrier;
+use tracing::{info, warn};
+
+const BUCKET: &str = "conditional-put-race-bucket";
+
+async fn cleanup_object(client: &Client, key: &str) {
+    if let Err(e) = client.delete_object().bucket(BUCKET).key(key).send().await {
+        warn!("Failed to delete object '{}' from bucket '{}' during cleanup: {:?}", key, BUCKET, e);
+    }
+}
+
+async fn conditional_put(
+    client: &Client,
+    key: &str,
+    data: &[u8],
+    client_id: usize,
+) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
+    let result = client
+        .put_object()
+        .bucket(BUCKET)
+        .key(key)
+        .body(Bytes::copy_from_slice(data).into())
+        .if_none_match("*")
+        .send()
+        .await;
+
+    match result {
+        Ok(resp) => {
+            info!("  Client {} SUCCEEDED - ETag: {}", client_id, resp.e_tag().unwrap_or("none"));
+            Ok(true)
+        }
+        Err(SdkError::ServiceError(e)) => {
+            let code = e.err().meta().code().unwrap_or("");
+            let message = e.err().meta().message().unwrap_or("");
+            if code == "PreconditionFailed" {
+                warn!("  Client {} got 412 PreconditionFailed", client_id);
+                Ok(false)
+            } else {
+                warn!("  Client {} got ServiceError: code={}, message={}", client_id, code, message);
+                Err(e.into_err().into())
+            }
+        }
+        Err(e) => {
+            warn!("  Client {} got non-ServiceError: {:?}", client_id, e);
+            Err(e.into())
+        }
+    }
+}
+
+async fn run_race_iteration(
+    clients: &[Client],
+    test_key: &str,
+    iteration: usize,
+) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
+    cleanup_object(&clients[0], test_key).await;
+    tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
+
+    let head_result = clients[0].head_object().bucket(BUCKET).key(test_key).send().await;
+
+    if head_result.is_ok() {
+        warn!("Warning: Object still exists after cleanup, skipping iteration {}", iteration);
+        return Ok(0);
+    }
+
+    info!("\n=== Iteration {} ===", iteration);
+    info!("Launching {} concurrent conditional PUTs to different nodes...", clients.len());
+
+    let barrier = Arc::new(Barrier::new(clients.len()));
+    let test_key = test_key.to_string();
+
+    let mut handles = vec![];
+    for (i, client) in clients.iter().enumerate() {
+        let client = client.clone();
+        let barrier = barrier.clone();
+        let key = test_key.clone();
+        let data = format!("data from client {}", i).into_bytes();
+
+        let handle = tokio::spawn(async move {
+            barrier.wait().await;
+            conditional_put(&client, &key, &data, i).await
+        });
+        handles.push(handle);
+    }
+
+    let mut success_count = 0;
+    let mut had_error = false;
+    for handle in handles {
+        match handle.await {
+            Ok(Ok(true)) => success_count += 1,
+            Ok(Ok(false)) => {}
+            Ok(Err(e)) => {
+                had_error = true;
+                info!("  Error: {}", e);
+            }
+            Err(e) => {
+                had_error = true;
+                info!("  Task error: {}", e);
+            }
+        }
+    }
+
+    info!("Result: {} out of {} succeeded", success_count, clients.len());
+
+    if success_count > 1 {
+        info!(">>> RACE CONDITION DETECTED!");
+    } else if success_count == 1 {
+        info!(">>> Correct behavior: exactly 1 writer succeeded.");
+    } else if had_error {
+        return Err("all conditional PUTs failed (e.g. cluster/bucket not ready)".into());
+    } else {
+        info!(">>> Unexpected: no writers succeeded.");
+    }
+
+    Ok(success_count)
+}
+
+#[tokio::test]
+#[serial]
+async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    crate::common::init_logging();
+    info!("Starting conditional PUT race test with auto cluster");
+
+    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+    cluster.start().await?;
+
+    cluster.create_test_bucket(BUCKET).await?;
+
+    let clients = cluster.create_all_clients()?;
+
+    let iterations = 5;
+    let mut races_detected = 0;
+    let mut correct_count = 0;
+    let mut error_count = 0;
+
+    for i in 1..=iterations {
+        let test_key = format!("race-test-{}-{}", std::process::id(), i);
+
+        match run_race_iteration(&clients, &test_key, i).await {
+            Ok(success_count) => {
+                if success_count > 1 {
+                    races_detected += 1;
+                } else if success_count == 1 {
+                    correct_count += 1;
+                }
+            }
+            Err(e) => {
+                error_count += 1;
+                warn!("Iteration {} failed (not a race; e.g. cluster/network): {}", i, e)
+            }
+        }
+
+        cleanup_object(&clients[0], &test_key).await;
+        tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
+    }
+
+    info!("\n====================================");
+    info!("SUMMARY");
+    info!("====================================");
+    info!("Total iterations:   {}", iterations);
+    info!("Correct (1 winner): {}", correct_count);
+    info!("Race conditions:    {}", races_detected);
+    info!("Errors (skipped):   {}", error_count);
+
+    assert_eq!(races_detected, 0, "Race conditions detected: {}/{}", races_detected, iterations);
+    assert_eq!(
+        error_count, 0,
+        "{} iteration(s) failed due to errors (e.g. cluster not ready)",
+        error_count
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    crate::common::init_logging();
+    info!("Starting basic conditional PUT test with auto cluster");
+
+    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+    cluster.start().await?;
+
+    cluster.create_test_bucket(BUCKET).await?;
+
+    let client = cluster.create_s3_client(0)?;
+    let test_key = "basic-conditional-put";
+    cleanup_object(&client, test_key).await;
+
+    let result = client
+        .put_object()
+        .bucket(BUCKET)
+        .key(test_key)
+        .body(Bytes::from("first write").into())
+        .if_none_match("*")
+        .send()
+        .await;
+    assert!(result.is_ok(), "First PUT with If-None-Match:* should succeed");
+
+    let result = client
+        .put_object()
+        .bucket(BUCKET)
+        .key(test_key)
+        .body(Bytes::from("second write").into())
+        .if_none_match("*")
+        .send()
+        .await;
+    assert!(result.is_err(), "Second PUT with If-None-Match:* should fail");
+
+    assert!(
+        matches!(result, Err(SdkError::ServiceError(_))),
+        "Expected ServiceError but got different error type"
+    );
+
+    if let Err(SdkError::ServiceError(e)) = result {
+        let code = e.err().meta().code().unwrap_or("");
+        assert_eq!(code, "PreconditionFailed");
+    }
+
+    cleanup_object(&client, test_key).await;
+    Ok(())
+}