test(e2e): add P0/P1 regression tests for recurring issue patterns (#5709)

Add 21 E2E regression tests across 7 new test files covering the most frequently regressing issue patterns identified from 5600+ issues in rustfs/rustfs. Each test references specific regression issue numbers and validates the exact failure path that caused the regression. Regression categories covered: - P0: Event notification startup race (rustfs#5387, #5681, #5401) - P0: Lifecycle/ILM rule persistence (rustfs#5407, #5167, #4963) - P0: Delete consistency (rustfs#5375, #4978, #760) - P1: Listing completeness (rustfs#4810, #5051, #3191) - P1: Bucket statistics accuracy (rustfs#5615, #3898, #1012) - P1: Distributed startup quorum (rustfs#5655, #2945) - P1: Tier/scanner persistence (rustfs#5218, #5013) Ref: https://github.com/rustfs/backlog/issues/1670
2026-08-31 09:18:28 +00:00 · 2026-08-04 23:21:01 +08:00
parent 4f43c0ca7e
commit 624a4ab837
8 changed files with 1976 additions and 0 deletions
@@ -0,0 +1,202 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression tests for distributed cluster startup and quorum.
+//!
+//! Covers the recurring pattern where multi-node clusters fail to start due to
+//! lock quorum issues, DNS resolution delays, or erasure quorum deadlocks.
+//! This has regressed 7+ times.
+//!
+//! ## Regression Issues
+//!
+//! - rustfs#5416: RustFS cannot cold-start with 2/3 quorum when Pod DNS missing
+//! - rustfs#2945: Distributed mode fails on K8s: erasure quorum deadlock
+//! - rustfs#2794: distributed deployment does not become ready
+//! - rustfs#2601: fresh pod immediately enters FaultyDisk state
+//! - rustfs#4040: Distributed startup can fail lock quorum before AppContext initializes
+//! - rustfs#5655: fix(ecstore): bootstrap fresh four-node clusters reliably
+//! - rustfs#4954: S3/health endpoint unavailability after multi-pool scale-up
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestClusterEnvironment, init_logging};
+    use aws_sdk_s3::primitives::ByteStream;
+    use serial_test::serial;
+    use std::error::Error;
+    use tokio::time::{Duration, sleep, timeout};
+    use tracing::{info, warn};
+
+    type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+
+    /// RT-10: Verify 4-node cluster starts successfully and all nodes are ready.
+    ///
+    /// Regression pattern: distributed startup fails with quorum deadlock or
+    /// lock acquisition timeout (rustfs#2945, rustfs#5655).
+    ///
+    /// Steps:
+    /// 1. Create a 4-node cluster
+    /// 2. Start all nodes simultaneously
+    /// 3. Verify all nodes report healthy
+    /// 4. Verify S3 operations work through any node
+    #[tokio::test]
+    #[serial]
+    async fn test_four_node_cluster_startup_and_health() -> TestResult {
+        init_logging();
+        info!("RT-10: 4-node cluster startup and health");
+
+        let mut cluster = RustFSTestClusterEnvironment::new(4).await.expect("create 4-node cluster");
+
+        cluster.start().await.expect("start 4-node cluster");
+
+        // Create a bucket and verify it's accessible from all nodes
+        cluster
+            .create_test_bucket("rt10-startup")
+            .await
+            .expect("create bucket on cluster");
+
+        let clients = cluster.create_all_clients().expect("create per-node clients");
+
+        // Verify S3 operations work from every node
+        for (i, client) in clients.iter().enumerate() {
+            client
+                .put_object()
+                .bucket("rt10-startup")
+                .key(format!("from-node-{i}.txt"))
+                .body(ByteStream::from_static(b"hello from node"))
+                .send()
+                .await
+                .unwrap_or_else(|e| panic!("PUT from node {i} failed: {e}"));
+        }
+
+        // Verify all objects are visible from node 0
+        let list = clients[0]
+            .list_objects_v2()
+            .bucket("rt10-startup")
+            .send()
+            .await
+            .expect("list objects from node 0");
+
+        assert_eq!(
+            list.contents().len(),
+            4,
+            "RT-10 FAIL: expected 4 objects (one per node), found {}",
+            list.contents().len()
+        );
+
+        info!("RT-10 PASS: 4-node cluster starts and serves S3 from all nodes");
+        Ok(())
+    }
+
+    /// RT-10b: Verify cluster handles node restart gracefully.
+    ///
+    /// Regression pattern: after a node restart, it cannot rejoin the cluster
+    /// or enters a faulty state (rustfs#2601).
+    #[tokio::test]
+    #[serial]
+    async fn test_cluster_survives_node_restart() -> TestResult {
+        init_logging();
+        info!("RT-10b: cluster survives node restart");
+
+        let mut cluster = RustFSTestClusterEnvironment::new(4).await.expect("create 4-node cluster");
+
+        cluster.start().await.expect("start cluster");
+
+        cluster.create_test_bucket("rt10b-restart").await.expect("create bucket");
+
+        // Write data
+        let clients = cluster.create_all_clients()?;
+        clients[0]
+            .put_object()
+            .bucket("rt10b-restart")
+            .key("before-restart.txt")
+            .body(ByteStream::from_static(b"persistent data"))
+            .send()
+            .await
+            .expect("put object before restart");
+
+        // Stop node 3
+        cluster.stop_node(3).expect("stop node 3");
+        sleep(Duration::from_secs(2)).await;
+
+        // Verify cluster still works with 3/4 nodes (quorum)
+        clients[0]
+            .put_object()
+            .bucket("rt10b-restart")
+            .key("during-offline.txt")
+            .body(ByteStream::from_static(b"written while node 3 down"))
+            .send()
+            .await
+            .expect("PUT should succeed with 3/4 nodes");
+
+        // Restart node 3
+        cluster.start_node(3).await.expect("restart node 3");
+
+        // Wait for node to rejoin
+        sleep(Duration::from_secs(3)).await;
+
+        // Verify the restarted node can serve reads
+        let list = clients[3]
+            .list_objects_v2()
+            .bucket("rt10b-restart")
+            .send()
+            .await
+            .expect("list from restarted node");
+
+        assert!(
+            list.contents().len() >= 2,
+            "RT-10b FAIL: restarted node sees {} objects, expected >= 2",
+            list.contents().len()
+        );
+
+        info!("RT-10b PASS: cluster survives and recovers from node restart");
+        Ok(())
+    }
+
+    /// RT-10c: Verify bucket creation persists across all nodes.
+    ///
+    /// Regression pattern: bucket metadata is not replicated to all nodes,
+    /// causing NoSuchBucket errors on some nodes (rustfs#3191).
+    #[tokio::test]
+    #[serial]
+    async fn test_bucket_visible_from_all_nodes() -> TestResult {
+        init_logging();
+        info!("RT-10c: bucket visible from all nodes");
+
+        let mut cluster = RustFSTestClusterEnvironment::new(4).await.expect("create 4-node cluster");
+
+        cluster.start().await.expect("start cluster");
+
+        cluster
+            .create_test_bucket("rt10c-bucket-visibility")
+            .await
+            .expect("create bucket");
+
+        let clients = cluster.create_all_clients()?;
+
+        // Verify the bucket is visible from every node
+        for (i, client) in clients.iter().enumerate() {
+            let resp = client
+                .list_objects_v2()
+                .bucket("rt10c-bucket-visibility")
+                .send()
+                .await
+                .unwrap_or_else(|e| panic!("list from node {i} failed (NoSuchBucket?): {e}"));
+
+            assert!(resp.contents().is_empty(), "RT-10c: fresh bucket should be empty on node {i}");
+        }
+
+        info!("RT-10c PASS: bucket visible from all 4 nodes");
+        Ok(())
+    }
+}