fix(ci): preserve cluster startup failure evidence (#8213)

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=a6aee967387226b6e6937a0e625db53048c8b2f502636820a09c6ad791341b44
-sha256-linux=6b8b8f3dbdd63414fc969716ae3d7c2ec71712395630bae8bd419de2f32195b3
+sha256-darwin=3b6ff112ff16acbaa4148c4736095a1b6b71cc11af0dd3513ff28c6bdc334ca8
+sha256-linux=4e8bfc01870565a0a608a1634e6919fc593ce120a970aa04d974f7edf2675c21

.github/workflows/ci.yml

@@ -1022,6 +1022,7 @@ jobs:
           RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
           RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
           RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
+          RUSTFS_HEAL_CHAOS_LOG_ROOT: ${{ runner.temp }}/rustfs-heal-chaos-logs
         run: python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
 
       - name: Upload junit
@@ -1045,6 +1046,15 @@ jobs:
           if-no-files-found: warn
           retention-days: 7
 
+      - name: Upload heal cluster logs
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-full-heal-logs-${{ github.run_number }}
+          path: ${{ runner.temp }}/rustfs-heal-chaos-logs
+          if-no-files-found: warn
+          retention-days: 7
+
   e2e-tests-rio-v2:
     name: End-to-End Tests (rio-v2)
     # Inherits the schedule/dispatch-only gate through needs: on every other

crates/e2e_test/src/common.rs

@@ -22,7 +22,8 @@
 //! - Common test constants and utilities
 
 use aws_sdk_s3::config::{Credentials, Region};
-use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
+use aws_sdk_s3::operation::list_buckets::ListBucketsError;
 use aws_sdk_s3::{Client, Config};
 use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
 use http::header::{CONTENT_TYPE, HOST};
@@ -62,6 +63,41 @@ const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
 const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
 const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);
 
+fn list_buckets_readiness_error(err: &SdkError<ListBucketsError>) -> String {
+    // SDK Display reports only the category. Do not expose raw response bodies or headers.
+    let service_code = err.as_service_error().and_then(|error| error.code()).filter(|code| {
+        matches!(
+            *code,
+            "ServiceUnavailable"
+                | "ServerNotInitialized"
+                | "InternalError"
+                | "AccessDenied"
+                | "InvalidAccessKeyId"
+                | "SignatureDoesNotMatch"
+        )
+    });
+    let transport = match err {
+        SdkError::DispatchFailure(failure) if failure.is_timeout() => Some("timeout"),
+        SdkError::DispatchFailure(failure) if failure.is_io() => Some("io"),
+        SdkError::DispatchFailure(failure) if failure.is_user() => Some("user"),
+        SdkError::DispatchFailure(failure) if failure.is_other() => Some("other"),
+        _ => None,
+    };
+    let mut source: Option<&(dyn std::error::Error + 'static)> = Some(err);
+    let mut io_error = None;
+    while let Some(error) = source {
+        if let Some(error) = error.downcast_ref::<std::io::Error>() {
+            io_error = Some((error.kind(), error.raw_os_error()));
+            break;
+        }
+        source = error.source();
+    }
+    format!(
+        "{err}; http_status={:?}; service_code={service_code:?}; transport={transport:?}; io_error={io_error:?}",
+        err.raw_response().map(|response| response.status().as_u16())
+    )
+}
+
 fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
     let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
     Some(log_dir.join(format!("{temp_name}.log")))
@@ -1576,7 +1612,7 @@ impl RustFSTestClusterEnvironment {
     ///
     /// Verifies service availability by calling the S3 `list_buckets` API against the requested node,
     /// retries up to 120 times with a 1-second interval between attempts.
-    async fn wait_for_node_service_ready(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    async fn wait_for_node_service_ready(&mut self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let client = self.create_s3_client(node_idx)?;
         let mut last_error = None;
 
@@ -1593,26 +1629,18 @@ impl RustFSTestClusterEnvironment {
             }
         }
 
-        let last_error = last_error.as_ref().map(|err| {
-            // SDK Display reports only the category. Do not expose raw response bodies or headers.
-            let service_code = err.as_service_error().and_then(|error| error.code()).filter(|code| {
-                matches!(
-                    *code,
-                    "ServiceUnavailable"
-                        | "ServerNotInitialized"
-                        | "InternalError"
-                        | "AccessDenied"
-                        | "InvalidAccessKeyId"
-                        | "SignatureDoesNotMatch"
-                )
-            });
-            format!(
-                "{err}; http_status={:?}; service_code={service_code:?}",
-                err.raw_response().map(|response| response.status().as_u16())
-            )
-        });
+        let last_error = last_error.as_ref().map(list_buckets_readiness_error);
+        // Observe liveness only after the existing readiness budget is exhausted, before cleanup.
+        let process_status = match self.nodes[node_idx].process.as_mut() {
+            Some(process) => match process.try_wait() {
+                Ok(Some(status)) => format!("exited ({status})"),
+                Ok(None) => "running".to_string(),
+                Err(error) => format!("unavailable (kind={:?}, os_code={:?})", error.kind(), error.raw_os_error()),
+            },
+            None => "not started".to_string(),
+        };
         Err(format!(
-            "Cluster node {node_idx} service failed to become ready; last ListBuckets error={last_error:?}; capture_log_path={:?}",
+            "Cluster node {node_idx} service failed to become ready; last ListBuckets error={last_error:?}; process_status={process_status}; capture_log_path={:?}",
             self.node_capture_log_paths[node_idx]
         )
         .into())
@@ -2126,6 +2154,34 @@ mod tests {
         verify_awscurl_path(&executable).expect("an available client with a working help command should pass");
     }
 
+    #[test]
+    fn readiness_error_reports_transport_without_sensitive_source_text() {
+        use aws_sdk_s3::error::ConnectorError;
+
+        let sensitive = "request credentials must not appear in diagnostics";
+        for (connector, category) in [
+            (
+                ConnectorError::io(std::io::Error::new(ErrorKind::ConnectionReset, sensitive).into()),
+                "io",
+            ),
+            (ConnectorError::timeout(sensitive.into()), "timeout"),
+            (ConnectorError::user(sensitive.into()), "user"),
+            (ConnectorError::other(sensitive.into(), None), "other"),
+        ] {
+            let diagnostic = list_buckets_readiness_error(&SdkError::dispatch_failure(connector));
+            assert!(diagnostic.contains(&format!("transport=Some(\"{category}\")")), "{diagnostic}");
+            assert!(diagnostic.contains("http_status=None"), "{diagnostic}");
+            assert!(!diagnostic.contains(sensitive), "{diagnostic}");
+            if category == "io" {
+                assert!(diagnostic.contains("io_error=Some((ConnectionReset, None))"), "{diagnostic}");
+            }
+        }
+        let os_error = std::io::Error::from_raw_os_error(13);
+        let expected = format!("io_error=Some(({:?}, Some(13)))", os_error.kind());
+        let diagnostic = list_buckets_readiness_error(&SdkError::dispatch_failure(ConnectorError::io(os_error.into())));
+        assert!(diagnostic.contains(&expected), "{diagnostic}");
+    }
+
     #[test]
     fn capture_log_path_uses_temp_directory_basename() {
         assert_eq!(

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -1453,7 +1453,12 @@ mod tests {
         let server_rust_log = std::env::var("RUSTFS_HEAL_CHAOS_SERVER_RUST_LOG")
             .unwrap_or_else(|_| "rustfs::heal::task=info,rustfs=error".to_string());
         cluster.set_env("RUST_LOG", server_rust_log);
-        let log_dir = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR").unwrap_or_else(|_| format!("{}/logs", cluster.temp_dir));
+        let log_dir = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR").unwrap_or_else(|_| {
+            // Keep explicit directories compatible and isolate parallel suite captures.
+            std::env::var("RUSTFS_HEAL_CHAOS_LOG_ROOT")
+                .map(|root| format!("{root}/{interruption_kind}-{}", uuid::Uuid::new_v4()))
+                .unwrap_or_else(|_| format!("{}/logs", cluster.temp_dir))
+        });
         std::fs::create_dir_all(&log_dir)?;
         for node_index in 0..cluster.nodes.len() {
             cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;