From 47af5565c77a1c4a21c1eb79176eec4f65040ca4 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 29 Sep 2026 08:50:09 +0800 Subject: [PATCH] fix(ci): preserve cluster startup failure evidence (#8213) --- .config/e2e-full-selection.txt | 4 +- .github/workflows/ci.yml | 10 ++ crates/e2e_test/src/common.rs | 98 +++++++++++++++---- .../src/heal_erasure_disk_rebuild_test.rs | 7 +- 4 files changed, 95 insertions(+), 24 deletions(-) diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index e09c9add1..77b7535bd 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=a6aee967387226b6e6937a0e625db53048c8b2f502636820a09c6ad791341b44 -sha256-linux=6b8b8f3dbdd63414fc969716ae3d7c2ec71712395630bae8bd419de2f32195b3 +sha256-darwin=3b6ff112ff16acbaa4148c4736095a1b6b71cc11af0dd3513ff28c6bdc334ca8 +sha256-linux=4e8bfc01870565a0a608a1634e6919fc593ce120a970aa04d974f7edf2675c21 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13bf73e21..759a3c510 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1022,6 +1022,7 @@ jobs: RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence + RUSTFS_HEAL_CHAOS_LOG_ROOT: ${{ runner.temp }}/rustfs-heal-chaos-logs run: python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test - name: Upload junit @@ -1045,6 +1046,15 @@ jobs: if-no-files-found: warn retention-days: 7 + - name: Upload heal cluster logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-full-heal-logs-${{ github.run_number }} + path: ${{ runner.temp }}/rustfs-heal-chaos-logs + if-no-files-found: warn + retention-days: 7 + e2e-tests-rio-v2: name: End-to-End Tests (rio-v2) # Inherits the schedule/dispatch-only gate through needs: on every other diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index 698df2f76..5b8e4bba5 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -22,7 +22,8 @@ //! - Common test constants and utilities use aws_sdk_s3::config::{Credentials, Region}; -use aws_sdk_s3::error::ProvideErrorMetadata; +use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError}; +use aws_sdk_s3::operation::list_buckets::ListBucketsError; use aws_sdk_s3::{Client, Config}; use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; use http::header::{CONTENT_TYPE, HOST}; @@ -62,6 +63,41 @@ const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port"; const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock"; const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30); +fn list_buckets_readiness_error(err: &SdkError) -> String { + // SDK Display reports only the category. Do not expose raw response bodies or headers. + let service_code = err.as_service_error().and_then(|error| error.code()).filter(|code| { + matches!( + *code, + "ServiceUnavailable" + | "ServerNotInitialized" + | "InternalError" + | "AccessDenied" + | "InvalidAccessKeyId" + | "SignatureDoesNotMatch" + ) + }); + let transport = match err { + SdkError::DispatchFailure(failure) if failure.is_timeout() => Some("timeout"), + SdkError::DispatchFailure(failure) if failure.is_io() => Some("io"), + SdkError::DispatchFailure(failure) if failure.is_user() => Some("user"), + SdkError::DispatchFailure(failure) if failure.is_other() => Some("other"), + _ => None, + }; + let mut source: Option<&(dyn std::error::Error + 'static)> = Some(err); + let mut io_error = None; + while let Some(error) = source { + if let Some(error) = error.downcast_ref::() { + io_error = Some((error.kind(), error.raw_os_error())); + break; + } + source = error.source(); + } + format!( + "{err}; http_status={:?}; service_code={service_code:?}; transport={transport:?}; io_error={io_error:?}", + err.raw_response().map(|response| response.status().as_u16()) + ) +} + fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option { let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy(); Some(log_dir.join(format!("{temp_name}.log"))) @@ -1576,7 +1612,7 @@ impl RustFSTestClusterEnvironment { /// /// Verifies service availability by calling the S3 `list_buckets` API against the requested node, /// retries up to 120 times with a 1-second interval between attempts. - async fn wait_for_node_service_ready(&self, node_idx: usize) -> Result<(), Box> { + async fn wait_for_node_service_ready(&mut self, node_idx: usize) -> Result<(), Box> { let client = self.create_s3_client(node_idx)?; let mut last_error = None; @@ -1593,26 +1629,18 @@ impl RustFSTestClusterEnvironment { } } - let last_error = last_error.as_ref().map(|err| { - // SDK Display reports only the category. Do not expose raw response bodies or headers. - let service_code = err.as_service_error().and_then(|error| error.code()).filter(|code| { - matches!( - *code, - "ServiceUnavailable" - | "ServerNotInitialized" - | "InternalError" - | "AccessDenied" - | "InvalidAccessKeyId" - | "SignatureDoesNotMatch" - ) - }); - format!( - "{err}; http_status={:?}; service_code={service_code:?}", - err.raw_response().map(|response| response.status().as_u16()) - ) - }); + let last_error = last_error.as_ref().map(list_buckets_readiness_error); + // Observe liveness only after the existing readiness budget is exhausted, before cleanup. + let process_status = match self.nodes[node_idx].process.as_mut() { + Some(process) => match process.try_wait() { + Ok(Some(status)) => format!("exited ({status})"), + Ok(None) => "running".to_string(), + Err(error) => format!("unavailable (kind={:?}, os_code={:?})", error.kind(), error.raw_os_error()), + }, + None => "not started".to_string(), + }; Err(format!( - "Cluster node {node_idx} service failed to become ready; last ListBuckets error={last_error:?}; capture_log_path={:?}", + "Cluster node {node_idx} service failed to become ready; last ListBuckets error={last_error:?}; process_status={process_status}; capture_log_path={:?}", self.node_capture_log_paths[node_idx] ) .into()) @@ -2126,6 +2154,34 @@ mod tests { verify_awscurl_path(&executable).expect("an available client with a working help command should pass"); } + #[test] + fn readiness_error_reports_transport_without_sensitive_source_text() { + use aws_sdk_s3::error::ConnectorError; + + let sensitive = "request credentials must not appear in diagnostics"; + for (connector, category) in [ + ( + ConnectorError::io(std::io::Error::new(ErrorKind::ConnectionReset, sensitive).into()), + "io", + ), + (ConnectorError::timeout(sensitive.into()), "timeout"), + (ConnectorError::user(sensitive.into()), "user"), + (ConnectorError::other(sensitive.into(), None), "other"), + ] { + let diagnostic = list_buckets_readiness_error(&SdkError::dispatch_failure(connector)); + assert!(diagnostic.contains(&format!("transport=Some(\"{category}\")")), "{diagnostic}"); + assert!(diagnostic.contains("http_status=None"), "{diagnostic}"); + assert!(!diagnostic.contains(sensitive), "{diagnostic}"); + if category == "io" { + assert!(diagnostic.contains("io_error=Some((ConnectionReset, None))"), "{diagnostic}"); + } + } + let os_error = std::io::Error::from_raw_os_error(13); + let expected = format!("io_error=Some(({:?}, Some(13)))", os_error.kind()); + let diagnostic = list_buckets_readiness_error(&SdkError::dispatch_failure(ConnectorError::io(os_error.into()))); + assert!(diagnostic.contains(&expected), "{diagnostic}"); + } + #[test] fn capture_log_path_uses_temp_directory_basename() { assert_eq!( diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs index c1a2c6af2..a427c290c 100644 --- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs +++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs @@ -1453,7 +1453,12 @@ mod tests { let server_rust_log = std::env::var("RUSTFS_HEAL_CHAOS_SERVER_RUST_LOG") .unwrap_or_else(|_| "rustfs::heal::task=info,rustfs=error".to_string()); cluster.set_env("RUST_LOG", server_rust_log); - let log_dir = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR").unwrap_or_else(|_| format!("{}/logs", cluster.temp_dir)); + let log_dir = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR").unwrap_or_else(|_| { + // Keep explicit directories compatible and isolate parallel suite captures. + std::env::var("RUSTFS_HEAL_CHAOS_LOG_ROOT") + .map(|root| format!("{root}/{interruption_kind}-{}", uuid::Uuid::new_v4())) + .unwrap_or_else(|_| format!("{}/logs", cluster.temp_dir)) + }); std::fs::create_dir_all(&log_dir)?; for node_index in 0..cluster.nodes.len() { cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;