From 6920abfe29eea028d01d3e42c8d5344af9789ac5 Mon Sep 17 00:00:00 2001
From: overtrue <anzhengchao@gmail.com>
Date: Wed, 9 Sep 2026 12:31:04 +0800
Subject: [PATCH] ci: retain full e2e recovery diagnostics

---
 .github/workflows/ci.yml                         | 11 +++++++++++
 .../src/heal_erasure_disk_rebuild_test.rs        | 16 ++++++++++++++--
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8af2c929d..cd9228206 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1002,11 +1002,22 @@ jobs:
       # debug binary; each test spawns its own rustfs server on a random port.
       - name: Run e2e full suite
         env:
+          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-full-logs
+          RUSTFS_HEAL_CHAOS_SERVER_RUST_LOG: rustfs::heal=debug,rustfs_heal=debug,rustfs_ecstore::disk=debug,rustfs_lock=debug,rustfs=error
           RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
           RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
           RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
         run: cargo nextest run --profile e2e-full -p e2e_test
 
+      - name: Upload e2e full diagnostics
+        if: failure()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: e2e-full-diagnostics-${{ github.run_number }}
+          path: ${{ runner.temp }}/rustfs-e2e-full-logs/
+          if-no-files-found: warn
+          retention-days: 7
+
       - name: Upload junit
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs
index 06f443276..c2f63155f 100644
--- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs
+++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs
@@ -1581,7 +1581,19 @@ mod tests {
                 }
             }
             if Instant::now() >= heal_deadline {
-                let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
+                let mut mismatched_manifests = Vec::new();
+                for expected in &expected_manifests {
+                    let actual = census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?;
+                    if !actual.matches_manifest(&expected.shard_census) {
+                        mismatched_manifests.push(serde_json::json!({
+                            "key": expected.key,
+                            "expected": expected.shard_census,
+                            "actual": actual,
+                        }));
+                    }
+                }
+                let matching = expected_manifests.len() - mismatched_manifests.len();
+                let mismatched_manifests = serde_json::Value::Array(mismatched_manifests);
                 let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
                 let pool_metadata =
                     census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?;
@@ -1604,7 +1616,7 @@ mod tests {
                     Err(_) => "replacement status request exceeded 5s diagnostic budget".to_string(),
                 };
                 return Err(format!(
-                    "root heal did not recover after {interruption_kind} within {heal_timeout_secs}s: baseline={matching}/{}, outage={outage_census:?}, pool_metadata={pool_metadata:?}, status={final_status}, task_status={task_status}, pre_interrupt_status={pre_interrupt_status}, pre_heal_replacement={pre_heal_replacement}, pre_interrupt_replacement={pre_interrupt_replacement}, replacement_status={replacement_status}",
+                    "root heal did not recover after {interruption_kind} within {heal_timeout_secs}s: baseline={matching}/{}, mismatched_manifests={mismatched_manifests}, outage={outage_census:?}, pool_metadata={pool_metadata:?}, status={final_status}, task_status={task_status}, pre_interrupt_status={pre_interrupt_status}, pre_heal_replacement={pre_heal_replacement}, pre_interrupt_replacement={pre_interrupt_replacement}, replacement_status={replacement_status}",
                     expected_manifests.len()
                 )
                 .into());