test(heal): read coordinator logs from the isolated path

(cherry picked from commit 618d38372cab961f293789a3bf08b8a8925e9771)

.github/workflows/ci.yml

@@ -995,6 +995,11 @@ jobs:
           cargo nextest list --profile e2e-full -p e2e_test --message-format json > "${NEXTEST_LISTING}"
           python3 ./scripts/check_test_wiring.py --check-profile e2e-full "${NEXTEST_LISTING}"
 
+      - name: Prepare Heal logs
+        run: |
+          heal_log_root=$(mktemp -d "${RUNNER_TEMP}/rustfs-heal-logs.XXXXXX")
+          echo "RUSTFS_HEAL_CHAOS_LOG_DIR=$heal_log_root" >> "$GITHUB_ENV"
+
       # Full single-node e2e lane (backlog#1149 ci-5). The e2e-full
       # default-filter in .config/nextest.toml is the single wiring mechanism —
       # extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
@@ -1004,8 +1009,18 @@ jobs:
           RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
           RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
           RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
+          RUSTFS_HEAL_CHAOS_SERVER_RUST_LOG: error,rustfs::heal::task=info,rustfs::app::object_usecase=warn,rustfs_ecstore::set_disk::ops::object=warn,rustfs_lock::distributed_lock=debug,rustfs_ecstore::cluster::rpc::remote_locker=warn
         run: cargo nextest run --profile e2e-full -p e2e_test
 
+      - name: Upload coordinator restart logs
+        if: always() && env.RUSTFS_HEAL_CHAOS_LOG_DIR != ''
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: heal-coordinator-restart-logs-${{ github.run_number }}-${{ github.run_attempt }}
+          path: ${{ env.RUSTFS_HEAL_CHAOS_LOG_DIR }}/coordinator_restart/**/node*.log
+          if-no-files-found: warn
+          retention-days: 7
+
       - name: Upload junit
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -923,7 +923,7 @@ mod tests {
                 latest_cycle_end = latest_cycle_end.max(cycle_end);
                 versions_observed |= versions_scanned > 0;
                 observations.push(format!(
-                    "node{node_index}: end={cycle_end}, versions={versions_scanned}, cycle={}, active={}, leader={}, result={}",
+                    "node{node_index}: end={cycle_end}, versions={versions_scanned}, cycle={}, active={}, leader={}, result={}, status={status}",
                     metrics["current_cycle"],
                     metrics["current_cycle_active"],
                     metrics["leader_lock_state"],
@@ -1047,6 +1047,8 @@ mod tests {
         let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
         cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
         cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
+        // Capture physical baselines after the PUT rename fanout has drained.
+        cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
         // Heal control uses the first lexicographically sorted grid host.
         // Keep that coordinator distinct from the remote target at index 1.
         cluster.nodes.sort_by(|left, right| left.url.cmp(&right.url));
@@ -1069,10 +1071,20 @@ mod tests {
         let server_rust_log = std::env::var("RUSTFS_HEAL_CHAOS_SERVER_RUST_LOG")
             .unwrap_or_else(|_| "rustfs::heal::task=info,rustfs=error".to_string());
         cluster.set_env("RUST_LOG", server_rust_log);
-        let log_dir = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR").unwrap_or_else(|_| format!("{}/logs", cluster.temp_dir));
+        let log_dir = match std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR") {
+            Ok(root) => PathBuf::from(root).join(interruption_kind).join(
+                Path::new(&cluster.temp_dir)
+                    .file_name()
+                    .ok_or("cluster temp directory has no basename")?,
+            ),
+            Err(_) => PathBuf::from(&cluster.temp_dir).join("logs"),
+        };
         std::fs::create_dir_all(&log_dir)?;
         for node_index in 0..cluster.nodes.len() {
-            cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
+            cluster.set_node_capture_log_path(
+                node_index,
+                log_dir.join(format!("node{node_index}.log")).to_string_lossy().into_owned(),
+            )?;
         }
         cluster.start_with_binary(&server_binary).await?;
         let clients = cluster.create_all_clients()?;
@@ -1326,7 +1338,7 @@ mod tests {
         let pre_interrupt_status: serde_json::Value = serde_json::from_str(&pre_interrupt_status_body)
             .map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {pre_interrupt_status_body}"))?;
         let pre_interrupt_replacement = replacement_recovery_status(&cluster).await?;
-        let coordinator_log = std::fs::read_to_string(format!("{log_dir}/node0.log"))?;
+        let coordinator_log = std::fs::read_to_string(log_dir.join("node0.log"))?;
         assert!(
             coordinator_log
                 .lines()