test(e2e): target multi-set outage heal candidate (#7653)

* test(e2e): target multi-set outage heal candidate Require the outage write used by EC8+4 multi-set root-heal evidence to miss the same erasure index owned by the selected replacement drive. This avoids accepting a candidate from a different set and turning a valid heal into a false negative. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(e2e): satisfy G14 heal lint gates Remove clippy-only noise from the G14 multi-set heal evidence test and align the admin route policy inventory with the registered heal catch-all route. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * update --------- Co-authored-by: zhi22915 <qiuzgang@gmail.com>
2026-09-11 13:29:12 +00:00 · 2026-09-11 08:22:10 +08:00
parent 77ba4f1b2e
commit 0ae38aabbc
8 changed files with 315 additions and 67 deletions
@@ -539,6 +539,13 @@ path = "junit.xml"
 filter = 'package(e2e_test)'
 test-group = 'e2e-cluster-nightly'

+# The EC8+4 multi-set heal proof deliberately uploads a larger workload so the
+# background root-heal pass can be interrupted after targeting the replacement
+# drive's erasure slots. Keep the extended budget scoped to this proof case.
+[[profile.e2e-nightly.overrides]]
+filter = 'package(e2e_test) & test(=heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart)'
+slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
+
 # ---------------------------------------------------------------------------
 # e2e-distributed profile — 4-node 4-disk Actions suite
 # ---------------------------------------------------------------------------
@@ -271,7 +271,7 @@ version = "1.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
 dependencies = [
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -282,7 +282,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
 dependencies = [
 "anstyle",
 "once_cell_polyfill",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -688,9 +688,9 @@ dependencies = [

 [[package]]
 name = "async-compat"
-version = "0.2.5"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1ba85bc55464dcbf728b56d97e119d673f4cf9062be330a9a26f3acf504a590"
+checksum = "4c97d7ff3c25d6c10d64170c12acaf5d4245e76dece3779c1d92b153a64f11df"
 dependencies = [
 "futures-core",
 "futures-io",
@@ -1906,7 +1906,7 @@ dependencies = [
 "maybe-owned",
 "rustix",
 "rustix-linux-procfs",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 "winx",
 ]

@@ -3942,7 +3942,7 @@ dependencies = [
 "libc",
 "option-ext",
 "redox_users",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -4287,7 +4287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
 "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -5155,9 +5155,9 @@ dependencies = [

 [[package]]
 name = "hickory-net"
-version = "0.26.2"
+version = "0.26.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "084e7bd6a377435d568f652153e571b50970d7ccc1d1eeec0519f834632287e1"
+checksum = "c480823ed7c2c5d0f09c41020cb6b7c28029ce60ec42dc942158dcf22f8e0a4d"
 dependencies = [
 "async-trait",
 "cfg-if",
@@ -5179,9 +5179,9 @@ dependencies = [

 [[package]]
 name = "hickory-proto"
-version = "0.26.2"
+version = "0.26.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e2da0694c15b44c6f68a6b05e0233617008c54080e31d6eb848d858a9c5b38d"
+checksum = "12b92608f679a6fa515dd1d15c1ff89443026e391200a2c840c7afcba482893d"
 dependencies = [
 "data-encoding",
 "idna",
@@ -5199,9 +5199,9 @@ dependencies = [

 [[package]]
 name = "hickory-resolver"
-version = "0.26.2"
+version = "0.26.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e4f9f4603319422d482738f3f6fe5aac03157fdbfed1cd85a3ff45adb09072f"
+checksum = "3f3da5255c95d5a716857d54b5b8f4e8d67c3484d3beaaaae2ce25063b3ba981"
 dependencies = [
 "cfg-if",
 "futures-util",
@@ -6963,7 +6963,7 @@ version = "0.50.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -8712,7 +8712,7 @@ dependencies = [
 "once_cell",
 "socket2",
 "tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -11065,7 +11065,7 @@ dependencies = [
 "errno",
 "libc",
 "linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -11148,7 +11148,7 @@ dependencies = [
 "security-framework",
 "security-framework-sys",
 "webpki-root-certs",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -11949,7 +11949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
 dependencies = [
 "libc",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -12100,7 +12100,7 @@ dependencies = [
 "cfg-if",
 "libc",
 "psm",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -12405,10 +12405,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
 "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
 "once_cell",
 "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -12761,9 +12761,9 @@ dependencies = [

 [[package]]
 name = "toml_edit"
-version = "0.25.13+spec-1.1.0"
+version = "0.25.14+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b"
+checksum = "d2195eec204e2764644a4ea619704f9fbe5e0673038eded55ad9956f24fca0cc"
 dependencies = [
 "indexmap 2.14.2",
 "toml_datetime",
@@ -13510,7 +13510,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -61,6 +61,7 @@ pub(crate) struct VersionShardCensus {
    pub has_xl_meta: bool,
    pub data_dir: Option<String>,
    pub erasure_index: Option<usize>,
+    pub erasure_distribution: Option<Vec<usize>>,
    pub data_blocks: Option<usize>,
    pub parity_blocks: Option<usize>,
    pub expected_part_numbers: BTreeSet<usize>,
@@ -90,6 +91,7 @@ impl VersionShardCensus {
            && manifest.is_complete()
            && self.data_dir == manifest.data_dir
            && self.erasure_index == manifest.erasure_index
+            && self.erasure_distribution == manifest.erasure_distribution
            && self.data_blocks == manifest.data_blocks
            && self.parity_blocks == manifest.parity_blocks
            && self.expected_part_numbers == manifest.expected_part_numbers
@@ -317,6 +319,7 @@ pub(crate) fn census_object_version_on_disk(
            has_xl_meta: false,
            data_dir: None,
            erasure_index: None,
+            erasure_distribution: None,
            data_blocks: None,
            parity_blocks: None,
            expected_part_numbers: BTreeSet::new(),
@@ -334,6 +337,7 @@ pub(crate) fn census_object_version_on_disk(
    };
    let data_dir = file_info.data_dir.map(|id| id.to_string());
    let erasure_index = Some(file_info.erasure.index);
+    let erasure_distribution = Some(file_info.erasure.distribution.clone());
    let inline_data_fingerprint = file_info.data.as_deref().map(shard_fingerprint).transpose()?;
    let part_dir = data_dir.as_ref().map_or_else(|| object_dir.clone(), |id| object_dir.join(id));
    let present_part_fingerprints = match std::fs::read_dir(&part_dir) {
@@ -366,6 +370,7 @@ pub(crate) fn census_object_version_on_disk(
        has_xl_meta: true,
        data_dir,
        erasure_index,
+        erasure_distribution,
        data_blocks: Some(file_info.erasure.data_blocks),
        parity_blocks: Some(file_info.erasure.parity_blocks),
        expected_part_numbers,
@@ -421,6 +426,7 @@ mod tests {
            has_xl_meta: true,
            data_dir: Some("data-dir".to_string()),
            erasure_index: Some(3),
+            erasure_distribution: Some(vec![1, 2, 3, 4]),
            data_blocks: Some(2),
            parity_blocks: Some(2),
            expected_part_numbers: BTreeSet::from([1]),
@@ -253,6 +253,7 @@ async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult<Vec<Ex
                has_xl_meta: false,
                data_dir: None,
                erasure_index: None,
+                erasure_distribution: None,
                data_blocks: None,
                parity_blocks: None,
                expected_part_numbers: Default::default(),
@@ -24,7 +24,7 @@ mod tests {
    use crate::storage_api::RUSTFS_META_BUCKET;
    use aws_sdk_s3::{
        error::{ProvideErrorMetadata, SdkError},
-        operation::put_object::PutObjectError,
+        operation::{delete_object::DeleteObjectError, put_object::PutObjectError},
        primitives::ByteStream,
    };
    use http::Method;
@@ -45,6 +45,7 @@ mod tests {

    struct ReplacementDriveSelection {
        replaced_disk: PathBuf,
+        drive_index: usize,
        replacement_format_path: PathBuf,
        replacement_format: Vec<u8>,
        expected_pool_metadata: Option<VersionShardCensus>,
@@ -461,6 +462,12 @@ mod tests {
        shard_census: VersionShardCensus,
    }

+    #[derive(Debug, Default)]
+    struct OutagePeerManifest {
+        erasure_indices: HashSet<usize>,
+        erasure_distribution: Option<Vec<usize>>,
+    }
+
    fn deterministic_object_body(len: usize, seed: u8) -> Vec<u8> {
        let mut value = seed;
        std::iter::repeat_with(|| {
@@ -486,6 +493,81 @@ mod tests {
        Ok(matching)
    }

+    fn collect_outage_peer_manifest(
+        cluster: &RustFSTestClusterEnvironment,
+        offline_node_index: usize,
+        bucket: &str,
+        key: &str,
+        erasure_set_drive_count: usize,
+    ) -> Result<OutagePeerManifest, Box<dyn Error + Send + Sync>> {
+        let mut manifest = OutagePeerManifest::default();
+        for (node_index, node) in cluster.nodes.iter().enumerate() {
+            if node_index == offline_node_index {
+                continue;
+            }
+            for (drive_index, drive) in node.data_dirs.iter().enumerate() {
+                let census = census_object_version_on_disk(Path::new(drive), bucket, key, None)?;
+                if !census.has_xl_meta {
+                    continue;
+                }
+                assert!(
+                    census.is_complete(),
+                    "online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
+                );
+                let erasure_index = census.erasure_index.ok_or_else(|| {
+                    format!("online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}")
+                })?;
+                assert!(
+                    (1..=erasure_set_drive_count).contains(&erasure_index),
+                    "online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
+                );
+                let distribution = census.erasure_distribution.as_ref().ok_or_else(|| {
+                    format!(
+                        "online node {node_index} drive {drive_index} outage-object shard has no erasure distribution: {census:?}"
+                    )
+                })?;
+                assert_eq!(
+                    distribution.len(),
+                    erasure_set_drive_count,
+                    "online node {node_index} drive {drive_index} outage-object distribution must match the erasure set: {census:?}"
+                );
+                match &manifest.erasure_distribution {
+                    Some(existing) => {
+                        assert_eq!(existing, distribution, "outage-object shards must agree on one erasure distribution")
+                    }
+                    None => manifest.erasure_distribution = Some(distribution.clone()),
+                }
+                assert!(
+                    manifest.erasure_indices.insert(erasure_index),
+                    "outage-object erasure index {erasure_index} is duplicated across online drives"
+                );
+            }
+        }
+        Ok(manifest)
+    }
+
+    fn outage_candidate_replacement_erasure_index(
+        peer_manifest: &OutagePeerManifest,
+        erasure_set_drive_count: usize,
+        replacement_set_slot: usize,
+    ) -> Option<usize> {
+        let distribution = peer_manifest.erasure_distribution.as_ref()?;
+        distribution.get(replacement_set_slot).copied().filter(|replacement_index| {
+            (1..=erasure_set_drive_count).contains(replacement_index)
+                && !peer_manifest.erasure_indices.contains(replacement_index)
+        })
+    }
+
+    fn outage_candidate_targets_replacement(
+        peer_manifest: &OutagePeerManifest,
+        erasure_set_drive_count: usize,
+        replacement_set_slot: usize,
+    ) -> bool {
+        let min_online_data_shards = erasure_set_drive_count.saturating_sub(4);
+        peer_manifest.erasure_indices.len() >= min_online_data_shards
+            && outage_candidate_replacement_erasure_index(peer_manifest, erasure_set_drive_count, replacement_set_slot).is_some()
+    }
+
    fn metadata_count(disk: &Path, bucket: &str, expected_manifests: &[PhysicalObjectManifest]) -> usize {
        expected_manifests
            .iter()
@@ -601,6 +683,10 @@ mod tests {
        error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable")
    }

+    fn is_service_unavailable_delete(error: &SdkError<DeleteObjectError>) -> bool {
+        error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable")
+    }
+
    fn select_replacement_drive(
        cluster: &RustFSTestClusterEnvironment,
        node_index: usize,
@@ -612,7 +698,7 @@ mod tests {
            .ok_or_else(|| format!("replacement node {node_index} is absent"))?;
        let mut incomplete_pool_metadata = Vec::new();

-        for drive in &node.data_dirs {
+        for (drive_index, drive) in node.data_dirs.iter().enumerate() {
            let replaced_disk = PathBuf::from(drive);
            let replacement_format_path = replaced_disk.join(".rustfs.sys").join("format.json");
            let replacement_format = std::fs::read(&replacement_format_path).map_err(|err| {
@@ -621,6 +707,7 @@ mod tests {
            if !require_pool_metadata {
                return Ok(ReplacementDriveSelection {
                    replaced_disk,
+                    drive_index,
                    replacement_format_path,
                    replacement_format,
                    expected_pool_metadata: None,
@@ -631,6 +718,7 @@ mod tests {
            if census.is_complete() {
                return Ok(ReplacementDriveSelection {
                    replaced_disk,
+                    drive_index,
                    replacement_format_path,
                    replacement_format,
                    expected_pool_metadata: Some(census),
@@ -1180,7 +1268,7 @@ mod tests {
    async fn test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart()
    -> Result<(), Box<dyn Error + Send + Sync>> {
        timeout(
-            Duration::from_secs(600),
+            Duration::from_secs(900),
            run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestartEc84MultiSet),
        )
        .await?
@@ -1332,10 +1420,13 @@ mod tests {

        let ReplacementDriveSelection {
            replaced_disk,
+            drive_index: replacement_drive_index,
            replacement_format_path,
            replacement_format,
            expected_pool_metadata,
        } = select_replacement_drive(&cluster, 1, background_enabled)?;
+        let replacement_global_drive_index = topology.drives_per_node + replacement_drive_index;
+        let replacement_set_slot = replacement_global_drive_index % erasure_set_drive_count;
        let default_online_object_count = if !outage_target_manifest_required { 64 } else { 24 };
        let online_object_count = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_COUNT")
            .ok()
@@ -1391,6 +1482,23 @@ mod tests {
            expected_manifests.push(PhysicalObjectManifest { key, shard_census });
            attempt_count += 1;
        }
+        for manifest in &expected_manifests {
+            let distribution = manifest
+                .shard_census
+                .erasure_distribution
+                .as_ref()
+                .ok_or_else(|| format!("replacement baseline manifest has no erasure distribution: {manifest:?}"))?;
+            assert_eq!(
+                distribution.len(),
+                erasure_set_drive_count,
+                "replacement baseline distribution must match the erasure set: {manifest:?}"
+            );
+            assert_eq!(
+                manifest.shard_census.erasure_index,
+                distribution.get(replacement_set_slot).copied(),
+                "replacement baseline shard must match the selected drive's erasure-set slot"
+            );
+        }

        if background_enabled {
            wait_for_scanner_cycle_after(&cluster, 0).await?;
@@ -1415,6 +1523,9 @@ mod tests {
        let mut outage_write_deferred_until_rejoin = false;
        let mut service_unavailable_outage_writes = 0usize;
        let mut last_service_unavailable = None;
+        let mut outage_peer_manifest = OutagePeerManifest::default();
+        let mut replacement_outage_erasure_index = None;
+        let mut rejected_outage_keys = Vec::new();
        for attempt in 0..max_outage_write_attempts {
            let candidate_key = format!("cluster/written-while-node-down-{attempt:04}.bin");
            let put_result = timeout(
@@ -1429,6 +1540,24 @@ mod tests {
            .await;
            match put_result {
                Ok(Ok(_)) => {
+                    if outage_target_manifest_required {
+                        let candidate_peer_manifest =
+                            collect_outage_peer_manifest(&cluster, 1, bucket, &candidate_key, erasure_set_drive_count)?;
+                        if !outage_candidate_targets_replacement(
+                            &candidate_peer_manifest,
+                            erasure_set_drive_count,
+                            replacement_set_slot,
+                        ) {
+                            rejected_outage_keys.push(candidate_key);
+                            continue;
+                        }
+                        replacement_outage_erasure_index = outage_candidate_replacement_erasure_index(
+                            &candidate_peer_manifest,
+                            erasure_set_drive_count,
+                            replacement_set_slot,
+                        );
+                        outage_peer_manifest = candidate_peer_manifest;
+                    }
                    outage_key = Some(candidate_key);
                    break;
                }
@@ -1456,57 +1585,37 @@ mod tests {
            }
        };

-        let mut outage_peer_erasure_indices = HashSet::new();
-        if !outage_write_deferred_until_rejoin {
-            for (node_index, node) in cluster.nodes.iter().enumerate() {
-                if node_index == 1 {
-                    continue;
-                }
-                for (drive_index, drive) in node.data_dirs.iter().enumerate() {
-                    let census = census_object_version_on_disk(Path::new(drive), bucket, &outage_key, None)?;
-                    if !census.has_xl_meta {
-                        continue;
-                    }
-                    assert!(
-                        census.is_complete(),
-                        "online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
-                    );
-                    let erasure_index = census.erasure_index.ok_or_else(|| {
-                        format!(
-                            "online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}"
-                        )
-                    })?;
-                    assert!(
-                        (1..=erasure_set_drive_count).contains(&erasure_index),
-                        "online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
-                    );
-                    assert!(
-                        outage_peer_erasure_indices.insert(erasure_index),
-                        "outage-object erasure index {erasure_index} is duplicated across online drives"
-                    );
-                }
-            }
+        if !outage_write_deferred_until_rejoin && outage_peer_manifest.erasure_indices.is_empty() {
+            outage_peer_manifest = collect_outage_peer_manifest(&cluster, 1, bucket, &outage_key, erasure_set_drive_count)?;
+            replacement_outage_erasure_index =
+                outage_candidate_replacement_erasure_index(&outage_peer_manifest, erasure_set_drive_count, replacement_set_slot);
        }
        assert!(
            outage_write_deferred_until_rejoin
-                || (!outage_peer_erasure_indices.is_empty() && outage_peer_erasure_indices.len() <= erasure_set_drive_count),
+                || (!outage_peer_manifest.erasure_indices.is_empty()
+                    && outage_peer_manifest.erasure_indices.len() <= erasure_set_drive_count),
            "outage-object must occupy one non-empty erasure set"
        );
        if outage_target_manifest_required {
            let min_online_data_shards = erasure_set_drive_count.saturating_sub(4);
            assert!(
-                outage_peer_erasure_indices.len() >= min_online_data_shards,
+                outage_peer_manifest.erasure_indices.len() >= min_online_data_shards,
                "online drives in the selected erasure set must retain at least the EC data quorum"
            );
        }
        let missing_outage_erasure_indices = (1..=erasure_set_drive_count)
-            .filter(|index| !outage_peer_erasure_indices.contains(index))
+            .filter(|index| !outage_peer_manifest.erasure_indices.contains(index))
            .collect::<HashSet<_>>();
        if outage_target_manifest_required {
            assert!(
                !missing_outage_erasure_indices.is_empty(),
                "the stopped target must account for at least one missing outage-object erasure index"
            );
+            assert_eq!(
+                replacement_outage_erasure_index.filter(|index| missing_outage_erasure_indices.contains(index)),
+                replacement_outage_erasure_index,
+                "the outage object must target the selected replacement drive's erasure-set slot"
+            );
        }

        let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
@@ -1531,6 +1640,24 @@ mod tests {
        }

        cluster.start_node_from_binary(1, &server_binary).await?;
+        for rejected_key in rejected_outage_keys {
+            let delete_deadline = Instant::now() + Duration::from_secs(60);
+            loop {
+                let delete_result = timeout(
+                    Duration::from_secs(30),
+                    clients[0].delete_object().bucket(bucket).key(&rejected_key).send(),
+                )
+                .await;
+                match delete_result {
+                    Ok(Ok(_)) => break,
+                    Ok(Err(error)) if is_service_unavailable_delete(&error) && Instant::now() < delete_deadline => {
+                        sleep(Duration::from_secs(1)).await;
+                    }
+                    Ok(Err(error)) => return Err(error.into()),
+                    Err(error) => return Err(error.into()),
+                }
+            }
+        }

        let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
        let recovery_deadline = Instant::now() + Duration::from_secs(60);
@@ -1995,9 +2122,9 @@ mod tests {
            assert_eq!(
                outage_census
                    .erasure_index
-                    .filter(|index| missing_outage_erasure_indices.contains(index)),
+                    .filter(|index| replacement_outage_erasure_index == Some(*index)),
                outage_census.erasure_index,
-                "the outage object must be rebuilt into one of the stopped node's missing erasure slots"
+                "the outage object must be rebuilt into the selected replacement drive's erasure-set slot"
            );
        }

@@ -2277,4 +2404,28 @@ mod tests {

        Ok(())
    }
+
+    #[test]
+    fn outage_candidate_must_target_replacement_erasure_index() {
+        let distribution = vec![4, 7, 10, 1, 5, 8, 11, 2, 6, 9, 12, 3];
+        let replacement_set_slot = 8;
+        let replacement_index = distribution[replacement_set_slot];
+        let peers_missing_replacement = OutagePeerManifest {
+            erasure_indices: (1..=12).filter(|index| *index != replacement_index).collect(),
+            erasure_distribution: Some(distribution.clone()),
+        };
+        assert!(outage_candidate_targets_replacement(&peers_missing_replacement, 12, replacement_set_slot));
+
+        let peers_missing_other_slot = OutagePeerManifest {
+            erasure_indices: (1..=12).filter(|index| *index != 9).collect(),
+            erasure_distribution: Some(distribution),
+        };
+        assert!(!outage_candidate_targets_replacement(&peers_missing_other_slot, 12, replacement_set_slot));
+
+        let insufficient_peer_shards = OutagePeerManifest {
+            erasure_indices: [1, 3, 4, 5, 6, 7, 8].into_iter().collect(),
+            erasure_distribution: Some(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]),
+        };
+        assert!(!outage_candidate_targets_replacement(&insufficient_peer_shards, 12, replacement_set_slot));
+    }
 }
@@ -3280,6 +3280,65 @@ mod heal_result_report_tests {
        );
    }

+    #[tokio::test]
+    async fn deep_heal_rebuilds_missing_part_when_metadata_remains_current() {
+        let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+        let bucket = "deep-heal-missing-part-current-meta";
+        let object = "object.bin";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let payload = vec![0x7b; 1024 * 1024];
+        let mut reader = PutObjReader::from_vec(payload);
+        set.put_object(
+            bucket,
+            object,
+            &mut reader,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("source object should be written before shard loss");
+        let source = disks[2]
+            .read_version("", bucket, object, "", &ReadOptions::default())
+            .await
+            .expect("source metadata should be readable");
+        let data_dir = source.data_dir.expect("non-inline source should have a data directory");
+        let missing_part = temp_dirs[1]
+            .path()
+            .join(bucket)
+            .join(object)
+            .join(data_dir.to_string())
+            .join("part.1");
+        tokio::fs::remove_file(&missing_part)
+            .await
+            .expect("target shard should be removed while xl.meta remains current");
+
+        let (result, error) = set
+            .heal_object(
+                bucket,
+                object,
+                "",
+                &HealOpts {
+                    no_lock: true,
+                    scan_mode: HealScanMode::Deep,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("deep heal should finish after a single shard is removed");
+
+        assert!(error.is_none(), "deep heal should recover the missing shard: {error:?}");
+        assert_eq!(result.after.drives[1].state, DriveState::Ok.to_string());
+        assert!(
+            missing_part.exists(),
+            "deep heal must reconstruct the missing shard on the original disk slot"
+        );
+    }
+
    #[tokio::test]
    async fn replacement_target_readback_checks_the_requested_historical_version() {
        let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
@@ -346,7 +346,7 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
    admin(HttpMethod::Post, "/rustfs/admin/v3/rebalance/stop", REBALANCE, RouteRiskLevel::High),
    admin(HttpMethod::Post, "/rustfs/admin/v3/heal/", HEAL, RouteRiskLevel::High),
    admin(HttpMethod::Post, "/rustfs/admin/v3/heal/{bucket}", HEAL, RouteRiskLevel::High),
-    admin(HttpMethod::Post, "/rustfs/admin/v3/heal/{bucket}/{prefix}", HEAL, RouteRiskLevel::High),
+    admin(HttpMethod::Post, "/rustfs/admin/v3/heal/{bucket}/{*prefix}", HEAL, RouteRiskLevel::High),
    admin(HttpMethod::Post, "/rustfs/admin/v3/background-heal/status", HEAL, RouteRiskLevel::High),
    admin(
        HttpMethod::Get,
@@ -83,9 +83,12 @@ runtime_profile_for() {
        background-target-crash|background-target-restart)
            echo "background-4x1"
            ;;
-        background-target-crash-ec8-4|background-target-restart-ec8-4|background-target-restart-ec8-4-multi-set)
+        background-target-crash-ec8-4|background-target-restart-ec8-4)
            echo "background-ec8-4"
            ;;
+        background-target-restart-ec8-4-multi-set)
+            echo "background-ec8-4-multi-set"
+            ;;
        background-target-crash-ec8-4-multi-pool)
            echo "background-ec8-4-multi-pool"
            ;;
@@ -111,6 +114,11 @@ apply_runtime_profile() {
            export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-8388608}"
            export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-180}"
            ;;
+        background-ec8-4-multi-set)
+            export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}"
+            export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-16777216}"
+            export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-240}"
+            ;;
        background-ec8-4-multi-pool)
            export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}"
            export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-4194304}"
@@ -319,9 +327,25 @@ if [[ "$NOFILE_SOFT" =~ ^[0-9]+$ && "$NOFILE_HARD" =~ ^[0-9]+$ && "$NOFILE_SOFT"
        ulimit -n "$NOFILE_HARD" || true
    fi
 fi
-mkdir -p "$(dirname "$RUN_DIR")"
-TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-scanner-heal-evidence.XXXXXX")"
-trap 'rm -rf "$TMP_DIR"' EXIT
+RUN_PARENT="$(dirname "$RUN_DIR")"
+mkdir -p "$RUN_PARENT"
+RUN_TMP_ROOT_CREATED=0
+if [[ -z "${TMPDIR:-}" ]]; then
+    RUN_TMP_ROOT="$RUN_PARENT/.tmp-$(basename "$RUN_DIR")"
+    mkdir -p "$RUN_TMP_ROOT"
+    export TMPDIR="$RUN_TMP_ROOT"
+    RUN_TMP_ROOT_CREATED=1
+else
+    RUN_TMP_ROOT=""
+fi
+TMP_DIR="$(mktemp -d "${TMPDIR%/}/rustfs-scanner-heal-evidence.XXXXXX")"
+cleanup_tmp() {
+    rm -rf "$TMP_DIR"
+    if [[ "$RUN_TMP_ROOT_CREATED" == 1 ]]; then
+        rm -rf "$RUN_TMP_ROOT"
+    fi
+}
+trap cleanup_tmp EXIT

 BUILD_FEATURES="${RUSTFS_BUILD_FEATURES:-}"
 TARGET_DIR="${CARGO_TARGET_DIR:-$ROOT/target}"