diff --git a/.config/nextest.toml b/.config/nextest.toml index 2e54a7180..804888924 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -539,6 +539,13 @@ path = "junit.xml" filter = 'package(e2e_test)' test-group = 'e2e-cluster-nightly' +# The EC8+4 multi-set heal proof deliberately uploads a larger workload so the +# background root-heal pass can be interrupted after targeting the replacement +# drive's erasure slots. Keep the extended budget scoped to this proof case. +[[profile.e2e-nightly.overrides]] +filter = 'package(e2e_test) & test(=heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart)' +slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" } + # --------------------------------------------------------------------------- # e2e-distributed profile — 4-node 4-disk Actions suite # --------------------------------------------------------------------------- diff --git a/Cargo.lock b/Cargo.lock index c00244594..5b6e0e875 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -271,7 +271,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -282,7 +282,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -688,9 +688,9 @@ dependencies = [ [[package]] name = "async-compat" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ba85bc55464dcbf728b56d97e119d673f4cf9062be330a9a26f3acf504a590" +checksum = "4c97d7ff3c25d6c10d64170c12acaf5d4245e76dece3779c1d92b153a64f11df" dependencies = [ "futures-core", "futures-io", @@ -1906,7 +1906,7 @@ dependencies = [ "maybe-owned", "rustix", "rustix-linux-procfs", - "windows-sys 0.60.2", + "windows-sys 0.61.2", "winx", ] @@ -3942,7 +3942,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4287,7 +4287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5155,9 +5155,9 @@ dependencies = [ [[package]] name = "hickory-net" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084e7bd6a377435d568f652153e571b50970d7ccc1d1eeec0519f834632287e1" +checksum = "c480823ed7c2c5d0f09c41020cb6b7c28029ce60ec42dc942158dcf22f8e0a4d" dependencies = [ "async-trait", "cfg-if", @@ -5179,9 +5179,9 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e2da0694c15b44c6f68a6b05e0233617008c54080e31d6eb848d858a9c5b38d" +checksum = "12b92608f679a6fa515dd1d15c1ff89443026e391200a2c840c7afcba482893d" dependencies = [ "data-encoding", "idna", @@ -5199,9 +5199,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4f9f4603319422d482738f3f6fe5aac03157fdbfed1cd85a3ff45adb09072f" +checksum = "3f3da5255c95d5a716857d54b5b8f4e8d67c3484d3beaaaae2ce25063b3ba981" dependencies = [ "cfg-if", "futures-util", @@ -6963,7 +6963,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8712,7 +8712,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -11065,7 +11065,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -11148,7 +11148,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -11949,7 +11949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -12100,7 +12100,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -12405,10 +12405,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -12761,9 +12761,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.13+spec-1.1.0" +version = "0.25.14+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +checksum = "d2195eec204e2764644a4ea619704f9fbe5e0673038eded55ad9956f24fca0cc" dependencies = [ "indexmap 2.14.2", "toml_datetime", @@ -13510,7 +13510,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/e2e_test/src/chaos.rs b/crates/e2e_test/src/chaos.rs index f0b2971e9..0fb8b924b 100644 --- a/crates/e2e_test/src/chaos.rs +++ b/crates/e2e_test/src/chaos.rs @@ -61,6 +61,7 @@ pub(crate) struct VersionShardCensus { pub has_xl_meta: bool, pub data_dir: Option, pub erasure_index: Option, + pub erasure_distribution: Option>, pub data_blocks: Option, pub parity_blocks: Option, pub expected_part_numbers: BTreeSet, @@ -90,6 +91,7 @@ impl VersionShardCensus { && manifest.is_complete() && self.data_dir == manifest.data_dir && self.erasure_index == manifest.erasure_index + && self.erasure_distribution == manifest.erasure_distribution && self.data_blocks == manifest.data_blocks && self.parity_blocks == manifest.parity_blocks && self.expected_part_numbers == manifest.expected_part_numbers @@ -317,6 +319,7 @@ pub(crate) fn census_object_version_on_disk( has_xl_meta: false, data_dir: None, erasure_index: None, + erasure_distribution: None, data_blocks: None, parity_blocks: None, expected_part_numbers: BTreeSet::new(), @@ -334,6 +337,7 @@ pub(crate) fn census_object_version_on_disk( }; let data_dir = file_info.data_dir.map(|id| id.to_string()); let erasure_index = Some(file_info.erasure.index); + let erasure_distribution = Some(file_info.erasure.distribution.clone()); let inline_data_fingerprint = file_info.data.as_deref().map(shard_fingerprint).transpose()?; let part_dir = data_dir.as_ref().map_or_else(|| object_dir.clone(), |id| object_dir.join(id)); let present_part_fingerprints = match std::fs::read_dir(&part_dir) { @@ -366,6 +370,7 @@ pub(crate) fn census_object_version_on_disk( has_xl_meta: true, data_dir, erasure_index, + erasure_distribution, data_blocks: Some(file_info.erasure.data_blocks), parity_blocks: Some(file_info.erasure.parity_blocks), expected_part_numbers, @@ -421,6 +426,7 @@ mod tests { has_xl_meta: true, data_dir: Some("data-dir".to_string()), erasure_index: Some(3), + erasure_distribution: Some(vec![1, 2, 3, 4]), data_blocks: Some(2), parity_blocks: Some(2), expected_part_numbers: BTreeSet::from([1]), diff --git a/crates/e2e_test/src/distributed/heal_test.rs b/crates/e2e_test/src/distributed/heal_test.rs index 3da4eb559..07d7cb29f 100644 --- a/crates/e2e_test/src/distributed/heal_test.rs +++ b/crates/e2e_test/src/distributed/heal_test.rs @@ -253,6 +253,7 @@ async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult, expected_pool_metadata: Option, @@ -461,6 +462,12 @@ mod tests { shard_census: VersionShardCensus, } + #[derive(Debug, Default)] + struct OutagePeerManifest { + erasure_indices: HashSet, + erasure_distribution: Option>, + } + fn deterministic_object_body(len: usize, seed: u8) -> Vec { let mut value = seed; std::iter::repeat_with(|| { @@ -486,6 +493,81 @@ mod tests { Ok(matching) } + fn collect_outage_peer_manifest( + cluster: &RustFSTestClusterEnvironment, + offline_node_index: usize, + bucket: &str, + key: &str, + erasure_set_drive_count: usize, + ) -> Result> { + let mut manifest = OutagePeerManifest::default(); + for (node_index, node) in cluster.nodes.iter().enumerate() { + if node_index == offline_node_index { + continue; + } + for (drive_index, drive) in node.data_dirs.iter().enumerate() { + let census = census_object_version_on_disk(Path::new(drive), bucket, key, None)?; + if !census.has_xl_meta { + continue; + } + assert!( + census.is_complete(), + "online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}" + ); + let erasure_index = census.erasure_index.ok_or_else(|| { + format!("online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}") + })?; + assert!( + (1..=erasure_set_drive_count).contains(&erasure_index), + "online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}" + ); + let distribution = census.erasure_distribution.as_ref().ok_or_else(|| { + format!( + "online node {node_index} drive {drive_index} outage-object shard has no erasure distribution: {census:?}" + ) + })?; + assert_eq!( + distribution.len(), + erasure_set_drive_count, + "online node {node_index} drive {drive_index} outage-object distribution must match the erasure set: {census:?}" + ); + match &manifest.erasure_distribution { + Some(existing) => { + assert_eq!(existing, distribution, "outage-object shards must agree on one erasure distribution") + } + None => manifest.erasure_distribution = Some(distribution.clone()), + } + assert!( + manifest.erasure_indices.insert(erasure_index), + "outage-object erasure index {erasure_index} is duplicated across online drives" + ); + } + } + Ok(manifest) + } + + fn outage_candidate_replacement_erasure_index( + peer_manifest: &OutagePeerManifest, + erasure_set_drive_count: usize, + replacement_set_slot: usize, + ) -> Option { + let distribution = peer_manifest.erasure_distribution.as_ref()?; + distribution.get(replacement_set_slot).copied().filter(|replacement_index| { + (1..=erasure_set_drive_count).contains(replacement_index) + && !peer_manifest.erasure_indices.contains(replacement_index) + }) + } + + fn outage_candidate_targets_replacement( + peer_manifest: &OutagePeerManifest, + erasure_set_drive_count: usize, + replacement_set_slot: usize, + ) -> bool { + let min_online_data_shards = erasure_set_drive_count.saturating_sub(4); + peer_manifest.erasure_indices.len() >= min_online_data_shards + && outage_candidate_replacement_erasure_index(peer_manifest, erasure_set_drive_count, replacement_set_slot).is_some() + } + fn metadata_count(disk: &Path, bucket: &str, expected_manifests: &[PhysicalObjectManifest]) -> usize { expected_manifests .iter() @@ -601,6 +683,10 @@ mod tests { error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable") } + fn is_service_unavailable_delete(error: &SdkError) -> bool { + error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable") + } + fn select_replacement_drive( cluster: &RustFSTestClusterEnvironment, node_index: usize, @@ -612,7 +698,7 @@ mod tests { .ok_or_else(|| format!("replacement node {node_index} is absent"))?; let mut incomplete_pool_metadata = Vec::new(); - for drive in &node.data_dirs { + for (drive_index, drive) in node.data_dirs.iter().enumerate() { let replaced_disk = PathBuf::from(drive); let replacement_format_path = replaced_disk.join(".rustfs.sys").join("format.json"); let replacement_format = std::fs::read(&replacement_format_path).map_err(|err| { @@ -621,6 +707,7 @@ mod tests { if !require_pool_metadata { return Ok(ReplacementDriveSelection { replaced_disk, + drive_index, replacement_format_path, replacement_format, expected_pool_metadata: None, @@ -631,6 +718,7 @@ mod tests { if census.is_complete() { return Ok(ReplacementDriveSelection { replaced_disk, + drive_index, replacement_format_path, replacement_format, expected_pool_metadata: Some(census), @@ -1180,7 +1268,7 @@ mod tests { async fn test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart() -> Result<(), Box> { timeout( - Duration::from_secs(600), + Duration::from_secs(900), run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestartEc84MultiSet), ) .await? @@ -1332,10 +1420,13 @@ mod tests { let ReplacementDriveSelection { replaced_disk, + drive_index: replacement_drive_index, replacement_format_path, replacement_format, expected_pool_metadata, } = select_replacement_drive(&cluster, 1, background_enabled)?; + let replacement_global_drive_index = topology.drives_per_node + replacement_drive_index; + let replacement_set_slot = replacement_global_drive_index % erasure_set_drive_count; let default_online_object_count = if !outage_target_manifest_required { 64 } else { 24 }; let online_object_count = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_COUNT") .ok() @@ -1391,6 +1482,23 @@ mod tests { expected_manifests.push(PhysicalObjectManifest { key, shard_census }); attempt_count += 1; } + for manifest in &expected_manifests { + let distribution = manifest + .shard_census + .erasure_distribution + .as_ref() + .ok_or_else(|| format!("replacement baseline manifest has no erasure distribution: {manifest:?}"))?; + assert_eq!( + distribution.len(), + erasure_set_drive_count, + "replacement baseline distribution must match the erasure set: {manifest:?}" + ); + assert_eq!( + manifest.shard_census.erasure_index, + distribution.get(replacement_set_slot).copied(), + "replacement baseline shard must match the selected drive's erasure-set slot" + ); + } if background_enabled { wait_for_scanner_cycle_after(&cluster, 0).await?; @@ -1415,6 +1523,9 @@ mod tests { let mut outage_write_deferred_until_rejoin = false; let mut service_unavailable_outage_writes = 0usize; let mut last_service_unavailable = None; + let mut outage_peer_manifest = OutagePeerManifest::default(); + let mut replacement_outage_erasure_index = None; + let mut rejected_outage_keys = Vec::new(); for attempt in 0..max_outage_write_attempts { let candidate_key = format!("cluster/written-while-node-down-{attempt:04}.bin"); let put_result = timeout( @@ -1429,6 +1540,24 @@ mod tests { .await; match put_result { Ok(Ok(_)) => { + if outage_target_manifest_required { + let candidate_peer_manifest = + collect_outage_peer_manifest(&cluster, 1, bucket, &candidate_key, erasure_set_drive_count)?; + if !outage_candidate_targets_replacement( + &candidate_peer_manifest, + erasure_set_drive_count, + replacement_set_slot, + ) { + rejected_outage_keys.push(candidate_key); + continue; + } + replacement_outage_erasure_index = outage_candidate_replacement_erasure_index( + &candidate_peer_manifest, + erasure_set_drive_count, + replacement_set_slot, + ); + outage_peer_manifest = candidate_peer_manifest; + } outage_key = Some(candidate_key); break; } @@ -1456,57 +1585,37 @@ mod tests { } }; - let mut outage_peer_erasure_indices = HashSet::new(); - if !outage_write_deferred_until_rejoin { - for (node_index, node) in cluster.nodes.iter().enumerate() { - if node_index == 1 { - continue; - } - for (drive_index, drive) in node.data_dirs.iter().enumerate() { - let census = census_object_version_on_disk(Path::new(drive), bucket, &outage_key, None)?; - if !census.has_xl_meta { - continue; - } - assert!( - census.is_complete(), - "online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}" - ); - let erasure_index = census.erasure_index.ok_or_else(|| { - format!( - "online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}" - ) - })?; - assert!( - (1..=erasure_set_drive_count).contains(&erasure_index), - "online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}" - ); - assert!( - outage_peer_erasure_indices.insert(erasure_index), - "outage-object erasure index {erasure_index} is duplicated across online drives" - ); - } - } + if !outage_write_deferred_until_rejoin && outage_peer_manifest.erasure_indices.is_empty() { + outage_peer_manifest = collect_outage_peer_manifest(&cluster, 1, bucket, &outage_key, erasure_set_drive_count)?; + replacement_outage_erasure_index = + outage_candidate_replacement_erasure_index(&outage_peer_manifest, erasure_set_drive_count, replacement_set_slot); } assert!( outage_write_deferred_until_rejoin - || (!outage_peer_erasure_indices.is_empty() && outage_peer_erasure_indices.len() <= erasure_set_drive_count), + || (!outage_peer_manifest.erasure_indices.is_empty() + && outage_peer_manifest.erasure_indices.len() <= erasure_set_drive_count), "outage-object must occupy one non-empty erasure set" ); if outage_target_manifest_required { let min_online_data_shards = erasure_set_drive_count.saturating_sub(4); assert!( - outage_peer_erasure_indices.len() >= min_online_data_shards, + outage_peer_manifest.erasure_indices.len() >= min_online_data_shards, "online drives in the selected erasure set must retain at least the EC data quorum" ); } let missing_outage_erasure_indices = (1..=erasure_set_drive_count) - .filter(|index| !outage_peer_erasure_indices.contains(index)) + .filter(|index| !outage_peer_manifest.erasure_indices.contains(index)) .collect::>(); if outage_target_manifest_required { assert!( !missing_outage_erasure_indices.is_empty(), "the stopped target must account for at least one missing outage-object erasure index" ); + assert_eq!( + replacement_outage_erasure_index.filter(|index| missing_outage_erasure_indices.contains(index)), + replacement_outage_erasure_index, + "the outage object must target the selected replacement drive's erasure-set slot" + ); } let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#; @@ -1531,6 +1640,24 @@ mod tests { } cluster.start_node_from_binary(1, &server_binary).await?; + for rejected_key in rejected_outage_keys { + let delete_deadline = Instant::now() + Duration::from_secs(60); + loop { + let delete_result = timeout( + Duration::from_secs(30), + clients[0].delete_object().bucket(bucket).key(&rejected_key).send(), + ) + .await; + match delete_result { + Ok(Ok(_)) => break, + Ok(Err(error)) if is_service_unavailable_delete(&error) && Instant::now() < delete_deadline => { + sleep(Duration::from_secs(1)).await; + } + Ok(Err(error)) => return Err(error.into()), + Err(error) => return Err(error.into()), + } + } + } let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url); let recovery_deadline = Instant::now() + Duration::from_secs(60); @@ -1995,9 +2122,9 @@ mod tests { assert_eq!( outage_census .erasure_index - .filter(|index| missing_outage_erasure_indices.contains(index)), + .filter(|index| replacement_outage_erasure_index == Some(*index)), outage_census.erasure_index, - "the outage object must be rebuilt into one of the stopped node's missing erasure slots" + "the outage object must be rebuilt into the selected replacement drive's erasure-set slot" ); } @@ -2277,4 +2404,28 @@ mod tests { Ok(()) } + + #[test] + fn outage_candidate_must_target_replacement_erasure_index() { + let distribution = vec![4, 7, 10, 1, 5, 8, 11, 2, 6, 9, 12, 3]; + let replacement_set_slot = 8; + let replacement_index = distribution[replacement_set_slot]; + let peers_missing_replacement = OutagePeerManifest { + erasure_indices: (1..=12).filter(|index| *index != replacement_index).collect(), + erasure_distribution: Some(distribution.clone()), + }; + assert!(outage_candidate_targets_replacement(&peers_missing_replacement, 12, replacement_set_slot)); + + let peers_missing_other_slot = OutagePeerManifest { + erasure_indices: (1..=12).filter(|index| *index != 9).collect(), + erasure_distribution: Some(distribution), + }; + assert!(!outage_candidate_targets_replacement(&peers_missing_other_slot, 12, replacement_set_slot)); + + let insufficient_peer_shards = OutagePeerManifest { + erasure_indices: [1, 3, 4, 5, 6, 7, 8].into_iter().collect(), + erasure_distribution: Some(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]), + }; + assert!(!outage_candidate_targets_replacement(&insufficient_peer_shards, 12, replacement_set_slot)); + } } diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index a983ad8b6..4f934b458 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -3280,6 +3280,65 @@ mod heal_result_report_tests { ); } + #[tokio::test] + async fn deep_heal_rebuilds_missing_part_when_metadata_remains_current() { + let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "deep-heal-missing-part-current-meta"; + let object = "object.bin"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let payload = vec![0x7b; 1024 * 1024]; + let mut reader = PutObjReader::from_vec(payload); + set.put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("source object should be written before shard loss"); + let source = disks[2] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("source metadata should be readable"); + let data_dir = source.data_dir.expect("non-inline source should have a data directory"); + let missing_part = temp_dirs[1] + .path() + .join(bucket) + .join(object) + .join(data_dir.to_string()) + .join("part.1"); + tokio::fs::remove_file(&missing_part) + .await + .expect("target shard should be removed while xl.meta remains current"); + + let (result, error) = set + .heal_object( + bucket, + object, + "", + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("deep heal should finish after a single shard is removed"); + + assert!(error.is_none(), "deep heal should recover the missing shard: {error:?}"); + assert_eq!(result.after.drives[1].state, DriveState::Ok.to_string()); + assert!( + missing_part.exists(), + "deep heal must reconstruct the missing shard on the original disk slot" + ); + } + #[tokio::test] async fn replacement_target_readback_checks_the_requested_historical_version() { let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 7a6137c21..f53366d6d 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -346,7 +346,7 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ admin(HttpMethod::Post, "/rustfs/admin/v3/rebalance/stop", REBALANCE, RouteRiskLevel::High), admin(HttpMethod::Post, "/rustfs/admin/v3/heal/", HEAL, RouteRiskLevel::High), admin(HttpMethod::Post, "/rustfs/admin/v3/heal/{bucket}", HEAL, RouteRiskLevel::High), - admin(HttpMethod::Post, "/rustfs/admin/v3/heal/{bucket}/{prefix}", HEAL, RouteRiskLevel::High), + admin(HttpMethod::Post, "/rustfs/admin/v3/heal/{bucket}/{*prefix}", HEAL, RouteRiskLevel::High), admin(HttpMethod::Post, "/rustfs/admin/v3/background-heal/status", HEAL, RouteRiskLevel::High), admin( HttpMethod::Get, diff --git a/scripts/run_scanner_heal_evidence_case.sh b/scripts/run_scanner_heal_evidence_case.sh index a351c01af..a8bc5042c 100755 --- a/scripts/run_scanner_heal_evidence_case.sh +++ b/scripts/run_scanner_heal_evidence_case.sh @@ -83,9 +83,12 @@ runtime_profile_for() { background-target-crash|background-target-restart) echo "background-4x1" ;; - background-target-crash-ec8-4|background-target-restart-ec8-4|background-target-restart-ec8-4-multi-set) + background-target-crash-ec8-4|background-target-restart-ec8-4) echo "background-ec8-4" ;; + background-target-restart-ec8-4-multi-set) + echo "background-ec8-4-multi-set" + ;; background-target-crash-ec8-4-multi-pool) echo "background-ec8-4-multi-pool" ;; @@ -111,6 +114,11 @@ apply_runtime_profile() { export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-8388608}" export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-180}" ;; + background-ec8-4-multi-set) + export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}" + export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-16777216}" + export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-240}" + ;; background-ec8-4-multi-pool) export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}" export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-4194304}" @@ -319,9 +327,25 @@ if [[ "$NOFILE_SOFT" =~ ^[0-9]+$ && "$NOFILE_HARD" =~ ^[0-9]+$ && "$NOFILE_SOFT" ulimit -n "$NOFILE_HARD" || true fi fi -mkdir -p "$(dirname "$RUN_DIR")" -TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-scanner-heal-evidence.XXXXXX")" -trap 'rm -rf "$TMP_DIR"' EXIT +RUN_PARENT="$(dirname "$RUN_DIR")" +mkdir -p "$RUN_PARENT" +RUN_TMP_ROOT_CREATED=0 +if [[ -z "${TMPDIR:-}" ]]; then + RUN_TMP_ROOT="$RUN_PARENT/.tmp-$(basename "$RUN_DIR")" + mkdir -p "$RUN_TMP_ROOT" + export TMPDIR="$RUN_TMP_ROOT" + RUN_TMP_ROOT_CREATED=1 +else + RUN_TMP_ROOT="" +fi +TMP_DIR="$(mktemp -d "${TMPDIR%/}/rustfs-scanner-heal-evidence.XXXXXX")" +cleanup_tmp() { + rm -rf "$TMP_DIR" + if [[ "$RUN_TMP_ROOT_CREATED" == 1 ]]; then + rm -rf "$RUN_TMP_ROOT" + fi +} +trap cleanup_tmp EXIT BUILD_FEATURES="${RUSTFS_BUILD_FEATURES:-}" TARGET_DIR="${CARGO_TARGET_DIR:-$ROOT/target}"