diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 7c228595d..67be1ce57 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -779,16 +779,11 @@ impl SetDisks { .await { Ok(m) => { - let derr = if !version_id.is_empty() { - DiskError::FileVersionNotFound - } else { - DiskError::FileNotFound - }; let mut t_errs = Vec::with_capacity(errs.len()); for _ in 0..errs.len() { t_errs.push(None); } - Ok((self.default_heal_result(m, &t_errs, bucket, object, version_id).await, Some(derr))) + Ok((self.default_heal_result(m, &t_errs, bucket, object, version_id).await, None)) } Err(err) => { error!( @@ -798,10 +793,9 @@ impl SetDisks { object, version_id, error = %err, - returned_error = %cannot_heal_err, "Heal object dangling cleanup could not prove object deletion" ); - Ok((result, Some(cannot_heal_err))) + Ok((result, Some(err))) } }; } @@ -1273,6 +1267,14 @@ impl SetDisks { return Box::pin(self.heal_object_with_explicit_version_regen(bucket, object, version_id, opts, false)).await; } + if opts.dry_run { + return Ok(( + self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id) + .await, + Some(err), + )); + } + if self .dangling_delete_safety(bucket, object, &parts_metadata, &errs, &disks) .await? @@ -1300,18 +1302,11 @@ impl SetDisks { ) .await { - Ok(m) => { - let err = if !version_id.is_empty() { - DiskError::FileVersionNotFound - } else { - DiskError::FileNotFound - }; - Ok((self.default_heal_result(m, &errs, bucket, object, version_id).await, Some(err))) - } - Err(_) => Ok(( + Ok(m) => Ok((self.default_heal_result(m, &errs, bucket, object, version_id).await, None)), + Err(cleanup_err) => Ok(( self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id) .await, - Some(err), + Some(cleanup_err), )), } } @@ -2123,7 +2118,7 @@ mod heal_result_report_tests { use crate::disk::endpoint::Endpoint; use crate::disk::error::DiskError; use crate::disk::format::FormatV3; - use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, new_disk}; + use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, STORAGE_FORMAT_FILE, new_disk}; use crate::error::Error; use crate::object_api::{ObjectOptions, PutObjReader}; use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated; @@ -2134,6 +2129,7 @@ mod heal_result_report_tests { config::storageclass, store::init_format::{load_format_erasure, save_format_file}, }; + use bytes::Bytes; use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode}; use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo, ObjectPartInfo, TRANSITION_COMPLETE}; use std::sync::{Arc, Mutex}; @@ -2394,6 +2390,20 @@ mod heal_result_report_tests { .expect("test metadata should be written"); } + async fn dangling_inline_test_fixture( + bucket: &str, + object: &str, + mod_time: OffsetDateTime, + ) -> (Vec, Arc, Vec>) { + let (temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[]).await; + let mut metadata = meta_regen_test_fileinfo(object, Uuid::nil(), mod_time.unix_timestamp(), 0); + metadata.data_dir = None; + metadata.set_inline_data(); + metadata.data = Some(Bytes::from_static(b"stale-inline-shard")); + seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata).await; + (temp_dirs, set, disks) + } + async fn formatted_single_disk_no_parity_set() -> (TempDir, Arc) { let format = FormatV3::new(1, 1); let dir = tempfile::tempdir().expect("tempdir should be created"); @@ -3068,6 +3078,109 @@ mod heal_result_report_tests { ); } + #[tokio::test] + #[serial_test::serial] + async fn heal_reports_success_after_dangling_inline_cleanup() { + temp_env::async_with_vars([("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS", Some("0"))], async { + let bucket = "bucket-dangling-inline-cleanup"; + let object = "stale.txt"; + let (temp_dirs, set, _disks) = + dangling_inline_test_fixture(bucket, object, OffsetDateTime::now_utc() - time::Duration::hours(2)).await; + + let (_, error) = set + .heal_object( + bucket, + object, + "", + &HealOpts { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("dangling cleanup should complete"); + + assert!(error.is_none(), "successful dangling cleanup must not be reported as FileNotFound"); + assert!( + temp_dirs.iter().all(|dir| !dir.path().join(bucket).join(object).exists()), + "successful dangling cleanup must remove the stale object from every disk" + ); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn heal_preserves_recent_dangling_inline_metadata_as_retryable() { + temp_env::async_with_vars([("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS", Some("3600"))], async { + let bucket = "bucket-dangling-inline-grace"; + let object = "recent.txt"; + let (temp_dirs, set, _disks) = dangling_inline_test_fixture(bucket, object, OffsetDateTime::now_utc()).await; + + let (_, error) = set + .heal_object( + bucket, + object, + "", + &HealOpts { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("grace-protected dangling metadata should return a typed heal result"); + + assert_eq!(error, Some(DiskError::ErasureReadQuorum)); + assert!( + temp_dirs[0] + .path() + .join(bucket) + .join(object) + .join(STORAGE_FORMAT_FILE) + .is_file(), + "grace-protected metadata must remain for a later heal pass" + ); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn dry_run_never_deletes_dangling_inline_metadata() { + temp_env::async_with_vars([("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS", Some("0"))], async { + let bucket = "bucket-dangling-inline-dry-run"; + let object = "stale.txt"; + let (temp_dirs, set, _disks) = + dangling_inline_test_fixture(bucket, object, OffsetDateTime::now_utc() - time::Duration::hours(2)).await; + + let (_, error) = set + .heal_object( + bucket, + object, + "", + &HealOpts { + dry_run: true, + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("dry-run should report the dangling candidate without mutation"); + + assert_eq!(error, Some(DiskError::FileNotFound)); + assert!( + temp_dirs[0] + .path() + .join(bucket) + .join(object) + .join(STORAGE_FORMAT_FILE) + .is_file(), + "dry-run must preserve dangling metadata even when grace is disabled" + ); + }) + .await; + } + #[tokio::test] async fn heal_meta_quorum_failure_preserves_candidate_when_required_shard_disk_is_offline() { let bucket = "bucket-delete-guard-offline"; diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index 736aede0b..364bb87da 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -168,6 +168,60 @@ impl ECStore { } } + /// Return every live erasure set selected by an object-heal scope. + pub async fn heal_erasure_set_scopes(&self, opts: &HealOpts) -> Result> { + let pools = self.get_pools_for_heal_object(opts)?; + let pool_meta = self.pool_meta.read().await; + let mut scopes = Vec::new(); + + for pool in pools { + let suspended_complete = pool_meta.is_suspended(pool.pool_idx).then(|| { + pool_meta + .pools + .get(pool.pool_idx) + .and_then(|status| status.decommission.as_ref()) + .is_some_and(|decommission| decommission.complete) + }); + if let Some(complete) = suspended_complete { + if opts.pool.is_some() { + return Err(if complete { + StorageError::InvalidArgument( + "heal".to_string(), + "pool".to_string(), + format!("heal pool {} has completed decommission", pool.pool_idx), + ) + } else { + Error::SlowDown + }); + } + continue; + } + + if let Some(set_idx) = opts.set { + if set_idx >= pool.disk_set.len() { + return Err(StorageError::InvalidArgument( + "heal".to_string(), + "set".to_string(), + format!( + "invalid heal set index {set_idx} for pool {} with {} sets", + pool.pool_idx, + pool.disk_set.len() + ), + )); + } + scopes.push((pool.pool_idx, set_idx)); + } else { + scopes.extend((0..pool.disk_set.len()).map(|set_idx| (pool.pool_idx, set_idx))); + } + } + + if scopes.is_empty() { + return Err(Error::SlowDown); + } + + Ok(scopes) + } + #[instrument(skip(self))] pub(super) async fn handle_heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { let mut r = HealResultItem { @@ -642,6 +696,40 @@ mod tests { } } + #[tokio::test] + async fn heal_erasure_set_scopes_follow_requested_pool_and_set() { + let store = minimal_heal_store().await; + + assert_eq!( + store + .heal_erasure_set_scopes(&HealOpts::default()) + .await + .expect("unscoped heal should enumerate every live set"), + vec![(0, 0), (1, 0)] + ); + assert_eq!( + store + .heal_erasure_set_scopes(&HealOpts { + pool: Some(1), + set: Some(0), + ..Default::default() + }) + .await + .expect("scoped heal should enumerate only its requested set"), + vec![(1, 0)] + ); + + let err = store + .heal_erasure_set_scopes(&HealOpts { + pool: Some(0), + set: Some(1), + ..Default::default() + }) + .await + .expect_err("an invalid set scope must fail closed"); + assert!(matches!(err, Error::InvalidArgument(..))); + } + fn pool_meta_with_decommission(info: PoolDecommissionInfo) -> PoolMeta { PoolMeta { pools: vec![PoolStatus { diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index a3e7c0cdf..8c2c5fc2f 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -428,6 +428,16 @@ pub trait HealStorageAPI: Send + Sync { include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)>; + /// Return the live erasure sets selected by this heal request. + /// + /// Recursive admin heals use these scopes with the cross-disk union walk so + /// objects surviving on only one returning disk are still discovered. The + /// `None` default preserves the read-quorum listing for alternate backends; + /// `Some(Vec::new())` means the selected topology currently has no live set. + async fn heal_erasure_set_scopes(&self, _opts: &HealOpts) -> Result>> { + Ok(None) + } + /// List versions for healing via a per-erasure-set DISK-WALK union enumerator /// (backlog#920). Unlike `list_objects_for_heal_page` (which reflects only the /// READ-QUORUM metadata view via `list_object_versions`), this surfaces every @@ -1294,6 +1304,14 @@ impl HealStorageAPI for ECStoreHealStorage { Ok((page_objects, next_token, list_info.is_truncated)) } + async fn heal_erasure_set_scopes(&self, opts: &HealOpts) -> Result>> { + self.ecstore + .heal_erasure_set_scopes(opts) + .await + .map(Some) + .map_err(Error::Storage) + } + async fn list_versions_for_heal_page_disk_walk( &self, set_disk_id: &str, diff --git a/crates/heal/src/heal/task/heal_bucket.rs b/crates/heal/src/heal/task/heal_bucket.rs index 8a4a28476..6237d246f 100644 --- a/crates/heal/src/heal/task/heal_bucket.rs +++ b/crates/heal/src/heal/task/heal_bucket.rs @@ -14,6 +14,7 @@ /// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline use super::*; use crate::heal::progress::{add_bytes, increment_counter, stable_generation}; +use crate::heal::utils::format_set_disk_id; impl HealTask { pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> { @@ -242,7 +243,6 @@ impl HealTask { #[hotpath::measure] async fn heal_bucket_objects(&self, bucket: &str, prefix: &str) -> Result<()> { - let mut continuation_token: Option = None; let mut scanned = 0u64; let mut healed = 0u64; let mut failed = 0u64; @@ -266,108 +266,101 @@ impl HealTask { set: self.options.set_index, }; - loop { - self.check_control_flags().await?; - let (objects, next_token, is_truncated) = self - .await_with_control( - self.storage - .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false), - ) - .await?; + let erasure_set_scopes = self + .await_with_control(self.storage.heal_erasure_set_scopes(&heal_opts)) + .await?; + let listing_scopes = match erasure_set_scopes { + None => vec![(None, heal_opts)], + Some(erasure_set_scopes) => erasure_set_scopes + .into_iter() + .map(|(pool_idx, set_idx)| { + let mut scoped_opts = heal_opts; + scoped_opts.pool = Some(pool_idx); + scoped_opts.set = Some(set_idx); + (Some(format_set_disk_id(pool_idx, set_idx)), scoped_opts) + }) + .collect(), + }; - let mut pending = objects; - let mut retry_attempt = 0_u32; - while !pending.is_empty() { - if retry_attempt > 0 { - self.await_with_control(async { - tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await; - Ok(()) - }) - .await?; - } - let mut retry = Vec::with_capacity(pending.len()); - for item in pending { - self.check_control_flags().await?; - let mut telemetry_unknown = false; - let object = item.name.as_str(); - { - let mut progress = self.progress.write().await; - progress.set_current_object(Some(format!("{bucket}/{object}"))); + for (set_disk_id, heal_opts) in listing_scopes { + let mut continuation_token: Option = None; + loop { + self.check_control_flags().await?; + let (objects, next_token, is_truncated) = if let Some(set_disk_id) = set_disk_id.as_deref() { + self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk( + set_disk_id, + bucket, + prefix, + continuation_token.as_deref(), + false, + )) + .await? + } else { + self.await_with_control(self.storage.list_objects_for_heal_page( + bucket, + prefix, + continuation_token.as_deref(), + false, + )) + .await? + }; + + let mut pending = objects; + let mut retry_attempt = 0_u32; + while !pending.is_empty() { + if retry_attempt > 0 { + self.await_with_control(async { + tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await; + Ok(()) + }) + .await?; } - - let mut terminal_outcome = true; - let error = match self - .await_with_control( - self.storage - .heal_object(bucket, object, item.version_id.as_deref(), &heal_opts), - ) - .await - { - Ok((result, None)) => { - telemetry_unknown |= !increment_counter(&mut healed); - telemetry_unknown |= !add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX)); - self.record_result_item(result).await; - None + let mut retry = Vec::with_capacity(pending.len()); + for item in pending { + self.check_control_flags().await?; + let mut telemetry_unknown = false; + let object = item.name.as_str(); + { + let mut progress = self.progress.write().await; + progress.set_current_object(Some(format!("{bucket}/{object}"))); } - Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => { - telemetry_unknown |= !increment_counter(&mut healed); - debug!( - target: "rustfs::heal::task", - event = EVENT_HEAL_BUCKET_RESULT, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - bucket, - object, - result = "object_dir_not_found_skipped", - "Heal bucket object-dir candidate skipped after not-found result" - ); - None - } - Ok((_, Some(err))) | Err(err) => Some(err), - }; - if let Some(err) = error { - if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) { - telemetry_unknown |= !increment_counter(&mut skipped); - warn!( - target: "rustfs::heal::task", - event = EVENT_HEAL_BUCKET_RESULT, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - bucket, - object, - result = "transient_skip", - error = %err, - "Heal bucket object repair skipped due to transient metadata error" - ); - } else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES { - terminal_outcome = false; - debug!( - target: "rustfs::heal::task", - event = EVENT_HEAL_BUCKET_RESULT, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - bucket, - object, - retry_attempt = retry_attempt.saturating_add(1), - error = %err, - result = "object_retry_scheduled", - "Heal bucket object retry scheduled" - ); - retry.push(item); - } else { - telemetry_unknown |= !increment_counter(&mut failed); - if err.is_recoverable_heal() { - retryable_failed = retryable_failed.saturating_add(1); - } else { - permanent_failed = permanent_failed.saturating_add(1); + let mut terminal_outcome = true; + let error = match self + .await_with_control( + self.storage + .heal_object(bucket, object, item.version_id.as_deref(), &heal_opts), + ) + .await + { + Ok((result, None)) => { + telemetry_unknown |= !increment_counter(&mut healed); + telemetry_unknown |= + !add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX)); + self.record_result_item(result).await; + None } - first_failed_object.get_or_insert_with(|| object.to_string()); - first_error.get_or_insert_with(|| err.to_string()); - if take_failure_log_sample(&mut failure_samples_logged) { + Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => { + telemetry_unknown |= !increment_counter(&mut healed); + debug!( + target: "rustfs::heal::task", + event = EVENT_HEAL_BUCKET_RESULT, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_TASK, + task_id = %self.id, + bucket, + object, + result = "object_dir_not_found_skipped", + "Heal bucket object-dir candidate skipped after not-found result" + ); + None + } + Ok((_, Some(err))) | Err(err) => Some(err), + }; + + if let Some(err) = error { + if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) { + telemetry_unknown |= !increment_counter(&mut skipped); warn!( target: "rustfs::heal::task", event = EVENT_HEAL_BUCKET_RESULT, @@ -376,41 +369,80 @@ impl HealTask { task_id = %self.id, bucket, object, - retry_attempt, + result = "transient_skip", error = %err, - result = "object_failed", - "Heal bucket object repair failed" + "Heal bucket object repair skipped due to transient metadata error" ); + } else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES { + terminal_outcome = false; + debug!( + target: "rustfs::heal::task", + event = EVENT_HEAL_BUCKET_RESULT, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_TASK, + task_id = %self.id, + bucket, + object, + retry_attempt = retry_attempt.saturating_add(1), + error = %err, + result = "object_retry_scheduled", + "Heal bucket object retry scheduled" + ); + retry.push(item); + } else { + telemetry_unknown |= !increment_counter(&mut failed); + if err.is_recoverable_heal() { + retryable_failed = retryable_failed.saturating_add(1); + } else { + permanent_failed = permanent_failed.saturating_add(1); + } + first_failed_object.get_or_insert_with(|| object.to_string()); + first_error.get_or_insert_with(|| err.to_string()); + if take_failure_log_sample(&mut failure_samples_logged) { + warn!( + target: "rustfs::heal::task", + event = EVENT_HEAL_BUCKET_RESULT, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_TASK, + task_id = %self.id, + bucket, + object, + retry_attempt, + error = %err, + result = "object_failed", + "Heal bucket object repair failed" + ); + } } } - } - if terminal_outcome { - telemetry_unknown |= !increment_counter(&mut scanned); - } + if terminal_outcome { + telemetry_unknown |= !increment_counter(&mut scanned); + } - if !terminal_outcome { - continue; - } + if !terminal_outcome { + continue; + } - let mut progress = self.progress.write().await; - progress.update_object_progress(scanned, healed, failed, skipped, bytes); - if telemetry_unknown { - progress.mark_unknown(); + let mut progress = self.progress.write().await; + progress.update_object_progress(scanned, healed, failed, skipped, bytes); + if telemetry_unknown { + progress.mark_unknown(); + } } + pending = retry; + retry_attempt = retry_attempt.saturating_add(1); } - pending = retry; - retry_attempt = retry_attempt.saturating_add(1); - } - if !is_truncated { - break; - } + if !is_truncated { + break; + } - continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?; - if continuation_token.is_none() { - // Truncated but no continuation token: end of listing. - break; + continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?; + if continuation_token.is_none() { + // Truncated but no continuation token: end of listing. + break; + } } } diff --git a/crates/heal/src/heal/task/tests.rs b/crates/heal/src/heal/task/tests.rs index 4d9635d01..bd2b23606 100644 --- a/crates/heal/src/heal/task/tests.rs +++ b/crates/heal/src/heal/task/tests.rs @@ -564,6 +564,8 @@ struct MockStorage { replacement_resume_disk: Mutex>, usage_baseline: Mutex>, usage_baseline_error: Mutex, + erasure_set_scopes: Mutex>, + disk_walk_calls: Mutex>, } #[test] @@ -946,6 +948,26 @@ impl HealStorageAPI for MockStorage { } } + async fn heal_erasure_set_scopes(&self, _opts: &HealOpts) -> Result>> { + let scopes = self.erasure_set_scopes.lock().unwrap().clone(); + Ok((!scopes.is_empty()).then_some(scopes)) + } + + async fn list_versions_for_heal_page_disk_walk( + &self, + set_disk_id: &str, + _bucket: &str, + _prefix: &str, + continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, + ) -> Result<(Vec, Option, bool)> { + self.disk_walk_calls.lock().unwrap().push(set_disk_id.to_string()); + if continuation_token.is_some() { + return Ok((Vec::new(), None, false)); + } + Ok((vec![heal_item(&format!("{set_disk_id}-object"))], None, false)) + } + async fn get_disk_for_resume(&self, _set_disk_id: &str) -> Result { self.resume_disk .lock() @@ -1081,6 +1103,47 @@ async fn test_recursive_bucket_heal_visits_objects() { assert_eq!(result_items.iter().filter(|item| item.object_size == 1).count(), 2); } +#[tokio::test] +async fn recursive_bucket_heal_uses_each_erasure_set_union_scope() { + let storage = Arc::new(MockStorage { + erasure_set_scopes: Mutex::new(vec![(0, 0), (1, 2)]), + ..Default::default() + }); + let request = HealRequest::new( + HealType::Bucket { + bucket: "bucket-a".to_string(), + }, + HealOptions { + recursive: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ); + let task = HealTask::from_request(request, storage.clone()); + + task.heal_bucket("bucket-a") + .await + .expect("recursive bucket heal should consume every selected set union"); + + assert_eq!( + storage.disk_walk_calls.lock().unwrap().as_slice(), + ["pool_0_set_0".to_string(), "pool_1_set_2".to_string()] + ); + assert_eq!( + storage.healed_objects.lock().unwrap().as_slice(), + ["pool_0_set_0-object".to_string(), "pool_1_set_2-object".to_string()] + ); + assert!( + !*storage.listed.lock().unwrap(), + "read-quorum listing must not hide returning-disk candidates" + ); + let object_opts = storage.object_heal_opts.lock().unwrap(); + assert_eq!(object_opts.len(), 2); + assert_eq!((object_opts[0].pool, object_opts[0].set), (Some(0), Some(0))); + assert_eq!((object_opts[1].pool, object_opts[1].set), (Some(1), Some(2))); +} + #[tokio::test] async fn result_items_are_bounded_and_report_truncation() { let storage = Arc::new(MockStorage::default()); diff --git a/crates/heal/tests/heal_b920_subquorum_union_test.rs b/crates/heal/tests/heal_b920_subquorum_union_test.rs index b38a90620..eabda8e11 100644 --- a/crates/heal/tests/heal_b920_subquorum_union_test.rs +++ b/crates/heal/tests/heal_b920_subquorum_union_test.rs @@ -354,9 +354,9 @@ mod serial_tests { with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts())) .await .expect("heal_object call must not itself error"); - // A dangling delete reports the version as gone (FileVersionNotFound), - // proving the destructive path fired for a genuinely torn write. - assert!(error.is_some(), "a torn (< data_blocks) version must NOT be silently treated as healed"); + // Successful cleanup is a successful heal outcome. The on-disk checks + // below prove that the torn version was deleted rather than healed. + assert!(error.is_none(), "successful torn-version cleanup must not report a heal error"); // Destructive-path PROOF: the stale minority copy on disk0 was purged by // the dangling delete (the guard correctly did NOT rescue a torn write). assert_eq!(