diff --git a/crates/e2e_test/src/chaos.rs b/crates/e2e_test/src/chaos.rs index c61ff5bc5..85c5b226d 100644 --- a/crates/e2e_test/src/chaos.rs +++ b/crates/e2e_test/src/chaos.rs @@ -40,6 +40,7 @@ use http::header::{CONTENT_TYPE, HOST}; use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; +use std::collections::BTreeSet; use std::error::Error; use std::path::{Path, PathBuf}; use tracing::info; @@ -48,6 +49,34 @@ use walkdir::WalkDir; type ChaosResult = Result>; +/// Physical `xl.meta` and shard-file census for one object version on one disk. +/// +/// A successful S3 GET only proves that a quorum can serve an object. Replacement +/// tests need this lower-level record to prove that the rebuilt target holds the +/// `xl.meta` selected for a specific version and every `part.N` it declares. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct VersionShardCensus { + pub version_id: Option, + pub has_xl_meta: bool, + pub data_dir: Option, + pub expected_part_numbers: BTreeSet, + pub present_part_numbers: BTreeSet, +} + +impl VersionShardCensus { + pub(crate) fn is_complete(&self) -> bool { + self.has_xl_meta && self.expected_part_numbers == self.present_part_numbers + } + + pub(crate) fn matches_manifest(&self, manifest: &Self) -> bool { + self.version_id == manifest.version_id + && self.is_complete() + && manifest.is_complete() + && self.data_dir == manifest.data_dir + && self.expected_part_numbers == manifest.expected_part_numbers + } +} + /// Single-node RustFS server with `disk_count` local volume directories that /// can be faulted individually while the server is running. pub struct DiskFaultHarness { @@ -219,6 +248,78 @@ impl DiskFaultHarness { pub fn object_metadata_exists_on_disk(&self, disk_index: usize, bucket: &str, key: &str) -> bool { self.disks[disk_index].join(bucket).join(key).join("xl.meta").is_file() } + + /// Census the physical files selected by `version_id` on one disk. + /// + /// Missing metadata and missing shard files are represented in the returned + /// census rather than as an error so callers can poll replacement progress. + /// Invalid metadata or an unknown requested version remains an error: treating + /// either as an incomplete rebuild would hide corruption or a wrong-version + /// recovery result. + pub(crate) fn census_object_version( + &self, + disk_index: usize, + bucket: &str, + key: &str, + version_id: Option<&str>, + ) -> ChaosResult { + census_object_version_on_disk(&self.disks[disk_index], bucket, key, version_id) + } +} + +/// Census one physical object version without requiring a single-node harness. +/// Cluster replacement tests use the same evidence as the disk-fault tests. +pub(crate) fn census_object_version_on_disk( + disk: &Path, + bucket: &str, + key: &str, + version_id: Option<&str>, +) -> ChaosResult { + let version_id = version_id.map(str::to_owned); + let object_dir = disk.join(bucket).join(key); + let meta_path = object_dir.join("xl.meta"); + if !meta_path.is_file() { + return Ok(VersionShardCensus { + version_id, + has_xl_meta: false, + data_dir: None, + expected_part_numbers: BTreeSet::new(), + present_part_numbers: BTreeSet::new(), + }); + } + + let metadata = rustfs_filemeta::FileMeta::load(&std::fs::read(&meta_path)?)?; + let file_info = metadata.into_fileinfo(bucket, key, version_id.as_deref().unwrap_or_default(), true, false, true)?; + let expected_part_numbers = if file_info.inline_data() { + BTreeSet::new() + } else { + file_info.parts.iter().map(|part| part.number).collect() + }; + let data_dir = file_info.data_dir.map(|id| id.to_string()); + let part_dir = data_dir.as_ref().map_or_else(|| object_dir.clone(), |id| object_dir.join(id)); + let present_part_numbers = match std::fs::read_dir(&part_dir) { + Ok(entries) => entries + .filter_map(Result::ok) + .filter_map(|entry| { + entry + .file_type() + .ok() + .filter(|kind| kind.is_file()) + .and_then(|_| entry.file_name().to_str().map(str::to_owned)) + }) + .filter_map(|name| name.strip_prefix("part.").and_then(|number| number.parse::().ok())) + .collect(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => BTreeSet::new(), + Err(error) => return Err(error.into()), + }; + + Ok(VersionShardCensus { + version_id, + has_xl_meta: true, + data_dir, + expected_part_numbers, + present_part_numbers, + }) } /// `POST` a signed (SigV4, service `s3`) admin request without relying on the diff --git a/crates/e2e_test/src/reliability_disk_fault_test.rs b/crates/e2e_test/src/reliability_disk_fault_test.rs index 9ff84ad90..1aa1a0d70 100644 --- a/crates/e2e_test/src/reliability_disk_fault_test.rs +++ b/crates/e2e_test/src/reliability_disk_fault_test.rs @@ -22,15 +22,16 @@ #[cfg(test)] mod tests { - use crate::chaos::{DiskFaultHarness, signed_admin_post}; + use crate::chaos::{DiskFaultHarness, VersionShardCensus, signed_admin_post}; use crate::common::init_logging; use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; - use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; + use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration}; use serial_test::serial; use sha2::{Digest, Sha256}; use std::collections::HashSet; - use tokio::time::{Duration, sleep, timeout}; + use std::error::Error; + use tokio::time::{Duration, Instant, interval, timeout}; use tracing::info; const GET_TIMEOUT: Duration = Duration::from_secs(60); @@ -271,12 +272,17 @@ mod tests { put_and_record(&client, bucket, "heal/nested/large.bin", payload(2 * 1024 * 1024, 34), &mut manifest).await?; verify_manifest(&client, bucket, &manifest, "baseline before disk replacement").await?; - for (key, _) in &manifest { - assert!( - harness.object_metadata_exists_on_disk(0, bucket, key), - "disk0 should hold xl.meta for {key} before replacement" - ); - } + let manifest_keys = manifest.iter().map(|(key, _)| key.clone()).collect::>(); + let target_manifest: Vec<(String, VersionShardCensus)> = manifest_keys + .iter() + .map(|key| { + let census = harness.census_object_version(0, bucket, key, None)?; + if !census.is_complete() { + return Err(format!("disk 0 has incomplete physical census for {key}: {census:?}").into()); + } + Ok((key.clone(), census)) + }) + .collect::>>()?; harness.kill_server(); harness.replace_disk_with_empty(0)?; @@ -287,21 +293,112 @@ mod tests { signed_admin_post(&heal_url, Some(heal_body), &harness.env.access_key, &harness.env.secret_key).await?; let client = harness.env.create_s3_client(); - let mut remaining: HashSet = manifest.iter().map(|(key, _)| key.clone()).collect(); + let mut remaining: HashSet = manifest_keys.iter().cloned().collect(); let heal_timeout_secs = std::env::var("RUSTFS_RELIABILITY_HEAL_TIMEOUT_SECS") .ok() .and_then(|value| value.parse::().ok()) .unwrap_or(120); + let deadline = Instant::now() + Duration::from_secs(heal_timeout_secs); + let mut retry = interval(Duration::from_secs(1)); - for _ in 0..heal_timeout_secs { - remaining.retain(|key| !harness.object_metadata_exists_on_disk(0, bucket, key)); + loop { + remaining.retain(|key| { + let expected = target_manifest + .iter() + .find(|(manifest_key, _)| manifest_key == key) + .map(|(_, manifest)| manifest) + .expect("every key has a physical manifest"); + harness + .census_object_version(0, bucket, key, None) + .map(|census| !census.matches_manifest(expected)) + .unwrap_or(true) + }); if remaining.is_empty() { verify_manifest(&client, bucket, &manifest, "after fresh-disk heal completed").await?; return Ok(()); } - sleep(Duration::from_secs(1)).await; + if Instant::now() >= deadline { + break; + } + retry.tick().await; } Err(format!("fresh-disk heal did not rebuild {remaining:?} on the replaced disk within {heal_timeout_secs}s").into()) } + + #[tokio::test] + #[serial] + async fn test_versioned_shard_census_selects_each_version_data_dir() -> Result<(), Box> { + init_logging(); + info!("Reliability: physical shard census selects the requested object version"); + + let mut harness = DiskFaultHarness::new(4).await?; + harness.start_server().await?; + let client = harness.env.create_s3_client(); + let bucket = "reliability-versioned-census"; + let key = "versions/large.bin"; + client.create_bucket().bucket(bucket).send().await?; + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration( + VersioningConfiguration::builder() + .status(BucketVersioningStatus::Enabled) + .build(), + ) + .send() + .await?; + + let first = client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from(payload(256 * 1024, 41))) + .send() + .await?; + let first_version = first.version_id().ok_or("first PUT did not return a version ID")?; + let second = client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from(payload(256 * 1024, 42))) + .send() + .await?; + let second_version = second.version_id().ok_or("second PUT did not return a version ID")?; + let delete = client.delete_object().bucket(bucket).key(key).send().await?; + let delete_version = delete.version_id().ok_or("delete marker did not return a version ID")?; + + let first_census = harness.census_object_version(0, bucket, key, Some(first_version))?; + let second_census = harness.census_object_version(0, bucket, key, Some(second_version))?; + let delete_census = harness.census_object_version(0, bucket, key, Some(delete_version))?; + assert!( + first_census.is_complete(), + "first version physical census is incomplete: {first_census:?}" + ); + assert!( + second_census.is_complete(), + "second version physical census is incomplete: {second_census:?}" + ); + assert_ne!( + first_census.data_dir, second_census.data_dir, + "distinct object versions must select distinct physical data directories" + ); + assert_eq!( + first_census.expected_part_numbers, second_census.expected_part_numbers, + "same single-part shape should expose the same part numbers" + ); + assert!( + delete_census.is_complete(), + "delete marker physical census is incomplete: {delete_census:?}" + ); + assert!( + delete_census.expected_part_numbers.is_empty(), + "delete marker must not declare object shards: {delete_census:?}" + ); + assert!( + delete_census.present_part_numbers.is_empty(), + "delete marker must not select stale object shards: {delete_census:?}" + ); + Ok(()) + } } diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 714557732..9a63c7cd9 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -326,8 +326,8 @@ pub mod disk { pub use crate::disk::local::ScanGuard; pub use crate::disk::{ BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, - CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, - FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, + CheckPartsResp, ConditionalFileUpdate, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, + DiskStore, FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count, diff --git a/crates/ecstore/src/client/api_get_object_file.rs b/crates/ecstore/src/client/api_get_object_file.rs index 5811b9e9c..05694e274 100644 --- a/crates/ecstore/src/client/api_get_object_file.rs +++ b/crates/ecstore/src/client/api_get_object_file.rs @@ -57,7 +57,7 @@ fn build_part_path(file_path: &Path) -> PathBuf { async fn open_download_part_file(file_part_path: &Path) -> io::Result { let mut options = OpenOptions::new(); - options.create(true).read(true).write(true); + options.create(true).truncate(false).read(true).write(true); #[cfg(not(windows))] options.mode(0o600); diff --git a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs index bb0dd1c97..dd1822548 100644 --- a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs @@ -784,7 +784,11 @@ impl PeerS3Client for LocalPeerS3Client { if opts.force_if_empty && !opts.force { for disk in local_disks.iter() { - if has_xlmeta_files(&disk.path().join(bucket)).await.map_err(Error::Io)? { + let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else { + continue; + }; + let bucket_path = bucket_path?; + if has_xlmeta_files(&bucket_path).await.map_err(Error::Io)? { return Err(Error::VolumeNotEmpty); } } diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index c35df9db9..acb8b53b9 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -1058,17 +1058,23 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets { for (i, set) in new_format_sets.iter().enumerate() { for (j, fm) in set.iter().enumerate() { if let Some(fm) = fm { - res.after.drives[i * self.set_drive_count + j].uuid = fm.erasure.this.to_string(); - res.after.drives[i * self.set_drive_count + j].state = DriveState::Ok.to_string(); tmp_new_formats[i * self.set_drive_count + j] = Some(fm.clone()); } } } // Save new formats `format.json` on unformatted disks. - for (fm, disk) in tmp_new_formats.iter_mut().zip(disks.iter()) { - if fm.is_some() && disk.is_some() && save_format_file(disk, fm).await.is_err() { - let _ = disk.as_ref().unwrap().close().await; - *fm = None; + for (index, (fm, disk)) in tmp_new_formats.iter_mut().zip(disks.iter()).enumerate() { + if fm.is_some() && disk.is_some() { + if let Err(err) = save_format_file(disk, fm).await { + if let Some(disk) = disk.as_ref() { + let _ = disk.close().await; + } + return Ok((res, Some(err.into()))); + } + if let Some(saved_format) = fm.as_ref() { + res.after.drives[index].uuid = saved_format.erasure.this.to_string(); + res.after.drives[index].state = DriveState::Ok.to_string(); + } } } @@ -2206,6 +2212,39 @@ mod tests { ); } + #[tokio::test] + #[serial] + async fn replacement_format_only_writes_the_requested_slot() { + let (_dirs, _ref_format, sets) = setup_heal_format_sets(1, false).await; + let target = sets.endpoints.endpoints.as_ref()[1].to_string(); + let untouched = sets.endpoints.endpoints.as_ref()[2].to_string(); + let set = set_level_heal_view(&sets).await; + + let (result, error) = set + .heal_replacement_format(false, std::slice::from_ref(&target)) + .await + .expect("target-scoped replacement format should run"); + + assert!(error.is_none(), "target format must not report an error: {error:?}"); + assert!( + result + .after + .drives + .iter() + .any(|drive| drive.endpoint == target && drive.state == DriveState::Ok.to_string()), + "requested replacement slot must be formatted" + ); + let untouched_format = std::path::Path::new(&sets.endpoints.endpoints.as_ref()[2].get_file_path()) + .join(crate::disk::RUSTFS_META_BUCKET) + .join(crate::disk::FORMAT_CONFIG_FILE); + assert!( + !tokio::fs::try_exists(untouched_format) + .await + .expect("untouched replacement format path should be inspectable"), + "unrequested slot {untouched} must remain unformatted" + ); + } + fn instance_ctx_test_pool_endpoints() -> (FormatV3, PoolEndpoints) { let format = FormatV3::new(1, 2); let endpoints = vec![ diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 3cafd621b..fee70bc97 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -152,6 +152,7 @@ const DISK_OPERATION_NAMES: &[&str] = &[ "read_parts", "read_multiple", "write_all", + "compare_and_update_file", "read_all", ]; @@ -1092,6 +1093,18 @@ impl LocalDiskWrapper { self.disk.get_object_path(volume, path) } + pub(crate) fn get_object_path_for_io(&self, volume: &str, path: &str) -> crate::disk::error::Result { + self.disk.get_object_path_for_io(volume, path) + } + + pub(crate) fn get_bucket_path_for_io(&self, volume: &str) -> crate::disk::error::Result { + self.disk.get_bucket_path_for_io(volume) + } + + pub fn replacement_mount_lease_root(&self) -> Option { + self.disk.replacement_mount_lease_root() + } + pub fn runtime_state(&self) -> RuntimeDriveHealthState { self.health.runtime_state() } @@ -1639,6 +1652,10 @@ impl LocalDiskWrapper { #[async_trait::async_trait] impl DiskAPI for LocalDiskWrapper { + fn has_replacement_mount_lease(&self) -> bool { + self.disk.has_replacement_mount_lease() + } + async fn read_metadata(&self, volume: &str, path: &str) -> Result { self.track_disk_health_with_op_and_timeout_action( "read_metadata", @@ -2140,6 +2157,22 @@ impl DiskAPI for LocalDiskWrapper { .await } + async fn compare_and_update_file( + &self, + volume: &str, + path: &str, + expected: Option, + replacement: Option, + ) -> Result { + self.track_disk_health_mutation( + "compare_and_update_file", + DiskMetricMutation::Write, + || async { self.disk.compare_and_update_file(volume, path, expected, replacement).await }, + get_max_timeout_duration(), + ) + .await + } + async fn read_all(&self, volume: &str, path: &str) -> Result { self.track_disk_health_with_op( "read_all", diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 58632adde..303564039 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -15,11 +15,13 @@ use crate::config::storageclass::DEFAULT_INLINE_BLOCK; use crate::crash_inject::{self, CrashPoint}; use crate::data_usage::local_snapshot::ensure_data_usage_layout; +#[cfg(test)] +use crate::disk::HEALING_MARKER_PATH; use crate::disk::disk_store::{get_drive_walkdir_stall_timeout, get_object_disk_read_timeout}; use crate::disk::{ BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, - CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, - DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, + CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, ConditionalFileUpdate, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo, + DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, @@ -540,34 +542,26 @@ fn empty_tree_io_error(err: rustix::io::Errno) -> std::io::Error { } #[cfg(unix)] -fn remove_empty_directory_tree_unix_with( +fn remove_empty_directory_tree_unix_at( + root_parent: impl std::os::fd::AsFd, + root_name: &std::ffi::CStr, root: &Path, mut before_descend: impl FnMut(&Path) -> std::io::Result<()>, mut before_remove: impl FnMut(&Path) -> std::io::Result<()>, ) -> std::io::Result<()> { use rustix::{ - fs::{AtFlags, Dir, Mode, OFlags, fstat, open, openat, statat, unlinkat}, + fs::{AtFlags, Dir, Mode, OFlags, fstat, openat, statat, unlinkat}, io::Errno, }; - use std::os::fd::AsFd; use std::os::unix::ffi::OsStrExt; let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC; - let root_parent_path = root - .parent() - .ok_or_else(|| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?; - let root_name = root - .file_name() - .ok_or_else(|| std::io::Error::from(ErrorKind::DirectoryNotEmpty))? - .as_bytes(); - let root_name = std::ffi::CString::new(root_name).map_err(|_| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?; - let root_parent = open(root_parent_path, flags, Mode::empty()).map_err(empty_tree_io_error)?; - let root_fd = openat(&root_parent, root_name.as_c_str(), flags, Mode::empty()).map_err(empty_tree_io_error)?; + let root_fd = openat(&root_parent, root_name, flags, Mode::empty()).map_err(empty_tree_io_error)?; // Each frame owns one directory iterator/FD, so memory and descriptors are // bounded by path depth rather than by the number of empty remnants. let mut stack = vec![EmptyDirectoryFrame { path: root.to_path_buf(), - name_in_parent: root_name, + name_in_parent: root_name.to_owned(), entries: Dir::new(root_fd).map_err(empty_tree_io_error)?, }]; @@ -630,6 +624,42 @@ fn remove_empty_directory_tree_unix_with( Ok(()) } +#[cfg(unix)] +fn remove_empty_directory_tree_unix_with( + root: &Path, + before_descend: impl FnMut(&Path) -> std::io::Result<()>, + before_remove: impl FnMut(&Path) -> std::io::Result<()>, +) -> std::io::Result<()> { + use rustix::fs::{Mode, OFlags, open}; + use std::os::unix::ffi::OsStrExt; + + let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC; + let root_parent_path = root + .parent() + .ok_or_else(|| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?; + let root_name = root + .file_name() + .ok_or_else(|| std::io::Error::from(ErrorKind::DirectoryNotEmpty))? + .as_bytes(); + let root_name = std::ffi::CString::new(root_name).map_err(|_| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?; + let root_parent = open(root_parent_path, flags, Mode::empty()).map_err(empty_tree_io_error)?; + remove_empty_directory_tree_unix_at(&root_parent, root_name.as_c_str(), root, before_descend, before_remove) +} + +#[cfg(target_os = "linux")] +async fn remove_empty_directory_tree_under_mount_lease( + mount_lease: &std::fs::File, + volume: &str, + root: PathBuf, +) -> std::io::Result<()> { + let root_parent = mount_lease.try_clone()?; + let root_name = std::ffi::CString::new(volume.as_bytes()).map_err(|_| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?; + tokio::task::spawn_blocking(move || { + remove_empty_directory_tree_unix_at(&root_parent, root_name.as_c_str(), &root, |_| Ok(()), |_| Ok(())) + }) + .await? +} + #[cfg(unix)] async fn remove_empty_directory_tree_with( root: &Path, @@ -639,7 +669,7 @@ async fn remove_empty_directory_tree_with( remove_empty_directory_tree_unix_with(root, before_descend, before_remove) } -#[cfg(unix)] +#[cfg(all(unix, not(target_os = "linux")))] async fn remove_empty_directory_tree(root: &Path) -> std::io::Result<()> { let root = root.to_path_buf(); tokio::task::spawn_blocking(move || remove_empty_directory_tree_unix_with(&root, |_| Ok(()), |_| Ok(()))).await? @@ -2897,10 +2927,14 @@ impl StdBackend { } } + fn io_root(&self) -> &Path { + &self.root + } + async fn open_file(&self, path: impl AsRef, mode: usize, skip_parent: impl AsRef) -> Result { let mut skip_parent = skip_parent.as_ref(); if skip_parent.as_os_str().is_empty() { - skip_parent = self.root.as_path(); + skip_parent = self.io_root(); } if let Some(parent) = path.as_ref().parent() @@ -3307,7 +3341,7 @@ impl LocalIoBackend for StdBackend { debug!(reason = "non_unix_platform", "zero_copy_fallback"); let access_check_start = metrics_enabled.then(std::time::Instant::now); - let volume_dir = local_disk_bucket_path(&self.root, volume)?; + let volume_dir = local_disk_bucket_path(self.io_root(), volume)?; if !skip_access_checks(volume) { access(&volume_dir) .await @@ -3318,7 +3352,7 @@ impl LocalIoBackend for StdBackend { } let path_resolve_start = metrics_enabled.then(std::time::Instant::now); - let file_path = local_disk_object_path(&self.root, volume, path)?; + let file_path = local_disk_object_path(self.io_root(), volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; if let Some(metrics) = metrics { record_mmap_copy_stage(metrics, metrics.path_resolve_stage, path_resolve_start); @@ -3372,14 +3406,14 @@ impl LocalIoBackend for StdBackend { } async fn open_read_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result { - let volume_dir = local_disk_bucket_path(&self.root, volume)?; + let volume_dir = local_disk_bucket_path(self.io_root(), volume)?; if !skip_access_checks(volume) { access(&volume_dir) .await .map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?; } - let file_path = local_disk_object_path(&self.root, volume, path)?; + let file_path = local_disk_object_path(self.io_root(), volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; let mut f = self.open_file_read_only(file_path).await?; @@ -3412,14 +3446,14 @@ impl LocalIoBackend for StdBackend { } async fn open_full_read(&self, volume: &str, path: &str) -> Result { - let volume_dir = local_disk_bucket_path(&self.root, volume)?; + let volume_dir = local_disk_bucket_path(self.io_root(), volume)?; if !skip_access_checks(volume) { access(&volume_dir) .await .map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?; } - let file_path = local_disk_object_path(&self.root, volume, path)?; + let file_path = local_disk_object_path(self.io_root(), volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; let f = self.open_file_read_only(file_path).await?; @@ -3430,8 +3464,8 @@ impl LocalIoBackend for StdBackend { async fn open_write(&self, volume: &str, path: &str, mode: WriteMode) -> Result { match mode { WriteMode::Truncate { size_hint } => { - let volume_dir = local_disk_bucket_path(&self.root, volume)?; - let file_path = local_disk_object_path(&self.root, volume, path)?; + let volume_dir = local_disk_bucket_path(self.io_root(), volume)?; + let file_path = local_disk_object_path(self.io_root(), volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; if let Some(parent) = file_path.parent() { @@ -3466,14 +3500,14 @@ impl LocalIoBackend for StdBackend { Ok(Box::new(FileCacheReclaimWriter::new(f, size_hint.max(0) as usize, reclaim_on_shutdown))) } WriteMode::Append => { - let volume_dir = local_disk_bucket_path(&self.root, volume)?; + let volume_dir = local_disk_bucket_path(self.io_root(), volume)?; if !skip_access_checks(volume) { access(&volume_dir) .await .map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?; } - let file_path = local_disk_object_path(&self.root, volume, path)?; + let file_path = local_disk_object_path(self.io_root(), volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; let f = self.open_file(file_path, O_CREATE | O_APPEND | O_WRONLY, volume_dir).await?; @@ -4471,7 +4505,19 @@ fn build_local_io_backend(root: PathBuf) -> Arc { pub struct LocalDisk { pub root: PathBuf, publication_root: os::PublicationRoot, + /// I/O root pinned to the mount instance that was opened while the disk + /// was initialized. On Linux this is `/proc/self/fd//.`; resolving + /// paths beneath it keeps repair I/O on that mount even if the configured + /// pathname is later covered by another mount. + io_root: PathBuf, + #[cfg(target_os = "linux")] + mount_lease: std::fs::File, + #[cfg(target_os = "linux")] + mount_lease_mount_id: Option, + /// Public path for callers that need the configured disk layout. Internal + /// disk I/O uses `io_format_path`, which is rooted at `mount_lease`. pub format_path: PathBuf, + io_format_path: PathBuf, pub format_info: RwLock, pub endpoint: Endpoint, pub disk_info_cache: Arc>, @@ -4583,7 +4629,143 @@ fn resolve_local_disk_root(ep_path: &str) -> Result { } } +#[cfg(target_os = "linux")] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ReplacementMountIdentity { + device: u64, + inode: u64, + mount_id: u64, +} + +#[cfg(target_os = "linux")] +fn replacement_mount_identity(metadata: &std::fs::Metadata, mount_id: u64) -> ReplacementMountIdentity { + use std::os::unix::fs::MetadataExt as _; + + ReplacementMountIdentity { + device: metadata.dev(), + inode: metadata.ino(), + mount_id, + } +} + +#[cfg(target_os = "linux")] +fn mount_id_for_fd(fd: &std::fs::File) -> Option { + use rustix::fs::{AtFlags, StatxFlags}; + use std::os::fd::AsRawFd as _; + + rustix::fs::statx(fd, "", AtFlags::EMPTY_PATH, StatxFlags::MNT_ID) + .ok() + .filter(|statx| StatxFlags::from_bits_retain(statx.stx_mask).contains(StatxFlags::MNT_ID)) + .map(|statx| statx.stx_mnt_id) + .or_else(|| mount_id_from_fdinfo(fd.as_raw_fd())) +} + +#[cfg(target_os = "linux")] +fn mount_id_for_path(path: &Path) -> Option { + use rustix::fs::{AtFlags, CWD, StatxFlags}; + + rustix::fs::statx(CWD, path, AtFlags::empty(), StatxFlags::MNT_ID) + .ok() + .filter(|statx| StatxFlags::from_bits_retain(statx.stx_mask).contains(StatxFlags::MNT_ID)) + .map(|statx| statx.stx_mnt_id) + .or_else(|| { + std::fs::canonicalize(path) + .ok() + .and_then(|path| mount_id_from_mountinfo(&path)) + }) +} + +#[cfg(target_os = "linux")] +fn mount_id_from_fdinfo(fd: std::os::fd::RawFd) -> Option { + std::fs::read_to_string(format!("/proc/self/fdinfo/{fd}")) + .ok()? + .lines() + .find_map(|line| line.strip_prefix("mnt_id:")?.trim().parse().ok()) +} + +#[cfg(target_os = "linux")] +fn mount_id_from_mountinfo(path: &Path) -> Option { + let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").ok()?; + mount_id_from_mountinfo_contents(&mountinfo, path) +} + +#[cfg(target_os = "linux")] +fn mount_id_from_mountinfo_contents(mountinfo: &str, path: &Path) -> Option { + let mountpoint = path + .to_string_lossy() + .replace('\\', "\\134") + .replace('\t', "\\011") + .replace('\n', "\\012") + .replace(' ', "\\040"); + mountinfo.lines().find_map(|line| { + let mut fields = line.split_whitespace(); + let mount_id = fields.next()?.parse().ok()?; + fields.next()?; + fields.next()?; + fields.next()?; + (fields.next()? == mountpoint).then_some(mount_id) + }) +} + impl LocalDisk { + #[cfg(target_os = "linux")] + fn open_mount_lease(root: &Path) -> Result<(std::fs::File, PathBuf, Option)> { + use rustix::fs::{Mode, OFlags, open}; + use std::os::fd::AsRawFd as _; + + let fd = open( + root, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(std::io::Error::from) + .map_err(DiskError::from)?; + let lease = std::fs::File::from(fd); + let io_root = PathBuf::from(format!("/proc/self/fd/{}/.", lease.as_raw_fd())); + let mount_id = mount_id_for_fd(&lease); + Ok((lease, io_root, mount_id)) + } + + #[cfg(not(target_os = "linux"))] + fn open_mount_lease(root: &Path) -> Result { + Ok(root.to_path_buf()) + } + + fn io_root(&self) -> &Path { + &self.io_root + } + + /// Auto-replacement is destructive, so it is admitted only when the + /// configured endpoint still names the directory held by `mount_lease`. + pub fn has_replacement_mount_lease(&self) -> bool { + #[cfg(target_os = "linux")] + { + let Ok(configured) = std::fs::metadata(self.endpoint.get_file_path()) else { + return false; + }; + let Ok(pinned) = self.mount_lease.metadata() else { + return false; + }; + let Some(configured_mount_id) = mount_id_for_path(Path::new(&self.endpoint.get_file_path())) else { + return false; + }; + let Some(pinned_mount_id) = self.mount_lease_mount_id else { + return false; + }; + replacement_mount_identity(&configured, configured_mount_id) == replacement_mount_identity(&pinned, pinned_mount_id) + } + + #[cfg(not(target_os = "linux"))] + false + } + + /// Return the descriptor-rooted path for the mount instance admitted for + /// automatic replacement. Callers must not derive destructive identity + /// from the mutable endpoint pathname. + pub fn replacement_mount_lease_root(&self) -> Option { + self.has_replacement_mount_lease().then(|| self.io_root.clone()) + } + pub async fn new(ep: &Endpoint, cleanup: bool) -> Result { debug!( event = EVENT_DISK_LOCAL_STARTUP_CLEANUP, @@ -4619,7 +4801,12 @@ impl LocalDisk { // cannot split ordinary IO from handle-relative publication. let root = publication_root.path().to_path_buf(); - ensure_data_usage_layout(&root) + #[cfg(target_os = "linux")] + let (mount_lease, io_root, mount_lease_mount_id) = Self::open_mount_lease(&root)?; + #[cfg(not(target_os = "linux"))] + let io_root = Self::open_mount_lease(&root)?; + + ensure_data_usage_layout(&io_root) .await .map_err(DiskError::from) .inspect_err(|err| { @@ -4631,7 +4818,7 @@ impl LocalDisk { if cleanup && let Err(err) = Self::cleanup_tmp_on_startup( - &root, + &io_root, &publication_root, startup_cleanup_ready.clone(), startup_cleanup_notify.clone(), @@ -4653,6 +4840,7 @@ impl LocalDisk { // Use optimized path resolution instead of absolutize_virtually let format_path = root.join(RUSTFS_META_BUCKET).join(super::FORMAT_CONFIG_FILE); + let io_format_path = io_root.join(RUSTFS_META_BUCKET).join(super::FORMAT_CONFIG_FILE); debug!( event = EVENT_DISK_LOCAL_STARTUP_CLEANUP, component = LOG_COMPONENT_ECSTORE, @@ -4662,8 +4850,8 @@ impl LocalDisk { state = "format_path_resolved", "Local disk format path resolved" ); - let (format_data, format_meta) = read_file_exists(&format_path).await.inspect_err(|err| { - log_startup_disk_error("read_format_json", &format_path, err); + let (format_data, format_meta) = read_file_exists(&io_format_path).await.inspect_err(|err| { + log_startup_disk_error("read_format_json", &io_format_path, err); })?; let mut id = None; @@ -4753,8 +4941,14 @@ impl LocalDisk { let mut disk = Self { root: root.clone(), publication_root, + io_root: io_root.clone(), + #[cfg(target_os = "linux")] + mount_lease, + #[cfg(target_os = "linux")] + mount_lease_mount_id, endpoint: ep.clone(), format_path, + io_format_path, format_info: RwLock::new(format_info), disk_info_cache: Arc::new(cache), scanning: Arc::new(AtomicU32::new(0)), @@ -4772,7 +4966,7 @@ impl LocalDisk { startup_cleanup_ready, startup_cleanup_notify, exit_signal: None, - io_backend: build_local_io_backend(root.clone()), + io_backend: build_local_io_backend(io_root.clone()), file_sync_permits: os::disk_file_sync_limiter(&root), snapshot_leases: Arc::new(Mutex::new(SnapshotLeaseRegistry::default())), }; @@ -4802,9 +4996,9 @@ impl LocalDisk { let (exit_tx, exit_rx) = tokio::sync::broadcast::channel(1); disk.exit_signal = Some(exit_tx); - let root = disk.root.clone(); + let io_root = disk.io_root.clone(); let publication_root = disk.publication_root.clone(); - tokio::spawn(Self::cleanup_deleted_objects_loop(root, publication_root, exit_rx)); + tokio::spawn(Self::cleanup_deleted_objects_loop(io_root, publication_root, exit_rx)); debug!( event = EVENT_DISK_LOCAL_STARTUP_CLEANUP, component = LOG_COMPONENT_ECSTORE, @@ -5050,7 +5244,7 @@ impl LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn check_format_json(&self) -> Result { - let md = fs::metadata(&self.format_path).await.map_err(to_unformatted_disk_error)?; + let md = fs::metadata(&self.io_format_path).await.map_err(to_unformatted_disk_error)?; Ok(md) } async fn make_meta_volumes(&self) -> Result<()> { @@ -5070,8 +5264,12 @@ impl LocalDisk { self.make_volumes(defaults).await } - // Optimized path resolution with caching + /// Resolve a caller-visible path beneath the configured disk root. pub fn resolve_abs_path(&self, path: impl AsRef) -> Result { + Ok(Self::resolve_abs_path_from(&self.root, path.as_ref())) + } + + fn io_resolve_abs_path(&self, path: impl AsRef) -> PathBuf { let path_ref = path.as_ref(); let path_str = path_ref.to_string_lossy(); @@ -5079,26 +5277,11 @@ impl LocalDisk { { let cache = self.path_cache.read(); if let Some(cached_path) = cache.get(path_str.as_ref()) { - return Ok(cached_path.clone()); + return cached_path.clone(); } } - // Calculate absolute path without using path_absolutize for better performance - let abs_path = if path_ref.is_absolute() { - path_ref.to_path_buf() - } else { - #[cfg(windows)] - { - self.root.join(path_str.replace('/', "\\")) - } - #[cfg(not(windows))] - { - self.root.join(path_ref) - } - }; - - // Normalize path components to avoid filesystem calls - let normalized = normalize_path_components(abs_path.as_path()); + let normalized = Self::resolve_abs_path_from(self.io_root(), path_ref); // Cache the result { @@ -5116,7 +5299,24 @@ impl LocalDisk { cache.insert(path_str.into_owned(), normalized.clone()); } - Ok(normalized) + normalized + } + + fn resolve_abs_path_from(root: &Path, path: &Path) -> PathBuf { + let abs_path = if path.is_absolute() { + path.to_path_buf() + } else { + #[cfg(windows)] + { + root.join(path.to_string_lossy().replace('/', "\\")) + } + #[cfg(not(windows))] + { + root.join(path) + } + }; + + normalize_path_components(abs_path) } // Get the absolute path of an object @@ -5129,13 +5329,29 @@ impl LocalDisk { local_disk_bucket_path(&self.root, bucket) } + pub(crate) fn get_object_path_for_io(&self, bucket: &str, key: &str) -> Result { + self.io_get_object_path(bucket, key) + } + + pub(crate) fn get_bucket_path_for_io(&self, bucket: &str) -> Result { + self.io_get_bucket_path(bucket) + } + + fn io_get_object_path(&self, bucket: &str, key: &str) -> Result { + local_disk_object_path(self.io_root(), bucket, key) + } + + fn io_get_bucket_path(&self, bucket: &str) -> Result { + local_disk_bucket_path(self.io_root(), bucket) + } + // Check if a path is valid fn check_valid_path>(&self, path: P) -> Result<()> { - check_local_disk_valid_path(&self.root, path) + check_local_disk_valid_path(self.io_root(), path) } fn reject_symlink_components(&self, path: &Path) -> Result<()> { - reject_local_disk_symlink_components(&self.root, path) + reject_local_disk_symlink_components(self.io_root(), path) } // Batch path generation with single lock acquisition @@ -5160,10 +5376,7 @@ impl LocalDisk { if !cache_misses.is_empty() { let mut new_entries = Vec::new(); for (i, _bucket, _key, cache_key) in cache_misses { - #[cfg(windows)] - let path = self.root.join(cache_key.replace('/', "\\")); - #[cfg(not(windows))] - let path = self.root.join(&cache_key); + let path = self.io_resolve_abs_path(&cache_key); results.push((i, path.clone())); new_entries.push((cache_key, path)); @@ -5186,7 +5399,7 @@ impl LocalDisk { // /// Write to the filesystem atomically. // /// This is done by first writing to a temporary location and then moving the file. // pub(crate) async fn prepare_file_write<'a>(&self, path: &'a PathBuf) -> Result> { - // let tmp_path = self.get_object_path(RUSTFS_META_TMP_BUCKET, Uuid::new_v4().to_string().as_str())?; + // let tmp_path = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, Uuid::new_v4().to_string().as_str())?; // debug!("prepare_file_write tmp_path:{:?}, path:{:?}", &tmp_path, &path); @@ -5211,7 +5424,7 @@ impl LocalDisk { // TODO: async notifications for disk space checks and trash cleanup - let trash_path = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; + let trash_path = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; // if let Some(parent) = trash_path.parent() { // if !parent.exists() { // fs::create_dir_all(parent).await?; @@ -5222,7 +5435,7 @@ impl LocalDisk { rename_all_ignore_missing_source( delete_path, trash_path, - self.get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?, + self.io_get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?, &self.publication_root, ) .await @@ -5240,11 +5453,11 @@ impl LocalDisk { }; if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR) { - let trash_path2 = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; + let trash_path2 = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; let _ = rename_all_ignore_missing_source( encode_dir_object(delete_path.to_string_lossy().as_ref()), trash_path2, - self.get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?, + self.io_get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?, &self.publication_root, ) .await; @@ -5272,14 +5485,14 @@ impl LocalDisk { } async fn delete_unleased(&self, volume: &str, path: &str, opt: &DeleteOptions) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; + let volume_dir = self.io_get_bucket_path(volume)?; if !skip_access_checks(volume) && let Err(e) = access(&volume_dir).await { return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); } - let file_path = self.get_object_path(volume, path)?; + let file_path = self.io_get_object_path(volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; self.delete_file(&volume_dir, &file_path, opt.recursive, opt.immediate) .await?; @@ -5461,9 +5674,9 @@ impl LocalDisk { } async fn read_listing_metadata(&self, volume: &str, object_name: &str) -> Result { - let object_dir = self.get_object_path(volume, object_name)?; + let object_dir = self.io_get_object_path(volume, object_name)?; let metadata_path = object_dir.join(STORAGE_FORMAT_FILE); - let volume_dir = self.get_bucket_path(volume)?; + let volume_dir = self.io_get_bucket_path(volume)?; let result = tokio::task::spawn_blocking(move || { let (bytes, _) = read_all_data_std(&metadata_path)?; let file_meta = FileMeta::load(&bytes).ok(); @@ -5565,8 +5778,8 @@ impl LocalDisk { } async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; - let xlpath = self.get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?; + let volume_dir = self.io_get_bucket_path(volume)?; + let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?; let object_dir = xlpath .parent() .ok_or_else(|| DiskError::other("missing object metadata parent"))?; @@ -5637,7 +5850,7 @@ impl LocalDisk { let vid = fi.version_id.unwrap_or_default(); let _ = fm.data.remove(vec![vid, dir]); - let dir_path = match self.get_object_path(volume, format!("{path}/{dir}").as_str()) { + let dir_path = match self.io_get_object_path(volume, format!("{path}/{dir}").as_str()) { Ok(dir_path) => dir_path, Err(err) => { if reserved_version_delete && let Some(rollback_dir) = rollback_dir { @@ -5888,12 +6101,12 @@ impl LocalDisk { } async fn write_all_meta(&self, volume: &str, path: &str, buf: &[u8], sync: bool) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; - let file_path = self.get_object_path(volume, path)?; + let volume_dir = self.io_get_bucket_path(volume)?; + let file_path = self.io_get_object_path(volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; - let tmp_volume_dir = self.get_bucket_path(RUSTFS_META_TMP_BUCKET)?; - let tmp_file_path = self.get_object_path(RUSTFS_META_TMP_BUCKET, Uuid::new_v4().to_string().as_str())?; + let tmp_volume_dir = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?; + let tmp_file_path = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, Uuid::new_v4().to_string().as_str())?; let durability = effective_durability(volume); @@ -5939,7 +6152,7 @@ impl LocalDisk { format_info.data.clone_from(&data); } - let volume_dir = self.get_bucket_path(volume)?; + let volume_dir = self.io_get_bucket_path(volume)?; // Files written here (format.json, ...) stay where they land — no // rename follows — so the new directory entry must be fsynced too. @@ -5958,7 +6171,7 @@ impl LocalDisk { // write_all_private with check_path_length #[tracing::instrument(level = "trace", skip_all)] async fn write_all_private(&self, volume: &str, path: &str, buf: Bytes, sync: SyncMode, skip_parent: &Path) -> Result<()> { - let file_path = self.get_object_path(volume, path)?; + let file_path = self.io_get_object_path(volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; self.write_all_internal(&file_path, InternalBuf::Owned(buf), sync, skip_parent) @@ -5977,7 +6190,7 @@ impl LocalDisk { skip_parent: &Path, ) -> Result<()> { let skip_parent = if skip_parent.as_os_str().is_empty() { - self.root.as_path() + self.io_root() } else { skip_parent }; @@ -6040,7 +6253,7 @@ impl LocalDisk { async fn open_file(&self, path: impl AsRef, mode: usize, skip_parent: impl AsRef) -> Result { let mut skip_parent = skip_parent.as_ref(); if skip_parent.as_os_str().is_empty() { - skip_parent = self.root.as_path(); + skip_parent = self.io_root(); } if let Some(parent) = path.as_ref().parent() @@ -6141,10 +6354,10 @@ impl LocalDisk { // `read_dir_entries_with_walk_stall` so wide prefixes can still be handled // as a single logical read in API semantics. let read_dir_started = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); - let dir_path_abs = self.get_object_path(&opts.bucket, current.trim_start_matches(SLASH_SEPARATOR))?; + let dir_path_abs = self.io_get_object_path(&opts.bucket, current.trim_start_matches(SLASH_SEPARATOR))?; let read_dir_result = match read_dir_entries_with_walk_stall(&dir_path_abs, -1, stall).await { Err(err) if err == Error::FileNotFound && !skip_access_checks(&opts.bucket) => { - let volume_dir = self.get_bucket_path(&opts.bucket)?; + let volume_dir = self.io_get_bucket_path(&opts.bucket)?; if let Err(access_err) = access(&volume_dir).await { Err(to_access_error(access_err, DiskError::VolumeAccessDenied).into()) } else { @@ -6457,7 +6670,7 @@ impl LocalDisk { // NOT an object, append to stack (with slash) // If dirObject, but no metadata (which is unexpected) we skip it. if !is_dir_obj - && !with_walk_stall_deadline(stall, is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?)) + && !with_walk_stall_deadline(stall, is_empty_dir(self.io_get_object_path(&opts.bucket, &meta.name)?)) .await? { meta.name.push_str(SLASH_SEPARATOR); @@ -6906,7 +7119,7 @@ fn normalize_path_components(path: impl AsRef) -> PathBuf { impl LocalDisk { async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result { let path = format!("{object}/{data_dir}"); - let data_path = self.get_object_path(volume, &path)?; + let data_path = self.io_get_object_path(volume, &path)?; match fs::metadata(&data_path).await { Ok(metadata) if metadata.is_dir() => {} Ok(_) => return Ok(false), @@ -6923,7 +7136,7 @@ impl LocalDisk { } async fn commit_reserved_version_delete(&self, volume: &str, object: &str, rollback_dir: Uuid) -> Result<()> { - let object_path = self.get_object_path(volume, object)?; + let object_path = self.io_get_object_path(volume, object)?; let mut entries = match fs::read_dir(object_path).await { Ok(entries) => entries, Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), @@ -6952,7 +7165,7 @@ impl LocalDisk { } async fn finish_version_delete(&self, volume: &str, object: &str, rollback_dir: Uuid) -> Result { - let object_path = self.get_object_path(volume, object)?; + let object_path = self.io_get_object_path(volume, object)?; let mut entries = match fs::read_dir(object_path).await { Ok(entries) => entries, Err(err) if err.kind() == ErrorKind::NotFound => return Ok(false), @@ -7207,7 +7420,7 @@ impl DiskAPI for LocalDisk { debug!("get_disk_id: read format.json"); - let b = fs::read(&self.format_path).await.map_err(to_unformatted_disk_error)?; + let b = fs::read(&self.io_format_path).await.map_err(to_unformatted_disk_error)?; let fm = FormatV3::try_from(b.as_slice()).map_err(|e| { warn!( @@ -7253,7 +7466,7 @@ impl DiskAPI for LocalDisk { } } - let p = self.get_object_path(volume, path)?; + let p = self.io_get_object_path(volume, path)?; let (data, _) = read_file_all(&p).await?; @@ -7266,6 +7479,98 @@ impl DiskAPI for LocalDisk { self.write_all_public(volume, path, data).await } + async fn compare_and_update_file( + &self, + volume: &str, + path: &str, + expected: Option, + replacement: Option, + ) -> Result { + #[cfg(unix)] + { + use rustix::fs::{FlockOperation, flock}; + use std::io::Write as _; + + let file_path = self.io_get_object_path(volume, path)?; + let lock_path = file_path.with_extension("rustfs-cas.lock"); + let path = path.to_string(); + let sync_metadata = effective_durability(volume).syncs_commit_metadata(); + return Ok(tokio::task::spawn_blocking(move || { + let lock = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(&lock_path)?; + flock(&lock, FlockOperation::NonBlockingLockExclusive).map_err(std::io::Error::from)?; + let result = (|| { + let current = match std::fs::read(&file_path) { + Ok(current) => Some(current), + Err(err) if err.kind() == ErrorKind::NotFound => None, + Err(err) => return Err(err), + }; + let matches = match (¤t, &expected) { + (None, None) => true, + (Some(current), Some(expected)) => current.as_slice() == expected.as_ref(), + _ => false, + }; + if !matches { + return Ok(match current { + None => ConditionalFileUpdate::Missing, + Some(_) => ConditionalFileUpdate::Mismatch, + }); + } + + match replacement { + Some(replacement) => { + let parent = file_path + .parent() + .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "conditional file has no parent"))?; + let temporary = parent.join(format!(".{}.{}.tmp", path.replace('/', "_"), Uuid::new_v4())); + let write_result = (|| -> std::io::Result<()> { + let mut staged = std::fs::OpenOptions::new().create_new(true).write(true).open(&temporary)?; + staged.write_all(&replacement)?; + if sync_metadata { + staged.sync_all()?; + } + std::fs::rename(&temporary, &file_path)?; + Ok(()) + })(); + if let Err(err) = write_result { + let _ = std::fs::remove_file(&temporary); + return Err(err); + } + if sync_metadata { + os::fsync_dir_std(parent)?; + } + } + None => { + std::fs::remove_file(&file_path)?; + if sync_metadata && let Some(parent) = file_path.parent() { + os::fsync_dir_std(parent)?; + } + } + } + Ok(ConditionalFileUpdate::Updated) + })(); + let _ = flock(&lock, FlockOperation::Unlock); + result + }) + .await + .map_err(DiskError::from)??); + } + + #[cfg(not(unix))] + { + let _ = (volume, path, expected, replacement); + Err(DiskError::MethodNotAllowed) + } + } + + fn has_replacement_mount_lease(&self) -> bool { + LocalDisk::has_replacement_mount_lease(self) + } + #[tracing::instrument(level = "trace", skip_all)] async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { crate::hp_guard!("LocalDisk::delete"); @@ -7286,7 +7591,7 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - let volume_dir = self.get_bucket_path(volume)?; + let volume_dir = self.io_get_bucket_path(volume)?; if !skip_access_checks(volume) && let Err(e) = access(&volume_dir).await { @@ -7313,7 +7618,7 @@ impl DiskAPI for LocalDisk { } else { checksum_info.algorithm }; - let part_path = self.get_object_path( + let part_path = self.io_get_object_path( volume, path_join_buf(&[ path, @@ -7366,7 +7671,7 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result> { - let volume_dir = self.get_bucket_path(bucket)?; + let volume_dir = self.io_get_bucket_path(bucket)?; let mut ret = vec![ObjectPartInfo::default(); paths.len()]; @@ -7380,7 +7685,7 @@ impl DiskAPI for LocalDisk { .unwrap_or_default(); if let Err(err) = access( - self.get_object_path( + self.io_get_object_path( bucket, path_join_buf(&[ path.parent().unwrap_or_else(|| Path::new("")).to_string_lossy().as_ref(), @@ -7400,7 +7705,11 @@ impl DiskAPI for LocalDisk { } let data = match self - .read_all_data(bucket, volume_dir.clone(), self.get_object_path(bucket, path.to_string_lossy().as_ref())?) + .read_all_data( + bucket, + volume_dir.clone(), + self.io_get_object_path(bucket, path.to_string_lossy().as_ref())?, + ) .await { Ok(data) => data, @@ -7433,15 +7742,15 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { let layout = fi.validate(ValidationMode::RequireErasure)?.ok_or(DiskError::FileCorrupt)?; - let volume_dir = self.get_bucket_path(volume)?; - let file_path = self.get_object_path(volume, path)?; + let volume_dir = self.io_get_bucket_path(volume)?; + let file_path = self.io_get_object_path(volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; let mut resp = CheckPartsResp { results: vec![0; fi.parts.len()], }; for (i, part) in fi.parts.iter().enumerate() { - let part_path = self.get_object_path( + let part_path = self.io_get_object_path( volume, path_join_buf(&[ path, @@ -7528,8 +7837,8 @@ impl DiskAPI for LocalDisk { dst_path: &str, meta: Bytes, ) -> Result<()> { - let src_volume_dir = self.get_bucket_path(src_volume)?; - let dst_volume_dir = self.get_bucket_path(dst_volume)?; + let src_volume_dir = self.io_get_bucket_path(src_volume)?; + let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; if !skip_access_checks(src_volume) { super::fs::access_std(&src_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?; } @@ -7537,10 +7846,10 @@ impl DiskAPI for LocalDisk { super::fs::access_std(&dst_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?; } - let src_file_path = self.get_object_path(src_volume, src_path)?; - let dst_file_path = self.get_object_path(dst_volume, dst_path)?; - let dst_meta_path = self.get_object_path(dst_volume, &format!("{dst_path}.meta"))?; - let transaction_path = self.get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?; + let src_file_path = self.io_get_object_path(src_volume, src_path)?; + let dst_file_path = self.io_get_object_path(dst_volume, dst_path)?; + let dst_meta_path = self.io_get_object_path(dst_volume, &format!("{dst_path}.meta"))?; + let transaction_path = self.io_get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?; for path in [&src_file_path, &dst_file_path, &dst_meta_path, &transaction_path] { check_path_length(path.to_string_lossy().as_ref())?; } @@ -7603,10 +7912,10 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> { - self.get_bucket_path(volume)?; - let current_data_path = self.get_object_path(volume, path)?; - let current_meta_path = self.get_object_path(volume, &format!("{path}.meta"))?; - let transaction_path = self.get_object_path(volume, &crate::disk::part_transaction_path(path))?; + self.io_get_bucket_path(volume)?; + let current_data_path = self.io_get_object_path(volume, path)?; + let current_meta_path = self.io_get_object_path(volume, &format!("{path}.meta"))?; + let transaction_path = self.io_get_object_path(volume, &crate::disk::part_transaction_path(path))?; for candidate in [¤t_data_path, ¤t_meta_path, &transaction_path] { check_path_length(candidate.to_string_lossy().as_ref())?; } @@ -7666,8 +7975,8 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> { - let src_volume_dir = self.get_bucket_path(src_volume)?; - let dst_volume_dir = self.get_bucket_path(dst_volume)?; + let src_volume_dir = self.io_get_bucket_path(src_volume)?; + let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; if !skip_access_checks(src_volume) { super::fs::access_std(&src_volume_dir).map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))? } @@ -7691,8 +8000,8 @@ impl DiskAPI for LocalDisk { return Err(DiskError::FileAccessDenied); } - let src_file_path = self.get_object_path(src_volume, src_path)?; - let dst_file_path = self.get_object_path(dst_volume, dst_path)?; + let src_file_path = self.io_get_object_path(src_volume, src_path)?; + let dst_file_path = self.io_get_object_path(dst_volume, dst_path)?; // warn!("rename_part src_file_path:{:?}, dst_file_path:{:?}", &src_file_path, &dst_file_path); @@ -7748,7 +8057,7 @@ impl DiskAPI for LocalDisk { let transaction_publish_meta = if src_is_dir { None } else { - let transaction_path = self.get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?; + let transaction_path = self.io_get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?; let transaction_meta_path = transaction_path.join(PART_TRANSACTION_NEW_META); match fs::read(&transaction_meta_path).await { Ok(expected_meta) => { @@ -7815,7 +8124,7 @@ impl DiskAPI for LocalDisk { } if let Some(transaction_publish_meta) = transaction_publish_meta { - let dst_meta_path = self.get_object_path(dst_volume, &format!("{dst_path}.meta"))?; + let dst_meta_path = self.io_get_object_path(dst_volume, &format!("{dst_path}.meta"))?; rename_all(&transaction_publish_meta, &dst_meta_path, &dst_volume_dir, &self.publication_root).await?; if durability.syncs_commit_metadata() && let Some(parent) = dst_meta_path.parent() @@ -7836,8 +8145,8 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { crate::hp_guard!("LocalDisk::rename_file"); - let src_volume_dir = self.get_bucket_path(src_volume)?; - let dst_volume_dir = self.get_bucket_path(dst_volume)?; + let src_volume_dir = self.io_get_bucket_path(src_volume)?; + let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; if !skip_access_checks(src_volume) { access(&src_volume_dir) .await @@ -7855,10 +8164,10 @@ impl DiskAPI for LocalDisk { return Err(Error::from(DiskError::FileAccessDenied)); } - let src_file_path = self.get_object_path(src_volume, src_path)?; + let src_file_path = self.io_get_object_path(src_volume, src_path)?; check_path_length(src_file_path.to_string_lossy().as_ref())?; - let dst_file_path = self.get_object_path(dst_volume, dst_path)?; + let dst_file_path = self.io_get_object_path(dst_volume, dst_path)?; check_path_length(dst_file_path.to_string_lossy().as_ref())?; if src_is_dir { @@ -7909,7 +8218,7 @@ impl DiskAPI for LocalDisk { async fn create_file(&self, origvolume: &str, volume: &str, path: &str, _file_size: i64) -> Result { crate::hp_guard!("LocalDisk::create_file"); if !origvolume.is_empty() { - let origvolume_dir = self.get_bucket_path(origvolume)?; + let origvolume_dir = self.io_get_bucket_path(origvolume)?; if !skip_access_checks(origvolume) { access(origvolume_dir) .await @@ -7994,7 +8303,7 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result> { if !origvolume.is_empty() { - let origvolume_dir = self.get_bucket_path(origvolume)?; + let origvolume_dir = self.io_get_bucket_path(origvolume)?; if !skip_access_checks(origvolume) && let Err(e) = access(origvolume_dir).await { @@ -8002,8 +8311,8 @@ impl DiskAPI for LocalDisk { } } - let volume_dir = self.get_bucket_path(volume)?; - let dir_path_abs = self.get_object_path(volume, dir_path.trim_start_matches(SLASH_SEPARATOR))?; + let volume_dir = self.io_get_bucket_path(volume)?; + let dir_path_abs = self.io_get_object_path(volume, dir_path.trim_start_matches(SLASH_SEPARATOR))?; // Whole-directory enumeration in one syscall path (see the wide-directory // stall hazard on this fn): with `count < 0` this reads every entry, and @@ -8039,7 +8348,7 @@ impl DiskAPI for LocalDisk { } let stall = opts.stall_timeout_duration(); - let volume_dir = self.get_bucket_path(&opts.bucket)?; + let volume_dir = self.io_get_bucket_path(&opts.bucket)?; if !skip_access_checks(&opts.bucket) && let Err(e) = with_walk_stall_deadline(stall, access(&volume_dir)).await? @@ -8077,8 +8386,8 @@ impl DiskAPI for LocalDisk { write_metacache_obj(&mut out, &meta).await?; objs_returned += 1; } else { - let fpath = - self.get_object_path(&opts.bucket, path_join_buf(&[opts.base_dir.as_str(), STORAGE_FORMAT_FILE]).as_str())?; + let fpath = self + .io_get_object_path(&opts.bucket, path_join_buf(&[opts.base_dir.as_str(), STORAGE_FORMAT_FILE]).as_str())?; if let Ok(meta) = with_walk_stall_deadline(stall, tokio::fs::metadata(&fpath)).await? && meta.is_file() @@ -8135,7 +8444,7 @@ impl DiskAPI for LocalDisk { // object commit is publishing into it. The peer's empty scan remains // optimistic; this lease establishes the local commit/delete order and // remains owned by any blocking syscall that outlives async cancellation. - let destination_object_path = self.get_object_path(dst_volume, dst_path)?; + let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; if fi.is_legacy_indexed_delete_marker() { fi.erasure.index = 0; @@ -8152,7 +8461,7 @@ impl DiskAPI for LocalDisk { .map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number)) .collect() }; - let src_volume_dir = self.get_bucket_path(src_volume)?; + let src_volume_dir = self.io_get_bucket_path(src_volume)?; if !skip_access_checks(src_volume) && let Err(e) = super::fs::access_std(&src_volume_dir) { @@ -8168,7 +8477,7 @@ impl DiskAPI for LocalDisk { return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); } - let dst_volume_dir = self.get_bucket_path(dst_volume)?; + let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; if !skip_access_checks(dst_volume) && let Err(e) = super::fs::access_std(&dst_volume_dir) { @@ -8185,8 +8494,8 @@ impl DiskAPI for LocalDisk { } // xl.meta path - let src_file_path = self.get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; - let dst_file_path = self.get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; + let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; + let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; // data_dir path let has_data_dir_path = { @@ -8200,11 +8509,11 @@ impl DiskAPI for LocalDisk { }; if let Some(data_dir) = has_data_dir { - let src_data_path = self.get_object_path( + let src_data_path = self.io_get_object_path( src_volume, rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(), )?; - let dst_data_path = self.get_object_path( + let dst_data_path = self.io_get_object_path( dst_volume, rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(), )?; @@ -9006,11 +9315,11 @@ impl DiskAPI for LocalDisk { return Err(Error::other("Invalid arguments specified")); } - let volume_dir = self.get_bucket_path(volume)?; + let volume_dir = self.io_get_bucket_path(volume)?; if let Err(e) = access(&volume_dir).await { if e.kind() == ErrorKind::NotFound { - os::make_dir_all(&volume_dir, self.root.as_path()).await?; + os::make_dir_all(&volume_dir, self.io_root()).await?; return Ok(()); } error!( @@ -9032,7 +9341,7 @@ impl DiskAPI for LocalDisk { async fn list_volumes(&self) -> Result> { let mut volumes = Vec::new(); - let entries = os::read_dir(&self.root, -1).await.map_err(to_volume_error)?; + let entries = os::read_dir(self.io_root(), -1).await.map_err(to_volume_error)?; for entry in entries { if !has_suffix(&entry, SLASH_SEPARATOR) || !Self::is_valid_volname(clean(&entry).as_str()) { @@ -9050,7 +9359,7 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn stat_volume(&self, volume: &str) -> Result { - let volume_dir = self.get_bucket_path(volume)?; + let volume_dir = self.io_get_bucket_path(volume)?; let meta = lstat(&volume_dir).await.map_err(to_volume_error)?; let modtime = match meta.modified() { @@ -9066,7 +9375,7 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> { - let volume_dir = self.get_bucket_path(volume)?; + let volume_dir = self.io_get_bucket_path(volume)?; if !skip_access_checks(volume) { access(&volume_dir) .await @@ -9074,7 +9383,7 @@ impl DiskAPI for LocalDisk { } for path in paths.iter() { - let file_path = self.get_object_path(volume, path)?; + let file_path = self.io_get_object_path(volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; @@ -9089,7 +9398,7 @@ impl DiskAPI for LocalDisk { } async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result { - let file_path = self.get_object_path(volume, path)?; + let file_path = self.io_get_object_path(volume, path)?; let key = SnapshotLeaseKey { volume: volume.to_string(), path: path.to_string(), @@ -9220,7 +9529,7 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> { if !fi.metadata.is_empty() { - let file_path = self.get_object_path(volume, path)?; + let file_path = self.io_get_object_path(volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; @@ -9257,7 +9566,7 @@ impl DiskAPI for LocalDisk { async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { crate::hp_guard!("LocalDisk::write_metadata"); fi.validate_for_metadata_read()?; - let p = self.get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?; + let p = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?; let mut meta = FileMeta::new(); if !fi.fresh { @@ -9292,7 +9601,7 @@ impl DiskAPI for LocalDisk { ) -> Result { crate::hp_guard!("LocalDisk::read_version"); if !org_volume.is_empty() { - let org_volume_path = self.get_bucket_path(org_volume)?; + let org_volume_path = self.io_get_bucket_path(org_volume)?; if !skip_access_checks(org_volume) { access(&org_volume_path) .await @@ -9300,8 +9609,8 @@ impl DiskAPI for LocalDisk { } } - let file_path = self.get_object_path(volume, path)?; - let volume_dir = self.get_bucket_path(volume)?; + let file_path = self.io_get_object_path(volume, path)?; + let volume_dir = self.io_get_bucket_path(volume)?; check_path_length(file_path.to_string_lossy().as_ref())?; @@ -9351,7 +9660,7 @@ impl DiskAPI for LocalDisk { fi.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string()).as_str(), part_path.as_str(), ]); - let part_path = self.get_object_path(volume, part_path.as_str())?; + let part_path = self.io_get_object_path(volume, part_path.as_str())?; if lstat(&part_path).await.is_err() { fi.set_inline_data(); return Ok(fi); @@ -9368,7 +9677,7 @@ impl DiskAPI for LocalDisk { fi.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string()).as_str(), format!("part.{}", fi.parts[0].number).as_str(), ]); - let part_path = self.get_object_path(volume, part_path.as_str())?; + let part_path = self.io_get_object_path(volume, part_path.as_str())?; let data = self.read_all_data(volume, volume_dir, part_path.clone()).await.map_err(|e| { warn!( @@ -9392,8 +9701,8 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { crate::hp_guard!("LocalDisk::read_xl"); - let file_path = self.get_object_path(volume, path)?; - let file_dir = self.get_bucket_path(volume)?; + let file_path = self.io_get_object_path(volume, path)?; + let file_dir = self.io_get_bucket_path(volume)?; let (buf, _) = self.read_raw(volume, file_dir, file_path, read_data).await?; @@ -9423,9 +9732,9 @@ impl DiskAPI for LocalDisk { .await; } - let volume_dir = self.get_bucket_path(volume)?; + let volume_dir = self.io_get_bucket_path(volume)?; - let file_path = self.get_object_path(volume, path)?; + let file_path = self.io_get_object_path(volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; @@ -9746,7 +10055,7 @@ impl DiskAPI for LocalDisk { let mut found = 0; for v in req.files.iter() { - let fpath = self.get_object_path(&req.bucket, format!("{}/{}", req.prefix, v).as_str())?; + let fpath = self.io_get_object_path(&req.bucket, format!("{}/{}", req.prefix, v).as_str())?; let mut res = ReadMultipleResp { bucket: req.bucket.clone(), prefix: req.prefix.clone(), @@ -9808,7 +10117,7 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn delete_volume(&self, volume: &str, force_delete: bool) -> Result<()> { - let p = self.get_bucket_path(volume)?; + let p = self.io_get_bucket_path(volume)?; let _volume_mutation_guard = os::disk_volume_mutation_lock(&self.root, volume).write_owned().await; // A streaming reader's snapshot lease defers the physical cleanup of @@ -9826,7 +10135,14 @@ impl DiskAPI for LocalDisk { let res = if force_delete { fs::remove_dir_all(&p).await } else { - remove_empty_directory_tree(&p).await + #[cfg(target_os = "linux")] + { + remove_empty_directory_tree_under_mount_lease(&self.mount_lease, volume, p.clone()).await + } + #[cfg(not(target_os = "linux"))] + { + remove_empty_directory_tree(&p).await + } }; if let Err(err) = res { @@ -9869,8 +10185,8 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn read_metadata(&self, volume: &str, path: &str) -> Result { crate::hp_guard!("LocalDisk::read_metadata"); - let file_path = self.get_object_path(volume, path)?; - let volume_dir = self.get_bucket_path(volume)?; + let file_path = self.io_get_object_path(volume, path)?; + let volume_dir = self.io_get_bucket_path(volume)?; let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?; Ok(data.into()) } @@ -12445,7 +12761,7 @@ mod test { .await .expect("staged part should be written"); let staged_metadata = disk - .get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{STORAGE_FORMAT_FILE}")) + .get_object_path_for_io(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{STORAGE_FORMAT_FILE}")) .expect("staged metadata path should resolve"); let (entered_tx, entered_rx) = mpsc::channel(); @@ -12618,9 +12934,12 @@ mod test { .expect("staged part should be written"); let backup_path = object_dir.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); + let backup_hook_path = disk + .get_object_path_for_io(bucket, &path_join_buf(&[object, &old_data_dir.to_string(), STORAGE_FORMAT_FILE_BACKUP])) + .expect("rollback backup io path should resolve"); let (entered_tx, entered_rx) = mpsc::channel(); let (release_tx, release_rx) = mpsc::channel(); - set_owned_file_write_before_open(&backup_path, move || { + set_owned_file_write_before_open(&backup_hook_path, move || { entered_tx.send(()).expect("signal backup write entry"); release_rx.recv().expect("wait until cancellation has been observed"); }); @@ -16888,6 +17207,26 @@ mod test { assert!(matches!(disk.stat_volume(bucket).await, Err(DiskError::VolumeNotFound))); } + #[cfg(target_os = "linux")] + #[tokio::test] + async fn delete_volume_non_force_removes_empty_bucket_under_mount_lease_root() { + let root = tempfile::tempdir().expect("temporary disk root should be created"); + let endpoint = Endpoint::try_from(root.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + let bucket = "mount-lease-empty-delete"; + + disk.make_volume(bucket).await.expect("bucket should be created"); + fs::create_dir_all(root.path().join(bucket).join("deleted/object/path")) + .await + .expect("empty object remnant should be created"); + + disk.delete_volume(bucket, false) + .await + .expect("non-force delete should remove empty remnants through the held mount lease"); + + assert!(!root.path().join(bucket).exists(), "empty bucket should be removed"); + } + #[tokio::test] async fn empty_tree_delete_preserves_xlmeta_published_after_scan() { let root = tempfile::tempdir().expect("temporary disk root should be created"); @@ -19810,4 +20149,235 @@ mod test { assert!(results[1].is_err()); assert!(matches!(results[1].as_ref().unwrap_err(), DiskError::Io(_))); } + + #[cfg(unix)] + #[tokio::test] + async fn conditional_file_update_never_deletes_a_new_owner() { + use tempfile::tempdir; + + let dir = tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + let owner_a = Bytes::from_static(b"owner-a"); + let owner_b = Bytes::from_static(b"owner-b"); + + assert_eq!( + disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, None, Some(owner_a.clone())) + .await + .expect("owner a should acquire marker"), + ConditionalFileUpdate::Updated + ); + assert_eq!( + disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Some(owner_a.clone()), Some(owner_b.clone()),) + .await + .expect("owner b should replace marker"), + ConditionalFileUpdate::Updated + ); + assert_eq!( + disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Some(owner_a), None) + .await + .expect("stale owner check should complete"), + ConditionalFileUpdate::Mismatch + ); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH) + .await + .expect("new owner marker should remain"), + owner_b + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn conditional_file_update_returns_would_block_when_marker_lock_is_contended() { + use rustix::fs::{FlockOperation, flock}; + + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + ensure_test_volume(&disk, RUSTFS_META_BUCKET).await; + let marker_path = disk + .get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH) + .expect("marker path should resolve"); + let lock_path = marker_path.with_extension("rustfs-cas.lock"); + let lock = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(lock_path) + .expect("marker lock should open"); + flock(&lock, FlockOperation::LockExclusive).expect("marker lock should be held"); + + let err = tokio::time::timeout( + Duration::from_secs(1), + disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, None, Some(Bytes::from_static(b"owner"))), + ) + .await + .expect("contended conditional update must not block") + .expect_err("contended conditional update must retry"); + + assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock)); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn replacement_io_paths_stay_under_the_mount_lease() { + use tempfile::tempdir; + + let dir = tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + + assert!(disk.has_replacement_mount_lease()); + let root_path = disk + .get_object_path("bucket", "object") + .expect("public object path should resolve"); + assert_eq!(root_path, disk.root.join("bucket/object")); + assert!(!root_path.starts_with("/proc/self/fd/")); + assert_eq!( + disk.get_bucket_path("bucket").expect("public bucket path should resolve"), + disk.root.join("bucket") + ); + assert_eq!( + disk.resolve_abs_path("bucket/object") + .expect("public absolute path should resolve"), + disk.root.join("bucket/object") + ); + + let object_path = disk + .io_get_object_path("bucket", "object") + .expect("lease I/O object path should resolve"); + assert!( + object_path.starts_with("/proc/self/fd/"), + "replacement I/O must resolve beneath the held directory descriptor" + ); + assert!( + std::fs::metadata(disk.io_root()) + .expect("lease I/O root must be stat-able") + .is_dir(), + "lease I/O root must remain usable as a directory path" + ); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn replacement_mount_lease_rejects_a_replaced_endpoint_path() { + use tempfile::tempdir; + + let dir = tempdir().expect("temp dir should be created"); + let root = dir.path().to_path_buf(); + let old_root = root.with_extension("leased"); + let endpoint = Endpoint::try_from(root.to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + + std::fs::rename(&root, &old_root).expect("move leased mount root aside"); + std::fs::create_dir(&root).expect("replace configured endpoint directory"); + + assert!( + !disk.has_replacement_mount_lease(), + "replacement admission must reject a path that no longer names the lease" + ); + assert_eq!( + disk.format_path, + root.join(RUSTFS_META_BUCKET).join(crate::disk::FORMAT_CONFIG_FILE), + "the public format path must keep configured-root semantics after replacement" + ); + disk.write_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Bytes::from_static(b"owner")) + .await + .expect("lease-root marker write should succeed"); + assert!(old_root.join(RUSTFS_META_BUCKET).join(HEALING_MARKER_PATH).exists()); + assert!(!root.join(RUSTFS_META_BUCKET).join(HEALING_MARKER_PATH).exists()); + + drop(disk); + std::fs::remove_dir_all(&old_root).expect("remove old leased root"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn startup_cleanup_uses_mount_lease_root_as_a_directory() { + use tempfile::tempdir; + + let dir = tempdir().expect("temp dir should be created"); + let tmp_leftover = dir.path().join(RUSTFS_META_TMP_BUCKET).join("leftover").join("data"); + fs::create_dir_all(tmp_leftover.parent().expect("tmp leftover should have a parent")) + .await + .expect("tmp leftover parent should be created"); + fs::write(&tmp_leftover, b"temporary") + .await + .expect("tmp leftover should be written"); + + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, true).await.expect("local disk should be created"); + + assert!(disk.has_replacement_mount_lease()); + assert!( + !tmp_leftover.exists(), + "startup tmp cleanup must be able to rename beneath the procfd I/O root" + ); + assert!(LocalDisk::meta_path(disk.io_root(), RUSTFS_META_TMP_DELETED_BUCKET).exists()); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn replacement_mount_lease_rejects_a_retargeted_endpoint_symlink() { + use std::os::unix::fs::symlink; + use tempfile::tempdir; + + let dir = tempdir().expect("temp dir should be created"); + let first_root = dir.path().join("first"); + let second_root = dir.path().join("second"); + let endpoint_path = dir.path().join("replacement"); + std::fs::create_dir(&first_root).expect("first replacement root should be created"); + std::fs::create_dir(&second_root).expect("second replacement root should be created"); + symlink(&first_root, &endpoint_path).expect("replacement endpoint symlink should be created"); + let endpoint = Endpoint::try_from(endpoint_path.to_str().expect("replacement endpoint should be utf8")) + .expect("replacement endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + + assert!(disk.has_replacement_mount_lease()); + assert!(disk.replacement_mount_lease_root().is_some()); + + std::fs::remove_file(&endpoint_path).expect("original endpoint symlink should be removed"); + symlink(&second_root, &endpoint_path).expect("replacement endpoint should be retargeted"); + + assert!( + !disk.has_replacement_mount_lease(), + "replacement admission must reject an endpoint symlink retargeted away from the held mount" + ); + assert!( + disk.replacement_mount_lease_root().is_none(), + "readiness must not expose an identity root after endpoint retargeting" + ); + + disk.write_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Bytes::from_static(b"owner")) + .await + .expect("lease-root marker write should succeed"); + assert!(first_root.join(RUSTFS_META_BUCKET).join(HEALING_MARKER_PATH).exists()); + assert!(!second_root.join(RUSTFS_META_BUCKET).join(HEALING_MARKER_PATH).exists()); + } + + #[cfg(target_os = "linux")] + #[test] + fn replacement_mount_identity_rejects_a_bind_remount_with_the_same_inode() { + let held = ReplacementMountIdentity { + device: 8, + inode: 42, + mount_id: 101, + }; + let rebound = ReplacementMountIdentity { mount_id: 102, ..held }; + + assert_ne!( + held, rebound, + "a bind remount can retain device and inode while changing the mount incarnation" + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn mountinfo_fallback_uses_the_exact_escaped_mountpoint() { + let mountinfo = "101 1 0:30 / / rw - rootfs rootfs rw\n202 101 0:42 / /mnt/replacement\\040disk rw - tmpfs tmpfs rw\n"; + assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement disk")), Some(202)); + assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement")), None); + } } diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index d916522a7..8112aa194 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -115,6 +115,15 @@ pub enum PartTransactionAction { Rollback, } +/// Result of an owner-aware file mutation. The disk applies the mutation only +/// while the current contents match the supplied expected value. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ConditionalFileUpdate { + Updated, + Missing, + Mismatch, +} + #[derive(Clone, Copy, Debug)] pub struct MmapCopyStageMetrics { pub(crate) path: &'static str, @@ -557,6 +566,26 @@ impl DiskAPI for Disk { } } + async fn compare_and_update_file( + &self, + volume: &str, + path: &str, + expected: Option, + replacement: Option, + ) -> Result { + match self { + Disk::Local(local_disk) => local_disk.compare_and_update_file(volume, path, expected, replacement).await, + Disk::Remote(remote_disk) => remote_disk.compare_and_update_file(volume, path, expected, replacement).await, + } + } + + fn has_replacement_mount_lease(&self) -> bool { + match self { + Disk::Local(local_disk) => local_disk.has_replacement_mount_lease(), + Disk::Remote(remote_disk) => remote_disk.has_replacement_mount_lease(), + } + } + #[tracing::instrument(level = "trace", skip_all)] async fn read_all(&self, volume: &str, path: &str) -> Result { match self { @@ -695,6 +724,34 @@ impl Disk { Disk::Remote(_) => None, } } + + pub(crate) fn get_object_path_for_io_if_local( + &self, + volume: &str, + path: &str, + ) -> Option> { + match self { + Disk::Local(w) => Some(w.get_object_path_for_io(volume, path)), + Disk::Remote(_) => None, + } + } + + pub(crate) fn get_bucket_path_for_io_if_local(&self, volume: &str) -> Option> { + match self { + Disk::Local(w) => Some(w.get_bucket_path_for_io(volume)), + Disk::Remote(_) => None, + } + } + + /// Return the descriptor-rooted mount path admitted for automatic + /// replacement, or `None` when the configured endpoint no longer names + /// that held mount instance. + pub fn replacement_mount_lease_root(&self) -> Option { + match self { + Disk::Local(local_disk) => local_disk.replacement_mount_lease_root(), + Disk::Remote(_) => None, + } + } } pub async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> Result { @@ -860,6 +917,24 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { // CleanAbandonedData async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()>; async fn read_all(&self, volume: &str, path: &str) -> Result; + /// Atomically replace or remove a small control file only when its current + /// contents match `expected`. Implementations that cannot provide this + /// cross-process guarantee must fail closed instead of emulating it with a + /// read-then-write sequence. + async fn compare_and_update_file( + &self, + _volume: &str, + _path: &str, + _expected: Option, + _replacement: Option, + ) -> Result { + Err(DiskError::MethodNotAllowed) + } + /// Whether local I/O is rooted at a held mount descriptor. Auto-replacement + /// refuses destructive work when this is false. + fn has_replacement_mount_lease(&self) -> bool { + false + } async fn disk_info(&self, opts: &DiskInfoOptions) -> Result; fn start_scan(&self) -> ScanGuard; } @@ -1612,6 +1687,7 @@ mod tests { let endpoint = Endpoint::try_from(test_dir).unwrap(); let local_disk = LocalDisk::new(&endpoint, false).await.unwrap(); + let expected_object_path = local_disk.root.join("test-bucket/test-object"); let disk = Disk::Local(Box::new(LocalDiskWrapper::new(Arc::new(local_disk), false))); // Test basic methods @@ -1626,6 +1702,19 @@ mod tests { // Test path method let path = disk.path(); assert!(path.exists()); + let object_path = disk + .get_object_path_if_local("test-bucket", "test-object") + .expect("local disk should expose an object path") + .expect("object path should resolve"); + assert_eq!(object_path, expected_object_path); + assert!(!object_path.starts_with("/proc/self/fd/")); + #[cfg(target_os = "linux")] + assert!( + disk.get_object_path_for_io_if_local("test-bucket", "test-object") + .expect("local disk should expose an I/O object path") + .expect("I/O object path should resolve") + .starts_with("/proc/self/fd/") + ); // Test disk location let location = disk.get_disk_location(); diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index ccce48958..806263cfa 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -84,11 +84,22 @@ pub(crate) mod fsync_dir_recorder { static RECORDED: Mutex> = Mutex::new(Vec::new()); pub(crate) fn record(dir: &Path) { - RECORDED.lock().expect("fsync dir recorder poisoned").push(dir.to_path_buf()); + let mut recorded = RECORDED.lock().expect("fsync dir recorder poisoned"); + recorded.push(dir.to_path_buf()); + if let Ok(canonical) = dir.canonicalize() + && canonical != dir + { + recorded.push(canonical); + } } pub(crate) fn was_fsynced(dir: &Path) -> bool { - RECORDED.lock().expect("fsync dir recorder poisoned").iter().any(|p| p == dir) + let canonical = dir.canonicalize().ok(); + RECORDED + .lock() + .expect("fsync dir recorder poisoned") + .iter() + .any(|p| p == dir || canonical.as_ref().is_some_and(|canonical| p == canonical)) } } @@ -330,7 +341,7 @@ pub(crate) mod file_sync_probe { use tokio::sync::Notify; use tokio::time::timeout; - static ROOT: RwLock> = RwLock::new(None); + static ROOTS: RwLock> = RwLock::new(Vec::new()); static BLOCK_MUTEX: Mutex<()> = Mutex::new(()); static BLOCK_CONDVAR: Condvar = Condvar::new(); static ACTIVE_CHANGED: Notify = Notify::const_new(); @@ -364,7 +375,7 @@ pub(crate) mod file_sync_probe { fn drop(&mut self) { release(); FAIL_ON_ATTEMPT.store(usize::MAX, Ordering::SeqCst); - *ROOT.write().expect("file sync probe lock poisoned") = None; + ROOTS.write().expect("file sync probe lock poisoned").clear(); } } @@ -377,7 +388,13 @@ pub(crate) mod file_sync_probe { let _guard = BLOCK_MUTEX.lock().expect("file sync probe blocker poisoned"); BLOCK.store(block, Ordering::SeqCst); } - *ROOT.write().expect("file sync probe lock poisoned") = Some(root.to_path_buf()); + let mut roots = vec![root.to_path_buf()]; + if let Ok(canonical) = root.canonicalize() + && canonical != root + { + roots.push(canonical); + } + *ROOTS.write().expect("file sync probe lock poisoned") = roots; ProbeGuard } @@ -398,11 +415,11 @@ pub(crate) mod file_sync_probe { } pub(super) fn enter(path: &Path) -> Option { - let enabled = ROOT - .read() - .expect("file sync probe lock poisoned") - .as_ref() - .is_some_and(|root| path.starts_with(root)); + let canonical_path = path.canonicalize().ok(); + let enabled = + ROOTS.read().expect("file sync probe lock poisoned").iter().any(|root| { + path.starts_with(root) || canonical_path.as_ref().is_some_and(|canonical| canonical.starts_with(root)) + }); if !enabled { return None; } diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 40bc2673b..1ddd04ee8 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -1439,7 +1439,7 @@ async fn try_create_bitrot_readers_via_batch_pread( if let Some(disk) = disk_op.as_ref() { let data_dir = files[idx].data_dir.unwrap_or_default(); let path_str = format!("{object}/{data_dir}/part.{part_number}"); - match disk.get_object_path_if_local(bucket, &path_str) { + match disk.get_object_path_for_io_if_local(bucket, &path_str) { Some(Ok(p)) => batch_items.push((idx, p, adj_off, adj_len)), _ => return None, } diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 67413a219..cf7f6c1e4 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -331,6 +331,85 @@ fn warn_heal_writer_failures( } impl SetDisks { + /// Read back one healed version from every explicitly admitted replacement + /// target. This is intentionally separate from the normal heal result: a + /// successful result describes the transaction attempt, while automatic + /// replacement completion needs physical evidence that survives a crash + /// before its checkpoint is persisted. + pub(crate) async fn replacement_targets_have_version( + &self, + bucket: &str, + object: &str, + version_id: &str, + targets: &[String], + ) -> disk::error::Result { + let disks = self.get_disks_internal().await; + let mut target_disks = Vec::with_capacity(targets.len()); + + for target in targets { + let Some(index) = self.set_endpoints.iter().position(|endpoint| endpoint.to_string() == *target) else { + return Ok(false); + }; + let Some(disk) = disks.get(index).and_then(Option::as_ref) else { + return Ok(false); + }; + target_disks.push(disk.clone()); + } + + let read_options = ReadOptions { + incl_free_versions: false, + read_data: true, + healing: true, + }; + let checks = target_disks.into_iter().map(|disk| { + let read_options = read_options.clone(); + async move { + let file_info = match disk.read_version("", bucket, object, version_id, &read_options).await { + Ok(file_info) => file_info, + Err( + DiskError::DiskNotFound + | DiskError::VolumeNotFound + | DiskError::FileNotFound + | DiskError::FileVersionNotFound + | DiskError::PathNotFound, + ) => return Ok(false), + Err(err) => return Err(err), + }; + if !file_info_is_valid_for_metadata(&file_info) { + return Ok(false); + } + if !version_id.is_empty() && file_info.version_id.as_ref().map(ToString::to_string).as_deref() != Some(version_id) + { + return Ok(false); + } + if file_info.is_canonical_delete_marker() || file_info.is_remote() { + return Ok(true); + } + if (file_info.data.is_some() || file_info.size == 0) && !file_info.parts.is_empty() { + return Ok(true); + } + + let check = match disk.check_parts(bucket, object, &file_info).await { + Ok(check) => check, + Err( + DiskError::DiskNotFound + | DiskError::VolumeNotFound + | DiskError::FileNotFound + | DiskError::FileVersionNotFound + | DiskError::PathNotFound, + ) => return Ok(false), + Err(err) => return Err(err), + }; + Ok(!check.results.is_empty() && check.results.iter().all(|result| *result == CHECK_PART_SUCCESS)) + } + }); + + Ok(futures::future::try_join_all(checks) + .await? + .into_iter() + .all(|committed| committed)) + } + #[tracing::instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))] pub(in crate::set_disk) async fn heal_object( &self, @@ -1711,19 +1790,35 @@ impl SetDisks { } } -// Heal operation family: the storage-api `HealOperations` contract stays -// implemented `for SetDisks` (contract bounds unchanged) but now lives beside -// its inherent helpers in the `set_disk::ops::heal` module. Bodies are moved -// unchanged; `get_pool_and_set` reads the core through `SetDisksCtx` to keep -// the Heal family aligned with the borrow pattern from #816. -#[async_trait::async_trait] -impl crate::storage_api_contracts::heal::HealOperations for SetDisks { - type Error = Error; - type HealResultItem = HealResultItem; - type HealOptions = HealOpts; +impl SetDisks { + pub(crate) async fn heal_replacement_format( + &self, + dry_run: bool, + targets: &[String], + ) -> Result<(HealResultItem, Option)> { + if targets.is_empty() { + return Err(Error::other("replacement format requires at least one target")); + } - #[tracing::instrument(skip(self))] - async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { + let mut target_slots = Vec::with_capacity(targets.len()); + for target in targets { + let Some(slot) = self.set_endpoints.iter().position(|endpoint| endpoint.to_string() == *target) else { + return Err(Error::other("replacement format target does not belong to the set")); + }; + if target_slots.contains(&slot) { + return Err(Error::other("replacement format target is duplicated")); + } + target_slots.push(slot); + } + + self.heal_format_for_slots(dry_run, Some(&target_slots)).await + } + + async fn heal_format_for_slots( + &self, + dry_run: bool, + target_slots: Option<&[usize]>, + ) -> Result<(HealResultItem, Option)> { let disks = self.disks.read().await.clone(); let (formats, errs) = load_format_erasure_all(&disks, true).await; if errs.iter().any(|err| { @@ -1785,21 +1880,43 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { if !dry_run { for (disk_idx, err) in errs.iter().enumerate() { - if !matches!(err, Some(DiskError::UnformattedDisk)) { + if !matches!(err, Some(DiskError::UnformattedDisk)) + || target_slots.is_some_and(|slots| !slots.contains(&disk_idx)) + { continue; } let mut new_format = ref_format.clone(); new_format.erasure.this = ref_format.erasure.sets[self.set_index][disk_idx]; - if save_format_file(&disks[disk_idx], &Some(new_format.clone())).await.is_ok() { - result.after.drives[disk_idx].uuid = new_format.erasure.this.to_string(); - result.after.drives[disk_idx].state = DriveState::Ok.to_string(); + match save_format_file(&disks[disk_idx], &Some(new_format.clone())).await { + Ok(()) => { + result.after.drives[disk_idx].uuid = new_format.erasure.this.to_string(); + result.after.drives[disk_idx].state = DriveState::Ok.to_string(); + } + Err(err) => return Ok((result, Some(err.into()))), } } } Ok((result, None)) } +} + +// Heal operation family: the storage-api `HealOperations` contract stays +// implemented `for SetDisks` (contract bounds unchanged) but now lives beside +// its inherent helpers in the `set_disk::ops::heal` module. Bodies are moved +// unchanged; `get_pool_and_set` reads the core through `SetDisksCtx` to keep +// the Heal family aligned with the borrow pattern from #816. +#[async_trait::async_trait] +impl crate::storage_api_contracts::heal::HealOperations for SetDisks { + type Error = Error; + type HealResultItem = HealResultItem; + type HealOptions = HealOpts; + + #[tracing::instrument(skip(self))] + async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { + self.heal_format_for_slots(dry_run, None).await + } #[tracing::instrument(skip(self))] async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { @@ -2397,6 +2514,50 @@ mod heal_result_report_tests { } } + #[tokio::test] + async fn replacement_target_readback_requires_the_committed_shard() { + let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "replacement-target-readback"; + let object = "object.bin"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let mut reader = PutObjReader::from_vec(vec![0x5a; 1024 * 1024]); + set.put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + let source = disks[2] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("source metadata should be readable"); + let data_dir = source.data_dir.expect("non-inline source should have a data directory"); + let targets = vec![set.set_endpoints[0].to_string(), set.set_endpoints[1].to_string()]; + + assert!( + set.replacement_targets_have_version(bucket, object, "", &targets) + .await + .expect("healthy target shards should be readable") + ); + + tokio::fs::remove_file( + temp_dirs[1] + .path() + .join(bucket) + .join(object) + .join(data_dir.to_string()) + .join("part.1"), + ) + .await + .expect("target shard should be removed after the initial commit"); + + assert!( + !set.replacement_targets_have_version(bucket, object, "", &targets) + .await + .expect("missing target shard should be observable") + ); + } + #[tokio::test] async fn format_heal_cached_layout_rejects_a_disk_from_another_slot() { let mut _temp_dirs = Vec::new(); diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 79a1d43a6..8dc2e8a13 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -52,7 +52,10 @@ fn validate_table_bucket_delete_allowed( async fn table_catalog_metadata_exists(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result { let local_disks = runtime_sources::local_disks_in(ctx).await; for disk in local_disks.iter() { - let catalog_path = disk.path().join(bucket).join(BUCKET_TABLE_RESERVED_PREFIX); + let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else { + continue; + }; + let catalog_path = bucket_path?.join(BUCKET_TABLE_RESERVED_PREFIX); if has_xlmeta_files(&catalog_path).await? { return Ok(true); } @@ -727,7 +730,10 @@ impl ECStore { if !opts.force { let local_disks = runtime_sources::local_disks_in(&self.ctx).await; for disk in local_disks.iter() { - let bucket_path = disk.path().join(bucket); + let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else { + continue; + }; + let bucket_path = bucket_path?; if has_xlmeta_files(&bucket_path).await? { return Err(StorageError::BucketNotEmpty(bucket.to_string())); } diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index fdfba048a..3efa4ed5f 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -97,6 +97,56 @@ impl ECStore { Ok((r, None)) } + #[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))] + pub async fn heal_replacement_format( + &self, + dry_run: bool, + pool_index: usize, + set_index: usize, + targets: &[String], + ) -> Result<(HealResultItem, Option)> { + let pool = self + .pools + .get(pool_index) + .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?; + let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| { + StorageError::InvalidArgument( + "heal".to_string(), + "set".to_string(), + format!("invalid heal set index {set_index} for pool {pool_index}"), + ) + })?; + + set.heal_replacement_format(dry_run, targets).await + } + + #[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))] + pub async fn replacement_targets_have_version( + &self, + bucket: &str, + object: &str, + version_id: &str, + pool_index: usize, + set_index: usize, + targets: &[String], + ) -> Result { + let pool = self + .pools + .get(pool_index) + .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?; + let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| { + StorageError::InvalidArgument( + "heal".to_string(), + "set".to_string(), + format!("invalid heal set index {set_index} for pool {pool_index}"), + ) + })?; + + set.replacement_targets_have_version(bucket, object, version_id, targets) + .await + .map_err(Into::into) + } + #[instrument(skip(self))] pub(super) async fn handle_heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { let res = self.peer_sys.heal_bucket(bucket, opts).await?; diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 695301976..376a8d717 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -14,7 +14,10 @@ use crate::heal::{ progress::HealProgress, - resume::{CheckpointManager, ResumeManager, ResumeUtils, compose_key}, + resume::{ + CheckpointManager, ReplacementTargetIdentity, ResumeManager, ResumeUtils, compose_key, + replacement_target_identities_match, + }, storage::{HealStorageAPI, next_heal_listing_token}, task::{demote_to_debug_when, is_missing_object_dir_heal_result, take_failure_log_sample}, }; @@ -22,6 +25,7 @@ use crate::{Error, Result}; use futures::{StreamExt, stream::FuturesUnordered}; use metrics::gauge; use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode}; +use rustfs_madmin::heal_commands::HealResultItem; use std::sync::{ Arc, atomic::{AtomicUsize, Ordering}, @@ -85,6 +89,16 @@ pub struct ErasureSetHealer { disk: DiskStore, heal_opts: HealOpts, source: HealRequestSource, + target_endpoints: Arc<[String]>, + replacement_task_id: Option, + replacement_target_identities: Option>, +} + +pub(crate) fn target_outcomes_complete(result: &HealResultItem, target_endpoints: &[String]) -> bool { + target_endpoints.iter().all(|endpoint| { + let mut drives = result.after.drives.iter().filter(|drive| drive.endpoint == *endpoint); + matches!(drives.next(), Some(drive) if drive.state == "ok") && drives.next().is_none() + }) } impl ErasureSetHealer { @@ -182,9 +196,46 @@ impl ErasureSetHealer { disk, heal_opts, source, + target_endpoints: Vec::new().into(), + replacement_task_id: None, + replacement_target_identities: None, } } + pub(crate) fn with_replacement_targets( + mut self, + mut target_endpoints: Vec, + replacement_task_id: Option, + ) -> Self { + target_endpoints.sort_unstable(); + target_endpoints.dedup(); + self.target_endpoints = target_endpoints.into(); + self.replacement_task_id = replacement_task_id; + self + } + + pub(crate) fn with_replacement_identity_fence( + mut self, + replacement_target_identities: Option>, + ) -> Self { + self.replacement_target_identities = replacement_target_identities.map(Into::into); + self + } + + async fn verify_replacement_identity_fence(&self, stage: &str) -> Result<()> { + let Some(expected_identities) = self.replacement_target_identities.as_ref() else { + return Ok(()); + }; + let actual_identities = self.storage.replacement_target_identities(&self.target_endpoints).await?; + if replacement_target_identities_match(expected_identities, &actual_identities) { + return Ok(()); + } + + Err(Error::TaskExecutionFailed { + message: format!("Replacement target changed during {stage}"), + }) + } + /// execute erasure set heal with resume #[tracing::instrument(skip(self, buckets), fields(set_disk_id = %set_disk_id, bucket_count = buckets.len()))] #[hotpath::measure] @@ -212,9 +263,15 @@ impl ErasureSetHealer { .await; result?; + self.verify_replacement_identity_fence("completion").await?; + + if self.replacement_task_id.is_some() { + // A replacement marker must outlive the successful data scan. The + // task clears that owner marker before deleting these artifacts. + resume_manager.mark_replacement_completed_and_verified().await?; + return Ok(()); + } - // The healing marker is cleared by the caller only after both cleanup - // operations succeed. Cleanup is idempotent, so a retry is safe. checkpoint_manager.cleanup().await?; resume_manager.cleanup().await?; Ok(()) @@ -222,6 +279,21 @@ impl ErasureSetHealer { /// get or create task id async fn get_or_create_task_id(&self, set_disk_id: &str) -> Result { + if let Some(task_id) = &self.replacement_task_id { + let manager = ResumeManager::load_replacement_intent(self.disk.clone(), task_id).await?; + let state = manager.get_state().await; + if !state.completed + && state.set_disk_id == set_disk_id + && state.replacement_targets.as_slice() == self.target_endpoints.as_ref() + && state.replacement_generation.as_deref() == Some(task_id.as_str()) + { + return Ok(task_id.clone()); + } + return Err(Error::TaskExecutionFailed { + message: format!("Replacement resume intent does not match task {task_id}"), + }); + } + // check if there are resumable tasks let resumable_tasks = ResumeUtils::get_resumable_tasks(&self.disk).await?; @@ -231,6 +303,7 @@ impl ErasureSetHealer { let state = manager.get_state().await; if !state.completed && state.set_disk_id == set_disk_id + && state.replacement_targets.as_slice() == self.target_endpoints.as_ref() && ResumeUtils::can_resume_task(&self.disk, &task_id).await { debug!( @@ -263,7 +336,7 @@ impl ErasureSetHealer { } // create new task id - let task_id = format!("{}_{}", set_disk_id, ResumeUtils::generate_task_id()); + let task_id = ResumeUtils::generate_task_id(); debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_RESUME_STATE, @@ -285,7 +358,12 @@ impl ErasureSetHealer { buckets: &[String], ) -> Result<(ResumeManager, CheckpointManager)> { // check if resume state exists - if ResumeManager::has_resume_state(&self.disk, task_id).await { + let has_resume_state = if self.replacement_task_id.is_some() { + ResumeManager::has_replacement_intent(&self.disk, task_id).await + } else { + ResumeManager::has_resume_state(&self.disk, task_id).await + }; + if has_resume_state { debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_RESUME_STATE, @@ -297,7 +375,11 @@ impl ErasureSetHealer { "Erasure set resume state loading" ); - let resume_manager = ResumeManager::load_from_disk(self.disk.clone(), task_id).await?; + let resume_manager = if self.replacement_task_id.is_some() { + ResumeManager::load_replacement_intent(self.disk.clone(), task_id).await? + } else { + ResumeManager::load_from_disk(self.disk.clone(), task_id).await? + }; let checkpoint_manager = if CheckpointManager::has_checkpoint(&self.disk, task_id).await { CheckpointManager::load_from_disk(self.disk.clone(), task_id).await? } else { @@ -340,6 +422,9 @@ impl ErasureSetHealer { buckets.to_vec(), ) .await?; + resume_manager + .set_replacement_targets(self.target_endpoints.as_ref().to_vec()) + .await?; let checkpoint_manager = CheckpointManager::new(self.disk.clone(), task_id.to_string()).await?; @@ -485,6 +570,12 @@ impl ErasureSetHealer { // later heal cycle via the same bounded-retry mechanism as failures — // never hot-retried in place here. if failed_objects > 0 || skipped_objects > 0 || failed_buckets > 0 { + if self.replacement_task_id.is_some() && resume_manager.schedule_retry().await? { + checkpoint_manager.reset_for_retry().await?; + return Err(Error::transient_skip(format!( + "Replacement erasure set heal incomplete: {failed_buckets} bucket(s) failed, {failed_objects} object(s) failed, {skipped_objects} object(s) skipped; retry scheduled" + ))); + } if resume_manager.schedule_retry().await? { // Both persistence layers must be reset together: schedule_retry // rewinds the resume state (cursor + counters), and the @@ -508,15 +599,15 @@ impl ErasureSetHealer { state = "retry_scheduled", "Erasure set heal pass finished with unhealed versions; scheduled full re-heal retry" ); - return Err(Error::other(format!( + return Err(Error::transient_skip(format!( "Erasure set heal incomplete: {failed_buckets} bucket(s) failed, {failed_objects} object(s) failed, {skipped_objects} object(s) skipped; retry scheduled" ))); } - // Retry budget exhausted: drop the resume/checkpoint state so this - // task does not loop, but keep the healing markers (return Err) so a - // later heal cycle / the background scanner starts a fresh attempt. - // Never silently claim a clean completion while objects are unhealed. + // Retry budget exhausted: keep the resume/checkpoint state while + // the replacement marker remains. A later repair must retain the + // durable evidence of the incomplete generation instead of + // starting from an indistinguishable blank state. error!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_RESUME_STATE, @@ -529,15 +620,16 @@ impl ErasureSetHealer { state = "failed_after_retries", "Erasure set heal exhausted retries with unrecovered versions" ); - checkpoint_manager.cleanup().await?; - resume_manager.cleanup().await?; return Err(Error::other(format!( "Erasure set heal exhausted retries with {failed_buckets} bucket(s) failed, {failed_objects} object(s) failed, {skipped_objects} object(s) skipped" ))); } - // no failures — mark task completed - resume_manager.mark_completed().await?; + // No failures — ordinary heals are complete now. Replacement heals + // atomically transition to Verified after the terminal identity fence. + if self.replacement_task_id.is_none() { + resume_manager.mark_completed().await?; + } debug!( target: "rustfs::heal::erasure_healer", @@ -628,6 +720,7 @@ impl ErasureSetHealer { matches!(self.heal_opts.scan_mode, HealScanMode::Deep) || matches!(self.source, HealRequestSource::AutoHeal); loop { + self.verify_replacement_identity_fence("page scan").await?; // Get one page of object versions let (objects, next_token, is_truncated) = if use_disk_walk { self.storage @@ -672,6 +765,8 @@ impl ErasureSetHealer { let set_label = set_disk_id.to_string(); let heal_opts = self.heal_opts; let semaphore = semaphore.clone(); + let target_endpoints = self.target_endpoints.clone(); + let replacement_commit_evidence_required = self.replacement_task_id.is_some(); page_tasks.push(async move { let permit = semaphore @@ -699,6 +794,35 @@ impl ErasureSetHealer { .heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts) .await { + Ok((result, None)) + if target_outcomes_complete(&result, &target_endpoints) => + { + if !replacement_commit_evidence_required { + Ok(true) + } else { + match storage + .replacement_targets_have_version( + &bucket_name, + &object_name, + version_id.as_deref(), + &heal_opts, + &target_endpoints, + ) + .await + { + Ok(true) => Ok(true), + Ok(false) => Err(Error::transient_skip(format!( + "Skipped heal for {bucket_name}/{object_name} because replacement target readback did not confirm the committed version" + ))), + Err(err) => Err(Error::transient_skip(format!( + "Skipped heal for {bucket_name}/{object_name} because replacement target readback failed: {err}" + ))), + } + } + } + Ok((_result, None)) if !target_endpoints.is_empty() => Err(Error::transient_skip(format!( + "Skipped heal for {bucket_name}/{object_name} because a replacement target was not committed" + ))), Ok((_result, None)) => Ok(true), Ok((_, Some(err))) if is_missing_object_dir_heal_result(&object_name, &err) => Ok(false), Ok((_, Some(err))) | Err(err) => match Self::classify_heal_object_error(&err) { @@ -1011,9 +1135,12 @@ mod resume_loop_tests { //! that emits programmable multi-version pages. These exercise the real loop //! logic (cursor seeding, per-version dedup, anti-loop guard, absence //! handling) — not merely a mock's own output. - use super::ErasureSetHealer; + use super::{ErasureSetHealer, target_outcomes_complete}; use crate::heal::progress::HealProgress; - use crate::heal::resume::{CheckpointManager, RESUME_CHECKPOINT_FILE, ResumeDeleteFailure, ResumeManager, compose_key}; + use crate::heal::resume::{ + CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils, + compose_key, + }; use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo, HealStorageAPI}; use crate::heal::storage_api::status::BucketInfo; use crate::heal::{ @@ -1021,8 +1148,8 @@ mod resume_loop_tests { }; use crate::{Error, Result}; use rustfs_common::heal_channel::{HealOpts, HealRequestSource}; - use rustfs_madmin::heal_commands::HealResultItem; - use std::collections::HashMap; + use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; + use std::collections::{HashMap, VecDeque}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use tempfile::TempDir; @@ -1037,6 +1164,53 @@ mod resume_loop_tests { } } + #[test] + fn target_outcomes_require_each_requested_endpoint_once_and_ok() { + let result = HealResultItem { + after: Infos { + drives: vec![ + HealDriveInfo { + endpoint: "replacement-a".to_string(), + state: "ok".to_string(), + ..Default::default() + }, + HealDriveInfo { + endpoint: "replacement-b".to_string(), + state: "missing".to_string(), + ..Default::default() + }, + ], + }, + ..Default::default() + }; + + assert!(target_outcomes_complete(&result, &["replacement-a".to_string()])); + assert!(!target_outcomes_complete( + &result, + &["replacement-a".to_string(), "replacement-b".to_string()] + )); + assert!(!target_outcomes_complete(&result, &["replacement-c".to_string()])); + + let duplicate = HealResultItem { + after: Infos { + drives: vec![ + HealDriveInfo { + endpoint: "replacement-a".to_string(), + state: "ok".to_string(), + ..Default::default() + }, + HealDriveInfo { + endpoint: "replacement-a".to_string(), + state: "missing".to_string(), + ..Default::default() + }, + ], + }, + ..Default::default() + }; + assert!(!target_outcomes_complete(&duplicate, &["replacement-a".to_string()])); + } + #[derive(Clone)] struct Page { items: Vec, @@ -1061,8 +1235,14 @@ mod resume_loop_tests { pages: Mutex, Page>>, /// per-`compose_key` heal outcome; default is `Ok` outcomes: Mutex>, + /// successful low-level result per `compose_key`; default has no drive outcomes. + results: Mutex>, + /// Target-specific physical readback evidence per `compose_key`; the + /// fake models a healthy backend unless a test explicitly revokes it. + replacement_commit_evidence: Mutex>, /// every heal_object call recorded as (name, version_id) heal_calls: Mutex)>>, + replacement_target_identity_sequences: Mutex>>, fail_listing: AtomicBool, } @@ -1073,6 +1253,15 @@ mod resume_loop_tests { fn set_outcome(&self, name: &str, version: Option<&str>, outcome: HealOutcome) { self.outcomes.lock().unwrap().insert(compose_key(name, version), outcome); } + fn set_result(&self, name: &str, version: Option<&str>, result: HealResultItem) { + self.results.lock().unwrap().insert(compose_key(name, version), result); + } + fn set_replacement_commit_evidence(&self, name: &str, version: Option<&str>, committed: bool) { + self.replacement_commit_evidence + .lock() + .unwrap() + .insert(compose_key(name, version), committed); + } fn calls(&self) -> Vec<(String, Option)> { self.heal_calls.lock().unwrap().clone() } @@ -1143,7 +1332,7 @@ mod resume_loop_tests { let key = compose_key(object, version_id); let outcome = self.outcomes.lock().unwrap().get(&key).cloned().unwrap_or(HealOutcome::Ok); match outcome { - HealOutcome::Ok => Ok((HealResultItem::default(), None)), + HealOutcome::Ok => Ok((self.results.lock().unwrap().get(&key).cloned().unwrap_or_default(), None)), HealOutcome::VersionNotFound => { Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::FileVersionNotFound)))) } @@ -1157,6 +1346,21 @@ mod resume_loop_tests { async fn heal_format(&self, _dry: bool) -> Result<(HealResultItem, Option)> { Ok((HealResultItem::default(), None)) } + async fn replacement_targets_have_version( + &self, + _bucket: &str, + object: &str, + version_id: Option<&str>, + _opts: &HealOpts, + _targets: &[String], + ) -> Result { + Ok(*self + .replacement_commit_evidence + .lock() + .unwrap() + .get(&compose_key(object, version_id)) + .unwrap_or(&true)) + } async fn list_objects_for_heal(&self, _b: &str, _p: &str) -> Result> { Ok(Vec::new()) } @@ -1179,6 +1383,13 @@ mod resume_loop_tests { async fn get_disk_for_resume(&self, _id: &str) -> Result { Err(Error::other("not implemented in tests")) } + async fn replacement_target_identities(&self, _targets: &[String]) -> Result> { + self.replacement_target_identity_sequences + .lock() + .unwrap() + .pop_front() + .ok_or_else(|| Error::other("replacement identity sequence exhausted")) + } } async fn make_disk(temp: &TempDir) -> DiskStore { @@ -1204,13 +1415,19 @@ mod resume_loop_tests { storage: Arc, resume: ResumeManager, checkpoint: CheckpointManager, + task_id: String, _temp: TempDir, } async fn make_env() -> Env { + make_env_with_targets(Vec::new()).await + } + + async fn make_env_with_targets(target_endpoints: Vec) -> Env { let temp = TempDir::new().unwrap(); let disk = make_disk(&temp).await; let storage = Arc::new(FakeStorage::default()); + let task_id = ResumeUtils::generate_task_id(); let healer = ErasureSetHealer::new( storage.clone(), Arc::new(RwLock::new(HealProgress::new())), @@ -1218,22 +1435,24 @@ mod resume_loop_tests { disk.clone(), HealOpts::default(), HealRequestSource::Internal, - ); + ) + .with_replacement_targets(target_endpoints, None); let resume = ResumeManager::new( disk.clone(), - "task".to_string(), + task_id.clone(), "erasure_set".to_string(), "pool_0_set_0".to_string(), vec!["b".to_string()], ) .await .unwrap(); - let checkpoint = CheckpointManager::new(disk, "task".to_string()).await.unwrap(); + let checkpoint = CheckpointManager::new(disk, task_id.clone()).await.unwrap(); Env { healer, storage, resume, checkpoint, + task_id, _temp: temp, } } @@ -1277,6 +1496,112 @@ mod resume_loop_tests { assert_eq!(env.resume.resume_cursor().await, None); } + #[tokio::test] + async fn replacement_targets_use_a_canonical_order() { + let env = make_env_with_targets(vec![ + "replacement-b".to_string(), + "replacement-a".to_string(), + "replacement-b".to_string(), + ]) + .await; + + assert_eq!(env.healer.target_endpoints.as_ref(), ["replacement-a", "replacement-b"]); + } + + #[tokio::test] + async fn replacement_identity_fence_rejects_a_remount_before_page_scan() { + let env = make_env_with_targets(vec!["replacement-a".to_string()]).await; + let expected_identity = ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "filesystem-a".to_string(), + }; + let remounted_identity = ReplacementTargetIdentity { + physical_device_ids: vec!["device-b".to_string()], + filesystem_identity: "filesystem-b".to_string(), + ..expected_identity.clone() + }; + env.storage + .replacement_target_identity_sequences + .lock() + .unwrap() + .push_back(vec![remounted_identity]); + let healer = ErasureSetHealer::new( + env.storage.clone(), + Arc::new(RwLock::new(HealProgress::new())), + CancellationToken::new(), + env.healer.disk.clone(), + HealOpts::default(), + HealRequestSource::AutoHeal, + ) + .with_replacement_targets(vec!["replacement-a".to_string()], Some("generation-a".to_string())) + .with_replacement_identity_fence(Some(vec![expected_identity])); + let mut current_object_index = 0; + let mut processed = 0; + let mut successful = 0; + let mut failed = 0; + let mut skipped = 0; + + let error = healer + .heal_bucket_with_resume( + "b", + "pool_0_set_0", + 0, + &mut current_object_index, + &mut processed, + &mut successful, + &mut failed, + &mut skipped, + &env.resume, + &env.checkpoint, + ) + .await + .expect_err("a remounted target must not begin a new page scan"); + + assert!(error.to_string().contains("page scan")); + assert!(env.storage.calls().is_empty()); + } + + #[tokio::test] + async fn replacement_generation_never_reuses_another_disk_cursor() { + let env = make_env_with_targets(vec!["replacement-a".to_string()]).await; + ResumeManager::new_replacement_intent( + env.healer.disk.clone(), + ResumeUtils::generate_task_id(), + "pool_0_set_0".to_string(), + vec!["b".to_string()], + vec!["replacement-a".to_string()], + vec![crate::heal::resume::ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("first replacement intent should persist"); + + let healer = ErasureSetHealer::new( + env.storage.clone(), + Arc::new(RwLock::new(HealProgress::new())), + CancellationToken::new(), + env.healer.disk.clone(), + HealOpts::default(), + HealRequestSource::AutoHeal, + ) + .with_replacement_targets(vec!["replacement-a".to_string()], Some(ResumeUtils::generate_task_id())); + + let error = healer + .get_or_create_task_id("pool_0_set_0") + .await + .expect_err("a second replacement must not reuse the first replacement cursor"); + assert!( + !error.to_string().contains("generation-a"), + "the previous replacement generation must not be selected" + ); + } + #[tokio::test] async fn object_timeout_aborts_the_bucket_page_immediately() { let env = make_env().await; @@ -1330,13 +1655,14 @@ mod resume_loop_tests { .await .expect("new heal should allocate a task id"); - assert_ne!(task_id, "task", "a completed resume state must not suppress a new heal"); + assert_ne!(task_id, env.task_id, "a completed resume state must not suppress a new heal"); + assert!(uuid::Uuid::parse_str(&task_id).is_ok(), "new resume task ids must be UUIDs"); } #[tokio::test] async fn cleanup_failure_keeps_erasure_set_heal_incomplete() { let env = make_env().await; - let checkpoint_path = format!("{BUCKET_META_PREFIX}/task_{RESUME_CHECKPOINT_FILE}"); + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{}_{RESUME_CHECKPOINT_FILE}", env.task_id); let _failure = ResumeDeleteFailure::install(checkpoint_path, crate::heal::DiskError::DiskAccessDenied); let error = env @@ -1346,7 +1672,7 @@ mod resume_loop_tests { .expect_err("checkpoint cleanup failure must fail the erasure-set heal"); assert!(matches!(error, Error::Disk(crate::heal::DiskError::DiskAccessDenied))); - let state = ResumeManager::load_from_disk(env.healer.disk.clone(), "task") + let state = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id) .await .expect("completed state must remain discoverable after cleanup failure") .get_state() @@ -1354,6 +1680,84 @@ mod resume_loop_tests { assert!(state.completed, "successful data heal must be persisted before cleanup is attempted"); } + #[tokio::test] + async fn replacement_completion_keeps_resume_artifacts_until_marker_cleanup() { + let env = make_env_with_targets(vec!["replacement-a".to_string()]).await; + let replacement_task_id = ResumeUtils::generate_task_id(); + ResumeManager::new_replacement_intent( + env.healer.disk.clone(), + replacement_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["b".to_string()], + vec!["replacement-a".to_string()], + vec![crate::heal::resume::ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + let checkpoint = CheckpointManager::new(env.healer.disk.clone(), replacement_task_id.clone()) + .await + .expect("replacement checkpoint should persist"); + let healer = ErasureSetHealer::new( + env.storage.clone(), + Arc::new(RwLock::new(HealProgress::new())), + CancellationToken::new(), + env.healer.disk.clone(), + HealOpts::default(), + HealRequestSource::AutoHeal, + ) + .with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone())); + + healer + .heal_erasure_set(&["b".to_string()], "pool_0_set_0") + .await + .expect("replacement data scan should complete"); + + let state = ResumeManager::load_replacement_intent(env.healer.disk.clone(), &replacement_task_id) + .await + .expect("verified replacement state must remain after data scan") + .get_state() + .await; + assert!(state.completed, "the verified state must record a completed data scan"); + assert_eq!(state.replacement_phase, crate::heal::resume::ReplacementPhase::Verified); + assert!( + CheckpointManager::has_checkpoint(&env.healer.disk, &replacement_task_id).await, + "the checkpoint must survive until the caller clears the healing marker" + ); + drop(checkpoint); + } + + #[tokio::test] + async fn retry_exhaustion_keeps_resume_artifacts_for_recovery() { + let env = make_env().await; + for _ in 0..3 { + assert!(env.resume.schedule_retry().await.expect("retry state should persist")); + env.checkpoint + .reset_for_retry() + .await + .expect("checkpoint reset should persist"); + } + env.storage.fail_listing(); + + env.healer + .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &env.resume, &env.checkpoint) + .await + .expect_err("exhausted retry state must report the incomplete heal"); + + assert!( + ResumeManager::has_resume_state(&env.healer.disk, &env.task_id).await, + "retry exhaustion must not delete the resumable state while a marker may remain" + ); + assert!( + CheckpointManager::has_checkpoint(&env.healer.disk, &env.task_id).await, + "retry exhaustion must retain the checkpoint with the resumable state" + ); + } + #[tokio::test] async fn retry_resume_repairs_checkpoint_after_crash_between_resets() { let env = make_env().await; @@ -1385,7 +1789,7 @@ mod resume_loop_tests { let (_, checkpoint) = env .healer - .initialize_resume_state("task", "pool_0_set_0", &["b".to_string()]) + .initialize_resume_state(&env.task_id, "pool_0_set_0", &["b".to_string()]) .await .expect("resume initialization should repair a stale checkpoint"); let checkpoint = checkpoint.get_checkpoint().await; @@ -1653,4 +2057,132 @@ mod resume_loop_tests { "the skipped set must be cleared so the retry re-heals the version" ); } + + #[tokio::test] + async fn replacement_target_missing_from_success_result_retries_the_full_pass() { + let env = make_env_with_targets(vec!["replacement-a".to_string()]).await; + env.storage.set_page( + None, + Page { + items: vec![item("object", Some("v1"), false)], + next: None, + truncated: false, + }, + ); + env.storage.set_result( + "object", + Some("v1"), + HealResultItem { + after: Infos { + drives: vec![HealDriveInfo { + endpoint: "replacement-a".to_string(), + state: "missing".to_string(), + ..Default::default() + }], + }, + ..Default::default() + }, + ); + + let result = env + .healer + .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &env.resume, &env.checkpoint) + .await; + + result.expect_err("a missing replacement target must not report completion"); + let state = env.resume.get_state().await; + assert!(!state.completed); + assert_eq!(state.retry_count, 1); + assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("v1".to_string()))]); + assert!(env.checkpoint.get_checkpoint().await.processed_objects.is_empty()); + } + + #[tokio::test] + async fn replacement_target_readback_evidence_must_confirm_the_healed_version() { + let env = make_env_with_targets(vec!["replacement-a".to_string()]).await; + let healer = ErasureSetHealer::new( + env.storage.clone(), + Arc::new(RwLock::new(HealProgress::new())), + CancellationToken::new(), + env.healer.disk.clone(), + HealOpts::default(), + HealRequestSource::AutoHeal, + ) + .with_replacement_targets(vec!["replacement-a".to_string()], Some("generation-a".to_string())); + env.storage.set_page( + None, + Page { + items: vec![item("object", Some("v1"), false)], + next: None, + truncated: false, + }, + ); + env.storage.set_result( + "object", + Some("v1"), + HealResultItem { + after: Infos { + drives: vec![HealDriveInfo { + endpoint: "replacement-a".to_string(), + state: "ok".to_string(), + ..Default::default() + }], + }, + ..Default::default() + }, + ); + env.storage.set_replacement_commit_evidence("object", Some("v1"), false); + + let result = healer + .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &env.resume, &env.checkpoint) + .await; + + result.expect_err("a success result without target readback evidence must retry"); + assert_eq!(env.resume.get_state().await.retry_count, 1); + assert!(env.checkpoint.get_checkpoint().await.processed_objects.is_empty()); + } + + #[tokio::test] + async fn manual_targeted_heal_keeps_existing_best_effort_result_semantics() { + let env = make_env_with_targets(vec!["replacement-a".to_string()]).await; + let healer = ErasureSetHealer::new( + env.storage.clone(), + Arc::new(RwLock::new(HealProgress::new())), + CancellationToken::new(), + env.healer.disk.clone(), + HealOpts::default(), + HealRequestSource::Admin, + ) + .with_replacement_targets(vec!["replacement-a".to_string()], None); + env.storage.set_page( + None, + Page { + items: vec![item("object", Some("v1"), false)], + next: None, + truncated: false, + }, + ); + env.storage.set_result( + "object", + Some("v1"), + HealResultItem { + after: Infos { + drives: vec![HealDriveInfo { + endpoint: "replacement-a".to_string(), + state: "ok".to_string(), + ..Default::default() + }], + }, + ..Default::default() + }, + ); + env.storage.set_replacement_commit_evidence("object", Some("v1"), false); + + healer + .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &env.resume, &env.checkpoint) + .await + .expect("manual targeted healing must retain its existing success semantics"); + + assert_eq!(env.resume.get_state().await.retry_count, 0); + } } diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 4e1151656..e256b1c18 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -14,6 +14,7 @@ use crate::heal::{ progress::{HealProgress, HealStatistics}, + resume::{ReplacementPhase, ResumeManager, ResumeState, ResumeUtils}, storage::HealStorageAPI, task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType, demote_to_debug_when}, }; @@ -36,7 +37,7 @@ use tokio::{ use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; -use super::{DiskError, HealDiskExt as _, local_disk_map_read}; +use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read}; const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60); const LOG_COMPONENT_HEAL: &str = "heal"; @@ -54,6 +55,16 @@ const EVENT_HEAL_UNCLEAN_SHUTDOWN: &str = "heal_unclean_shutdown"; const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3; const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30); +fn durable_replacement_recovery_is_due(state: &ResumeState, task_id: &str) -> bool { + state.replacement_generation.as_deref() == Some(task_id) + && !state.replacement_targets.is_empty() + && ((!state.completed + && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) + && state.retry_count >= state.max_retries) + || (state.completed + && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending))) +} + // Admission/scheduler outcomes for per-object requests (Object/Metadata/MRF/ // ECDecode) log via demote_to_debug_when! — MRF, autoheal, and scanner // recovery loops submit those per object, so a full queue or a retry storm @@ -795,6 +806,12 @@ pub struct HealManager { task_aliases: Arc>>, /// Heal tasks waiting for a retry backoff to expire. retrying_heals: Arc>>, + /// Surviving disks that hold durable replacement intents, keyed by task ID. + /// This is rebuilt from durable state after restart and never crosses the + /// public request boundary. + replacement_recovery_anchors: Arc>>, + /// Set IDs whose durable replacement metadata is corrupt or conflicting. + replacement_recovery_blocked_sets: Arc>>, /// Storage layer interface storage: Arc, /// Cancel token @@ -812,6 +829,7 @@ struct HealQueueContext<'a> { active_heals: &'a Arc>>>, completed_heals: &'a Arc>>, retrying_heals: &'a Arc>>, + replacement_recovery_anchors: &'a Arc>>, config: &'a Arc>, statistics: &'a Arc>, storage: &'a Arc, @@ -1200,6 +1218,28 @@ impl HealManager { .retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id); } + fn block_replacement_recovery_set(&self, set_disk_id: &str) { + self.replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned") + .insert(set_disk_id.to_string()); + } + + fn replacement_recovery_set_is_blocked(&self, set_disk_id: &str) -> bool { + self.replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned") + .contains(set_disk_id) + } + + async fn validate_replacement_recovery_records(disk: &crate::heal::DiskStore) -> Result<()> { + ResumeUtils::migrate_legacy_replacement_records(disk).await?; + for task_id in ResumeUtils::get_replacement_intent_tasks(disk).await? { + ResumeManager::load_replacement_intent(disk.clone(), &task_id).await?; + } + Ok(()) + } + /// Create new HealManager pub fn new(storage: Arc, config: Option) -> Self { Self::new_with_workload_provider(storage, config, None) @@ -1220,6 +1260,8 @@ impl HealManager { completed_heals: Arc::new(Mutex::new(HashMap::new())), task_aliases: Arc::new(Mutex::new(HashMap::new())), retrying_heals: Arc::new(Mutex::new(HashMap::new())), + replacement_recovery_anchors: Arc::new(std::sync::Mutex::new(HashMap::new())), + replacement_recovery_blocked_sets: Arc::new(std::sync::Mutex::new(HashSet::new())), storage, cancel_token: CancellationToken::new(), statistics: Arc::new(RwLock::new(HealStatistics::new())), @@ -1257,6 +1299,10 @@ impl HealManager { // start scheduler self.start_scheduler().await?; + // Recover durable replacement intents before the scanner can admit a + // competing task for the same set. + self.process_unclean_shutdown().await; + // start auto disk scanner to heal unformatted disks if self.config.read().await.enable_auto_heal { self.start_auto_disk_scanner().await?; @@ -1271,10 +1317,6 @@ impl HealManager { ); } - // Detect a previous unclean shutdown (crash/power loss) and proactively - // verify all local erasure sets instead of waiting for the periodic scanner. - self.process_unclean_shutdown().await; - info!( target: "rustfs::heal::manager", event = EVENT_HEAL_MANAGER_STATE, @@ -1293,10 +1335,16 @@ impl HealManager { async fn process_unclean_shutdown(&self) { let mut unclean = false; let mut set_disk_ids = HashSet::new(); + let mut replacement_intents = HashMap::, Vec, String)>::new(); + let mut replacement_restarts = HashMap::)>::new(); + let mut conflicted_replacement_sets = HashSet::new(); { - let local_disk_map = local_disk_map_read().await; - for disk in local_disk_map.values().flatten() { + let local_disks = { + let local_disk_map = local_disk_map_read().await; + local_disk_map.values().flatten().cloned().collect::>() + }; + for disk in &local_disks { let endpoint = disk.endpoint(); match disk .read_all(super::RUSTFS_META_BUCKET, super::UNCLEAN_SHUTDOWN_MARKER_PATH) @@ -1336,8 +1384,230 @@ impl HealManager { ); } - if let Some(set_disk_id) = crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx) { - set_disk_ids.insert(set_disk_id); + let disk_set_disk_id = crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx); + if let Some(set_disk_id) = &disk_set_disk_id { + set_disk_ids.insert(set_disk_id.clone()); + } + + // Legacy flat records are inspected only while starting. The + // periodic scanner lists the dedicated replacement directory. + if let Err(error) = ResumeUtils::migrate_legacy_replacement_records(disk).await { + if let Some(set_disk_id) = &disk_set_disk_id { + self.block_replacement_recovery_set(set_disk_id); + } + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + endpoint = %endpoint, + error = %error, + "Legacy replacement recovery migration failed" + ); + } + let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await { + Ok(task_ids) => task_ids, + Err(error) => { + if let Some(set_disk_id) = &disk_set_disk_id { + self.block_replacement_recovery_set(set_disk_id); + } + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + endpoint = %endpoint, + error = %error, + "Replacement recovery discovery failed" + ); + continue; + } + }; + for task_id in replacement_task_ids { + let manager = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await { + Ok(manager) => manager, + Err(error) => { + if let Some(set_disk_id) = &disk_set_disk_id { + self.block_replacement_recovery_set(set_disk_id); + } + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + endpoint = %endpoint, + task_id, + error = %error, + "Replacement recovery intent load failed" + ); + continue; + } + }; + let state = manager.get_state().await; + let active_replacement = !state.completed + && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding); + let verified_replacement = state.completed + && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending); + if (active_replacement || verified_replacement) + && state.replacement_generation.as_deref() == Some(task_id.as_str()) + && !state.replacement_targets.is_empty() + { + if matches!(state.replacement_phase, ReplacementPhase::CleanupPending) { + replacement_intents.entry(task_id).or_insert(( + state.set_disk_id, + state.replacement_targets, + state.replacement_buckets, + endpoint.to_string(), + )); + continue; + } + match self.storage.replacement_target_identities(&state.replacement_targets).await { + Ok(identities) if identities == state.replacement_target_identities => { + let resume_endpoint = endpoint.to_string(); + match replacement_intents.entry(task_id) { + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(( + state.set_disk_id, + state.replacement_targets, + state.replacement_buckets, + resume_endpoint, + )); + } + std::collections::hash_map::Entry::Occupied(entry) => { + let (existing_set_disk_id, existing_targets, existing_buckets, existing_anchor) = + entry.get(); + if existing_set_disk_id != &state.set_disk_id + || existing_targets != &state.replacement_targets + || existing_buckets != &state.replacement_buckets + || existing_anchor != &resume_endpoint + { + conflicted_replacement_sets.insert(state.set_disk_id.clone()); + self.block_replacement_recovery_set(&state.set_disk_id); + } + } + } + } + Ok(_) => { + if manager.abandon_replacement_intent().await.is_ok() { + replacement_restarts + .entry(task_id) + .or_insert((state.set_disk_id, state.replacement_targets)); + } + } + Err(_) => {} + } + } + } + } + } + + if !unclean && replacement_intents.is_empty() && replacement_restarts.is_empty() { + return; + } + + let mut recovery_by_set = HashMap::, Vec, Vec, Option)>>::new(); + for (task_id, (set_disk_id, heal_endpoints, buckets, resume_endpoint)) in replacement_intents { + recovery_by_set + .entry(set_disk_id) + .or_default() + .push((Some(task_id), heal_endpoints, buckets, Some(resume_endpoint))); + } + for (_abandoned_task_id, (set_disk_id, heal_endpoints)) in replacement_restarts { + recovery_by_set + .entry(set_disk_id) + .or_default() + .push((None, heal_endpoints, Vec::new(), None)); + } + + for (set_disk_id, mut recoveries) in recovery_by_set { + let Ok((pool_index, set_index)) = crate::heal::utils::parse_set_disk_id(&set_disk_id) else { + continue; + }; + if self.replacement_recovery_set_is_blocked(&set_disk_id) { + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + set_disk_id, + recovery_count = recoveries.len(), + "Replacement recovery deferred because durable recovery validation is blocked" + ); + continue; + } + if conflicted_replacement_sets.contains(&set_disk_id) || recoveries.len() != 1 { + self.block_replacement_recovery_set(&set_disk_id); + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + set_disk_id, + recovery_count = recoveries.len(), + "Replacement recovery deferred because multiple durable generations exist" + ); + continue; + } + let reuse_single_generation = recoveries.len() == 1 && recoveries[0].0.is_some(); + let mut heal_endpoints = recoveries + .iter_mut() + .flat_map(|(_, targets, _, _)| std::mem::take(targets)) + .collect::>(); + heal_endpoints.sort_unstable(); + heal_endpoints.dedup(); + let buckets = if reuse_single_generation { + std::mem::take(&mut recoveries[0].2) + } else { + Vec::new() + }; + let mut req = HealRequest::new( + HealType::ErasureSet { + buckets, + set_disk_id: set_disk_id.clone(), + }, + HealOptions { + pool_index: Some(pool_index), + set_index: Some(set_index), + timeout: None, + ..HealOptions::default() + }, + HealPriority::Low, + ); + if reuse_single_generation && let Some(task_id) = recoveries[0].0.take() { + req.id = task_id; + } + let recovery_anchor = reuse_single_generation.then(|| recoveries[0].3.take()).flatten(); + req.source = HealRequestSource::AutoHeal; + req.heal_endpoints = heal_endpoints; + let request_id = req.id.clone(); + if let Some(anchor) = &recovery_anchor { + self.replacement_recovery_anchors + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(request_id.clone(), anchor.clone()); + } + match self.submit_heal_request(req).await { + Ok(HealAdmissionResult::Accepted) => {} + Ok(_) => { + self.replacement_recovery_anchors + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(&request_id); + } + Err(err) => { + self.replacement_recovery_anchors + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(&request_id); + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + set_disk_id, + error = %err, + "Replacement recovery enqueue failed" + ); } } } @@ -2100,6 +2370,7 @@ impl HealManager { let active_heals = self.active_heals.clone(); let completed_heals = self.completed_heals.clone(); let retrying_heals = self.retrying_heals.clone(); + let replacement_recovery_anchors = self.replacement_recovery_anchors.clone(); let cancel_token = self.cancel_token.clone(); let statistics = self.statistics.clone(); let storage = self.storage.clone(); @@ -2129,6 +2400,7 @@ impl HealManager { active_heals: &active_heals, completed_heals: &completed_heals, retrying_heals: &retrying_heals, + replacement_recovery_anchors: &replacement_recovery_anchors, config: &config, statistics: &statistics, storage: &storage, @@ -2144,6 +2416,7 @@ impl HealManager { active_heals: &active_heals, completed_heals: &completed_heals, retrying_heals: &retrying_heals, + replacement_recovery_anchors: &replacement_recovery_anchors, config: &config, statistics: &statistics, storage: &storage, @@ -2165,8 +2438,10 @@ impl HealManager { let config = self.config.clone(); let heal_queue = self.heal_queue.clone(); let active_heals = self.active_heals.clone(); - let cancel_token = self.cancel_token.clone(); let storage = self.storage.clone(); + let replacement_recovery_anchors = self.replacement_recovery_anchors.clone(); + let replacement_recovery_blocked_sets = self.replacement_recovery_blocked_sets.clone(); + let cancel_token = self.cancel_token.clone(); let notify = self.notify.clone(); let mut duration = { let config = config.read().await; @@ -2210,18 +2485,106 @@ impl HealManager { } _ = interval.tick() => { // Build list of endpoints that need healing - let mut endpoints = Vec::new(); - let mut seen_returning_sets = HashSet::new(); - let local_disk_map = local_disk_map_read().await; - for disk in local_disk_map.values().flatten() { + let mut endpoints = HashMap::>::new(); + let mut durable_recoveries = HashMap::, Vec, String)>::new(); + let mut conflicted_recovery_sets = HashSet::::new(); + let local_disks = { + let local_disk_map = local_disk_map_read().await; + local_disk_map.values().flatten().cloned().collect::>() + }; + let local_endpoints = local_disks.iter().map(|disk| disk.endpoint()).collect::>(); + let blocked_sets = replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned") + .clone(); + if !blocked_sets.is_empty() { + let mut retry_succeeded = HashSet::new(); + let mut retry_failed = HashSet::new(); + for disk in &local_disks { + let endpoint = disk.endpoint(); + let Some(set_disk_id) = + crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx) + else { + continue; + }; + if !blocked_sets.contains(&set_disk_id) { + continue; + } + match Self::validate_replacement_recovery_records(disk).await { + Ok(()) => { + retry_succeeded.insert(set_disk_id); + } + Err(error) => { + retry_failed.insert(set_disk_id.clone()); + conflicted_recovery_sets.insert(set_disk_id); + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + error = %error, + "Replacement recovery retry failed" + ); + } + } + } + let mut blocked = replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned"); + for set_disk_id in retry_succeeded { + if !retry_failed.contains(&set_disk_id) { + blocked.remove(&set_disk_id); + } + } + } + for disk in &local_disks { let endpoint = disk.endpoint(); - let runtime_state = disk.runtime_state(); - let set_disk_id = - crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx); + let runtime_state = disk.runtime_state(); + let set_disk_id = + crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx); + if set_disk_id.as_ref().is_some_and(|set_disk_id| { + replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned") + .contains(set_disk_id) + }) { + skipped_invalid_count += 1; + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_DISK, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + set_disk_id = set_disk_id.as_deref().unwrap_or_default(), + disk_state = "replacement_recovery_blocked", + "Heal auto-scan replacement deferred because durable recovery is blocked" + ); + continue; + } // detect unformatted disk via get_disk_id() match disk.get_disk_id().await { Err(DiskError::UnformattedDisk) => { + if !super::replacement_readiness::auto_replacement_target_ready(disk, &local_disks) + .await + { + skipped_invalid_count += 1; + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_DISK, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + disk_state = "replacement_path_unavailable", + "Heal auto-scan replacement deferred" + ); + continue; + } + let Some(set_disk_id) = set_disk_id else { + skipped_invalid_count += 1; + continue; + }; candidate_count += 1; debug!( target: "rustfs::heal::manager", @@ -2232,7 +2595,7 @@ impl HealManager { disk_state = "unformatted", "Heal auto-scan candidate detected" ); - endpoints.push(endpoint); + endpoints.entry(set_disk_id).or_default().push(endpoint); } Err(e) => { warn!( @@ -2247,10 +2610,7 @@ impl HealManager { ); } Ok(_) => { - if runtime_state.as_str() == "returning" - && let Some(set_disk_id) = set_disk_id - && seen_returning_sets.insert(set_disk_id.clone()) - { + if runtime_state.as_str() == "returning" && let Some(set_disk_id) = set_disk_id { candidate_count += 1; debug!( target: "rustfs::heal::manager", @@ -2262,12 +2622,160 @@ impl HealManager { disk_state = "returning", "Heal auto-scan returning disk candidate detected" ); - endpoints.push(endpoint); + endpoints.entry(set_disk_id).or_default().push(endpoint); } } } } + // Once formatting succeeds a replacement is no longer + // discoverable as UnformattedDisk. Re-admit exactly one + // incomplete durable generation per set after bounded + // scheduler retries are exhausted, or re-admit its + // verified terminal cleanup. Multiple generations are a + // durable conflict: leave every marker/state intact and + // require reconciliation rather than choosing one. + for disk in &local_disks { + let endpoint = disk.endpoint(); + let disk_set_disk_id = + crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx); + let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await { + Ok(task_ids) => task_ids, + Err(error) => { + if let Some(set_disk_id) = &disk_set_disk_id { + conflicted_recovery_sets.insert(set_disk_id.clone()); + } + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + error = %error, + "Replacement recovery discovery failed" + ); + continue; + } + }; + for task_id in replacement_task_ids { + let resume_manager = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await { + Ok(resume_manager) => resume_manager, + Err(error) => { + if let Some(set_disk_id) = &disk_set_disk_id { + conflicted_recovery_sets.insert(set_disk_id.clone()); + } + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + task_id, + error = %error, + "Replacement recovery intent load failed" + ); + continue; + } + }; + let state = resume_manager.get_state().await; + if !durable_replacement_recovery_is_due(&state, &task_id) { + continue; + } + if !matches!(state.replacement_phase, ReplacementPhase::CleanupPending) { + let Ok(identities) = storage.replacement_target_identities(&state.replacement_targets).await else { + continue; + }; + if identities != state.replacement_target_identities { + continue; + } + } + let targets = state + .replacement_targets + .iter() + .filter_map(|target| { + local_endpoints + .iter() + .find(|endpoint| endpoint.to_string() == *target) + .cloned() + }) + .collect::>(); + if targets.len() != state.replacement_targets.len() { + continue; + } + let Some(set_disk_id) = crate::heal::utils::format_set_disk_id_from_i32( + targets[0].pool_idx, + targets[0].set_idx, + ) else { + continue; + }; + if targets.iter().any(|target| { + crate::heal::utils::format_set_disk_id_from_i32(target.pool_idx, target.set_idx) + .as_deref() + != Some(set_disk_id.as_str()) + }) { + continue; + } + let resume_endpoint = disk.endpoint().to_string(); + match durable_recoveries.get(&set_disk_id) { + Some((existing_task_id, _, _, existing_anchor)) + if existing_task_id != &task_id || existing_anchor != &resume_endpoint => { + replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned") + .insert(set_disk_id.clone()); + conflicted_recovery_sets.insert(set_disk_id); + } + Some(_) => {} + None => { + durable_recoveries.insert( + set_disk_id, + (task_id, targets, state.replacement_buckets, resume_endpoint), + ); + } + } + } + } + + for set_disk_id in &conflicted_recovery_sets { + durable_recoveries.remove(set_disk_id); + endpoints.remove(set_disk_id); + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + set_disk_id, + result = "durable_generation_conflict", + "Replacement recovery deferred because multiple durable generations exist" + ); + } + + for (set_disk_id, (_, targets, _, _)) in &durable_recoveries { + let expected = targets.iter().map(ToString::to_string).collect::>(); + let observed = endpoints + .get(set_disk_id) + .map(|endpoints| endpoints.iter().map(ToString::to_string).collect::>()) + .unwrap_or_default(); + if !observed.is_subset(&expected) { + replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned") + .insert(set_disk_id.clone()); + conflicted_recovery_sets.insert(set_disk_id.clone()); + continue; + } + endpoints.entry(set_disk_id.clone()).or_default().extend(targets.clone()); + } + for set_disk_id in &conflicted_recovery_sets { + durable_recoveries.remove(set_disk_id); + endpoints.remove(set_disk_id); + } + + for target_endpoints in endpoints.values_mut() { + target_endpoints.sort_by_key(ToString::to_string); + target_endpoints.dedup_by(|left, right| left.to_string() == right.to_string()); + } + if endpoints.is_empty() { debug!( target: "rustfs::heal::manager", @@ -2280,40 +2788,26 @@ impl HealManager { continue; } - // Get bucket list for erasure set healing - let buckets = match storage.list_buckets().await { - Ok(buckets) => buckets.iter().map(|b| b.name.clone()).collect::>(), - Err(e) => { - error!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - state = "bucket_list_failed", - error = %e, - "Heal auto-scan bucket listing failed" - ); - continue; - } - }; - - // Create erasure set heal requests for each endpoint - for ep in endpoints { - let Some(set_disk_id) = - crate::heal::utils::format_set_disk_id_from_i32(ep.pool_idx, ep.set_idx) - else { - warn!( + // Admit one set task with every ready replacement target. Queue deduplication is + // set-scoped, so admitting endpoints independently would silently drop later targets. + for (set_disk_id, endpoints) in endpoints { + if replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned") + .contains(&set_disk_id) + { + skipped_invalid_count += 1; + debug!( target: "rustfs::heal::manager", event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint = %ep, - result = "skipped_invalid_set_disk_id", - "Heal auto-scan enqueue skipped" + set_disk_id, + result = "replacement_recovery_blocked", + "Heal auto-scan replacement admission deferred because durable recovery is blocked" ); - skipped_invalid_count += 1; continue; - }; + } // skip if already queued or healing // Use consistent lock order: queue first, then active_heals to avoid deadlock let mut skip = false; @@ -2343,7 +2837,7 @@ impl HealManager { event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint = %ep, + endpoint_count = endpoints.len(), set_disk_id, result = "skipped_duplicate", "Heal auto-scan duplicate skipped" @@ -2351,25 +2845,49 @@ impl HealManager { continue; } - // enqueue erasure set heal request for this disk + // enqueue erasure set heal request for all ready replacements in this set + let recovery = durable_recoveries.remove(&set_disk_id); let mut req = HealRequest::new( HealType::ErasureSet { - buckets: buckets.clone(), + buckets: recovery + .as_ref() + .map(|(_, _, buckets, _)| buckets.clone()) + .unwrap_or_default(), set_disk_id: set_disk_id.clone(), }, HealOptions { + pool_index: endpoints + .first() + .and_then(|endpoint| usize::try_from(endpoint.pool_idx).ok()), + set_index: endpoints + .first() + .and_then(|endpoint| usize::try_from(endpoint.set_idx).ok()), timeout: None, ..HealOptions::default() }, HealPriority::Low, ); + let recovery_anchor = recovery.as_ref().map(|(_, _, _, anchor)| anchor.clone()); + if let Some((task_id, _, _, _)) = recovery { + req.id = task_id; + } req.source = HealRequestSource::AutoHeal; - req.heal_endpoints = vec![ep.to_string()]; + req.heal_endpoints = endpoints.iter().map(ToString::to_string).collect(); + let request_id = req.id.clone(); + let endpoint_count = req.heal_endpoints.len(); let config = config.read().await; let mut queue = heal_queue.lock().await; let admission = Self::admit_request_to_queue(&mut queue, req, &config, "auto_scan"); let should_notify = matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable; + if matches!(admission, HealAdmissionResult::Accepted) + && let Some(anchor) = recovery_anchor + { + replacement_recovery_anchors + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(request_id, anchor); + } drop(queue); drop(config); if matches!(admission, HealAdmissionResult::Accepted) { @@ -2382,9 +2900,9 @@ impl HealManager { event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint = %ep, + endpoint_count, set_disk_id, - bucket_count = buckets.len(), + bucket_count = 0, result = "enqueued", "Heal auto-scan task enqueued" ); @@ -2405,9 +2923,9 @@ impl HealManager { event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint = %ep, + endpoint_count, set_disk_id, - bucket_count = buckets.len(), + bucket_count = 0, admission = admission.result_label(), reason = admission.reason_label(), result = "not_enqueued", @@ -2445,6 +2963,7 @@ impl HealManager { active_heals, completed_heals, retrying_heals, + replacement_recovery_anchors, config, statistics, storage, @@ -2509,7 +3028,16 @@ impl HealManager { { *running_per_set.entry(set_key).or_insert(0) += 1; } - let task = Arc::new(HealTask::from_request(request, storage.clone())); + let replacement_resume_endpoint = replacement_recovery_anchors + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .get(&request.id) + .cloned(); + let task = Arc::new(HealTask::from_replacement_recovery_request( + request, + storage.clone(), + replacement_resume_endpoint, + )); let task_id = task.id.clone(); active_heals_guard.insert(task_id.clone(), task.clone()); publish_active_heal_count(&active_heals_guard); @@ -2518,6 +3046,7 @@ impl HealManager { let heal_queue_clone = heal_queue.clone(); let completed_heals_clone = completed_heals.clone(); let retrying_heals_clone = retrying_heals.clone(); + let replacement_recovery_anchors_clone = replacement_recovery_anchors.clone(); let statistics_clone = statistics.clone(); let notify_clone = notify.clone(); let manager_cancel_token = cancel_token.clone(); @@ -2592,6 +3121,12 @@ impl HealManager { }); let retry_request_for_queue = retry_request; let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new()); + if retry_request_for_queue.is_none() { + replacement_recovery_anchors_clone + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(&task_id); + } let mut active_heals_guard = active_heals_clone.lock().await; // Keep retry ownership continuous: status snapshots acquire // these locks in the same active -> retrying order. @@ -2961,8 +3496,45 @@ mod tests { use rustfs_common::heal_channel::{HealOpts, HealRequestSource}; use rustfs_concurrency::{WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot}; use rustfs_madmin::heal_commands::HealResultItem; + use tempfile::TempDir; - use super::super::{DiskStore, Endpoint, storage_api::status::BucketInfo}; + use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo}; + + #[tokio::test] + async fn auto_replacement_path_requires_a_non_root_mount() { + let temp = TempDir::new().expect("temporary replacement root should be created"); + let ready = Endpoint::try_from(temp.path().to_string_lossy().as_ref()).expect("replacement endpoint should parse"); + let missing = Endpoint::try_from(temp.path().join("missing").to_string_lossy().as_ref()) + .expect("missing replacement endpoint should parse"); + + let ready_disk = new_disk( + &ready, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("temporary disk should initialize"); + assert!( + !super::super::replacement_readiness::auto_replacement_target_ready(&ready_disk, std::slice::from_ref(&ready_disk),) + .await + ); + assert!( + matches!( + new_disk( + &missing, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await, + Err(DiskError::VolumeNotFound) + ), + "a missing replacement path must be rejected before admission" + ); + } #[derive(Debug)] struct FixedWorkloadProvider { @@ -2988,6 +3560,7 @@ mod tests { active_heals: &manager.active_heals, completed_heals: &manager.completed_heals, retrying_heals: &manager.retrying_heals, + replacement_recovery_anchors: &manager.replacement_recovery_anchors, config: &manager.config, statistics: &manager.statistics, storage: &manager.storage, @@ -3842,6 +4415,116 @@ mod tests { assert!(retry_error.contains("Storage resources are insufficient")); } + #[test] + fn test_retry_request_for_durable_replacement_retry_signal() { + let storage: Arc = Arc::new(MockStorage); + let task = HealTask::from_request( + HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ), + storage, + ); + let result = Err(Error::transient_skip("Replacement erasure set heal incomplete; retry scheduled")); + + let (retry_request, retry_delay, retry_error) = + retry_request_for_result(&task, &result).expect("typed replacement retry signal must be scheduled"); + + assert_eq!(retry_request.id, task.id); + assert_eq!(retry_request.retry_attempts, 1); + assert!(retry_delay > Duration::ZERO); + assert!(retry_error.contains("Replacement erasure set heal incomplete")); + } + + #[test] + fn durable_replacement_recovery_re_admits_only_the_matching_generation() { + let task_id = "replacement-generation"; + let mut state = crate::heal::resume::ResumeState::new( + task_id.to_string(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ); + state.replacement_generation = Some(task_id.to_string()); + state.replacement_phase = ReplacementPhase::Intent; + state.replacement_targets = vec!["replacement-a".to_string()]; + assert!(!durable_replacement_recovery_is_due(&state, task_id)); + + state.retry_count = state.max_retries; + assert!(durable_replacement_recovery_is_due(&state, task_id)); + + state.completed = true; + state.retry_count = 0; + state.replacement_phase = ReplacementPhase::Verified; + assert!( + durable_replacement_recovery_is_due(&state, task_id), + "verified terminal cleanup must be re-admitted without re-running recovery" + ); + + state.replacement_phase = ReplacementPhase::CleanupPending; + assert!( + durable_replacement_recovery_is_due(&state, task_id), + "cleanup-pending terminal cleanup must be periodically re-admitted" + ); + + state.completed = false; + state.replacement_generation = Some("another-generation".to_string()); + assert!( + !durable_replacement_recovery_is_due(&state, task_id), + "a task must not adopt another generation's durable intent" + ); + + state.replacement_generation = Some(task_id.to_string()); + state.replacement_targets.clear(); + assert!( + !durable_replacement_recovery_is_due(&state, task_id), + "a durable retry without a target is not safe to re-admit" + ); + + state.replacement_targets = vec!["replacement-a".to_string()]; + state.replacement_phase = ReplacementPhase::Verified; + assert!( + !durable_replacement_recovery_is_due(&state, task_id), + "a task must not adopt another generation's terminal cleanup" + ); + } + + #[test] + fn replacement_recovery_blocker_is_set_scoped() { + let manager = HealManager::new(Arc::new(MockStorage), None); + + manager.block_replacement_recovery_set("pool_0_set_0"); + + assert!(manager.replacement_recovery_set_is_blocked("pool_0_set_0")); + assert!(!manager.replacement_recovery_set_is_blocked("pool_0_set_1")); + } + + #[test] + fn replacement_recovery_blocks_only_confirmed_conflicts() { + assert!(crate::heal::resume::replacement_recovery_error_requires_block( + &Error::TaskExecutionFailed { + message: "replacement recovery conflict: proof mismatch".to_string(), + } + )); + assert!(crate::heal::resume::replacement_recovery_error_requires_block( + &Error::TaskExecutionFailed { + message: "replacement recovery corruption: malformed legacy intent".to_string(), + } + )); + assert!(!crate::heal::resume::replacement_recovery_error_requires_block(&Error::Disk( + DiskError::Timeout + ))); + assert!(!crate::heal::resume::replacement_recovery_error_requires_block( + &Error::TaskExecutionFailed { + message: "Failed to list replacement recovery records: temporary I/O error".to_string(), + } + )); + } + #[test] fn test_retry_request_for_scoped_slowdown_preserves_scope() { let storage: Arc = Arc::new(MockStorage); diff --git a/crates/heal/src/heal/mod.rs b/crates/heal/src/heal/mod.rs index 7fc50323a..0881e5853 100644 --- a/crates/heal/src/heal/mod.rs +++ b/crates/heal/src/heal/mod.rs @@ -17,6 +17,7 @@ pub mod erasure_healer; pub mod event; pub mod manager; pub mod progress; +pub(crate) mod replacement_readiness; pub mod resume; pub mod storage; pub(crate) mod storage_api; @@ -25,8 +26,8 @@ pub mod utils; use storage_api::owner::{ ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET, - EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore, - EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations, + EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, + EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations, ecstore_local_disk_map_read, }; #[cfg(test)] @@ -76,56 +77,155 @@ pub(crate) const HEALING_MARKER_PATH: &str = ECSTORE_HEALING_MARKER_PATH; /// Write the healing marker on the local disks matching `endpoints` so their /// `DiskInfo.healing` reports true while the erasure-set heal rebuilds them. -pub(crate) async fn set_healing_markers(endpoints: &[String], set_disk_id: &str) { - apply_healing_markers(endpoints, Some(set_disk_id)).await; +pub(crate) async fn set_healing_markers(endpoints: &[String], marker: &str) -> crate::Result<()> { + apply_healing_markers(endpoints, Some(marker), None, false).await } -/// Remove the healing markers written by [`set_healing_markers`]. -pub(crate) async fn clear_healing_markers(endpoints: &[String]) { - apply_healing_markers(endpoints, None).await; +/// Remove an owner marker after the replacement scan's verified state is +/// durable. A missing marker is idempotent here because a crash may have +/// happened after the previous terminal clear and before resume cleanup. +pub(crate) async fn clear_healing_markers_after_verified(endpoints: &[String], marker: &str) -> crate::Result<()> { + apply_healing_markers(endpoints, None, Some(marker), true).await } -async fn apply_healing_markers(endpoints: &[String], set_disk_id: Option<&str>) { +#[cfg(test)] +fn marker_matches(current: &[u8], expected_marker: Option<&str>) -> bool { + expected_marker.is_some_and(|expected| current == expected.as_bytes()) +} + +async fn apply_healing_markers( + endpoints: &[String], + marker: Option<&str>, + expected_marker: Option<&str>, + allow_missing: bool, +) -> crate::Result<()> { if endpoints.is_empty() { - return; + return Ok(()); } - let local_disk_map = local_disk_map_read().await; - for disk in local_disk_map.values().flatten() { - let endpoint = EcstoreDiskAPI::endpoint(disk.as_ref()).to_string(); - if !endpoints.iter().any(|candidate| candidate == &endpoint) { - continue; + let mut local_disks = std::collections::HashMap::new(); + { + let local_disk_map = local_disk_map_read().await; + for disk in local_disk_map.values().flatten() { + local_disks.insert(EcstoreDiskAPI::endpoint(disk.as_ref()).to_string(), disk.clone()); } - let result = match set_disk_id { - Some(set_disk_id) => { - EcstoreDiskAPI::write_all( + } + + let mut matched_endpoints = std::collections::HashSet::new(); + let mut targets = Vec::with_capacity(endpoints.len()); + for endpoint in endpoints { + if !matched_endpoints.insert(endpoint.clone()) { + return Err(DiskError::other("healing marker endpoint is duplicated").into()); + } + let Some(disk) = local_disks.remove(endpoint) else { + return Err(DiskError::other("healing marker target is unavailable").into()); + }; + targets.push(disk); + } + + apply_healing_markers_to_targets(targets, marker, expected_marker, allow_missing).await +} + +async fn apply_healing_markers_to_targets( + targets: Vec, + marker: Option<&str>, + expected_marker: Option<&str>, + allow_missing: bool, +) -> crate::Result<()> { + apply_healing_markers_to_targets_with_after_acquire(targets, marker, expected_marker, allow_missing, |_| {}).await +} + +async fn apply_healing_markers_to_targets_with_after_acquire( + targets: Vec, + marker: Option<&str>, + expected_marker: Option<&str>, + allow_missing: bool, + mut after_acquire: F, +) -> crate::Result<()> +where + F: FnMut(&DiskStore), +{ + let marker_bytes = marker.map(|marker| EcstoreDiskBytes::copy_from_slice(marker.as_bytes())); + let expected_bytes = expected_marker.map(|marker| EcstoreDiskBytes::copy_from_slice(marker.as_bytes())); + let mut newly_acquired = Vec::new(); + for disk in targets { + let result = match marker_bytes.as_ref() { + Some(marker) => { + match EcstoreDiskAPI::compare_and_update_file( disk.as_ref(), RUSTFS_META_BUCKET, HEALING_MARKER_PATH, - EcstoreDiskBytes::copy_from_slice(set_disk_id.as_bytes()), + None, + Some(marker.clone()), ) .await + { + Ok(EcstoreConditionalFileUpdate::Updated) => { + newly_acquired.push(disk.clone()); + after_acquire(&disk); + Ok(()) + } + Ok(EcstoreConditionalFileUpdate::Mismatch) => match EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + HEALING_MARKER_PATH, + Some(marker.clone()), + Some(marker.clone()), + ) + .await + { + Ok(EcstoreConditionalFileUpdate::Updated) => Ok(()), + Ok(_) => Err(DiskError::other("healing marker ownership changed")), + Err(err) => Err(err), + }, + Ok(EcstoreConditionalFileUpdate::Missing) => Err(DiskError::other("healing marker disappeared")), + Err(err) => Err(err), + } + } + None => { + match EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + HEALING_MARKER_PATH, + expected_bytes.clone(), + None, + ) + .await + { + Ok(EcstoreConditionalFileUpdate::Updated) => Ok(()), + Ok(EcstoreConditionalFileUpdate::Missing) if allow_missing => Ok(()), + Ok(EcstoreConditionalFileUpdate::Missing) => Err(DiskError::other("healing marker is missing")), + Ok(EcstoreConditionalFileUpdate::Mismatch) => Err(DiskError::other("healing marker ownership changed")), + Err(err) => Err(err), + } } - None => match EcstoreDiskAPI::delete( - disk.as_ref(), - RUSTFS_META_BUCKET, - HEALING_MARKER_PATH, - EcstoreDeleteOptions::default(), - ) - .await - { - Err(DiskError::FileNotFound) => Ok(()), - other => other, - }, }; if let Err(err) = result { - tracing::warn!( - endpoint = %endpoint, - action = if set_disk_id.is_some() { "set" } else { "clear" }, - error = ?err, - "failed to update healing marker" - ); + if let Some(marker) = marker_bytes.as_ref() { + let mut rollback_error = None; + for acquired in newly_acquired.iter().rev() { + if let Err(rollback) = EcstoreDiskAPI::compare_and_update_file( + acquired.as_ref(), + RUSTFS_META_BUCKET, + HEALING_MARKER_PATH, + Some(marker.clone()), + None, + ) + .await + { + rollback_error.get_or_insert(rollback); + } + } + if let Some(rollback) = rollback_error { + return Err(DiskError::other(format!( + "healing marker acquisition failed ({err}) and owner-safe rollback failed ({rollback})" + )) + .into()); + } + } + return Err(err.into()); } } + Ok(()) } pub(crate) type DiskError = EcstoreDiskError; @@ -203,3 +303,195 @@ where pub type HealObjectInfo = ::ObjectInfo; pub type HealObjectOptions = ::ObjectOptions; pub type HealPutObjReader = ::PutObjectReader; + +#[cfg(test)] +mod tests { + use super::{ + DiskError, DiskOption, Endpoint, HEALING_MARKER_PATH, RUSTFS_META_BUCKET, apply_healing_markers_to_targets, + apply_healing_markers_to_targets_with_after_acquire, marker_matches, new_disk, + }; + use crate::{ + Error, + heal::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes}, + }; + use tempfile::TempDir; + + async fn make_marker_disk(temp: &TempDir, name: &str) -> super::DiskStore { + let path = temp.path().join(name); + std::fs::create_dir_all(&path).expect("marker disk directory should be created"); + let endpoint = Endpoint::try_from(path.to_string_lossy().as_ref()).expect("marker disk endpoint should be valid"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("marker disk should initialize"); + let metadata_volume = disk.make_volume(RUSTFS_META_BUCKET).await; + assert!( + matches!(metadata_volume, Ok(()) | Err(DiskError::VolumeExists)), + "marker metadata volume should exist: {metadata_volume:?}" + ); + disk + } + + #[test] + fn marker_clear_requires_the_current_owner_token() { + assert!(marker_matches(b"set:task-a", Some("set:task-a"))); + assert!(!marker_matches(b"set:task-b", Some("set:task-a"))); + assert!(!marker_matches(b"set:task-a", None)); + } + + #[tokio::test] + async fn marker_acquisition_rolls_back_after_second_disk_ownership_conflict() { + let temp = TempDir::new().expect("marker test directory should be created"); + let first = make_marker_disk(&temp, "first").await; + let second = make_marker_disk(&temp, "second").await; + let owner_b = EcstoreDiskBytes::from_static(b"owner-b"); + + assert_eq!( + EcstoreDiskAPI::compare_and_update_file( + second.as_ref(), + RUSTFS_META_BUCKET, + HEALING_MARKER_PATH, + None, + Some(owner_b.clone()), + ) + .await + .expect("second disk owner should acquire marker"), + EcstoreConditionalFileUpdate::Updated + ); + + let err = apply_healing_markers_to_targets(vec![first.clone(), second.clone()], Some("owner-a"), None, false) + .await + .expect_err("second disk ownership must reject the partial acquisition"); + assert!(matches!(err, Error::Disk(DiskError::Io(ref io)) if io.to_string() == "healing marker ownership changed")); + assert!(matches!( + EcstoreDiskAPI::read_all(first.as_ref(), RUSTFS_META_BUCKET, HEALING_MARKER_PATH).await, + Err(DiskError::FileNotFound) + )); + assert_eq!( + EcstoreDiskAPI::read_all(second.as_ref(), RUSTFS_META_BUCKET, HEALING_MARKER_PATH) + .await + .expect("conflicting owner marker must remain"), + owner_b + ); + } + + #[tokio::test] + async fn marker_acquisition_rolls_back_after_second_disk_io_error() { + let temp = TempDir::new().expect("marker test directory should be created"); + let first = make_marker_disk(&temp, "first").await; + let second_path = temp.path().join("second"); + std::fs::create_dir_all(&second_path).expect("second marker disk directory should be created"); + let second_endpoint = + Endpoint::try_from(second_path.to_string_lossy().as_ref()).expect("second marker endpoint should be valid"); + let second = new_disk( + &second_endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("second marker disk should initialize"); + std::fs::remove_dir_all(second_path.join(RUSTFS_META_BUCKET)) + .expect("second marker metadata directory should be removed for the I/O failure fixture"); + std::fs::write(second_path.join(RUSTFS_META_BUCKET), b"not a directory") + .expect("second marker volume should become an I/O failure fixture"); + + let err = apply_healing_markers_to_targets(vec![first.clone(), second], Some("owner-a"), None, false) + .await + .expect_err("second disk I/O failure must reject the partial acquisition"); + assert!( + matches!(err, Error::Disk(DiskError::FileAccessDenied)), + "second marker operation must report its mapped filesystem failure: {err:?}" + ); + assert!(matches!( + EcstoreDiskAPI::read_all(first.as_ref(), RUSTFS_META_BUCKET, HEALING_MARKER_PATH).await, + Err(DiskError::FileNotFound) + )); + } + + #[tokio::test] + async fn marker_acquisition_reports_an_owner_safe_rollback_io_failure() { + let temp = TempDir::new().expect("marker test directory should be created"); + let first = make_marker_disk(&temp, "first").await; + let second = make_marker_disk(&temp, "second").await; + let owner_b = EcstoreDiskBytes::from_static(b"owner-b"); + let first_path = EcstoreDiskAPI::path(first.as_ref()); + let moved_metadata_path = first_path.join("metadata-before-rollback"); + + assert_eq!( + EcstoreDiskAPI::compare_and_update_file( + second.as_ref(), + RUSTFS_META_BUCKET, + HEALING_MARKER_PATH, + None, + Some(owner_b), + ) + .await + .expect("second disk owner should acquire marker"), + EcstoreConditionalFileUpdate::Updated + ); + + let err = + apply_healing_markers_to_targets_with_after_acquire(vec![first, second], Some("owner-a"), None, false, |disk| { + let metadata_path = EcstoreDiskAPI::path(disk.as_ref()).join(RUSTFS_META_BUCKET); + std::fs::rename(&metadata_path, &moved_metadata_path) + .expect("first marker metadata should move after acquisition"); + std::fs::write(&metadata_path, b"not a directory") + .expect("first marker metadata should become a rollback I/O failure fixture"); + }) + .await + .expect_err("rollback I/O failure must remain visible to the caller"); + let message = err.to_string(); + assert!(message.contains("healing marker acquisition failed")); + assert!(message.contains("owner-safe rollback failed")); + assert!(moved_metadata_path.join(HEALING_MARKER_PATH).exists()); + } + + #[tokio::test] + async fn concurrent_marker_acquisition_has_one_owner_on_every_disk() { + let temp = TempDir::new().expect("marker test directory should be created"); + let first = make_marker_disk(&temp, "first").await; + let second = make_marker_disk(&temp, "second").await; + let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(3)); + + let owner_a_barrier = barrier.clone(); + let owner_a_first = first.clone(); + let owner_a_second = second.clone(); + let owner_a = tokio::spawn(async move { + owner_a_barrier.wait().await; + apply_healing_markers_to_targets(vec![owner_a_first, owner_a_second], Some("owner-a"), None, false).await + }); + let owner_b_barrier = barrier.clone(); + let owner_b_first = first.clone(); + let owner_b_second = second.clone(); + let owner_b = tokio::spawn(async move { + owner_b_barrier.wait().await; + apply_healing_markers_to_targets(vec![owner_b_first, owner_b_second], Some("owner-b"), None, false).await + }); + + barrier.wait().await; + let owner_a_result = owner_a.await.expect("owner a task should join"); + let owner_b_result = owner_b.await.expect("owner b task should join"); + assert_ne!( + owner_a_result.is_ok(), + owner_b_result.is_ok(), + "exactly one owner must acquire both markers" + ); + + let winning_marker = if owner_a_result.is_ok() { b"owner-a" } else { b"owner-b" }; + for disk in [&first, &second] { + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, HEALING_MARKER_PATH) + .await + .expect("every disk must retain the winning owner marker"), + EcstoreDiskBytes::from_static(winning_marker) + ); + } + } +} diff --git a/crates/heal/src/heal/replacement_readiness.rs b/crates/heal/src/heal/replacement_readiness.rs new file mode 100644 index 000000000..f0ac9b640 --- /dev/null +++ b/crates/heal/src/heal/replacement_readiness.rs @@ -0,0 +1,160 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{fs, path::Path}; + +#[cfg(test)] +use super::Endpoint; +use super::{DiskStore, HealDiskExt as _, local_disk_map_read, resume::ReplacementTargetIdentity}; + +pub(crate) async fn auto_replacement_target_ready(disk: &DiskStore, local_disks: &[DiskStore]) -> bool { + auto_replacement_target_identity(disk, local_disks).await.is_some() +} + +pub(crate) async fn auto_replacement_target_identity( + disk: &DiskStore, + local_disks: &[DiskStore], +) -> Option { + let lease_root = disk.replacement_mount_lease_root()?; + let endpoint = disk.endpoint().to_string(); + let sibling_lease_roots = local_disks + .iter() + .filter(|sibling| sibling.endpoint().is_local && sibling.endpoint().to_string() != endpoint) + .map(|sibling| sibling.replacement_mount_lease_root()) + .collect::>>()?; + tokio::task::spawn_blocking(move || { + let canonical_path = fs::canonicalize(&lease_root).ok()?; + let metadata = fs::metadata(&lease_root).ok()?; + let Ok(target_device_ids) = rustfs_utils::os::get_physical_device_ids(lease_root.to_string_lossy().as_ref()) else { + return None; + }; + let Ok(root_device_ids) = rustfs_utils::os::get_physical_device_ids("/") else { + return None; + }; + if target_device_ids.is_empty() + || root_device_ids.is_empty() + || target_device_ids.iter().any(|target| root_device_ids.contains(target)) + || !rustfs_utils::os::is_mount_point(&canonical_path).unwrap_or(false) + { + return None; + } + + if sibling_lease_roots.iter().any(|sibling_lease_root| { + rustfs_utils::os::get_physical_device_ids(sibling_lease_root.to_string_lossy().as_ref()) + .map(|ids| ids.iter().any(|id| target_device_ids.contains(id))) + .unwrap_or(true) + }) { + return None; + } + + let filesystem_identity = filesystem_identity(&metadata, &canonical_path)?; + + Some(ReplacementTargetIdentity { + endpoint, + canonical_path: canonical_path.to_string_lossy().into_owned(), + physical_device_ids: target_device_ids, + filesystem_identity, + }) + }) + .await + .ok() + .flatten() +} + +pub(crate) async fn auto_replacement_targets_ready(targets: &[String]) -> bool { + auto_replacement_target_identities(targets).await.is_some() +} + +pub(crate) async fn auto_replacement_target_identities(targets: &[String]) -> Option> { + let local_disk_map = local_disk_map_read().await; + let local_disks = local_disk_map + .values() + .flatten() + .filter(|disk| disk.endpoint().is_local) + .cloned() + .collect::>(); + drop(local_disk_map); + + let mut identities = Vec::with_capacity(targets.len()); + for target in targets { + let disk = local_disks.iter().find(|disk| disk.endpoint().to_string() == *target)?; + identities.push(auto_replacement_target_identity(disk, &local_disks).await?); + } + identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint)); + identities.dedup_by(|left, right| left.endpoint == right.endpoint); + (identities.len() == targets.len()).then_some(identities) +} + +#[cfg(target_os = "linux")] +fn filesystem_identity(metadata: &fs::Metadata, canonical_path: &Path) -> Option { + use std::os::unix::fs::MetadataExt as _; + + let escaped_path = canonical_path.to_string_lossy().replace(' ', "\\040"); + let mountinfo = fs::read_to_string("/proc/self/mountinfo").ok()?; + let mount_id = mountinfo.lines().find_map(|line| { + let mut fields = line.split_whitespace(); + let mount_id = fields.next()?; + fields.next()?; + fields.next()?; + fields.next()?; + (fields.next()? == escaped_path).then_some(mount_id) + })?; + Some(format!("{mount_id}:{}:{}", metadata.dev(), metadata.ino())) +} + +#[cfg(all(unix, not(target_os = "linux")))] +fn filesystem_identity(metadata: &fs::Metadata, _canonical_path: &Path) -> Option { + use std::os::unix::fs::MetadataExt as _; + + Some(format!("{}:{}", metadata.dev(), metadata.ino())) +} + +#[cfg(not(unix))] +fn filesystem_identity(_metadata: &fs::Metadata, _canonical_path: &Path) -> Option { + None +} + +#[cfg(test)] +mod tests { + use super::super::{DiskOption, new_disk}; + use super::*; + use tempfile::TempDir; + + #[tokio::test] + async fn runtime_environment_cannot_bypass_mount_admission() { + temp_env::async_with_vars( + [ + ("RUSTFS_TEST_AUTO_REPLACEMENT_READINESS_BYPASS", Some("1")), + ("RUSTFS_E2E_AUTO_REPLACEMENT_READINESS_BYPASS", Some("1")), + ], + async { + let temp = TempDir::new().expect("temporary replacement root should be created"); + let endpoint = + Endpoint::try_from(temp.path().to_string_lossy().as_ref()).expect("replacement endpoint should parse"); + + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("temporary disk should initialize"); + assert!(!auto_replacement_target_ready(&disk, std::slice::from_ref(&disk)).await); + }, + ) + .await; + } +} diff --git a/crates/heal/src/heal/resume.rs b/crates/heal/src/heal/resume.rs index cd310e3c1..01601a761 100644 --- a/crates/heal/src/heal/resume.rs +++ b/crates/heal/src/heal/resume.rs @@ -17,14 +17,17 @@ use serde::{Deserialize, Serialize}; #[cfg(test)] use std::collections::HashMap; use std::collections::HashSet; -use std::path::Path; +use std::path::{Component, Path}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::sync::RwLock; use tracing::{debug, warn}; use uuid::Uuid; -use super::{BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET}; +use super::{ + BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET, + storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskBytes}, +}; const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_RESUME: &str = "resume"; @@ -33,18 +36,247 @@ const EVENT_HEAL_CHECKPOINT_STATE: &str = "heal_checkpoint_state"; /// resume state file constants const RESUME_STATE_FILE: &str = "ahm_resume_state.json"; +// Replacement intents must not use the ordinary resume-state suffix. Older +// binaries enumerate that suffix and can otherwise resume a replacement as a +// normal heal without its identity and format fences. +const REPLACEMENT_INTENT_FILE: &str = "ahm_replacement_intent.json"; const RESUME_PROGRESS_FILE: &str = "ahm_progress.json"; pub(super) const RESUME_CHECKPOINT_FILE: &str = "ahm_checkpoint.json"; +const REPLACEMENT_COMPLETION_PROOF_FILE: &str = "ahm_replacement_completion_proof.json"; +const REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement"; +const REPLACEMENT_INTENT_SEAL_FILE: &str = "ahm_replacement_intent_seal"; +const LEGACY_REPLACEMENT_RECOVERY_MARKER_FILE: &str = "ahm_replacement_recovery.json"; +const REPLACEMENT_RECOVERY_CONFLICT_PREFIX: &str = "replacement recovery conflict:"; +const REPLACEMENT_RECOVERY_CORRUPTION_PREFIX: &str = "replacement recovery corruption:"; +const CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA: u32 = 1; /// Current on-disk schema version for `ResumeState`. Snapshots written by an /// older schema (which tracked latest-only object names and a positional /// cursor) are incompatible with the per-version resume cursor, so they are /// discarded on load and the scan restarts from the beginning. -const CURRENT_RESUME_SCHEMA: u32 = 2; +const CURRENT_RESUME_SCHEMA: u32 = 5; /// Current on-disk schema version for `ResumeCheckpoint`. Same rationale as /// `CURRENT_RESUME_SCHEMA`: pre-per-version dedup identities are not comparable /// to the new `compose_key` identities, so a stale checkpoint is discarded. -const CURRENT_CHECKPOINT_SCHEMA: u32 = 2; +const CURRENT_CHECKPOINT_SCHEMA: u32 = 5; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReplacementPhase { + #[default] + None, + Intent, + Rebuilding, + Verified, + CleanupPending, + Abandoned, +} + +/// Target-specific state for a durable automatic replacement generation. +/// +/// This is deliberately separate from the legacy background-heal status +/// contract. Consumers must treat [`Self::Unknown`] as non-definitive. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReplacementRecoveryState { + WaitingForReplacement, + Running, + Incomplete, + Unrecoverable, + CleanupPending, + Completed, + Unknown, +} + +/// Read-only status derived from one durable replacement generation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReplacementRecoveryRecord { + pub task_id: String, + pub state: ReplacementRecoveryState, + pub generation: Option, + pub set_disk_id: Option, + pub target_slots: Vec, + pub reason: Option, + pub verified_at: Option, +} + +impl ReplacementRecoveryRecord { + fn from_state(state: ResumeState) -> Option { + if !is_replacement_intent(&state) { + return None; + } + + let invariant_holds = state.replacement_generation.as_deref() == Some(state.task_id.as_str()) + && replacement_targets_match_identities(&state.replacement_targets, &state.replacement_target_identities); + if !invariant_holds { + return Some(Self::unknown( + state.task_id, + "durable replacement state violates its generation or target identity binding", + )); + } + + let (state_kind, reason) = if !state.completed && state.retry_count >= state.max_retries { + ( + ReplacementRecoveryState::Unrecoverable, + Some("replacement retry budget exhausted".to_string()), + ) + } else if let Some(reason) = state.error_message.clone() { + (ReplacementRecoveryState::Incomplete, Some(reason)) + } else { + match state.replacement_phase { + ReplacementPhase::Intent => (ReplacementRecoveryState::WaitingForReplacement, None), + ReplacementPhase::Rebuilding => (ReplacementRecoveryState::Running, None), + ReplacementPhase::Verified | ReplacementPhase::CleanupPending => (ReplacementRecoveryState::CleanupPending, None), + ReplacementPhase::Abandoned => ( + ReplacementRecoveryState::Unrecoverable, + Some("replacement generation was abandoned".to_string()), + ), + ReplacementPhase::None => (ReplacementRecoveryState::Unknown, Some("replacement phase is missing".to_string())), + } + }; + + Some(Self { + task_id: state.task_id, + state: state_kind, + generation: state.replacement_generation, + set_disk_id: Some(state.set_disk_id), + target_slots: state.replacement_targets, + reason, + verified_at: None, + }) + } + + fn from_completion_proof(proof: &ReplacementCompletionProof) -> Self { + Self { + task_id: proof.task_id.clone(), + state: ReplacementRecoveryState::Completed, + generation: Some(proof.replacement_generation.clone()), + set_disk_id: Some(proof.set_disk_id.clone()), + target_slots: proof.replacement_targets.clone(), + reason: None, + verified_at: Some(proof.verified_at), + } + } + + fn unknown(task_id: String, reason: &str) -> Self { + Self { + task_id, + state: ReplacementRecoveryState::Unknown, + generation: None, + set_disk_id: None, + target_slots: Vec::new(), + reason: Some(reason.to_string()), + verified_at: None, + } + } +} + +fn replacement_targets_match_identities(targets: &[String], identities: &[ReplacementTargetIdentity]) -> bool { + !targets.is_empty() + && targets.len() == identities.len() + && targets.iter().collect::>().len() == targets.len() + && identities.iter().map(|identity| &identity.endpoint).eq(targets.iter()) +} + +/// Stable evidence for the mounted replacement instance that owns a repair +/// generation. Endpoint text alone is not sufficient because a later disk can +/// be mounted at the same configured path. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ReplacementTargetIdentity { + pub endpoint: String, + pub canonical_path: String, + pub physical_device_ids: Vec, + pub filesystem_identity: String, +} + +/// Durable terminal evidence for one automatic replacement generation. This +/// lives on the healthy non-target anchor rather than in the resumable state, +/// because resume cleanup must not erase proof that the generation completed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct ReplacementCompletionProof { + pub schema_version: u32, + pub task_id: String, + pub replacement_generation: String, + pub set_disk_id: String, + pub replacement_targets: Vec, + pub replacement_target_identities: Vec, + pub verified_at: u64, +} + +impl ReplacementCompletionProof { + fn from_state(state: &ResumeState, verified_at: u64) -> Result { + let replacement_generation = state + .replacement_generation + .clone() + .ok_or_else(|| Error::TaskExecutionFailed { + message: format!("Replacement completion has no generation for task {}", state.task_id), + })?; + if replacement_generation != state.task_id + || state.replacement_targets.is_empty() + || state + .replacement_target_identities + .iter() + .map(|identity| &identity.endpoint) + .collect::>() + != state.replacement_targets.iter().collect::>() + { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement completion identity does not match task {}", state.task_id), + }); + } + + Ok(Self { + schema_version: CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA, + task_id: state.task_id.clone(), + replacement_generation, + set_disk_id: state.set_disk_id.clone(), + replacement_targets: state.replacement_targets.clone(), + replacement_target_identities: state.replacement_target_identities.clone(), + verified_at, + }) + } + + fn matches_state(&self, state: &ResumeState) -> bool { + self.schema_version == CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA + && self.task_id == state.task_id + && state.replacement_generation.as_deref() == Some(self.replacement_generation.as_str()) + && self.set_disk_id == state.set_disk_id + && self.replacement_targets == state.replacement_targets + && self.replacement_target_identities == state.replacement_target_identities + } + + fn validate(&self, expected_task_id: &str) -> Result<()> { + if self.schema_version != CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement completion proof schema {} is unsupported", self.schema_version), + }); + } + validate_resume_task_id(expected_task_id)?; + if self.task_id != expected_task_id + || self.replacement_generation != self.task_id + || self.set_disk_id.is_empty() + || self.verified_at == 0 + || !replacement_targets_match_identities(&self.replacement_targets, &self.replacement_target_identities) + { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement completion proof does not match task {expected_task_id}"), + }); + } + Ok(()) + } +} + +pub(crate) fn replacement_target_identities_match( + expected: &[ReplacementTargetIdentity], + actual: &[ReplacementTargetIdentity], +) -> bool { + let mut expected = expected.to_vec(); + let mut actual = actual.to_vec(); + expected.sort_by(|left, right| left.endpoint.cmp(&right.endpoint)); + actual.sort_by(|left, right| left.endpoint.cmp(&right.endpoint)); + expected == actual +} /// Build the canonical, provably-injective dedup identity for an object /// version. Length-prefixing the object key makes the encoding injective: no @@ -94,6 +326,61 @@ fn path_to_str(path: &Path) -> Result<&str> { .ok_or_else(|| Error::other(format!("Invalid UTF-8 path: {path:?}"))) } +fn replacement_recovery_conflict(message: impl std::fmt::Display) -> Error { + Error::TaskExecutionFailed { + message: format!("{REPLACEMENT_RECOVERY_CONFLICT_PREFIX} {message}"), + } +} + +fn replacement_recovery_corruption(message: impl std::fmt::Display) -> Error { + Error::TaskExecutionFailed { + message: format!("{REPLACEMENT_RECOVERY_CORRUPTION_PREFIX} {message}"), + } +} + +pub(crate) fn replacement_recovery_error_requires_block(error: &Error) -> bool { + matches!( + error, + Error::TaskExecutionFailed { message } + if message.starts_with(REPLACEMENT_RECOVERY_CONFLICT_PREFIX) + || message.starts_with(REPLACEMENT_RECOVERY_CORRUPTION_PREFIX) + ) +} + +fn replacement_recovery_corruption_for_state_load(message: impl std::fmt::Display, error: Error) -> Error { + if replacement_recovery_error_requires_block(&error) { + error + } else if matches!( + error, + Error::TaskExecutionFailed { .. } | Error::Serialization(_) | Error::InvalidCheckpoint(_) + ) { + replacement_recovery_corruption(format!("{message}: {error}")) + } else { + error + } +} + +/// Resume task IDs become part of metadata file names. Persisted filenames are +/// untrusted, so only accept a UUID encoded as one normal path component. +fn validate_resume_task_id(task_id: &str) -> Result<()> { + let mut components = Path::new(task_id).components(); + let is_single_normal_component = matches!(components.next(), Some(Component::Normal(_))) && components.next().is_none(); + + let Ok(uuid) = Uuid::parse_str(task_id) else { + return Err(Error::TaskExecutionFailed { + message: "Invalid resume task id".to_string(), + }); + }; + + if is_single_normal_component && uuid.hyphenated().to_string() == task_id { + return Ok(()); + } + + Err(Error::TaskExecutionFailed { + message: "Invalid resume task id".to_string(), + }) +} + #[cfg(test)] pub(super) struct ResumeDeleteFailure { path: String, @@ -141,6 +428,53 @@ fn injected_resume_delete_error(_path: &str) -> Option { None } +#[cfg(test)] +struct ReplacementProofWriteFailure { + path: String, +} + +#[cfg(test)] +fn replacement_proof_write_failures() -> &'static Mutex> { + static FAILURES: std::sync::OnceLock>> = std::sync::OnceLock::new(); + FAILURES.get_or_init(|| Mutex::new(HashMap::new())) +} + +#[cfg(test)] +impl ReplacementProofWriteFailure { + fn install(path: String, error: DiskError) -> Self { + let previous = replacement_proof_write_failures() + .lock() + .expect("replacement proof write failure registry should not poison") + .insert(path.clone(), error); + assert!(previous.is_none(), "replacement proof write failure already installed"); + Self { path } + } +} + +#[cfg(test)] +impl Drop for ReplacementProofWriteFailure { + fn drop(&mut self) { + replacement_proof_write_failures() + .lock() + .expect("replacement proof write failure registry should not poison") + .remove(&self.path); + } +} + +#[cfg(test)] +fn injected_replacement_proof_write_error(path: &str) -> Option { + replacement_proof_write_failures() + .lock() + .expect("replacement proof write failure registry should not poison") + .get(path) + .cloned() +} + +#[cfg(not(test))] +fn injected_replacement_proof_write_error(_path: &str) -> Option { + None +} + async fn delete_resume_file(disk: &DiskStore, path: &Path) -> Result<()> { let path_str = path_to_str(path)?; if let Some(err) = injected_resume_delete_error(path_str) { @@ -152,8 +486,36 @@ async fn delete_resume_file(disk: &DiskStore, path: &Path) -> Result<()> { } } +fn replacement_completion_proof_path(task_id: &str) -> std::path::PathBuf { + replacement_recovery_dir().join(format!("{task_id}_{REPLACEMENT_COMPLETION_PROOF_FILE}")) +} + +fn legacy_replacement_completion_proof_path(task_id: &str) -> std::path::PathBuf { + Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{REPLACEMENT_COMPLETION_PROOF_FILE}")) +} + +fn replacement_recovery_dir() -> std::path::PathBuf { + Path::new(BUCKET_META_PREFIX).join(REPLACEMENT_RECOVERY_DIR) +} + +async fn ensure_replacement_recovery_dir(disk: &DiskStore) -> std::result::Result<(), DiskError> { + let volume = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/{REPLACEMENT_RECOVERY_DIR}"); + match super::storage_api::owner::EcstoreDiskAPI::make_volume(disk.as_ref(), &volume).await { + Ok(()) | Err(DiskError::VolumeExists) => Ok(()), + Err(error) => Err(error), + } +} + +fn replacement_intent_seal_path(task_id: &str) -> std::path::PathBuf { + replacement_recovery_dir().join(format!("{task_id}_{REPLACEMENT_INTENT_SEAL_FILE}")) +} + +fn legacy_replacement_recovery_marker_path(task_id: &str) -> std::path::PathBuf { + Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{LEGACY_REPLACEMENT_RECOVERY_MARKER_FILE}")) +} + /// resume state -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ResumeState { /// on-disk schema version; absent in legacy snapshots (defaults to 0) #[serde(default)] @@ -169,6 +531,22 @@ pub struct ResumeState { /// set disk identifier (for erasure set tasks) #[serde(default)] pub set_disk_id: String, + #[serde(default)] + pub replacement_targets: Vec, + #[serde(default)] + pub replacement_target_identities: Vec, + /// Immutable bucket plan for a replacement generation. `pending_buckets` + /// shrinks during a pass and must never be reused as a positional resume + /// input after restart. + #[serde(default)] + pub replacement_buckets: Vec, + /// A task-scoped generation assigned before formatting an automatic + /// replacement. A new unformatted replacement receives a new generation + /// and therefore cannot reuse an older disk's cursor or checkpoint. + #[serde(default)] + pub replacement_generation: Option, + #[serde(default)] + pub replacement_phase: ReplacementPhase, /// start time pub start_time: u64, /// last update time @@ -209,6 +587,11 @@ impl ResumeState { task_id, task_type, set_disk_id, + replacement_targets: Vec::new(), + replacement_target_identities: Vec::new(), + replacement_buckets: Vec::new(), + replacement_generation: None, + replacement_phase: ReplacementPhase::None, start_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(), last_update: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(), completed: false, @@ -227,6 +610,23 @@ impl ResumeState { } } + fn replacement_intent( + task_id: String, + task_type: String, + set_disk_id: String, + buckets: Vec, + replacement_targets: Vec, + replacement_target_identities: Vec, + ) -> Self { + let mut state = Self::new(task_id.clone(), task_type, set_disk_id, buckets); + state.replacement_targets = replacement_targets; + state.replacement_target_identities = replacement_target_identities; + state.replacement_buckets = state.pending_buckets.clone(); + state.replacement_generation = Some(task_id); + state.replacement_phase = ReplacementPhase::Intent; + state + } + pub fn update_progress(&mut self, processed: u64, successful: u64, failed: u64, skipped: u64) { self.processed_objects = processed; self.successful_objects = successful; @@ -319,6 +719,38 @@ pub struct ResumeManager { disk: DiskStore, state: Arc>, throttle: Mutex, + state_file: ResumeStateFile, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum ResumeStateFile { + Ordinary, + ReplacementIntent, + LegacyReplacementIntent, +} + +impl ResumeStateFile { + fn path(self, task_id: &str) -> std::path::PathBuf { + match self { + Self::Ordinary => Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_STATE_FILE}")), + Self::ReplacementIntent => replacement_recovery_dir().join(format!("{task_id}_{REPLACEMENT_INTENT_FILE}")), + Self::LegacyReplacementIntent => Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{REPLACEMENT_INTENT_FILE}")), + } + } +} + +fn is_replacement_intent(state: &ResumeState) -> bool { + state.replacement_generation.as_deref() == Some(state.task_id.as_str()) + && !state.replacement_targets.is_empty() + && replacement_targets_match_identities(&state.replacement_targets, &state.replacement_target_identities) + && matches!( + state.replacement_phase, + ReplacementPhase::Intent + | ReplacementPhase::Rebuilding + | ReplacementPhase::Verified + | ReplacementPhase::CleanupPending + | ReplacementPhase::Abandoned + ) } impl ResumeManager { @@ -330,38 +762,493 @@ impl ResumeManager { set_disk_id: String, buckets: Vec, ) -> Result { + validate_resume_task_id(&task_id)?; let state = ResumeState::new(task_id, task_type, set_disk_id, buckets); let manager = Self { disk, state: Arc::new(RwLock::new(state)), throttle: Mutex::new(PersistThrottle::new()), + state_file: ResumeStateFile::Ordinary, }; // save initial state - if let Err(e) = manager.save_state().await { - warn!( - target: "rustfs::heal::resume", - event = EVENT_HEAL_RESUME_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_RESUME, - state = "initial_save_failed", - error = %e, - "Heal resume state persistence failed" - ); - } + manager.save_state().await?; Ok(manager) } - /// load resume state from disk + /// Persist the automatic replacement intent before the target is formatted. + /// Reusing the same task id is idempotent for scheduler retries, while a new + /// admission obtains a new task id and cannot inherit stale progress. + pub async fn new_replacement_intent( + disk: DiskStore, + task_id: String, + set_disk_id: String, + buckets: Vec, + mut replacement_targets: Vec, + mut replacement_target_identities: Vec, + ) -> Result { + validate_resume_task_id(&task_id)?; + replacement_targets.sort_unstable(); + replacement_targets.dedup(); + replacement_target_identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint)); + replacement_target_identities.dedup_by(|left, right| left.endpoint == right.endpoint); + if replacement_target_identities + .iter() + .map(|identity| &identity.endpoint) + .collect::>() + != replacement_targets.iter().collect::>() + { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement identities do not match targets for task {task_id}"), + }); + } + + let recovery_expected = if Self::has_replacement_intent(&disk, &task_id).await { + match Self::load_replacement_intent(disk.clone(), &task_id).await { + Ok(manager) => { + let state = manager.get_state().await; + if state.set_disk_id != set_disk_id + || state.replacement_targets != replacement_targets + || state.replacement_target_identities != replacement_target_identities + || state.replacement_generation.as_deref() != Some(task_id.as_str()) + || !matches!( + state.replacement_phase, + ReplacementPhase::Intent + | ReplacementPhase::Rebuilding + | ReplacementPhase::Verified + | ReplacementPhase::CleanupPending + ) + { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement intent does not match task {task_id}"), + }); + } + manager.ensure_replacement_intent_seal().await?; + return Ok(manager); + } + Err(error) => match Self::torn_replacement_intent_bytes(&disk, &task_id).await? { + Some(expected) => { + warn!( + target: "rustfs::heal::resume", + event = EVENT_HEAL_RESUME_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_RESUME, + task_id, + error = %error, + state = "torn_replacement_intent_recovered", + "Replacing a torn replacement intent before formatting" + ); + Some(expected) + } + None => return Err(error), + }, + } + } else { + None + }; + + let state = ResumeState::replacement_intent( + task_id, + "erasure_set".to_string(), + set_disk_id, + buckets, + replacement_targets, + replacement_target_identities, + ); + let manager = Self { + disk, + state: Arc::new(RwLock::new(state)), + throttle: Mutex::new(PersistThrottle::new()), + state_file: ResumeStateFile::ReplacementIntent, + }; + manager.publish_new_replacement_intent(recovery_expected).await?; + manager.ensure_replacement_intent_seal().await?; + Ok(manager) + } + + /// Seal a durably published intent before the caller may format a target. + /// A torn intent without this seal is known to have failed before its + /// creator returned and can be atomically recreated on retry. + async fn ensure_replacement_intent_seal(&self) -> Result<()> { + let task_id = self.state.read().await.task_id.clone(); + validate_resume_task_id(&task_id)?; + let path = replacement_intent_seal_path(&task_id); + let path = path_to_str(&path)?; + match self.disk.read_all(RUSTFS_META_BUCKET, path).await { + Ok(_) => return Ok(()), + Err(DiskError::FileNotFound) => {} + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to read replacement intent seal: {error}"), + }); + } + } + self.disk + .write_all(RUSTFS_META_BUCKET, path, b"sealed".as_slice().into()) + .await + .map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to save replacement intent seal: {error}"), + }) + } + + pub async fn mark_replacement_rebuilding( + &self, + mut replacement_target_identities: Vec, + ) -> Result<()> { + replacement_target_identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint)); + replacement_target_identities.dedup_by(|left, right| left.endpoint == right.endpoint); + let mut state = self.state.write().await; + if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement intent is not active for task {}", state.task_id), + }); + } + if replacement_target_identities + .iter() + .map(|identity| &identity.endpoint) + .collect::>() + != state.replacement_targets.iter().collect::>() + { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement identities do not match targets for task {}", state.task_id), + }); + } + if !replacement_target_identities_match(&state.replacement_target_identities, &replacement_target_identities) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement target changed after format for task {}", state.task_id), + }); + } + state.replacement_phase = ReplacementPhase::Rebuilding; + state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + drop(state); + self.save_state_strict().await + } + + /// Persist survivor-anchor completion proof before transitioning this + /// resumable state to `Verified`. If proof persistence fails, this state + /// stays rebuildable and the caller must retain the healing marker. + pub async fn mark_replacement_completed_and_verified(&self) -> Result<()> { + let state = self.state.read().await.clone(); + if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement verification is not active for task {}", state.task_id), + }); + } + let proof = self.write_replacement_completion_proof(&state, None).await?; + + let mut state = self.state.write().await; + if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement verification changed for task {}", state.task_id), + }); + } + state.mark_completed(); + state.replacement_phase = ReplacementPhase::Verified; + state.last_update = proof.verified_at; + drop(state); + self.save_state_strict().await + } + + /// Verify or backfill the terminal proof before marker removal or resume + /// cleanup. This supports restart recovery from a `Verified` state written + /// by a prior binary that did not yet have a separate proof record. + pub(crate) async fn ensure_replacement_completion_proof(&self) -> Result { + let state = self.state.read().await.clone(); + if !state.completed || !matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement completion is not verified for task {}", state.task_id), + }); + } + self.write_replacement_completion_proof(&state, Some(state.last_update)).await + } + + /// Record that the healing markers have been removed, so a later retry can + /// safely delete the remaining resume artifacts without touching markers. + pub async fn mark_replacement_cleanup_pending(&self) -> Result<()> { + let mut state = self.state.write().await; + if !state.completed || !matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement cleanup is not ready for task {}", state.task_id), + }); + } + state.replacement_phase = ReplacementPhase::CleanupPending; + state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + drop(state); + self.save_state_strict().await + } + + /// Load the durable terminal proof from the healthy survivor anchor. + pub(crate) async fn load_replacement_completion_proof(disk: DiskStore, task_id: &str) -> Result { + Self::replacement_completion_proof_if_present(disk, task_id) + .await? + .ok_or_else(|| Error::TaskExecutionFailed { + message: format!("Failed to read replacement completion proof: proof is missing for task {task_id}"), + }) + } + + async fn replacement_completion_proof_if_present( + disk: DiskStore, + task_id: &str, + ) -> Result> { + validate_resume_task_id(task_id)?; + let mut proofs = Vec::new(); + for path in [ + replacement_completion_proof_path(task_id), + legacy_replacement_completion_proof_path(task_id), + ] { + let path_str = path_to_str(&path)?; + let bytes = match disk.read_all(RUSTFS_META_BUCKET, path_str).await { + Ok(bytes) => bytes, + Err(DiskError::FileNotFound) => continue, + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to read replacement completion proof: {error}"), + }); + } + }; + let proof: ReplacementCompletionProof = + serde_json::from_slice(&bytes).map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to deserialize replacement completion proof: {error}"), + })?; + proof.validate(task_id)?; + proofs.push(proof); + } + + match proofs.as_slice() { + [] => Ok(None), + [proof] => Ok(Some(proof.clone())), + [proof, legacy_proof] if proof == legacy_proof => Ok(Some(proof.clone())), + _ => Err(replacement_recovery_conflict(format!( + "Replacement completion proof conflicts with legacy proof for task {task_id}" + ))), + } + } + + /// Reconcile the proof-first publication order after a crash. A matching + /// proof is durable evidence that rebuilding finished, so it must win over + /// an older active state before a retry may format the target again. + async fn reconcile_replacement_completion_proof(&self) -> Result<()> { + let task_id = self.state.read().await.task_id.clone(); + let Some(proof) = Self::replacement_completion_proof_if_present(self.disk.clone(), &task_id).await? else { + return Ok(()); + }; + + let mut state = self.state.write().await; + if !proof.matches_state(&state) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement completion proof does not match active intent for task {}", state.task_id), + }); + } + if state.completed && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) { + return Ok(()); + } + if state.completed || !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) { + return Err(replacement_recovery_conflict(format!( + "Replacement completion proof conflicts with state for task {}", + state.task_id + ))); + } + + state.mark_completed(); + state.replacement_phase = ReplacementPhase::Verified; + state.last_update = proof.verified_at; + drop(state); + self.save_state_strict().await + } + + async fn migrate_legacy_replacement_completion_proof(disk: &DiskStore, task_id: &str) -> Result { + validate_resume_task_id(task_id)?; + let legacy_path = legacy_replacement_completion_proof_path(task_id); + let legacy_path_str = path_to_str(&legacy_path)?; + let legacy_bytes = match disk.read_all(RUSTFS_META_BUCKET, legacy_path_str).await { + Ok(bytes) => bytes, + Err(DiskError::FileNotFound) => return Ok(false), + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to read legacy replacement completion proof: {error}"), + }); + } + }; + let legacy_proof: ReplacementCompletionProof = serde_json::from_slice(&legacy_bytes).map_err(|error| { + replacement_recovery_corruption(format!("Failed to deserialize legacy replacement completion proof: {error}")) + })?; + legacy_proof + .validate(task_id) + .map_err(|error| replacement_recovery_corruption(format!("Invalid legacy replacement completion proof: {error}")))?; + + ensure_replacement_recovery_dir(disk) + .await + .map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to create replacement recovery directory: {error}"), + })?; + let path = replacement_completion_proof_path(task_id); + let path_str = path_to_str(&path)?; + for _ in 0..2 { + match disk.read_all(RUSTFS_META_BUCKET, path_str).await { + Ok(bytes) => { + let proof: ReplacementCompletionProof = + serde_json::from_slice(&bytes).map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to deserialize replacement completion proof: {error}"), + })?; + proof.validate(task_id).map_err(|error| { + replacement_recovery_corruption(format!("Invalid replacement completion proof: {error}")) + })?; + if proof != legacy_proof { + return Err(replacement_recovery_conflict(format!( + "Replacement completion proof conflicts with legacy proof for task {task_id}" + ))); + } + delete_resume_file(disk, &legacy_path).await?; + return Ok(true); + } + Err(DiskError::FileNotFound) => {} + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to read replacement completion proof: {error}"), + }); + } + } + + match super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + path_str, + None, + Some(legacy_bytes.clone()), + ) + .await + { + Ok(EcstoreConditionalFileUpdate::Updated) => { + delete_resume_file(disk, &legacy_path).await?; + return Ok(true); + } + Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue, + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to migrate replacement completion proof: {error}"), + }); + } + } + } + + Err(Error::TaskExecutionFailed { + message: format!("Replacement completion proof changed while migrating task {task_id}"), + }) + } + + pub async fn abandon_replacement_intent(&self) -> Result<()> { + let mut state = self.state.write().await; + if matches!(state.replacement_phase, ReplacementPhase::Abandoned) { + return Ok(()); + } + state.replacement_phase = ReplacementPhase::Abandoned; + state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + drop(state); + self.save_state_strict().await + } + + pub async fn set_replacement_targets(&self, replacement_targets: Vec) -> Result<()> { + { + let mut state = self.state.write().await; + state.replacement_targets = replacement_targets; + } + self.save_state().await + } + + /// Load an ordinary resume state from disk. Replacement intents have a + /// separate namespace so they cannot be mistaken for ordinary work by an + /// older binary. pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result { - let state_data = Self::read_state_file(&disk, task_id).await?; + Self::load_from_disk_at(disk, task_id, ResumeStateFile::Ordinary).await + } + + /// Load a replacement intent, migrating a legacy flat record only after + /// the dedicated replacement record has been written. + pub async fn load_replacement_intent(disk: DiskStore, task_id: &str) -> Result { + if Self::has_state_file(&disk, task_id, ResumeStateFile::ReplacementIntent).await { + let isolated = Self::load_from_disk_at(disk.clone(), task_id, ResumeStateFile::ReplacementIntent).await?; + isolated.ensure_replacement_intent_seal().await?; + for legacy_file in [ResumeStateFile::LegacyReplacementIntent, ResumeStateFile::Ordinary] { + if !Self::has_state_file(&disk, task_id, legacy_file).await { + continue; + } + let legacy = Self::load_from_disk_at(disk.clone(), task_id, legacy_file) + .await + .map_err(|error| { + replacement_recovery_corruption_for_state_load( + format!("Failed to load legacy replacement state {task_id}"), + error, + ) + })?; + let legacy_state = legacy.get_state().await; + if !is_replacement_intent(&legacy_state) || legacy_state != isolated.get_state().await { + return Err(replacement_recovery_conflict(format!( + "Replacement intent has conflicting legacy state for task {task_id}" + ))); + } + legacy.cleanup().await?; + } + delete_resume_file(&disk, &legacy_replacement_recovery_marker_path(task_id)).await?; + isolated.reconcile_replacement_completion_proof().await?; + return Ok(isolated); + } + + let legacy_file = if Self::has_state_file(&disk, task_id, ResumeStateFile::LegacyReplacementIntent).await { + ResumeStateFile::LegacyReplacementIntent + } else { + ResumeStateFile::Ordinary + }; + let legacy = Self::load_from_disk_at(disk.clone(), task_id, legacy_file) + .await + .map_err(|error| { + replacement_recovery_corruption_for_state_load( + format!("Failed to load legacy replacement state {task_id}"), + error, + ) + })?; + if !is_replacement_intent(&legacy.get_state().await) { + return Err(replacement_recovery_corruption(format!( + "Resume state is not a replacement intent for task {task_id}" + ))); + } + + let migrated = Self { + disk, + state: legacy.state.clone(), + throttle: Mutex::new(PersistThrottle::new()), + state_file: ResumeStateFile::ReplacementIntent, + }; + migrated.save_state_strict().await?; + migrated.ensure_replacement_intent_seal().await?; + legacy.cleanup().await?; + delete_resume_file(&migrated.disk, &legacy_replacement_recovery_marker_path(task_id)).await?; + migrated.reconcile_replacement_completion_proof().await?; + Ok(migrated) + } + + async fn load_from_disk_at(disk: DiskStore, task_id: &str, state_file: ResumeStateFile) -> Result { + validate_resume_task_id(task_id)?; + let state_data = Self::read_state_file(&disk, task_id, state_file).await?; let mut state: ResumeState = serde_json::from_slice(&state_data).map_err(|e| Error::TaskExecutionFailed { message: format!("Failed to deserialize resume state: {e}"), })?; + if state.task_id != task_id { + return Err(Error::TaskExecutionFailed { + message: "Resume state task id does not match filename".to_string(), + }); + } // A snapshot written by an older schema tracked a latest-only positional // cursor that is meaningless under per-version resume. Discard the stale // progress so the scan restarts cleanly, then stamp the current schema. + if state.schema_version > CURRENT_RESUME_SCHEMA { + return Err(Error::TaskExecutionFailed { + message: format!( + "Resume state schema {} is newer than supported schema {CURRENT_RESUME_SCHEMA}", + state.schema_version + ), + }); + } if state.schema_version < CURRENT_RESUME_SCHEMA { warn!( target: "rustfs::heal::resume", @@ -380,6 +1267,7 @@ impl ResumeManager { state.failed_objects = 0; state.skipped_objects = 0; state.completed = false; + state.completed_buckets.clear(); state.schema_version = CURRENT_RESUME_SCHEMA; } @@ -387,12 +1275,35 @@ impl ResumeManager { disk, state: Arc::new(RwLock::new(state)), throttle: Mutex::new(PersistThrottle::new()), + state_file, }) } /// check if resume state exists pub async fn has_resume_state(disk: &DiskStore, task_id: &str) -> bool { - let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_STATE_FILE}")); + Self::has_state_file(disk, task_id, ResumeStateFile::Ordinary).await + } + + /// Check for a replacement intent in its dedicated namespace. Flat and + /// ordinary legacy records are recognized only for migration compatibility. + pub async fn has_replacement_intent(disk: &DiskStore, task_id: &str) -> bool { + if Self::has_state_file(disk, task_id, ResumeStateFile::ReplacementIntent).await { + return true; + } + if Self::has_state_file(disk, task_id, ResumeStateFile::LegacyReplacementIntent).await { + return true; + } + match Self::load_from_disk_at(disk.clone(), task_id, ResumeStateFile::Ordinary).await { + Ok(manager) => is_replacement_intent(&manager.get_state().await), + Err(_) => false, + } + } + + async fn has_state_file(disk: &DiskStore, task_id: &str, state_file: ResumeStateFile) -> bool { + if validate_resume_task_id(task_id).is_err() { + return false; + } + let file_path = state_file.path(task_id); match path_to_str(&file_path) { Ok(path_str) => match disk.read_all(RUSTFS_META_BUCKET, path_str).await { Ok(data) => !data.is_empty(), @@ -402,6 +1313,30 @@ impl ResumeManager { } } + async fn torn_replacement_intent_bytes(disk: &DiskStore, task_id: &str) -> Result> { + validate_resume_task_id(task_id)?; + let path = ResumeStateFile::ReplacementIntent.path(task_id); + let path = path_to_str(&path)?; + match disk.read_all(RUSTFS_META_BUCKET, path).await { + Ok(bytes) if serde_json::from_slice::(&bytes).is_err() => { + let seal = replacement_intent_seal_path(task_id); + let seal = path_to_str(&seal)?; + match disk.read_all(RUSTFS_META_BUCKET, seal).await { + Err(DiskError::FileNotFound) => Ok(Some(bytes)), + Ok(_) => Ok(None), + Err(error) => Err(Error::TaskExecutionFailed { + message: format!("Failed to read replacement intent seal: {error}"), + }), + } + } + Ok(_) => Ok(None), + Err(DiskError::FileNotFound) => Ok(None), + Err(error) => Err(Error::TaskExecutionFailed { + message: format!("Failed to read replacement intent: {error}"), + }), + } + } + /// get current state pub async fn get_state(&self) -> ResumeState { self.state.read().await.clone() @@ -504,14 +1439,19 @@ impl ResumeManager { /// cleanup resume state pub async fn cleanup(&self) -> Result<()> { - let task_id = self.state.read().await.task_id.clone(); + let state = self.state.read().await; + let task_id = state.task_id.clone(); + drop(state); + validate_resume_task_id(&task_id)?; - let state_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_STATE_FILE}")); + let state_file = self.state_file.path(&task_id); let progress_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_PROGRESS_FILE}")); delete_resume_file(&self.disk, &progress_file).await?; - // Delete the state file last so a partial cleanup remains discoverable. delete_resume_file(&self.disk, &state_file).await?; + if matches!(self.state_file, ResumeStateFile::ReplacementIntent) { + delete_resume_file(&self.disk, &replacement_intent_seal_path(&task_id)).await?; + } debug!( target: "rustfs::heal::resume", @@ -527,16 +1467,141 @@ impl ResumeManager { /// save state to disk async fn save_state(&self) -> Result<()> { - let state = self.state.read().await; - let state_data = serde_json::to_vec(&*state).map_err(|e| Error::TaskExecutionFailed { - message: format!("Failed to serialize resume state: {e}"), - })?; + self.save_state_with_unformatted_policy(true).await + } - let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", state.task_id, RESUME_STATE_FILE)); + async fn save_state_strict(&self) -> Result<()> { + self.save_state_with_unformatted_policy(false).await + } + + async fn publish_new_replacement_intent(&self, expected: Option) -> Result<()> { + let state = self.state.read().await.clone(); + validate_resume_task_id(&state.task_id)?; + let state_data = EcstoreDiskBytes::from(serde_json::to_vec(&state).map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to serialize resume state: {error}"), + })?); + let path = self.state_file.path(&state.task_id); + let path = path_to_str(&path)?; + + ensure_replacement_recovery_dir(&self.disk) + .await + .map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to create replacement recovery directory: {error}"), + })?; + match super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file( + self.disk.as_ref(), + RUSTFS_META_BUCKET, + path, + expected, + Some(state_data), + ) + .await + { + Ok(EcstoreConditionalFileUpdate::Updated) => Ok(()), + Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => { + Err(Error::TaskExecutionFailed { + message: format!("Replacement intent changed before publication for task {}", state.task_id), + }) + } + Err(error) => Err(Error::TaskExecutionFailed { + message: format!("Failed to save resume state: {error}"), + }), + } + } + + async fn write_replacement_completion_proof( + &self, + state: &ResumeState, + verified_at: Option, + ) -> Result { + ensure_replacement_recovery_dir(&self.disk) + .await + .map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to create replacement recovery directory: {error}"), + })?; + let path = replacement_completion_proof_path(&state.task_id); + let path_str = path_to_str(&path)?; + let proof = ReplacementCompletionProof::from_state( + state, + verified_at.unwrap_or_else(|| SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs()), + )?; + let proof_data = EcstoreDiskBytes::from(serde_json::to_vec(&proof).map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to serialize replacement completion proof: {e}"), + })?); + if let Some(error) = injected_replacement_proof_write_error(path_str) { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to save replacement completion proof: {error}"), + }); + } + + // Publish through the disk CAS primitive: `write_all` can expose a + // partially written proof to a crash/restart reader. If a prior + // version left torn bytes behind, replace exactly the observed bytes; + // a concurrently published valid proof is never overwritten. + for _ in 0..2 { + let expected = match self.disk.read_all(RUSTFS_META_BUCKET, path_str).await { + Ok(existing) => match serde_json::from_slice::(&existing) { + Ok(existing_proof) => { + existing_proof.validate(&state.task_id)?; + if existing_proof.matches_state(state) { + return Ok(existing_proof); + } + return Err(Error::TaskExecutionFailed { + message: format!("Replacement completion proof does not match task {}", state.task_id), + }); + } + Err(_) => Some(existing), + }, + Err(DiskError::FileNotFound) => None, + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to read replacement completion proof: {error}"), + }); + } + }; + + match super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file( + self.disk.as_ref(), + RUSTFS_META_BUCKET, + path_str, + expected, + Some(proof_data.clone()), + ) + .await + { + Ok(EcstoreConditionalFileUpdate::Updated) => return Ok(proof), + Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue, + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to save replacement completion proof: {error}"), + }); + } + } + } + + Err(Error::TaskExecutionFailed { + message: format!("Replacement completion proof changed while publishing task {}", state.task_id), + }) + } + + async fn save_state_with_unformatted_policy(&self, allow_unformatted: bool) -> Result<()> { + let state = self.state.read().await.clone(); + validate_resume_task_id(&state.task_id)?; + let state_data = EcstoreDiskBytes::from(serde_json::to_vec(&state).map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to serialize resume state: {e}"), + })?); + + let file_path = self.state_file.path(&state.task_id); let path_str = path_to_str(&file_path)?; - if let Err(e) = self.disk.write_all(RUSTFS_META_BUCKET, path_str, state_data.into()).await { - if matches!(e, DiskError::UnformattedDisk) { + let write_result = match self.state_file { + ResumeStateFile::Ordinary | ResumeStateFile::LegacyReplacementIntent => { + self.disk.write_all(RUSTFS_META_BUCKET, path_str, state_data).await + } + ResumeStateFile::ReplacementIntent => self.write_replacement_intent_state(path_str, state_data).await, + }; + if let Err(e) = write_result { + if allow_unformatted && matches!(e, DiskError::UnformattedDisk) { warn!( target: "rustfs::heal::resume", event = EVENT_HEAL_RESUME_STATE, @@ -565,17 +1630,45 @@ impl ResumeManager { Ok(()) } + async fn write_replacement_intent_state( + &self, + path: &str, + state_data: EcstoreDiskBytes, + ) -> std::result::Result<(), DiskError> { + ensure_replacement_recovery_dir(&self.disk).await?; + for _ in 0..2 { + let expected = match self.disk.read_all(RUSTFS_META_BUCKET, path).await { + Ok(existing) => Some(existing), + Err(DiskError::FileNotFound) => None, + Err(error) => return Err(error), + }; + match super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file( + self.disk.as_ref(), + RUSTFS_META_BUCKET, + path, + expected, + Some(state_data.clone()), + ) + .await + { + Ok(EcstoreConditionalFileUpdate::Updated) => return Ok(()), + Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue, + Err(error) => return Err(error), + } + } + Err(DiskError::other("replacement intent changed while publishing")) + } + /// read state file from disk - async fn read_state_file(disk: &DiskStore, task_id: &str) -> Result> { - let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_STATE_FILE}")); + async fn read_state_file(disk: &DiskStore, task_id: &str, state_file: ResumeStateFile) -> Result> { + validate_resume_task_id(task_id)?; + let file_path = state_file.path(task_id); let path_str = path_to_str(&file_path)?; disk.read_all(RUSTFS_META_BUCKET, path_str) .await .map(|bytes| bytes.to_vec()) - .map_err(|e| Error::TaskExecutionFailed { - message: format!("Failed to read resume state file: {e}"), - }) + .map_err(Error::Disk) } } @@ -666,6 +1759,7 @@ pub struct CheckpointManager { impl CheckpointManager { /// create new checkpoint manager pub async fn new(disk: DiskStore, task_id: String) -> Result { + validate_resume_task_id(&task_id)?; let checkpoint = ResumeCheckpoint::new(task_id); let manager = Self { disk, @@ -690,16 +1784,30 @@ impl CheckpointManager { /// load checkpoint from disk pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result { + validate_resume_task_id(task_id)?; let checkpoint_data = Self::read_checkpoint_file(&disk, task_id).await?; let mut checkpoint: ResumeCheckpoint = serde_json::from_slice(&checkpoint_data).map_err(|e| Error::TaskExecutionFailed { message: format!("Failed to deserialize checkpoint: {e}"), })?; + if checkpoint.task_id != task_id { + return Err(Error::TaskExecutionFailed { + message: "Resume checkpoint task id does not match filename".to_string(), + }); + } // A checkpoint from an older schema stored latest-only dedup identities // that are not comparable to the new per-version `compose_key` // identities. Discard the stale sets and position, then stamp the // current schema so the scan restarts cleanly. + if checkpoint.schema_version > CURRENT_CHECKPOINT_SCHEMA { + return Err(Error::TaskExecutionFailed { + message: format!( + "Checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}", + checkpoint.schema_version + ), + }); + } if checkpoint.schema_version < CURRENT_CHECKPOINT_SCHEMA { warn!( target: "rustfs::heal::resume", @@ -715,6 +1823,7 @@ impl CheckpointManager { checkpoint.processed_objects.clear(); checkpoint.failed_objects.clear(); checkpoint.skipped_objects.clear(); + checkpoint.current_bucket_index = 0; checkpoint.current_object_index = 0; checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA; } @@ -728,6 +1837,9 @@ impl CheckpointManager { /// check if checkpoint exists pub async fn has_checkpoint(disk: &DiskStore, task_id: &str) -> bool { + if validate_resume_task_id(task_id).is_err() { + return false; + } let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); match path_to_str(&file_path) { Ok(path_str) => match disk.read_all(RUSTFS_META_BUCKET, path_str).await { @@ -814,6 +1926,7 @@ impl CheckpointManager { /// cleanup checkpoint pub async fn cleanup(&self) -> Result<()> { let task_id = self.checkpoint.read().await.task_id.clone(); + validate_resume_task_id(&task_id)?; let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); delete_resume_file(&self.disk, &checkpoint_file).await?; @@ -833,6 +1946,7 @@ impl CheckpointManager { /// save checkpoint to disk async fn save_checkpoint(&self) -> Result<()> { let checkpoint = self.checkpoint.read().await; + validate_resume_task_id(&checkpoint.task_id)?; let checkpoint_data = serde_json::to_vec(&*checkpoint).map_err(|e| Error::TaskExecutionFailed { message: format!("Failed to serialize checkpoint: {e}"), })?; @@ -861,6 +1975,7 @@ impl CheckpointManager { /// read checkpoint file from disk async fn read_checkpoint_file(disk: &DiskStore, task_id: &str) -> Result> { + validate_resume_task_id(task_id)?; let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); let path_str = path_to_str(&file_path)?; @@ -913,7 +2028,7 @@ impl ResumeUtils { if entry.ends_with(&format!("_{RESUME_STATE_FILE}")) { // Extract task ID from filename: {task_id}_ahm_resume_state.json if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}")) - && !task_id.is_empty() + && validate_resume_task_id(task_id).is_ok() { task_ids.push(task_id.to_string()); } @@ -932,6 +2047,145 @@ impl ResumeUtils { Ok(task_ids) } + /// Return replacement intent task IDs from the dedicated recovery + /// directory. Periodic recovery must never enumerate the ordinary resume + /// directory, whose cardinality is unrelated to replacement work. + pub async fn get_replacement_intent_tasks(disk: &DiskStore) -> Result> { + let entries = Self::replacement_recovery_entries(disk).await?; + let suffix = format!("_{REPLACEMENT_INTENT_FILE}"); + let mut task_ids = HashSet::new(); + + for entry in entries { + if let Some(task_id) = entry.strip_suffix(&suffix) + && validate_resume_task_id(task_id).is_ok() + { + task_ids.insert(task_id.to_string()); + continue; + } + } + + let mut task_ids = task_ids.into_iter().collect::>(); + task_ids.sort_unstable(); + Ok(task_ids) + } + + async fn replacement_recovery_entries(disk: &DiskStore) -> Result> { + let recovery_dir = replacement_recovery_dir(); + let recovery_dir = path_to_str(&recovery_dir)?; + match disk.list_dir("", RUSTFS_META_BUCKET, recovery_dir, -1).await { + Ok(entries) => Ok(entries), + Err(DiskError::FileNotFound) => Ok(Vec::new()), + Err(error) => Err(Error::TaskExecutionFailed { + message: format!("Failed to list replacement recovery records: {error}"), + }), + } + } + + /// Migrate flat replacement artifacts from earlier builds exactly once at + /// manager startup. The normal scanner only uses the dedicated directory; + /// ordinary resume JSON is never read on its periodic path. + pub async fn migrate_legacy_replacement_records(disk: &DiskStore) -> Result<()> { + let entries = disk + .list_dir("", RUSTFS_META_BUCKET, BUCKET_META_PREFIX, -1) + .await + .map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to list legacy replacement records: {error}"), + })?; + let ordinary_suffix = format!("_{RESUME_STATE_FILE}"); + let intent_suffix = format!("_{REPLACEMENT_INTENT_FILE}"); + let proof_suffix = format!("_{REPLACEMENT_COMPLETION_PROOF_FILE}"); + let mut ordinary_task_ids = HashSet::new(); + let mut intent_task_ids = HashSet::new(); + let mut proof_task_ids = HashSet::new(); + + for entry in entries { + if let Some(task_id) = entry.strip_suffix(&intent_suffix) + && validate_resume_task_id(task_id).is_ok() + { + intent_task_ids.insert(task_id.to_string()); + continue; + } + if let Some(task_id) = entry.strip_suffix(&ordinary_suffix) + && validate_resume_task_id(task_id).is_ok() + { + ordinary_task_ids.insert(task_id.to_string()); + continue; + } + if let Some(task_id) = entry.strip_suffix(&proof_suffix) + && validate_resume_task_id(task_id).is_ok() + { + proof_task_ids.insert(task_id.to_string()); + } + } + + let mut state_task_ids = intent_task_ids.into_iter().collect::>(); + state_task_ids.extend(ordinary_task_ids); + state_task_ids.sort_unstable(); + state_task_ids.dedup(); + for task_id in state_task_ids { + let has_flat_intent = ResumeManager::has_state_file(disk, &task_id, ResumeStateFile::LegacyReplacementIntent).await; + if !has_flat_intent { + let manager = ResumeManager::load_from_disk(disk.clone(), &task_id).await.map_err(|error| { + replacement_recovery_corruption_for_state_load( + format!("Failed to load legacy replacement recovery candidate {task_id}"), + error, + ) + })?; + if !is_replacement_intent(&manager.get_state().await) { + continue; + } + } + ResumeManager::load_replacement_intent(disk.clone(), &task_id).await?; + } + + for task_id in proof_task_ids { + ResumeManager::migrate_legacy_replacement_completion_proof(disk, &task_id).await?; + } + Ok(()) + } + + /// Return all durable replacement states and completion proofs stored on + /// one survivor disk. Unlike the legacy resumable-task helper, listing + /// failures are returned to the caller so an observability surface cannot + /// silently turn an unreadable durable record into a green result. + pub async fn get_replacement_recovery_records(disk: &DiskStore) -> Result> { + let entries = Self::replacement_recovery_entries(disk).await?; + let proof_suffix = format!("_{REPLACEMENT_COMPLETION_PROOF_FILE}"); + let mut records = Vec::new(); + let mut intent_task_ids = HashSet::new(); + + for task_id in Self::get_replacement_intent_tasks(disk).await? { + let state = ResumeManager::load_replacement_intent(disk.clone(), &task_id) + .await? + .get_state() + .await; + intent_task_ids.insert(task_id.clone()); + records.push(ReplacementRecoveryRecord::from_state(state).unwrap_or_else(|| { + ReplacementRecoveryRecord::unknown( + task_id, + "isolated replacement intent violates its generation or target identity binding", + ) + })); + } + + for entry in entries { + let Some(task_id) = entry.strip_suffix(&proof_suffix) else { + continue; + }; + if validate_resume_task_id(task_id).is_err() { + continue; + } + if intent_task_ids.contains(task_id) { + continue; + } + let proof = ResumeManager::load_replacement_completion_proof(disk.clone(), task_id).await?; + records.push(ReplacementRecoveryRecord::from_completion_proof(&proof)); + } + + records.sort_by(|left, right| left.task_id.cmp(&right.task_id).then(left.state.cmp(&right.state))); + Ok(records) + } + /// cleanup expired resume states pub async fn cleanup_expired_states(disk: &DiskStore, max_age_hours: u64) -> Result<()> { let task_ids = Self::get_resumable_tasks(disk).await?; @@ -940,7 +2194,17 @@ impl ResumeUtils { for task_id in task_ids { if let Ok(resume_manager) = ResumeManager::load_from_disk(disk.clone(), &task_id).await { let state = resume_manager.get_state().await; - let age_hours = (current_time - state.last_update) / 3600; + let age_hours = current_time.saturating_sub(state.last_update) / 3600; + + if !state.completed && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) + { + continue; + } + if state.completed + && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) + { + continue; + } if age_hours > max_age_hours { debug!( @@ -970,6 +2234,39 @@ impl ResumeUtils { } } + for task_id in Self::get_replacement_intent_tasks(disk).await? { + if let Ok(resume_manager) = ResumeManager::load_replacement_intent(disk.clone(), &task_id).await { + let state = resume_manager.get_state().await; + let age_hours = current_time.saturating_sub(state.last_update) / 3600; + + if !state.completed && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) + { + continue; + } + if state.completed + && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) + { + continue; + } + + if age_hours > max_age_hours + && let Err(e) = resume_manager.cleanup().await + { + warn!( + target: "rustfs::heal::resume", + event = EVENT_HEAL_RESUME_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_RESUME, + task_id, + age_hours, + state = "expired_cleanup_failed", + error = %e, + "Replacement intent cleanup failed" + ); + } + } + } + Ok(()) } } @@ -978,6 +2275,32 @@ impl ResumeUtils { mod tests { use super::*; + async fn schema_test_disk() -> (tempfile::TempDir, DiskStore) { + use super::super::{DiskOption, Endpoint, new_disk}; + + let temp_dir = tempfile::TempDir::new().expect("create schema test directory"); + let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create schema test disk endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("create schema test disk"); + match disk.make_volume(RUSTFS_META_BUCKET).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(error) => panic!("create metadata volume: {error}"), + } + match disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(error) => panic!("create resume metadata volume: {error}"), + } + + (temp_dir, disk) + } + #[tokio::test] async fn test_resume_state_creation() { let task_id = ResumeUtils::generate_task_id(); @@ -991,6 +2314,1231 @@ mod tests { assert_eq!(state.pending_buckets.len(), 2); } + #[test] + fn replacement_intent_binds_a_generation_before_format() { + let state = ResumeState::replacement_intent( + "generation-a".to_string(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket-a".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ); + + assert_eq!(state.replacement_generation.as_deref(), Some("generation-a")); + assert_eq!(state.replacement_phase, ReplacementPhase::Intent); + assert_eq!(state.replacement_targets, ["replacement-a"]); + assert!(state.resume_cursor.is_none(), "a new replacement must start from the beginning"); + + let mut state = state; + state.complete_bucket("bucket-a"); + assert_eq!( + state.replacement_buckets, + ["bucket-a"], + "recovery must retain the original positional bucket plan" + ); + } + + #[tokio::test] + async fn replacement_terminal_phases_are_durable() { + use super::super::{DiskOption, Endpoint, new_disk}; + use tempfile::TempDir; + + let temp_dir = TempDir::new().expect("create replacement phase test directory"); + let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create test disk endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("create test disk"); + match disk.make_volume(RUSTFS_META_BUCKET).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(err) => panic!("create metadata volume for replacement phase test: {err}"), + } + + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + manager + .mark_replacement_completed_and_verified() + .await + .expect("completion and verified phase must persist together"); + + let verified = ResumeManager::load_replacement_intent(disk.clone(), &task_id) + .await + .expect("verified phase must survive a restart") + .get_state() + .await; + assert!(verified.completed); + assert_eq!(verified.replacement_phase, ReplacementPhase::Verified); + + let resumed = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["ignored-after-restart".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("verified replacement must be reopenable for terminal cleanup"); + assert_eq!( + resumed.get_state().await.replacement_buckets, + ["bucket"], + "terminal recovery must preserve the original generation bucket plan" + ); + + manager + .mark_replacement_cleanup_pending() + .await + .expect("cleanup-pending phase must persist after marker removal"); + let cleanup_pending = ResumeManager::load_replacement_intent(disk, &task_id) + .await + .expect("cleanup-pending phase must survive a restart") + .get_state() + .await; + assert!(cleanup_pending.completed); + assert_eq!(cleanup_pending.replacement_phase, ReplacementPhase::CleanupPending); + } + + #[tokio::test] + async fn replacement_proof_before_verified_state_is_reconciled_after_restart() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + + let proof = ReplacementCompletionProof::from_state(&manager.get_state().await, 42) + .expect("active replacement state should build a completion proof"); + let proof_path = replacement_completion_proof_path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + proof_path.to_str().expect("completion proof path must be UTF-8"), + serde_json::to_vec(&proof) + .expect("completion proof fixture should serialize") + .into(), + ) + .await + .expect("proof-first crash fixture should persist"); + + let recovered = ResumeManager::load_replacement_intent(disk.clone(), &task_id) + .await + .expect("matching completion proof must prevent another rebuild") + .get_state() + .await; + assert!(recovered.completed); + assert_eq!(recovered.replacement_phase, ReplacementPhase::Verified); + assert_eq!(recovered.last_update, proof.verified_at); + + let records = ResumeUtils::get_replacement_recovery_records(&disk) + .await + .expect("reconciled replacement state should be observable"); + assert_eq!(records.len(), 1); + assert_eq!(records[0].state, ReplacementRecoveryState::CleanupPending); + } + + #[tokio::test] + async fn replacement_proof_conflicting_with_active_state_fails_closed_after_restart() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + + let mut proof = ReplacementCompletionProof::from_state(&manager.get_state().await, 42) + .expect("active replacement state should build a completion proof"); + proof.set_disk_id = "pool_0_set_1".to_string(); + let proof_path = replacement_completion_proof_path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + proof_path.to_str().expect("completion proof path must be UTF-8"), + serde_json::to_vec(&proof) + .expect("completion proof fixture should serialize") + .into(), + ) + .await + .expect("conflicting proof fixture should persist"); + + let error = match ResumeManager::load_replacement_intent(disk, &task_id).await { + Ok(_) => panic!("a mismatched proof must not permit another rebuild"), + Err(error) => error, + }; + assert!(error.to_string().contains("does not match active intent")); + } + + #[tokio::test] + async fn replacement_intent_is_not_an_ordinary_resumable_task() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist in its isolated namespace"); + + assert!( + !ResumeManager::has_resume_state(&disk, &task_id).await, + "an old ordinary-resume lookup must not discover a replacement intent" + ); + assert!(ResumeManager::has_replacement_intent(&disk, &task_id).await); + assert!( + !ResumeUtils::get_resumable_tasks(&disk) + .await + .expect("ordinary resume listing should succeed") + .contains(&task_id), + "the old filename enumeration must not return replacement work" + ); + assert_eq!( + ResumeUtils::get_replacement_intent_tasks(&disk) + .await + .expect("replacement intent listing should succeed"), + vec![task_id.clone()] + ); + assert_eq!( + ResumeManager::load_replacement_intent(disk, &task_id) + .await + .expect("new replacement reader should load the isolated state") + .get_state() + .await, + manager.get_state().await + ); + } + + #[tokio::test] + async fn replacement_intent_recovers_from_torn_publication_before_formatting() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let intent_path = ResumeStateFile::ReplacementIntent.path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + intent_path.to_str().expect("replacement intent path must be UTF-8"), + b"{torn replacement intent".as_slice().into(), + ) + .await + .expect("torn replacement intent fixture should persist"); + + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("a retry must atomically replace only a torn pre-format intent"); + + let recovered = ResumeManager::load_replacement_intent(disk, &task_id) + .await + .expect("recovered intent should be readable after restart") + .get_state() + .await; + assert_eq!(recovered, manager.get_state().await); + assert_eq!(recovered.replacement_phase, ReplacementPhase::Intent); + } + + #[tokio::test] + async fn torn_intent_recovery_cas_preserves_a_concurrent_valid_binding() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let intent_path = ResumeStateFile::ReplacementIntent.path(&task_id); + let intent_path = intent_path.to_str().expect("replacement intent path must be UTF-8"); + let torn = EcstoreDiskBytes::from_static(b"{torn replacement intent"); + disk.write_all(RUSTFS_META_BUCKET, intent_path, torn) + .await + .expect("torn intent fixture should persist"); + + let expected = ResumeManager::torn_replacement_intent_bytes(&disk, &task_id) + .await + .expect("torn intent should be recoverable before a seal exists") + .expect("torn intent bytes should be retained as the CAS precondition"); + let winner = ResumeState::replacement_intent( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_winner".to_string(), + vec!["winner-bucket".to_string()], + vec!["replacement-winner".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-winner".to_string(), + canonical_path: "/mnt/replacement-winner".to_string(), + physical_device_ids: vec!["device-winner".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ); + let winner_bytes = EcstoreDiskBytes::from(serde_json::to_vec(&winner).expect("winner intent should serialize")); + disk.write_all(RUSTFS_META_BUCKET, intent_path, winner_bytes.clone()) + .await + .expect("concurrent valid intent fixture should persist"); + + let loser = ResumeManager { + disk: disk.clone(), + state: Arc::new(RwLock::new(ResumeState::replacement_intent( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_loser".to_string(), + vec!["loser-bucket".to_string()], + vec!["replacement-loser".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-loser".to_string(), + canonical_path: "/mnt/replacement-loser".to_string(), + physical_device_ids: vec!["device-loser".to_string()], + filesystem_identity: "4:5:6".to_string(), + }], + ))), + throttle: Mutex::new(PersistThrottle::new()), + state_file: ResumeStateFile::ReplacementIntent, + }; + let error = match loser.publish_new_replacement_intent(Some(expected)).await { + Ok(()) => panic!("a stale torn-intent recovery must not overwrite a concurrent valid binding"), + Err(error) => error, + }; + assert!(error.to_string().contains("changed before publication")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, intent_path) + .await + .expect("concurrent valid intent must remain durable"), + winner_bytes + ); + } + + #[tokio::test] + async fn replacement_intent_does_not_recreate_torn_state_after_seal_publication() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let intent_path = ResumeStateFile::ReplacementIntent.path(&task_id); + let intent_path = intent_path.to_str().expect("replacement intent path must be UTF-8"); + let torn = b"{torn replacement intent"; + disk.write_all(RUSTFS_META_BUCKET, intent_path, torn.as_slice().into()) + .await + .expect("torn replacement intent fixture should persist"); + let marker_path = replacement_intent_seal_path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + marker_path.to_str().expect("replacement seal path must be UTF-8"), + b"sealed".as_slice().into(), + ) + .await + .expect("replacement seal fixture should persist"); + + let error = match ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + { + Ok(_) => panic!("a seal means a torn state may have crossed the format boundary"), + Err(error) => error, + }; + assert!(error.to_string().contains("Failed to deserialize resume state")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, intent_path) + .await + .expect("torn intent must remain for operator recovery"), + torn.as_slice() + ); + } + + #[tokio::test] + async fn replacement_intent_migrates_from_legacy_resume_filename() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let legacy = ResumeState::replacement_intent( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ); + let legacy_path = ResumeStateFile::Ordinary.path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + legacy_path.to_str().expect("legacy resume path must be UTF-8"), + serde_json::to_vec(&legacy) + .expect("serialize legacy replacement state") + .into(), + ) + .await + .expect("write legacy replacement state"); + + let migrated = ResumeManager::load_replacement_intent(disk.clone(), &task_id) + .await + .expect("new binary should migrate a legacy replacement state"); + assert_eq!(migrated.get_state().await, legacy); + assert!( + !ResumeManager::has_resume_state(&disk, &task_id).await, + "migration must remove the old-binary-visible state only after the new state is durable" + ); + assert!(ResumeManager::has_replacement_intent(&disk, &task_id).await); + } + + #[tokio::test] + async fn ordinary_targeted_resume_is_not_migrated_as_a_replacement_intent() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new( + disk.clone(), + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ) + .await + .expect("ordinary targeted resume should persist"); + manager + .set_replacement_targets(vec!["manual-target".to_string()]) + .await + .expect("ordinary targeted resume should retain its target filter"); + + assert!(!ResumeManager::has_replacement_intent(&disk, &task_id).await); + assert!(ResumeManager::load_replacement_intent(disk.clone(), &task_id).await.is_err()); + assert!(ResumeManager::has_resume_state(&disk, &task_id).await); + } + + #[tokio::test] + async fn malformed_isolated_replacement_intent_is_reported_as_unknown() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let malformed = ResumeState::new( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ); + let path = ResumeStateFile::ReplacementIntent.path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + path.to_str().expect("isolated replacement path must be UTF-8"), + serde_json::to_vec(&malformed) + .expect("malformed replacement fixture should serialize") + .into(), + ) + .await + .expect("malformed isolated replacement state should persist"); + + let records = ResumeUtils::get_replacement_recovery_records(&disk) + .await + .expect("isolated replacement listing should succeed"); + assert_eq!(records.len(), 1); + assert_eq!(records[0].task_id, task_id); + assert_eq!(records[0].state, ReplacementRecoveryState::Unknown); + } + + #[tokio::test] + async fn startup_migration_moves_flat_replacement_artifacts_to_dedicated_directory() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let mut state = ResumeState::replacement_intent( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ); + state.mark_completed(); + state.replacement_phase = ReplacementPhase::Verified; + let legacy_intent = ResumeStateFile::LegacyReplacementIntent.path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + legacy_intent.to_str().expect("legacy intent path must be UTF-8"), + serde_json::to_vec(&state) + .expect("serialize legacy replacement intent") + .into(), + ) + .await + .expect("write legacy replacement intent"); + let proof = ReplacementCompletionProof::from_state(&state, state.last_update).expect("build legacy completion proof"); + let legacy_proof = legacy_replacement_completion_proof_path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + legacy_proof.to_str().expect("legacy proof path must be UTF-8"), + serde_json::to_vec(&proof).expect("serialize legacy completion proof").into(), + ) + .await + .expect("write legacy completion proof"); + + ResumeUtils::migrate_legacy_replacement_records(&disk) + .await + .expect("startup migration should move flat replacement artifacts"); + + assert_eq!( + ResumeUtils::get_replacement_intent_tasks(&disk) + .await + .expect("dedicated intent listing should succeed"), + vec![task_id.clone()] + ); + assert_eq!( + ResumeManager::load_replacement_completion_proof(disk.clone(), &task_id) + .await + .expect("dedicated completion proof should be readable"), + proof + ); + assert!(matches!( + disk.read_all(RUSTFS_META_BUCKET, legacy_intent.to_str().expect("legacy intent path must be UTF-8")) + .await, + Err(DiskError::FileNotFound) + )); + assert!(matches!( + disk.read_all(RUSTFS_META_BUCKET, legacy_proof.to_str().expect("legacy proof path must be UTF-8")) + .await, + Err(DiskError::FileNotFound) + )); + } + + #[tokio::test] + async fn startup_migration_moves_ordinary_replacement_resume_to_dedicated_directory() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let state = ResumeState::replacement_intent( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ); + let legacy_resume = ResumeStateFile::Ordinary.path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + legacy_resume.to_str().expect("legacy resume path must be UTF-8"), + serde_json::to_vec(&state) + .expect("serialize legacy ordinary replacement state") + .into(), + ) + .await + .expect("write legacy ordinary replacement state"); + + ResumeUtils::migrate_legacy_replacement_records(&disk) + .await + .expect("startup migration should move ordinary replacement state"); + + assert_eq!( + ResumeUtils::get_replacement_intent_tasks(&disk) + .await + .expect("dedicated replacement listing should succeed"), + vec![task_id.clone()] + ); + assert_eq!( + ResumeManager::load_replacement_intent(disk.clone(), &task_id) + .await + .expect("migrated replacement intent should be readable") + .get_state() + .await, + state + ); + assert!(matches!( + disk.read_all(RUSTFS_META_BUCKET, legacy_resume.to_str().expect("legacy resume path must be UTF-8")) + .await, + Err(DiskError::FileNotFound) + )); + } + + #[tokio::test] + async fn startup_migration_reports_corrupt_ordinary_replacement_candidate() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let legacy_resume = ResumeStateFile::Ordinary.path(&task_id); + let legacy_resume = legacy_resume.to_str().expect("legacy resume path must be UTF-8"); + disk.write_all(RUSTFS_META_BUCKET, legacy_resume, b"{corrupt replacement resume".as_slice().into()) + .await + .expect("corrupt legacy replacement candidate should persist"); + + let error = ResumeUtils::migrate_legacy_replacement_records(&disk) + .await + .expect_err("a corrupt UUID-named legacy candidate must fail closed"); + + assert!(replacement_recovery_error_requires_block(&error)); + assert!(error.to_string().contains("replacement recovery corruption")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, legacy_resume) + .await + .expect("corrupt legacy state must remain for operator recovery"), + b"{corrupt replacement resume".as_slice() + ); + } + + #[tokio::test] + async fn startup_migration_reports_corrupt_flat_replacement_intent() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let legacy_intent = ResumeStateFile::LegacyReplacementIntent.path(&task_id); + let legacy_intent = legacy_intent.to_str().expect("legacy intent path must be UTF-8"); + disk.write_all(RUSTFS_META_BUCKET, legacy_intent, b"{corrupt replacement intent".as_slice().into()) + .await + .expect("corrupt legacy replacement intent should persist"); + + let error = ResumeUtils::migrate_legacy_replacement_records(&disk) + .await + .expect_err("a corrupt flat legacy intent must fail closed"); + + assert!(replacement_recovery_error_requires_block(&error)); + assert!(error.to_string().contains("replacement recovery corruption")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, legacy_intent) + .await + .expect("corrupt legacy intent must remain for operator recovery"), + b"{corrupt replacement intent".as_slice() + ); + } + + #[tokio::test] + async fn startup_migration_preserves_conflicting_dedicated_and_legacy_state() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("dedicated replacement intent should persist"); + let dedicated_path = ResumeStateFile::ReplacementIntent.path(&task_id); + let dedicated_bytes = disk + .read_all(RUSTFS_META_BUCKET, dedicated_path.to_str().expect("dedicated intent path must be UTF-8")) + .await + .expect("dedicated replacement intent should be readable"); + + let legacy_state = ResumeState::replacement_intent( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_1".to_string(), + vec!["other-bucket".to_string()], + vec!["replacement-b".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-b".to_string(), + canonical_path: "/mnt/replacement-b".to_string(), + physical_device_ids: vec!["device-b".to_string()], + filesystem_identity: "4:5:6".to_string(), + }], + ); + let legacy_path = ResumeStateFile::LegacyReplacementIntent.path(&task_id); + let legacy_bytes = serde_json::to_vec(&legacy_state).expect("serialize conflicting legacy replacement state"); + disk.write_all( + RUSTFS_META_BUCKET, + legacy_path.to_str().expect("legacy intent path must be UTF-8"), + legacy_bytes.clone().into(), + ) + .await + .expect("conflicting legacy replacement intent should persist"); + + let error = ResumeUtils::migrate_legacy_replacement_records(&disk) + .await + .expect_err("conflicting replacement states must fail closed"); + assert!(error.to_string().contains("conflicting legacy state")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, dedicated_path.to_str().expect("dedicated intent path must be UTF-8"),) + .await + .expect("dedicated state must remain after conflict"), + dedicated_bytes + ); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, legacy_path.to_str().expect("legacy intent path must be UTF-8"),) + .await + .expect("legacy state must remain after conflict"), + legacy_bytes + ); + } + + #[tokio::test] + async fn startup_migration_preserves_conflicting_dedicated_and_legacy_proof() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("dedicated replacement intent should persist"); + manager + .mark_replacement_completed_and_verified() + .await + .expect("dedicated completion proof should persist"); + let dedicated_path = replacement_completion_proof_path(&task_id); + let dedicated_bytes = disk + .read_all(RUSTFS_META_BUCKET, dedicated_path.to_str().expect("dedicated proof path must be UTF-8")) + .await + .expect("dedicated completion proof should be readable"); + + let mut legacy_proof = ReplacementCompletionProof::from_state(&manager.get_state().await, 42) + .expect("legacy completion proof fixture should build"); + legacy_proof.set_disk_id = "pool_0_set_1".to_string(); + let legacy_path = legacy_replacement_completion_proof_path(&task_id); + let legacy_bytes = serde_json::to_vec(&legacy_proof).expect("serialize conflicting legacy completion proof"); + disk.write_all( + RUSTFS_META_BUCKET, + legacy_path.to_str().expect("legacy proof path must be UTF-8"), + legacy_bytes.clone().into(), + ) + .await + .expect("conflicting legacy completion proof should persist"); + + let error = ResumeUtils::migrate_legacy_replacement_records(&disk) + .await + .expect_err("conflicting completion proofs must fail closed"); + assert!(error.to_string().contains("conflicts with legacy proof")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, dedicated_path.to_str().expect("dedicated proof path must be UTF-8"),) + .await + .expect("dedicated proof must remain after conflict"), + dedicated_bytes + ); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, legacy_path.to_str().expect("legacy proof path must be UTF-8"),) + .await + .expect("legacy proof must remain after conflict"), + legacy_bytes + ); + let error = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await { + Ok(_) => panic!("a conflicting legacy proof must not be ignored during recovery"), + Err(error) => error, + }; + assert!(error.to_string().contains("conflicts with legacy proof")); + } + + #[tokio::test] + async fn replacement_discovery_does_not_read_ordinary_resume_directory() { + let (_temp_dir, disk) = schema_test_disk().await; + for _ in 0..3 { + ResumeManager::new( + disk.clone(), + ResumeUtils::generate_task_id(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ) + .await + .expect("ordinary resume state should persist"); + } + let corrupt_task_id = ResumeUtils::generate_task_id(); + let corrupt_path = ResumeStateFile::Ordinary.path(&corrupt_task_id); + disk.write_all( + RUSTFS_META_BUCKET, + corrupt_path.to_str().expect("ordinary resume path must be UTF-8"), + b"not-json".to_vec().into(), + ) + .await + .expect("corrupt ordinary resume state should persist"); + + let replacement_task_id = ResumeUtils::generate_task_id(); + ResumeManager::new_replacement_intent( + disk.clone(), + replacement_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist in the dedicated directory"); + + assert_eq!( + ResumeUtils::get_replacement_intent_tasks(&disk) + .await + .expect("dedicated replacement listing should not parse ordinary JSON"), + vec![replacement_task_id] + ); + } + + #[tokio::test] + async fn empty_replacement_recovery_directory_is_not_an_error() { + let (_temp_dir, disk) = schema_test_disk().await; + assert!( + ResumeUtils::get_replacement_intent_tasks(&disk) + .await + .expect("missing recovery directory should be empty") + .is_empty() + ); + assert!( + ResumeUtils::get_replacement_recovery_records(&disk) + .await + .expect("missing recovery directory should have no records") + .is_empty() + ); + ResumeUtils::cleanup_expired_states(&disk, 0) + .await + .expect("missing recovery directory should not block expiry cleanup"); + } + + #[tokio::test] + async fn replacement_completion_proof_survives_resume_cleanup() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let identity = ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }; + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![identity.clone()], + ) + .await + .expect("replacement intent should persist"); + manager + .mark_replacement_completed_and_verified() + .await + .expect("verified replacement must persist proof before completion"); + + let proof = ResumeManager::load_replacement_completion_proof(disk.clone(), &task_id) + .await + .expect("completion proof must be readable from the survivor anchor"); + assert_eq!(proof.task_id, task_id); + assert_eq!(proof.replacement_generation, proof.task_id); + assert_eq!(proof.set_disk_id, "pool_0_set_0"); + assert_eq!(proof.replacement_targets, ["replacement-a"]); + assert_eq!(proof.replacement_target_identities, vec![identity]); + assert!(proof.verified_at > 0); + + manager.cleanup().await.expect("resume cleanup should succeed"); + assert!( + !ResumeManager::has_replacement_intent(&disk, &proof.task_id).await, + "completion cleanup must remove the resumable state" + ); + assert_eq!( + ResumeManager::load_replacement_completion_proof(disk, &proof.task_id) + .await + .expect("survivor proof must outlive resume cleanup"), + proof + ); + } + + #[tokio::test] + async fn replacement_recovery_records_distinguish_active_and_proven_completion() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + + let active = ResumeUtils::get_replacement_recovery_records(&disk) + .await + .expect("active replacement record should be readable"); + assert_eq!(active.len(), 1); + assert_eq!(active[0].task_id, task_id); + assert_eq!(active[0].state, ReplacementRecoveryState::WaitingForReplacement); + assert_eq!(active[0].generation.as_deref(), Some(task_id.as_str())); + + manager + .mark_replacement_completed_and_verified() + .await + .expect("completion proof should persist"); + + let cleanup_pending = ResumeUtils::get_replacement_recovery_records(&disk) + .await + .expect("cleanup-pending replacement record should be readable"); + assert_eq!(cleanup_pending.len(), 1); + assert_eq!(cleanup_pending[0].state, ReplacementRecoveryState::CleanupPending); + + manager.cleanup().await.expect("resume state cleanup should succeed"); + + let completed = ResumeUtils::get_replacement_recovery_records(&disk) + .await + .expect("completion proof should remain readable"); + assert_eq!(completed.len(), 1); + assert_eq!(completed[0].state, ReplacementRecoveryState::Completed); + assert!(completed[0].verified_at.is_some()); + } + + #[tokio::test] + async fn replacement_completion_write_failure_cannot_mark_completed() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk, + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + let proof_path = replacement_completion_proof_path(&task_id) + .to_str() + .expect("completion proof path must be UTF-8") + .to_string(); + let _failure = ReplacementProofWriteFailure::install(proof_path, DiskError::DiskAccessDenied); + + let error = manager + .mark_replacement_completed_and_verified() + .await + .expect_err("completion must fail closed when durable proof cannot be written"); + assert!(error.to_string().contains("Failed to save replacement completion proof")); + let state = manager.get_state().await; + assert!(!state.completed, "a failed proof write must not produce a completed state"); + assert_eq!(state.replacement_phase, ReplacementPhase::Intent); + } + + #[tokio::test] + async fn replacement_completion_repairs_torn_proof_without_wedging_rebuild() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let identity = ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }; + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![identity.clone()], + ) + .await + .expect("replacement intent should persist"); + manager + .mark_replacement_rebuilding(vec![identity]) + .await + .expect("replacement fixture should enter rebuilding before proof publication"); + let proof_path = replacement_completion_proof_path(&task_id); + let proof_path = proof_path.to_str().expect("completion proof path must be UTF-8"); + disk.write_all(RUSTFS_META_BUCKET, proof_path, b"{torn completion proof".as_slice().into()) + .await + .expect("torn proof fixture should persist"); + + manager + .mark_replacement_completed_and_verified() + .await + .expect("a rebuilding generation must replace only its torn completion proof"); + + let proof = ResumeManager::load_replacement_completion_proof(disk, &task_id) + .await + .expect("repaired completion proof should be durable and readable"); + assert_eq!(proof.task_id, task_id); + assert_eq!(proof.replacement_generation, proof.task_id); + } + + #[tokio::test] + async fn replacement_completion_does_not_replace_a_valid_mismatched_proof() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + let mut conflicting = ReplacementCompletionProof::from_state(&manager.get_state().await, 1) + .expect("replacement state should build a proof fixture"); + conflicting.set_disk_id = "pool_0_set_1".to_string(); + let proof_path = replacement_completion_proof_path(&task_id); + let proof_path = proof_path.to_str().expect("completion proof path must be UTF-8"); + disk.write_all( + RUSTFS_META_BUCKET, + proof_path, + serde_json::to_vec(&conflicting) + .expect("proof fixture should serialize") + .into(), + ) + .await + .expect("conflicting proof fixture should persist"); + + let error = manager + .mark_replacement_completed_and_verified() + .await + .expect_err("a distinct durable generation binding must not be overwritten"); + assert!(error.to_string().contains("does not match task")); + assert_eq!( + ResumeManager::load_replacement_completion_proof(disk, &task_id) + .await + .expect("valid conflicting proof should remain intact"), + conflicting + ); + } + + #[tokio::test] + async fn cleanup_expired_states_keeps_all_durable_replacement_phases() { + use super::super::{DiskOption, Endpoint, new_disk}; + use tempfile::TempDir; + + let temp_dir = TempDir::new().expect("create replacement expiry test directory"); + let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create test disk endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("create test disk"); + match disk.make_volume(RUSTFS_META_BUCKET).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(err) => panic!("create metadata volume for replacement expiry test: {err}"), + } + + let target = ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }; + let intent_task_id = ResumeUtils::generate_task_id(); + let rebuilding_task_id = ResumeUtils::generate_task_id(); + let verified_task_id = ResumeUtils::generate_task_id(); + let cleanup_pending_task_id = ResumeUtils::generate_task_id(); + let abandoned_task_id = ResumeUtils::generate_task_id(); + let ordinary_task_id = ResumeUtils::generate_task_id(); + let intent = ResumeManager::new_replacement_intent( + disk.clone(), + intent_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![target.clone()], + ) + .await + .expect("replacement intent should persist"); + let rebuilding = ResumeManager::new_replacement_intent( + disk.clone(), + rebuilding_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![target.clone()], + ) + .await + .expect("replacement rebuilding state should persist"); + rebuilding + .mark_replacement_rebuilding(vec![target.clone()]) + .await + .expect("replacement rebuilding phase should persist"); + let verified = ResumeManager::new_replacement_intent( + disk.clone(), + verified_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![target.clone()], + ) + .await + .expect("replacement verified state should persist"); + verified + .mark_replacement_completed_and_verified() + .await + .expect("replacement verified phase should persist"); + let cleanup_pending = ResumeManager::new_replacement_intent( + disk.clone(), + cleanup_pending_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![target.clone()], + ) + .await + .expect("replacement cleanup-pending state should persist"); + cleanup_pending + .mark_replacement_completed_and_verified() + .await + .expect("replacement completion should persist"); + cleanup_pending + .mark_replacement_cleanup_pending() + .await + .expect("replacement cleanup-pending phase should persist"); + let abandoned = ResumeManager::new_replacement_intent( + disk.clone(), + abandoned_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![target], + ) + .await + .expect("replacement abandoned state should persist"); + abandoned + .abandon_replacement_intent() + .await + .expect("replacement abandoned phase should persist"); + let ordinary = ResumeManager::new( + disk.clone(), + ordinary_task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ) + .await + .expect("ordinary resume state should persist"); + + for manager in [&intent, &rebuilding, &verified, &cleanup_pending, &abandoned, &ordinary] { + manager.state.write().await.last_update = 0; + manager.save_state_strict().await.expect("persist expired resume state"); + } + + ResumeUtils::cleanup_expired_states(&disk, 0) + .await + .expect("replacement expiry cleanup should complete"); + + for (task_id, expected_phase) in [ + (intent_task_id.as_str(), ReplacementPhase::Intent), + (rebuilding_task_id.as_str(), ReplacementPhase::Rebuilding), + (verified_task_id.as_str(), ReplacementPhase::Verified), + (cleanup_pending_task_id.as_str(), ReplacementPhase::CleanupPending), + ] { + let state = ResumeManager::load_replacement_intent(disk.clone(), task_id) + .await + .expect("durable replacement state must survive expiry cleanup") + .get_state() + .await; + assert_eq!(state.replacement_phase, expected_phase); + } + assert!( + !ResumeManager::has_replacement_intent(&disk, &abandoned_task_id).await, + "an abandoned replacement must expire" + ); + assert!( + !ResumeManager::has_resume_state(&disk, &ordinary_task_id).await, + "an ordinary expired resume must expire" + ); + } + #[tokio::test] async fn test_resume_state_progress() { let task_id = ResumeUtils::generate_task_id(); @@ -1011,6 +3559,98 @@ mod tests { assert_eq!(progress, 10.0); } + #[tokio::test] + async fn replacement_intent_rejects_a_new_mount_at_the_same_endpoint() { + use super::super::{DiskOption, Endpoint, new_disk}; + use tempfile::TempDir; + + let temp_dir = TempDir::new().unwrap(); + let disk_path = temp_dir.path().join("resume_disk"); + std::fs::create_dir_all(&disk_path).unwrap(); + let endpoint = Endpoint::try_from(disk_path.to_string_lossy().as_ref()).unwrap(); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .unwrap(); + let _ = disk.make_volume(RUSTFS_META_BUCKET).await; + let _ = disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await; + + let task_id = ResumeUtils::generate_task_id(); + let targets = vec!["replacement-a".to_string()]; + let first = ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }; + ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket-a".to_string()], + targets.clone(), + vec![first.clone()], + ) + .await + .unwrap(); + + let reused = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket-b".to_string()], + targets.clone(), + vec![first.clone()], + ) + .await + .unwrap(); + assert_eq!( + reused.get_state().await.replacement_buckets, + ["bucket-a"], + "retries must keep the first generation's bucket plan" + ); + + let mut second = ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: first.physical_device_ids.clone(), + filesystem_identity: first.filesystem_identity.clone(), + }; + for changed_identity in [ + { + second.physical_device_ids = vec!["device-b".to_string()]; + second.clone() + }, + { + second.physical_device_ids = first.physical_device_ids.clone(); + second.filesystem_identity = "4:5:6".to_string(); + second.clone() + }, + { + second.filesystem_identity = first.filesystem_identity.clone(); + second.canonical_path = "/mnt/replacement-b".to_string(); + second.clone() + }, + ] { + let result = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket-a".to_string()], + targets.clone(), + vec![changed_identity], + ) + .await; + assert!(result.is_err(), "a new mounted instance must not reuse the old replacement cursor"); + } + temp_dir.close().unwrap(); + } + #[test] fn reset_for_retry_clears_progress_but_keeps_retry_budget() { // backlog#855 / #799 B6: a retry must re-scan from the start without @@ -1144,24 +3784,7 @@ mod tests { #[tokio::test] async fn test_resumestate_schema_v0_discarded_on_load() { - use super::super::{DiskOption, Endpoint, new_disk}; - use tempfile::TempDir; - - let temp_dir = TempDir::new().unwrap(); - let disk_path = temp_dir.path().join("test_disk"); - std::fs::create_dir_all(&disk_path).unwrap(); - let endpoint = Endpoint::try_from(disk_path.to_string_lossy().as_ref()).unwrap(); - let disk = new_disk( - &endpoint, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .unwrap(); - let _ = disk.make_volume(RUSTFS_META_BUCKET).await; - let _ = disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await; + let (temp_dir, disk) = schema_test_disk().await; // Legacy snapshot: no schema_version, a stale positional cursor and progress. let legacy = r#"{ @@ -1185,12 +3808,14 @@ mod tests { "max_retries": 3, "resume_cursor": "v1:stale-token" }"#; - let file_path = format!("{BUCKET_META_PREFIX}/old-task_{RESUME_STATE_FILE}"); + let task_id = "00000000-0000-4000-8000-000000000001"; + let legacy = legacy.replace("old-task", task_id); + let file_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); disk.write_all(RUSTFS_META_BUCKET, &file_path, legacy.as_bytes().to_vec().into()) .await - .unwrap(); + .expect("write legacy resume state"); - let manager = ResumeManager::load_from_disk(disk.clone(), "old-task").await.unwrap(); + let manager = ResumeManager::load_from_disk(disk.clone(), task_id).await.unwrap(); let state = manager.get_state().await; assert_eq!(state.schema_version, CURRENT_RESUME_SCHEMA, "schema must be stamped current"); assert_eq!(state.resume_cursor, None, "stale cursor must be cleared"); @@ -1198,33 +3823,19 @@ mod tests { assert_eq!(state.successful_objects, 0); assert_eq!(state.failed_objects, 0); assert!(!state.completed); - temp_dir.close().unwrap(); + temp_dir.close().expect("remove schema test directory"); } #[tokio::test] - async fn test_checkpoint_schema_v0_discarded_on_load() { - use super::super::{DiskOption, Endpoint, new_disk}; - use tempfile::TempDir; + async fn test_checkpoint_schema_v4_discarded_on_load() { + let (temp_dir, disk) = schema_test_disk().await; - let temp_dir = TempDir::new().unwrap(); - let disk_path = temp_dir.path().join("test_disk"); - std::fs::create_dir_all(&disk_path).unwrap(); - let endpoint = Endpoint::try_from(disk_path.to_string_lossy().as_ref()).unwrap(); - let disk = new_disk( - &endpoint, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .unwrap(); - let _ = disk.make_volume(RUSTFS_META_BUCKET).await; - let _ = disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await; - - // Legacy checkpoint: no schema_version, stale position and dedup sets. + // The previous checkpoint schema is unsafe once its paired resume + // state is discarded: retaining either position would skip work. + let task_id = "00000000-0000-4000-8000-000000000002"; let legacy = r#"{ - "task_id": "old-task", + "schema_version": 4, + "task_id": "00000000-0000-4000-8000-000000000002", "checkpoint_time": 1700000000, "current_bucket_index": 2, "current_object_index": 500, @@ -1232,19 +3843,95 @@ mod tests { "failed_objects": ["c"], "skipped_objects": ["d"] }"#; - let file_path = format!("{BUCKET_META_PREFIX}/old-task_{RESUME_CHECKPOINT_FILE}"); + let file_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); disk.write_all(RUSTFS_META_BUCKET, &file_path, legacy.as_bytes().to_vec().into()) .await - .unwrap(); + .expect("write legacy checkpoint"); - let manager = CheckpointManager::load_from_disk(disk.clone(), "old-task").await.unwrap(); + let manager = CheckpointManager::load_from_disk(disk.clone(), task_id).await.unwrap(); let checkpoint = manager.get_checkpoint().await; assert_eq!(checkpoint.schema_version, CURRENT_CHECKPOINT_SCHEMA, "schema must be stamped current"); + assert_eq!(checkpoint.current_bucket_index, 0, "stale bucket position must be reset"); assert_eq!(checkpoint.current_object_index, 0, "stale position must be reset"); assert!(checkpoint.processed_objects.is_empty()); assert!(checkpoint.failed_objects.is_empty()); assert!(checkpoint.skipped_objects.is_empty()); - temp_dir.close().unwrap(); + temp_dir.close().expect("remove schema test directory"); + } + + #[tokio::test] + async fn current_normal_resume_schema_preserves_progress() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let mut state = ResumeState::new( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket-b".to_string()], + ); + state.resume_cursor = Some("opaque-marker".to_string()); + state.processed_objects = 7; + state.successful_objects = 6; + state.failed_objects = 1; + state.completed_buckets = vec!["bucket-a".to_string()]; + let file_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); + let state_data = serde_json::to_vec(&state).expect("serialize current normal resume state"); + disk.write_all(RUSTFS_META_BUCKET, &file_path, state_data.into()) + .await + .expect("write current normal resume state"); + + let restored = ResumeManager::load_from_disk(disk.clone(), &task_id) + .await + .expect("load current normal resume state") + .get_state() + .await; + + assert_eq!(restored.schema_version, CURRENT_RESUME_SCHEMA); + assert_eq!(restored.resume_cursor.as_deref(), Some("opaque-marker")); + assert_eq!(restored.processed_objects, 7); + assert_eq!(restored.successful_objects, 6); + assert_eq!(restored.failed_objects, 1); + assert_eq!(restored.completed_buckets, ["bucket-a"]); + assert!(restored.replacement_targets.is_empty()); + assert_eq!(restored.replacement_generation, None); + assert_eq!(restored.replacement_phase, ReplacementPhase::None); + temp_dir.close().expect("remove schema test directory"); + } + + #[tokio::test] + async fn future_resume_and_checkpoint_schemas_are_rejected() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let mut state = ResumeState::new(task_id.clone(), "erasure_set".to_string(), "pool_0_set_0".to_string(), Vec::new()); + state.schema_version = CURRENT_RESUME_SCHEMA + 1; + let state_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); + let state_data = serde_json::to_vec(&state).expect("serialize future resume state"); + disk.write_all(RUSTFS_META_BUCKET, &state_path, state_data.into()) + .await + .expect("write future resume state"); + + let resume_error = match ResumeManager::load_from_disk(disk.clone(), &task_id).await { + Ok(_) => panic!("future resume schema must not load"), + Err(error) => error, + }; + assert!(matches!(resume_error, Error::TaskExecutionFailed { .. })); + assert!(resume_error.to_string().contains("newer than supported schema")); + + let mut checkpoint = ResumeCheckpoint::new(task_id.clone()); + checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1; + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); + let checkpoint_data = serde_json::to_vec(&checkpoint).expect("serialize future checkpoint"); + disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, checkpoint_data.into()) + .await + .expect("write future checkpoint"); + + let checkpoint_error = match CheckpointManager::load_from_disk(disk.clone(), &task_id).await { + Ok(_) => panic!("future checkpoint schema must not load"), + Err(error) => error, + }; + assert!(matches!(checkpoint_error, Error::TaskExecutionFailed { .. })); + assert!(checkpoint_error.to_string().contains("newer than supported schema")); + temp_dir.close().expect("remove schema test directory"); } #[test] @@ -1279,7 +3966,7 @@ mod tests { Err(err) => panic!("create metadata volume for resume persistence test: {err}"), } - let task_id = "completion-persistence".to_string(); + let task_id = ResumeUtils::generate_task_id(); let manager = ResumeManager::new( disk.clone(), task_id.clone(), @@ -1335,6 +4022,50 @@ mod tests { .expect("missing checkpoint must be idempotent success"); } + #[tokio::test] + async fn checkpoint_rejects_a_task_id_mismatched_to_its_file_name() { + use super::super::{DiskOption, Endpoint, new_disk}; + use tempfile::TempDir; + + let temp_dir = TempDir::new().expect("create checkpoint binding test directory"); + let endpoint = + Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create checkpoint binding test endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("create checkpoint binding test disk"); + match disk.make_volume(RUSTFS_META_BUCKET).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(err) => panic!("create checkpoint binding metadata volume: {err}"), + } + + let requested_task_id = ResumeUtils::generate_task_id(); + let checkpoint = ResumeCheckpoint::new(ResumeUtils::generate_task_id()); + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{requested_task_id}_{RESUME_CHECKPOINT_FILE}"); + disk.write_all( + RUSTFS_META_BUCKET, + &checkpoint_path, + serde_json::to_vec(&checkpoint) + .expect("serialize mismatched checkpoint") + .into(), + ) + .await + .expect("persist mismatched checkpoint"); + + let error = match CheckpointManager::load_from_disk(disk, &requested_task_id).await { + Ok(_) => panic!("checkpoint task id must be bound to its file name"), + Err(error) => error, + }; + + assert!(error.to_string().contains("does not match")); + temp_dir.close().expect("remove checkpoint binding test directory"); + } + #[tokio::test] async fn test_resume_utils() { let task_id1 = ResumeUtils::generate_task_id(); @@ -1343,6 +4074,9 @@ mod tests { assert_ne!(task_id1, task_id2); assert_eq!(task_id1.len(), 36); // UUID length assert_eq!(task_id2.len(), 36); + assert!(validate_resume_task_id(&task_id1).is_ok()); + assert!(validate_resume_task_id(&format!("pool_0_set_0_{task_id1}")).is_err()); + assert!(validate_resume_task_id(&task_id1.to_uppercase()).is_err()); } #[tokio::test] @@ -1369,9 +4103,9 @@ mod tests { // Create some test resume state files let task_ids = vec![ - "test-task-1".to_string(), - "test-task-2".to_string(), - "test-task-3".to_string(), + ResumeUtils::generate_task_id(), + ResumeUtils::generate_task_id(), + ResumeUtils::generate_task_id(), ]; // Save resume state files for each task @@ -1397,6 +4131,8 @@ mod tests { "task4_ahm_checkpoint.json", "task5_ahm_progress.json", "_ahm_resume_state.json", // Invalid: empty task ID + "not-a-uuid_ahm_resume_state.json", + "00000000-0000-4000-8000-000000000001_extra_ahm_resume_state.json", ]; for file_name in non_resume_files { @@ -1419,8 +4155,87 @@ mod tests { assert!(!found_task_ids.contains(&"".to_string())); assert!(!found_task_ids.contains(&"task4".to_string())); assert!(!found_task_ids.contains(&"task5".to_string())); + assert!(!found_task_ids.contains(&"not-a-uuid".to_string())); + + let error = match ResumeManager::load_from_disk(disk.clone(), "../not-a-uuid").await { + Ok(_) => panic!("a traversal-like task id must be rejected before reading metadata"), + Err(error) => error, + }; + assert!(matches!( + error, + Error::TaskExecutionFailed { message } if message == "Invalid resume task id" + )); // Clean up temp_dir.close().unwrap(); } + + #[tokio::test] + async fn resume_state_rejects_filename_and_json_task_id_mismatch() { + use super::super::{DiskOption, Endpoint, new_disk}; + use tempfile::TempDir; + + let temp_dir = TempDir::new().expect("create resume mismatch test directory"); + let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create test disk endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("create resume mismatch test disk"); + match disk.make_volume(RUSTFS_META_BUCKET).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(error) => panic!("create metadata volume for resume mismatch test: {error}"), + } + + let filename_task_id = ResumeUtils::generate_task_id(); + let state = ResumeState::new( + ResumeUtils::generate_task_id(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ); + let path = format!("{BUCKET_META_PREFIX}/{filename_task_id}_{RESUME_STATE_FILE}"); + disk.write_all( + RUSTFS_META_BUCKET, + &path, + serde_json::to_vec(&state).expect("serialize mismatched resume state").into(), + ) + .await + .expect("write mismatched resume state"); + + let error = match ResumeManager::load_from_disk(disk.clone(), &filename_task_id).await { + Ok(_) => panic!("resume state task id must match its filename"), + Err(error) => error, + }; + assert!(matches!( + error, + Error::TaskExecutionFailed { message } if message == "Resume state task id does not match filename" + )); + + let checkpoint_filename_task_id = ResumeUtils::generate_task_id(); + let checkpoint = ResumeCheckpoint::new(ResumeUtils::generate_task_id()); + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{checkpoint_filename_task_id}_{RESUME_CHECKPOINT_FILE}"); + disk.write_all( + RUSTFS_META_BUCKET, + &checkpoint_path, + serde_json::to_vec(&checkpoint) + .expect("serialize mismatched resume checkpoint") + .into(), + ) + .await + .expect("write mismatched resume checkpoint"); + + let error = match CheckpointManager::load_from_disk(disk, &checkpoint_filename_task_id).await { + Ok(_) => panic!("resume checkpoint task id must match its filename"), + Err(error) => error, + }; + assert!(matches!( + error, + Error::TaskExecutionFailed { message } if message == "Resume checkpoint task id does not match filename" + )); + } } diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index a9d9f04ca..f5101ac1c 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -26,7 +26,7 @@ use super::storage_api::storage::{ BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _, ObjectOperations as _, StorageAdminApi, }; -use super::{DiskStore, ECStore, Endpoint, StorageError}; +use super::{DiskStore, ECStore, Endpoint, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity}; pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader}; const LOG_COMPONENT_HEAL: &str = "heal"; @@ -37,6 +37,11 @@ const EVENT_HEAL_STORAGE_OBJECT_VERIFY: &str = "heal_storage_object_verify"; const EVENT_HEAL_STORAGE_ADMIN_OP: &str = "heal_storage_admin_op"; const EVENT_HEAL_STORAGE_REPAIR_OP: &str = "heal_storage_repair_op"; +pub enum ReplacementResumeDisk { + Fresh, + Existing(DiskStore), +} + pub(crate) fn next_heal_listing_token( bucket: &str, prefix: &str, @@ -354,6 +359,42 @@ pub trait HealStorageAPI: Send + Sync { /// Heal format using ecstore async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)>; + /// Heal only the explicitly admitted replacement targets in one erasure set. + /// + /// The default is deliberately fail-closed so alternate storage + /// implementations cannot accidentally fall back to the global format path. + async fn heal_replacement_format( + &self, + _dry_run: bool, + _pool_index: usize, + _set_index: usize, + _targets: &[String], + ) -> Result<(HealResultItem, Option)> { + Err(Error::other("target-scoped replacement format is unsupported")) + } + + /// Recheck admitted replacement targets immediately before destructive work. + async fn replacement_targets_ready(&self, _targets: &[String]) -> Result { + Ok(false) + } + + /// Read target-specific physical evidence for one replacement version. + /// + /// This is only used by automatic replacement healing after the normal + /// transaction returns success. The conservative default prevents an + /// alternate backend from turning an unverified replacement into a + /// completed generation. + async fn replacement_targets_have_version( + &self, + _bucket: &str, + _object: &str, + _version_id: Option<&str>, + _opts: &HealOpts, + _targets: &[String], + ) -> Result { + Ok(false) + } + /// List object versions for healing (returns all versions, may use significant memory for large buckets) /// /// WARNING: This method loads all object versions into memory at once. For buckets with many @@ -390,8 +431,30 @@ pub trait HealStorageAPI: Send + Sync { self.list_objects_for_heal_page(bucket, prefix, continuation_token).await } - /// Get disk for resume functionality + /// Get disk for resume functionality. async fn get_disk_for_resume(&self, set_disk_id: &str) -> Result; + + /// Get a healthy non-target disk for durable replacement state. + async fn get_disk_for_resume_excluding(&self, _set_disk_id: &str, _excluded_targets: &[String]) -> Result { + Err(Error::other("target-excluding resume disk selection is unsupported")) + } + + /// Reopen the exact surviving disk that owns an existing replacement + /// intent. Falling back to another disk would create a second copy of the + /// same generation and split its progress. + async fn get_replacement_resume_disk( + &self, + _set_disk_id: &str, + _task_id: &str, + _excluded_targets: &[String], + ) -> Result { + Err(Error::other("durable replacement resume selection is unsupported")) + } + + /// Capture the mounted replacement instance before it is formatted. + async fn replacement_target_identities(&self, _targets: &[String]) -> Result> { + Err(Error::other("replacement target identity collection is unsupported")) + } } /// ECStore Heal storage layer implementation @@ -1306,6 +1369,44 @@ impl HealStorageAPI for ECStoreHealStorage { } } + async fn heal_replacement_format( + &self, + dry_run: bool, + pool_index: usize, + set_index: usize, + targets: &[String], + ) -> Result<(HealResultItem, Option)> { + self.ecstore + .heal_replacement_format(dry_run, pool_index, set_index, targets) + .await + .map(|(result, error)| (result, error.map(Error::Storage))) + .map_err(Error::Storage) + } + + async fn replacement_targets_ready(&self, targets: &[String]) -> Result { + Ok(super::replacement_readiness::auto_replacement_targets_ready(targets).await) + } + + async fn replacement_targets_have_version( + &self, + bucket: &str, + object: &str, + version_id: Option<&str>, + opts: &HealOpts, + targets: &[String], + ) -> Result { + let pool_index = opts + .pool + .ok_or_else(|| Error::other("replacement target readback is missing pool scope"))?; + let set_index = opts + .set + .ok_or_else(|| Error::other("replacement target readback is missing set scope"))?; + self.ecstore + .replacement_targets_have_version(bucket, object, version_id.unwrap_or(""), pool_index, set_index, targets) + .await + .map_err(Error::Storage) + } + async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result> { debug!( target: "rustfs::heal::storage", @@ -1543,6 +1644,10 @@ impl HealStorageAPI for ECStoreHealStorage { } async fn get_disk_for_resume(&self, set_disk_id: &str) -> Result { + self.get_disk_for_resume_excluding(set_disk_id, &[]).await + } + + async fn get_disk_for_resume_excluding(&self, set_disk_id: &str, excluded_targets: &[String]) -> Result { debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_ADMIN_OP, @@ -1564,8 +1669,18 @@ impl HealStorageAPI for ECStoreHealStorage { message: format!("Failed to get disks for pool {pool_idx} set {set_idx}: {e}"), })?; - // Find the first available disk - if let Some(disk_store) = disks.into_iter().flatten().next() { + // The replacement target is unformatted before repair and must never + // host the intent that authorizes its own formatting. + for disk_store in disks.into_iter().flatten() { + if !disk_store.endpoint().is_local { + continue; + } + if excluded_targets.contains(&disk_store.endpoint().to_string()) { + continue; + } + if !matches!(disk_store.get_disk_id().await, Ok(Some(id)) if !id.is_nil()) { + continue; + } debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_ADMIN_OP, @@ -1584,6 +1699,43 @@ impl HealStorageAPI for ECStoreHealStorage { message: format!("No available disk found for set_disk_id: {set_disk_id}"), }) } + + async fn get_replacement_resume_disk( + &self, + set_disk_id: &str, + task_id: &str, + excluded_targets: &[String], + ) -> Result { + let (pool_idx, set_idx) = crate::heal::utils::parse_set_disk_id(set_disk_id)?; + let disks = StorageAdminApi::disk_set_inventory(self.ecstore.as_ref(), DiskSetSelector::new(pool_idx, set_idx)) + .await + .map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to get disks for pool {pool_idx} set {set_idx}: {e}"), + })?; + let mut existing = None; + for disk_store in disks.into_iter().flatten() { + if !disk_store.endpoint().is_local || excluded_targets.contains(&disk_store.endpoint().to_string()) { + continue; + } + if !matches!(disk_store.get_disk_id().await, Ok(Some(id)) if !id.is_nil()) { + continue; + } + if super::resume::ResumeManager::has_replacement_intent(&disk_store, task_id).await + && existing.replace(disk_store).is_some() + { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement resume intent is duplicated for set_disk_id: {set_disk_id}"), + }); + } + } + Ok(existing.map_or(ReplacementResumeDisk::Fresh, ReplacementResumeDisk::Existing)) + } + + async fn replacement_target_identities(&self, targets: &[String]) -> Result> { + super::replacement_readiness::auto_replacement_target_identities(targets) + .await + .ok_or_else(|| Error::other("replacement target is not a stable mounted disk")) + } } #[cfg(test)] diff --git a/crates/heal/src/heal/storage_api.rs b/crates/heal/src/heal/storage_api.rs index 061ca33e0..417998e39 100644 --- a/crates/heal/src/heal/storage_api.rs +++ b/crates/heal/src/heal/storage_api.rs @@ -16,8 +16,9 @@ pub(crate) use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint; pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult}; pub(crate) use rustfs_ecstore::api::disk::{ - BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, DeleteOptions as EcstoreDeleteOptions, - DiskAPI as EcstoreDiskAPI, DiskStore as EcstoreDiskStore, HEALING_MARKER_PATH as ECSTORE_HEALING_MARKER_PATH, + BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, + ConditionalFileUpdate as EcstoreConditionalFileUpdate, DeleteOptions as EcstoreDeleteOptions, DiskAPI as EcstoreDiskAPI, + DiskStore as EcstoreDiskStore, HEALING_MARKER_PATH as ECSTORE_HEALING_MARKER_PATH, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, }; #[cfg(test)] @@ -32,8 +33,9 @@ pub(crate) mod owner { pub(crate) use super::{ ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET, - EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore, - EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ecstore_local_disk_map_read, + EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, + EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, + ecstore_local_disk_map_read, }; #[cfg(test)] diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index 780ccf022..ddf83646e 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -13,8 +13,12 @@ // limitations under the License. use crate::heal::{ - DiskError, EcstoreError, ErasureSetHealer, + DiskError, EcstoreError, ErasureSetHealer, HealDiskExt as _, + erasure_healer::target_outcomes_complete, progress::HealProgress, + resume::{ + CheckpointManager, ReplacementPhase, ReplacementTargetIdentity, ResumeManager, replacement_target_identities_match, + }, storage::{HealStorageAPI, next_heal_listing_token}, }; use crate::{Error, Result}; @@ -350,6 +354,9 @@ pub struct HealTask { pub retry_attempts: u32, /// Endpoints of the disks being rebuilt (see `HealRequest::heal_endpoints`). pub heal_endpoints: Vec, + /// Durable resume anchor injected by the manager for an existing automatic + /// replacement generation. + replacement_resume_endpoint: Option, /// Task status pub status: Arc>, /// Progress tracking @@ -376,6 +383,24 @@ pub struct HealTask { } impl HealTask { + async fn verify_replacement_identity_fence( + &self, + expected_identities: &[ReplacementTargetIdentity], + set_disk_id: &str, + stage: &str, + ) -> Result<()> { + let actual_identities = self + .await_with_control(self.storage.replacement_target_identities(&self.heal_endpoints)) + .await?; + if replacement_target_identities_match(expected_identities, &actual_identities) { + return Ok(()); + } + + Err(Error::TaskExecutionFailed { + message: format!("Replacement target changed during {stage} for automatic heal {set_disk_id}"), + }) + } + pub fn from_request(request: HealRequest, storage: Arc) -> Self { Self { id: request.id, @@ -385,6 +410,7 @@ impl HealTask { source: request.source, retry_attempts: request.retry_attempts, heal_endpoints: request.heal_endpoints, + replacement_resume_endpoint: None, status: Arc::new(RwLock::new(HealTaskStatus::Pending)), progress: Arc::new(RwLock::new(HealProgress::new())), result_items: Arc::new(RwLock::new(Vec::new())), @@ -416,6 +442,16 @@ impl HealTask { } } + pub(crate) fn from_replacement_recovery_request( + request: HealRequest, + storage: Arc, + replacement_resume_endpoint: Option, + ) -> Self { + let mut task = Self::from_request(request, storage); + task.replacement_resume_endpoint = replacement_resume_endpoint; + task + } + pub fn metric_type_label(&self) -> &'static str { match &self.heal_type { HealType::Cluster => "cluster", @@ -2131,7 +2167,87 @@ impl HealTask { progress.update_progress(0, 4, 0, 0); } - let buckets = if buckets.is_empty() { + let is_auto_replacement = matches!(self.source, HealRequestSource::AutoHeal) && !self.heal_endpoints.is_empty(); + let replacement_resume_disk = if is_auto_replacement { + let mut requested_targets = self.heal_endpoints.clone(); + requested_targets.sort_unstable(); + requested_targets.dedup(); + let selection = self + .await_with_control( + self.storage + .get_replacement_resume_disk(&set_disk_id, &self.id, &self.heal_endpoints), + ) + .await?; + let disk = match selection { + crate::heal::storage::ReplacementResumeDisk::Existing(disk) => { + if let Some(anchor) = &self.replacement_resume_endpoint + && disk.endpoint().to_string() != *anchor + { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement resume anchor changed for automatic heal {set_disk_id}"), + }); + } + Some(disk) + } + crate::heal::storage::ReplacementResumeDisk::Fresh => { + if self.replacement_resume_endpoint.is_some() { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement resume anchor is unavailable for automatic heal {set_disk_id}"), + }); + } + None + } + }; + if let Some(disk) = disk.as_ref() + && ResumeManager::has_replacement_intent(disk, &self.id).await + { + let resume_manager = ResumeManager::load_replacement_intent(disk.clone(), &self.id).await?; + let state = resume_manager.get_state().await; + if state.completed + && matches!(state.replacement_phase, ReplacementPhase::CleanupPending) + && state.set_disk_id == set_disk_id + && state.replacement_targets == requested_targets + && state.replacement_generation.as_deref() == Some(self.id.as_str()) + { + resume_manager.ensure_replacement_completion_proof().await?; + if CheckpointManager::has_checkpoint(disk, &self.id).await { + CheckpointManager::load_from_disk(disk.clone(), &self.id) + .await? + .cleanup() + .await?; + } + resume_manager.cleanup().await?; + return Ok(()); + } + } + disk + } else { + None + }; + + if is_auto_replacement + && !self + .await_with_control(self.storage.replacement_targets_ready(&self.heal_endpoints)) + .await? + { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement target is no longer ready for automatic heal {set_disk_id}"), + }); + } + + let replacement_resume_disk = if is_auto_replacement { + Some(match replacement_resume_disk { + Some(disk) => disk, + None => { + self.await_with_control(self.storage.get_disk_for_resume_excluding(&set_disk_id, &self.heal_endpoints)) + .await? + } + }) + } else { + None + }; + + let mut buckets = if buckets.is_empty() { debug!( target: "rustfs::heal::task", event = EVENT_HEAL_ERASURE_SET_STAGE, @@ -2148,6 +2264,50 @@ impl HealTask { buckets }; + // Persist automatic replacement intent on a surviving disk before the + // first target format write. A task retry keeps this id; a newly + // admitted blank replacement gets a fresh id and cannot reuse cursor + // progress from an older disk at the same endpoint. + let replacement_resume = if is_auto_replacement { + let identities = self + .await_with_control(self.storage.replacement_target_identities(&self.heal_endpoints)) + .await?; + let disk = replacement_resume_disk.clone().ok_or_else(|| Error::TaskExecutionFailed { + message: format!("Replacement resume disk is missing for automatic heal {set_disk_id}"), + })?; + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + self.id.clone(), + set_disk_id.clone(), + buckets.clone(), + self.heal_endpoints.clone(), + identities.clone(), + ) + .await?; + buckets = manager.get_state().await.replacement_buckets; + Some((disk, manager, identities)) + } else { + None + }; + + let healing_marker = format!("{set_disk_id}:{}", self.id); + if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() { + let state = resume_manager.get_state().await; + if state.completed && matches!(state.replacement_phase, ReplacementPhase::Verified) { + resume_manager.ensure_replacement_completion_proof().await?; + super::clear_healing_markers_after_verified(&self.heal_endpoints, &healing_marker).await?; + resume_manager.mark_replacement_cleanup_pending().await?; + if CheckpointManager::has_checkpoint(disk, &self.id).await { + CheckpointManager::load_from_disk(disk.clone(), &self.id) + .await? + .cleanup() + .await?; + } + resume_manager.cleanup().await?; + return Ok(()); + } + } + // Step 1: Perform disk format heal using ecstore debug!( target: "rustfs::heal::task", @@ -2159,7 +2319,32 @@ impl HealTask { stage = "heal_format", "Heal erasure set stage entered" ); - let format_result = self.await_with_control(self.storage.heal_format(self.options.dry_run)).await; + if is_auto_replacement { + let Some((_, _, expected_identities)) = replacement_resume.as_ref() else { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement intent is missing for automatic heal {set_disk_id}"), + }); + }; + self.verify_replacement_identity_fence(expected_identities, &set_disk_id, "format") + .await?; + } + let format_result = if is_auto_replacement { + let pool_index = self.options.pool_index.ok_or_else(|| Error::TaskExecutionFailed { + message: format!("Missing pool scope for automatic replacement heal {set_disk_id}"), + })?; + let set_index = self.options.set_index.ok_or_else(|| Error::TaskExecutionFailed { + message: format!("Missing set scope for automatic replacement heal {set_disk_id}"), + })?; + self.await_with_control(self.storage.heal_replacement_format( + self.options.dry_run, + pool_index, + set_index, + &self.heal_endpoints, + )) + .await + } else { + self.await_with_control(self.storage.heal_format(self.options.dry_run)).await + }; match format_result { Ok((result, error)) => { @@ -2209,6 +2394,22 @@ impl HealTask { "Heal erasure set format repaired" ); } + if !self.options.dry_run && !target_outcomes_complete(&result, &self.heal_endpoints) { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to verify formatted replacement targets for {set_disk_id}"), + }); + } + if let Some((_, replacement_resume, expected_identities)) = &replacement_resume { + let identities = self + .await_with_control(self.storage.replacement_target_identities(&self.heal_endpoints)) + .await?; + if !replacement_target_identities_match(expected_identities, &identities) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement target changed after format for automatic heal {set_disk_id}"), + }); + } + replacement_resume.mark_replacement_rebuilding(identities).await?; + } } Err(Error::TaskCancelled) => return Err(Error::TaskCancelled), Err(Error::TaskTimeout) => return Err(Error::TaskTimeout), @@ -2241,7 +2442,7 @@ impl HealTask { // The rebuilt disks are formatted now: mark them as healing so // DiskInfo.healing reflects the rebuild until it completes. - super::set_healing_markers(&self.heal_endpoints, &set_disk_id).await; + super::set_healing_markers(&self.heal_endpoints, &healing_marker).await?; // Step 2: Get disk for resume functionality debug!( @@ -2254,9 +2455,14 @@ impl HealTask { stage = "resolve_resume_disk", "Heal erasure set stage entered" ); - let disk = self - .await_with_control(self.storage.get_disk_for_resume(&set_disk_id)) - .await?; + let replacement_target_identities = replacement_resume.as_ref().map(|(_, _, identities)| identities.clone()); + let disk = match replacement_resume.as_ref() { + Some((disk, _, _)) => disk.clone(), + None => { + self.await_with_control(self.storage.get_disk_for_resume(&set_disk_id)) + .await? + } + }; { let mut progress = self.progress.write().await; @@ -2280,6 +2486,10 @@ impl HealTask { for bucket in buckets.iter() { // Check control flags before starting each bucket heal self.check_control_flags().await?; + if let Some(expected_identities) = replacement_target_identities.as_ref() { + self.verify_replacement_identity_fence(expected_identities, &set_disk_id, "bucket prepass") + .await?; + } let heal_result = self .await_with_control(self.storage.heal_bucket(bucket, &bucket_heal_opts)) .await; @@ -2334,7 +2544,9 @@ impl HealTask { disk, heal_opts, self.source, - ); + ) + .with_replacement_targets(self.heal_endpoints.clone(), is_auto_replacement.then(|| self.id.clone())) + .with_replacement_identity_fence(replacement_target_identities.clone()); { let mut progress = self.progress.write().await; @@ -2358,9 +2570,27 @@ impl HealTask { // Keep the markers on failure: the resume state also persists, and the // next run of this set heal re-marks and eventually clears them. - if result.is_ok() { - super::clear_healing_markers(&self.heal_endpoints).await; - } + let result = match result { + Ok(()) => { + if let Some(expected_identities) = replacement_target_identities.as_ref() { + self.verify_replacement_identity_fence(expected_identities, &set_disk_id, "marker completion") + .await?; + } + super::clear_healing_markers_after_verified(&self.heal_endpoints, &healing_marker).await?; + if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() { + resume_manager.mark_replacement_cleanup_pending().await?; + if CheckpointManager::has_checkpoint(disk, &self.id).await { + CheckpointManager::load_from_disk(disk.clone(), &self.id) + .await? + .cleanup() + .await?; + } + resume_manager.cleanup().await?; + } + Ok(()) + } + Err(err) => Err(err), + }; { let mut progress = self.progress.write().await; @@ -2420,12 +2650,353 @@ mod tests { use super::super::{DiskOption, DiskStore, Endpoint, HealDiskExt as _, new_disk}; use super::*; use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo}; - use rustfs_madmin::heal_commands::HealResultItem; + use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; use std::collections::{HashMap, VecDeque}; use std::sync::Mutex; use tempfile::TempDir; use super::super::storage_api::status::BucketInfo; + + #[test] + fn format_result_requires_every_requested_target_to_be_ok() { + let result = HealResultItem { + after: Infos { + drives: vec![ + HealDriveInfo { + endpoint: "disk-a".to_string(), + state: "ok".to_string(), + ..Default::default() + }, + HealDriveInfo { + endpoint: "disk-b".to_string(), + state: "missing".to_string(), + ..Default::default() + }, + ], + }, + ..Default::default() + }; + + assert!(target_outcomes_complete(&result, &["disk-a".to_string()])); + assert!(!target_outcomes_complete(&result, &["disk-a".to_string(), "disk-b".to_string()])); + assert!(!target_outcomes_complete(&result, &["disk-c".to_string()])); + } + + #[tokio::test] + async fn automatic_replacement_uses_target_scoped_format() { + let temp = TempDir::new().expect("temporary resume disk directory should be created"); + let disk = make_resume_disk(&temp).await; + let storage = Arc::new(MockStorage { + replacement_targets_ready: Mutex::new(true), + resume_disk: Mutex::new(Some(disk)), + ..Default::default() + }); + let mut request = HealRequest::new( + HealType::ErasureSet { + buckets: Vec::new(), + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + pool_index: Some(0), + set_index: Some(0), + ..Default::default() + }, + HealPriority::Low, + ); + request.source = HealRequestSource::AutoHeal; + request.heal_endpoints = vec!["replacement-a".to_string()]; + let task = HealTask::from_request(request, storage.clone()); + + task.execute() + .await + .expect_err("the mock has no local replacement marker target"); + + assert_eq!( + *storage.global_format_calls.lock().unwrap(), + 0, + "automatic replacement must not call global format" + ); + assert_eq!( + storage.replacement_format_calls.lock().unwrap().as_slice(), + &[(0, 0, vec!["replacement-a".to_string()])], + "automatic replacement must pass the exact pool, set, and target" + ); + } + + #[tokio::test] + async fn automatic_replacement_persists_intent_before_format() { + let storage = Arc::new(MockStorage { + replacement_targets_ready: Mutex::new(true), + ..Default::default() + }); + let mut request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + pool_index: Some(0), + set_index: Some(0), + ..HealOptions::default() + }, + HealPriority::Low, + ); + request.source = HealRequestSource::AutoHeal; + request.heal_endpoints = vec!["replacement-a".to_string()]; + + HealTask::from_request(request, storage.clone()) + .execute() + .await + .expect_err("intent persistence needs a healthy non-target disk"); + + assert!( + storage.replacement_format_calls.lock().unwrap().is_empty(), + "format must not start before the durable replacement intent exists" + ); + } + + #[tokio::test] + async fn recovered_replacement_never_uses_a_fresh_resume_disk() { + let storage = Arc::new(MockStorage { + replacement_targets_ready: Mutex::new(true), + ..Default::default() + }); + let mut request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + pool_index: Some(0), + set_index: Some(0), + ..HealOptions::default() + }, + HealPriority::Low, + ); + request.source = HealRequestSource::AutoHeal; + request.heal_endpoints = vec!["replacement-a".to_string()]; + + let error = HealTask::from_replacement_recovery_request(request, storage.clone(), Some("survivor-a".to_string())) + .execute() + .await + .expect_err("a durable recovery must not fall back to another resume disk"); + + assert!(error.to_string().contains("resume anchor is unavailable")); + assert!( + storage.replacement_format_calls.lock().unwrap().is_empty(), + "an unavailable durable anchor must block formatting before any write" + ); + assert!(!*storage.listed.lock().unwrap(), "an unavailable durable anchor must not list buckets"); + } + + #[tokio::test] + async fn automatic_replacement_rejects_a_new_identity_after_format() { + let temp = TempDir::new().expect("temporary resume disk directory should be created"); + let disk = make_resume_disk(&temp).await; + let first_identity = replacement_identity("replacement-a", "device-a", "filesystem-a"); + let second_identity = replacement_identity("replacement-a", "device-b", "filesystem-b"); + let storage = Arc::new(MockStorage { + replacement_targets_ready: Mutex::new(true), + replacement_target_identity_sequences: Mutex::new(VecDeque::from([ + vec![first_identity.clone()], + vec![first_identity.clone()], + vec![second_identity], + ])), + resume_disk: Mutex::new(Some(disk.clone())), + ..Default::default() + }); + let mut request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + pool_index: Some(0), + set_index: Some(0), + ..HealOptions::default() + }, + HealPriority::Low, + ); + request.source = HealRequestSource::AutoHeal; + request.heal_endpoints = vec!["replacement-a".to_string()]; + let task = HealTask::from_request(request, storage.clone()); + + let error = task + .execute() + .await + .expect_err("a remounted target after format must fail closed"); + + assert!(error.to_string().contains("changed after format")); + assert_eq!(storage.replacement_format_calls.lock().unwrap().len(), 1); + assert!(storage.bucket_heal_calls.lock().unwrap().is_empty()); + assert!(storage.heal_object_calls.lock().unwrap().is_empty()); + + let state = ResumeManager::load_replacement_intent(disk, &task.id) + .await + .expect("durable replacement intent should remain available") + .get_state() + .await; + assert_eq!(state.replacement_phase, crate::heal::resume::ReplacementPhase::Intent); + assert_eq!(state.replacement_target_identities, vec![first_identity]); + } + + #[tokio::test] + async fn automatic_replacement_reuses_an_existing_non_target_resume_anchor() { + let temp = TempDir::new().expect("temporary resume disk directory should be created"); + let anchor = make_resume_disk(&temp).await; + let task_id = crate::heal::resume::ResumeUtils::generate_task_id(); + let identity = ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/replacement/replacement-a".to_string(), + physical_device_ids: vec!["replacement-a".to_string()], + filesystem_identity: "identity-replacement-a".to_string(), + }; + ResumeManager::new_replacement_intent( + anchor.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket-a".to_string()], + vec!["replacement-a".to_string()], + vec![identity], + ) + .await + .expect("existing intent should be stored on the non-target anchor"); + let storage = Arc::new(MockStorage { + replacement_targets_ready: Mutex::new(true), + replacement_resume_disk: Mutex::new(Some(anchor.clone())), + ..Default::default() + }); + let mut request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + pool_index: Some(0), + set_index: Some(0), + ..HealOptions::default() + }, + HealPriority::Low, + ); + request.id = task_id.clone(); + request.source = HealRequestSource::AutoHeal; + request.heal_endpoints = vec!["replacement-a".to_string()]; + + HealTask::from_request(request, storage.clone()) + .execute() + .await + .expect_err("the test has no mounted marker target after format"); + + assert_eq!( + storage.replacement_format_calls.lock().unwrap().len(), + 1, + "an existing non-target anchor must be reused instead of falling back to a fresh anchor" + ); + assert!( + storage.resume_disk.lock().unwrap().is_none(), + "the fresh resume-anchor fallback must remain unused" + ); + let state = ResumeManager::load_replacement_intent(anchor, &task_id) + .await + .expect("the existing non-target anchor should retain the generation") + .get_state() + .await; + assert_eq!(state.replacement_phase, ReplacementPhase::Rebuilding); + } + + #[tokio::test] + async fn automatic_replacement_defers_before_bucket_listing_when_target_is_unready() { + let storage = Arc::new(MockStorage::default()); + let mut request = HealRequest::new( + HealType::ErasureSet { + buckets: Vec::new(), + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + pool_index: Some(0), + set_index: Some(0), + ..Default::default() + }, + HealPriority::Low, + ); + request.source = HealRequestSource::AutoHeal; + request.heal_endpoints = vec!["replacement-a".to_string()]; + + HealTask::from_request(request, storage.clone()) + .execute() + .await + .expect_err("an unsafe replacement must defer before any scan work"); + + assert!(!*storage.listed.lock().unwrap(), "unsafe targets must not list buckets"); + assert!( + storage.replacement_format_calls.lock().unwrap().is_empty(), + "unsafe targets must not format" + ); + } + + #[tokio::test] + async fn cleanup_pending_recovery_skips_target_readiness_and_format() { + let temp = TempDir::new().expect("temporary resume disk directory should be created"); + let anchor = make_resume_disk(&temp).await; + let task_id = crate::heal::resume::ResumeUtils::generate_task_id(); + let identity = replacement_identity("replacement-a", "device-a", "filesystem-a"); + let resume_manager = ResumeManager::new_replacement_intent( + anchor.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket-a".to_string()], + vec!["replacement-a".to_string()], + vec![identity], + ) + .await + .expect("terminal replacement state should persist on the survivor anchor"); + resume_manager + .mark_replacement_completed_and_verified() + .await + .expect("terminal replacement proof should persist before cleanup"); + resume_manager + .mark_replacement_cleanup_pending() + .await + .expect("failed cleanup must retain a cleanup-pending state"); + + let storage = Arc::new(MockStorage { + replacement_resume_disk: Mutex::new(Some(anchor.clone())), + ..Default::default() + }); + let mut request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + pool_index: Some(0), + set_index: Some(0), + ..HealOptions::default() + }, + HealPriority::Low, + ); + request.id = task_id.clone(); + request.source = HealRequestSource::AutoHeal; + request.heal_endpoints = vec!["replacement-a".to_string()]; + + HealTask::from_replacement_recovery_request(request, storage.clone(), Some(anchor.endpoint().to_string())) + .execute() + .await + .expect("cleanup-pending recovery must not require a mounted replacement target"); + + assert!( + !ResumeManager::has_resume_state(&anchor, &task_id).await, + "terminal cleanup must remove the retained resume state" + ); + assert_eq!(*storage.global_format_calls.lock().unwrap(), 0); + assert!( + storage.replacement_format_calls.lock().unwrap().is_empty(), + "terminal cleanup must not format replacement targets" + ); + assert!(storage.bucket_heal_calls.lock().unwrap().is_empty()); + assert!(!*storage.listed.lock().unwrap()); + } + #[derive(Default)] struct MockStorage { listed: Mutex, @@ -2440,6 +3011,10 @@ mod tests { heal_object_outcomes: Mutex>>, deleted_objects: Mutex>, format_no_heal_required: Mutex, + global_format_calls: Mutex, + replacement_format_calls: Mutex)>>, + replacement_targets_ready: Mutex, + replacement_target_identity_sequences: Mutex>>, listed_prefixes: Mutex>, truncate_without_token: Mutex, include_object_dir_candidate: Mutex, @@ -2448,6 +3023,7 @@ mod tests { bucket_heal_calls: Mutex>, block_heal_object: Mutex, resume_disk: Mutex>, + replacement_resume_disk: Mutex>, } #[test] @@ -2519,6 +3095,19 @@ mod tests { } } + fn replacement_identity( + endpoint: &str, + physical_device_id: &str, + filesystem_identity: &str, + ) -> crate::heal::resume::ReplacementTargetIdentity { + crate::heal::resume::ReplacementTargetIdentity { + endpoint: endpoint.to_string(), + canonical_path: format!("/replacement/{endpoint}"), + physical_device_ids: vec![physical_device_id.to_string()], + filesystem_identity: filesystem_identity.to_string(), + } + } + enum MockHealObjectOutcome { OkWithOtherError(&'static str), ErrOther(&'static str), @@ -2721,6 +3310,7 @@ mod tests { } async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option)> { + *self.global_format_calls.lock().unwrap() += 1; let no_heal_required = *self.format_no_heal_required.lock().unwrap(); if no_heal_required { Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::NoHealRequired)))) @@ -2729,6 +3319,39 @@ mod tests { } } + async fn heal_replacement_format( + &self, + _dry_run: bool, + pool_index: usize, + set_index: usize, + targets: &[String], + ) -> Result<(HealResultItem, Option)> { + self.replacement_format_calls + .lock() + .unwrap() + .push((pool_index, set_index, targets.to_vec())); + Ok(( + HealResultItem { + after: Infos { + drives: targets + .iter() + .map(|endpoint| HealDriveInfo { + endpoint: endpoint.clone(), + state: "ok".to_string(), + ..Default::default() + }) + .collect(), + }, + ..Default::default() + }, + None, + )) + } + + async fn replacement_targets_ready(&self, _targets: &[String]) -> Result { + Ok(*self.replacement_targets_ready.lock().unwrap()) + } + async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> Result> { Ok(vec![heal_item("object-a"), heal_item("object-b")]) } @@ -2772,6 +3395,43 @@ mod tests { .clone() .ok_or_else(|| Error::other("not implemented in tests")) } + + async fn get_disk_for_resume_excluding(&self, set_disk_id: &str, _excluded_targets: &[String]) -> Result { + self.get_disk_for_resume(set_disk_id).await + } + + async fn get_replacement_resume_disk( + &self, + _set_disk_id: &str, + _task_id: &str, + _excluded_targets: &[String], + ) -> Result { + if let Some(disk) = self.replacement_resume_disk.lock().unwrap().clone() { + return Ok(crate::heal::storage::ReplacementResumeDisk::Existing(disk)); + } + Ok(crate::heal::storage::ReplacementResumeDisk::Fresh) + } + + async fn replacement_target_identities( + &self, + targets: &[String], + ) -> Result> { + if !*self.replacement_targets_ready.lock().unwrap() { + return Err(Error::other("replacement target is not ready")); + } + if let Some(identities) = self.replacement_target_identity_sequences.lock().unwrap().pop_front() { + return Ok(identities); + } + Ok(targets + .iter() + .map(|endpoint| crate::heal::resume::ReplacementTargetIdentity { + endpoint: endpoint.clone(), + canonical_path: format!("/replacement/{endpoint}"), + physical_device_ids: vec![endpoint.clone()], + filesystem_identity: format!("identity-{endpoint}"), + }) + .collect()) + } } #[tokio::test] diff --git a/crates/heal/src/lib.rs b/crates/heal/src/lib.rs index b6acde862..7c156304d 100644 --- a/crates/heal/src/lib.rs +++ b/crates/heal/src/lib.rs @@ -18,9 +18,12 @@ pub mod heal; pub use error::{Error, Result}; pub use heal::{ HealManager, HealOperationsSnapshot, HealOptions, HealPriority, HealPriorityCounts, HealRequest, HealSourceCounts, HealType, - channel::HealChannelProcessor, progress::HealProgress, + channel::HealChannelProcessor, + progress::HealProgress, + resume::{ReplacementRecoveryRecord, ReplacementRecoveryState, ResumeUtils}, }; use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; +use std::collections::BTreeMap; use std::future::Future; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; @@ -73,6 +76,17 @@ static GLOBAL_HEAL_RUNTIME_INIT: Mutex<()> = Mutex::const_new(()); static GLOBAL_HEAL_ACTIVE_TASKS: AtomicU64 = AtomicU64::new(0); static GLOBAL_HEAL_QUEUE_LENGTH: AtomicU64 = AtomicU64::new(0); +/// Local view of durable replacement recovery state. `definitive` only covers +/// the local survivor-disk records; a distributed caller must additionally +/// establish that every peer returned a compatible snapshot. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReplacementRecoverySnapshot { + pub records: Vec, + pub definitive: bool, + pub reason: Option, +} + #[cfg(test)] #[derive(Default)] struct HealRuntimeInitTestHook { @@ -243,6 +257,81 @@ pub async fn current_heal_progress_snapshot() -> Option { } } +/// Read all local survivor-disk replacement records without conflating an I/O +/// failure or conflicting copies with successful completion. +pub async fn current_replacement_recovery_snapshot() -> ReplacementRecoverySnapshot { + if !heal_runtime_initialized() { + return ReplacementRecoverySnapshot { + records: Vec::new(), + definitive: false, + reason: Some("heal runtime is not initialized".to_string()), + }; + } + + let disks = { + let local_disk_map = heal::local_disk_map_read().await; + local_disk_map.values().flatten().cloned().collect::>() + }; + if disks.is_empty() { + return ReplacementRecoverySnapshot { + records: Vec::new(), + definitive: false, + reason: Some("no local survivor disks are available".to_string()), + }; + } + + let mut records = BTreeMap::::new(); + let mut reason = None; + for disk in disks { + match ResumeUtils::get_replacement_recovery_records(&disk).await { + Ok(disk_records) => { + for record in disk_records { + let task_id = record.task_id.clone(); + if matches!(record.state, ReplacementRecoveryState::Unknown) { + reason.get_or_insert_with(|| "invalid durable replacement record".to_string()); + } + match records.entry(task_id.clone()) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(record); + } + std::collections::btree_map::Entry::Occupied(entry) if entry.get() == &record => {} + std::collections::btree_map::Entry::Occupied(entry) + if matches!(entry.get().state, ReplacementRecoveryState::CleanupPending) + && matches!(record.state, ReplacementRecoveryState::Completed) => {} + std::collections::btree_map::Entry::Occupied(mut entry) + if matches!(entry.get().state, ReplacementRecoveryState::Completed) + && matches!(record.state, ReplacementRecoveryState::CleanupPending) => + { + entry.insert(record); + } + std::collections::btree_map::Entry::Occupied(mut entry) => { + entry.insert(ReplacementRecoveryRecord { + task_id, + state: ReplacementRecoveryState::Unknown, + generation: None, + set_disk_id: None, + target_slots: Vec::new(), + reason: Some("conflicting durable replacement records across survivor disks".to_string()), + verified_at: None, + }); + reason.get_or_insert_with(|| "conflicting durable replacement records".to_string()); + } + } + } + } + Err(error) => { + reason.get_or_insert_with(|| format!("failed to read local replacement recovery records: {error}")); + } + } + } + + ReplacementRecoverySnapshot { + records: records.into_values().collect(), + definitive: reason.is_none(), + reason, + } +} + fn usize_to_u64_saturated(value: usize) -> u64 { u64::try_from(value).unwrap_or(u64::MAX) } diff --git a/crates/utils/src/os/linux.rs b/crates/utils/src/os/linux.rs index fdf9d75af..d3d39b608 100644 --- a/crates/utils/src/os/linux.rs +++ b/crates/utils/src/os/linux.rs @@ -122,6 +122,34 @@ pub fn same_disk(disk1: &str, disk2: &str) -> std::io::Result { Ok(stat1.st_dev == stat2.st_dev) } +/// Return whether `path` is an exact Linux mount point. +/// +/// Device numbers alone are insufficient here: bind mounts may deliberately +/// share a device number with their source. Auto-format callers use this to +/// refuse an unmounted mountpoint directory rather than writing into its +/// parent filesystem. +pub fn is_mount_point(path: &Path) -> std::io::Result { + let metadata = fs::symlink_metadata(path)?; + if !metadata.file_type().is_dir() || metadata.file_type().is_symlink() { + return Ok(false); + } + + let canonical_path = fs::canonicalize(path)?; + let mountinfo = fs::read_to_string("/proc/self/mountinfo")?; + let canonical_path = canonical_path.to_string_lossy().replace(' ', "\\040"); + Ok(mountinfo + .lines() + .any(|line| mountinfo_path(line) == Some(canonical_path.as_ref()))) +} + +fn mountinfo_path(line: &str) -> Option<&str> { + let mut fields = line.split_whitespace(); + for _ in 0..4 { + fields.next()?; + } + fields.next() +} + /// Resolve the leaf physical device identities backing a local filesystem path. /// /// Linux block stacks such as partitions, `dm-*`, or software RAID can all @@ -415,6 +443,13 @@ mod tests { assert_eq!(paths, vec!["/data/my disk".to_string()]); } + #[test] + fn mountinfo_path_extracts_the_mountpoint() { + let line = "42 31 8:1 / /data/replacement rw,relatime - ext4 /dev/sda1 rw"; + + assert_eq!(mountinfo_path(line), Some("/data/replacement")); + } + #[test] fn reject_relative_path_for_cross_device_validation() { let err = ensure_no_sub_mounts("relative/path", &[]).unwrap_err(); diff --git a/crates/utils/src/os/mod.rs b/crates/utils/src/os/mod.rs index 0ab228b37..0d5ed3c22 100644 --- a/crates/utils/src/os/mod.rs +++ b/crates/utils/src/os/mod.rs @@ -24,14 +24,15 @@ mod unix; mod windows; #[cfg(target_os = "linux")] -pub use linux::{check_cross_device_mounts, get_drive_stats, get_info, get_physical_device_ids, same_disk}; +pub use linux::{check_cross_device_mounts, get_drive_stats, get_info, get_physical_device_ids, is_mount_point, same_disk}; #[cfg(all(unix, not(target_os = "linux")))] -pub use unix::{check_cross_device_mounts, get_drive_stats, get_info, get_physical_device_ids, same_disk}; +pub use unix::{check_cross_device_mounts, get_drive_stats, get_info, get_physical_device_ids, is_mount_point, same_disk}; #[cfg(target_os = "windows")] pub use windows::{ - check_cross_device_mounts, get_drive_stats, get_info, get_physical_device_ids, get_volume_serial_number, same_disk, + check_cross_device_mounts, get_drive_stats, get_info, get_physical_device_ids, get_volume_serial_number, is_mount_point, + same_disk, }; #[derive(Debug, Default, PartialEq)] diff --git a/crates/utils/src/os/unix.rs b/crates/utils/src/os/unix.rs index 8a9f92d54..d0785d36c 100644 --- a/crates/utils/src/os/unix.rs +++ b/crates/utils/src/os/unix.rs @@ -93,6 +93,20 @@ pub fn same_disk(disk1: &str, disk2: &str) -> std::io::Result { Ok(stat1.st_dev == stat2.st_dev) } +/// Return whether `path` is mounted separately from its parent directory. +/// +/// This is intentionally conservative on non-Linux Unix platforms. Linux +/// uses mountinfo to recognize bind mounts; elsewhere a shared device number +/// cannot distinguish a bind mount from a normal directory safely. +pub fn is_mount_point(path: &Path) -> std::io::Result { + let metadata = std::fs::symlink_metadata(path)?; + if !metadata.file_type().is_dir() || metadata.file_type().is_symlink() { + return Ok(false); + } + let parent = path.parent().ok_or_else(|| Error::other("mount point has no parent"))?; + Ok(rustix::fs::stat(path)?.st_dev != rustix::fs::stat(parent)?.st_dev) +} + pub fn get_physical_device_ids(disk: &str) -> std::io::Result> { let stat = rustix::fs::stat(disk)?; let major = rustix::fs::major(stat.st_dev); diff --git a/crates/utils/src/os/windows.rs b/crates/utils/src/os/windows.rs index c1e94c3d3..237adbed3 100644 --- a/crates/utils/src/os/windows.rs +++ b/crates/utils/src/os/windows.rs @@ -158,6 +158,12 @@ pub fn same_disk(disk1: &str, disk2: &str) -> std::io::Result { Ok(volume1 == volume2) } +/// Automatic replacement formatting is disabled until Windows has an +/// equivalent mount identity probe. +pub fn is_mount_point(_path: &std::path::Path) -> std::io::Result { + Ok(false) +} + pub fn get_physical_device_ids(disk: &str) -> std::io::Result> { let path_wide = to_wide_path(Path::new(disk)); let volume = get_volume_name(&path_wide)?; diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index 2499f2647..4830e3a6b 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -213,6 +213,16 @@ missed work. Scanner-originated object checks should appear under under `admin/high`. If scanner work grows but admin work remains blocked, treat that as heal queue pressure rather than scanner pacing pressure. +## Replacement Recovery Completion + +`POST /v3/background-heal/status` is an execution-queue view. `state=idle`, zero queue and active counts, an online disk, a readable object, or acceptance of an Admin deep-heal request do not independently prove that a replacement disk contains every erasure shard. + +Treat replacement recovery as verified only after the repair task has completed for the exact replacement instance and an operator has confirmed the target disk contains the expected `xl.meta` and data parts for every relevant object version. A replacement that is not mounted, is unsafe to format, loses its marker, or returns a partial target outcome must be treated as deferred or incomplete rather than complete. + +The v3 route and its peer status protocol preserve their existing fields for mixed-version clusters. A new node must not infer replacement completion from an old or unavailable peer; regard that information as unknown or degraded until every required peer can report the same replacement instance and verified completion. Do not automate destructive replacement actions from an `idle` observation alone. + +Replacement resume and checkpoint files use an independent on-disk schema. A newer reader rejects a future schema rather than continuing with data it cannot interpret, while an older binary cannot safely enforce the new generation fence because it may ignore fields it does not know. Do not roll a cluster back after a replacement generation has started. Complete that recovery with the current-or-newer release; if it cannot complete, keep that version for diagnosis rather than deleting its durable records or continuing with an older binary. + ## Reading Replication Repair `metrics.replication_repair`, `metrics.current_cycle_replication_repair`, and