Compare commits

...

7 Commits

Author SHA1 Message Date
Hauser Zhi c1044cd873 perf(ecstore): reuse the pinned replacement mount identity
Capture device, inode, and mount ID when opening the pinned root. Reuse that identity for live endpoint checks and avoid a duplicate lookup when the endpoint is the expected mount root.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-19 22:09:21 +08:00
Hauser Zhi 1680db7ed8 fix(ci): remove stale form helper and unused hotpath dependencies
Remove the test helper left by the gateway merge. Keep profiling feature forwarding in protos and activate rio instrumentation only for profiling builds.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-19 21:41:38 +08:00
Hauser Zhi 161d0c7ae8 Merge branch 'main' into houseme/fix-replacement-rejoin-3283 2026-11-19 20:56:54 +08:00
Hauser Zhi 19a45dcb7b test(heal): reuse the privileged replacement mount fixture
Exercise configured subdirectories and unmount rejection in the existing independent-mount case. Reuse the mount namespace and keep the existing ignored-test ceiling.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-19 18:26:27 +08:00
Hauser Zhi 674e55fa6d refactor(heal): probe replacement mounts as one pinned batch
Resolve each target once and check mount identity on the same handles before and after taking execution leases. Verify mount boundaries with bounded kernel lookups and resolve each local device once per batch.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-19 18:17:46 +08:00
Hauser Zhi 7cf2c173ea Merge origin/main into replacement mount fix
Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-19 18:03:03 +08:00
Hauser Zhi 6b40ead7c1 fix(heal): admit replacement data directories on configured mounts
Verify the mount ID of the pinned data directory against the expected mount root and retain the existing physical-device checks. Reject the newly exposed directory when the intended mount is missing.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-19 16:50:09 +08:00
12 changed files with 349 additions and 267 deletions
Generated
-1
View File
@@ -11094,7 +11094,6 @@ name = "rustfs-protos"
version = "1.0.5"
dependencies = [
"flatbuffers",
"hotpath",
"hyper",
"hyper-util",
"prost 0.14.6",
+3
View File
@@ -32,6 +32,9 @@ pub const DEFAULT_HEAL_BITROT_CYCLE_SECS: u64 = 30 * 24 * 60 * 60;
/// - Note: Enabling auto-heal can improve system resilience by automatically addressing issues, but may increase resource usage; evaluate based on your operational requirements.
pub const ENV_HEAL_AUTO_HEAL_ENABLE: &str = "RUSTFS_HEAL_AUTO_HEAL_ENABLE";
/// Colon-separated Linux mount roots for automatic replacement of data directories below a mount point.
pub const ENV_HEAL_REPLACEMENT_MOUNT_ROOTS: &str = "RUSTFS_HEAL_REPLACEMENT_MOUNT_ROOTS";
/// Environment variable name that specifies the heal queue size.
///
/// - Purpose: Set the maximum number of heal requests that can be queued.
+1
View File
@@ -469,6 +469,7 @@ pub mod disk {
pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
pub use crate::disk::BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION;
pub use crate::disk::ReplacementMount;
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
pub use crate::disk::disk_store::get_object_disk_read_timeout;
#[cfg(feature = "test-util")]
+99 -83
View File
@@ -8268,8 +8268,9 @@ pub struct LocalDisk {
io_root: PathBuf,
#[cfg(target_os = "linux")]
mount_lease: Arc<std::fs::File>,
// The descriptor pins this identity even if the configured path is remounted.
#[cfg(target_os = "linux")]
mount_lease_mount_id: Option<u64>,
mount_lease_identity: Option<ReplacementMountIdentity>,
/// Public path for callers that need the configured disk layout. Internal
/// disk I/O uses `io_format_path`, which is rooted at `mount_lease`.
pub format_path: PathBuf,
@@ -8469,72 +8470,50 @@ struct ReplacementMountIdentity {
}
#[cfg(target_os = "linux")]
fn replacement_mount_identity(metadata: &std::fs::Metadata, mount_id: u64) -> ReplacementMountIdentity {
use std::os::unix::fs::MetadataExt as _;
ReplacementMountIdentity {
device: metadata.dev(),
inode: metadata.ino(),
mount_id,
}
}
#[cfg(target_os = "linux")]
fn mount_id_for_fd(fd: &std::fs::File) -> Option<u64> {
use rustix::fs::{AtFlags, StatxFlags};
use std::os::fd::AsRawFd as _;
rustix::fs::statx(fd, "", AtFlags::EMPTY_PATH, StatxFlags::MNT_ID)
.ok()
.filter(|statx| StatxFlags::from_bits_retain(statx.stx_mask).contains(StatxFlags::MNT_ID))
.map(|statx| statx.stx_mnt_id)
.or_else(|| mount_id_from_fdinfo(fd.as_raw_fd()))
}
#[cfg(target_os = "linux")]
fn mount_id_for_path(path: &Path) -> Option<u64> {
use rustix::fs::{AtFlags, CWD, StatxFlags};
rustix::fs::statx(CWD, path, AtFlags::empty(), StatxFlags::MNT_ID)
.ok()
.filter(|statx| StatxFlags::from_bits_retain(statx.stx_mask).contains(StatxFlags::MNT_ID))
.map(|statx| statx.stx_mnt_id)
.or_else(|| {
std::fs::canonicalize(path)
.ok()
.and_then(|path| mount_id_from_mountinfo(&path))
fn statx_replacement_mount_identity(
dir: impl std::os::fd::AsFd,
path: &Path,
flags: rustix::fs::AtFlags,
) -> Option<ReplacementMountIdentity> {
use rustix::fs::{FileType, StatxFlags, makedev, statx};
let fields = StatxFlags::TYPE | StatxFlags::INO | StatxFlags::MNT_ID;
let stat = statx(dir, path, flags, fields).ok()?;
(StatxFlags::from_bits_retain(stat.stx_mask).contains(fields) && FileType::from_raw_mode(stat.stx_mode.into()).is_dir())
.then_some(ReplacementMountIdentity {
device: makedev(stat.stx_dev_major, stat.stx_dev_minor),
inode: stat.stx_ino,
mount_id: stat.stx_mnt_id,
})
}
#[cfg(target_os = "linux")]
fn mount_id_from_fdinfo(fd: std::os::fd::RawFd) -> Option<u64> {
std::fs::read_to_string(format!("/proc/self/fdinfo/{fd}"))
fn replacement_mount_identity_for_fd(directory: &std::fs::File) -> Option<ReplacementMountIdentity> {
statx_replacement_mount_identity(directory, Path::new(""), rustix::fs::AtFlags::EMPTY_PATH)
.or_else(|| replacement_mount_identity_from_fdinfo(directory))
}
#[cfg(target_os = "linux")]
fn replacement_mount_identity_for_path(path: &Path) -> Option<ReplacementMountIdentity> {
use rustix::fs::{AtFlags, CWD, Mode, OFlags, open};
statx_replacement_mount_identity(CWD, path, AtFlags::empty()).or_else(|| {
let directory = std::fs::File::from(open(path, OFlags::PATH | OFlags::DIRECTORY | OFlags::CLOEXEC, Mode::empty()).ok()?);
replacement_mount_identity_from_fdinfo(&directory)
})
}
#[cfg(target_os = "linux")]
fn replacement_mount_identity_from_fdinfo(directory: &std::fs::File) -> Option<ReplacementMountIdentity> {
use std::os::{fd::AsRawFd as _, unix::fs::MetadataExt as _};
let fd = directory.as_raw_fd();
let mount_id = std::fs::read_to_string(format!("/proc/self/fdinfo/{fd}"))
.ok()?
.lines()
.find_map(|line| line.strip_prefix("mnt_id:")?.trim().parse().ok())
}
#[cfg(target_os = "linux")]
fn mount_id_from_mountinfo(path: &Path) -> Option<u64> {
let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").ok()?;
mount_id_from_mountinfo_contents(&mountinfo, path)
}
#[cfg(target_os = "linux")]
fn mount_id_from_mountinfo_contents(mountinfo: &str, path: &Path) -> Option<u64> {
let mountpoint = path
.to_string_lossy()
.replace('\\', "\\134")
.replace('\t', "\\011")
.replace('\n', "\\012")
.replace(' ', "\\040");
mountinfo.lines().find_map(|line| {
let mut fields = line.split_whitespace();
let mount_id = fields.next()?.parse().ok()?;
fields.next()?;
fields.next()?;
fields.next()?;
(fields.next()? == mountpoint).then_some(mount_id)
.find_map(|line| line.strip_prefix("mnt_id:")?.trim().parse().ok())?;
let metadata = directory.metadata().ok()?;
Some(ReplacementMountIdentity {
device: metadata.dev(),
inode: metadata.ino(),
mount_id,
})
}
@@ -8592,26 +8571,55 @@ impl LocalDisk {
/// configured endpoint still names the directory held by `mount_lease`.
pub fn has_replacement_mount_lease(&self) -> bool {
#[cfg(target_os = "linux")]
{
let Ok(configured) = std::fs::metadata(self.endpoint.get_file_path()) else {
return false;
};
let Ok(pinned) = self.mount_lease.metadata() else {
return false;
};
let Some(configured_mount_id) = mount_id_for_path(Path::new(&self.endpoint.get_file_path())) else {
return false;
};
let Some(pinned_mount_id) = self.mount_lease_mount_id else {
return false;
};
replacement_mount_identity(&configured, configured_mount_id) == replacement_mount_identity(&pinned, pinned_mount_id)
}
return self
.current_replacement_mount_identity(Path::new(&self.endpoint.get_file_path()))
.is_some();
#[cfg(not(target_os = "linux"))]
false
}
#[cfg(target_os = "linux")]
fn current_replacement_mount_identity(&self, configured_path: &Path) -> Option<ReplacementMountIdentity> {
let pinned = self.mount_lease_identity?;
(replacement_mount_identity_for_path(configured_path)? == pinned).then_some(pinned)
}
pub(super) fn replacement_mount(&self, _mount_roots: &[PathBuf]) -> Option<super::ReplacementMount> {
#[cfg(target_os = "linux")]
{
let configured_path = self.endpoint.get_file_path();
let configured_path = Path::new(&configured_path);
let canonical_path = std::fs::canonicalize(&self.io_root).ok()?;
let expected = _mount_roots
.iter()
.filter(|root| root.is_absolute() && (configured_path.starts_with(root) || canonical_path.starts_with(root)))
.max_by_key(|root| root.components().count())
.unwrap_or(&canonical_path);
// The expected root must be its own mount boundary. If it is
// unmounted, the exposed directory and its parent share a mount ID.
if !canonical_path.starts_with(expected) {
return None;
}
// When the configured endpoint is the expected mount root, this
// live observation also supplies the mount ID for the boundary.
let identity = self.current_replacement_mount_identity(configured_path)?;
if (expected != configured_path && replacement_mount_identity_for_path(expected)?.mount_id != identity.mount_id)
|| replacement_mount_identity_for_path(expected.parent()?)?.mount_id == identity.mount_id
{
return None;
}
Some(super::ReplacementMount {
canonical_path,
physical_device_ids: rustfs_utils::os::get_physical_device_ids(self.io_root.to_str()?).ok()?,
filesystem_identity: format!("{}:{}:{}", identity.mount_id, identity.device, identity.inode),
})
}
#[cfg(not(target_os = "linux"))]
None
}
/// Return the descriptor-rooted path for the mount instance admitted for
/// automatic replacement. Callers must not derive destructive identity
/// from the mutable endpoint pathname.
@@ -8665,10 +8673,10 @@ impl LocalDisk {
PathBuf::from(format!(".rustfs-format-control/{format_device:x}/{}", Uuid::from_bytes(format_file_id)));
#[cfg(target_os = "linux")]
let (mount_lease, io_root, mount_lease_mount_id) = (
let (mount_lease, io_root, mount_lease_identity) = (
publication_root.directory().clone(),
publication_root.io_path().to_path_buf(),
mount_id_for_fd(publication_root.directory()),
replacement_mount_identity_for_fd(publication_root.directory()),
);
#[cfg(not(target_os = "linux"))]
let io_root = Self::open_mount_lease(&root)?;
@@ -8851,7 +8859,7 @@ impl LocalDisk {
#[cfg(target_os = "linux")]
mount_lease: mount_lease.clone(),
#[cfg(target_os = "linux")]
mount_lease_mount_id,
mount_lease_identity,
endpoint: ep.clone(),
format_path,
io_format_path,
@@ -41370,10 +41378,18 @@ mod test {
#[cfg(target_os = "linux")]
#[test]
fn mountinfo_fallback_uses_the_exact_escaped_mountpoint() {
let mountinfo = "101 1 0:30 / / rw - rootfs rootfs rw\n202 101 0:42 / /mnt/replacement\\040disk rw - tmpfs tmpfs rw\n";
assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement disk")), Some(202));
assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement")), None);
fn mount_id_fdinfo_can_identify_a_directory_below_a_mount() {
let temp = tempfile::tempdir().expect("mount fixture directory");
let child_path = temp.path().join("data");
std::fs::create_dir(&child_path).expect("create data subdirectory");
let parent = std::fs::File::open(temp.path()).expect("open parent directory");
let child = std::fs::File::open(&child_path).expect("open data subdirectory");
let parent_identity = replacement_mount_identity_from_fdinfo(&parent).expect("read parent identity");
let child_identity = replacement_mount_identity_from_fdinfo(&child).expect("read child identity");
assert_eq!(child_identity.mount_id, parent_identity.mount_id);
assert_ne!(child_identity.inode, parent_identity.inode);
assert_eq!(replacement_mount_identity_for_fd(&child), Some(child_identity));
assert_eq!(replacement_mount_identity_for_path(&child_path), Some(child_identity));
}
#[cfg(not(windows))]
+17
View File
@@ -239,6 +239,14 @@ pub enum PoolMetaNativeRecoveryResponse {
pub type DiskStore = Arc<Disk>;
/// Filesystem evidence for a replacement data directory on its expected mount.
#[derive(Debug)]
pub struct ReplacementMount {
pub canonical_path: PathBuf,
pub physical_device_ids: Vec<String>,
pub filesystem_identity: String,
}
pub trait DiskFileWriter: AsyncWrite + Send + Sync + Unpin + std::any::Any {}
impl<T: AsyncWrite + Send + Sync + Unpin + 'static> DiskFileWriter for T {}
@@ -1583,6 +1591,15 @@ impl Disk {
}
}
/// Observe the expected mount through the same local disk handle that will
/// execute recovery. Call from a blocking context.
pub fn replacement_mount(&self, mount_roots: &[PathBuf]) -> Option<ReplacementMount> {
match self {
Self::Local(disk) => disk.get_disk().replacement_mount(mount_roots),
Self::Remote(_) => None,
}
}
pub async fn acquire_replacement_execution_lease(&self) -> Result<std::sync::Arc<local::ReplacementExecutionLease>> {
match self {
Self::Local(disk) => disk.get_disk().acquire_replacement_execution_lease().await,
+13 -22
View File
@@ -14,12 +14,9 @@
use super::{
DiskError, DiskStore, HEALING_MARKER_PATH, RUSTFS_META_BUCKET,
replacement_readiness::{auto_replacement_target_identities, replacement_target_disk},
replacement_readiness::{ReplacementTargets, local_replacement_disks, probe_replacement_mounts},
resume::ReplacementTargetIdentity,
storage_api::{
EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreReplacementExecutionLease,
ecstore_local_disk_map_read,
},
storage_api::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreReplacementExecutionLease},
};
use crate::{Error, Result};
use std::{future::Future, sync::Arc};
@@ -35,33 +32,27 @@ pub struct ReplacementExecution {
impl ReplacementExecution {
pub(crate) async fn acquire(targets: &[String]) -> Result<Arc<Self>> {
let identities = auto_replacement_target_identities(targets)
let targets = ReplacementTargets::open(targets)
.await
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is unavailable".to_string()))?;
let identities = targets
.identities()
.await
.ok_or_else(|| Error::ReplacementTargetNotReady("target mount admission failed".to_string()))?;
let local_disks = ecstore_local_disk_map_read()
.await
.values()
.flatten()
.filter(|disk| EcstoreDiskAPI::is_local(disk.as_ref()))
.cloned()
.collect::<Vec<_>>();
let mut disks = Vec::with_capacity(identities.len());
let mut leases = Vec::with_capacity(identities.len());
// Identities are sorted by endpoint, the same order on every opener.
for identity in &identities {
let disk = replacement_target_disk(&identity.endpoint, &local_disks)
.await
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is unavailable".to_string()))?;
// Take leases in endpoint order. Revalidate those same handles against
// the current inventory so a reopened disk cannot supply their proof.
for disk in &targets.disks {
leases.push(disk.acquire_replacement_execution_lease().await?);
disks.push(disk);
}
if auto_replacement_target_identities(targets).await.as_ref() != Some(&identities) {
let local_disks = local_replacement_disks().await;
if probe_replacement_mounts(&targets.disks, &local_disks).await.as_ref() != Some(&identities) {
return Err(Error::ReplacementTargetNotReady(
"target changed while acquiring execution leases".to_string(),
));
}
Ok(Arc::new(Self {
disks,
disks: targets.disks,
identities,
_leases: leases,
}))
+203 -126
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{fs, path::Path};
use std::collections::HashSet;
use super::{
DiskOption, DiskStore, Endpoint, HealDiskExt as _, local_disk_map_read, new_disk, resume::ReplacementTargetIdentity,
@@ -24,12 +24,15 @@ use super::{
/// Directory-backed deployments already declare, through
/// `RUSTFS_UNSAFE_BYPASS_DISK_CHECK`, that their endpoints are plain
/// directories sharing a device with the host root. Those endpoints can never
/// satisfy [`auto_replacement_target_identity`], so without this fallback a
/// pass the independent-mount check, so without this fallback a
/// runtime-wiped or replaced directory disk would stay deferred forever. The
/// admission check itself is never bypassed; the fallback only routes the heal
/// through the ordinary format path that formats every unformatted disk in the
/// set, which is exactly what the pre-admission `heal_disk` path did.
pub(crate) fn directory_backed_replacement_fallback_enabled() -> bool {
if cfg!(target_os = "linux") && std::env::var_os(rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS).is_some() {
return false;
}
rustfs_utils::get_env_bool_with_aliases(
rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK,
&[rustfs_config::ENV_MINIO_CI],
@@ -38,53 +41,66 @@ pub(crate) fn directory_backed_replacement_fallback_enabled() -> bool {
}
pub(crate) async fn auto_replacement_target_ready(disk: &DiskStore, local_disks: &[DiskStore]) -> bool {
auto_replacement_target_identity(disk, local_disks).await.is_some()
probe_replacement_mounts(std::slice::from_ref(disk), local_disks)
.await
.is_some()
}
pub(crate) async fn auto_replacement_target_identity(
disk: &DiskStore,
pub(super) async fn probe_replacement_mounts(
disks: &[DiskStore],
local_disks: &[DiskStore],
) -> Option<ReplacementTargetIdentity> {
let lease_root = disk.replacement_mount_lease_root()?;
let endpoint = disk.endpoint().to_string();
let sibling_lease_roots = local_disks
.iter()
.filter(|sibling| sibling.endpoint().is_local && sibling.endpoint().to_string() != endpoint)
.map(|sibling| sibling.replacement_mount_lease_root())
.collect::<Option<Vec<_>>>()?;
) -> Option<Vec<ReplacementTargetIdentity>> {
if disks.is_empty() {
return Some(Vec::new());
}
let disks = disks.to_vec();
let local_disks = local_disks.to_vec();
tokio::task::spawn_blocking(move || {
let canonical_path = fs::canonicalize(&lease_root).ok()?;
let metadata = fs::metadata(&lease_root).ok()?;
let Ok(target_device_ids) = rustfs_utils::os::get_physical_device_ids(lease_root.to_string_lossy().as_ref()) else {
return None;
};
let Ok(root_device_ids) = rustfs_utils::os::get_physical_device_ids("/") else {
return None;
};
if target_device_ids.is_empty()
|| root_device_ids.is_empty()
|| target_device_ids.iter().any(|target| root_device_ids.contains(target))
|| !rustfs_utils::os::is_mount_point(&canonical_path).unwrap_or(false)
{
return None;
let mount_roots = std::env::var_os(rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS)
.as_ref()
.into_iter()
.flat_map(std::env::split_paths)
.collect::<Vec<_>>();
let mut identities = Vec::with_capacity(disks.len());
for disk in &disks {
let mount = disk.replacement_mount(&mount_roots)?;
if mount.physical_device_ids.is_empty() {
return None;
}
identities.push(ReplacementTargetIdentity {
endpoint: disk.endpoint().to_string(),
canonical_path: mount.canonical_path.into_os_string().into_string().ok()?,
physical_device_ids: mount.physical_device_ids,
filesystem_identity: mount.filesystem_identity,
});
}
if sibling_lease_roots.iter().any(|sibling_lease_root| {
rustfs_utils::os::get_physical_device_ids(sibling_lease_root.to_string_lossy().as_ref())
.map(|ids| ids.iter().any(|id| target_device_ids.contains(id)))
.unwrap_or(true)
}) {
let mut target_devices = HashSet::new();
for target in &identities {
if target
.physical_device_ids
.iter()
.any(|device| !target_devices.insert(device.as_str()))
{
return None;
}
}
let root_devices = rustfs_utils::os::get_physical_device_ids("/").ok()?;
if root_devices.is_empty() || root_devices.iter().any(|device| target_devices.contains(device.as_str())) {
return None;
}
let filesystem_identity = filesystem_identity(&metadata, &canonical_path)?;
Some(ReplacementTargetIdentity {
endpoint,
canonical_path: canonical_path.to_string_lossy().into_owned(),
physical_device_ids: target_device_ids,
filesystem_identity,
})
let targets: HashSet<_> = identities.iter().map(|identity| identity.endpoint.as_str()).collect();
for sibling in &local_disks {
let endpoint = sibling.endpoint();
if endpoint.is_local && !targets.contains(endpoint.to_string().as_str()) {
let root = sibling.replacement_mount_lease_root()?;
let devices = rustfs_utils::os::get_physical_device_ids(root.to_str()?).ok()?;
if devices.iter().any(|device| target_devices.contains(device.as_str())) {
return None;
}
}
}
Some(identities)
})
.await
.ok()
@@ -106,72 +122,76 @@ fn local_replacement_endpoint(target: &str, local_grid_hosts: &[String]) -> Opti
Some(endpoint)
}
pub(super) async fn replacement_target_disk(target: &str, local_disks: &[DiskStore]) -> Option<DiskStore> {
if let Some(disk) = local_disks.iter().find(|disk| disk.endpoint().to_string() == target) {
return Some(disk.clone());
}
let local_grid_hosts = local_disks.iter().map(|disk| disk.endpoint().grid_host()).collect::<Vec<_>>();
let endpoint = local_replacement_endpoint(target, &local_grid_hosts)?;
new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.ok()
}
pub(crate) async fn auto_replacement_target_identities(targets: &[String]) -> Option<Vec<ReplacementTargetIdentity>> {
let local_disk_map = local_disk_map_read().await;
let local_disks = local_disk_map
pub(super) async fn local_replacement_disks() -> Vec<DiskStore> {
local_disk_map_read()
.await
.values()
.flatten()
.filter(|disk| disk.endpoint().is_local)
.cloned()
.collect::<Vec<_>>();
drop(local_disk_map);
.collect()
}
let mut identities = Vec::with_capacity(targets.len());
for target in targets {
let disk = replacement_target_disk(target, &local_disks).await?;
identities.push(auto_replacement_target_identity(&disk, &local_disks).await?);
pub(super) struct ReplacementTargets {
pub(super) disks: Vec<DiskStore>,
local_disks: Vec<DiskStore>,
}
impl ReplacementTargets {
pub(super) async fn open(targets: &[String]) -> Option<Self> {
let mut ordered: Vec<_> = targets.iter().map(String::as_str).collect();
ordered.sort_unstable();
if ordered.windows(2).any(|pair| pair[0] == pair[1]) {
return None;
}
let registry = local_disk_map_read().await;
let requested: Vec<_> = ordered
.into_iter()
.map(|target| {
let disk = registry
.get(target)
.and_then(Option::as_ref)
.filter(|disk| disk.endpoint().is_local)
.cloned();
(target, disk)
})
.collect();
let local_disks = registry
.values()
.flatten()
.filter(|disk| disk.endpoint().is_local)
.cloned()
.collect::<Vec<_>>();
drop(registry);
let mut disks = Vec::with_capacity(requested.len());
let mut hosts = None;
for (target, disk) in requested {
let disk = match disk {
Some(disk) => disk,
None => {
let local_hosts = hosts
.get_or_insert_with(|| local_disks.iter().map(|disk| disk.endpoint().grid_host()).collect::<Vec<_>>());
let endpoint = local_replacement_endpoint(target, local_hosts)?;
new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.ok()?
}
};
disks.push(disk);
}
Some(Self { disks, local_disks })
}
identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
identities.dedup_by(|left, right| left.endpoint == right.endpoint);
(identities.len() == targets.len()).then_some(identities)
}
#[cfg(target_os = "linux")]
fn filesystem_identity(metadata: &fs::Metadata, canonical_path: &Path) -> Option<String> {
use std::os::unix::fs::MetadataExt as _;
let escaped_path = canonical_path.to_string_lossy().replace(' ', "\\040");
let mountinfo = fs::read_to_string("/proc/self/mountinfo").ok()?;
let mount_id = mountinfo.lines().find_map(|line| {
let mut fields = line.split_whitespace();
let mount_id = fields.next()?;
fields.next()?;
fields.next()?;
fields.next()?;
(fields.next()? == escaped_path).then_some(mount_id)
})?;
Some(format!("{mount_id}:{}:{}", metadata.dev(), metadata.ino()))
}
#[cfg(all(unix, not(target_os = "linux")))]
fn filesystem_identity(metadata: &fs::Metadata, _canonical_path: &Path) -> Option<String> {
use std::os::unix::fs::MetadataExt as _;
Some(format!("{}:{}", metadata.dev(), metadata.ino()))
}
#[cfg(not(unix))]
fn filesystem_identity(_metadata: &fs::Metadata, _canonical_path: &Path) -> Option<String> {
None
pub(super) async fn identities(&self) -> Option<Vec<ReplacementTargetIdentity>> {
probe_replacement_mounts(&self.disks, &self.local_disks).await
}
}
#[cfg(test)]
@@ -269,8 +289,8 @@ mod tests {
mod linux_privileged_tests {
use super::*;
use std::error::Error;
use std::path::Path;
use std::process::Command;
use std::{fs, path::Path};
const ENABLE_ENV: &str = "RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS";
const NAMESPACE_ENV: &str = "RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS_IN_NAMESPACE";
@@ -396,39 +416,86 @@ mod tests {
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS=1"]
fn auto_replacement_readiness_accepts_an_independent_mount() -> Result<(), Box<dyn Error + Send + Sync>> {
run_privileged_mount_test(|mut mounts| async move {
let temp = TempDir::new().expect("temporary replacement roots should be created");
let target = temp.path().join("target");
let temp = TempDir::new()?;
let parent = temp.path().join("parent");
let target = parent.join("replacement disk");
let data = target.join("data");
let sibling = temp.path().join("sibling");
std::fs::create_dir(&target).expect("target mountpoint should be created");
std::fs::create_dir(&sibling).expect("sibling mountpoint should be created");
fs::create_dir(&parent)?;
mounts.mount_tmpfs(&parent, "rustfs-readiness-parent")?;
// Leave a directory at the same path below the target mount.
fs::create_dir_all(&data)?;
mounts.mount_tmpfs(&target, "rustfs-readiness-target")?;
fs::create_dir(&data)?;
fs::create_dir(&sibling)?;
mounts.mount_tmpfs(&sibling, "rustfs-readiness-sibling")?;
let target_endpoint = Endpoint::try_from(target.to_string_lossy().as_ref())?;
let sibling_endpoint = Endpoint::try_from(sibling.to_string_lossy().as_ref())?;
let target_disk = new_disk(
&target_endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await?;
let sibling_disk = new_disk(
&sibling_endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await?;
let options = DiskOption {
cleanup: false,
health_check: false,
};
let target_disk = new_disk(&target_endpoint, &options).await?;
let sibling_disk = new_disk(&sibling_endpoint, &options).await?;
let identity = auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await;
assert!(
identity.is_some(),
auto_replacement_target_ready(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await,
"a separately mounted replacement target with no sibling device overlap must be admitted"
);
Ok(())
let disks = [target_disk, sibling_disk];
let mut identities = probe_replacement_mounts(&disks, &disks)
.await
.expect("independently mounted replacement targets should be admitted together");
assert_eq!(identities[0].endpoint, disks[0].endpoint().to_string());
assert_eq!(identities[1].endpoint, disks[1].endpoint().to_string());
identities.sort_unstable_by(|left, right| left.endpoint.cmp(&right.endpoint));
let endpoints = [identities[1].endpoint.clone(), identities[0].endpoint.clone()];
let execution = crate::heal::replacement_execution::ReplacementExecution::acquire(&endpoints).await?;
assert_eq!(execution.identities(), identities);
assert!(
crate::heal::replacement_execution::ReplacementExecution::acquire(&endpoints)
.await
.is_err(),
"a second executor must not take leases on the same targets"
);
drop(execution);
let endpoint = Endpoint::try_from(path_to_string(&data, "data directory")?.as_str())?;
let disk = new_disk(&endpoint, &options).await?;
let key = rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS;
temp_env::async_with_vars([(key, None::<&str>)], async {
assert!(!auto_replacement_target_ready(&disk, &[]).await);
})
.await;
let roots = std::env::join_paths([&parent, &target])?;
temp_env::async_with_vars(
[
(key, Some(roots.to_str().ok_or("mount roots are not UTF-8")?)),
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true")),
],
async {
assert!(!directory_backed_replacement_fallback_enabled());
let identities = probe_replacement_mounts(std::slice::from_ref(&disk), &[])
.await
.expect("data directory on its configured independent mount should be admitted");
assert_eq!(Path::new(&identities[0].canonical_path), data.canonicalize()?);
let reopened = new_disk(&endpoint, &options).await?;
assert_eq!(probe_replacement_mounts(&[reopened], &[]).await, Some(identities));
run_command("umount", &["-l", &path_to_string(&target, "replacement mount")?])?;
assert!(!auto_replacement_target_ready(&disk, &[]).await);
let exposed_disk = new_disk(&endpoint, &options).await?;
assert!(
!auto_replacement_target_ready(&exposed_disk, &[]).await,
"a new handle on the exposed parent filesystem must also be rejected"
);
Ok(())
},
)
.await
})
}
@@ -467,10 +534,20 @@ mod tests {
.await?;
assert!(
auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()])
!auto_replacement_target_ready(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await,
"replacement readiness must reject a target sharing its physical device with a sibling endpoint"
);
assert!(
probe_replacement_mounts(&[target_disk.clone(), sibling_disk], &[])
.await
.is_none(),
"replacement readiness must reject a target sharing its physical device with a sibling endpoint"
"a batch must reject overlapping targets even before they are registered"
);
assert!(auto_replacement_target_ready(&target_disk, &[]).await);
mounts.mount_bind(&source, &target)?;
assert!(
!auto_replacement_target_ready(&target_disk, &[]).await,
"rebinding the same inode must invalidate the held mount identity"
);
Ok(())
})
+11 -18
View File
@@ -38,7 +38,7 @@ use super::storage_api::storage::{
ObjectIO as _, ObjectOperations as _, StorageAdminApi,
};
use super::{
DiskError, DiskStore, ECStore, HealDiskExt as _, StorageError, local_disk_map_read,
DiskError, DiskStore, ECStore, HealDiskExt as _, StorageError,
resume::{ReplacementRecoveryCandidate, ReplacementTargetIdentity, merge_replacement_recovery_candidate},
};
pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
@@ -2457,9 +2457,13 @@ impl HealStorageAPI for ECStoreHealStorage {
}
async fn replacement_target_identities(&self, targets: &[String]) -> Result<Vec<ReplacementTargetIdentity>> {
super::replacement_readiness::auto_replacement_target_identities(targets)
let error = || Error::ReplacementTargetNotReady("replacement target is not a stable mounted disk".to_string());
super::replacement_readiness::ReplacementTargets::open(targets)
.await
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is not a stable mounted disk".to_string()))
.ok_or_else(error)?
.identities()
.await
.ok_or_else(error)
}
async fn replacement_targets_ready_for_retry(
@@ -2468,25 +2472,14 @@ impl HealStorageAPI for ECStoreHealStorage {
targets: &[String],
expected_identities: &[ReplacementTargetIdentity],
) -> Result<bool> {
let identities = match self.replacement_target_identities(targets).await {
Ok(identities) => identities,
Err(_) => return Ok(false),
let Some(targets) = super::replacement_readiness::ReplacementTargets::open(targets).await else {
return Ok(false);
};
if identities != expected_identities {
if targets.identities().await.as_deref() != Some(expected_identities) {
return Ok(false);
}
let local_disks = local_disk_map_read()
.await
.values()
.flatten()
.filter(|disk| disk.endpoint().is_local)
.cloned()
.collect::<Vec<_>>();
for target in targets {
let Some(disk) = local_disks.iter().find(|disk| disk.endpoint().to_string() == *target) else {
return Ok(false);
};
for disk in targets.disks {
let endpoint = disk.endpoint();
if crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx).as_deref()
!= Some(set_disk_id)
-5
View File
@@ -38,8 +38,6 @@ default = []
# event. Requires `RUSTFLAGS="--cfg hyper_unstable_tracing"`.
diagnostic-internode-http2-ping = ["dep:hyper", "rustfs-io-metrics/diagnostic-internode-http2-ping"]
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
"rustfs-common/hotpath",
"rustfs-config/hotpath",
"rustfs-io-metrics/hotpath",
@@ -48,7 +46,6 @@ hotpath = [
]
hotpath-alloc = [
"hotpath",
"hotpath/hotpath-alloc",
"rustfs-common/hotpath-alloc",
"rustfs-config/hotpath-alloc",
"rustfs-io-metrics/hotpath-alloc",
@@ -57,7 +54,6 @@ hotpath-alloc = [
]
hotpath-cpu = [
"hotpath",
"hotpath/hotpath-cpu",
"rustfs-common/hotpath-cpu",
"rustfs-config/hotpath-cpu",
"rustfs-io-metrics/hotpath-cpu",
@@ -66,7 +62,6 @@ hotpath-cpu = [
]
[dependencies]
hotpath.workspace = true
rustfs-common.workspace = true
rustfs-heal-contracts.workspace = true
rustfs-io-metrics.workspace = true
+1 -1
View File
@@ -59,7 +59,7 @@ hotpath-cpu = [
]
[dependencies]
hotpath.workspace = true
hotpath = { workspace = true, optional = true }
arc-swap.workspace = true
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "time"] }
rand = { workspace = true, features = ["serde"] }
@@ -232,7 +232,7 @@ Heal selects the authoritative version from quorum metadata, refuses when more s
### 10.2 Drive replacement
1. Confirm the affected set still meets read quorum for its objects (impossible at `EC:0`, see §5) and record pool index, set index, slot, and the failed disk's UUID from the pool status.
2. Replace the drive at the **same** endpoint and mount path with an empty filesystem owned by RustFS. Do not copy the old drive's directories; a copied `format.json` claims an identity the set already tracks.
2. Replace the drive at the **same** endpoint and mount path with an empty filesystem owned by RustFS. Do not copy the old drive's directories. A copied `format.json` claims an identity the set already tracks. On Linux, automatic replacement checks that the endpoint is an independent mount by default. When the endpoint is a data directory below the mount, set `RUSTFS_HEAL_REPLACEMENT_MOUNT_ROOTS` on that node to the colon-separated absolute mount paths, for example `/srv/disk1:/srv/disk2` for endpoints `/srv/disk1/rustfs` and `/srv/disk2/rustfs`. Use canonical mount paths without trailing slashes. The most specific matching root must be the active mount that contains the endpoint. The root must remain independent of the system and sibling drives. Setting this variable disables the unsafe directory-backed heal fallback on that node.
3. Bring the node back. The empty drive is not a format-quorum member, so `retain_format_quorum_members` keeps its slot offline; the endpoint monitor (`monitor_and_connect_endpoints_task` in `crates/ecstore/src/core/sets.rs`) retries the connection every 15 seconds. The erasure-set heal task (`crates/heal/src/heal/task/heal_erasure_set.rs`) writes the format into the original slot, through `heal_replacement_format` when an automatic replacement intent exists (schema 7 records under `.rustfs.sys/buckets/ahm-replacement/`, created when startup or the disk scanner observes a changed mount identity) or through `heal_format` for an admin erasure-set heal (`POST /rustfs/admin/v3/heal/` with `pool` and `set`), and then rebuilds the shards. Legacy schema 5/6 intents follow [replacement-generation-recovery.md](replacement-generation-recovery.md). If the drive held the node's pool metadata, follow [pool-metadata-recovery.md](pool-metadata-recovery.md#disk-replacement-and-metadata-erasure) first.
4. Watch `GET /rustfs/admin/v4/heal/replacement-recovery`, background heal status, disk health, and bitrot counters. Completion criteria are in [scanner-runtime-controls.md](scanner-runtime-controls.md#replacement-recovery-completion): an `idle` queue or a readable object is not proof.
5. The replacement is done when recovery reports `completed` for that instance and sampled objects have `xl.meta` and parts on the new drive.
@@ -185,16 +185,6 @@ impl PostObjectFormPolicy {
Ok(Self(values))
}
#[cfg(test)]
pub(crate) fn with_metadata(mut self, metadata: &[(String, String)]) -> S3Result<Self> {
for (name, value) in metadata {
let name = HeaderName::from_bytes(format!("x-amz-meta-{name}").as_bytes()).map_err(|_| invalid_post_form_field())?;
let value = HeaderValue::from_str(value).map_err(|_| invalid_post_form_field())?;
self.0.push((name, Some(value)));
}
Ok(self)
}
pub(crate) fn install<R: AuthorizationRequest + ?Sized>(self, req: &mut R) {
let mut headers = req.headers().clone();
for (name, value) in self.0 {