mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-19 14:32:28 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c1044cd873 | |||
| 1680db7ed8 | |||
| 161d0c7ae8 | |||
| 19a45dcb7b | |||
| 674e55fa6d | |||
| 7cf2c173ea | |||
| 6b40ead7c1 |
Generated
-1
@@ -11094,7 +11094,6 @@ name = "rustfs-protos"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"prost 0.14.6",
|
||||
|
||||
@@ -32,6 +32,9 @@ pub const DEFAULT_HEAL_BITROT_CYCLE_SECS: u64 = 30 * 24 * 60 * 60;
|
||||
/// - Note: Enabling auto-heal can improve system resilience by automatically addressing issues, but may increase resource usage; evaluate based on your operational requirements.
|
||||
pub const ENV_HEAL_AUTO_HEAL_ENABLE: &str = "RUSTFS_HEAL_AUTO_HEAL_ENABLE";
|
||||
|
||||
/// Colon-separated Linux mount roots for automatic replacement of data directories below a mount point.
|
||||
pub const ENV_HEAL_REPLACEMENT_MOUNT_ROOTS: &str = "RUSTFS_HEAL_REPLACEMENT_MOUNT_ROOTS";
|
||||
|
||||
/// Environment variable name that specifies the heal queue size.
|
||||
///
|
||||
/// - Purpose: Set the maximum number of heal requests that can be queued.
|
||||
|
||||
@@ -469,6 +469,7 @@ pub mod disk {
|
||||
pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
|
||||
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
|
||||
pub use crate::disk::BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION;
|
||||
pub use crate::disk::ReplacementMount;
|
||||
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
#[cfg(feature = "test-util")]
|
||||
|
||||
@@ -8268,8 +8268,9 @@ pub struct LocalDisk {
|
||||
io_root: PathBuf,
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease: Arc<std::fs::File>,
|
||||
// The descriptor pins this identity even if the configured path is remounted.
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease_mount_id: Option<u64>,
|
||||
mount_lease_identity: Option<ReplacementMountIdentity>,
|
||||
/// Public path for callers that need the configured disk layout. Internal
|
||||
/// disk I/O uses `io_format_path`, which is rooted at `mount_lease`.
|
||||
pub format_path: PathBuf,
|
||||
@@ -8469,72 +8470,50 @@ struct ReplacementMountIdentity {
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn replacement_mount_identity(metadata: &std::fs::Metadata, mount_id: u64) -> ReplacementMountIdentity {
|
||||
use std::os::unix::fs::MetadataExt as _;
|
||||
|
||||
ReplacementMountIdentity {
|
||||
device: metadata.dev(),
|
||||
inode: metadata.ino(),
|
||||
mount_id,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_for_fd(fd: &std::fs::File) -> Option<u64> {
|
||||
use rustix::fs::{AtFlags, StatxFlags};
|
||||
use std::os::fd::AsRawFd as _;
|
||||
|
||||
rustix::fs::statx(fd, "", AtFlags::EMPTY_PATH, StatxFlags::MNT_ID)
|
||||
.ok()
|
||||
.filter(|statx| StatxFlags::from_bits_retain(statx.stx_mask).contains(StatxFlags::MNT_ID))
|
||||
.map(|statx| statx.stx_mnt_id)
|
||||
.or_else(|| mount_id_from_fdinfo(fd.as_raw_fd()))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_for_path(path: &Path) -> Option<u64> {
|
||||
use rustix::fs::{AtFlags, CWD, StatxFlags};
|
||||
|
||||
rustix::fs::statx(CWD, path, AtFlags::empty(), StatxFlags::MNT_ID)
|
||||
.ok()
|
||||
.filter(|statx| StatxFlags::from_bits_retain(statx.stx_mask).contains(StatxFlags::MNT_ID))
|
||||
.map(|statx| statx.stx_mnt_id)
|
||||
.or_else(|| {
|
||||
std::fs::canonicalize(path)
|
||||
.ok()
|
||||
.and_then(|path| mount_id_from_mountinfo(&path))
|
||||
fn statx_replacement_mount_identity(
|
||||
dir: impl std::os::fd::AsFd,
|
||||
path: &Path,
|
||||
flags: rustix::fs::AtFlags,
|
||||
) -> Option<ReplacementMountIdentity> {
|
||||
use rustix::fs::{FileType, StatxFlags, makedev, statx};
|
||||
let fields = StatxFlags::TYPE | StatxFlags::INO | StatxFlags::MNT_ID;
|
||||
let stat = statx(dir, path, flags, fields).ok()?;
|
||||
(StatxFlags::from_bits_retain(stat.stx_mask).contains(fields) && FileType::from_raw_mode(stat.stx_mode.into()).is_dir())
|
||||
.then_some(ReplacementMountIdentity {
|
||||
device: makedev(stat.stx_dev_major, stat.stx_dev_minor),
|
||||
inode: stat.stx_ino,
|
||||
mount_id: stat.stx_mnt_id,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_from_fdinfo(fd: std::os::fd::RawFd) -> Option<u64> {
|
||||
std::fs::read_to_string(format!("/proc/self/fdinfo/{fd}"))
|
||||
fn replacement_mount_identity_for_fd(directory: &std::fs::File) -> Option<ReplacementMountIdentity> {
|
||||
statx_replacement_mount_identity(directory, Path::new(""), rustix::fs::AtFlags::EMPTY_PATH)
|
||||
.or_else(|| replacement_mount_identity_from_fdinfo(directory))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn replacement_mount_identity_for_path(path: &Path) -> Option<ReplacementMountIdentity> {
|
||||
use rustix::fs::{AtFlags, CWD, Mode, OFlags, open};
|
||||
statx_replacement_mount_identity(CWD, path, AtFlags::empty()).or_else(|| {
|
||||
let directory = std::fs::File::from(open(path, OFlags::PATH | OFlags::DIRECTORY | OFlags::CLOEXEC, Mode::empty()).ok()?);
|
||||
replacement_mount_identity_from_fdinfo(&directory)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn replacement_mount_identity_from_fdinfo(directory: &std::fs::File) -> Option<ReplacementMountIdentity> {
|
||||
use std::os::{fd::AsRawFd as _, unix::fs::MetadataExt as _};
|
||||
let fd = directory.as_raw_fd();
|
||||
let mount_id = std::fs::read_to_string(format!("/proc/self/fdinfo/{fd}"))
|
||||
.ok()?
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("mnt_id:")?.trim().parse().ok())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_from_mountinfo(path: &Path) -> Option<u64> {
|
||||
let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").ok()?;
|
||||
mount_id_from_mountinfo_contents(&mountinfo, path)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_from_mountinfo_contents(mountinfo: &str, path: &Path) -> Option<u64> {
|
||||
let mountpoint = path
|
||||
.to_string_lossy()
|
||||
.replace('\\', "\\134")
|
||||
.replace('\t', "\\011")
|
||||
.replace('\n', "\\012")
|
||||
.replace(' ', "\\040");
|
||||
mountinfo.lines().find_map(|line| {
|
||||
let mut fields = line.split_whitespace();
|
||||
let mount_id = fields.next()?.parse().ok()?;
|
||||
fields.next()?;
|
||||
fields.next()?;
|
||||
fields.next()?;
|
||||
(fields.next()? == mountpoint).then_some(mount_id)
|
||||
.find_map(|line| line.strip_prefix("mnt_id:")?.trim().parse().ok())?;
|
||||
let metadata = directory.metadata().ok()?;
|
||||
Some(ReplacementMountIdentity {
|
||||
device: metadata.dev(),
|
||||
inode: metadata.ino(),
|
||||
mount_id,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8592,26 +8571,55 @@ impl LocalDisk {
|
||||
/// configured endpoint still names the directory held by `mount_lease`.
|
||||
pub fn has_replacement_mount_lease(&self) -> bool {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let Ok(configured) = std::fs::metadata(self.endpoint.get_file_path()) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(pinned) = self.mount_lease.metadata() else {
|
||||
return false;
|
||||
};
|
||||
let Some(configured_mount_id) = mount_id_for_path(Path::new(&self.endpoint.get_file_path())) else {
|
||||
return false;
|
||||
};
|
||||
let Some(pinned_mount_id) = self.mount_lease_mount_id else {
|
||||
return false;
|
||||
};
|
||||
replacement_mount_identity(&configured, configured_mount_id) == replacement_mount_identity(&pinned, pinned_mount_id)
|
||||
}
|
||||
return self
|
||||
.current_replacement_mount_identity(Path::new(&self.endpoint.get_file_path()))
|
||||
.is_some();
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn current_replacement_mount_identity(&self, configured_path: &Path) -> Option<ReplacementMountIdentity> {
|
||||
let pinned = self.mount_lease_identity?;
|
||||
(replacement_mount_identity_for_path(configured_path)? == pinned).then_some(pinned)
|
||||
}
|
||||
|
||||
pub(super) fn replacement_mount(&self, _mount_roots: &[PathBuf]) -> Option<super::ReplacementMount> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let configured_path = self.endpoint.get_file_path();
|
||||
let configured_path = Path::new(&configured_path);
|
||||
let canonical_path = std::fs::canonicalize(&self.io_root).ok()?;
|
||||
let expected = _mount_roots
|
||||
.iter()
|
||||
.filter(|root| root.is_absolute() && (configured_path.starts_with(root) || canonical_path.starts_with(root)))
|
||||
.max_by_key(|root| root.components().count())
|
||||
.unwrap_or(&canonical_path);
|
||||
// The expected root must be its own mount boundary. If it is
|
||||
// unmounted, the exposed directory and its parent share a mount ID.
|
||||
if !canonical_path.starts_with(expected) {
|
||||
return None;
|
||||
}
|
||||
// When the configured endpoint is the expected mount root, this
|
||||
// live observation also supplies the mount ID for the boundary.
|
||||
let identity = self.current_replacement_mount_identity(configured_path)?;
|
||||
if (expected != configured_path && replacement_mount_identity_for_path(expected)?.mount_id != identity.mount_id)
|
||||
|| replacement_mount_identity_for_path(expected.parent()?)?.mount_id == identity.mount_id
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(super::ReplacementMount {
|
||||
canonical_path,
|
||||
physical_device_ids: rustfs_utils::os::get_physical_device_ids(self.io_root.to_str()?).ok()?,
|
||||
filesystem_identity: format!("{}:{}:{}", identity.mount_id, identity.device, identity.inode),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
None
|
||||
}
|
||||
|
||||
/// Return the descriptor-rooted path for the mount instance admitted for
|
||||
/// automatic replacement. Callers must not derive destructive identity
|
||||
/// from the mutable endpoint pathname.
|
||||
@@ -8665,10 +8673,10 @@ impl LocalDisk {
|
||||
PathBuf::from(format!(".rustfs-format-control/{format_device:x}/{}", Uuid::from_bytes(format_file_id)));
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let (mount_lease, io_root, mount_lease_mount_id) = (
|
||||
let (mount_lease, io_root, mount_lease_identity) = (
|
||||
publication_root.directory().clone(),
|
||||
publication_root.io_path().to_path_buf(),
|
||||
mount_id_for_fd(publication_root.directory()),
|
||||
replacement_mount_identity_for_fd(publication_root.directory()),
|
||||
);
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let io_root = Self::open_mount_lease(&root)?;
|
||||
@@ -8851,7 +8859,7 @@ impl LocalDisk {
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease: mount_lease.clone(),
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease_mount_id,
|
||||
mount_lease_identity,
|
||||
endpoint: ep.clone(),
|
||||
format_path,
|
||||
io_format_path,
|
||||
@@ -41370,10 +41378,18 @@ mod test {
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn mountinfo_fallback_uses_the_exact_escaped_mountpoint() {
|
||||
let mountinfo = "101 1 0:30 / / rw - rootfs rootfs rw\n202 101 0:42 / /mnt/replacement\\040disk rw - tmpfs tmpfs rw\n";
|
||||
assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement disk")), Some(202));
|
||||
assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement")), None);
|
||||
fn mount_id_fdinfo_can_identify_a_directory_below_a_mount() {
|
||||
let temp = tempfile::tempdir().expect("mount fixture directory");
|
||||
let child_path = temp.path().join("data");
|
||||
std::fs::create_dir(&child_path).expect("create data subdirectory");
|
||||
let parent = std::fs::File::open(temp.path()).expect("open parent directory");
|
||||
let child = std::fs::File::open(&child_path).expect("open data subdirectory");
|
||||
let parent_identity = replacement_mount_identity_from_fdinfo(&parent).expect("read parent identity");
|
||||
let child_identity = replacement_mount_identity_from_fdinfo(&child).expect("read child identity");
|
||||
assert_eq!(child_identity.mount_id, parent_identity.mount_id);
|
||||
assert_ne!(child_identity.inode, parent_identity.inode);
|
||||
assert_eq!(replacement_mount_identity_for_fd(&child), Some(child_identity));
|
||||
assert_eq!(replacement_mount_identity_for_path(&child_path), Some(child_identity));
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
|
||||
@@ -239,6 +239,14 @@ pub enum PoolMetaNativeRecoveryResponse {
|
||||
|
||||
pub type DiskStore = Arc<Disk>;
|
||||
|
||||
/// Filesystem evidence for a replacement data directory on its expected mount.
|
||||
#[derive(Debug)]
|
||||
pub struct ReplacementMount {
|
||||
pub canonical_path: PathBuf,
|
||||
pub physical_device_ids: Vec<String>,
|
||||
pub filesystem_identity: String,
|
||||
}
|
||||
|
||||
pub trait DiskFileWriter: AsyncWrite + Send + Sync + Unpin + std::any::Any {}
|
||||
impl<T: AsyncWrite + Send + Sync + Unpin + 'static> DiskFileWriter for T {}
|
||||
|
||||
@@ -1583,6 +1591,15 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
/// Observe the expected mount through the same local disk handle that will
|
||||
/// execute recovery. Call from a blocking context.
|
||||
pub fn replacement_mount(&self, mount_roots: &[PathBuf]) -> Option<ReplacementMount> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.get_disk().replacement_mount(mount_roots),
|
||||
Self::Remote(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn acquire_replacement_execution_lease(&self) -> Result<std::sync::Arc<local::ReplacementExecutionLease>> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.get_disk().acquire_replacement_execution_lease().await,
|
||||
|
||||
@@ -14,12 +14,9 @@
|
||||
|
||||
use super::{
|
||||
DiskError, DiskStore, HEALING_MARKER_PATH, RUSTFS_META_BUCKET,
|
||||
replacement_readiness::{auto_replacement_target_identities, replacement_target_disk},
|
||||
replacement_readiness::{ReplacementTargets, local_replacement_disks, probe_replacement_mounts},
|
||||
resume::ReplacementTargetIdentity,
|
||||
storage_api::{
|
||||
EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreReplacementExecutionLease,
|
||||
ecstore_local_disk_map_read,
|
||||
},
|
||||
storage_api::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreReplacementExecutionLease},
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use std::{future::Future, sync::Arc};
|
||||
@@ -35,33 +32,27 @@ pub struct ReplacementExecution {
|
||||
|
||||
impl ReplacementExecution {
|
||||
pub(crate) async fn acquire(targets: &[String]) -> Result<Arc<Self>> {
|
||||
let identities = auto_replacement_target_identities(targets)
|
||||
let targets = ReplacementTargets::open(targets)
|
||||
.await
|
||||
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is unavailable".to_string()))?;
|
||||
let identities = targets
|
||||
.identities()
|
||||
.await
|
||||
.ok_or_else(|| Error::ReplacementTargetNotReady("target mount admission failed".to_string()))?;
|
||||
let local_disks = ecstore_local_disk_map_read()
|
||||
.await
|
||||
.values()
|
||||
.flatten()
|
||||
.filter(|disk| EcstoreDiskAPI::is_local(disk.as_ref()))
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let mut disks = Vec::with_capacity(identities.len());
|
||||
let mut leases = Vec::with_capacity(identities.len());
|
||||
// Identities are sorted by endpoint, the same order on every opener.
|
||||
for identity in &identities {
|
||||
let disk = replacement_target_disk(&identity.endpoint, &local_disks)
|
||||
.await
|
||||
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is unavailable".to_string()))?;
|
||||
// Take leases in endpoint order. Revalidate those same handles against
|
||||
// the current inventory so a reopened disk cannot supply their proof.
|
||||
for disk in &targets.disks {
|
||||
leases.push(disk.acquire_replacement_execution_lease().await?);
|
||||
disks.push(disk);
|
||||
}
|
||||
if auto_replacement_target_identities(targets).await.as_ref() != Some(&identities) {
|
||||
let local_disks = local_replacement_disks().await;
|
||||
if probe_replacement_mounts(&targets.disks, &local_disks).await.as_ref() != Some(&identities) {
|
||||
return Err(Error::ReplacementTargetNotReady(
|
||||
"target changed while acquiring execution leases".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(Arc::new(Self {
|
||||
disks,
|
||||
disks: targets.disks,
|
||||
identities,
|
||||
_leases: leases,
|
||||
}))
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{fs, path::Path};
|
||||
use std::collections::HashSet;
|
||||
|
||||
use super::{
|
||||
DiskOption, DiskStore, Endpoint, HealDiskExt as _, local_disk_map_read, new_disk, resume::ReplacementTargetIdentity,
|
||||
@@ -24,12 +24,15 @@ use super::{
|
||||
/// Directory-backed deployments already declare, through
|
||||
/// `RUSTFS_UNSAFE_BYPASS_DISK_CHECK`, that their endpoints are plain
|
||||
/// directories sharing a device with the host root. Those endpoints can never
|
||||
/// satisfy [`auto_replacement_target_identity`], so without this fallback a
|
||||
/// pass the independent-mount check, so without this fallback a
|
||||
/// runtime-wiped or replaced directory disk would stay deferred forever. The
|
||||
/// admission check itself is never bypassed; the fallback only routes the heal
|
||||
/// through the ordinary format path that formats every unformatted disk in the
|
||||
/// set, which is exactly what the pre-admission `heal_disk` path did.
|
||||
pub(crate) fn directory_backed_replacement_fallback_enabled() -> bool {
|
||||
if cfg!(target_os = "linux") && std::env::var_os(rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS).is_some() {
|
||||
return false;
|
||||
}
|
||||
rustfs_utils::get_env_bool_with_aliases(
|
||||
rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK,
|
||||
&[rustfs_config::ENV_MINIO_CI],
|
||||
@@ -38,53 +41,66 @@ pub(crate) fn directory_backed_replacement_fallback_enabled() -> bool {
|
||||
}
|
||||
|
||||
pub(crate) async fn auto_replacement_target_ready(disk: &DiskStore, local_disks: &[DiskStore]) -> bool {
|
||||
auto_replacement_target_identity(disk, local_disks).await.is_some()
|
||||
probe_replacement_mounts(std::slice::from_ref(disk), local_disks)
|
||||
.await
|
||||
.is_some()
|
||||
}
|
||||
|
||||
pub(crate) async fn auto_replacement_target_identity(
|
||||
disk: &DiskStore,
|
||||
pub(super) async fn probe_replacement_mounts(
|
||||
disks: &[DiskStore],
|
||||
local_disks: &[DiskStore],
|
||||
) -> Option<ReplacementTargetIdentity> {
|
||||
let lease_root = disk.replacement_mount_lease_root()?;
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
let sibling_lease_roots = local_disks
|
||||
.iter()
|
||||
.filter(|sibling| sibling.endpoint().is_local && sibling.endpoint().to_string() != endpoint)
|
||||
.map(|sibling| sibling.replacement_mount_lease_root())
|
||||
.collect::<Option<Vec<_>>>()?;
|
||||
) -> Option<Vec<ReplacementTargetIdentity>> {
|
||||
if disks.is_empty() {
|
||||
return Some(Vec::new());
|
||||
}
|
||||
let disks = disks.to_vec();
|
||||
let local_disks = local_disks.to_vec();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let canonical_path = fs::canonicalize(&lease_root).ok()?;
|
||||
let metadata = fs::metadata(&lease_root).ok()?;
|
||||
let Ok(target_device_ids) = rustfs_utils::os::get_physical_device_ids(lease_root.to_string_lossy().as_ref()) else {
|
||||
return None;
|
||||
};
|
||||
let Ok(root_device_ids) = rustfs_utils::os::get_physical_device_ids("/") else {
|
||||
return None;
|
||||
};
|
||||
if target_device_ids.is_empty()
|
||||
|| root_device_ids.is_empty()
|
||||
|| target_device_ids.iter().any(|target| root_device_ids.contains(target))
|
||||
|| !rustfs_utils::os::is_mount_point(&canonical_path).unwrap_or(false)
|
||||
{
|
||||
return None;
|
||||
let mount_roots = std::env::var_os(rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS)
|
||||
.as_ref()
|
||||
.into_iter()
|
||||
.flat_map(std::env::split_paths)
|
||||
.collect::<Vec<_>>();
|
||||
let mut identities = Vec::with_capacity(disks.len());
|
||||
for disk in &disks {
|
||||
let mount = disk.replacement_mount(&mount_roots)?;
|
||||
if mount.physical_device_ids.is_empty() {
|
||||
return None;
|
||||
}
|
||||
identities.push(ReplacementTargetIdentity {
|
||||
endpoint: disk.endpoint().to_string(),
|
||||
canonical_path: mount.canonical_path.into_os_string().into_string().ok()?,
|
||||
physical_device_ids: mount.physical_device_ids,
|
||||
filesystem_identity: mount.filesystem_identity,
|
||||
});
|
||||
}
|
||||
|
||||
if sibling_lease_roots.iter().any(|sibling_lease_root| {
|
||||
rustfs_utils::os::get_physical_device_ids(sibling_lease_root.to_string_lossy().as_ref())
|
||||
.map(|ids| ids.iter().any(|id| target_device_ids.contains(id)))
|
||||
.unwrap_or(true)
|
||||
}) {
|
||||
let mut target_devices = HashSet::new();
|
||||
for target in &identities {
|
||||
if target
|
||||
.physical_device_ids
|
||||
.iter()
|
||||
.any(|device| !target_devices.insert(device.as_str()))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
}
|
||||
let root_devices = rustfs_utils::os::get_physical_device_ids("/").ok()?;
|
||||
if root_devices.is_empty() || root_devices.iter().any(|device| target_devices.contains(device.as_str())) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let filesystem_identity = filesystem_identity(&metadata, &canonical_path)?;
|
||||
|
||||
Some(ReplacementTargetIdentity {
|
||||
endpoint,
|
||||
canonical_path: canonical_path.to_string_lossy().into_owned(),
|
||||
physical_device_ids: target_device_ids,
|
||||
filesystem_identity,
|
||||
})
|
||||
let targets: HashSet<_> = identities.iter().map(|identity| identity.endpoint.as_str()).collect();
|
||||
for sibling in &local_disks {
|
||||
let endpoint = sibling.endpoint();
|
||||
if endpoint.is_local && !targets.contains(endpoint.to_string().as_str()) {
|
||||
let root = sibling.replacement_mount_lease_root()?;
|
||||
let devices = rustfs_utils::os::get_physical_device_ids(root.to_str()?).ok()?;
|
||||
if devices.iter().any(|device| target_devices.contains(device.as_str())) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(identities)
|
||||
})
|
||||
.await
|
||||
.ok()
|
||||
@@ -106,72 +122,76 @@ fn local_replacement_endpoint(target: &str, local_grid_hosts: &[String]) -> Opti
|
||||
Some(endpoint)
|
||||
}
|
||||
|
||||
pub(super) async fn replacement_target_disk(target: &str, local_disks: &[DiskStore]) -> Option<DiskStore> {
|
||||
if let Some(disk) = local_disks.iter().find(|disk| disk.endpoint().to_string() == target) {
|
||||
return Some(disk.clone());
|
||||
}
|
||||
|
||||
let local_grid_hosts = local_disks.iter().map(|disk| disk.endpoint().grid_host()).collect::<Vec<_>>();
|
||||
let endpoint = local_replacement_endpoint(target, &local_grid_hosts)?;
|
||||
|
||||
new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
pub(crate) async fn auto_replacement_target_identities(targets: &[String]) -> Option<Vec<ReplacementTargetIdentity>> {
|
||||
let local_disk_map = local_disk_map_read().await;
|
||||
let local_disks = local_disk_map
|
||||
pub(super) async fn local_replacement_disks() -> Vec<DiskStore> {
|
||||
local_disk_map_read()
|
||||
.await
|
||||
.values()
|
||||
.flatten()
|
||||
.filter(|disk| disk.endpoint().is_local)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
drop(local_disk_map);
|
||||
.collect()
|
||||
}
|
||||
|
||||
let mut identities = Vec::with_capacity(targets.len());
|
||||
for target in targets {
|
||||
let disk = replacement_target_disk(target, &local_disks).await?;
|
||||
identities.push(auto_replacement_target_identity(&disk, &local_disks).await?);
|
||||
pub(super) struct ReplacementTargets {
|
||||
pub(super) disks: Vec<DiskStore>,
|
||||
local_disks: Vec<DiskStore>,
|
||||
}
|
||||
|
||||
impl ReplacementTargets {
|
||||
pub(super) async fn open(targets: &[String]) -> Option<Self> {
|
||||
let mut ordered: Vec<_> = targets.iter().map(String::as_str).collect();
|
||||
ordered.sort_unstable();
|
||||
if ordered.windows(2).any(|pair| pair[0] == pair[1]) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let registry = local_disk_map_read().await;
|
||||
let requested: Vec<_> = ordered
|
||||
.into_iter()
|
||||
.map(|target| {
|
||||
let disk = registry
|
||||
.get(target)
|
||||
.and_then(Option::as_ref)
|
||||
.filter(|disk| disk.endpoint().is_local)
|
||||
.cloned();
|
||||
(target, disk)
|
||||
})
|
||||
.collect();
|
||||
let local_disks = registry
|
||||
.values()
|
||||
.flatten()
|
||||
.filter(|disk| disk.endpoint().is_local)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
drop(registry);
|
||||
let mut disks = Vec::with_capacity(requested.len());
|
||||
let mut hosts = None;
|
||||
for (target, disk) in requested {
|
||||
let disk = match disk {
|
||||
Some(disk) => disk,
|
||||
None => {
|
||||
let local_hosts = hosts
|
||||
.get_or_insert_with(|| local_disks.iter().map(|disk| disk.endpoint().grid_host()).collect::<Vec<_>>());
|
||||
let endpoint = local_replacement_endpoint(target, local_hosts)?;
|
||||
new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.ok()?
|
||||
}
|
||||
};
|
||||
disks.push(disk);
|
||||
}
|
||||
Some(Self { disks, local_disks })
|
||||
}
|
||||
identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
identities.dedup_by(|left, right| left.endpoint == right.endpoint);
|
||||
(identities.len() == targets.len()).then_some(identities)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn filesystem_identity(metadata: &fs::Metadata, canonical_path: &Path) -> Option<String> {
|
||||
use std::os::unix::fs::MetadataExt as _;
|
||||
|
||||
let escaped_path = canonical_path.to_string_lossy().replace(' ', "\\040");
|
||||
let mountinfo = fs::read_to_string("/proc/self/mountinfo").ok()?;
|
||||
let mount_id = mountinfo.lines().find_map(|line| {
|
||||
let mut fields = line.split_whitespace();
|
||||
let mount_id = fields.next()?;
|
||||
fields.next()?;
|
||||
fields.next()?;
|
||||
fields.next()?;
|
||||
(fields.next()? == escaped_path).then_some(mount_id)
|
||||
})?;
|
||||
Some(format!("{mount_id}:{}:{}", metadata.dev(), metadata.ino()))
|
||||
}
|
||||
|
||||
#[cfg(all(unix, not(target_os = "linux")))]
|
||||
fn filesystem_identity(metadata: &fs::Metadata, _canonical_path: &Path) -> Option<String> {
|
||||
use std::os::unix::fs::MetadataExt as _;
|
||||
|
||||
Some(format!("{}:{}", metadata.dev(), metadata.ino()))
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn filesystem_identity(_metadata: &fs::Metadata, _canonical_path: &Path) -> Option<String> {
|
||||
None
|
||||
pub(super) async fn identities(&self) -> Option<Vec<ReplacementTargetIdentity>> {
|
||||
probe_replacement_mounts(&self.disks, &self.local_disks).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -269,8 +289,8 @@ mod tests {
|
||||
mod linux_privileged_tests {
|
||||
use super::*;
|
||||
use std::error::Error;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
use std::{fs, path::Path};
|
||||
|
||||
const ENABLE_ENV: &str = "RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS";
|
||||
const NAMESPACE_ENV: &str = "RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS_IN_NAMESPACE";
|
||||
@@ -396,39 +416,86 @@ mod tests {
|
||||
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS=1"]
|
||||
fn auto_replacement_readiness_accepts_an_independent_mount() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
run_privileged_mount_test(|mut mounts| async move {
|
||||
let temp = TempDir::new().expect("temporary replacement roots should be created");
|
||||
let target = temp.path().join("target");
|
||||
let temp = TempDir::new()?;
|
||||
let parent = temp.path().join("parent");
|
||||
let target = parent.join("replacement disk");
|
||||
let data = target.join("data");
|
||||
let sibling = temp.path().join("sibling");
|
||||
std::fs::create_dir(&target).expect("target mountpoint should be created");
|
||||
std::fs::create_dir(&sibling).expect("sibling mountpoint should be created");
|
||||
fs::create_dir(&parent)?;
|
||||
mounts.mount_tmpfs(&parent, "rustfs-readiness-parent")?;
|
||||
// Leave a directory at the same path below the target mount.
|
||||
fs::create_dir_all(&data)?;
|
||||
mounts.mount_tmpfs(&target, "rustfs-readiness-target")?;
|
||||
fs::create_dir(&data)?;
|
||||
fs::create_dir(&sibling)?;
|
||||
mounts.mount_tmpfs(&sibling, "rustfs-readiness-sibling")?;
|
||||
|
||||
let target_endpoint = Endpoint::try_from(target.to_string_lossy().as_ref())?;
|
||||
let sibling_endpoint = Endpoint::try_from(sibling.to_string_lossy().as_ref())?;
|
||||
let target_disk = new_disk(
|
||||
&target_endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let sibling_disk = new_disk(
|
||||
&sibling_endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let options = DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
};
|
||||
let target_disk = new_disk(&target_endpoint, &options).await?;
|
||||
let sibling_disk = new_disk(&sibling_endpoint, &options).await?;
|
||||
|
||||
let identity = auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await;
|
||||
assert!(
|
||||
identity.is_some(),
|
||||
auto_replacement_target_ready(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await,
|
||||
"a separately mounted replacement target with no sibling device overlap must be admitted"
|
||||
);
|
||||
Ok(())
|
||||
let disks = [target_disk, sibling_disk];
|
||||
let mut identities = probe_replacement_mounts(&disks, &disks)
|
||||
.await
|
||||
.expect("independently mounted replacement targets should be admitted together");
|
||||
assert_eq!(identities[0].endpoint, disks[0].endpoint().to_string());
|
||||
assert_eq!(identities[1].endpoint, disks[1].endpoint().to_string());
|
||||
identities.sort_unstable_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
let endpoints = [identities[1].endpoint.clone(), identities[0].endpoint.clone()];
|
||||
let execution = crate::heal::replacement_execution::ReplacementExecution::acquire(&endpoints).await?;
|
||||
assert_eq!(execution.identities(), identities);
|
||||
assert!(
|
||||
crate::heal::replacement_execution::ReplacementExecution::acquire(&endpoints)
|
||||
.await
|
||||
.is_err(),
|
||||
"a second executor must not take leases on the same targets"
|
||||
);
|
||||
drop(execution);
|
||||
|
||||
let endpoint = Endpoint::try_from(path_to_string(&data, "data directory")?.as_str())?;
|
||||
let disk = new_disk(&endpoint, &options).await?;
|
||||
let key = rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS;
|
||||
temp_env::async_with_vars([(key, None::<&str>)], async {
|
||||
assert!(!auto_replacement_target_ready(&disk, &[]).await);
|
||||
})
|
||||
.await;
|
||||
|
||||
let roots = std::env::join_paths([&parent, &target])?;
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(key, Some(roots.to_str().ok_or("mount roots are not UTF-8")?)),
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true")),
|
||||
],
|
||||
async {
|
||||
assert!(!directory_backed_replacement_fallback_enabled());
|
||||
let identities = probe_replacement_mounts(std::slice::from_ref(&disk), &[])
|
||||
.await
|
||||
.expect("data directory on its configured independent mount should be admitted");
|
||||
assert_eq!(Path::new(&identities[0].canonical_path), data.canonicalize()?);
|
||||
let reopened = new_disk(&endpoint, &options).await?;
|
||||
assert_eq!(probe_replacement_mounts(&[reopened], &[]).await, Some(identities));
|
||||
|
||||
run_command("umount", &["-l", &path_to_string(&target, "replacement mount")?])?;
|
||||
assert!(!auto_replacement_target_ready(&disk, &[]).await);
|
||||
|
||||
let exposed_disk = new_disk(&endpoint, &options).await?;
|
||||
assert!(
|
||||
!auto_replacement_target_ready(&exposed_disk, &[]).await,
|
||||
"a new handle on the exposed parent filesystem must also be rejected"
|
||||
);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
}
|
||||
|
||||
@@ -467,10 +534,20 @@ mod tests {
|
||||
.await?;
|
||||
|
||||
assert!(
|
||||
auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()])
|
||||
!auto_replacement_target_ready(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await,
|
||||
"replacement readiness must reject a target sharing its physical device with a sibling endpoint"
|
||||
);
|
||||
assert!(
|
||||
probe_replacement_mounts(&[target_disk.clone(), sibling_disk], &[])
|
||||
.await
|
||||
.is_none(),
|
||||
"replacement readiness must reject a target sharing its physical device with a sibling endpoint"
|
||||
"a batch must reject overlapping targets even before they are registered"
|
||||
);
|
||||
assert!(auto_replacement_target_ready(&target_disk, &[]).await);
|
||||
mounts.mount_bind(&source, &target)?;
|
||||
assert!(
|
||||
!auto_replacement_target_ready(&target_disk, &[]).await,
|
||||
"rebinding the same inode must invalidate the held mount identity"
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
|
||||
@@ -38,7 +38,7 @@ use super::storage_api::storage::{
|
||||
ObjectIO as _, ObjectOperations as _, StorageAdminApi,
|
||||
};
|
||||
use super::{
|
||||
DiskError, DiskStore, ECStore, HealDiskExt as _, StorageError, local_disk_map_read,
|
||||
DiskError, DiskStore, ECStore, HealDiskExt as _, StorageError,
|
||||
resume::{ReplacementRecoveryCandidate, ReplacementTargetIdentity, merge_replacement_recovery_candidate},
|
||||
};
|
||||
pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
|
||||
@@ -2457,9 +2457,13 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
|
||||
async fn replacement_target_identities(&self, targets: &[String]) -> Result<Vec<ReplacementTargetIdentity>> {
|
||||
super::replacement_readiness::auto_replacement_target_identities(targets)
|
||||
let error = || Error::ReplacementTargetNotReady("replacement target is not a stable mounted disk".to_string());
|
||||
super::replacement_readiness::ReplacementTargets::open(targets)
|
||||
.await
|
||||
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is not a stable mounted disk".to_string()))
|
||||
.ok_or_else(error)?
|
||||
.identities()
|
||||
.await
|
||||
.ok_or_else(error)
|
||||
}
|
||||
|
||||
async fn replacement_targets_ready_for_retry(
|
||||
@@ -2468,25 +2472,14 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
targets: &[String],
|
||||
expected_identities: &[ReplacementTargetIdentity],
|
||||
) -> Result<bool> {
|
||||
let identities = match self.replacement_target_identities(targets).await {
|
||||
Ok(identities) => identities,
|
||||
Err(_) => return Ok(false),
|
||||
let Some(targets) = super::replacement_readiness::ReplacementTargets::open(targets).await else {
|
||||
return Ok(false);
|
||||
};
|
||||
if identities != expected_identities {
|
||||
if targets.identities().await.as_deref() != Some(expected_identities) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let local_disks = local_disk_map_read()
|
||||
.await
|
||||
.values()
|
||||
.flatten()
|
||||
.filter(|disk| disk.endpoint().is_local)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
for target in targets {
|
||||
let Some(disk) = local_disks.iter().find(|disk| disk.endpoint().to_string() == *target) else {
|
||||
return Ok(false);
|
||||
};
|
||||
for disk in targets.disks {
|
||||
let endpoint = disk.endpoint();
|
||||
if crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx).as_deref()
|
||||
!= Some(set_disk_id)
|
||||
|
||||
@@ -38,8 +38,6 @@ default = []
|
||||
# event. Requires `RUSTFLAGS="--cfg hyper_unstable_tracing"`.
|
||||
diagnostic-internode-http2-ping = ["dep:hyper", "rustfs-io-metrics/diagnostic-internode-http2-ping"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"rustfs-common/hotpath",
|
||||
"rustfs-config/hotpath",
|
||||
"rustfs-io-metrics/hotpath",
|
||||
@@ -48,7 +46,6 @@ hotpath = [
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-common/hotpath-alloc",
|
||||
"rustfs-config/hotpath-alloc",
|
||||
"rustfs-io-metrics/hotpath-alloc",
|
||||
@@ -57,7 +54,6 @@ hotpath-alloc = [
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-common/hotpath-cpu",
|
||||
"rustfs-config/hotpath-cpu",
|
||||
"rustfs-io-metrics/hotpath-cpu",
|
||||
@@ -66,7 +62,6 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
rustfs-heal-contracts.workspace = true
|
||||
rustfs-io-metrics.workspace = true
|
||||
|
||||
@@ -59,7 +59,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
arc-swap.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "time"] }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
|
||||
@@ -232,7 +232,7 @@ Heal selects the authoritative version from quorum metadata, refuses when more s
|
||||
### 10.2 Drive replacement
|
||||
|
||||
1. Confirm the affected set still meets read quorum for its objects (impossible at `EC:0`, see §5) and record pool index, set index, slot, and the failed disk's UUID from the pool status.
|
||||
2. Replace the drive at the **same** endpoint and mount path with an empty filesystem owned by RustFS. Do not copy the old drive's directories; a copied `format.json` claims an identity the set already tracks.
|
||||
2. Replace the drive at the **same** endpoint and mount path with an empty filesystem owned by RustFS. Do not copy the old drive's directories. A copied `format.json` claims an identity the set already tracks. On Linux, automatic replacement checks that the endpoint is an independent mount by default. When the endpoint is a data directory below the mount, set `RUSTFS_HEAL_REPLACEMENT_MOUNT_ROOTS` on that node to the colon-separated absolute mount paths, for example `/srv/disk1:/srv/disk2` for endpoints `/srv/disk1/rustfs` and `/srv/disk2/rustfs`. Use canonical mount paths without trailing slashes. The most specific matching root must be the active mount that contains the endpoint. The root must remain independent of the system and sibling drives. Setting this variable disables the unsafe directory-backed heal fallback on that node.
|
||||
3. Bring the node back. The empty drive is not a format-quorum member, so `retain_format_quorum_members` keeps its slot offline; the endpoint monitor (`monitor_and_connect_endpoints_task` in `crates/ecstore/src/core/sets.rs`) retries the connection every 15 seconds. The erasure-set heal task (`crates/heal/src/heal/task/heal_erasure_set.rs`) writes the format into the original slot, through `heal_replacement_format` when an automatic replacement intent exists (schema 7 records under `.rustfs.sys/buckets/ahm-replacement/`, created when startup or the disk scanner observes a changed mount identity) or through `heal_format` for an admin erasure-set heal (`POST /rustfs/admin/v3/heal/` with `pool` and `set`), and then rebuilds the shards. Legacy schema 5/6 intents follow [replacement-generation-recovery.md](replacement-generation-recovery.md). If the drive held the node's pool metadata, follow [pool-metadata-recovery.md](pool-metadata-recovery.md#disk-replacement-and-metadata-erasure) first.
|
||||
4. Watch `GET /rustfs/admin/v4/heal/replacement-recovery`, background heal status, disk health, and bitrot counters. Completion criteria are in [scanner-runtime-controls.md](scanner-runtime-controls.md#replacement-recovery-completion): an `idle` queue or a readable object is not proof.
|
||||
5. The replacement is done when recovery reports `completed` for that instance and sampled objects have `xl.meta` and parts on the new drive.
|
||||
|
||||
@@ -185,16 +185,6 @@ impl PostObjectFormPolicy {
|
||||
Ok(Self(values))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_metadata(mut self, metadata: &[(String, String)]) -> S3Result<Self> {
|
||||
for (name, value) in metadata {
|
||||
let name = HeaderName::from_bytes(format!("x-amz-meta-{name}").as_bytes()).map_err(|_| invalid_post_form_field())?;
|
||||
let value = HeaderValue::from_str(value).map_err(|_| invalid_post_form_field())?;
|
||||
self.0.push((name, Some(value)));
|
||||
}
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
pub(crate) fn install<R: AuthorizationRequest + ?Sized>(self, req: &mut R) {
|
||||
let mut headers = req.headers().clone();
|
||||
for (name, value) in self.0 {
|
||||
|
||||
Reference in New Issue
Block a user