Compare commits

...

4 Commits

Author SHA1 Message Date
Hauser Zhi efdbba2c01 Merge remote-tracking branch 'origin/main' into houseme/fix-delete-full-disk 2026-11-16 11:55:10 +08:00
Hauser Zhi 8f6ebd261c perf(ecstore): retry concurrent full-disk deletes
Serialize ENOSPC waiters per physical disk and give each safe metadata primitive one bounded retry against shared released capacity.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-16 11:55:02 +08:00
Hauser Zhi 67eb18dd46 refactor(ecstore): recover delete metadata space on demand
Share a pinned reserve owner per physical disk and keep healthy deletion free of reserve filesystem probes. Retry only disposable staging and idempotent metadata primitives after actual ENOSPC, preserving transaction publication and quorum boundaries.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-16 11:29:05 +08:00
Hauser Zhi 23aee8b58a fix(ecstore): reserve emergency space for delete journals
Keep bounded physical metadata headroom for recoverable deletion on Unix without bypassing quorum or rollback.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-16 10:56:33 +08:00
4 changed files with 920 additions and 103 deletions
+23
View File
@@ -22,6 +22,7 @@ use self::commit::lock_rename_commit_directories;
mod commit;
pub(super) mod delete_recovery;
mod delete_reserve;
#[cfg(all(unix, test))]
#[path = "fd_cache_tests.rs"]
mod fd_cache_tests;
@@ -8246,6 +8247,7 @@ pub struct LocalDisk {
pub root: PathBuf,
publication_root: os::PublicationRoot,
publication_history: Option<os::NativePublicationRegistration>,
delete_reserve: Arc<delete_reserve::DeleteReserve>,
multipart_proof_authority: OnceLock<crate::disk::multipart_source::ProofVerificationContext>,
pub(crate) usage_watch_binding: Arc<super::usage_watch::DiskUsageWatchBinding>,
multipart_index: Arc<multipart_index::IndexOwner>,
@@ -8661,6 +8663,16 @@ impl LocalDisk {
#[cfg(not(target_os = "linux"))]
let io_root = Self::open_mount_lease(&root)?;
// A persisted reserve can also make a full disk writable after restart.
let reserve_root = io_root.clone();
let lease = os::acquire_namespace_mutation_lease_with_owner(&io_root, None).await;
let delete_reserve = os::run_blocking_namespace_operation_with_sync(lease, true, move || {
let reserve = delete_reserve::DeleteReserve::acquire(&reserve_root)?;
let _ = reserve.release_if_low();
Ok(reserve)
})
.await
.map_err(DiskError::from)?;
Self::check_startup_writable(&publication_root, &root).await?;
multipart_source_state::invalidate_active_multipart_sources_under(&io_root);
multipart_source_state::invalidate_active_multipart_sources_under(&root);
@@ -8677,6 +8689,16 @@ impl LocalDisk {
log_startup_disk_error("ensure_data_usage_layout", &root, err);
})?;
let reserve = delete_reserve.clone();
let lease = os::acquire_namespace_mutation_lease_with_owner(&io_root, None).await;
os::run_blocking_namespace_operation_with_sync(lease, true, move || {
// Reservation is best effort: a legacy full disk must remain readable.
let _ = reserve.initialize();
Ok(())
})
.await
.map_err(DiskError::from)?;
let startup_cleanup_ready = Arc::new(AtomicU32::new(u32::from(!cleanup)));
let startup_cleanup_notify = Arc::new(Notify::new());
@@ -8808,6 +8830,7 @@ impl LocalDisk {
let multipart_index = multipart_index::IndexOwner::acquire(&publication_root, &root, &io_root)?;
let blocking_queue = multipart_index.attach_blocking_queue(&root, &io_root);
let mut disk = Self {
delete_reserve,
multipart_index,
root: root.clone(),
publication_root,
+292 -103
View File
@@ -1227,6 +1227,16 @@ fn read_child_regular_bytes(
std::fs::read(fallback_path)
}
fn stage_recovery_file(root: &Path, bytes: &[u8], sync: bool) -> std::io::Result<tempfile::NamedTempFile> {
std::fs::create_dir_all(root)?;
let mut file = tempfile::NamedTempFile::new_in(root)?;
std::io::Write::write_all(file.as_file_mut(), bytes)?;
if sync && !bytes.is_empty() {
file.as_file().sync_data()?;
}
Ok(file)
}
#[cfg(unix)]
fn create_and_write_child_file(
parent_fd: Option<&std::fs::File>,
@@ -1872,34 +1882,40 @@ impl LocalDisk {
let record_destination = destination.clone();
let staging_root = temporary;
let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, owner.clone()).await;
let reserve = self.delete_reserve.clone();
let (staged, target) = os::run_blocking_namespace_operation_with_sync(lease, sync, move || {
std::fs::create_dir_all(&staging_root)?;
let staged = Arc::new(
tempfile::Builder::new()
.prefix("delete-preparation-")
.tempdir_in(&staging_root)?,
);
let missing_marker = original.is_none() && target.is_some();
for (name, bytes) in [
(STORAGE_FORMAT_FILE_BACKUP, original.as_deref()),
(TARGET, target.as_deref()),
(PREPARED, Some(journal.as_slice())),
(DELETE_MARKER_ROLLBACK_FILE, missing_marker.then_some(&[][..])),
] {
let Some(bytes) = bytes else {
continue;
};
#[cfg(test)]
run_owned_file_before_open(&record_destination.join(name));
let mut file = std::fs::File::create(staged.path().join(name))?;
std::io::Write::write_all(&mut file, bytes)?;
if sync && !bytes.is_empty() {
file.sync_data()?;
let staged = reserve.retry_after_enospc(|| {
std::fs::create_dir_all(&staging_root)?;
let staged = Arc::new(
tempfile::Builder::new()
.prefix("delete-preparation-")
.tempdir_in(&staging_root)?,
);
let missing_marker = original.is_none() && target.is_some();
for (name, bytes) in [
(STORAGE_FORMAT_FILE_BACKUP, original.as_deref()),
(TARGET, target.as_deref()),
(PREPARED, Some(journal.as_slice())),
(DELETE_MARKER_ROLLBACK_FILE, missing_marker.then_some(&[][..])),
] {
let Some(bytes) = bytes else {
continue;
};
#[cfg(test)]
run_owned_file_before_open(&record_destination.join(name));
let mut file = std::fs::File::create(staged.path().join(name))?;
std::io::Write::write_all(&mut file, bytes)?;
if sync && !bytes.is_empty() {
file.sync_data()?;
}
}
}
if sync {
os::fsync_dir_std(staged.path())?;
}
if sync {
os::fsync_dir_std(staged.path())?;
}
#[cfg(test)]
preparation_io_fault::run(&record_destination)?;
Ok(staged)
})?;
Ok((staged, target))
})
.await
@@ -2812,6 +2828,7 @@ impl LocalDisk {
let object_str = object.to_owned();
let recovery_scope = recovery_operation_scope();
let reserve = self.delete_reserve.clone();
type FusedStage1Outcome = Option<(Option<Uuid>, PathBuf, PreparedDeleteReceiptEntry)>;
let res = os::run_blocking_namespace_operation_with_sync(
lease,
@@ -2976,52 +2993,61 @@ impl LocalDisk {
Ok(None) => return Ok(Ok(None)),
Err(err) => return Ok(Err(err)),
};
let staged = match tempfile::Builder::new().prefix("delete-prep-").tempdir_in(&staging_root) {
Ok(dir) => dir,
Err(err) if err.kind() == ErrorKind::NotFound => {
std::fs::create_dir_all(&staging_root)?;
tempfile::Builder::new().prefix("delete-prep-").tempdir_in(&staging_root)?
let (staged, rollback_dir_fd) = reserve.retry_after_enospc(|| {
let staged = match tempfile::Builder::new().prefix("delete-prep-").tempdir_in(&staging_root) {
Ok(dir) => dir,
Err(err) if err.kind() == ErrorKind::NotFound => {
std::fs::create_dir_all(&staging_root)?;
tempfile::Builder::new().prefix("delete-prep-").tempdir_in(&staging_root)?
}
Err(err) => return Err(err),
};
let staged_dir = staged.path().to_path_buf();
#[cfg(unix)]
let rollback_dir_fd = open_dir_fd(&staged_dir).ok().map(Arc::new);
#[cfg(not(unix))]
let rollback_dir_fd: Option<Arc<std::fs::File>> = None;
let backup_path = staged_dir.join(STORAGE_FORMAT_FILE_BACKUP);
if link_child_files(
file_path_dir.as_deref(),
STORAGE_FORMAT_FILE,
&xl_path,
rollback_dir_fd.as_deref(),
STORAGE_FORMAT_FILE_BACKUP,
&backup_path,
)
.is_err()
{
let file = create_and_write_child_file(
rollback_dir_fd.as_deref(),
STORAGE_FORMAT_FILE_BACKUP,
&backup_path,
&buf,
)?;
if sync && !buf.is_empty() {
file.sync_data()?;
}
}
Err(err) => return Err(err),
};
let staged_dir = staged.path().to_path_buf();
#[cfg(unix)]
let rollback_dir_fd = open_dir_fd(&staged_dir).ok().map(Arc::new);
#[cfg(not(unix))]
let rollback_dir_fd: Option<Arc<std::fs::File>> = None;
let backup_path = staged_dir.join(STORAGE_FORMAT_FILE_BACKUP);
if link_child_files(
file_path_dir.as_deref(),
STORAGE_FORMAT_FILE,
&xl_path,
rollback_dir_fd.as_deref(),
STORAGE_FORMAT_FILE_BACKUP,
&backup_path,
)
.is_err()
{
let file =
create_and_write_child_file(rollback_dir_fd.as_deref(), STORAGE_FORMAT_FILE_BACKUP, &backup_path, &buf)?;
if sync && !buf.is_empty() {
let mut written_files = Vec::with_capacity(2);
for (name, bytes) in [(TARGET, target.as_deref()), (PREPARED, Some(journal_raw.as_slice()))] {
let Some(bytes) = bytes else {
continue;
};
let file = create_and_write_child_file(rollback_dir_fd.as_deref(), name, &staged_dir.join(name), bytes)?;
if sync && !bytes.is_empty() {
written_files.push(file);
}
}
for file in written_files {
file.sync_data()?;
}
}
let mut written_files = Vec::with_capacity(2);
for (name, bytes) in [(TARGET, target.as_deref()), (PREPARED, Some(journal_raw.as_slice()))] {
let Some(bytes) = bytes else {
continue;
};
let file = create_and_write_child_file(rollback_dir_fd.as_deref(), name, &staged_dir.join(name), bytes)?;
if sync && !bytes.is_empty() {
written_files.push(file);
if sync {
os::fsync_directory_handle_std(&staged_dir, rollback_dir_fd.as_deref())?;
}
}
for file in written_files {
file.sync_data()?;
}
if sync {
os::fsync_directory_handle_std(&staged_dir, rollback_dir_fd.as_deref())?;
}
#[cfg(test)]
preparation_io_fault::run(&destination)?;
Ok((staged, rollback_dir_fd))
})?;
let tx_name = transaction.to_string();
if !destination_missing {
match std::fs::symlink_metadata(&destination) {
@@ -3040,7 +3066,9 @@ impl LocalDisk {
}
let staged_dir = staged.keep();
os::record_native_publication(&destination);
if let Err(err) = rename_into_dir_fd(&staged_dir, file_path_dir.as_deref(), &tx_name, &destination) {
if let Err(err) = reserve
.retry_after_enospc(|| rename_into_dir_fd(&staged_dir, file_path_dir.as_deref(), &tx_name, &destination))
{
let _ = std::fs::remove_dir_all(&staged_dir);
return Err(err);
}
@@ -3058,14 +3086,16 @@ impl LocalDisk {
let reserved_name = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
let reserved_marker = old_path.join(&reserved_name);
let old_dir_file = open_child_dir_fd(file_path_dir.as_deref(), &old_dir_name, &old_path);
let reserve_res = (|| -> std::io::Result<()> {
let reserve_res = reserve.retry_after_enospc(|| -> std::io::Result<()> {
#[cfg(test)]
preparation_io_fault::run(&reserved_marker)?;
let marker = create_and_write_child_file(old_dir_file.as_ref(), &reserved_name, &reserved_marker, &[])?;
if sync {
marker.sync_all()?;
os::fsync_directory_handle_std(&old_path, old_dir_file.as_ref())?;
}
Ok(())
})();
});
if let Err(err) = reserve_res {
let _ = unlink_child_file(old_dir_file.as_ref(), &reserved_name, &reserved_marker);
let _ = std::fs::remove_dir_all(&destination);
@@ -3087,19 +3117,20 @@ impl LocalDisk {
Err(_) => false,
};
let write_res = if linked {
let rename_res = rename_into_dir_fd(&linked_xl, file_path_dir.as_deref(), STORAGE_FORMAT_FILE, &xl_path);
let rename_res = reserve.retry_after_enospc(|| {
rename_into_dir_fd(&linked_xl, file_path_dir.as_deref(), STORAGE_FORMAT_FILE, &xl_path)
});
if rename_res.is_err() {
let _ = std::fs::remove_file(&linked_xl);
}
rename_res
} else {
(|| -> std::io::Result<()> {
let mut staged_xl = tempfile::NamedTempFile::new_in(&staging_root)?;
std::io::Write::write_all(staged_xl.as_file_mut(), target_bytes)?;
if sync && !target_bytes.is_empty() {
staged_xl.as_file().sync_data()?;
}
rename_into_dir_fd(staged_xl.path(), file_path_dir.as_deref(), STORAGE_FORMAT_FILE, &xl_path)
let staged_xl =
reserve.retry_after_enospc(|| stage_recovery_file(&staging_root, target_bytes, sync))?;
reserve.retry_after_enospc(|| {
rename_into_dir_fd(staged_xl.path(), file_path_dir.as_deref(), STORAGE_FORMAT_FILE, &xl_path)
})
})()
};
if let Err(err) = write_res {
@@ -3242,6 +3273,7 @@ impl LocalDisk {
let volume_str = volume.to_owned();
let object_str = object.to_owned();
let reserve = self.delete_reserve.clone();
type PublishDeleteFastReturn = (DeleteRecoveryJournal, bool, bool, Option<(PathBuf, PublishedDeleteReceiptEntry)>);
let res = os::run_blocking_namespace_operation_with_sync(
lease,
@@ -3254,18 +3286,25 @@ impl LocalDisk {
let tmp_receipt = staging_root.join(format!("receipt-{transaction}.tmp"));
let receipt_bytes = format_commit_receipt_bytes(&journal_raw, false);
let write_res = (|| -> std::io::Result<()> {
let mut file = std::fs::File::create(&tmp_receipt)?;
std::io::Write::write_all(&mut file, &receipt_bytes)?;
if sync && !receipt_bytes.is_empty() {
file.sync_data()?;
}
reserve.retry_after_enospc(|| {
let mut file = std::fs::File::create(&tmp_receipt)?;
std::io::Write::write_all(&mut file, &receipt_bytes)?;
if sync && !receipt_bytes.is_empty() {
file.sync_data()?;
}
#[cfg(test)]
preparation_io_fault::run(&receipt_path)?;
Ok(())
})?;
os::record_native_publication(&receipt_path);
rename_into_dir_fd(
&tmp_receipt,
rollback_dir_fd.as_deref(),
DELETE_QUORUM_COMMITTED_FILE,
&receipt_path,
)?;
reserve.retry_after_enospc(|| {
rename_into_dir_fd(
&tmp_receipt,
rollback_dir_fd.as_deref(),
DELETE_QUORUM_COMMITTED_FILE,
&receipt_path,
)
})?;
if sync {
os::fsync_directory_handle_std(&rollback, rollback_dir_fd.as_deref())?;
}
@@ -3358,14 +3397,9 @@ impl LocalDisk {
finished: false,
};
let receipt_bytes = serde_json::to_vec(&receipt).map_err(std::io::Error::other)?;
std::fs::create_dir_all(&staging_root)?;
let mut staged = tempfile::NamedTempFile::new_in(&staging_root)?;
std::io::Write::write_all(staged.as_file_mut(), &receipt_bytes)?;
if sync && !receipt_bytes.is_empty() {
staged.as_file().sync_data()?;
}
let staged = reserve.retry_after_enospc(|| stage_recovery_file(&staging_root, &receipt_bytes, sync))?;
os::record_native_publication(&receipt_path);
std::fs::rename(staged.path(), &receipt_path)?;
reserve.retry_after_enospc(|| std::fs::rename(staged.path(), &receipt_path))?;
if sync {
os::fsync_dir_std(&rollback)?;
}
@@ -3517,6 +3551,7 @@ impl LocalDisk {
.collect()
};
let reserve = self.delete_reserve.clone();
let res = os::run_blocking_namespace_operation_with_sync(lease, sync, move || -> std::io::Result<()> {
let persist_receipt = |finished: bool| -> std::io::Result<()> {
let receipt_bytes = match &cached_journal_raw {
@@ -3531,14 +3566,9 @@ impl LocalDisk {
serde_json::to_vec(&receipt).map_err(std::io::Error::other)?
}
};
std::fs::create_dir_all(&staging_root)?;
let mut staged_receipt = tempfile::NamedTempFile::new_in(&staging_root)?;
std::io::Write::write_all(staged_receipt.as_file_mut(), &receipt_bytes)?;
if sync && !receipt_bytes.is_empty() {
staged_receipt.as_file().sync_data()?;
}
let staged_receipt = reserve.retry_after_enospc(|| stage_recovery_file(&staging_root, &receipt_bytes, sync))?;
os::record_native_publication(&receipt_path);
std::fs::rename(staged_receipt.path(), &receipt_path)?;
reserve.retry_after_enospc(|| std::fs::rename(staged_receipt.path(), &receipt_path))?;
if sync {
let _ = os::fsync_dir_std(&rollback);
}
@@ -3724,6 +3754,11 @@ impl LocalDisk {
for claim in claims {
claim.finish(ok);
}
if ok && !self.delete_reserve.is_armed() {
let reserve = self.delete_reserve.clone();
let lease = os::acquire_namespace_mutation_lease_with_owner(self.io_root(), None).await;
let _ = os::run_blocking_namespace_operation_with_sync(lease, true, move || reserve.replenish()).await;
}
res.map(|()| true).map_err(|err| to_file_error(err).into())
}
}
@@ -3859,6 +3894,43 @@ mod fused_prepare_checkpoint {
}
}
#[cfg(test)]
mod preparation_io_fault {
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{LazyLock, Mutex};
type Hook = Box<dyn FnOnce() -> std::io::Result<()> + Send>;
static HOOKS: LazyLock<Mutex<HashMap<PathBuf, Hook>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
pub(super) struct Guard(PathBuf);
impl Drop for Guard {
fn drop(&mut self) {
HOOKS.lock().expect("checkpoint hooks").remove(&self.0);
}
}
pub(super) fn install(root: &Path, hook: impl FnOnce() -> std::io::Result<()> + Send + 'static) -> Guard {
assert!(
HOOKS
.lock()
.expect("checkpoint hooks")
.insert(root.to_path_buf(), Box::new(hook))
.is_none()
);
Guard(root.to_path_buf())
}
pub(super) fn run(root: &Path) -> std::io::Result<()> {
let hook = HOOKS.lock().expect("checkpoint hooks").remove(root);
match hook {
Some(hook) => hook(),
None => Ok(()),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -8484,6 +8556,55 @@ mod tests {
#[tokio::test]
async fn delete_recovery_fused_unversioned_physical_delete_retains_receipt_until_finalize() {
assert_unversioned_delete_reserve_recovery(None).await;
}
#[cfg(unix)]
#[tokio::test]
async fn delete_recovery_fused_enospc_recovers_scratch_marker_and_receipt_without_replaying_transaction() {
for stage in ["scratch", "marker", "receipt"] {
assert_unversioned_delete_reserve_recovery(Some(stage)).await;
}
}
#[cfg(unix)]
#[tokio::test]
async fn delete_recovery_fallback_enospc_recovers_before_activation_and_preserves_original() {
for kind in [ErrorKind::StorageFull, ErrorKind::PermissionDenied] {
let (_temp, disk, info, _) = fixture().await;
let root = disk.io_get_object_path("recovery", "object").expect("object path");
let original = std::fs::read(root.join(STORAGE_FORMAT_FILE)).expect("original metadata");
let transaction = Uuid::new_v4();
let destination = root.join(transaction.to_string());
let _fault = preparation_io_fault::install(&destination, move || Err(std::io::Error::from(kind)));
let result = disk
.prepare_delete_recovery(
"recovery",
"object",
Some(&original),
&[info],
None,
DeletePreparationContext {
transaction,
bucket_incarnation: None,
namespace_owner: None,
},
)
.await;
if kind == ErrorKind::StorageFull {
assert!(result.is_ok(), "scratch ENOSPC must recover: {result:?}");
assert!(destination.join(PREPARED).is_file());
assert!(!disk.delete_reserve.is_armed());
} else {
assert!(result.is_err(), "non-capacity failures must not retry");
assert!(!destination.exists());
assert!(disk.delete_reserve.is_armed());
}
assert_eq!(std::fs::read(root.join(STORAGE_FORMAT_FILE)).expect("unchanged metadata"), original);
}
}
async fn assert_unversioned_delete_reserve_recovery(fault: Option<&str>) {
let (_temp, disk, mut info, _) = fixture().await;
let root = disk.io_get_object_path("recovery", "object").expect("object path");
@@ -8506,7 +8627,35 @@ mod tests {
.await
.expect("write xl.meta");
#[cfg(unix)]
let reserve_path = disk.io_root().join(".rustfs.sys/delete-reserve");
#[cfg(unix)]
assert_eq!(
std::fs::metadata(&reserve_path).expect("initialized reserve").len(),
super::super::delete_reserve::RESERVE_BYTES
);
let transaction = Uuid::new_v4();
let rollback = root.join(transaction.to_string());
let _fault = fault.map(|stage| {
let path = match stage {
"scratch" => rollback.clone(),
"marker" => root
.join(data_dir.to_string())
.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")),
"receipt" => rollback.join(DELETE_QUORUM_COMMITTED_FILE),
_ => panic!("unknown fault stage"),
};
preparation_io_fault::install(&path, || Err(std::io::Error::from(ErrorKind::StorageFull)))
});
#[cfg(unix)]
let checkpoints = Arc::new(std::sync::atomic::AtomicUsize::new(0));
#[cfg(unix)]
let _checkpoint = {
let checkpoints = checkpoints.clone();
fused_prepare_checkpoint::install(&root, move || {
checkpoints.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
})
};
let opts = DeleteOptions {
recoverable_delete: true,
..Default::default()
@@ -8516,6 +8665,32 @@ mod tests {
.await
.expect("fused unversioned physical delete must succeed");
assert_eq!(fused, Some(()));
#[cfg(unix)]
assert_eq!(
std::fs::metadata(&reserve_path).expect("released reserve").len(),
if matches!(fault, Some("scratch" | "marker")) {
0
} else {
super::super::delete_reserve::RESERVE_BYTES
},
"only an actual capacity failure should consume the reserve"
);
#[cfg(unix)]
assert_eq!(
checkpoints.load(std::sync::atomic::Ordering::Relaxed),
1,
"transaction publication must not replay"
);
assert!(
std::fs::read_dir(disk.io_root().join(RUSTFS_META_TMP_BUCKET))
.expect("staging root")
.all(|entry| !entry
.expect("staging entry")
.file_name()
.to_string_lossy()
.starts_with("delete-prep-")),
"failed scratch attempts must be cleaned up"
);
let (journal, sole_transaction, target_satisfied) = disk
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
@@ -8525,6 +8700,15 @@ mod tests {
assert!(sole_transaction);
assert!(target_satisfied);
assert!(journal.target_digest.is_none());
#[cfg(unix)]
assert_eq!(
std::fs::metadata(&reserve_path).expect("reserve after receipt").len(),
if fault.is_some() {
0
} else {
super::super::delete_reserve::RESERVE_BYTES
}
);
let rollback = root.join(transaction.to_string());
assert!(rollback.join(DELETE_QUORUM_COMMITTED_FILE).is_file());
@@ -8534,6 +8718,11 @@ mod tests {
.await
.expect("local finalize must succeed");
assert!(finalized);
#[cfg(unix)]
assert_eq!(
std::fs::metadata(&reserve_path).expect("replenished reserve").len(),
super::super::delete_reserve::RESERVE_BYTES
);
assert!(!rollback.exists(), "rollback directory must be pruned");
assert!(!root.join(data_dir.to_string()).exists(), "physical data_dir must be removed");
assert!(!root.exists(), "empty object directory must be pruned");
@@ -0,0 +1,597 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Bounded emergency headroom for durable deletion. Healthy operations access
//! only the shared owner's atomic state. Only allocation failures consume the
//! reserve, without holding its mutex across transaction publication or quorum.
use std::{io, path::Path, sync::Arc};
#[cfg(unix)]
pub(super) const RESERVE_BYTES: u64 = 8 * 1024 * 1024;
pub(super) struct DeleteReserve {
#[cfg(unix)]
inner: unix::State,
}
impl DeleteReserve {
pub(super) fn acquire(root: &Path) -> io::Result<Arc<Self>> {
#[cfg(unix)]
{
unix::acquire(root)
}
#[cfg(not(unix))]
{
let _ = root;
Ok(Arc::new(Self {}))
}
}
pub(super) fn is_armed(&self) -> bool {
#[cfg(unix)]
{
self.inner.is_armed()
}
#[cfg(not(unix))]
{
true
}
}
/// Populate the reserve on the disk's dedicated fsync lane.
pub(super) fn initialize(&self) -> io::Result<()> {
self.replenish()
}
/// Refill only when sufficient free space remains for recovery. An armed
/// owner returns immediately without opening files or probing capacity.
pub(super) fn replenish(&self) -> io::Result<()> {
#[cfg(unix)]
{
self.inner.replenish()
}
#[cfg(not(unix))]
{
Ok(())
}
}
/// Probe capacity during startup so a persisted reserve can make the disk
/// writable before its normal writeability check.
pub(super) fn release_if_low(&self) -> io::Result<()> {
#[cfg(unix)]
{
self.inner.release_if_low()
}
#[cfg(not(unix))]
{
Ok(())
}
}
/// Retry an allocation-safe scratch operation once after ENOSPC. The first
/// waiter releases the reserve; concurrent waiters serialize behind it and
/// may use the same released capacity. The caller must ensure that a failed
/// attempt leaves only disposable scratch or an idempotent primitive under
/// the existing namespace owner. Never replay a transaction or a
/// publication-plus-sync sequence with this helper.
///
/// The owner mutex covers release and the retry, excluding replenishment
/// during this bounded operation. Later transaction phases and unrelated
/// allocators can still compete for the released space.
pub(super) fn retry_after_enospc<T>(&self, mut operation: impl FnMut() -> io::Result<T>) -> io::Result<T> {
match operation() {
#[cfg(unix)]
Err(error) if error.kind() == io::ErrorKind::StorageFull => self.inner.retry_after_enospc(operation, error),
result => result,
}
}
}
#[cfg(unix)]
mod unix {
use super::*;
use parking_lot::Mutex;
use rustix::fs::{Mode, OFlags, mkdirat, open, openat};
use std::{
collections::HashMap,
fs::File,
io::{Seek, SeekFrom, Write},
os::unix::fs::MetadataExt,
sync::{
OnceLock, Weak,
atomic::{AtomicBool, Ordering},
},
};
const METADATA_DIR: &str = ".rustfs.sys";
const RESERVE_NAME: &str = "delete-reserve";
type RootIdentity = (u64, u64);
type OwnerRegistry = Mutex<HashMap<RootIdentity, Weak<DeleteReserve>>>;
static OWNERS: OnceLock<OwnerRegistry> = OnceLock::new();
pub(super) struct State {
root: File,
mutation: Mutex<()>,
armed: AtomicBool,
}
pub(super) fn acquire(path: &Path) -> io::Result<Arc<DeleteReserve>> {
let root = File::from(open(
path,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
)?);
let metadata = root.metadata()?;
let identity = (metadata.dev(), metadata.ino());
let mut registry = OWNERS.get_or_init(Mutex::default).lock();
if let Some(owner) = registry.get(&identity).and_then(Weak::upgrade) {
return Ok(owner);
}
registry.retain(|_, owner| owner.strong_count() != 0);
let owner = Arc::new(DeleteReserve {
inner: State {
root,
mutation: Mutex::new(()),
armed: AtomicBool::new(false),
},
});
registry.insert(identity, Arc::downgrade(&owner));
Ok(owner)
}
fn metadata_directory(root: &File, create: bool) -> io::Result<File> {
if create {
match mkdirat(root, METADATA_DIR, Mode::from_raw_mode(0o700)) {
Ok(()) => root.sync_all()?,
Err(rustix::io::Errno::EXIST) => {}
Err(error) => return Err(error.into()),
}
}
Ok(File::from(openat(
root,
METADATA_DIR,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
)?))
}
fn reserve_file(parent: &File, create: bool) -> io::Result<File> {
let flags = OFlags::RDWR | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC;
let file = File::from(openat(
parent,
RESERVE_NAME,
if create { flags | OFlags::CREATE } else { flags },
Mode::from_raw_mode(0o600),
)?);
let metadata = file.metadata()?;
if !metadata.is_file() || metadata.nlink() != 1 || metadata.len() > RESERVE_BYTES {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid delete reserve file"));
}
Ok(file)
}
fn absent_to_none<T>(result: io::Result<T>) -> io::Result<Option<T>> {
match result {
Ok(value) => Ok(Some(value)),
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
Err(error) => Err(error),
}
}
impl State {
pub(super) fn is_armed(&self) -> bool {
self.armed.load(Ordering::Acquire)
}
pub(super) fn replenish(&self) -> io::Result<()> {
if self.is_armed() {
return Ok(());
}
let _guard = self.mutation.lock();
if self.is_armed() {
return Ok(());
}
let existing = match absent_to_none(metadata_directory(&self.root, false))? {
Some(parent) => absent_to_none(reserve_file(&parent, false))?.map(|file| (parent, file)),
None => None,
};
if let Some((_, file)) = &existing {
let metadata = file.metadata()?;
if metadata.len() == RESERVE_BYTES && metadata.blocks().saturating_mul(512) >= RESERVE_BYTES {
self.armed.store(true, Ordering::Release);
return Ok(());
}
}
// A depleted reserve must not take newly released recovery space
// away from a disk that is still nearly full.
if available_bytes(&self.root)? < RESERVE_BYTES * 2 {
return Ok(());
}
let (parent, mut file) = match existing {
Some(existing) => existing,
None => {
let parent = metadata_directory(&self.root, true)?;
let file = reserve_file(&parent, true)?;
(parent, file)
}
};
let metadata = file.metadata()?;
// A length alone does not reserve blocks. Repair sparse files left
// by interrupted allocation or older installations.
if metadata.blocks().saturating_mul(512) < metadata.len() {
file.set_len(0)?;
}
let length = file.seek(SeekFrom::End(0))?;
let mut remaining = RESERVE_BYTES - length;
let mut bytes = [0_u8; 64 * 1024];
while remaining != 0 {
let count = usize::try_from(remaining.min(64 * 1024)).map_err(io::Error::other)?;
// Zero blocks can become holes on transparent-compression filesystems.
rand::fill(&mut bytes[..count]);
file.write_all(&bytes[..count])?;
remaining -= u64::try_from(count).map_err(io::Error::other)?;
}
file.sync_all()?;
if file.metadata()?.blocks().saturating_mul(512) < RESERVE_BYTES {
return Err(io::Error::new(
io::ErrorKind::Unsupported,
"filesystem did not allocate delete reserve blocks",
));
}
parent.sync_all()?;
self.armed.store(true, Ordering::Release);
Ok(())
}
fn release_locked(&self) -> io::Result<bool> {
let Some(parent) = absent_to_none(metadata_directory(&self.root, false))? else {
self.armed.store(false, Ordering::Release);
return Ok(false);
};
let Some(file) = absent_to_none(reserve_file(&parent, false))? else {
self.armed.store(false, Ordering::Release);
return Ok(false);
};
let metadata = file.metadata()?;
if metadata.len() == 0 {
self.armed.store(false, Ordering::Release);
return Ok(false);
}
file.set_len(0)?;
self.armed.store(false, Ordering::Release);
// The reserve contains no recovery authority. Truncation releases
// blocks immediately and needs no durability barrier before retry.
Ok(metadata.blocks() != 0)
}
pub(super) fn release_if_low(&self) -> io::Result<()> {
if available_bytes(&self.root)? < RESERVE_BYTES / 2 {
let _guard = self.mutation.lock();
self.release_locked()?;
}
Ok(())
}
pub(super) fn retry_after_enospc<T>(
&self,
mut operation: impl FnMut() -> io::Result<T>,
original: io::Error,
) -> io::Result<T> {
let _guard = self.mutation.lock();
match self.release_locked() {
// A peer may have consumed the shared reserve while this
// operation was failing. Waiting for the mutation lock and
// replaying once lets it use the capacity that peer released.
// The public contract only accepts disposable scratch or an
// idempotent primitive, so the retry remains safe even when no
// reserve was present. Invalid reserve state still fails closed.
Ok(_) => operation(),
Err(_) => Err(original),
}
}
}
fn available_bytes(root: &File) -> io::Result<u64> {
let stat = rustix::fs::fstatvfs(root)?;
let block_size = if stat.f_frsize == 0 { stat.f_bsize } else { stat.f_frsize };
let available = u128::from(stat.f_bavail) * u128::from(block_size);
Ok(u64::try_from(available).unwrap_or(u64::MAX))
}
#[cfg(test)]
mod tests {
use super::*;
fn reserve_path(root: &Path) -> std::path::PathBuf {
root.join(METADATA_DIR).join(RESERVE_NAME)
}
#[test]
fn delete_reserve_allocates_retries_once_and_replenishes_real_blocks() {
let root = tempfile::tempdir().expect("create disk root");
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
owner.initialize().expect("allocate reserve");
let path = reserve_path(root.path());
let initial = std::fs::metadata(&path).expect("reserve metadata");
assert_eq!(initial.len(), RESERVE_BYTES);
assert!(initial.blocks() * 512 >= RESERVE_BYTES);
assert!(owner.is_armed());
let mut calls = 0;
let result = owner
.retry_after_enospc(|| {
calls += 1;
if calls == 1 {
Err(io::Error::new(io::ErrorKind::StorageFull, "scratch allocation"))
} else {
Ok(42)
}
})
.expect("retry scratch allocation");
assert_eq!(result, 42);
assert_eq!(calls, 2);
assert!(!owner.is_armed());
assert_eq!(std::fs::metadata(&path).expect("released reserve").len(), 0);
owner.replenish().expect("replenish reserve");
assert!(owner.is_armed());
assert_eq!(std::fs::metadata(&path).expect("replenished reserve").len(), RESERVE_BYTES);
}
#[test]
fn delete_reserve_retries_once_without_local_reserve_capacity() {
let root = tempfile::tempdir().expect("create disk root");
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
let mut calls = 0;
let error = owner
.retry_after_enospc::<()>(|| {
calls += 1;
Err(io::Error::new(io::ErrorKind::StorageFull, format!("allocation {calls}")))
})
.expect_err("retry without a reserve still has no capacity");
assert_eq!(calls, 2);
assert_eq!(error.to_string(), "allocation 2");
owner.initialize().expect("allocate reserve");
let parent = metadata_directory(&owner.inner.root, false).expect("metadata directory");
let reserve = reserve_file(&parent, false).expect("reserve file");
reserve.set_len(0).expect("deplete reserve");
calls = 0;
assert!(
owner
.retry_after_enospc::<()>(|| {
calls += 1;
Err(io::Error::new(io::ErrorKind::StorageFull, "empty reserve"))
})
.is_err()
);
assert_eq!(calls, 2);
}
#[test]
fn delete_reserve_retry_ignores_other_errors_and_stays_bounded() {
let root = tempfile::tempdir().expect("create disk root");
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
owner.initialize().expect("allocate reserve");
let mut calls = 0;
let error = owner
.retry_after_enospc::<()>(|| {
calls += 1;
Err(io::Error::new(io::ErrorKind::PermissionDenied, "permission failure"))
})
.expect_err("permission failures must propagate");
assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
assert_eq!(calls, 1);
assert!(owner.is_armed());
calls = 0;
let error = owner
.retry_after_enospc::<()>(|| {
calls += 1;
Err(io::Error::new(io::ErrorKind::StorageFull, format!("allocation {calls}")))
})
.expect_err("second allocation failure propagates");
assert_eq!(calls, 2);
assert_eq!(error.to_string(), "allocation 2");
}
#[test]
fn delete_reserve_rejects_symlinks_and_hardlinks_without_hiding_operation_errors() {
use std::os::unix::fs::symlink;
let root = tempfile::tempdir().expect("create disk root");
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
let external = tempfile::tempdir().expect("create external directory");
let victim = external.path().join("victim");
std::fs::write(&victim, b"preserve").expect("write victim");
let metadata = root.path().join(METADATA_DIR);
symlink(external.path(), &metadata).expect("symlink metadata directory");
assert!(owner.replenish().is_err());
std::fs::remove_file(&metadata).expect("remove directory link");
std::fs::create_dir(&metadata).expect("create metadata directory");
let reserve = reserve_path(root.path());
symlink(&victim, &reserve).expect("symlink reserve");
assert!(owner.replenish().is_err());
let mut calls = 0;
let error = owner
.retry_after_enospc::<()>(|| {
calls += 1;
Err(io::Error::new(io::ErrorKind::StorageFull, "original error"))
})
.expect_err("invalid reserve cannot mask allocation failure");
assert_eq!(calls, 1);
assert_eq!(error.to_string(), "original error");
std::fs::remove_file(&reserve).expect("remove reserve link");
std::fs::hard_link(&victim, &reserve).expect("hardlink reserve");
assert!(owner.replenish().is_err());
assert!(owner.inner.release_locked().is_err());
assert_eq!(std::fs::read(&victim).expect("read victim"), b"preserve");
}
#[test]
fn delete_reserve_repairs_partial_and_sparse_files() {
let root = tempfile::tempdir().expect("create disk root");
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
let parent = metadata_directory(&owner.inner.root, true).expect("create metadata directory");
let mut file = reserve_file(&parent, true).expect("create reserve file");
file.write_all(b"partial allocation").expect("write partial reserve");
owner.replenish().expect("finish partial allocation");
let metadata = file.metadata().expect("partial reserve metadata");
assert_eq!(metadata.len(), RESERVE_BYTES);
assert!(metadata.blocks() * 512 >= RESERVE_BYTES);
file.set_len(0).expect("truncate reserve");
file.set_len(RESERVE_BYTES).expect("create sparse reserve");
owner.inner.armed.store(false, Ordering::Release);
assert!(file.metadata().expect("sparse reserve metadata").blocks() * 512 < RESERVE_BYTES);
owner.replenish().expect("repair sparse allocation");
let metadata = file.metadata().expect("repaired reserve metadata");
assert_eq!(metadata.len(), RESERVE_BYTES);
assert!(metadata.blocks() * 512 >= RESERVE_BYTES);
}
#[test]
fn delete_reserve_shares_reopened_owner_and_serializes_recovery_with_replenishment() {
let root = tempfile::tempdir().expect("create disk root");
let first = DeleteReserve::acquire(root.path()).expect("open first root");
let second = DeleteReserve::acquire(root.path()).expect("reopen root");
assert!(Arc::ptr_eq(&first, &second));
first.initialize().expect("allocate reserve");
let (retry_entered_tx, retry_entered_rx) = std::sync::mpsc::channel();
let (allow_retry_tx, allow_retry_rx) = std::sync::mpsc::channel();
let (replenished_tx, replenished_rx) = std::sync::mpsc::channel();
let recovery_owner = first.clone();
let root_path = root.path().to_path_buf();
std::thread::scope(|scope| {
let recovery = scope.spawn(move || {
let mut calls = 0;
recovery_owner
.retry_after_enospc(|| {
calls += 1;
if calls == 1 {
return Err(io::Error::new(io::ErrorKind::StorageFull, "scratch allocation"));
}
assert!(recovery_owner.inner.mutation.try_lock().is_none(), "retry must retain the reserve mutex");
retry_entered_tx.send(()).expect("notify retry");
allow_retry_rx
.recv_timeout(std::time::Duration::from_secs(10))
.expect("allow retry completion");
assert_eq!(std::fs::metadata(reserve_path(&root_path)).expect("depleted reserve").len(), 0);
Ok(())
})
.expect("retry allocation");
});
retry_entered_rx
.recv_timeout(std::time::Duration::from_secs(10))
.expect("retry entered");
let replenish = scope.spawn(move || {
second.replenish().expect("concurrent replenishment");
replenished_tx.send(()).expect("notify replenishment");
});
assert!(matches!(replenished_rx.try_recv(), Err(std::sync::mpsc::TryRecvError::Empty)));
allow_retry_tx.send(()).expect("finish retry");
recovery.join().expect("recovery worker");
replenish.join().expect("replenishment worker");
});
assert!(first.is_armed());
assert_eq!(std::fs::metadata(reserve_path(root.path())).expect("final reserve").len(), RESERVE_BYTES);
}
#[test]
fn delete_reserve_gives_concurrent_enospc_waiters_one_retry() {
let root = tempfile::tempdir().expect("create disk root");
let first = DeleteReserve::acquire(root.path()).expect("open first owner");
let second = DeleteReserve::acquire(root.path()).expect("open second owner");
first.initialize().expect("allocate reserve");
let (first_retry_tx, first_retry_rx) = std::sync::mpsc::channel();
let (allow_first_tx, allow_first_rx) = std::sync::mpsc::channel();
let (second_failed_tx, second_failed_rx) = std::sync::mpsc::channel();
let (second_done_tx, second_done_rx) = std::sync::mpsc::channel();
std::thread::scope(|scope| {
let first_waiter = scope.spawn(move || {
let mut calls = 0;
first
.retry_after_enospc(|| {
calls += 1;
if calls == 1 {
return Err(io::Error::new(io::ErrorKind::StorageFull, "first ENOSPC"));
}
first_retry_tx.send(()).expect("notify first retry");
allow_first_rx
.recv_timeout(std::time::Duration::from_secs(10))
.expect("allow first retry to finish");
Ok(())
})
.expect("first waiter retries");
assert_eq!(calls, 2);
});
first_retry_rx
.recv_timeout(std::time::Duration::from_secs(10))
.expect("first waiter holds recovery lock");
let second_waiter = scope.spawn(move || {
let mut calls = 0;
second
.retry_after_enospc(|| {
calls += 1;
if calls == 1 {
second_failed_tx.send(()).expect("notify second ENOSPC");
return Err(io::Error::new(io::ErrorKind::StorageFull, "second ENOSPC"));
}
Ok(())
})
.expect("second waiter retries after shared release");
assert_eq!(calls, 2);
second_done_tx.send(()).expect("notify second completion");
});
second_failed_rx
.recv_timeout(std::time::Duration::from_secs(10))
.expect("second waiter observed ENOSPC");
assert!(matches!(second_done_rx.try_recv(), Err(std::sync::mpsc::TryRecvError::Empty)));
allow_first_tx.send(()).expect("finish first retry");
first_waiter.join().expect("first waiter");
second_done_rx
.recv_timeout(std::time::Duration::from_secs(10))
.expect("second waiter completed its bounded retry");
second_waiter.join().expect("second waiter");
});
assert_eq!(std::fs::metadata(reserve_path(root.path())).expect("depleted reserve").len(), 0);
}
#[test]
fn delete_reserve_rejects_oversize_files_and_fifos() {
let root = tempfile::tempdir().expect("create disk root");
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
let parent = metadata_directory(&owner.inner.root, true).expect("create metadata directory");
let file = reserve_file(&parent, true).expect("create reserve file");
file.set_len(RESERVE_BYTES + 1).expect("create oversized reserve");
assert!(owner.replenish().is_err());
assert!(owner.inner.release_locked().is_err());
assert_eq!(file.metadata().expect("oversize metadata").len(), RESERVE_BYTES + 1);
let path = reserve_path(root.path());
std::fs::remove_file(&path).expect("remove oversized file");
assert!(
std::process::Command::new("mkfifo")
.arg(&path)
.status()
.expect("create FIFO fixture")
.success()
);
assert!(owner.replenish().is_err());
assert!(owner.inner.release_locked().is_err());
use std::os::unix::fs::FileTypeExt;
assert!(std::fs::symlink_metadata(path).expect("FIFO metadata").file_type().is_fifo());
}
}
}
@@ -68,3 +68,11 @@ No new environment variable, admin authorization action, or recovery command is
## Delete recovery during rolling upgrades
S3 deletes that require bounded recovery need peers that support the bounded prepare protocol. An older peer rejects that prepare before staging a mutation. If too few upgraded drives can satisfy the write quorum, the delete can fail with a retryable error until more peers are upgraded. Ordinary recovery inspection and settlement continue to use the original protocol.
## Full data disks and deletion recovery
An ordinary unversioned DELETE permanently removes data, but RustFS first persists a recoverable delete transaction so a failed quorum can restore the object. A filesystem with no allocatable space can reject that journal before any data is removed. This is distinct from PUT pool-capacity admission; changing only that admission does not make deletes work on a physically full disk. Full-disk failures retain the existing S3 `503 ServiceUnavailable` response.
On Unix, each local disk attempts to keep an 8 MiB physically allocated emergency reserve at `.rustfs.sys/delete-reserve`. The reserve uses incompressible content and is armed only after synced allocation is verified, so sparse or transparently compressed zero blocks cannot be mistaken for recovery headroom. Reopened disk handles share one owner pinned to the physical root. Healthy deletion does not open reserve paths or query capacity. If a repeatable scratch write or an idempotent delete-metadata primitive fails with ENOSPC, the first waiter releases existing reserve blocks and retries that operation once; concurrent waiters serialize behind it and also get one bounded retry against the released capacity. Replenishment cannot race those retries. A failed scratch attempt is discarded before retry; the transaction, publication-plus-sync sequences, and destructive cleanup are never replayed as a unit. Invalid reserve state preserves the original operation error. Successful local finalization only attempts replenishment after reserve depletion and with at least 16 MiB free, on the dedicated fsync lane. Startup may release a persisted reserve below 4 MiB available space before the writable probe. The reserve is outside temporary-object cleanup and changes no object formats, quorum, retention checks, or rollback rules. Rollback to an older server leaves an unused reserve file consuming at most 8 MiB per disk.
The reserve is bounded recovery headroom, not a guarantee under arbitrary concurrent writes. Large version histories, inode exhaustion, unrelated writers, or multiple simultaneous deletes can still exhaust space. A first upgrade on an already-full disk may have no reserve to release. Reservation failure does not disable otherwise usable storage, and Windows currently retains its existing behavior. Stop uploads and create filesystem headroom if API deletion still fails; do not remove object directories manually because that bypasses metadata and erasure consistency. Versioned DELETE normally creates a marker and does not reclaim the prior payload; permanently deleting a permitted version requires its version ID.