gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-29 16:37:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(heal): reconcile dangling objects after node reconnect (#6567)

Browse Source
This commit is contained in:
Henry Guo
2026-08-25 21:20:47 +08:00
committed by GitHub
parent b4a78fc907
commit 9db29c8a6f
6 changed files with 458 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/ops/heal.rs
+132 -19
View File
@@ -779,16 +779,11 @@ impl SetDisks {
.await
{
Ok(m) => {
let derr = if !version_id.is_empty() {
DiskError::FileVersionNotFound
} else {
DiskError::FileNotFound
};
let mut t_errs = Vec::with_capacity(errs.len());
for _ in 0..errs.len() {
t_errs.push(None);
}
Ok((self.default_heal_result(m, &t_errs, bucket, object, version_id).await, Some(derr)))
Ok((self.default_heal_result(m, &t_errs, bucket, object, version_id).await, None))
}
Err(err) => {
error!(
@@ -798,10 +793,9 @@ impl SetDisks {
object,
version_id,
error = %err,
returned_error = %cannot_heal_err,
"Heal object dangling cleanup could not prove object deletion"
);
Ok((result, Some(cannot_heal_err)))
Ok((result, Some(err)))
}
};
}
@@ -1273,6 +1267,14 @@ impl SetDisks {
return Box::pin(self.heal_object_with_explicit_version_regen(bucket, object, version_id, opts, false)).await;
}
if opts.dry_run {
return Ok((
self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id)
.await,
Some(err),
));
}
if self
.dangling_delete_safety(bucket, object, &parts_metadata, &errs, &disks)
.await?
@@ -1300,18 +1302,11 @@ impl SetDisks {
)
.await
{
Ok(m) => {
let err = if !version_id.is_empty() {
DiskError::FileVersionNotFound
} else {
DiskError::FileNotFound
};
Ok((self.default_heal_result(m, &errs, bucket, object, version_id).await, Some(err)))
}
Err(_) => Ok((
Ok(m) => Ok((self.default_heal_result(m, &errs, bucket, object, version_id).await, None)),
Err(cleanup_err) => Ok((
self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id)
.await,
Some(err),
Some(cleanup_err),
)),
}
}
@@ -2123,7 +2118,7 @@ mod heal_result_report_tests {
use crate::disk::endpoint::Endpoint;
use crate::disk::error::DiskError;
use crate::disk::format::FormatV3;
use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, new_disk};
use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, STORAGE_FORMAT_FILE, new_disk};
use crate::error::Error;
use crate::object_api::{ObjectOptions, PutObjReader};
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
@@ -2134,6 +2129,7 @@ mod heal_result_report_tests {
config::storageclass,
store::init_format::{load_format_erasure, save_format_file},
};
use bytes::Bytes;
use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode};
use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo, ObjectPartInfo, TRANSITION_COMPLETE};
use std::sync::{Arc, Mutex};
@@ -2394,6 +2390,20 @@ mod heal_result_report_tests {
.expect("test metadata should be written");
}
async fn dangling_inline_test_fixture(
bucket: &str,
object: &str,
mod_time: OffsetDateTime,
) -> (Vec<TempDir>, Arc<SetDisks>, Vec<Option<DiskStore>>) {
let (temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[]).await;
let mut metadata = meta_regen_test_fileinfo(object, Uuid::nil(), mod_time.unix_timestamp(), 0);
metadata.data_dir = None;
metadata.set_inline_data();
metadata.data = Some(Bytes::from_static(b"stale-inline-shard"));
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata).await;
(temp_dirs, set, disks)
}
async fn formatted_single_disk_no_parity_set() -> (TempDir, Arc<SetDisks>) {
let format = FormatV3::new(1, 1);
let dir = tempfile::tempdir().expect("tempdir should be created");
@@ -3068,6 +3078,109 @@ mod heal_result_report_tests {
);
}
#[tokio::test]
#[serial_test::serial]
async fn heal_reports_success_after_dangling_inline_cleanup() {
temp_env::async_with_vars([("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS", Some("0"))], async {
let bucket = "bucket-dangling-inline-cleanup";
let object = "stale.txt";
let (temp_dirs, set, _disks) =
dangling_inline_test_fixture(bucket, object, OffsetDateTime::now_utc() - time::Duration::hours(2)).await;
let (_, error) = set
.heal_object(
bucket,
object,
"",
&HealOpts {
no_lock: true,
..Default::default()
},
)
.await
.expect("dangling cleanup should complete");
assert!(error.is_none(), "successful dangling cleanup must not be reported as FileNotFound");
assert!(
temp_dirs.iter().all(|dir| !dir.path().join(bucket).join(object).exists()),
"successful dangling cleanup must remove the stale object from every disk"
);
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn heal_preserves_recent_dangling_inline_metadata_as_retryable() {
temp_env::async_with_vars([("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS", Some("3600"))], async {
let bucket = "bucket-dangling-inline-grace";
let object = "recent.txt";
let (temp_dirs, set, _disks) = dangling_inline_test_fixture(bucket, object, OffsetDateTime::now_utc()).await;
let (_, error) = set
.heal_object(
bucket,
object,
"",
&HealOpts {
no_lock: true,
..Default::default()
},
)
.await
.expect("grace-protected dangling metadata should return a typed heal result");
assert_eq!(error, Some(DiskError::ErasureReadQuorum));
assert!(
temp_dirs[0]
.path()
.join(bucket)
.join(object)
.join(STORAGE_FORMAT_FILE)
.is_file(),
"grace-protected metadata must remain for a later heal pass"
);
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn dry_run_never_deletes_dangling_inline_metadata() {
temp_env::async_with_vars([("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS", Some("0"))], async {
let bucket = "bucket-dangling-inline-dry-run";
let object = "stale.txt";
let (temp_dirs, set, _disks) =
dangling_inline_test_fixture(bucket, object, OffsetDateTime::now_utc() - time::Duration::hours(2)).await;
let (_, error) = set
.heal_object(
bucket,
object,
"",
&HealOpts {
dry_run: true,
no_lock: true,
..Default::default()
},
)
.await
.expect("dry-run should report the dangling candidate without mutation");
assert_eq!(error, Some(DiskError::FileNotFound));
assert!(
temp_dirs[0]
.path()
.join(bucket)
.join(object)
.join(STORAGE_FORMAT_FILE)
.is_file(),
"dry-run must preserve dangling metadata even when grace is disabled"
);
})
.await;
}
#[tokio::test]
async fn heal_meta_quorum_failure_preserves_candidate_when_required_shard_disk_is_offline() {
let bucket = "bucket-delete-guard-offline";
crates/ecstore/src/store/heal.rs
+88
View File
@@ -168,6 +168,60 @@ impl ECStore {
}
}
/// Return every live erasure set selected by an object-heal scope.
pub async fn heal_erasure_set_scopes(&self, opts: &HealOpts) -> Result<Vec<(usize, usize)>> {
let pools = self.get_pools_for_heal_object(opts)?;
let pool_meta = self.pool_meta.read().await;
let mut scopes = Vec::new();
for pool in pools {
let suspended_complete = pool_meta.is_suspended(pool.pool_idx).then(|| {
pool_meta
.pools
.get(pool.pool_idx)
.and_then(|status| status.decommission.as_ref())
.is_some_and(|decommission| decommission.complete)
});
if let Some(complete) = suspended_complete {
if opts.pool.is_some() {
return Err(if complete {
StorageError::InvalidArgument(
"heal".to_string(),
"pool".to_string(),
format!("heal pool {} has completed decommission", pool.pool_idx),
)
} else {
Error::SlowDown
});
}
continue;
}
if let Some(set_idx) = opts.set {
if set_idx >= pool.disk_set.len() {
return Err(StorageError::InvalidArgument(
"heal".to_string(),
"set".to_string(),
format!(
"invalid heal set index {set_idx} for pool {} with {} sets",
pool.pool_idx,
pool.disk_set.len()
),
));
}
scopes.push((pool.pool_idx, set_idx));
} else {
scopes.extend((0..pool.disk_set.len()).map(|set_idx| (pool.pool_idx, set_idx)));
}
}
if scopes.is_empty() {
return Err(Error::SlowDown);
}
Ok(scopes)
}
#[instrument(skip(self))]
pub(super) async fn handle_heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
let mut r = HealResultItem {
@@ -642,6 +696,40 @@ mod tests {
}
}
#[tokio::test]
async fn heal_erasure_set_scopes_follow_requested_pool_and_set() {
let store = minimal_heal_store().await;
assert_eq!(
store
.heal_erasure_set_scopes(&HealOpts::default())
.await
.expect("unscoped heal should enumerate every live set"),
vec![(0, 0), (1, 0)]
);
assert_eq!(
store
.heal_erasure_set_scopes(&HealOpts {
pool: Some(1),
set: Some(0),
..Default::default()
})
.await
.expect("scoped heal should enumerate only its requested set"),
vec![(1, 0)]
);
let err = store
.heal_erasure_set_scopes(&HealOpts {
pool: Some(0),
set: Some(1),
..Default::default()
})
.await
.expect_err("an invalid set scope must fail closed");
assert!(matches!(err, Error::InvalidArgument(..)));
}
fn pool_meta_with_decommission(info: PoolDecommissionInfo) -> PoolMeta {
PoolMeta {
pools: vec![PoolStatus {