gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 05:06:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): bound orphan heal discovery fallback

Browse Source
This commit is contained in:
马登山
2026-08-22 19:35:56 +08:00
parent 27a921372c
commit f40f8179a4
5 changed files with 272 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner_folder.rs
+78 -12
View File
@@ -43,7 +43,10 @@ use rustfs_common::metrics::{
UpdateCurrentPathFn, current_path_updater, global_metrics,
};
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit, trace_subscriber_count};
use rustfs_filemeta::{MAX_META_CACHE_HEAL_CANDIDATES, MetaCacheEntries, MetaCacheEntry, MetaCacheHealCandidateKind};
use rustfs_filemeta::{
MAX_META_CACHE_HEAL_CANDIDATES, MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS, MetaCacheEntries, MetaCacheEntry,
MetaCacheHealCandidateKind,
};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, VersioningConfiguration};
use time::OffsetDateTime;
@@ -100,6 +103,7 @@ const METRIC_SCANNER_HEAL_DISCOVERY_CANDIDATES_TOTAL: &str = "rustfs_scanner_hea
const METRIC_SCANNER_HEAL_DISCOVERY_SUB_QUORUM_TOTAL: &str = "rustfs_scanner_heal_discovery_sub_quorum_total";
const METRIC_SCANNER_HEAL_DISCOVERY_UNVERIFIED_TOTAL: &str = "rustfs_scanner_heal_discovery_unverified_total";
const METRIC_SCANNER_HEAL_DISCOVERY_QUEUED_TOTAL: &str = "rustfs_scanner_heal_discovery_queued_total";
const METRIC_SCANNER_HEAL_DISCOVERY_TRUNCATED_TOTAL: &str = "rustfs_scanner_heal_discovery_truncated_total";
const MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET: usize = 128;
// --- scanner excess alerts as S3 notification events (rustfs/backlog#1868) --
@@ -1877,6 +1881,7 @@ impl FolderScanner {
let mut partial_closed = false;
let mut finished_closed = false;
let mut seen_heal_candidates: HashSet<(String, Option<String>, MetaCacheHealCandidateKind)> = HashSet::new();
let mut seen_truncated_objects: HashSet<String> = HashSet::new();
loop {
if agreed_closed && partial_closed && finished_closed {
@@ -1917,8 +1922,12 @@ impl FolderScanner {
counter!(METRIC_SCANNER_HEAL_DISCOVERY_UNVERIFIED_TOTAL).increment(
u64::try_from(discovery.unverified_count).unwrap_or(u64::MAX),
);
if discovery.truncated {
counter!(METRIC_SCANNER_HEAL_DISCOVERY_TRUNCATED_TOTAL).increment(1);
}
for candidate in discovery.candidates {
let sub_quorum_candidate = candidate.replica_count < disks_quorum;
let version_id = candidate.validated_version().map(|id| id.to_string());
let identity = (candidate.object.clone(), version_id.clone(), candidate.kind.clone());
if seen_heal_candidates.len() >= MAX_META_CACHE_HEAL_CANDIDATES
@@ -1929,25 +1938,82 @@ impl FolderScanner {
if !seen_heal_candidates.insert(identity) {
continue;
}
let mut request = build_object_heal_request(
bucket.clone(),
candidate.object.clone(),
version_id.clone(),
self.scan_mode,
HealChannelPriority::High,
);
if candidate.is_unversioned() {
request.remove_corrupted = Some(false);
}
let request = if candidate.is_unversioned() {
build_non_destructive_object_heal_request(
bucket.clone(),
candidate.object.clone(),
self.scan_mode,
HealChannelPriority::High,
)
} else {
build_object_heal_request(
bucket.clone(),
candidate.object.clone(),
version_id.clone(),
self.scan_mode,
HealChannelPriority::High,
)
};
(self.update_current_path)(&candidate.object).await;
let admission = self.send_required_scanner_heal_request(
PendingScannerHealKind::Object,
bucket.clone(),
Some(candidate.object.clone()),
version_id,
version_id.clone(),
request,
)
.await?;
if admission.is_admitted() {
counter!(METRIC_SCANNER_HEAL_DISCOVERY_QUEUED_TOTAL).increment(1);
} else if sub_quorum_candidate {
self.mark_pending_scanner_heal_reason(
PendingScannerHealKind::Object,
&bucket,
Some(&candidate.object),
version_id.as_deref(),
"sub_quorum_metadata",
);
}
found_objects = true;
}
// A bounded candidate union may overflow for an
// object with a very long version history. Keep
// that overflow explicit and issue one safe,
// versionless inspection request per object so
// the dropped versions are not silently treated
// as absent. This continuation is deliberately
// outside the versioned candidate cap and always
// disables destructive cleanup.
for object in discovery.truncated_objects {
if seen_truncated_objects.len() >= MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS
&& !seen_truncated_objects.contains(&object)
{
continue;
}
if !seen_truncated_objects.insert(object.clone()) {
continue;
}
let identity = (object.clone(), None, MetaCacheHealCandidateKind::UnversionedObject);
if !seen_heal_candidates.insert(identity) {
continue;
}
let request = build_non_destructive_object_heal_request(
bucket.clone(),
object.clone(),
self.scan_mode,
HealChannelPriority::High,
);
(self.update_current_path)(&object).await;
let admission = self
.send_required_scanner_heal_request(
PendingScannerHealKind::Object,
bucket.clone(),
Some(object.clone()),
None,
request,
)
.await?;
if admission.is_admitted() {
counter!(METRIC_SCANNER_HEAL_DISCOVERY_QUEUED_TOTAL).increment(1);
}
crates/scanner/src/scanner_folder/item_actions.rs
+15
View File
@@ -90,6 +90,21 @@ pub(super) fn build_object_heal_request(
}
}
/// Build the versionless inspection request used when discovery cannot prove
/// a destructive version identity (for example an unversioned object or a
/// bounded candidate overflow). The explicit flag is the fail-closed safety
/// boundary; callers must not reconstruct it with the destructive default.
pub(super) fn build_non_destructive_object_heal_request(
bucket: String,
object: String,
scan_mode: HealScanMode,
priority: HealChannelPriority,
) -> HealChannelRequest {
let mut request = build_object_heal_request(bucket, object, None, scan_mode, priority);
request.remove_corrupted = Some(false);
request
}
#[cfg(test)]
pub(super) fn resolve_object_heal_entry(
entries: &MetaCacheEntries,
crates/scanner/src/scanner_folder/ledger.rs
+37 -9
View File
@@ -105,6 +105,29 @@ impl FolderScanner {
}
}
/// Preserve the discovery reason when a candidate could not be admitted
/// immediately. The existing string field is intentionally reused so the
/// scanner's map-encoded cache schema stays backward compatible.
pub(super) fn mark_pending_scanner_heal_reason(
&mut self,
kind: PendingScannerHealKind,
bucket: &str,
object: Option<&str>,
version_id: Option<&str>,
reason: &str,
) {
if let Some(entry) = self
.new_cache
.info
.pending_heals
.iter_mut()
.find(|entry| pending_scanner_heal_matches(entry, kind, bucket, object, version_id))
{
entry.last_admission_reason = reason.to_string();
self.sync_pending_heals();
}
}
pub(super) fn prune_pending_scanner_heals(&mut self) {
let now = Self::now_secs();
let before_expiry = self.new_cache.info.pending_heals.len();
@@ -305,17 +328,22 @@ pub(super) fn build_pending_scanner_heal_request(entry: &PendingScannerHeal) ->
match entry.kind {
PendingScannerHealKind::Bucket => Some(build_bucket_heal_request(entry.bucket.clone(), HealChannelPriority::High)),
PendingScannerHealKind::Object => entry.object.as_ref().map(|object| {
let mut request = build_object_heal_request(
entry.bucket.clone(),
object.clone(),
entry.version_id.clone(),
entry.scan_mode,
HealChannelPriority::High,
);
if entry.version_id.is_none() {
request.remove_corrupted = Some(false);
build_non_destructive_object_heal_request(
entry.bucket.clone(),
object.clone(),
entry.scan_mode,
HealChannelPriority::High,
)
} else {
build_object_heal_request(
entry.bucket.clone(),
object.clone(),
entry.version_id.clone(),
entry.scan_mode,
HealChannelPriority::High,
)
}
request
}),
}
}
crates/scanner/src/scanner_folder/tests.rs
+40
View File
@@ -982,6 +982,21 @@ fn test_build_object_heal_request_omits_nil_version_id() {
assert_eq!(request.recreate_missing, Some(false));
}
#[test]
fn test_build_non_destructive_object_heal_request_disables_removal() {
let request = build_non_destructive_object_heal_request(
"bucket".to_string(),
"path/to/object".to_string(),
HealScanMode::Deep,
HealChannelPriority::High,
);
assert_eq!(request.object_version_id, None);
assert_eq!(request.remove_corrupted, Some(false));
assert_eq!(request.recreate_missing, Some(false));
assert_eq!(request.source, HealRequestSource::Scanner);
}
#[test]
fn test_build_bucket_heal_request_disables_recreate_for_scanner() {
let request = build_bucket_heal_request("bucket".to_string(), HealChannelPriority::Low);
@@ -1132,6 +1147,31 @@ fn test_pending_heal_reconstructs_unversioned_request_without_removal() {
assert_eq!(request.recreate_missing, Some(false));
}
#[tokio::test]
async fn test_pending_heal_reason_preserves_sub_quorum_discovery() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
scanner.update_pending_scanner_heal_after_admission(
PendingScannerHealKind::Object,
"bucket",
Some("object"),
Some("version-a"),
HealScanMode::Deep,
HealAdmissionResult::Full,
);
scanner.mark_pending_scanner_heal_reason(
PendingScannerHealKind::Object,
"bucket",
Some("object"),
Some("version-a"),
"sub_quorum_metadata",
);
assert_eq!(scanner.new_cache.info.pending_heals.len(), 1);
assert_eq!(scanner.new_cache.info.pending_heals[0].last_admission_reason, "sub_quorum_metadata");
}
#[test]
fn test_pending_heal_retry_candidates_respect_cap_and_order() {
let pending: Vec<PendingScannerHeal> = (0..(MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET + 2))