gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-03 18:55:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): back off clean idle scans across erasure clusters (#4984)

Browse Source 
* fix(scanner): back off clean single-disk cycles

* fix(scanner): extend idle backoff across erasure clusters

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-07-18 10:49:45 +08:00
committed by GitHub
parent 5ec124bf23
commit 889a45ad4d
23 changed files with 3542 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/store/bucket.rs
+30
View File
@@ -392,16 +392,28 @@ mod tests {
}
async fn create_bucket_with_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
let generation_before_make = ecstore.scanner_namespace_mutation_generation();
ecstore
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_make.saturating_add(1),
"successful bucket creation should advance scanner namespace activity"
);
let generation_before_put = ecstore.scanner_namespace_mutation_generation();
let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
ecstore
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("object should be written");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_put.saturating_add(1),
"successful object creation should advance scanner namespace activity"
);
ecstore
.get_object_info(bucket, object, &ObjectOptions::default())
.await
@@ -499,6 +511,7 @@ mod tests {
create_bucket_with_object(&ecstore, &bucket, object).await;
assert!(metadata_sys::get(&bucket).await.is_ok());
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
ecstore
.delete_bucket(
&bucket,
@@ -509,6 +522,11 @@ mod tests {
)
.await
.expect("MarkDelete should not reject non-empty bucket data");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_delete.saturating_add(1),
"successful bucket deletion should advance scanner namespace activity"
);
assert!(
any_disk_has_object_metadata(&disk_paths, &bucket).await,
@@ -536,6 +554,7 @@ mod tests {
write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
assert!(any_disk_path_exists(&disk_paths, &metadata_prefix).await);
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
ecstore
.delete_bucket(
&bucket,
@@ -547,6 +566,11 @@ mod tests {
)
.await
.expect("Purge should force-delete bucket data");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_delete.saturating_add(1),
"successful bucket purge should advance scanner namespace activity"
);
assert!(!any_disk_path_exists(&disk_paths, &bucket).await, "Purge should remove the bucket volume");
assert!(
@@ -568,12 +592,18 @@ mod tests {
create_bucket_with_object(&ecstore, &bucket, object).await;
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
let err = ecstore
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect_err("default S3 DeleteBucket should reject non-empty buckets");
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_delete,
"failed bucket deletion must not advance scanner namespace activity"
);
assert!(
any_disk_has_object_metadata(&disk_paths, &bucket).await,
"failed default S3 DeleteBucket must keep object data"
crates/ecstore/src/store/list_objects.rs
+37 -7
View File
@@ -541,6 +541,7 @@ impl PersistentListMetadataObject {
static PERSISTENT_KEY_ONLY_INDEX_CACHE: OnceCell<RwLock<Option<PersistentKeyOnlyIndexCache>>> = OnceCell::const_new();
static LIST_OBJECTS_NAMESPACE_JOURNAL_LOCK: OnceCell<RwLock<()>> = OnceCell::const_new();
static LIST_OBJECTS_MUTATION_SEQUENCE: AtomicU64 = AtomicU64::new(0);
static SCANNER_NAMESPACE_MUTATION_GENERATION: AtomicU64 = AtomicU64::new(0);
static LIST_OBJECTS_BUCKET_MUTATION_SEQUENCE: OnceCell<RwLock<HashMap<String, u64>>> = OnceCell::const_new();
static LIST_OBJECTS_NAMESPACE_JOURNAL_DEGRADED_BUCKETS: OnceCell<RwLock<HashSet<String>>> = OnceCell::const_new();
static LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_CONFIG: OnceCell<Option<NamespaceMutationJournalChaosConfig>> = OnceCell::const_new();
@@ -607,6 +608,19 @@ async fn advance_list_objects_mutation_sequence(bucket: &str, sequence: u64) ->
sequence
}
pub(super) fn scanner_namespace_mutation_generation() -> u64 {
SCANNER_NAMESPACE_MUTATION_GENERATION.load(Ordering::Acquire)
}
pub(super) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) {
if bucket == RUSTFS_META_BUCKET {
return;
}
let _ = SCANNER_NAMESPACE_MUTATION_GENERATION
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(delta)));
}
pub(super) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 {
observe_list_objects_mutations(store, bucket, 1).await.unwrap_or_default()
}
@@ -621,6 +635,7 @@ async fn observe_list_objects_mutations_with_store(store: Option<&ECStore>, buck
}
let delta = u64::try_from(count).unwrap_or(u64::MAX);
observe_scanner_namespace_mutations(bucket, delta);
let next = LIST_OBJECTS_MUTATION_SEQUENCE
.fetch_add(delta, Ordering::AcqRel)
.saturating_add(delta);
@@ -641,6 +656,7 @@ async fn current_list_objects_mutation_sequence(bucket: &str) -> u64 {
#[cfg(test)]
async fn reset_list_objects_mutation_sequences_for_test() {
LIST_OBJECTS_MUTATION_SEQUENCE.store(0, Ordering::Release);
SCANNER_NAMESPACE_MUTATION_GENERATION.store(0, Ordering::Release);
let sequences = list_objects_bucket_mutation_sequence().await;
sequences.write().await.clear();
let degraded = list_objects_namespace_journal_degraded_buckets().await;
@@ -6646,11 +6662,11 @@ mod test {
ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend,
NamespaceMutationJournalSnapshot, NamespaceMutationJournalStatus, PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER,
PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER,
PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex, PersistentListMetadataObject, VerifiedIndexCandidateStats,
VersionMarker, current_list_objects_mutation_sequence, encode_persistent_list_metadata_object,
enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results,
latest_listing_allow_agreed_objects, latest_listing_object_quorum, latest_listing_raw_min_disks,
latest_listing_required_object_quorum, list_marker_key, list_metadata_resolution_params,
PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex, PersistentListMetadataObject, RUSTFS_META_BUCKET,
VerifiedIndexCandidateStats, VersionMarker, current_list_objects_mutation_sequence,
encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum,
fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects, latest_listing_object_quorum,
latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key, list_metadata_resolution_params,
list_objects_from_metadata_snapshot_candidates, list_objects_from_verified_index_candidates,
list_objects_from_verified_index_candidates_with_optional_stats, list_objects_from_verified_index_candidates_with_stats,
list_objects_index_mode_from_env, list_objects_index_provider_from_env, list_objects_index_provider_state_from_env,
@@ -6664,8 +6680,9 @@ mod test {
parse_version_marker, persist_observed_list_objects_mutation, persistent_key_only_index_has_complete_metadata_snapshot,
persistent_key_only_index_health, persistent_key_only_index_matches_provider, record_list_objects_index_opt_in_fallback,
reset_list_objects_mutation_sequences_for_test, resolve_agreed_listing_entry, resolve_listing_entries,
select_list_index_provider_source_mode, select_list_index_source_mode, send_or_cancel, version_marker_for_entries,
walk_result_from_set_errors, write_namespace_mutation_journal_state, write_persistent_key_only_index,
scanner_namespace_mutation_generation, select_list_index_provider_source_mode, select_list_index_source_mode,
send_or_cancel, version_marker_for_entries, walk_result_from_set_errors, write_namespace_mutation_journal_state,
write_persistent_key_only_index,
};
use crate::cache_value::metacache_set::{FallbackClaimTracker, TestReaderBehavior, list_path_raw};
use crate::disk::{DiskAPI, DiskOption, endpoint::Endpoint, error::DiskError, new_disk};
@@ -8362,6 +8379,19 @@ mod test {
assert_eq!(current_list_objects_mutation_sequence("bucket-b").await, 0);
}
#[tokio::test]
#[serial_test::serial]
async fn scanner_namespace_generation_tracks_only_user_namespace_mutations() {
reset_list_objects_mutation_sequences_for_test().await;
assert_eq!(scanner_namespace_mutation_generation(), 0);
assert_eq!(observe_list_objects_mutations_with_store(None, RUSTFS_META_BUCKET, 3).await, Some(3));
assert_eq!(scanner_namespace_mutation_generation(), 0);
assert_eq!(observe_list_objects_mutations_with_store(None, "photos", 2).await, Some(5));
assert_eq!(scanner_namespace_mutation_generation(), 2);
}
#[test]
fn list_objects_index_provider_state_uses_lifecycle_active_generation() {
let provider = ListObjectsIndexProviderState::from_kind(ListObjectsIndexProviderKind::WalkerKeyOnly);
crates/ecstore/src/store/mod.rs
+14 -2
View File
@@ -318,6 +318,10 @@ impl ECStore {
pub async fn setup_is_erasure_sd(&self) -> bool {
self.ctx.is_erasure_sd().await
}
pub fn scanner_namespace_mutation_generation(&self) -> u64 {
list_objects::scanner_namespace_mutation_generation()
}
}
// impl Clone for ECStore {
@@ -436,7 +440,11 @@ impl BucketOperations for ECStore {
#[instrument(skip(self))]
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
self.handle_make_bucket(bucket, opts).await
let result = self.handle_make_bucket(bucket, opts).await;
if result.is_ok() {
list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
result
}
#[instrument(skip(self))]
@@ -449,7 +457,11 @@ impl BucketOperations for ECStore {
}
#[instrument(skip(self))]
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
self.handle_delete_bucket(bucket, opts).await
let result = self.handle_delete_bucket(bucket, opts).await;
if result.is_ok() {
list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
result
}
}