From 889a45ad4d21e741768cc408ea2b3172788359f6 Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Sat, 18 Jul 2026 10:49:45 +0800 Subject: [PATCH] fix(scanner): back off clean idle scans across erasure clusters (#4984) * fix(scanner): back off clean single-disk cycles * fix(scanner): extend idle backoff across erasure clusters --------- Co-authored-by: Henry Guo --- crates/common/src/metrics.rs | 46 + .../e2e_test/src/reliant/grpc_lock_server.rs | 7 + crates/ecstore/src/api/mod.rs | 2 +- crates/ecstore/src/cluster/rpc/mod.rs | 1 + .../src/cluster/rpc/peer_rest_client.rs | 95 +- .../ecstore/src/services/notification_sys.rs | 124 +- crates/ecstore/src/store/bucket.rs | 30 + crates/ecstore/src/store/list_objects.rs | 44 +- crates/ecstore/src/store/mod.rs | 16 +- .../src/generated/proto_gen/node_service.rs | 60 + crates/protos/src/node.proto | 10 + crates/scanner/src/lib.rs | 7 +- crates/scanner/src/runtime_config.rs | 76 +- crates/scanner/src/scanner.rs | 2471 ++++++++++++++++- crates/scanner/src/scanner_io.rs | 460 ++- docs/operations/scanner-runtime-controls.md | 40 +- rustfs/src/admin/handlers/scanner.rs | 71 +- rustfs/src/admin/storage_api.rs | 4 +- rustfs/src/app/bucket_usecase.rs | 117 +- rustfs/src/app/storage_api.rs | 4 +- rustfs/src/storage/rpc/node_service.rs | 60 +- rustfs/src/storage/rpc/node_service/bucket.rs | 6 +- rustfs/src/storage/storage_api.rs | 32 +- 23 files changed, 3542 insertions(+), 241 deletions(-) diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index aea19129a..0f218c8c3 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -2434,6 +2434,18 @@ impl Metrics { self.current_scan_cycle_work_active.store(false, Ordering::Relaxed); } + pub fn current_scan_cycle_has_unresolved_heal_work(&self) -> bool { + if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) { + return false; + } + + let source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values()); + [ScannerWorkSource::Heal, ScannerWorkSource::Bitrot] + .into_iter() + .filter_map(|source| source_work.get(source.index())) + .any(|work| work.queued > 0 || work.skipped > 0 || work.failed > 0 || work.missed > 0) + } + fn scan_cycle_work_snapshot(&self) -> ScanCycleWorkSnapshot { ScanCycleWorkSnapshot { objects_scanned: self.lifetime(Metric::ScanObject), @@ -3361,6 +3373,40 @@ mod tests { metrics.finish_scan_cycle_work(start); } + #[test] + fn unresolved_heal_work_only_reflects_the_active_cycle() { + let metrics = Metrics::new(); + assert!(!metrics.current_scan_cycle_has_unresolved_heal_work()); + + let start = metrics.start_scan_cycle_work(); + metrics.record_scanner_source_missed(ScannerWorkSource::Heal, 1); + assert!(metrics.current_scan_cycle_has_unresolved_heal_work()); + + metrics.finish_scan_cycle_work(start); + assert!(!metrics.current_scan_cycle_has_unresolved_heal_work()); + + let start = metrics.start_scan_cycle_work(); + metrics.record_scanner_source_queued(ScannerWorkSource::Heal, 1); + assert!(metrics.current_scan_cycle_has_unresolved_heal_work()); + metrics.finish_scan_cycle_work(start); + + let start = metrics.start_scan_cycle_work(); + metrics.record_scanner_source_work( + ScannerWorkSource::Bitrot, + ScannerSourceWorkUpdate { + skipped: 1, + ..Default::default() + }, + ); + assert!(metrics.current_scan_cycle_has_unresolved_heal_work()); + metrics.finish_scan_cycle_work(start); + + let start = metrics.start_scan_cycle_work(); + metrics.record_scanner_source_failed(ScannerWorkSource::Bitrot, 1); + assert!(metrics.current_scan_cycle_has_unresolved_heal_work()); + metrics.finish_scan_cycle_work(start); + } + #[tokio::test] async fn report_marks_transition_failures_as_blocked_lifecycle_control() { let metrics = Metrics::new(); diff --git a/crates/e2e_test/src/reliant/grpc_lock_server.rs b/crates/e2e_test/src/reliant/grpc_lock_server.rs index 97b92c383..2525be58b 100644 --- a/crates/e2e_test/src/reliant/grpc_lock_server.rs +++ b/crates/e2e_test/src/reliant/grpc_lock_server.rs @@ -798,6 +798,13 @@ impl NodeService for MinimalLockNodeService { Err(Status::unimplemented("lock-only test server")) } + async fn scanner_activity( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("lock-only test server")) + } + async fn background_heal_status( &self, _request: Request, diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 8c9d26431..c72758f5a 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -376,7 +376,7 @@ pub mod rio { pub mod rpc { pub use crate::cluster::rpc::{ LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG, - SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, TonicInterceptor, gen_tonic_signature_interceptor, + SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth, verify_rpc_signature, }; } diff --git a/crates/ecstore/src/cluster/rpc/mod.rs b/crates/ecstore/src/cluster/rpc/mod.rs index f6f5ec284..3d7ed80f1 100644 --- a/crates/ecstore/src/cluster/rpc/mod.rs +++ b/crates/ecstore/src/cluster/rpc/mod.rs @@ -34,6 +34,7 @@ pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport; pub use internode_data_transport::build_internode_data_transport_from_env; pub use peer_rest_client::{ PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, + ScannerPeerActivity, }; pub(crate) use peer_s3_client::heal_bucket_local_on_disks; pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys}; diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 05897f49f..005e346f3 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -34,9 +34,9 @@ use rustfs_protos::proto_gen::node_service::{ GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest, GetSysConfigRequest, GetSysErrorsRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, - LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ServerInfoRequest, - SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, - node_service_client::NodeServiceClient, + LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ScannerActivityRequest, + ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest, + StopRebalanceRequest, node_service_client::NodeServiceClient, }; use rustfs_utils::XHost; use serde::{Deserialize, Serialize as _}; @@ -62,6 +62,31 @@ pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1; pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2; const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60; const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30); +const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScannerPeerActivity { + pub instance_id: String, + pub namespace_generation: u64, + pub maintenance_generation: u64, +} + +fn decode_scanner_activity(response: ScannerActivityResponse) -> Result { + let instance_id = response.instance_id; + if instance_id.len() != 32 + || !instance_id + .as_bytes() + .iter() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte)) + { + return Err(Error::other("peer returned an invalid scanner activity instance ID")); + } + Ok(ScannerPeerActivity { + instance_id, + namespace_generation: response.namespace_generation, + maintenance_generation: response.maintenance_generation, + }) +} #[derive(Clone, Debug)] pub struct PeerLiveEventsBatch { @@ -639,12 +664,13 @@ impl PeerRestClient { Err(Error::NotImplemented) } - pub async fn load_bucket_metadata(&self, bucket: &str) -> Result<()> { + pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> { self.finalize_result( async { let mut client = self.get_client().await?; let request = Request::new(LoadBucketMetadataRequest { bucket: bucket.to_string(), + scanner_maintenance_change, }); let response = client.load_bucket_metadata(request).await?.into_inner(); @@ -908,6 +934,25 @@ impl PeerRestClient { .await } + pub async fn scanner_activity(&self) -> Result { + self.finalize_result( + async { + let mut client = self + .get_client() + .await? + .max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE) + .max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE); + let response = client + .scanner_activity(Request::new(ScannerActivityRequest {})) + .await? + .into_inner(); + decode_scanner_activity(response) + } + .await, + ) + .await + } + pub async fn get_metacache_listing(&self) -> Result<()> { warn!("get_metacache_listing is not implemented in PeerRestClient"); Err(Error::NotImplemented) @@ -1157,6 +1202,48 @@ mod tests { ) } + #[test] + fn scanner_activity_requires_restart_safe_peer_identity() { + let missing_instance = ScannerActivityResponse { + instance_id: String::new(), + namespace_generation: 7, + maintenance_generation: 3, + }; + assert!( + decode_scanner_activity(missing_instance) + .expect_err("an empty instance ID is not restart safe") + .to_string() + .contains("instance ID") + ); + + let malformed_instance = ScannerActivityResponse { + instance_id: "ABCDEF0123456789ABCDEF0123456789".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + }; + assert!( + decode_scanner_activity(malformed_instance) + .expect_err("activity instance IDs must use the canonical lowercase hex form") + .to_string() + .contains("instance ID") + ); + + let activity = decode_scanner_activity(ScannerActivityResponse { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + }) + .expect("complete activity responses should be accepted"); + assert_eq!( + activity, + ScannerPeerActivity { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + } + ); + } + #[test] fn peer_rest_client_marks_network_like_errors() { assert!(PeerRestClient::is_network_like_error(&Error::other("transport error"))); diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index 3555159d5..7560aca9e 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::cluster::rpc::PeerRestClient; +use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity}; use crate::diagnostics::admin_server_info::get_commit_id; use crate::disk::DiskAPI; use crate::error::{Error, Result}; @@ -41,6 +41,7 @@ const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification"; const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation"; +const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5); /// Cached result from the last successful admin call to a peer. struct PeerAdminCache { @@ -91,7 +92,6 @@ pub fn get_global_notification_sys() -> Option> { pub struct NotificationSys { pub peer_clients: Vec>, - #[allow(dead_code)] pub all_peer_clients: Vec>, peer_admin_caches: Vec>, } @@ -756,6 +756,14 @@ impl NotificationSys { } pub async fn load_bucket_metadata(&self, bucket: &str) -> Result<()> { + self.load_bucket_metadata_with_scanner_maintenance(bucket, false).await + } + + pub async fn load_bucket_metadata_for_scanner_maintenance(&self, bucket: &str) -> Result<()> { + self.load_bucket_metadata_with_scanner_maintenance(bucket, true).await + } + + async fn load_bucket_metadata_with_scanner_maintenance(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> { let operation = format!("load_bucket_metadata({bucket})"); let mut failures = Vec::new(); let mut futures = Vec::with_capacity(self.peer_clients.len()); @@ -763,7 +771,12 @@ impl NotificationSys { if let Some(client) = client { let host = client.host.to_string(); let b = bucket.to_string(); - futures.push(async move { client.load_bucket_metadata(&b).await.map_err(|err| (host, err)) }); + futures.push(async move { + client + .load_bucket_metadata(&b, scanner_maintenance_change) + .await + .map_err(|err| (host, err)) + }); } else { failures.push(format!("peer[{idx}] {operation} failed: peer is not reachable")); } @@ -976,6 +989,36 @@ impl NotificationSys { join_all(futures).await } + pub async fn scanner_activity_snapshots(&self) -> Result> { + if self.peer_clients.is_empty() { + return Err(Error::other("scanner activity probe has no remote peers")); + } + if self.all_peer_clients.len() != self.peer_clients.len() + 1 { + return Err(Error::other(format!( + "scanner activity peer topology is incomplete: {} remote peers for {} cluster members", + self.peer_clients.len(), + self.all_peer_clients.len() + ))); + } + + let mut futures = Vec::with_capacity(self.peer_clients.len()); + for (idx, client) in self.peer_clients.iter().cloned().enumerate() { + futures.push(async move { + let client = client.ok_or_else(|| Error::other(format!("scanner activity peer[{idx}] is unreachable")))?; + let host = client.grid_host.clone(); + scanner_activity_with_timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, &host, client.scanner_activity()) + .await + .map(|activity| (host, activity)) + }); + } + + let mut generations = Vec::with_capacity(futures.len()); + for result in join_all(futures).await { + generations.push(result?); + } + Ok(generations) + } + pub async fn reload_site_replication_config(&self) -> Vec { let mut futures = Vec::with_capacity(self.peer_clients.len()); for client in self.peer_clients.iter() { @@ -1029,6 +1072,15 @@ impl NotificationSys { } } +async fn scanner_activity_with_timeout(timeout_duration: Duration, host: &str, activity: F) -> Result +where + F: Future>, +{ + timeout(timeout_duration, activity) + .await + .map_err(|_| Error::other(format!("scanner activity peer {host} timed out after {timeout_duration:?}")))? +} + async fn call_peer_with_timeout( timeout_dur: Duration, host_label: &str, @@ -1568,6 +1620,72 @@ mod tests { assert!(msg.contains("peer[0]")); } + #[tokio::test] + async fn scanner_activity_probe_reports_unreachable_peers() { + let sys = NotificationSys { + peer_clients: vec![None], + all_peer_clients: vec![None, None], + peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())], + }; + + let err = sys + .scanner_activity_snapshots() + .await + .expect_err("unreachable peers must disable scanner idle backoff"); + + assert!(err.to_string().contains("scanner activity peer[0] is unreachable")); + } + + #[tokio::test] + async fn scanner_activity_probe_rejects_an_empty_peer_set() { + let sys = NotificationSys { + peer_clients: Vec::new(), + all_peer_clients: Vec::new(), + peer_admin_caches: Vec::new(), + }; + + let err = sys + .scanner_activity_snapshots() + .await + .expect_err("a missing peer set must disable scanner idle backoff"); + + assert!(err.to_string().contains("no remote peers")); + } + + #[tokio::test] + async fn scanner_activity_probe_rejects_an_incomplete_peer_topology() { + let client = PeerRestClient::new( + "127.0.0.1:9000".to_string().try_into().expect("peer host should parse"), + "http://127.0.0.1:9000".to_string(), + ); + let sys = NotificationSys { + peer_clients: vec![Some(client)], + all_peer_clients: vec![None], + peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())], + }; + + let err = sys + .scanner_activity_snapshots() + .await + .expect_err("an incomplete peer topology must disable scanner idle backoff"); + + assert!(err.to_string().contains("peer topology is incomplete")); + } + + #[tokio::test] + async fn scanner_activity_probe_times_out() { + let err = scanner_activity_with_timeout( + Duration::from_millis(5), + "peer-1", + std::future::pending::>(), + ) + .await + .expect_err("a stalled peer must not block scanner scheduling"); + + assert!(err.to_string().contains("timed out")); + assert!(err.to_string().contains("peer-1")); + } + #[tokio::test] async fn load_bucket_metadata_reports_unreachable_peers() { let sys = NotificationSys { diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 9a28d1905..04b3aea33 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -392,16 +392,28 @@ mod tests { } async fn create_bucket_with_object(ecstore: &Arc, bucket: &str, object: &str) { + let generation_before_make = ecstore.scanner_namespace_mutation_generation(); ecstore .make_bucket(bucket, &MakeBucketOptions::default()) .await .expect("bucket should be created"); + assert_eq!( + ecstore.scanner_namespace_mutation_generation(), + generation_before_make.saturating_add(1), + "successful bucket creation should advance scanner namespace activity" + ); + let generation_before_put = ecstore.scanner_namespace_mutation_generation(); let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec()); ecstore .put_object(bucket, object, &mut reader, &ObjectOptions::default()) .await .expect("object should be written"); + assert_eq!( + ecstore.scanner_namespace_mutation_generation(), + generation_before_put.saturating_add(1), + "successful object creation should advance scanner namespace activity" + ); ecstore .get_object_info(bucket, object, &ObjectOptions::default()) .await @@ -499,6 +511,7 @@ mod tests { create_bucket_with_object(&ecstore, &bucket, object).await; assert!(metadata_sys::get(&bucket).await.is_ok()); + let generation_before_delete = ecstore.scanner_namespace_mutation_generation(); ecstore .delete_bucket( &bucket, @@ -509,6 +522,11 @@ mod tests { ) .await .expect("MarkDelete should not reject non-empty bucket data"); + assert_eq!( + ecstore.scanner_namespace_mutation_generation(), + generation_before_delete.saturating_add(1), + "successful bucket deletion should advance scanner namespace activity" + ); assert!( any_disk_has_object_metadata(&disk_paths, &bucket).await, @@ -536,6 +554,7 @@ mod tests { write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await; assert!(any_disk_path_exists(&disk_paths, &metadata_prefix).await); + let generation_before_delete = ecstore.scanner_namespace_mutation_generation(); ecstore .delete_bucket( &bucket, @@ -547,6 +566,11 @@ mod tests { ) .await .expect("Purge should force-delete bucket data"); + assert_eq!( + ecstore.scanner_namespace_mutation_generation(), + generation_before_delete.saturating_add(1), + "successful bucket purge should advance scanner namespace activity" + ); assert!(!any_disk_path_exists(&disk_paths, &bucket).await, "Purge should remove the bucket volume"); assert!( @@ -568,12 +592,18 @@ mod tests { create_bucket_with_object(&ecstore, &bucket, object).await; + let generation_before_delete = ecstore.scanner_namespace_mutation_generation(); let err = ecstore .delete_bucket(&bucket, &DeleteBucketOptions::default()) .await .expect_err("default S3 DeleteBucket should reject non-empty buckets"); assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket)); + assert_eq!( + ecstore.scanner_namespace_mutation_generation(), + generation_before_delete, + "failed bucket deletion must not advance scanner namespace activity" + ); assert!( any_disk_has_object_metadata(&disk_paths, &bucket).await, "failed default S3 DeleteBucket must keep object data" diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 3fe9af5d1..f0458cbe1 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -541,6 +541,7 @@ impl PersistentListMetadataObject { static PERSISTENT_KEY_ONLY_INDEX_CACHE: OnceCell>> = OnceCell::const_new(); static LIST_OBJECTS_NAMESPACE_JOURNAL_LOCK: OnceCell> = OnceCell::const_new(); static LIST_OBJECTS_MUTATION_SEQUENCE: AtomicU64 = AtomicU64::new(0); +static SCANNER_NAMESPACE_MUTATION_GENERATION: AtomicU64 = AtomicU64::new(0); static LIST_OBJECTS_BUCKET_MUTATION_SEQUENCE: OnceCell>> = OnceCell::const_new(); static LIST_OBJECTS_NAMESPACE_JOURNAL_DEGRADED_BUCKETS: OnceCell>> = OnceCell::const_new(); static LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_CONFIG: OnceCell> = OnceCell::const_new(); @@ -607,6 +608,19 @@ async fn advance_list_objects_mutation_sequence(bucket: &str, sequence: u64) -> sequence } +pub(super) fn scanner_namespace_mutation_generation() -> u64 { + SCANNER_NAMESPACE_MUTATION_GENERATION.load(Ordering::Acquire) +} + +pub(super) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) { + if bucket == RUSTFS_META_BUCKET { + return; + } + + let _ = SCANNER_NAMESPACE_MUTATION_GENERATION + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(delta))); +} + pub(super) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 { observe_list_objects_mutations(store, bucket, 1).await.unwrap_or_default() } @@ -621,6 +635,7 @@ async fn observe_list_objects_mutations_with_store(store: Option<&ECStore>, buck } let delta = u64::try_from(count).unwrap_or(u64::MAX); + observe_scanner_namespace_mutations(bucket, delta); let next = LIST_OBJECTS_MUTATION_SEQUENCE .fetch_add(delta, Ordering::AcqRel) .saturating_add(delta); @@ -641,6 +656,7 @@ async fn current_list_objects_mutation_sequence(bucket: &str) -> u64 { #[cfg(test)] async fn reset_list_objects_mutation_sequences_for_test() { LIST_OBJECTS_MUTATION_SEQUENCE.store(0, Ordering::Release); + SCANNER_NAMESPACE_MUTATION_GENERATION.store(0, Ordering::Release); let sequences = list_objects_bucket_mutation_sequence().await; sequences.write().await.clear(); let degraded = list_objects_namespace_journal_degraded_buckets().await; @@ -6646,11 +6662,11 @@ mod test { ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend, NamespaceMutationJournalSnapshot, NamespaceMutationJournalStatus, PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER, PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER, - PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex, PersistentListMetadataObject, VerifiedIndexCandidateStats, - VersionMarker, current_list_objects_mutation_sequence, encode_persistent_list_metadata_object, - enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results, - latest_listing_allow_agreed_objects, latest_listing_object_quorum, latest_listing_raw_min_disks, - latest_listing_required_object_quorum, list_marker_key, list_metadata_resolution_params, + PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex, PersistentListMetadataObject, RUSTFS_META_BUCKET, + VerifiedIndexCandidateStats, VersionMarker, current_list_objects_mutation_sequence, + encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum, + fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects, latest_listing_object_quorum, + latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key, list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates, list_objects_from_verified_index_candidates, list_objects_from_verified_index_candidates_with_optional_stats, list_objects_from_verified_index_candidates_with_stats, list_objects_index_mode_from_env, list_objects_index_provider_from_env, list_objects_index_provider_state_from_env, @@ -6664,8 +6680,9 @@ mod test { parse_version_marker, persist_observed_list_objects_mutation, persistent_key_only_index_has_complete_metadata_snapshot, persistent_key_only_index_health, persistent_key_only_index_matches_provider, record_list_objects_index_opt_in_fallback, reset_list_objects_mutation_sequences_for_test, resolve_agreed_listing_entry, resolve_listing_entries, - select_list_index_provider_source_mode, select_list_index_source_mode, send_or_cancel, version_marker_for_entries, - walk_result_from_set_errors, write_namespace_mutation_journal_state, write_persistent_key_only_index, + scanner_namespace_mutation_generation, select_list_index_provider_source_mode, select_list_index_source_mode, + send_or_cancel, version_marker_for_entries, walk_result_from_set_errors, write_namespace_mutation_journal_state, + write_persistent_key_only_index, }; use crate::cache_value::metacache_set::{FallbackClaimTracker, TestReaderBehavior, list_path_raw}; use crate::disk::{DiskAPI, DiskOption, endpoint::Endpoint, error::DiskError, new_disk}; @@ -8362,6 +8379,19 @@ mod test { assert_eq!(current_list_objects_mutation_sequence("bucket-b").await, 0); } + #[tokio::test] + #[serial_test::serial] + async fn scanner_namespace_generation_tracks_only_user_namespace_mutations() { + reset_list_objects_mutation_sequences_for_test().await; + + assert_eq!(scanner_namespace_mutation_generation(), 0); + assert_eq!(observe_list_objects_mutations_with_store(None, RUSTFS_META_BUCKET, 3).await, Some(3)); + assert_eq!(scanner_namespace_mutation_generation(), 0); + + assert_eq!(observe_list_objects_mutations_with_store(None, "photos", 2).await, Some(5)); + assert_eq!(scanner_namespace_mutation_generation(), 2); + } + #[test] fn list_objects_index_provider_state_uses_lifecycle_active_generation() { let provider = ListObjectsIndexProviderState::from_kind(ListObjectsIndexProviderKind::WalkerKeyOnly); diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index eeb56476b..181930128 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -318,6 +318,10 @@ impl ECStore { pub async fn setup_is_erasure_sd(&self) -> bool { self.ctx.is_erasure_sd().await } + + pub fn scanner_namespace_mutation_generation(&self) -> u64 { + list_objects::scanner_namespace_mutation_generation() + } } // impl Clone for ECStore { @@ -436,7 +440,11 @@ impl BucketOperations for ECStore { #[instrument(skip(self))] async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> { - self.handle_make_bucket(bucket, opts).await + let result = self.handle_make_bucket(bucket, opts).await; + if result.is_ok() { + list_objects::observe_scanner_namespace_mutations(bucket, 1); + } + result } #[instrument(skip(self))] @@ -449,7 +457,11 @@ impl BucketOperations for ECStore { } #[instrument(skip(self))] async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { - self.handle_delete_bucket(bucket, opts).await + let result = self.handle_delete_bucket(bucket, opts).await; + if result.is_ok() { + list_objects::observe_scanner_namespace_mutations(bucket, 1); + } + result } } diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index f381ffc80..795ac3e5b 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -923,6 +923,8 @@ pub struct GetAllBucketStatsResponse { pub struct LoadBucketMetadataRequest { #[prost(string, tag = "1")] pub bucket: ::prost::alloc::string::String, + #[prost(bool, tag = "2")] + pub scanner_maintenance_change: bool, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadBucketMetadataResponse { @@ -1067,6 +1069,17 @@ pub struct SignalServiceResponse { pub error_info: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ScannerActivityRequest {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ScannerActivityResponse { + #[prost(string, tag = "1")] + pub instance_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub namespace_generation: u64, + #[prost(uint64, tag = "3")] + pub maintenance_generation: u64, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BackgroundHealStatusRequest {} #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BackgroundHealStatusResponse { @@ -2351,6 +2364,21 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "SignalService")); self.inner.unary(req, path, codec).await } + pub async fn scanner_activity( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ScannerActivity"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "ScannerActivity")); + self.inner.unary(req, path, codec).await + } pub async fn background_heal_status( &mut self, request: impl tonic::IntoRequest, @@ -2825,6 +2853,10 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn scanner_activity( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn background_heal_status( &self, request: tonic::Request, @@ -4933,6 +4965,34 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/ScannerActivity" => { + #[allow(non_camel_case_types)] + struct ScannerActivitySvc(pub Arc); + impl tonic::server::UnaryService for ScannerActivitySvc { + type Response = super::ScannerActivityResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::scanner_activity(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ScannerActivitySvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/BackgroundHealStatus" => { #[allow(non_camel_case_types)] struct BackgroundHealStatusSvc(pub Arc); diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index a587b64ad..3543d589b 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -649,6 +649,7 @@ message GetAllBucketStatsResponse { message LoadBucketMetadataRequest { string bucket = 1; + bool scanner_maintenance_change = 2; } message LoadBucketMetadataResponse { @@ -756,6 +757,14 @@ message SignalServiceResponse { optional string error_info = 2; } +message ScannerActivityRequest {} + +message ScannerActivityResponse { + string instance_id = 1; + uint64 namespace_generation = 2; + uint64 maintenance_generation = 3; +} + message BackgroundHealStatusRequest {} message BackgroundHealStatusResponse { @@ -944,6 +953,7 @@ service NodeService { // rpc VerifyBinary() returns () {}; // rpc CommitBinary() returns () {}; rpc SignalService(SignalServiceRequest) returns (SignalServiceResponse) {}; + rpc ScannerActivity(ScannerActivityRequest) returns (ScannerActivityResponse) {}; rpc BackgroundHealStatus(BackgroundHealStatusRequest) returns (BackgroundHealStatusResponse) {}; rpc GetMetacacheListing(GetMetacacheListingRequest) returns (GetMetacacheListingResponse) {}; rpc UpdateMetacacheListing(UpdateMetacacheListingRequest) returns (UpdateMetacacheListingResponse) {}; diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index e4c3dd5d2..f8bb760d9 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -58,8 +58,11 @@ pub use data_usage_define::*; pub use error::ScannerError; pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config}; pub use rustfs_common::last_minute; -pub use scanner::init_data_scanner; -pub use scanner_io::{clear_dirty_usage_bucket, record_dirty_usage_bucket}; +pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status}; +pub use scanner_io::{ + clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, + scanner_maintenance_generation, +}; pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER}; use std::sync::atomic::{AtomicU64, Ordering}; pub use storage_api::ScannerReplicationConfig as ReplicationConfig; diff --git a/crates/scanner/src/runtime_config.rs b/crates/scanner/src/runtime_config.rs index eea497898..da951339d 100644 --- a/crates/scanner/src/runtime_config.rs +++ b/crates/scanner/src/runtime_config.rs @@ -38,11 +38,14 @@ use std::fmt; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{LazyLock, RwLock}; use std::time::Duration; +use tokio::sync::Notify; use tracing::warn; const LOG_COMPONENT_SCANNER: &str = "scanner"; +#[cfg(test)] const LOG_SUBSYSTEM_RUNTIME: &str = "runtime"; const LOG_SUBSYSTEM_RUNTIME_CONFIG: &str = "runtime_config"; +#[cfg(test)] const EVENT_SCANNER_RUNTIME_CONFIG: &str = "scanner_runtime_config"; const EVENT_SCANNER_RUNTIME_CONFIG_PARSE: &str = "scanner_runtime_config_parse"; @@ -52,6 +55,8 @@ const MAX_SCANNER_DELAY_FACTOR: f64 = 10_000.0; const SCANNER_DELAY_RANGE_REASON: &str = "expected scanner delay between 0 and 10000"; static SCANNER_DEFAULT_CYCLE_SECS: AtomicU64 = AtomicU64::new(NO_DEFAULT_CYCLE_OVERRIDE); +static SCANNER_RUNTIME_CONFIG_GENERATION: AtomicU64 = AtomicU64::new(0); +static SCANNER_RUNTIME_CONFIG_NOTIFY: LazyLock = LazyLock::new(Notify::new); static SCANNER_RUNTIME_CONFIG: LazyLock> = LazyLock::new(|| RwLock::new(lookup_scanner_runtime_config(None).unwrap_or_default())); @@ -678,6 +683,7 @@ fn current_server_config() -> Option { resolve_scanner_server_config() } +#[cfg(test)] pub(crate) fn resolve_scanner_runtime_config_from_global() -> ScannerRuntimeConfig { let config = current_server_config(); match lookup_scanner_runtime_config(config.as_ref()) { @@ -705,9 +711,19 @@ pub fn apply_scanner_runtime_config(config: &ServerConfig) -> Result<(), Scanner validate_scanner_runtime_config(config)?; let resolved = lookup_scanner_runtime_config(Some(config))?; apply_resolved_runtime_config(resolved); + SCANNER_RUNTIME_CONFIG_GENERATION.fetch_add(1, Ordering::AcqRel); + SCANNER_RUNTIME_CONFIG_NOTIFY.notify_one(); Ok(()) } +pub(crate) fn scanner_runtime_config_generation() -> u64 { + SCANNER_RUNTIME_CONFIG_GENERATION.load(Ordering::Acquire) +} + +pub(crate) async fn scanner_runtime_config_changed() { + SCANNER_RUNTIME_CONFIG_NOTIFY.notified().await; +} + pub(crate) fn refresh_scanner_runtime_config_from_global() -> Result<(), ScannerRuntimeConfigError> { let config = current_server_config(); let resolved = lookup_scanner_runtime_config(config.as_ref())?; @@ -846,11 +862,11 @@ mod tests { use crate::init_ecstore_config_for_scanner_tests; use rustfs_config::server_config::{Config as ServerConfig, KVS}; use rustfs_config::{ - DEFAULT_DELIMITER, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE, - ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE, - HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, - SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS, - ScannerSpeed, + DEFAULT_DELIMITER, DEFAULT_HEAL_BITROT_CYCLE_SECS, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, + ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, + HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, + SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, + SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed, }; use serial_test::serial; use std::collections::HashMap; @@ -927,6 +943,22 @@ mod tests { }); } + #[test] + #[serial] + fn scanner_runtime_config_normalizes_persisted_default_speed() { + let config = server_config_with_scanner(&[(SCANNER_SPEED, "default")]); + + with_var_unset(ENV_SCANNER_SPEED, || { + with_var_unset(ENV_SCANNER_CYCLE, || { + let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config"); + + assert_eq!(resolved.speed, ScannerSpeed::Default); + assert_eq!(resolved.speed_source, ScannerRuntimeConfigSource::Default); + assert_eq!(resolved.cycle_interval_source, ScannerRuntimeConfigSource::Default); + }); + }); + } + #[test] #[serial] fn scanner_runtime_config_prefers_env_over_persisted_config() { @@ -974,6 +1006,23 @@ mod tests { super::refresh_scanner_runtime_config_for_tests(); } + #[test] + #[serial] + fn scanner_runtime_config_normalizes_persisted_default_bitrot_cycles() { + let default_cycle = DEFAULT_HEAL_BITROT_CYCLE_SECS.to_string(); + for config in [ + server_config_with_scanner_and_heal(&[], &[(HEAL_BITROT_CYCLE, default_cycle.as_str())]), + server_config_with_scanner(&[(SCANNER_BITROT_CYCLE, default_cycle.as_str())]), + ] { + with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || { + let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config"); + + assert_eq!(resolved.bitrot_cycle, Some(Duration::from_secs(DEFAULT_HEAL_BITROT_CYCLE_SECS))); + assert_eq!(resolved.bitrot_cycle_source, ScannerRuntimeConfigSource::Default); + }); + } + } + #[test] fn scanner_runtime_config_rejects_invalid_persisted_speed() { let config = server_config_with_scanner(&[(SCANNER_SPEED, "warp")]); @@ -1058,6 +1107,23 @@ mod tests { super::refresh_scanner_runtime_config_for_tests(); } + #[test] + #[serial] + fn applied_runtime_config_is_the_authoritative_scheduler_state() { + let config = server_config_with_scanner(&[(SCANNER_CYCLE, "321")]); + + with_var_unset(ENV_SCANNER_CYCLE, || { + let generation = super::scanner_runtime_config_generation(); + super::apply_scanner_runtime_config(&config).expect("scanner runtime config should apply"); + + let applied = super::current_scanner_runtime_config(); + assert_eq!(applied.cycle_interval, Duration::from_secs(321)); + assert_eq!(applied.cycle_interval_source, ScannerRuntimeConfigSource::Config); + assert!(super::scanner_runtime_config_generation() > generation); + }); + super::refresh_scanner_runtime_config_for_tests(); + } + #[test] #[serial] fn scanner_runtime_config_status_reports_persisted_pacing_overrides() { diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 26cf044f3..5c4caa2d9 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -12,17 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Arc; +use std::collections::BTreeMap; +use std::future::Future; +use std::sync::{Arc, LazyLock, RwLock}; use crate::ScannerObjectIO; use crate::data_usage_define::{BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH}; use crate::runtime_config::{ - refresh_scanner_runtime_config_from_global, resolve_scanner_runtime_config_from_global, scanner_bitrot_cycle, - scanner_cycle_interval, scanner_start_delay, set_scanner_default_cycle_secs, + ScannerRuntimeConfig, ScannerRuntimeConfigSource, refresh_scanner_runtime_config_from_global, scanner_bitrot_cycle, + scanner_cycle_interval, scanner_runtime_config_changed, scanner_runtime_config_generation, scanner_start_delay, + set_scanner_default_cycle_secs, }; use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason}; use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob}; -use crate::scanner_io::{ScannerIO, dirty_usage_bucket_notified, dirty_usage_buckets_pending}; +use crate::scanner_io::{ + ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified, dirty_usage_buckets_pending, dirty_usage_generation, + scanner_maintenance_changed, scanner_maintenance_generation, +}; use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed}; use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError}; use chrono::{DateTime, Utc}; @@ -60,9 +66,69 @@ const EVENT_SCANNER_PERSIST_STATE: &str = "scanner_persist_state"; const EVENT_SCANNER_RUNTIME_CONFIG: &str = "scanner_runtime_config"; const EVENT_SCANNER_BACKGROUND_HEAL_STATE: &str = "scanner_background_heal_state"; const METRIC_SCANNER_LEADER_LOCK_TOTAL: &str = "rustfs_scanner_leader_lock_total"; +const CLEAN_IDLE_MAX_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60); +const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2; +const SCANNER_LEADER_LOCK_POLL_INTERVAL: Duration = Duration::from_secs(1); +const MAINTENANCE_FEATURE_INSPECTION_TIMEOUT: Duration = Duration::from_secs(30); +const MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5 * 60); +const MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(60 * 60); +const MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS: usize = 2; #[cfg(test)] const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS"; +#[derive(Clone, Copy, Debug, Serialize)] +#[non_exhaustive] +pub struct ScannerCycleScheduleStatus { + effective_interval_seconds: u64, + clean_idle_backoff_enabled: bool, + clean_idle_backoff_multiplier: u64, +} + +impl Default for ScannerCycleScheduleStatus { + fn default() -> Self { + Self { + effective_interval_seconds: 0, + clean_idle_backoff_enabled: false, + clean_idle_backoff_multiplier: 1, + } + } +} + +impl ScannerCycleScheduleStatus { + pub fn effective_interval_seconds(self) -> u64 { + self.effective_interval_seconds + } +} + +static SCANNER_CYCLE_SCHEDULE: LazyLock> = + LazyLock::new(|| RwLock::new(ScannerCycleScheduleStatus::default())); + +pub fn scanner_cycle_schedule_status() -> ScannerCycleScheduleStatus { + *SCANNER_CYCLE_SCHEDULE.read().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +fn record_scanner_cycle_schedule( + effective_interval: Duration, + clean_idle_backoff_enabled: bool, + clean_idle_backoff_multiplier: u64, +) { + let effective_interval_seconds = effective_interval + .as_secs() + .saturating_add(u64::from(effective_interval.subsec_nanos() != 0)); + let mut schedule = SCANNER_CYCLE_SCHEDULE + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + *schedule = ScannerCycleScheduleStatus { + effective_interval_seconds, + clean_idle_backoff_enabled, + clean_idle_backoff_multiplier: clean_idle_backoff_multiplier.max(1), + }; +} + +fn reset_scanner_cycle_schedule() { + record_scanner_cycle_schedule(Duration::ZERO, false, 1); +} + /// Returns the base cycle interval. /// Priority order: /// 1. RUSTFS_SCANNER_CYCLE (if set, overrides everything) @@ -92,7 +158,14 @@ fn scanner_cycle_max_duration() -> Option { } fn resolve_scanner_runtime_config() -> crate::runtime_config::ScannerRuntimeConfig { - resolve_scanner_runtime_config_from_global() + #[cfg(test)] + { + crate::runtime_config::resolve_scanner_runtime_config_from_global() + } + #[cfg(not(test))] + { + crate::runtime_config::current_scanner_runtime_config() + } } fn scan_cycle_partial_reason(reason: Option) -> ScanCyclePartialReason { @@ -125,34 +198,555 @@ fn randomized_cycle_delay_for(interval: Duration) -> Duration { delay.max(Duration::from_secs(1)) } +fn cap_clean_idle_cycle_delay(delay: Duration, max_interval: Duration, enabled: bool) -> Duration { + if !enabled { + return delay; + } + + let max_interval = max_interval.max(Duration::from_secs(1)); + if delay <= max_interval { + return delay; + } + + // Reflect positive jitter below the cap instead of collapsing every + // positive sample onto the same instant once backoff reaches its ceiling. + max_interval + .saturating_sub(delay.saturating_sub(max_interval)) + .max(Duration::from_secs(1)) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ScannerCycleWakeReason { Timer, DirtyUsage, + ClusterActivity, + ClusterMaintenance, + ClusterActivityUnavailable, + RuntimeConfig, + MaintenanceConfig, + LeaderLockLost, Cancelled, } -async fn wait_for_next_scanner_cycle(ctx: &CancellationToken, delay: Duration) -> ScannerCycleWakeReason { - if dirty_usage_buckets_pending() { - return ScannerCycleWakeReason::DirtyUsage; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ScannerCycleOutcome { + Completed, + CompletedWithPendingMaintenance, + Partial, + Failed, +} + +pub(crate) fn scanner_cycle_outcome_with_pending_maintenance( + outcome: ScannerCycleOutcome, + pending_maintenance_work: bool, +) -> ScannerCycleOutcome { + if outcome == ScannerCycleOutcome::Completed && pending_maintenance_work { + ScannerCycleOutcome::CompletedWithPendingMaintenance + } else { + outcome + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct ScannerCleanIdleBackoff { + interval_multiplier: u32, +} + +impl Default for ScannerCleanIdleBackoff { + fn default() -> Self { + Self { interval_multiplier: 1 } + } +} + +impl ScannerCleanIdleBackoff { + fn reset(&mut self) { + self.interval_multiplier = 1; } + fn effective_interval(self, base_interval: Duration, max_interval: Duration, enabled: bool) -> Duration { + let base_interval = base_interval.max(Duration::from_secs(1)); + if !enabled { + return base_interval; + } + + let max_interval = max_interval.max(base_interval); + base_interval.saturating_mul(self.interval_multiplier).min(max_interval) + } + + fn record_cycle( + &mut self, + base_interval: Duration, + max_interval: Duration, + enabled: bool, + wake_reason: ScannerCycleWakeReason, + outcome: ScannerCycleOutcome, + dirty_work_observed: bool, + ) { + if !enabled + || wake_reason != ScannerCycleWakeReason::Timer + || outcome != ScannerCycleOutcome::Completed + || dirty_work_observed + { + self.reset(); + return; + } + + let max_interval = max_interval.max(base_interval.max(Duration::from_secs(1))); + if self.effective_interval(base_interval, max_interval, true) < max_interval { + self.interval_multiplier = self.interval_multiplier.saturating_mul(CLEAN_IDLE_BACKOFF_FACTOR); + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct ScannerMaintenanceInspectionRetry { + consecutive_failures: u32, + retry_at: Option, +} + +impl ScannerMaintenanceInspectionRetry { + fn from_features(features: ScannerMaintenanceFeatures, now: Instant) -> Self { + let mut retry = Self::default(); + retry.record_inspection(features, now); + retry + } + + fn reset(&mut self) { + self.consecutive_failures = 0; + self.retry_at = None; + } + + fn retry_interval(self) -> Option { + if self.consecutive_failures == 0 { + return None; + } + + let exponent = self.consecutive_failures.saturating_sub(1).min(31); + let multiplier = 1u32.checked_shl(exponent).unwrap_or(u32::MAX); + Some( + MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL + .saturating_mul(multiplier) + .min(MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL), + ) + } + + fn record_inspection(&mut self, features: ScannerMaintenanceFeatures, now: Instant) { + if !features.inspection_failed { + self.reset(); + return; + } + + self.consecutive_failures = self.consecutive_failures.saturating_add(1); + self.retry_at = self.retry_interval().map(|interval| now + interval); + } + + fn retry_due(self, features: ScannerMaintenanceFeatures, wake_reason: ScannerCycleWakeReason, now: Instant) -> bool { + features.inspection_failed + && wake_reason == ScannerCycleWakeReason::Timer + && self.retry_at.is_some_and(|retry_at| now >= retry_at) + } +} + +fn scanner_cycle_observed_dirty_work( + pending_before_wait: bool, + generation_before_wait: u64, + generation_after_cycle: u64, +) -> bool { + pending_before_wait || generation_before_wait != generation_after_cycle +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct ScannerCycleWaitPlan { + effective_interval: Duration, + clean_idle_max_interval: Duration, + delay: Duration, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct ScannerCycleObservedGenerations { + dirty_usage: u64, + runtime_config: u64, + maintenance: u64, +} + +const LOCAL_SCANNER_ACTIVITY_NODE: &str = ""; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct ScannerNodeActivity { + instance_id: String, + namespace_generation: u64, + maintenance_generation: u64, +} + +type ScannerActivitySnapshot = BTreeMap; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ScannerActivityObservation { + NotRequired, + Unchanged, + Changed, + MaintenanceChanged, + Unverified, +} + +fn scanner_cycle_wait_plan( + runtime_config: &ScannerRuntimeConfig, + clean_idle_backoff: ScannerCleanIdleBackoff, + clean_idle_backoff_enabled: bool, + jitter: impl FnOnce(Duration) -> Duration, +) -> ScannerCycleWaitPlan { + let clean_idle_max_interval = scanner_clean_idle_max_interval(runtime_config.cycle_interval, runtime_config); + let effective_interval = + clean_idle_backoff.effective_interval(runtime_config.cycle_interval, clean_idle_max_interval, clean_idle_backoff_enabled); + let delay = cap_clean_idle_cycle_delay(jitter(effective_interval), clean_idle_max_interval, clean_idle_backoff_enabled); + + ScannerCycleWaitPlan { + effective_interval, + clean_idle_max_interval, + delay, + } +} + +fn record_scanner_cycle_result( + clean_idle_backoff: &mut ScannerCleanIdleBackoff, + runtime_config: &ScannerRuntimeConfig, + clean_idle_backoff_enabled: bool, + wake_reason: ScannerCycleWakeReason, + outcome: ScannerCycleOutcome, + dirty_work_observed: bool, +) { + clean_idle_backoff.record_cycle( + runtime_config.cycle_interval, + scanner_clean_idle_max_interval(runtime_config.cycle_interval, runtime_config), + clean_idle_backoff_enabled, + wake_reason, + outcome, + dirty_work_observed, + ); +} + +fn scanner_clean_idle_backoff_configured(runtime_config: &ScannerRuntimeConfig) -> bool { + let bitrot_cycle_allows_backoff = + runtime_config.bitrot_cycle.is_none() || runtime_config.bitrot_cycle_source == ScannerRuntimeConfigSource::Default; + runtime_config.cycle_interval_source == ScannerRuntimeConfigSource::Default && bitrot_cycle_allows_backoff +} + +fn scanner_clean_idle_max_interval(base_interval: Duration, runtime_config: &ScannerRuntimeConfig) -> Duration { + let policy_max = CLEAN_IDLE_MAX_INTERVAL.max(base_interval); + let Some(bitrot_cycle) = runtime_config.bitrot_cycle else { + return policy_max; + }; + if runtime_config.bitrot_cycle_source != ScannerRuntimeConfigSource::Default { + return policy_max; + } + + let selection_window = heal_object_select_prob(); + if selection_window == 0 { + return policy_max; + } + + bitrot_cycle + .checked_div(selection_window) + .unwrap_or(base_interval) + .max(base_interval) + .min(policy_max) +} + +fn scanner_clean_idle_backoff_enabled( + topology_supported: bool, + cluster_activity_ready: bool, + features: ScannerMaintenanceFeatures, + runtime_config: &ScannerRuntimeConfig, +) -> bool { + topology_supported + && cluster_activity_ready + && !features.needs_regular_cycle() + && scanner_clean_idle_backoff_configured(runtime_config) +} + +fn scanner_activity_probe_required( + topology_supported: bool, + backoff_blocked: bool, + features: ScannerMaintenanceFeatures, + runtime_config: &ScannerRuntimeConfig, +) -> bool { + topology_supported + && !backoff_blocked + && !features.needs_regular_cycle() + && scanner_clean_idle_backoff_configured(runtime_config) +} + +fn scanner_activity_observed_work(observation: ScannerActivityObservation) -> bool { + matches!( + observation, + ScannerActivityObservation::Changed + | ScannerActivityObservation::MaintenanceChanged + | ScannerActivityObservation::Unverified + ) +} + +fn scanner_activity_backoff_blocked_after_wake(currently_blocked: bool, wake_reason: ScannerCycleWakeReason) -> bool { + match wake_reason { + ScannerCycleWakeReason::ClusterMaintenance => true, + ScannerCycleWakeReason::MaintenanceConfig => false, + _ => currently_blocked, + } +} + +async fn wait_for_next_scanner_cycle( + ctx: &CancellationToken, + delay: Duration, + dirty_usage_generation_seen: u64, + runtime_config_generation: u64, + maintenance_generation: u64, + is_lock_lost: F, +) -> ScannerCycleWakeReason +where + F: Fn() -> bool, +{ let sleep = tokio::time::sleep(delay); tokio::pin!(sleep); + let lock_poll = tokio::time::sleep(SCANNER_LEADER_LOCK_POLL_INTERVAL); + tokio::pin!(lock_poll); loop { + if is_lock_lost() { + return ScannerCycleWakeReason::LeaderLockLost; + } + if scanner_runtime_config_generation() != runtime_config_generation { + return ScannerCycleWakeReason::RuntimeConfig; + } + if scanner_maintenance_generation() != maintenance_generation { + return ScannerCycleWakeReason::MaintenanceConfig; + } + if dirty_usage_buckets_pending() && dirty_usage_generation() != dirty_usage_generation_seen { + return ScannerCycleWakeReason::DirtyUsage; + } + tokio::select! { _ = ctx.cancelled() => return ScannerCycleWakeReason::Cancelled, _ = &mut sleep => return ScannerCycleWakeReason::Timer, + _ = &mut lock_poll => { + if is_lock_lost() { + return ScannerCycleWakeReason::LeaderLockLost; + } + lock_poll.as_mut().reset(Instant::now() + SCANNER_LEADER_LOCK_POLL_INTERVAL); + } _ = dirty_usage_bucket_notified() => { - if dirty_usage_buckets_pending() { + if scanner_runtime_config_generation() != runtime_config_generation { + return ScannerCycleWakeReason::RuntimeConfig; + } + if scanner_maintenance_generation() != maintenance_generation { + return ScannerCycleWakeReason::MaintenanceConfig; + } + if dirty_usage_buckets_pending() && dirty_usage_generation() != dirty_usage_generation_seen { return ScannerCycleWakeReason::DirtyUsage; } } + _ = scanner_runtime_config_changed() => { + if scanner_runtime_config_generation() != runtime_config_generation { + return ScannerCycleWakeReason::RuntimeConfig; + } + } + _ = scanner_maintenance_changed() => { + if scanner_maintenance_generation() != maintenance_generation { + return ScannerCycleWakeReason::MaintenanceConfig; + } + } } } } +async fn wait_for_next_scanner_cycle_with_activity( + ctx: &CancellationToken, + delay: Duration, + activity_poll_interval: Option, + activity_seen: &mut Option, + generations: ScannerCycleObservedGenerations, + is_lock_lost: F, + mut probe_activity: Probe, +) -> ScannerCycleWakeReason +where + F: Fn() -> bool, + Probe: FnMut() -> ProbeFuture, + ProbeFuture: Future>, +{ + let deadline = Instant::now() + delay; + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return ScannerCycleWakeReason::Timer; + } + let wait_slice = activity_poll_interval + .map(|interval| interval.max(Duration::from_secs(1)).min(remaining)) + .unwrap_or(remaining); + let wake_reason = wait_for_next_scanner_cycle( + ctx, + wait_slice, + generations.dirty_usage, + generations.runtime_config, + generations.maintenance, + &is_lock_lost, + ) + .await; + if wake_reason != ScannerCycleWakeReason::Timer || Instant::now() >= deadline { + return wake_reason; + } + + let Some(_) = activity_poll_interval else { + return ScannerCycleWakeReason::Timer; + }; + if is_lock_lost() { + return ScannerCycleWakeReason::LeaderLockLost; + } + + let probe = probe_activity(); + tokio::pin!(probe); + let lock_lost = async { + loop { + tokio::time::sleep(SCANNER_LEADER_LOCK_POLL_INTERVAL).await; + if is_lock_lost() { + break; + } + } + }; + tokio::pin!(lock_lost); + let probe_result = tokio::select! { + result = &mut probe => result, + _ = ctx.cancelled() => return ScannerCycleWakeReason::Cancelled, + _ = &mut lock_lost => return ScannerCycleWakeReason::LeaderLockLost, + }; + + let had_baseline = activity_seen.is_some(); + let (observation, probe_error) = apply_scanner_activity_probe_result(activity_seen, probe_result); + if let Some(err) = probe_error { + log_scanner_activity_probe_error(had_baseline, &err); + } + match observation { + ScannerActivityObservation::Unchanged | ScannerActivityObservation::NotRequired => {} + ScannerActivityObservation::Changed => return ScannerCycleWakeReason::ClusterActivity, + ScannerActivityObservation::MaintenanceChanged => return ScannerCycleWakeReason::ClusterMaintenance, + ScannerActivityObservation::Unverified => return ScannerCycleWakeReason::ClusterActivityUnavailable, + } + } +} + +fn log_scanner_activity_probe_error(had_baseline: bool, err: &str) { + if had_baseline { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_CYCLE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "cluster_activity_probe_failed", + error = %err, + "Scanner cluster activity probe failed; preserving the base cycle" + ); + } else { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_CYCLE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "cluster_activity_probe_unavailable", + error = %err, + "Scanner cluster activity probe remains unavailable" + ); + } +} + +fn compare_scanner_activity(previous: &ScannerActivitySnapshot, current: &ScannerActivitySnapshot) -> ScannerActivityObservation { + if previous == current { + return ScannerActivityObservation::Unchanged; + } + + for (host, current_activity) in current { + let Some(previous_activity) = previous.get(host) else { + continue; + }; + if host != LOCAL_SCANNER_ACTIVITY_NODE + && previous_activity.instance_id == current_activity.instance_id + && previous_activity.maintenance_generation != current_activity.maintenance_generation + { + return ScannerActivityObservation::MaintenanceChanged; + } + } + + ScannerActivityObservation::Changed +} + +fn apply_scanner_activity_probe_result( + activity_seen: &mut Option, + result: Result, +) -> (ScannerActivityObservation, Option) { + match result { + Ok(current) => { + let observation = match activity_seen.as_ref() { + Some(previous) => compare_scanner_activity(previous, ¤t), + None => ScannerActivityObservation::Unverified, + }; + *activity_seen = Some(current); + (observation, None) + } + Err(err) => { + *activity_seen = None; + (ScannerActivityObservation::Unverified, Some(err)) + } + } +} + +async fn observe_scanner_activity( + storeapi: &Arc, + distributed: bool, + activity_seen: &mut Option, +) -> ScannerActivityObservation { + let had_baseline = activity_seen.is_some(); + let (observation, probe_error) = + apply_scanner_activity_probe_result(activity_seen, probe_scanner_activity(storeapi, distributed).await); + if let Some(err) = probe_error { + log_scanner_activity_probe_error(had_baseline, &err); + } + observation +} + +async fn probe_scanner_activity(storeapi: &Arc, distributed: bool) -> Result { + let mut snapshot = ScannerActivitySnapshot::from([( + LOCAL_SCANNER_ACTIVITY_NODE.to_string(), + ScannerNodeActivity { + instance_id: crate::scanner_io::scanner_activity_epoch().to_string(), + namespace_generation: storeapi.scanner_namespace_mutation_generation(), + maintenance_generation: scanner_maintenance_generation(), + }, + )]); + if !distributed { + return Ok(snapshot); + } + + let notification_system = storeapi + .notification_system() + .ok_or_else(|| "notification system is not initialized".to_string())?; + let peers = notification_system + .scanner_activity_snapshots() + .await + .map_err(|err| err.to_string())?; + for (host, activity) in peers { + if snapshot + .insert( + host.clone(), + ScannerNodeActivity { + instance_id: activity.instance_id, + namespace_generation: activity.namespace_generation, + maintenance_generation: activity.maintenance_generation, + }, + ) + .is_some() + { + return Err(format!("duplicate scanner activity peer: {host}")); + } + } + Ok(snapshot) +} + fn initial_scanner_delay_for(start_delay_secs: Option) -> Duration { start_delay_secs .map(|secs| randomized_cycle_delay_for(Duration::from_secs(secs))) @@ -262,7 +856,7 @@ async fn initial_scanner_startup_usage_state(storeapi: &Arc) -> (bool, } pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc) { - let startup_features = configure_scanner_defaults(&storeapi).await; + let (startup_features, startup_maintenance_generation) = configure_scanner_defaults(&ctx, &storeapi).await; // Force init global sleeper so config is read once at startup. let _ = &*SCANNER_SLEEPER; if let Err(err) = refresh_scanner_runtime_config_from_global() { @@ -312,7 +906,14 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc) { break; } - if let Err(e) = run_data_scanner(ctx_clone.clone(), storeapi_clone.clone()).await { + if let Err(e) = run_data_scanner_with_maintenance_state( + ctx_clone.clone(), + storeapi_clone.clone(), + startup_features, + startup_maintenance_generation, + ) + .await + { error!( target: "rustfs::scanner", event = EVENT_SCANNER_CYCLE_STATE, @@ -346,6 +947,47 @@ impl ScannerMaintenanceFeatures { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum MaintenanceInspectionDecision { + Accept, + Retry, + PreserveBaseCycle, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum MaintenanceInspectionAttempt { + Completed(ScannerMaintenanceFeatures), + TimedOut, + Cancelled, +} + +async fn wait_for_maintenance_feature_inspection( + ctx: &CancellationToken, + inspection: F, + timeout: Duration, +) -> MaintenanceInspectionAttempt +where + F: Future, +{ + tokio::select! { + _ = ctx.cancelled() => MaintenanceInspectionAttempt::Cancelled, + result = tokio::time::timeout(timeout, inspection) => match result { + Ok(features) => MaintenanceInspectionAttempt::Completed(features), + Err(_) => MaintenanceInspectionAttempt::TimedOut, + }, + } +} + +fn maintenance_inspection_decision(generation: u64, current_generation: u64, attempts: usize) -> MaintenanceInspectionDecision { + if generation == current_generation { + MaintenanceInspectionDecision::Accept + } else if attempts < MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS { + MaintenanceInspectionDecision::Retry + } else { + MaintenanceInspectionDecision::PreserveBaseCycle + } +} + fn single_disk_default_cycle_secs(_features: ScannerMaintenanceFeatures) -> Option { None } @@ -432,9 +1074,87 @@ async fn detect_scanner_maintenance_features(storeapi: &Arc) -> Scanner features } -async fn configure_scanner_defaults(storeapi: &Arc) -> ScannerMaintenanceFeatures { - if scanner_is_erasure_sd().await { - let features = detect_scanner_maintenance_features(storeapi).await; +async fn detect_stable_scanner_maintenance_features( + ctx: &CancellationToken, + storeapi: &Arc, +) -> Option<(ScannerMaintenanceFeatures, u64)> { + detect_stable_scanner_maintenance_features_with( + ctx, + || detect_scanner_maintenance_features(storeapi), + MAINTENANCE_FEATURE_INSPECTION_TIMEOUT, + ) + .await +} + +async fn detect_stable_scanner_maintenance_features_with( + ctx: &CancellationToken, + mut inspect: F, + timeout: Duration, +) -> Option<(ScannerMaintenanceFeatures, u64)> +where + F: FnMut() -> Fut, + Fut: Future, +{ + let mut attempts = 0usize; + loop { + attempts += 1; + let generation = scanner_maintenance_generation(); + let mut features = match wait_for_maintenance_feature_inspection(ctx, inspect(), timeout).await { + MaintenanceInspectionAttempt::Completed(features) => features, + MaintenanceInspectionAttempt::Cancelled => return None, + MaintenanceInspectionAttempt::TimedOut => { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_RUNTIME_CONFIG, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + timeout = ?timeout, + state = "maintenance_feature_inspection_timed_out", + "Scanner maintenance feature inspection timed out; preserving the base cycle" + ); + ScannerMaintenanceFeatures { + inspection_failed: true, + ..Default::default() + } + } + }; + let current_generation = scanner_maintenance_generation(); + match maintenance_inspection_decision(generation, current_generation, attempts) { + MaintenanceInspectionDecision::Accept => return Some((features, current_generation)), + MaintenanceInspectionDecision::Retry => {} + MaintenanceInspectionDecision::PreserveBaseCycle => { + features.inspection_failed = true; + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_RUNTIME_CONFIG, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + attempts = MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS, + state = "maintenance_feature_inspection_unstable", + "Scanner maintenance configuration changed repeatedly during inspection; preserving the base cycle" + ); + return Some((features, current_generation)); + } + } + } +} + +async fn configure_scanner_defaults( + ctx: &CancellationToken, + storeapi: &Arc, +) -> (ScannerMaintenanceFeatures, Option) { + if storeapi.setup_is_erasure_sd().await { + let (features, maintenance_generation) = detect_stable_scanner_maintenance_features(ctx, storeapi) + .await + .unwrap_or_else(|| { + ( + ScannerMaintenanceFeatures { + inspection_failed: true, + ..Default::default() + }, + scanner_maintenance_generation(), + ) + }); let default_cycle_secs = single_disk_default_cycle_secs(features); set_scanner_default_speed(single_disk_default_speed()); set_scanner_default_cycle_secs(default_cycle_secs); @@ -453,11 +1173,11 @@ async fn configure_scanner_defaults(storeapi: &Arc) -> ScannerMaintenan state = "single_disk_defaults_applied", "Scanner defaults applied" ); - features + (features, Some(maintenance_generation)) } else { set_scanner_default_speed(ScannerSpeed::Default); set_scanner_default_cycle_secs(None); - ScannerMaintenanceFeatures::default() + (ScannerMaintenanceFeatures::default(), None) } } @@ -686,8 +1406,23 @@ async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle) { global_metrics().set_cycle(Some(cycle_info.clone())).await; } -async fn persist_scanner_cycle_state(storeapi: Arc, cycle_info: &CurrentCycle) { - let cycle_info_buf = cycle_info.marshal().unwrap_or_default(); +async fn persist_scanner_cycle_state(storeapi: Arc, cycle_info: &CurrentCycle) -> bool { + let cycle_info_buf = match cycle_info.marshal() { + Ok(buf) => buf, + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "encode_failed", + error = %e, + "Scanner state encoding failed" + ); + return false; + } + }; let mut buf = Vec::with_capacity(cycle_info_buf.len() + 8); buf.extend_from_slice(&cycle_info.next.to_le_bytes()); @@ -704,6 +1439,7 @@ async fn persist_scanner_cycle_state(storeapi: Arc, cycle_ error = %e, "Scanner state persistence failed" ); + false } else { debug!( target: "rustfs::scanner", @@ -714,10 +1450,11 @@ async fn persist_scanner_cycle_state(storeapi: Arc, cycle_ state = "saved", "Scanner state saved" ); + true } } -async fn finalize_partial_scan_cycle(storeapi: Arc, cycle_info: &mut CurrentCycle) { +async fn finalize_partial_scan_cycle(storeapi: Arc, cycle_info: &mut CurrentCycle) -> bool { // A budget-limited cycle is deliberate pacing, not a failure. The cycle counter // must still advance (and persist) because per-bucket next_cycle is stamped from // it and compacted folders are only rescanned when their hash matches @@ -725,11 +1462,15 @@ async fn finalize_partial_scan_cycle(storeapi: Arc, cycle_ // expiry and usage refresh on every folder outside the stuck window. cycle_info.next += 1; mark_scan_cycle_idle(cycle_info).await; - persist_scanner_cycle_state(storeapi, cycle_info).await; + persist_scanner_cycle_state(storeapi, cycle_info).await } #[instrument(skip_all)] -async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc, cycle_info: &mut CurrentCycle) { +async fn run_data_scanner_cycle( + ctx: &CancellationToken, + storeapi: &Arc, + cycle_info: &mut CurrentCycle, +) -> ScannerCycleOutcome { let _activity_guard = ScannerActivityGuard::new(); if let Err(err) = refresh_scanner_runtime_config_from_global() { warn!( @@ -745,6 +1486,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc let configured_cycle_interval = scanner_cycle_interval(); let configured_bitrot_cycle = scanner_bitrot_cycle(); let cycle_budget_config = scanner_cycle_budget_config(); + let usage_persist_timeout = resolve_scanner_runtime_config().cache_save_timeout; global_metrics().record_scanner_cycle_config( configured_cycle_interval, configured_bitrot_cycle, @@ -791,65 +1533,117 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc let (sender, receiver) = mpsc::channel::(1); let storeapi_clone = storeapi.clone(); let ctx_clone = ctx.clone(); - tokio::spawn(async move { - store_data_usage_in_backend(ctx_clone, storeapi_clone, receiver).await; - }); + let mut usage_persist_task = + tokio::spawn(async move { store_data_usage_in_backend_with_outcome(ctx_clone, storeapi_clone, receiver).await }); let done_cycle = Metrics::time(Metric::ScanCycle); let cycle_start = std::time::Instant::now(); let cycle_work_start = global_metrics().start_scan_cycle_work(); let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config); - if let Err(e) = storeapi + let scan_result = storeapi .clone() - .nsscanner(cycle_budget.token(), cycle_budget.clone(), sender, cycle_info.current, scan_mode) - .await + .nsscanner_with_status(cycle_budget.token(), cycle_budget.clone(), sender, cycle_info.current, scan_mode) + .await; + let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled(); + let usage_persist_outcome = match wait_for_data_usage_persist_task(ctx, &mut usage_persist_task, usage_persist_timeout).await { - let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled(); - global_metrics().finish_scan_cycle_work(cycle_work_start); - if budget_elapsed { - warn!( + DataUsagePersistTaskResult::Completed(outcome) => outcome, + DataUsagePersistTaskResult::JoinFailed(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + cycle = cycle_info.current, + state = "usage_persist_task_failed", + error = %err, + "Scanner data usage persistence task failed" + ); + DataUsagePersistOutcome::Failed + } + DataUsagePersistTaskResult::Cancelled => { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + cycle = cycle_info.current, + state = "usage_persist_task_cancelled", + "Scanner data usage persistence task cancelled" + ); + DataUsagePersistOutcome::Failed + } + DataUsagePersistTaskResult::TimedOut => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + cycle = cycle_info.current, + timeout = ?usage_persist_timeout, + state = "usage_persist_task_timed_out", + "Scanner data usage persistence task timed out" + ); + DataUsagePersistOutcome::Failed + } + }; + let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work(); + global_metrics().finish_scan_cycle_work(cycle_work_start); + + let scan_cycle_result = match scan_result { + Ok(result) => result, + Err(e) => { + error!( target: "rustfs::scanner", event = EVENT_SCANNER_CYCLE_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_RUNTIME, cycle = cycle_info.current, + scan_mode = ?scan_mode, + state = "failed", duration = ?now.elapsed(), - reason = ?cycle_budget.reason(), - max_duration = ?cycle_budget.max_duration(), - max_objects = ?cycle_budget.max_objects(), - max_directories = ?cycle_budget.max_directories(), - state = "budget_reached", - "Scanner cycle budget reached" + error = %e, + "Scanner cycle failed" ); - let budget_reason = cycle_budget.reason(); - emit_scan_cycle_partial_with_source( - cycle_start.elapsed(), - scan_cycle_partial_reason(budget_reason), - scan_cycle_partial_source(budget_reason), - ); - finalize_partial_scan_cycle(storeapi.clone(), cycle_info).await; - return; + emit_scan_cycle_complete(false, cycle_start.elapsed()); + if !ctx.is_cancelled() + && let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, false) + { + save_background_heal_info(storeapi.clone(), new_heal_info).await; + } + mark_scan_cycle_idle(cycle_info).await; + return ScannerCycleOutcome::Failed; } - error!( + }; + if ctx.is_cancelled() { + debug!( target: "rustfs::scanner", event = EVENT_SCANNER_CYCLE_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_RUNTIME, cycle = cycle_info.current, - scan_mode = ?scan_mode, - state = "failed", - duration = ?now.elapsed(), - error = %e, - "Scanner cycle failed" + state = "cancelled_before_commit", + "Scanner cycle stopped before committing cycle state" ); emit_scan_cycle_complete(false, cycle_start.elapsed()); - if let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, false) { - save_background_heal_info(storeapi.clone(), new_heal_info).await; - } mark_scan_cycle_idle(cycle_info).await; - return; + return ScannerCycleOutcome::Failed; } - if cycle_budget.budget_elapsed() && !ctx.is_cancelled() { + if usage_persist_outcome == DataUsagePersistOutcome::Failed { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + cycle = cycle_info.current, + state = "usage_not_durable", + "Scanner cycle completed without a durable data usage snapshot" + ); + emit_scan_cycle_complete(false, cycle_start.elapsed()); + mark_scan_cycle_idle(cycle_info).await; + return ScannerCycleOutcome::Failed; + } + if budget_elapsed { warn!( target: "rustfs::scanner", event = EVENT_SCANNER_CYCLE_STATE, @@ -864,28 +1658,88 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc state = "budget_reached", "Scanner cycle budget reached" ); - global_metrics().finish_scan_cycle_work(cycle_work_start); let budget_reason = cycle_budget.reason(); emit_scan_cycle_partial_with_source( cycle_start.elapsed(), scan_cycle_partial_reason(budget_reason), scan_cycle_partial_source(budget_reason), ); - finalize_partial_scan_cycle(storeapi.clone(), cycle_info).await; - return; - } - done_cycle(); - global_metrics().finish_scan_cycle_work(cycle_work_start); - emit_scan_cycle_complete(true, cycle_start.elapsed()); - if let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, true) { - save_background_heal_info(storeapi.clone(), new_heal_info).await; + return if finalize_partial_scan_cycle(storeapi.clone(), cycle_info).await { + ScannerCycleOutcome::Partial + } else { + ScannerCycleOutcome::Failed + }; } + let (completion_outcome, scanner_pending_maintenance_work) = + finalize_scanner_cycle_result(scan_cycle_result, usage_persist_outcome); + let pending_maintenance_work = scanner_pending_maintenance_work || unresolved_heal_work; + match completion_outcome { + ScannerCycleOutcome::Failed => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + cycle = cycle_info.current, + outcome = ?usage_persist_outcome, + state = "usage_not_durable", + "Scanner cycle completed without a durable data usage snapshot" + ); + emit_scan_cycle_complete(false, cycle_start.elapsed()); + mark_scan_cycle_idle(cycle_info).await; + return ScannerCycleOutcome::Failed; + } + ScannerCycleOutcome::Partial => { + if ctx.is_cancelled() { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_CYCLE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + cycle = cycle_info.current, + state = "incomplete_cancelled", + "Scanner cycle stopped before a complete usage snapshot was produced" + ); + } else { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_CYCLE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + cycle = cycle_info.current, + state = "incomplete", + "Scanner cycle ended without a complete usage snapshot" + ); + } + emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None); + return if finalize_partial_scan_cycle(storeapi.clone(), cycle_info).await { + ScannerCycleOutcome::Partial + } else { + ScannerCycleOutcome::Failed + }; + } + ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance => {} + } cycle_info.next += 1; cycle_info.current = 0; cycle_info.cycle_completed.push(Utc::now()); global_metrics().clear_current_scan_mode(); + retain_recent_cycle_completions(&mut cycle_info.cycle_completed); + global_metrics().set_cycle(Some(cycle_info.clone())).await; + if !persist_scanner_cycle_state(storeapi.clone(), cycle_info).await { + mark_scan_cycle_idle(cycle_info).await; + emit_scan_cycle_complete(false, cycle_start.elapsed()); + return ScannerCycleOutcome::Failed; + } + + done_cycle(); + emit_scan_cycle_complete(true, cycle_start.elapsed()); + if let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, true) { + save_background_heal_info(storeapi.clone(), new_heal_info).await; + } + info!( target: "rustfs::scanner", event = EVENT_SCANNER_CYCLE_STATE, @@ -899,15 +1753,41 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc "Scanner cycle completed" ); - retain_recent_cycle_completions(&mut cycle_info.cycle_completed); - global_metrics().set_cycle(Some(cycle_info.clone())).await; + scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, pending_maintenance_work) +} - persist_scanner_cycle_state(storeapi.clone(), cycle_info).await; +async fn record_scanner_leader_lock_lost(message: &'static str) { + reset_scanner_cycle_schedule(); + record_scanner_leader_lock_state("lost"); + global_metrics() + .record_scanner_leader_liveness("lost", false, "leader lock refresh quorum lost") + .await; + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_LOCK_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + lock_name = "leader.lock", + state = "lost", + reason = message, + "Scanner leader lock lost" + ); } pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc) -> Result<(), ScannerError> { + let (maintenance_features, maintenance_generation) = configure_scanner_defaults(&ctx, &storeapi).await; + run_data_scanner_with_maintenance_state(ctx, storeapi, maintenance_features, maintenance_generation).await +} + +async fn run_data_scanner_with_maintenance_state( + ctx: CancellationToken, + storeapi: Arc, + mut maintenance_features: ScannerMaintenanceFeatures, + mut maintenance_generation_seen: Option, +) -> Result<(), ScannerError> { + reset_scanner_cycle_schedule(); // Acquire leader lock (write lock) to ensure only one scanner runs - let _guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await { + let guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await { Ok(ns_lock) => match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await { Ok(guard) => { record_scanner_leader_lock_state("acquired"); @@ -959,6 +1839,36 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc) -> return Ok(()); } }; + let single_disk = storeapi.setup_is_erasure_sd().await; + let erasure = storeapi.setup_is_erasure().await; + let distributed = storeapi.setup_is_dist_erasure().await; + let clean_idle_topology_supported = single_disk || erasure; + let mut dirty_usage_generation_seen = dirty_usage_generation(); + let mut runtime_config_generation_seen = scanner_runtime_config_generation(); + let mut clean_idle_backoff = ScannerCleanIdleBackoff::default(); + let initial_runtime_config = resolve_scanner_runtime_config(); + if clean_idle_topology_supported + && scanner_clean_idle_backoff_configured(&initial_runtime_config) + && maintenance_generation_seen.is_none() + { + let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else { + global_metrics().set_cycle(None).await; + return Ok(()); + }; + maintenance_features = features; + maintenance_generation_seen = Some(generation); + } + let mut maintenance_inspection_retry = ScannerMaintenanceInspectionRetry::from_features(maintenance_features, Instant::now()); + let mut scanner_activity_seen = None; + let mut scanner_activity_backoff_blocked = false; + if scanner_activity_probe_required( + clean_idle_topology_supported, + scanner_activity_backoff_blocked, + maintenance_features, + &initial_runtime_config, + ) { + observe_scanner_activity(&storeapi, distributed, &mut scanner_activity_seen).await; + } let mut cycle_info = CurrentCycle::default(); let buf = read_config(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH) @@ -984,7 +1894,57 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc) -> if !ctx.is_cancelled() { // Preserve previous behavior: run one cycle immediately after lock acquisition. - run_data_scanner_cycle(&ctx, &storeapi, &mut cycle_info).await; + let dirty_generation_before_cycle = dirty_usage_generation(); + let dirty_usage_pending_before_cycle = dirty_usage_buckets_pending(); + let maintenance_generation_before_cycle = scanner_maintenance_generation(); + if guard.is_lock_lost() { + record_scanner_leader_lock_lost("Scanner leader lock lost before the initial cycle").await; + global_metrics().set_cycle(None).await; + return Ok(()); + } + let initial_outcome = run_data_scanner_cycle(&ctx, &storeapi, &mut cycle_info).await; + dirty_usage_generation_seen = dirty_generation_before_cycle; + if guard.is_lock_lost() { + record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await; + global_metrics().set_cycle(None).await; + return Ok(()); + } + let runtime_config = resolve_scanner_runtime_config(); + let scanner_activity_observation = if scanner_activity_probe_required( + clean_idle_topology_supported, + scanner_activity_backoff_blocked, + maintenance_features, + &runtime_config, + ) { + observe_scanner_activity(&storeapi, distributed, &mut scanner_activity_seen).await + } else { + scanner_activity_seen = None; + ScannerActivityObservation::NotRequired + }; + if scanner_activity_observation == ScannerActivityObservation::MaintenanceChanged { + scanner_activity_backoff_blocked = true; + } + let scanner_activity_ready = !scanner_activity_backoff_blocked && scanner_activity_seen.is_some(); + let backoff_enabled = scanner_clean_idle_backoff_enabled( + clean_idle_topology_supported, + scanner_activity_ready, + maintenance_features, + &runtime_config, + ); + record_scanner_cycle_result( + &mut clean_idle_backoff, + &runtime_config, + backoff_enabled, + ScannerCycleWakeReason::Timer, + initial_outcome, + scanner_cycle_observed_dirty_work( + dirty_usage_pending_before_cycle, + dirty_generation_before_cycle, + dirty_usage_generation(), + ) || maintenance_generation_before_cycle != scanner_maintenance_generation() + || scanner_activity_observed_work(scanner_activity_observation), + ); + runtime_config_generation_seen = scanner_runtime_config_generation(); } loop { @@ -992,27 +1952,228 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc) -> break; } - match wait_for_next_scanner_cycle(&ctx, randomized_cycle_delay()).await { - ScannerCycleWakeReason::Cancelled => break, - ScannerCycleWakeReason::Timer => { - run_data_scanner_cycle(&ctx, &storeapi, &mut cycle_info).await; - } - ScannerCycleWakeReason::DirtyUsage => { - debug!( - target: "rustfs::scanner", - event = EVENT_SCANNER_CYCLE_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - state = "dirty_usage_wakeup", - "Scanner cycle woke for dirty usage work" + let runtime_config = resolve_scanner_runtime_config(); + if clean_idle_topology_supported && scanner_clean_idle_backoff_configured(&runtime_config) { + let current_generation = scanner_maintenance_generation(); + if maintenance_generation_seen != Some(current_generation) { + scanner_activity_seen = None; + scanner_activity_backoff_blocked = scanner_activity_backoff_blocked_after_wake( + scanner_activity_backoff_blocked, + ScannerCycleWakeReason::MaintenanceConfig, ); - run_data_scanner_cycle(&ctx, &storeapi, &mut cycle_info).await; + let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else { + break; + }; + maintenance_features = features; + maintenance_generation_seen = Some(generation); + maintenance_inspection_retry.record_inspection(features, Instant::now()); } } + if !scanner_activity_probe_required( + clean_idle_topology_supported, + scanner_activity_backoff_blocked, + maintenance_features, + &runtime_config, + ) { + scanner_activity_seen = None; + } + let scanner_activity_ready = !scanner_activity_backoff_blocked && scanner_activity_seen.is_some(); + let backoff_enabled = scanner_clean_idle_backoff_enabled( + clean_idle_topology_supported, + scanner_activity_ready, + maintenance_features, + &runtime_config, + ); + let wait_plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for); + let dirty_generation_before_wait = dirty_usage_generation(); + let dirty_usage_pending_before_wait = dirty_usage_buckets_pending(); + let maintenance_generation_before_wait = scanner_maintenance_generation(); + record_scanner_cycle_schedule( + wait_plan.effective_interval, + backoff_enabled, + u64::from(clean_idle_backoff.interval_multiplier), + ); + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_CYCLE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + configured_interval = ?runtime_config.cycle_interval, + effective_interval = ?wait_plan.effective_interval, + clean_idle_max_interval = ?wait_plan.clean_idle_max_interval, + scheduled_delay = ?wait_plan.delay, + interval_multiplier = clean_idle_backoff.interval_multiplier, + clean_idle_backoff_enabled = backoff_enabled, + lifecycle_active = maintenance_features.lifecycle, + replication_active = maintenance_features.replication, + feature_inspection_failed = maintenance_features.inspection_failed, + state = "wait_scheduled", + "Scanner cycle wait scheduled" + ); + + let activity_poll_interval = backoff_enabled.then_some(runtime_config.cycle_interval.max(Duration::from_secs(1))); + let wake_reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + wait_plan.delay, + activity_poll_interval, + &mut scanner_activity_seen, + ScannerCycleObservedGenerations { + dirty_usage: dirty_usage_generation_seen, + runtime_config: runtime_config_generation_seen, + maintenance: maintenance_generation_before_wait, + }, + || guard.is_lock_lost(), + || probe_scanner_activity(&storeapi, distributed), + ) + .await; + scanner_activity_backoff_blocked = + scanner_activity_backoff_blocked_after_wake(scanner_activity_backoff_blocked, wake_reason); + match wake_reason { + ScannerCycleWakeReason::Cancelled => break, + ScannerCycleWakeReason::LeaderLockLost => { + record_scanner_leader_lock_lost("Scanner leader lock lost while waiting for the next cycle").await; + break; + } + ScannerCycleWakeReason::RuntimeConfig => { + runtime_config_generation_seen = scanner_runtime_config_generation(); + maintenance_generation_seen = None; + scanner_activity_seen = None; + clean_idle_backoff.reset(); + continue; + } + ScannerCycleWakeReason::MaintenanceConfig => { + maintenance_generation_seen = None; + scanner_activity_seen = None; + clean_idle_backoff.reset(); + continue; + } + ScannerCycleWakeReason::ClusterMaintenance => { + clean_idle_backoff.reset(); + } + ScannerCycleWakeReason::Timer + | ScannerCycleWakeReason::DirtyUsage + | ScannerCycleWakeReason::ClusterActivity + | ScannerCycleWakeReason::ClusterActivityUnavailable => {} + } + + if wake_reason == ScannerCycleWakeReason::DirtyUsage { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_CYCLE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "dirty_usage_wakeup", + "Scanner cycle woke for dirty usage work" + ); + } + if matches!( + wake_reason, + ScannerCycleWakeReason::ClusterActivity + | ScannerCycleWakeReason::ClusterMaintenance + | ScannerCycleWakeReason::ClusterActivityUnavailable + ) { + let cluster_activity_verified = wake_reason == ScannerCycleWakeReason::ClusterActivity; + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_CYCLE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "cluster_activity_wakeup", + cluster_activity_verified, + "Scanner cycle woke for cluster activity" + ); + } + + if guard.is_lock_lost() { + record_scanner_leader_lock_lost("Scanner leader lock lost before starting the next cycle").await; + break; + } + let dirty_generation_before_cycle = dirty_usage_generation(); + let outcome = run_data_scanner_cycle(&ctx, &storeapi, &mut cycle_info).await; + dirty_usage_generation_seen = dirty_generation_before_cycle; + if guard.is_lock_lost() { + record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await; + break; + } + let current_runtime_generation = scanner_runtime_config_generation(); + let runtime_config_changed = current_runtime_generation != runtime_config_generation_seen; + runtime_config_generation_seen = current_runtime_generation; + if runtime_config_changed { + maintenance_generation_seen = None; + clean_idle_backoff.reset(); + } + + let runtime_config = resolve_scanner_runtime_config(); + let current_maintenance_generation = scanner_maintenance_generation(); + let maintenance_config_changed = + maintenance_generation_seen.is_some_and(|generation| generation != current_maintenance_generation); + let retry_failed_inspection = maintenance_inspection_retry.retry_due(maintenance_features, wake_reason, Instant::now()); + if clean_idle_topology_supported + && scanner_clean_idle_backoff_configured(&runtime_config) + && (maintenance_config_changed || retry_failed_inspection) + { + let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else { + break; + }; + maintenance_features = features; + maintenance_generation_seen = Some(generation); + maintenance_inspection_retry.record_inspection(features, Instant::now()); + } + + if runtime_config_changed { + clean_idle_backoff.reset(); + continue; + } + if maintenance_config_changed { + scanner_activity_seen = None; + scanner_activity_backoff_blocked = scanner_activity_backoff_blocked_after_wake( + scanner_activity_backoff_blocked, + ScannerCycleWakeReason::MaintenanceConfig, + ); + clean_idle_backoff.reset(); + continue; + } + + let scanner_activity_observation = if scanner_activity_probe_required( + clean_idle_topology_supported, + scanner_activity_backoff_blocked, + maintenance_features, + &runtime_config, + ) { + observe_scanner_activity(&storeapi, distributed, &mut scanner_activity_seen).await + } else { + scanner_activity_seen = None; + ScannerActivityObservation::NotRequired + }; + if scanner_activity_observation == ScannerActivityObservation::MaintenanceChanged { + scanner_activity_backoff_blocked = true; + } + let scanner_activity_ready = !scanner_activity_backoff_blocked && scanner_activity_seen.is_some(); + let backoff_enabled = scanner_clean_idle_backoff_enabled( + clean_idle_topology_supported, + scanner_activity_ready, + maintenance_features, + &runtime_config, + ); + record_scanner_cycle_result( + &mut clean_idle_backoff, + &runtime_config, + backoff_enabled, + wake_reason, + outcome, + scanner_cycle_observed_dirty_work( + dirty_usage_pending_before_wait, + dirty_generation_before_wait, + dirty_usage_generation(), + ) || scanner_activity_observed_work(scanner_activity_observation), + ); } global_metrics().set_cycle(None).await; - global_metrics().record_scanner_leader_liveness("stopped", false, "").await; + reset_scanner_cycle_schedule(); + if !guard.is_lock_lost() { + global_metrics().record_scanner_leader_liveness("stopped", false, "").await; + } debug!( target: "rustfs::scanner", @@ -1041,6 +2202,82 @@ impl Drop for ScannerScanModeGuard { } } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum DataUsagePersistOutcome { + #[default] + NoUpdate, + Current, + Saved, + Failed, +} + +#[derive(Debug)] +enum DataUsagePersistTaskResult { + Completed(DataUsagePersistOutcome), + Cancelled, + TimedOut, + JoinFailed(tokio::task::JoinError), +} + +async fn wait_for_data_usage_persist_task( + ctx: &CancellationToken, + task: &mut tokio::task::JoinHandle, + timeout: Duration, +) -> DataUsagePersistTaskResult { + tokio::select! { + biased; + result = &mut *task => match result { + Ok(outcome) => DataUsagePersistTaskResult::Completed(outcome), + Err(err) => DataUsagePersistTaskResult::JoinFailed(err), + }, + _ = ctx.cancelled() => { + task.abort(); + let _ = (&mut *task).await; + DataUsagePersistTaskResult::Cancelled + }, + _ = tokio::time::sleep(timeout) => { + task.abort(); + let _ = (&mut *task).await; + DataUsagePersistTaskResult::TimedOut + } + } +} + +fn scanner_cycle_completion_outcome( + scan_status: ScannerCycleStatus, + usage_persist_outcome: DataUsagePersistOutcome, + has_dirty_usage: bool, + has_failed_dirty_usage: bool, +) -> ScannerCycleOutcome { + match (scan_status, usage_persist_outcome) { + (_, DataUsagePersistOutcome::Failed) => ScannerCycleOutcome::Failed, + (ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Saved) if !has_failed_dirty_usage => { + ScannerCycleOutcome::Partial + } + (ScannerCycleStatus::Incomplete, _) => ScannerCycleOutcome::Failed, + (ScannerCycleStatus::Complete, DataUsagePersistOutcome::Saved) => ScannerCycleOutcome::Completed, + (ScannerCycleStatus::Complete, DataUsagePersistOutcome::Current) if !has_dirty_usage => ScannerCycleOutcome::Completed, + (ScannerCycleStatus::Complete, _) => ScannerCycleOutcome::Failed, + } +} + +fn finalize_scanner_cycle_result( + scan_cycle_result: crate::scanner_io::ScannerCycleResult, + usage_persist_outcome: DataUsagePersistOutcome, +) -> (ScannerCycleOutcome, bool) { + let completion_outcome = scanner_cycle_completion_outcome( + scan_cycle_result.status, + usage_persist_outcome, + scan_cycle_result.has_dirty_usage_to_acknowledge(), + scan_cycle_result.has_failed_dirty_usage(), + ); + let pending_maintenance_work = scan_cycle_result.has_pending_maintenance_work(); + if usage_persist_outcome == DataUsagePersistOutcome::Saved { + scan_cycle_result.acknowledge_durable_usage(); + } + (completion_outcome, pending_maintenance_work) +} + /// Decide whether an incoming usage snapshot must be skipped as stale, given the local /// wall clock `now`. Mirrors `stale_data_usage_persist_reason` in /// `crates/ecstore/src/data_usage/mod.rs` — keep the two consistent. @@ -1070,9 +2307,18 @@ fn stale_data_usage_update_reason( pub async fn store_data_usage_in_backend( ctx: CancellationToken, storeapi: Arc, - mut receiver: mpsc::Receiver, + receiver: mpsc::Receiver, ) { + let _ = store_data_usage_in_backend_with_outcome(ctx, storeapi, receiver).await; +} + +async fn store_data_usage_in_backend_with_outcome( + ctx: CancellationToken, + storeapi: Arc, + mut receiver: mpsc::Receiver, +) -> DataUsagePersistOutcome { let mut attempts = 1u32; + let mut outcome = DataUsagePersistOutcome::NoUpdate; while let Some(data_usage_info) = receiver.recv().await { let _activity_guard = ScannerActivityGuard::new(); @@ -1097,10 +2343,10 @@ pub async fn store_data_usage_in_backend( "Scanner stale data usage update skipped" ); global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::SkippedStale); + outcome = DataUsagePersistOutcome::Current; continue; } - // Serialize to JSON let data = match serde_json::to_vec(&data_usage_info) { Ok(data) => data, Err(e) => { @@ -1115,13 +2361,12 @@ pub async fn store_data_usage_in_backend( "Scanner data usage encode failed" ); global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::EncodeFailed); + outcome = DataUsagePersistOutcome::Failed; continue; } }; - let backup_data = (attempts > 10).then(|| data.clone()); - // Save main configuration let done_save = Metrics::time(Metric::SaveUsage); let save_result = save_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data).await; done_save(); @@ -1138,11 +2383,12 @@ pub async fn store_data_usage_in_backend( "Scanner data usage save failed" ); global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed); + outcome = DataUsagePersistOutcome::Failed; } else { replace_bucket_usage_memory_from_info(&data_usage_info).await; global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); + outcome = DataUsagePersistOutcome::Saved; - // Save a backup only after the primary usage object is durable. if let Some(data) = backup_data { let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); let done_save = Metrics::time(Metric::SaveUsage); @@ -1165,6 +2411,8 @@ pub async fn store_data_usage_in_backend( attempts += 1; } + + outcome } #[cfg(test)] @@ -1178,12 +2426,25 @@ mod tests { use serial_test::serial; use std::collections::HashMap; use std::io::Cursor; + use std::task::Poll; use temp_env::{with_var, with_var_unset}; use tokio::io::AsyncReadExt; use tokio::sync::Mutex; const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60; + fn assert_run_data_scanner_signature(_run: F) + where + F: Fn(CancellationToken, Arc) -> Fut, + Fut: Future>, + { + } + + #[test] + fn run_data_scanner_keeps_its_two_argument_api() { + assert_run_data_scanner_signature(run_data_scanner); + } + struct ScannerDefaultSpeedGuard; impl ScannerDefaultSpeedGuard { @@ -1515,7 +2776,7 @@ mod tests { started: Utc::now(), }; - finalize_partial_scan_cycle(store.clone(), &mut cycle_info).await; + assert!(finalize_partial_scan_cycle(store.clone(), &mut cycle_info).await); assert_eq!(cycle_info.next, 13); assert_eq!(cycle_info.current, 0); @@ -1536,6 +2797,26 @@ mod tests { global_metrics().set_cycle(None).await; } + #[tokio::test] + #[serial] + async fn test_finalize_partial_scan_cycle_reports_persist_failure() { + let store = Arc::new(MemoryConfigStore::default()); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + store.fail_put_number.lock().await.insert(key, 1); + let mut cycle_info = CurrentCycle { + current: 12, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + + assert!(!finalize_partial_scan_cycle(store, &mut cycle_info).await); + assert_eq!(cycle_info.next, 13); + assert_eq!(cycle_info.current, 0); + + global_metrics().set_cycle(None).await; + } + #[tokio::test] async fn test_store_data_usage_in_backend_preserves_newer_snapshot() { let store = Arc::new(MemoryConfigStore::default()); @@ -1557,7 +2838,7 @@ mod tests { sender.send(older).await.expect("older usage snapshot should enqueue"); drop(sender); - store_data_usage_in_backend(ctx, store.clone(), receiver).await; + let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; let objects = store.objects.lock().await; let saved = objects @@ -1567,6 +2848,7 @@ mod tests { assert_eq!(saved.buckets_count, 2); assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20))); + assert_eq!(outcome, DataUsagePersistOutcome::Current); } #[tokio::test] @@ -1596,7 +2878,7 @@ mod tests { .expect("untimestamped usage snapshot should enqueue"); drop(sender); - store_data_usage_in_backend(ctx, store.clone(), receiver).await; + let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; let objects = store.objects.lock().await; let saved = objects @@ -1606,6 +2888,7 @@ mod tests { assert_eq!(saved.buckets_count, 2); assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20))); + assert_eq!(outcome, DataUsagePersistOutcome::Current); } fn usage_with_last_update(last_update: Option) -> DataUsageInfo { @@ -1704,7 +2987,7 @@ mod tests { } drop(sender); - store_data_usage_in_backend(ctx, store.clone(), receiver).await; + let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; let objects = store.objects.lock().await; assert_eq!( @@ -1712,13 +2995,188 @@ mod tests { Some(&old_backup), "primary save failure must not overwrite the previous backup" ); - let saved = objects .get(&main_key) .expect("last successful primary usage snapshot should remain saved"); let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); assert_eq!(saved.buckets_count, 10); assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10))); + assert_eq!(outcome, DataUsagePersistOutcome::Failed); + } + + #[tokio::test] + async fn test_store_data_usage_in_backend_reports_missing_snapshot() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(1); + let ctx = CancellationToken::new(); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome(ctx, store, receiver).await; + + assert_eq!(outcome, DataUsagePersistOutcome::NoUpdate); + } + + #[test] + fn test_scanner_cycle_completion_prioritizes_persist_failure() { + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Failed, true, true), + ScannerCycleOutcome::Failed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::NoUpdate, true, true), + ScannerCycleOutcome::Failed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Saved, true, false), + ScannerCycleOutcome::Partial + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Saved, true, true), + ScannerCycleOutcome::Failed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Saved, true, false), + ScannerCycleOutcome::Completed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Current, false, false), + ScannerCycleOutcome::Completed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Current, true, false), + ScannerCycleOutcome::Failed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::NoUpdate, false, false), + ScannerCycleOutcome::Failed + ); + } + + #[test] + #[serial] + fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() { + crate::scanner_io::clear_dirty_usage_bucket("photos"); + crate::scanner_io::record_dirty_usage_bucket("photos"); + let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests(); + + let unsaved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot.clone())); + let (outcome, _) = finalize_scanner_cycle_result(unsaved, DataUsagePersistOutcome::NoUpdate); + assert_eq!(outcome, ScannerCycleOutcome::Failed); + assert!(crate::scanner_io::dirty_usage_buckets_pending()); + + let saved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot)); + let (outcome, _) = finalize_scanner_cycle_result(saved, DataUsagePersistOutcome::Saved); + assert_eq!(outcome, ScannerCycleOutcome::Completed); + assert!(!crate::scanner_io::dirty_usage_buckets_pending()); + } + + #[tokio::test] + async fn data_usage_persist_wait_aborts_when_scanner_is_cancelled() { + let ctx = CancellationToken::new(); + let mut task = tokio::spawn(async { + std::future::pending::<()>().await; + DataUsagePersistOutcome::Saved + }); + ctx.cancel(); + + let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(60)).await; + + assert!(matches!(result, DataUsagePersistTaskResult::Cancelled)); + assert!(task.is_finished()); + } + + #[tokio::test(start_paused = true)] + async fn data_usage_persist_wait_aborts_after_timeout() { + let ctx = CancellationToken::new(); + let mut task = tokio::spawn(async { + std::future::pending::<()>().await; + DataUsagePersistOutcome::Saved + }); + + let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(30)).await; + + assert!(matches!(result, DataUsagePersistTaskResult::TimedOut)); + assert!(task.is_finished()); + } + + #[tokio::test(start_paused = true)] + async fn maintenance_feature_inspection_preserves_base_cycle_after_timeout() { + let ctx = CancellationToken::new(); + + let result = wait_for_maintenance_feature_inspection( + &ctx, + std::future::pending::(), + Duration::from_secs(30), + ) + .await; + + assert_eq!(result, MaintenanceInspectionAttempt::TimedOut); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn stable_maintenance_detection_preserves_base_cycle_after_timeout() { + let ctx = CancellationToken::new(); + + let (features, generation) = detect_stable_scanner_maintenance_features_with( + &ctx, + std::future::pending::, + Duration::from_secs(30), + ) + .await + .expect("timeout should preserve the scanner rather than stop it"); + + assert!(features.inspection_failed); + assert_eq!(generation, scanner_maintenance_generation()); + assert!(!scanner_clean_idle_backoff_enabled( + true, + true, + features, + &ScannerRuntimeConfig::default() + )); + } + + #[tokio::test(start_paused = true)] + async fn failed_maintenance_inspection_uses_bounded_retry_backoff() { + let failed = ScannerMaintenanceFeatures { + inspection_failed: true, + ..Default::default() + }; + let mut retry = ScannerMaintenanceInspectionRetry::from_features(failed, Instant::now()); + + assert_eq!(retry.retry_interval(), Some(MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL)); + assert!(!retry.retry_due(failed, ScannerCycleWakeReason::Timer, Instant::now())); + tokio::time::advance(MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL).await; + assert!(retry.retry_due(failed, ScannerCycleWakeReason::Timer, Instant::now())); + assert!(!retry.retry_due(failed, ScannerCycleWakeReason::DirtyUsage, Instant::now())); + + retry.record_inspection(failed, Instant::now()); + assert_eq!( + retry.retry_interval(), + Some(MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL.saturating_mul(2)) + ); + for _ in 0..8 { + retry.record_inspection(failed, Instant::now()); + } + assert_eq!(retry.retry_interval(), Some(MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL)); + + retry.record_inspection(ScannerMaintenanceFeatures::default(), Instant::now()); + assert_eq!(retry, ScannerMaintenanceInspectionRetry::default()); + } + + #[tokio::test] + async fn maintenance_feature_inspection_stops_on_cancellation() { + let ctx = CancellationToken::new(); + ctx.cancel(); + + let result = wait_for_maintenance_feature_inspection( + &ctx, + std::future::pending::(), + Duration::from_secs(30), + ) + .await; + + assert_eq!(result, MaintenanceInspectionAttempt::Cancelled); } #[test] @@ -1789,6 +3247,411 @@ mod tests { assert_eq!(single_disk_default_speed(), ScannerSpeed::Default); } + #[test] + fn test_maintenance_feature_inspection_is_bounded_and_conservative() { + assert_eq!(maintenance_inspection_decision(1, 1, 1), MaintenanceInspectionDecision::Accept); + assert_eq!(maintenance_inspection_decision(1, 2, 1), MaintenanceInspectionDecision::Retry); + assert_eq!( + maintenance_inspection_decision(1, 2, MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS), + MaintenanceInspectionDecision::PreserveBaseCycle + ); + } + + #[test] + fn clean_idle_backoff_grows_to_cap() { + let base_interval = Duration::from_secs(60); + let max_interval = CLEAN_IDLE_MAX_INTERVAL; + let mut backoff = ScannerCleanIdleBackoff::default(); + + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), Duration::from_secs(60)); + for expected_secs in [ + 120, 240, 480, 960, 1_920, 3_840, 7_680, 15_360, 30_720, 61_440, 86_400, 86_400, + ] { + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + assert_eq!( + backoff.effective_interval(base_interval, max_interval, true), + Duration::from_secs(expected_secs) + ); + } + } + + #[test] + fn scanner_cycle_wait_plan_drives_growth_resets_and_bitrot_cap() { + let runtime_config = ScannerRuntimeConfig { + cycle_interval: Duration::from_secs(60), + bitrot_cycle: None, + ..Default::default() + }; + let mut clean_idle_backoff = ScannerCleanIdleBackoff::default(); + + let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, std::convert::identity); + assert_eq!(plan.delay, Duration::from_secs(60)); + + for expected in [120, 240] { + record_scanner_cycle_result( + &mut clean_idle_backoff, + &runtime_config, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, std::convert::identity); + assert_eq!(plan.delay, Duration::from_secs(expected)); + } + + for (wake_reason, outcome, dirty_work_observed) in [ + (ScannerCycleWakeReason::Timer, ScannerCycleOutcome::Completed, true), + (ScannerCycleWakeReason::Timer, ScannerCycleOutcome::Partial, false), + (ScannerCycleWakeReason::Timer, ScannerCycleOutcome::Failed, false), + (ScannerCycleWakeReason::Timer, ScannerCycleOutcome::CompletedWithPendingMaintenance, false), + (ScannerCycleWakeReason::DirtyUsage, ScannerCycleOutcome::Completed, false), + ] { + record_scanner_cycle_result( + &mut clean_idle_backoff, + &runtime_config, + true, + wake_reason, + outcome, + dirty_work_observed, + ); + let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, std::convert::identity); + assert_eq!(plan.effective_interval, Duration::from_secs(60)); + assert_eq!(plan.delay, Duration::from_secs(60)); + + record_scanner_cycle_result( + &mut clean_idle_backoff, + &runtime_config, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + } + + clean_idle_backoff.reset(); + for _ in 0..32 { + record_scanner_cycle_result( + &mut clean_idle_backoff, + &runtime_config, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + } + let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, |interval| interval.mul_f64(1.1)); + assert_eq!(plan.effective_interval, CLEAN_IDLE_MAX_INTERVAL); + assert!(plan.delay < CLEAN_IDLE_MAX_INTERVAL); + assert_eq!( + plan.delay, + CLEAN_IDLE_MAX_INTERVAL.saturating_sub(CLEAN_IDLE_MAX_INTERVAL.mul_f64(1.1) - CLEAN_IDLE_MAX_INTERVAL) + ); + } + + #[test] + #[serial] + fn scanner_cycle_schedule_status_reports_effective_backoff() { + record_scanner_cycle_schedule(Duration::from_millis(86_400_001), true, 2_048); + + let status = scanner_cycle_schedule_status(); + + assert_eq!(status.effective_interval_seconds, 86_401); + assert!(status.clean_idle_backoff_enabled); + assert_eq!(status.clean_idle_backoff_multiplier, 2_048); + + reset_scanner_cycle_schedule(); + let status = scanner_cycle_schedule_status(); + assert_eq!(status.effective_interval_seconds, 0); + assert!(!status.clean_idle_backoff_enabled); + assert_eq!(status.clean_idle_backoff_multiplier, 1); + } + + #[test] + fn clean_idle_backoff_resets_for_non_idle_work() { + let base_interval = Duration::from_secs(60); + let max_interval = CLEAN_IDLE_MAX_INTERVAL; + let mut backoff = ScannerCleanIdleBackoff::default(); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), Duration::from_secs(240)); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::DirtyUsage, + ScannerCycleOutcome::Completed, + false, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Partial, + false, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Failed, + false, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + true, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::CompletedWithPendingMaintenance, + false, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); + } + + #[test] + fn test_dirty_work_is_observed_across_cycle_waits() { + assert!(scanner_cycle_observed_dirty_work(true, 7, 7)); + assert!(scanner_cycle_observed_dirty_work(false, 7, 8)); + assert!(!scanner_cycle_observed_dirty_work(false, 7, 7)); + } + + #[test] + fn clean_idle_backoff_never_shortens_base_interval() { + let base_interval = Duration::from_secs(48 * 60 * 60); + let mut backoff = ScannerCleanIdleBackoff::default(); + + backoff.record_cycle( + base_interval, + CLEAN_IDLE_MAX_INTERVAL, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + + assert_eq!(backoff.effective_interval(base_interval, CLEAN_IDLE_MAX_INTERVAL, true), base_interval); + } + + #[test] + fn clean_idle_backoff_resets_while_disabled() { + let base_interval = Duration::from_secs(60); + let max_interval = CLEAN_IDLE_MAX_INTERVAL; + let mut backoff = ScannerCleanIdleBackoff::default(); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), Duration::from_secs(240)); + + backoff.record_cycle( + base_interval, + max_interval, + false, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + + assert_eq!(backoff.effective_interval(base_interval, max_interval, false), base_interval); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); + } + + #[test] + fn clean_idle_backoff_policy_preserves_explicit_and_maintenance_cycles() { + let no_features = ScannerMaintenanceFeatures::default(); + let default_config = ScannerRuntimeConfig::default(); + assert!(scanner_clean_idle_backoff_enabled(true, true, no_features, &default_config)); + assert!(!scanner_clean_idle_backoff_enabled(false, true, no_features, &default_config)); + assert!(!scanner_clean_idle_backoff_enabled(true, false, no_features, &default_config)); + + for source in [ScannerRuntimeConfigSource::Env, ScannerRuntimeConfigSource::Config] { + let mut config = default_config.clone(); + config.cycle_interval_source = source; + assert!(!scanner_clean_idle_backoff_enabled(true, true, no_features, &config)); + } + + for source in [ + ScannerRuntimeConfigSource::Env, + ScannerRuntimeConfigSource::Config, + ScannerRuntimeConfigSource::ScannerCompatConfig, + ] { + let mut explicit_bitrot_config = default_config.clone(); + explicit_bitrot_config.bitrot_cycle = Some(Duration::from_secs(60 * 60)); + explicit_bitrot_config.bitrot_cycle_source = source; + assert!(!scanner_clean_idle_backoff_enabled(true, true, no_features, &explicit_bitrot_config)); + + explicit_bitrot_config.bitrot_cycle = None; + assert!(scanner_clean_idle_backoff_enabled(true, true, no_features, &explicit_bitrot_config)); + } + + for features in [ + ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }, + ScannerMaintenanceFeatures { + replication: true, + ..Default::default() + }, + ScannerMaintenanceFeatures { + inspection_failed: true, + ..Default::default() + }, + ] { + assert!(!scanner_clean_idle_backoff_enabled(true, true, features, &default_config)); + } + } + + #[test] + fn clean_idle_backoff_requires_activity_probes() { + let default_config = ScannerRuntimeConfig::default(); + let no_features = ScannerMaintenanceFeatures::default(); + assert!(scanner_activity_probe_required(true, false, no_features, &default_config)); + assert!(!scanner_activity_probe_required(false, false, no_features, &default_config)); + assert!(!scanner_activity_probe_required(true, true, no_features, &default_config)); + + let mut explicit_cycle = default_config.clone(); + explicit_cycle.cycle_interval_source = ScannerRuntimeConfigSource::Env; + assert!(!scanner_activity_probe_required(true, false, no_features, &explicit_cycle)); + + let lifecycle = ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }; + assert!(!scanner_activity_probe_required(true, false, lifecycle, &default_config)); + } + + #[test] + #[serial] + fn clean_idle_cap_preserves_default_bitrot_coverage_window() { + let config = ScannerRuntimeConfig { + bitrot_cycle: Some(Duration::from_secs(30 * 24 * 60 * 60)), + bitrot_cycle_source: ScannerRuntimeConfigSource::Default, + ..Default::default() + }; + + with_var("RUSTFS_HEAL_OBJECT_SELECT_PROB", Some("1024"), || { + let max_interval = scanner_clean_idle_max_interval(Duration::from_secs(60), &config); + assert_eq!(max_interval, Duration::from_millis(2_531_250)); + let positive_jitter = max_interval.mul_f64(1.1); + let actual_delay = cap_clean_idle_cycle_delay(positive_jitter, max_interval, true); + assert!(actual_delay < max_interval); + assert_eq!(actual_delay, max_interval.saturating_sub(positive_jitter - max_interval)); + assert!(actual_delay.saturating_mul(1024) <= config.bitrot_cycle.expect("bitrot cycle should be configured")); + }); + } + + #[test] + #[serial] + fn clean_idle_cap_allows_policy_max_when_bitrot_is_disabled() { + let config = ScannerRuntimeConfig { + bitrot_cycle: None, + ..Default::default() + }; + + assert_eq!(scanner_clean_idle_max_interval(Duration::from_secs(60), &config), CLEAN_IDLE_MAX_INTERVAL); + } + + #[test] + #[serial] + fn clean_idle_cap_never_shortens_the_base_cycle() { + let config = ScannerRuntimeConfig { + bitrot_cycle: Some(Duration::from_secs(60)), + bitrot_cycle_source: ScannerRuntimeConfigSource::Default, + ..Default::default() + }; + + with_var("RUSTFS_HEAL_OBJECT_SELECT_PROB", Some("1024"), || { + assert_eq!(scanner_clean_idle_max_interval(Duration::from_secs(60), &config), Duration::from_secs(60)); + }); + } + #[test] fn test_single_disk_default_cycle_preserves_regular_cycle_for_lifecycle() { assert_eq!( @@ -1892,29 +3755,431 @@ mod tests { #[tokio::test] #[serial] async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() { - crate::scanner_io::clear_dirty_usage_bucket("photos"); - crate::scanner_io::record_dirty_usage_bucket("photos"); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let ctx = CancellationToken::new(); - let reason = tokio::time::timeout(Duration::from_secs(1), wait_for_next_scanner_cycle(&ctx, Duration::from_secs(60))) + let dirty_generation = crate::scanner_io::dirty_usage_generation(); + let mut wait = Box::pin(wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + dirty_generation, + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + || false, + )); + assert!(matches!(futures::poll!(&mut wait), Poll::Pending)); + + crate::scanner_io::record_dirty_usage_bucket("photos"); + let reason = tokio::time::timeout(Duration::from_secs(1), wait) .await .expect("dirty usage should wake scanner before timer"); assert_eq!(reason, ScannerCycleWakeReason::DirtyUsage); - crate::scanner_io::clear_dirty_usage_bucket("photos"); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); } #[tokio::test] #[serial] - async fn test_wait_for_next_scanner_cycle_sees_existing_dirty_usage() { - crate::scanner_io::clear_dirty_usage_bucket("photos"); + async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let dirty_generation = crate::scanner_io::dirty_usage_generation(); crate::scanner_io::record_dirty_usage_bucket("photos"); let ctx = CancellationToken::new(); - let reason = wait_for_next_scanner_cycle(&ctx, Duration::from_secs(60)).await; + let reason = wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + dirty_generation, + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + || false, + ) + .await; assert_eq!(reason, ScannerCycleWakeReason::DirtyUsage); - crate::scanner_io::clear_dirty_usage_bucket("photos"); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn test_wait_for_next_scanner_cycle_retries_stable_dirty_usage_on_timer() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + crate::scanner_io::record_dirty_usage_bucket("photos"); + let dirty_generation = crate::scanner_io::dirty_usage_generation(); + let ctx = CancellationToken::new(); + let wait = wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + dirty_generation, + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + || false, + ); + + let reason = wait.await; + + assert_eq!(reason, ScannerCycleWakeReason::Timer); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + } + + #[tokio::test] + #[serial] + async fn test_wait_for_next_scanner_cycle_wakes_for_repeated_dirty_bucket() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + crate::scanner_io::record_dirty_usage_bucket("photos"); + let dirty_generation = crate::scanner_io::dirty_usage_generation(); + let ctx = CancellationToken::new(); + let mut wait = Box::pin(wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + dirty_generation, + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + || false, + )); + assert!(matches!(futures::poll!(&mut wait), Poll::Pending)); + + crate::scanner_io::record_dirty_usage_bucket("photos"); + let reason = tokio::time::timeout(Duration::from_secs(1), wait) + .await + .expect("a newer mutation of an already-dirty bucket should wake scanner"); + + assert_eq!(reason, ScannerCycleWakeReason::DirtyUsage); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + } + + #[tokio::test] + #[serial] + async fn test_wait_for_next_scanner_cycle_reschedules_for_runtime_config() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let observed_generation = crate::runtime_config::scanner_runtime_config_generation(); + let ctx = CancellationToken::new(); + let mut wait = Box::pin(wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + crate::scanner_io::dirty_usage_generation(), + observed_generation, + crate::scanner_io::scanner_maintenance_generation(), + || false, + )); + assert!(matches!(futures::poll!(&mut wait), Poll::Pending)); + + let mut config = rustfs_config::server_config::Config::new(); + config.set_defaults(); + crate::runtime_config::apply_scanner_runtime_config(&config).expect("default scanner config should apply"); + let reason = tokio::time::timeout(Duration::from_secs(1), wait) + .await + .expect("runtime config should wake scanner before timer"); + + assert_eq!(reason, ScannerCycleWakeReason::RuntimeConfig); + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + } + + #[tokio::test] + #[serial] + async fn test_wait_for_next_scanner_cycle_reschedules_for_maintenance_change() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let observed_generation = crate::scanner_io::scanner_maintenance_generation(); + let ctx = CancellationToken::new(); + let mut wait = Box::pin(wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + crate::scanner_io::dirty_usage_generation(), + crate::runtime_config::scanner_runtime_config_generation(), + observed_generation, + || false, + )); + assert!(matches!(futures::poll!(&mut wait), Poll::Pending)); + + crate::scanner_io::record_scanner_maintenance_change("photos"); + let reason = tokio::time::timeout(Duration::from_secs(1), wait) + .await + .expect("maintenance change should wake scanner before timer"); + + assert_eq!(reason, ScannerCycleWakeReason::MaintenanceConfig); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + } + + #[tokio::test] + async fn test_wait_for_next_scanner_cycle_stops_after_leader_lock_loss() { + let ctx = CancellationToken::new(); + let reason = wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + crate::scanner_io::dirty_usage_generation(), + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + || true, + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::LeaderLockLost); + } + + fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_generation: u64) -> ScannerNodeActivity { + ScannerNodeActivity { + instance_id: epoch.to_string(), + namespace_generation, + maintenance_generation, + } + } + + #[test] + fn scanner_activity_observation_requires_a_complete_baseline() { + let mut seen = None; + let first = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + + let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(first.clone())); + assert_eq!(observation, ScannerActivityObservation::Unverified); + assert!(error.is_none()); + + let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(first)); + assert_eq!(observation, ScannerActivityObservation::Unchanged); + assert!(error.is_none()); + + let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]); + let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(changed)); + assert_eq!(observation, ScannerActivityObservation::Changed); + assert!(error.is_none()); + + let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-b", 8, 0))]); + let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(restarted)); + assert_eq!(observation, ScannerActivityObservation::Changed); + assert!(error.is_none()); + + let (observation, error) = + apply_scanner_activity_probe_result(&mut seen, Err("peer does not support activity probes".to_string())); + assert_eq!(observation, ScannerActivityObservation::Unverified); + assert_eq!(error.as_deref(), Some("peer does not support activity probes")); + assert!(seen.is_none()); + } + + #[test] + fn remote_maintenance_change_is_distinct_from_namespace_activity() { + let previous = BTreeMap::from([ + (LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("local", 5, 2)), + ("node-2".to_string(), scanner_node_activity("remote", 7, 3)), + ]); + let remote_maintenance_changed = BTreeMap::from([ + (LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("local", 5, 2)), + ("node-2".to_string(), scanner_node_activity("remote", 7, 4)), + ]); + assert_eq!( + compare_scanner_activity(&previous, &remote_maintenance_changed), + ScannerActivityObservation::MaintenanceChanged + ); + + let local_maintenance_changed = BTreeMap::from([ + (LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("local", 5, 3)), + ("node-2".to_string(), scanner_node_activity("remote", 7, 3)), + ]); + assert_eq!( + compare_scanner_activity(&previous, &local_maintenance_changed), + ScannerActivityObservation::Changed + ); + } + + #[test] + fn local_maintenance_wakeup_releases_a_remote_maintenance_block() { + let blocked = scanner_activity_backoff_blocked_after_wake(false, ScannerCycleWakeReason::ClusterMaintenance); + assert!(blocked); + + let unblocked = scanner_activity_backoff_blocked_after_wake(blocked, ScannerCycleWakeReason::MaintenanceConfig); + assert!(!unblocked); + assert!(scanner_activity_backoff_blocked_after_wake( + blocked, + ScannerCycleWakeReason::ClusterActivity + )); + } + + #[test] + fn scanner_activity_after_a_cycle_restores_the_base_interval() { + let runtime_config = ScannerRuntimeConfig { + cycle_interval: Duration::from_secs(60), + ..Default::default() + }; + let mut backoff = ScannerCleanIdleBackoff { interval_multiplier: 8 }; + + record_scanner_cycle_result( + &mut backoff, + &runtime_config, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + scanner_activity_observed_work(ScannerActivityObservation::Changed), + ); + + let plan = scanner_cycle_wait_plan(&runtime_config, backoff, true, std::convert::identity); + assert_eq!(plan.effective_interval, Duration::from_secs(60)); + assert_eq!(plan.delay, Duration::from_secs(60)); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn distributed_clean_idle_wait_wakes_at_base_interval_for_remote_activity() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]); + + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + Duration::from_secs(120), + Some(Duration::from_secs(60)), + &mut seen, + ScannerCycleObservedGenerations { + dirty_usage: crate::scanner_io::dirty_usage_generation(), + runtime_config: crate::runtime_config::scanner_runtime_config_generation(), + maintenance: crate::scanner_io::scanner_maintenance_generation(), + }, + || false, + || std::future::ready(Ok(changed.clone())), + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::ClusterActivity); + assert_eq!(seen, Some(changed)); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 4))]); + + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + Duration::from_secs(120), + Some(Duration::from_secs(60)), + &mut seen, + ScannerCycleObservedGenerations { + dirty_usage: crate::scanner_io::dirty_usage_generation(), + runtime_config: crate::runtime_config::scanner_runtime_config_generation(), + maintenance: crate::scanner_io::scanner_maintenance_generation(), + }, + || false, + || std::future::ready(Ok(changed.clone())), + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::ClusterMaintenance); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn distributed_clean_idle_wait_fails_closed_when_a_peer_is_unverifiable() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + Duration::from_secs(120), + Some(Duration::from_secs(60)), + &mut seen, + ScannerCycleObservedGenerations { + dirty_usage: crate::scanner_io::dirty_usage_generation(), + runtime_config: crate::runtime_config::scanner_runtime_config_generation(), + maintenance: crate::scanner_io::scanner_maintenance_generation(), + }, + || false, + || std::future::ready(Err("node-2 is unreachable".to_string())), + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::ClusterActivityUnavailable); + assert!(seen.is_none()); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn distributed_clean_idle_wait_keeps_the_extended_deadline_when_peers_are_clean() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let expected = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let mut seen = Some(expected.clone()); + + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + Duration::from_secs(120), + Some(Duration::from_secs(60)), + &mut seen, + ScannerCycleObservedGenerations { + dirty_usage: crate::scanner_io::dirty_usage_generation(), + runtime_config: crate::runtime_config::scanner_runtime_config_generation(), + maintenance: crate::scanner_io::scanner_maintenance_generation(), + }, + || false, + || std::future::ready(Ok(expected.clone())), + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::Timer); + assert_eq!(seen, Some(expected)); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn scanner_activity_probe_wait_is_cancellation_aware() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let cancel = ctx.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(61)).await; + cancel.cancel(); + }); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + Duration::from_secs(120), + Some(Duration::from_secs(60)), + &mut seen, + ScannerCycleObservedGenerations { + dirty_usage: crate::scanner_io::dirty_usage_generation(), + runtime_config: crate::runtime_config::scanner_runtime_config_generation(), + maintenance: crate::scanner_io::scanner_maintenance_generation(), + }, + || false, + std::future::pending::>, + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::Cancelled); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn scanner_activity_probe_wait_stops_after_leader_lock_loss() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let lock_lost = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let lose_lock = Arc::clone(&lock_lost); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(61)).await; + lose_lock.store(true, std::sync::atomic::Ordering::Release); + }); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + Duration::from_secs(120), + Some(Duration::from_secs(60)), + &mut seen, + ScannerCycleObservedGenerations { + dirty_usage: crate::scanner_io::dirty_usage_generation(), + runtime_config: crate::runtime_config::scanner_runtime_config_generation(), + maintenance: crate::scanner_io::scanner_maintenance_generation(), + }, + || lock_lost.load(std::sync::atomic::Ordering::Acquire), + std::future::pending::>, + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::LeaderLockLost); } #[test] diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 9f4811689..f5d4a5bf2 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -31,7 +31,7 @@ use rustfs_utils::path::path_join_buf; use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration}; use std::collections::{HashMap, HashSet}; use std::path::Path; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::{LazyLock, Mutex as StdMutex, MutexGuard}; use std::time::{Instant, SystemTime}; use std::{fmt::Debug, sync::Arc}; @@ -72,6 +72,13 @@ const METRIC_SCANNER_DISK_BUCKET_SCANS_QUEUED: &str = "rustfs_scanner_disk_bucke pub type DirtyUsageBuckets = HashMap; +#[derive(Clone, Debug)] +struct DirtyUsageSnapshot { + buckets: Arc, + generation: u64, + covers_all_pending: bool, +} + pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool { matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR)) } @@ -115,6 +122,7 @@ pub struct ScannerBucketScanPlan { buckets: Vec, dirty_usage_buckets: Arc, failed_dirty_buckets: Arc>>, + pending_maintenance_work: Arc, } impl ScannerBucketScanPlan { @@ -122,11 +130,13 @@ impl ScannerBucketScanPlan { buckets: Vec, dirty_usage_buckets: Arc, failed_dirty_buckets: Arc>>, + pending_maintenance_work: Arc, ) -> Self { Self { buckets, dirty_usage_buckets, failed_dirty_buckets, + pending_maintenance_work, } } } @@ -134,6 +144,9 @@ impl ScannerBucketScanPlan { static DIRTY_USAGE_BUCKET_GENERATION: AtomicU64 = AtomicU64::new(0); static DIRTY_USAGE_BUCKETS: LazyLock> = LazyLock::new(|| StdMutex::new(HashMap::new())); static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock = LazyLock::new(Notify::new); +static SCANNER_ACTIVITY_EPOCH: LazyLock = LazyLock::new(|| format!("{:032x}", rand::random::())); +static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0); +static SCANNER_MAINTENANCE_NOTIFY: LazyLock = LazyLock::new(Notify::new); fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> { DIRTY_USAGE_BUCKETS.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) @@ -148,9 +161,9 @@ pub fn record_dirty_usage_bucket(bucket: &str) { return; } - let generation = DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel) + 1; let pending_buckets = { let mut dirty_buckets = dirty_usage_buckets(); + let generation = DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel) + 1; dirty_buckets.insert(bucket.to_string(), generation); dirty_buckets.len() }; @@ -158,6 +171,32 @@ pub fn record_dirty_usage_bucket(bucket: &str) { DIRTY_USAGE_BUCKET_NOTIFY.notify_one(); } +pub fn record_scanner_maintenance_change(bucket: &str) { + if bucket.is_empty() { + return; + } + + SCANNER_MAINTENANCE_GENERATION.fetch_add(1, Ordering::AcqRel); + SCANNER_MAINTENANCE_NOTIFY.notify_one(); + record_dirty_usage_bucket(bucket); +} + +pub fn scanner_maintenance_generation() -> u64 { + SCANNER_MAINTENANCE_GENERATION.load(Ordering::Acquire) +} + +pub(crate) async fn scanner_maintenance_changed() { + SCANNER_MAINTENANCE_NOTIFY.notified().await; +} + +pub fn scanner_activity_epoch() -> &'static str { + SCANNER_ACTIVITY_EPOCH.as_str() +} + +pub(crate) fn dirty_usage_generation() -> u64 { + DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire) +} + pub fn clear_dirty_usage_bucket(bucket: &str) { if bucket.is_empty() { return; @@ -166,25 +205,39 @@ pub fn clear_dirty_usage_bucket(bucket: &str) { let pending_buckets = { let mut dirty_buckets = dirty_usage_buckets(); dirty_buckets.remove(bucket); + DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel); dirty_buckets.len() }; global_metrics().record_scanner_dirty_usage_clear(usize_to_u64_saturated(pending_buckets)); } -fn snapshot_dirty_usage_buckets(buckets: &[BucketInfo]) -> DirtyUsageBuckets { - let snapshot = { +fn snapshot_dirty_usage_buckets(buckets: &[BucketInfo], absent_generation_cutoff: u64) -> DirtyUsageSnapshot { + let (snapshot, generation, covers_all_pending) = { let dirty_buckets = dirty_usage_buckets(); - buckets + let listed_buckets = dirty_buckets + .values() + .any(|generation| *generation > absent_generation_cutoff) + .then(|| buckets.iter().map(|bucket| bucket.name.as_str()).collect::>()); + let snapshot = dirty_buckets .iter() - .filter_map(|bucket| { - dirty_buckets - .get(&bucket.name) - .map(|generation| (bucket.name.clone(), *generation)) + .filter(|(bucket, generation)| { + **generation <= absent_generation_cutoff + || listed_buckets + .as_ref() + .is_some_and(|listed_buckets| listed_buckets.contains(bucket.as_str())) }) - .collect::() + .map(|(bucket, generation)| (bucket.clone(), *generation)) + .collect::(); + let generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire); + let covers_all_pending = generation == absent_generation_cutoff && snapshot.len() == dirty_buckets.len(); + (snapshot, generation, covers_all_pending) }; global_metrics().record_scanner_dirty_usage_cycle_snapshot(usize_to_u64_saturated(snapshot.len())); - snapshot + DirtyUsageSnapshot { + buckets: Arc::new(snapshot), + generation, + covers_all_pending, + } } pub(crate) fn dirty_usage_buckets_pending() -> bool { @@ -205,6 +258,9 @@ fn clear_dirty_usage_buckets(snapshot: &DirtyUsageBuckets) { cleared_buckets += 1; } } + if cleared_buckets > 0 { + DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel); + } (cleared_buckets, dirty_buckets.len()) }; global_metrics() @@ -237,13 +293,8 @@ async fn record_failed_dirty_bucket(failed_buckets: &Arc>> failed_buckets.lock().await.insert(bucket.to_string()); } -fn dirty_usage_snapshot_covers_current(snapshot: &DirtyUsageBuckets) -> bool { - let dirty_buckets = dirty_usage_buckets(); - dirty_buckets.iter().all(|(bucket, generation)| { - snapshot - .get(bucket) - .is_some_and(|snapshot_generation| snapshot_generation == generation) - }) +fn dirty_usage_snapshot_covers_current(snapshot: &DirtyUsageSnapshot) -> bool { + snapshot.covers_all_pending && DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire) == snapshot.generation } #[cfg(test)] @@ -252,10 +303,15 @@ fn dirty_usage_bucket_count() -> usize { } #[cfg(test)] -fn clear_dirty_usage_buckets_for_tests() { +pub(crate) fn clear_dirty_usage_buckets_for_tests() { dirty_usage_buckets().clear(); } +#[cfg(test)] +pub(crate) fn dirty_usage_buckets_for_tests() -> DirtyUsageBuckets { + dirty_usage_buckets().clone() +} + fn bucket_usage_scan_order( buckets: &[BucketInfo], old_cache: &DataUsageCache, @@ -479,6 +535,34 @@ fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option ScannerCycleStatus { + if completed_all_sets && !budget_elapsed && !cancelled && !has_failed_buckets && dirty_usage_current { + ScannerCycleStatus::Complete + } else { + ScannerCycleStatus::Incomplete + } +} + +fn scanner_results_have_pending_maintenance_work(results: &[DataUsageCache]) -> bool { + results.iter().any(|result| !result.info.pending_heals.is_empty()) +} + +fn pending_maintenance_work_for_cycle(pending: &AtomicBool, results: &[DataUsageCache]) -> bool { + pending.load(Ordering::Acquire) || scanner_results_have_pending_maintenance_work(results) +} + +fn record_bucket_pending_maintenance_work(cache: &DataUsageCache, pending: &AtomicBool) { + if !cache.info.pending_heals.is_empty() { + pending.store(true, Ordering::Release); + } +} + fn is_xl_meta_path(path: &str) -> bool { Path::new(path) .file_name() @@ -588,7 +672,9 @@ mod publish_gate_tests { async fn send_cache_root_entry_info( bucket_result_tx: &Arc>>, cache: &DataUsageCache, + pending_maintenance_work: &AtomicBool, ) -> std::result::Result<(), mpsc::error::SendError> { + record_bucket_pending_maintenance_work(cache, pending_maintenance_work); bucket_result_tx.lock().await.send(cache_root_entry_info(cache)).await } @@ -656,6 +742,18 @@ pub trait ScannerIO: Send + Sync + Debug + 'static { ) -> Result<()>; } +#[async_trait::async_trait] +pub(crate) trait ScannerIOCycle: Send + Sync + Debug + 'static { + async fn nsscanner_with_status( + &self, + ctx: CancellationToken, + budget: Arc, + updates: mpsc::Sender, + want_cycle: u64, + scan_mode: HealScanMode, + ) -> Result; +} + #[async_trait::async_trait] pub trait ScannerIOCache: Send + Sync + Debug + 'static { async fn nsscanner_cache( @@ -689,9 +787,61 @@ pub enum ScannerDiskScanOutcome { Partial(DataUsageCache), } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ScannerCycleStatus { + Complete, + Incomplete, +} + +#[derive(Debug)] +pub(crate) struct ScannerCycleResult { + pub(crate) status: ScannerCycleStatus, + dirty_usage_clear: Option, + failed_dirty_usage: bool, + pending_maintenance_work: bool, +} + +impl ScannerCycleResult { + pub(crate) fn new(status: ScannerCycleStatus, dirty_usage_clear: Option) -> Self { + Self { + status, + dirty_usage_clear, + failed_dirty_usage: false, + pending_maintenance_work: false, + } + } + + fn with_failed_dirty_usage(mut self, failed_dirty_usage: bool) -> Self { + self.failed_dirty_usage = failed_dirty_usage; + self + } + + fn with_pending_maintenance_work(mut self, pending_maintenance_work: bool) -> Self { + self.pending_maintenance_work = pending_maintenance_work; + self + } + + pub(crate) fn acknowledge_durable_usage(self) { + if let Some(snapshot) = self.dirty_usage_clear { + clear_dirty_usage_buckets(&snapshot); + } + } + + pub(crate) fn has_dirty_usage_to_acknowledge(&self) -> bool { + self.dirty_usage_clear.as_ref().is_some_and(|snapshot| !snapshot.is_empty()) + } + + pub(crate) fn has_failed_dirty_usage(&self) -> bool { + self.failed_dirty_usage + } + + pub(crate) fn has_pending_maintenance_work(&self) -> bool { + self.pending_maintenance_work + } +} + #[async_trait::async_trait] impl ScannerIO for ECStore { - #[tracing::instrument(skip(self, budget, updates))] async fn nsscanner( &self, ctx: CancellationToken, @@ -700,13 +850,40 @@ impl ScannerIO for ECStore { want_cycle: u64, scan_mode: HealScanMode, ) -> Result<()> { + // Preserve the public API's pre-existing completion semantics for + // embedders. The main scanner uses nsscanner_with_status so it can + // delay this acknowledgement until usage persistence succeeds. + ScannerIOCycle::nsscanner_with_status(self, ctx, budget, updates, want_cycle, scan_mode) + .await? + .acknowledge_durable_usage(); + Ok(()) + } +} + +#[async_trait::async_trait] +impl ScannerIOCycle for ECStore { + #[tracing::instrument(skip(self, budget, updates))] + async fn nsscanner_with_status( + &self, + ctx: CancellationToken, + budget: Arc, + updates: mpsc::Sender, + want_cycle: u64, + scan_mode: HealScanMode, + ) -> Result { let child_token = ctx.child_token(); + let dirty_generation_before_bucket_list = dirty_usage_generation(); let all_buckets = self.list_bucket(&BucketOptions::default()).await?; + let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list)); if all_buckets.is_empty() { reset_set_scan_gauges(); - if let Err(e) = updates.send(DataUsageInfo::default()).await { + let empty_usage = DataUsageInfo { + last_update: Some(SystemTime::now()), + ..Default::default() + }; + if let Err(e) = updates.send(empty_usage).await { error!( target: "rustfs::scanner::io", event = EVENT_SCANNER_SET_STATE, @@ -717,7 +894,14 @@ impl ScannerIO for ECStore { "Scanner set state update failed" ); } - return Ok(()); + let status = if dirty_usage_snapshot_covers_current(&dirty_usage_snapshot) { + ScannerCycleStatus::Complete + } else { + ScannerCycleStatus::Incomplete + }; + let dirty_usage_clear = + (status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone()); + return Ok(ScannerCycleResult::new(status, dirty_usage_clear)); } let mut total_results = 0; @@ -735,12 +919,12 @@ impl ScannerIO for ECStore { "Scanner set state update detected missing disk sets" ); reset_set_scan_gauges(); - return Ok(()); + return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)); } let set_scan_limit = scanner_max_concurrent_set_scans(total_results); - let dirty_usage_buckets = Arc::new(snapshot_dirty_usage_buckets(&all_buckets)); let failed_dirty_buckets = Arc::new(Mutex::new(HashSet::::new())); + let pending_maintenance_work = Arc::new(AtomicBool::new(false)); record_set_scan_concurrency_limit(set_scan_limit); debug!( target: "rustfs::scanner::io", @@ -794,8 +978,12 @@ impl ScannerIO for ECStore { }); wait_futs.push(receiver_fut); - let scan_plan = - ScannerBucketScanPlan::new(all_buckets.clone(), dirty_usage_buckets.clone(), failed_dirty_buckets.clone()); + let scan_plan = ScannerBucketScanPlan::new( + all_buckets.clone(), + dirty_usage_snapshot.buckets.clone(), + failed_dirty_buckets.clone(), + pending_maintenance_work.clone(), + ); // Spawn task to run the scanner let scanner_fut = tokio::spawn(async move { let permit_wait = child_token_clone.clone(); @@ -870,7 +1058,7 @@ impl ScannerIO for ECStore { let results_mutex_for_updates = results_mutex.clone(); let budget_for_updates = budget.clone(); let child_token_for_updates = child_token.clone(); - let dirty_usage_buckets_for_updates = dirty_usage_buckets.clone(); + let dirty_usage_snapshot_for_updates = dirty_usage_snapshot.clone(); tokio::spawn(async move { let mut last_update = SystemTime::UNIX_EPOCH; let mut has_sent_once = false; @@ -893,7 +1081,7 @@ impl ScannerIO for ECStore { &all_buckets_clone, budget_for_updates.budget_elapsed(), child_token_for_updates.is_cancelled(), - dirty_usage_snapshot_covers_current(dirty_usage_buckets_for_updates.as_ref()), + dirty_usage_snapshot_covers_current(dirty_usage_snapshot_for_updates.as_ref()), ) }; @@ -912,7 +1100,7 @@ impl ScannerIO for ECStore { &all_buckets_clone, budget_for_updates.budget_elapsed(), child_token_for_updates.is_cancelled(), - dirty_usage_snapshot_covers_current(dirty_usage_buckets_for_updates.as_ref()), + dirty_usage_snapshot_covers_current(dirty_usage_snapshot_for_updates.as_ref()), ) }; @@ -954,16 +1142,27 @@ impl ScannerIO for ECStore { let completed_all_sets = results.iter().all(|result| result.info.last_update.is_some()); let result = finalize_nsscanner_result(&results, first_err); let failed_buckets = failed_dirty_buckets.lock().await.clone(); - if let Some(clear_snapshot) = should_clear_dirty_usage_snapshot( + let pending_maintenance_work = pending_maintenance_work_for_cycle(&pending_maintenance_work, &results); + let budget_elapsed = budget.budget_elapsed(); + let dirty_usage_current = dirty_usage_snapshot_covers_current(&dirty_usage_snapshot); + let cycle_status = classify_nsscanner_cycle( + completed_all_sets, + budget_elapsed, + ctx.is_cancelled(), + !failed_buckets.is_empty(), + dirty_usage_current, + ); + let dirty_usage_clear = should_clear_dirty_usage_snapshot( result.is_ok(), completed_all_sets, - budget.budget_elapsed(), - &dirty_usage_buckets, + budget_elapsed, + &dirty_usage_snapshot.buckets, &failed_buckets, - ) { - clear_dirty_usage_buckets(&clear_snapshot); - } - result + ); + result?; + Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear) + .with_failed_dirty_usage(!failed_buckets.is_empty()) + .with_pending_maintenance_work(pending_maintenance_work)) } } @@ -983,6 +1182,7 @@ impl ScannerIOCache for SetDisks { buckets, dirty_usage_buckets, failed_dirty_buckets, + pending_maintenance_work, } = scan_plan; let pool_label = self.pool_index.to_string(); let set_label = self.set_index.to_string(); @@ -1127,6 +1327,7 @@ impl ScannerIOCache for SetDisks { let pool_label_clone = pool_label.clone(); let set_label_clone = set_label.clone(); let failed_dirty_buckets_clone = failed_dirty_buckets.clone(); + let pending_maintenance_work_clone = pending_maintenance_work.clone(); futs.push(tokio::spawn(async move { loop { let Some(bucket) = bucket_rx_mutex_clone.lock().await.recv().await else { @@ -1322,7 +1523,6 @@ impl ScannerIOCache for SetDisks { continue; } }; - debug!( target: "rustfs::scanner::io", event = EVENT_SCANNER_DISK_BUCKET_STATE, @@ -1349,7 +1549,9 @@ impl ScannerIOCache for SetDisks { "Scanner root entry publish started" ); - if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await { + if let Err(e) = + send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache, &pending_maintenance_work_clone).await + { record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; error!( target: "rustfs::scanner::io", @@ -1761,10 +1963,10 @@ mod tests { clear_dirty_usage_buckets_for_tests(); record_dirty_usage_bucket("photos"); let buckets = vec![bucket_info("photos")]; - let snapshot = snapshot_dirty_usage_buckets(&buckets); + let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation()); record_dirty_usage_bucket("photos"); - clear_dirty_usage_buckets(&snapshot); + clear_dirty_usage_buckets(&snapshot.buckets); assert_eq!(dirty_usage_bucket_count(), 1); clear_dirty_usage_buckets_for_tests(); @@ -1776,7 +1978,7 @@ mod tests { clear_dirty_usage_buckets_for_tests(); record_dirty_usage_bucket("photos"); let buckets = vec![bucket_info("photos")]; - let snapshot = snapshot_dirty_usage_buckets(&buckets); + let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation()); assert!(dirty_usage_snapshot_covers_current(&snapshot)); @@ -1786,6 +1988,83 @@ mod tests { clear_dirty_usage_buckets_for_tests(); } + #[test] + #[serial] + fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() { + clear_dirty_usage_buckets_for_tests(); + record_dirty_usage_bucket("photos"); + record_dirty_usage_bucket("temporarily-omitted"); + let generation_before_bucket_list = dirty_usage_generation(); + + let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], generation_before_bucket_list); + + assert!(snapshot.buckets.contains_key("photos")); + assert!(snapshot.buckets.contains_key("temporarily-omitted")); + assert!(dirty_usage_buckets().contains_key("temporarily-omitted")); + assert!(dirty_usage_snapshot_covers_current(&snapshot)); + + ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone())) + .acknowledge_durable_usage(); + assert!(!dirty_usage_buckets().contains_key("temporarily-omitted")); + clear_dirty_usage_buckets_for_tests(); + } + + #[test] + #[serial] + fn dirty_usage_snapshot_preserves_an_absent_bucket_recorded_after_listing_started() { + clear_dirty_usage_buckets_for_tests(); + let generation_before_bucket_list = dirty_usage_generation(); + record_dirty_usage_bucket("new-or-racing-bucket"); + + let snapshot = snapshot_dirty_usage_buckets(&[], generation_before_bucket_list); + + assert!(!snapshot.buckets.contains_key("new-or-racing-bucket")); + assert!(!dirty_usage_snapshot_covers_current(&snapshot)); + assert!(dirty_usage_buckets().contains_key("new-or-racing-bucket")); + clear_dirty_usage_buckets_for_tests(); + } + + #[test] + #[serial] + fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() { + clear_dirty_usage_buckets_for_tests(); + let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation()); + assert!(dirty_usage_snapshot_covers_current(&snapshot)); + + record_dirty_usage_bucket("photos"); + + assert!(!dirty_usage_snapshot_covers_current(&snapshot)); + assert!(dirty_usage_buckets().contains_key("photos")); + clear_dirty_usage_buckets_for_tests(); + } + + #[test] + #[serial] + fn deleting_a_bucket_during_listing_invalidates_the_resulting_usage_snapshot() { + clear_dirty_usage_buckets_for_tests(); + let generation_before_bucket_list = dirty_usage_generation(); + + record_dirty_usage_bucket("photos"); + let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], generation_before_bucket_list); + + assert!(!dirty_usage_snapshot_covers_current(&snapshot)); + assert!(dirty_usage_buckets().contains_key("photos")); + clear_dirty_usage_buckets_for_tests(); + } + + #[test] + #[serial] + fn scanner_maintenance_change_advances_generation_and_marks_usage_dirty() { + clear_dirty_usage_buckets_for_tests(); + let generation = scanner_maintenance_generation(); + + record_scanner_maintenance_change("photos"); + + assert!(scanner_maintenance_generation() > generation); + assert!(dirty_usage_buckets().contains_key("photos")); + clear_dirty_usage_buckets_for_tests(); + } + #[test] #[serial] fn dirty_usage_clear_excludes_failed_buckets() { @@ -1793,9 +2072,9 @@ mod tests { record_dirty_usage_bucket("photos"); record_dirty_usage_bucket("videos"); let buckets = vec![bucket_info("photos"), bucket_info("videos")]; - let snapshot = snapshot_dirty_usage_buckets(&buckets); + let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation()); let failed_buckets = HashSet::from(["videos".to_string()]); - let clear_snapshot = dirty_usage_buckets_excluding_failed(&snapshot, &failed_buckets); + let clear_snapshot = dirty_usage_buckets_excluding_failed(&snapshot.buckets, &failed_buckets); clear_dirty_usage_buckets(&clear_snapshot); @@ -1818,6 +2097,23 @@ mod tests { assert!(!clear_snapshot.contains_key("videos")); } + #[test] + #[serial] + fn dirty_usage_is_acknowledged_only_after_durable_usage_confirmation() { + clear_dirty_usage_buckets_for_tests(); + record_dirty_usage_bucket("photos"); + let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation()); + + let unconfirmed = ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone())); + drop(unconfirmed); + assert!(dirty_usage_buckets().contains_key("photos")); + + let confirmed = ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone())); + confirmed.acknowledge_durable_usage(); + assert!(!dirty_usage_buckets().contains_key("photos")); + clear_dirty_usage_buckets_for_tests(); + } + #[test] #[serial] fn clear_dirty_usage_bucket_removes_deleted_bucket_marker() { @@ -1828,9 +2124,9 @@ mod tests { clear_dirty_usage_bucket("photos"); let buckets = vec![bucket_info("photos"), bucket_info("videos")]; - let snapshot = snapshot_dirty_usage_buckets(&buckets); - assert!(!snapshot.contains_key("photos")); - assert!(snapshot.contains_key("videos")); + let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation()); + assert!(!snapshot.buckets.contains_key("photos")); + assert!(snapshot.buckets.contains_key("videos")); assert_eq!(dirty_usage_bucket_count(), 1); clear_dirty_usage_buckets_for_tests(); } @@ -1896,6 +2192,78 @@ mod tests { assert!(err.to_string().contains("set failed")); } + #[test] + fn scanner_cycle_status_requires_a_clean_complete_snapshot() { + assert_eq!(classify_nsscanner_cycle(true, false, false, false, true), ScannerCycleStatus::Complete); + + for status in [ + classify_nsscanner_cycle(false, false, false, false, true), + classify_nsscanner_cycle(true, true, false, false, true), + classify_nsscanner_cycle(true, false, true, false, true), + classify_nsscanner_cycle(true, false, false, true, true), + classify_nsscanner_cycle(true, false, false, false, false), + ] { + assert_eq!(status, ScannerCycleStatus::Incomplete); + } + } + + #[test] + fn scanner_cycle_surfaces_persisted_pending_heal_work() { + let clean = DataUsageCache::default(); + assert!(!scanner_results_have_pending_maintenance_work(std::slice::from_ref(&clean))); + + let mut pending = clean; + pending.info.pending_heals.push(crate::PendingScannerHeal { + kind: crate::PendingScannerHealKind::Object, + bucket: "photos".to_string(), + object: Some("image.jpg".to_string()), + version_id: None, + scan_mode: HealScanMode::Normal, + first_seen: 1, + last_attempt: 1, + attempts: 1, + last_admission_result: "queue_full".to_string(), + last_admission_reason: "capacity".to_string(), + }); + + assert!(scanner_results_have_pending_maintenance_work(&[pending])); + } + + #[tokio::test] + async fn bucket_cache_pending_heal_reaches_cycle_maintenance_state() { + let pending_maintenance_work = Arc::new(AtomicBool::new(false)); + let mut bucket_cache = DataUsageCache::default(); + bucket_cache.info.pending_heals.push(crate::PendingScannerHeal { + kind: crate::PendingScannerHealKind::Object, + bucket: "photos".to_string(), + object: Some("image.jpg".to_string()), + version_id: None, + scan_mode: HealScanMode::Normal, + first_seen: 1, + last_attempt: 1, + attempts: 1, + last_admission_result: "queue_full".to_string(), + last_admission_reason: "capacity".to_string(), + }); + let (sender, mut receiver) = mpsc::channel(1); + let sender = Arc::new(Mutex::new(sender)); + + send_cache_root_entry_info(&sender, &bucket_cache, &pending_maintenance_work) + .await + .expect("bucket result should send"); + + let cycle_pending = pending_maintenance_work_for_cycle(&pending_maintenance_work, &[]); + assert!(cycle_pending); + assert_eq!( + crate::scanner::scanner_cycle_outcome_with_pending_maintenance( + crate::scanner::ScannerCycleOutcome::Completed, + cycle_pending, + ), + crate::scanner::ScannerCycleOutcome::CompletedWithPendingMaintenance + ); + assert!(receiver.recv().await.is_some()); + } + #[test] #[serial] fn scanner_concurrency_limit_preserves_available_when_unconfigured() { diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index 26b8bc80c..0881848e8 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -70,6 +70,38 @@ sleep multiplier, maximum wait, and cycle interval. Use `scanner.delay`, `scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis needs a precise override. +## Single-disk clean-idle scheduling + +An erasure single-disk deployment using the built-in cycle and bitrot defaults +automatically backs off repeated clean idle scans instead of walking the same +unchanged namespace every minute. Each successful timer-driven cycle that +finds no dirty usage or unresolved maintenance work doubles the next interval. +The status endpoint reports the effective interval and multiplier. + +The backoff is reset to the base interval by object or bucket mutations, +lifecycle or replication configuration changes, partial or failed cycles, +usage persistence failures, and unresolved scanner-originated heal or bitrot +work. Active lifecycle or replication rules keep the base cadence. An explicit +cycle, a non-default persisted speed, any environment speed or start-delay +override, an environment bitrot override, or a non-default persisted active +bitrot cycle also keeps the configured cadence rather than applying the +automatic policy. Persisting `scanner.speed=default` or the default bitrot cycle +is normalized to the built-in default and therefore keeps automatic scheduling +enabled. + +Lifecycle and replication configuration inspection is bounded so a slow +metadata read cannot stall scanner startup or scheduling. A failed or timed-out +inspection keeps the base cadence and is retried after 5 minutes, doubling up +to a maximum of 60 minutes while failures continue. A lifecycle or replication +configuration change wakes the scanner and retries inspection immediately. + +With the default 30-day bitrot cycle, the clean-idle interval is capped at the +bitrot cycle divided by the object selection window. With the default selection +window this is about 42 minutes, which preserves the intended wall-clock bitrot +coverage. If periodic bitrot is disabled, the clean-idle policy cap is 24 hours. +The effective interval is jittered by up to 10 percent to avoid synchronized +scanner starts. + ## Status Endpoint The scanner status route is: @@ -79,9 +111,12 @@ GET /v3/scanner/status ``` The request must be authenticated with an admin identity that has -`ServerInfoAdminAction`. The JSON response has two top-level objects: +`ServerInfoAdminAction`. The JSON response has three scanner-specific top-level +objects: - `runtime_config`: the effective runtime controls and their value sources. +- `cycle_schedule`: the current effective cycle interval and clean-idle + backoff state. - `metrics`: scanner work, pressure, checkpoint, lifecycle, replication, heal, bitrot, and alert counters. @@ -93,6 +128,9 @@ runtime_config.delay.value runtime_config.max_wait_seconds.value runtime_config.cycle_interval_seconds.value runtime_config.bitrot_cycle_seconds.value +cycle_schedule.effective_interval_seconds +cycle_schedule.clean_idle_backoff_enabled +cycle_schedule.clean_idle_backoff_multiplier metrics.pacing_pressure.primary_pressure metrics.pacing_pressure.last_cycle_budget_limited metrics.lifecycle_transition.current_queued diff --git a/rustfs/src/admin/handlers/scanner.rs b/rustfs/src/admin/handlers/scanner.rs index 1a308ecec..859d676d7 100644 --- a/rustfs/src/admin/handlers/scanner.rs +++ b/rustfs/src/admin/handlers/scanner.rs @@ -37,6 +37,7 @@ struct ScannerStatusResponse { disabled_reason: Option, freshness: ScannerFreshnessStatus, metrics: ScannerMetricsReport, + cycle_schedule: rustfs_scanner::ScannerCycleScheduleStatus, runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus, } @@ -55,13 +56,15 @@ fn scanner_disabled_reason(enabled: bool) -> Option { fn scanner_freshness_status( metrics: &ScannerMetricsReport, runtime_config: &rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus, + effective_cycle_interval_seconds: u64, ) -> ScannerFreshnessStatus { const FRESHNESS_MULTIPLIER: u64 = 2; - let max_expected_age_seconds = runtime_config + let expected_cycle_interval_seconds = runtime_config .cycle_interval_seconds .value - .saturating_mul(FRESHNESS_MULTIPLIER); + .max(effective_cycle_interval_seconds); + let max_expected_age_seconds = expected_cycle_interval_seconds.saturating_mul(FRESHNESS_MULTIPLIER); if metrics.last_cycle_end_unix_secs == 0 { return ScannerFreshnessStatus { state: "unknown", @@ -90,6 +93,23 @@ fn scanner_freshness_status( } } +fn scanner_status_response( + enabled: bool, + metrics: ScannerMetricsReport, + runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus, + cycle_schedule: rustfs_scanner::ScannerCycleScheduleStatus, +) -> ScannerStatusResponse { + let freshness = scanner_freshness_status(&metrics, &runtime_config, cycle_schedule.effective_interval_seconds()); + ScannerStatusResponse { + enabled, + disabled_reason: scanner_disabled_reason(enabled), + freshness, + metrics, + cycle_schedule, + runtime_config, + } +} + pub fn register_scanner_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( Method::GET, @@ -142,14 +162,8 @@ impl Operation for ScannerStatusHandler { let enabled = scanner_enabled_from_env(); let metrics = current_scanner_metrics_report().await; let runtime_config = rustfs_scanner::scanner_runtime_config_status(); - let freshness = scanner_freshness_status(&metrics, &runtime_config); - let response = ScannerStatusResponse { - enabled, - disabled_reason: scanner_disabled_reason(enabled), - freshness, - metrics, - runtime_config, - }; + let cycle_schedule = rustfs_scanner::scanner_cycle_schedule_status(); + let response = scanner_status_response(enabled, metrics, runtime_config, cycle_schedule); let body = serde_json::to_vec(&response).map_err(|err| { S3Error::with_message(S3ErrorCode::InternalError, format!("failed to encode scanner status: {err}")) })?; @@ -174,7 +188,7 @@ mod tests { let mut runtime_config = rustfs_scanner::scanner_runtime_config_status(); runtime_config.cycle_interval_seconds.value = 60; - let freshness = scanner_freshness_status(&metrics, &runtime_config); + let freshness = scanner_freshness_status(&metrics, &runtime_config, 0); assert_eq!(freshness.state, "unknown"); assert_eq!(freshness.last_cycle_end_unix_secs, 0); @@ -191,10 +205,43 @@ mod tests { let mut runtime_config = rustfs_scanner::scanner_runtime_config_status(); runtime_config.cycle_interval_seconds.value = 60; - let freshness = scanner_freshness_status(&metrics, &runtime_config); + let freshness = scanner_freshness_status(&metrics, &runtime_config, 0); assert_eq!(freshness.state, "stale"); assert_eq!(freshness.max_expected_age_seconds, 120); assert_eq!(freshness.reason, Some("last cycle is older than freshness window")); } + + #[test] + fn scanner_freshness_uses_effective_clean_idle_interval() { + let metrics = ScannerMetricsReport { + last_cycle_end_unix_secs: u64::try_from(Utc::now().timestamp().max(0)) + .expect("non-negative timestamp should fit in u64") + .saturating_sub(300), + ..Default::default() + }; + let mut runtime_config = rustfs_scanner::scanner_runtime_config_status(); + runtime_config.cycle_interval_seconds.value = 60; + + let freshness = scanner_freshness_status(&metrics, &runtime_config, 3_600); + + assert_eq!(freshness.state, "fresh"); + assert_eq!(freshness.max_expected_age_seconds, 7_200); + assert_eq!(freshness.reason, None); + } + + #[test] + fn scanner_status_serializes_cycle_schedule_contract() { + let response = scanner_status_response( + true, + ScannerMetricsReport::default(), + rustfs_scanner::scanner_runtime_config_status(), + rustfs_scanner::ScannerCycleScheduleStatus::default(), + ); + + let encoded = serde_json::to_value(response).expect("scanner status should serialize"); + assert_eq!(encoded["cycle_schedule"]["effective_interval_seconds"], 0); + assert_eq!(encoded["cycle_schedule"]["clean_idle_backoff_enabled"], false); + assert_eq!(encoded["cycle_schedule"]["clean_idle_backoff_multiplier"], 1); + } } diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 72fbd6a8c..2de3f05bf 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -245,11 +245,11 @@ pub(crate) mod metadata_sys { } pub(crate) async fn update(bucket: &str, config_file: &str, data: Vec) -> Result { - super::ecstore_bucket::metadata_sys::update(bucket, config_file, data).await + crate::storage::storage_api::update_bucket_metadata_config(bucket, config_file, data).await } pub(crate) async fn delete(bucket: &str, config_file: &str) -> Result { - super::ecstore_bucket::metadata_sys::delete(bucket, config_file).await + crate::storage::storage_api::delete_bucket_metadata_config(bucket, config_file).await } pub(crate) async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> { diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 4eabbb720..c7c91c3b1 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -485,16 +485,29 @@ fn notify_bucket_metadata_reload( bucket: String, operation: &'static str, request_context: Option, + scanner_maintenance_change: bool, ) { + record_local_scanner_maintenance_reload(&bucket, scanner_maintenance_change); spawn_background_with_context(request_context, async move { - if let Some(notification_sys) = current_notification_system() - && let Err(err) = notification_sys.load_bucket_metadata(&bucket).await - { - warn!(bucket = %bucket, error = %err, "failed to notify peers after {operation}"); + if let Some(notification_sys) = current_notification_system() { + let result = if scanner_maintenance_change { + notification_sys.load_bucket_metadata_for_scanner_maintenance(&bucket).await + } else { + notification_sys.load_bucket_metadata(&bucket).await + }; + if let Err(err) = result { + warn!(bucket = %bucket, error = %err, "failed to notify peers after {operation}"); + } } }); } +fn record_local_scanner_maintenance_reload(bucket: &str, scanner_maintenance_change: bool) { + if scanner_maintenance_change { + rustfs_scanner::record_scanner_maintenance_change(bucket); + } +} + /// Notify peers to drop their cached metadata for a bucket that was just deleted, /// so they stop serving stale bucket configuration. Runs in the background to /// avoid blocking the delete response. @@ -1205,7 +1218,9 @@ impl DefaultBucketUsecase { // Invalidate bucket validation cache crate::storage::invalidate_bucket_validation_cache(&input.bucket); - rustfs_scanner::clear_dirty_usage_bucket(&input.bucket); + // Re-evaluate lifecycle/replication after bucket removal and keep an + // absent-bucket dirty marker until a complete usage snapshot is durable. + rustfs_scanner::record_scanner_maintenance_change(&input.bucket); if let Err(err) = remove_bucket_usage_from_backend(store.clone(), &input.bucket).await { warn!(bucket = %input.bucket, error = ?err, "failed to remove deleted bucket from data usage"); } @@ -1338,7 +1353,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "delete bucket encryption", request_context); + notify_bucket_metadata_reload(bucket.clone(), "delete bucket encryption", request_context, false); let item = sr_bucket_meta_item(bucket.clone(), "sse-config"); if let Err(err) = site_replication_bucket_meta_hook(item).await { @@ -1369,7 +1384,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "delete bucket cors", request_context); + notify_bucket_metadata_reload(bucket.clone(), "delete bucket cors", request_context, false); let item = sr_bucket_meta_item(bucket.clone(), "cors-config"); if let Err(err) = site_replication_bucket_meta_hook(item).await { @@ -1400,14 +1415,13 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "delete bucket lifecycle", request_context); + notify_bucket_metadata_reload(bucket.clone(), "delete bucket lifecycle", request_context, true); let item = sr_bucket_meta_item(bucket.clone(), "lc-config"); if let Err(err) = site_replication_bucket_meta_hook(item).await { warn!(bucket = %bucket, error = ?err, "site replication bucket lifecycle delete hook failed"); } - rustfs_scanner::record_dirty_usage_bucket(&bucket); Ok(S3Response::new(DeleteBucketLifecycleOutput::default())) } @@ -1431,7 +1445,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "delete bucket policy", request_context); + notify_bucket_metadata_reload(bucket.clone(), "delete bucket policy", request_context, false); let item = sr_bucket_meta_item(bucket.clone(), "policy"); if let Err(err) = site_replication_bucket_meta_hook(item).await { @@ -1481,7 +1495,7 @@ impl DefaultBucketUsecase { } drop(targets_guard); - notify_bucket_metadata_reload(bucket.clone(), "delete bucket replication", request_context); + notify_bucket_metadata_reload(bucket.clone(), "delete bucket replication", request_context, true); let item = sr_bucket_meta_item(bucket.clone(), "replication-config"); if let Err(err) = site_replication_bucket_meta_hook(item).await { @@ -1490,7 +1504,6 @@ impl DefaultBucketUsecase { info!(bucket = %bucket, "deleted bucket replication config"); - rustfs_scanner::record_dirty_usage_bucket(&bucket); Ok(S3Response::new(DeleteBucketReplicationOutput::default())) } @@ -1506,7 +1519,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "delete bucket tagging", request_context); + notify_bucket_metadata_reload(bucket.clone(), "delete bucket tagging", request_context, false); let item = sr_bucket_meta_item(bucket.clone(), "tags"); if let Err(err) = site_replication_bucket_meta_hook(item).await { @@ -1538,7 +1551,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "delete public access block", request_context); + notify_bucket_metadata_reload(bucket.clone(), "delete public access block", request_context, false); Ok(S3Response::with_status(DeletePublicAccessBlockOutput::default(), StatusCode::NO_CONTENT)) } @@ -1990,7 +2003,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket encryption", request_context); + notify_bucket_metadata_reload(bucket.clone(), "put bucket encryption", request_context, false); let mut item = sr_bucket_meta_item(bucket.clone(), "sse-config"); item.sse_config = Some( @@ -2051,7 +2064,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket lifecycle", request_context); + notify_bucket_metadata_reload(bucket.clone(), "put bucket lifecycle", request_context, true); let mut item = sr_bucket_meta_item(bucket.clone(), "lc-config"); item.expiry_lc_config = @@ -2096,7 +2109,6 @@ impl DefaultBucketUsecase { }); } - rustfs_scanner::record_dirty_usage_bucket(&bucket); Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default())) } @@ -2129,7 +2141,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket notification", request_context); + notify_bucket_metadata_reload(bucket.clone(), "put bucket notification", request_context, false); let region = resolve_notification_region(self.global_region(), request_region); let notify = current_notify_interface_for_context(self.context.as_deref()); @@ -2232,7 +2244,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket policy", request_context); + notify_bucket_metadata_reload(bucket.clone(), "put bucket policy", request_context, false); let mut item = sr_bucket_meta_item(bucket.clone(), "policy"); item.policy = Some(serde_json::from_str(&policy).map_err(|e| s3_error!(InvalidArgument, "parse policy failed {:?}", e))?); @@ -2266,7 +2278,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket cors", request_context); + notify_bucket_metadata_reload(bucket.clone(), "put bucket cors", request_context, false); let mut item = sr_bucket_meta_item(bucket.clone(), "cors-config"); item.cors = @@ -2307,7 +2319,7 @@ impl DefaultBucketUsecase { .map_err(ApiError::from)?; drop(targets_guard); - notify_bucket_metadata_reload(bucket.clone(), "put bucket replication", request_context); + notify_bucket_metadata_reload(bucket.clone(), "put bucket replication", request_context, true); let mut item = sr_bucket_meta_item(bucket.clone(), "replication-config"); item.replication_config = Some( @@ -2317,7 +2329,6 @@ impl DefaultBucketUsecase { warn!(bucket = %bucket, error = ?err, "site replication bucket replication-config hook failed"); } - rustfs_scanner::record_dirty_usage_bucket(&bucket); Ok(S3Response::new(PutBucketReplicationOutput::default())) } @@ -2347,7 +2358,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put public access block", request_context); + notify_bucket_metadata_reload(bucket.clone(), "put public access block", request_context, false); Ok(S3Response::new(PutPublicAccessBlockOutput::default())) } @@ -2375,7 +2386,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket tagging", request_context); + notify_bucket_metadata_reload(bucket.clone(), "put bucket tagging", request_context, false); let mut item = sr_bucket_meta_item(bucket.clone(), "tags"); item.tags = Some(serialize_config(&tagging).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?); @@ -2407,7 +2418,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket versioning", request_context); + notify_bucket_metadata_reload(bucket.clone(), "put bucket versioning", request_context, false); let mut item = sr_bucket_meta_item(bucket.clone(), "version-config"); item.versioning = Some( @@ -2654,23 +2665,23 @@ mod tests { #[test] fn bucket_metadata_config_changes_notify_peer_metadata_reload() { let source = include_str!("bucket_usecase.rs"); - for (method, operation) in [ - ("execute_delete_bucket_policy", "delete bucket policy"), - ("execute_put_bucket_policy", "put bucket policy"), - ("execute_delete_public_access_block", "delete public access block"), - ("execute_put_public_access_block", "put public access block"), - ("execute_delete_bucket_lifecycle", "delete bucket lifecycle"), - ("execute_put_bucket_lifecycle_configuration", "put bucket lifecycle"), - ("execute_put_bucket_versioning", "put bucket versioning"), - ("execute_delete_bucket_tagging", "delete bucket tagging"), - ("execute_put_bucket_tagging", "put bucket tagging"), - ("execute_delete_bucket_replication", "delete bucket replication"), - ("execute_put_bucket_replication", "put bucket replication"), - ("execute_delete_bucket_cors", "delete bucket cors"), - ("execute_put_bucket_cors", "put bucket cors"), - ("execute_delete_bucket_encryption", "delete bucket encryption"), - ("execute_put_bucket_encryption", "put bucket encryption"), - ("execute_put_bucket_notification_configuration", "put bucket notification"), + for (method, operation, scanner_maintenance_change) in [ + ("execute_delete_bucket_policy", "delete bucket policy", false), + ("execute_put_bucket_policy", "put bucket policy", false), + ("execute_delete_public_access_block", "delete public access block", false), + ("execute_put_public_access_block", "put public access block", false), + ("execute_delete_bucket_lifecycle", "delete bucket lifecycle", true), + ("execute_put_bucket_lifecycle_configuration", "put bucket lifecycle", true), + ("execute_put_bucket_versioning", "put bucket versioning", false), + ("execute_delete_bucket_tagging", "delete bucket tagging", false), + ("execute_put_bucket_tagging", "put bucket tagging", false), + ("execute_delete_bucket_replication", "delete bucket replication", true), + ("execute_put_bucket_replication", "put bucket replication", true), + ("execute_delete_bucket_cors", "delete bucket cors", false), + ("execute_put_bucket_cors", "put bucket cors", false), + ("execute_delete_bucket_encryption", "delete bucket encryption", false), + ("execute_put_bucket_encryption", "put bucket encryption", false), + ("execute_put_bucket_notification_configuration", "put bucket notification", false), ] { let body = usecase_method_source(source, method); assert!( @@ -2681,9 +2692,31 @@ mod tests { body.contains(operation), "{method} should identify the bucket metadata operation in reload logs" ); + let expected_reload = format!( + "notify_bucket_metadata_reload(bucket.clone(), \"{operation}\", request_context, {scanner_maintenance_change});" + ); + assert!( + body.contains(&expected_reload), + "{method} should propagate scanner_maintenance_change={scanner_maintenance_change}" + ); } } + #[test] + #[serial_test::serial] + fn scanner_maintenance_reload_marks_only_scanner_owned_config_changes() { + const BUCKET: &str = "scanner-maintenance-reload-test"; + rustfs_scanner::clear_dirty_usage_bucket(BUCKET); + let before = rustfs_scanner::scanner_maintenance_generation(); + + record_local_scanner_maintenance_reload(BUCKET, false); + assert_eq!(rustfs_scanner::scanner_maintenance_generation(), before); + + record_local_scanner_maintenance_reload(BUCKET, true); + assert_eq!(rustfs_scanner::scanner_maintenance_generation(), before.saturating_add(1)); + rustfs_scanner::clear_dirty_usage_bucket(BUCKET); + } + fn replication_rule_for_target(arn: &str) -> ReplicationRule { ReplicationRule { delete_marker_replication: None, diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index d575f79f9..29728d8ef 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -473,7 +473,7 @@ pub(crate) mod bucket { bucket: &str, config_file: &str, ) -> Result { - crate::storage::storage_api::ecstore_bucket::metadata_sys::delete(bucket, config_file).await + crate::storage::storage_api::delete_bucket_metadata_config(bucket, config_file).await } pub(crate) async fn get_bucket_policy( @@ -551,7 +551,7 @@ pub(crate) mod bucket { config_file: &str, data: Vec, ) -> Result { - crate::storage::storage_api::ecstore_bucket::metadata_sys::update(bucket, config_file, data).await + crate::storage::storage_api::update_bucket_metadata_config(bucket, config_file, data).await } } diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 102cc5ac5..f9a1fd930 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -55,6 +55,14 @@ const EVENT_RPC_RESPONSE_EMITTED: &str = "rpc_response_emitted"; const EVENT_RPC_BACKGROUND_TASK_SPAWNED: &str = "rpc_background_task_spawned"; const EVENT_RPC_BACKGROUND_TASK_FAILED: &str = "rpc_background_task_failed"; +fn scanner_activity_response(namespace_generation: u64) -> ScannerActivityResponse { + ScannerActivityResponse { + instance_id: rustfs_scanner::scanner_activity_epoch().to_string(), + namespace_generation, + maintenance_generation: rustfs_scanner::scanner_maintenance_generation(), + } +} + macro_rules! log_load_rebalance_meta_rejected { ($reason:expr, $start_rebalance:expr) => { warn!( @@ -980,6 +988,16 @@ impl Node for NodeService { } } + async fn scanner_activity( + &self, + _request: Request, + ) -> Result, Status> { + let store = self + .resolve_object_store() + .ok_or_else(|| Status::unavailable("storage layer is not initialized"))?; + Ok(Response::new(scanner_activity_response(store.scanner_namespace_mutation_generation()))) + } + async fn background_heal_status( &self, _request: Request, @@ -1216,7 +1234,7 @@ mod tests { use super::{ CollectMetricsOpts, Error, MetricType, Node as _, NodeService, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, - background_rebalance_start_error_message, make_server, stop_rebalance_response, + background_rebalance_start_error_message, make_server, scanner_activity_response, stop_rebalance_response, }; use bytes::Bytes; use rustfs_protos::models::PingBodyBuilder; @@ -1232,9 +1250,10 @@ mod tests { LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest, MakeVolumesRequest, Mss, PingRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest, - RenamePartRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest, StatVolumeRequest, - StopRebalanceRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, - WriteMetadataRequest, WriteRequest, node_service_client::NodeServiceClient, node_service_server::NodeServiceServer, + RenamePartRequest, ScannerActivityRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest, + StatVolumeRequest, StopRebalanceRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, + WriteAllRequest, WriteMetadataRequest, WriteRequest, node_service_client::NodeServiceClient, + node_service_server::NodeServiceServer, }; use std::collections::HashMap; use tokio::net::TcpListener; @@ -2561,8 +2580,12 @@ mod tests { #[tokio::test] async fn test_load_bucket_metadata_empty_bucket() { let service = create_test_node_service(); + let maintenance_generation = rustfs_scanner::scanner_maintenance_generation(); - let request = Request::new(LoadBucketMetadataRequest { bucket: "".to_string() }); + let request = Request::new(LoadBucketMetadataRequest { + bucket: "".to_string(), + scanner_maintenance_change: true, + }); let response = service.load_bucket_metadata(request).await; assert!(response.is_ok()); @@ -2571,6 +2594,11 @@ mod tests { assert!(!load_response.success); assert!(load_response.error_info.is_some()); assert!(load_response.error_info.unwrap().contains("bucket name is missing")); + assert_eq!( + rustfs_scanner::scanner_maintenance_generation(), + maintenance_generation, + "rejected metadata reloads must not advance scanner maintenance activity" + ); } #[tokio::test] @@ -2580,6 +2608,7 @@ mod tests { let request = Request::new(LoadBucketMetadataRequest { bucket: "test-bucket".to_string(), + scanner_maintenance_change: false, }); let response = service.load_bucket_metadata(request).await; @@ -2829,6 +2858,27 @@ mod tests { assert_eq!(signal_response.error_info.as_deref(), Some("unsupported service signal: 99")); } + #[tokio::test] + async fn test_scanner_activity_requires_storage_layer() { + let service = create_test_node_service(); + + let err = service + .scanner_activity(Request::new(ScannerActivityRequest {})) + .await + .expect_err("activity queries must fail closed before storage is initialized"); + + assert_eq!(err.code(), tonic::Code::Unavailable); + } + + #[test] + fn test_scanner_activity_response_uses_process_epoch_and_generations() { + let response = scanner_activity_response(17); + + assert_eq!(response.instance_id, rustfs_scanner::scanner_activity_epoch()); + assert_eq!(response.namespace_generation, 17); + assert_eq!(response.maintenance_generation, rustfs_scanner::scanner_maintenance_generation()); + } + #[tokio::test] async fn test_signal_service_rejects_non_dynamic_subsystem() { let service = create_test_node_service(); diff --git a/rustfs/src/storage/rpc/node_service/bucket.rs b/rustfs/src/storage/rpc/node_service/bucket.rs index 68549eec7..3e7c76586 100644 --- a/rustfs/src/storage/rpc/node_service/bucket.rs +++ b/rustfs/src/storage/rpc/node_service/bucket.rs @@ -57,6 +57,7 @@ impl NodeService { request: Request, ) -> Result, Status> { let request = request.into_inner(); + let scanner_maintenance_change = request.scanner_maintenance_change; let bucket = request.bucket; if bucket.is_empty() { return Ok(Response::new(LoadBucketMetadataResponse { @@ -74,12 +75,15 @@ impl NodeService { match load_bucket_metadata(store, &bucket).await { Ok(meta) => { - if let Err(err) = set_bucket_metadata(bucket, meta).await { + if let Err(err) = set_bucket_metadata(bucket.clone(), meta).await { return Ok(Response::new(LoadBucketMetadataResponse { success: false, error_info: Some(err.to_string()), })); }; + if scanner_maintenance_change { + rustfs_scanner::record_scanner_maintenance_change(&bucket); + } Ok(Response::new(LoadBucketMetadataResponse { success: true, error_info: None, diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index ea3f71bbb..f7b636eda 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -1169,7 +1169,9 @@ pub(crate) fn get_global_bucket_metadata_sys() -> Option Result { - ecstore_bucket::metadata_sys::delete(bucket, config_file).await + let updated_at = ecstore_bucket::metadata_sys::delete(bucket, config_file).await?; + record_scanner_maintenance_config_change(bucket, config_file); + Ok(updated_at) } pub(crate) async fn get_bucket_metadata(bucket: &str) -> Result> { @@ -1241,7 +1243,22 @@ pub(crate) async fn update_bucket_metadata_config( config_file: &str, data: Vec, ) -> Result { - ecstore_bucket::metadata_sys::update(bucket, config_file, data).await + let updated_at = ecstore_bucket::metadata_sys::update(bucket, config_file, data).await?; + record_scanner_maintenance_config_change(bucket, config_file); + Ok(updated_at) +} + +fn record_scanner_maintenance_config_change(bucket: &str, config_file: &str) { + if scanner_maintenance_config_file(config_file) { + rustfs_scanner::record_scanner_maintenance_change(bucket); + } +} + +fn scanner_maintenance_config_file(config_file: &str) -> bool { + matches!( + config_file, + ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG | ecstore_bucket::metadata::BUCKET_REPLICATION_CONFIG + ) } pub(crate) fn add_object_lock_years(dt: time::OffsetDateTime, years: i32) -> time::OffsetDateTime { @@ -1465,7 +1482,9 @@ pub(crate) async fn init_compression_total_memory_from_backend(store: Arc