mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 15:37:02 +00:00
fix(scanner): back off clean idle scans across erasure clusters (#4984)
* fix(scanner): back off clean single-disk cycles * fix(scanner): extend idle backoff across erasure clusters --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
@@ -2434,6 +2434,18 @@ impl Metrics {
|
||||
self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn current_scan_cycle_has_unresolved_heal_work(&self) -> bool {
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
|
||||
[ScannerWorkSource::Heal, ScannerWorkSource::Bitrot]
|
||||
.into_iter()
|
||||
.filter_map(|source| source_work.get(source.index()))
|
||||
.any(|work| work.queued > 0 || work.skipped > 0 || work.failed > 0 || work.missed > 0)
|
||||
}
|
||||
|
||||
fn scan_cycle_work_snapshot(&self) -> ScanCycleWorkSnapshot {
|
||||
ScanCycleWorkSnapshot {
|
||||
objects_scanned: self.lifetime(Metric::ScanObject),
|
||||
@@ -3361,6 +3373,40 @@ mod tests {
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unresolved_heal_work_only_reflects_the_active_cycle() {
|
||||
let metrics = Metrics::new();
|
||||
assert!(!metrics.current_scan_cycle_has_unresolved_heal_work());
|
||||
|
||||
let start = metrics.start_scan_cycle_work();
|
||||
metrics.record_scanner_source_missed(ScannerWorkSource::Heal, 1);
|
||||
assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
|
||||
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
assert!(!metrics.current_scan_cycle_has_unresolved_heal_work());
|
||||
|
||||
let start = metrics.start_scan_cycle_work();
|
||||
metrics.record_scanner_source_queued(ScannerWorkSource::Heal, 1);
|
||||
assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
|
||||
let start = metrics.start_scan_cycle_work();
|
||||
metrics.record_scanner_source_work(
|
||||
ScannerWorkSource::Bitrot,
|
||||
ScannerSourceWorkUpdate {
|
||||
skipped: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
|
||||
let start = metrics.start_scan_cycle_work();
|
||||
metrics.record_scanner_source_failed(ScannerWorkSource::Bitrot, 1);
|
||||
assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_marks_transition_failures_as_blocked_lifecycle_control() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -798,6 +798,13 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn scanner_activity(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ScannerActivityRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::ScannerActivityResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn background_heal_status(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::BackgroundHealStatusRequest>,
|
||||
|
||||
@@ -376,7 +376,7 @@ pub mod rio {
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, TonicInterceptor, gen_tonic_signature_interceptor,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, verify_rpc_signature,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
pub use internode_data_transport::build_internode_data_transport_from_env;
|
||||
pub use peer_rest_client::{
|
||||
PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
ScannerPeerActivity,
|
||||
};
|
||||
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
|
||||
pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys};
|
||||
|
||||
@@ -34,9 +34,9 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest, GetSysConfigRequest,
|
||||
GetSysErrorsRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
|
||||
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ServerInfoRequest,
|
||||
SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ScannerActivityRequest,
|
||||
ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest,
|
||||
StopRebalanceRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use rustfs_utils::XHost;
|
||||
use serde::{Deserialize, Serialize as _};
|
||||
@@ -62,6 +62,31 @@ pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
|
||||
pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
|
||||
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
|
||||
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerPeerActivity {
|
||||
pub instance_id: String,
|
||||
pub namespace_generation: u64,
|
||||
pub maintenance_generation: u64,
|
||||
}
|
||||
|
||||
fn decode_scanner_activity(response: ScannerActivityResponse) -> Result<ScannerPeerActivity> {
|
||||
let instance_id = response.instance_id;
|
||||
if instance_id.len() != 32
|
||||
|| !instance_id
|
||||
.as_bytes()
|
||||
.iter()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte))
|
||||
{
|
||||
return Err(Error::other("peer returned an invalid scanner activity instance ID"));
|
||||
}
|
||||
Ok(ScannerPeerActivity {
|
||||
instance_id,
|
||||
namespace_generation: response.namespace_generation,
|
||||
maintenance_generation: response.maintenance_generation,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerLiveEventsBatch {
|
||||
@@ -639,12 +664,13 @@ impl PeerRestClient {
|
||||
Err(Error::NotImplemented)
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str) -> Result<()> {
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(LoadBucketMetadataRequest {
|
||||
bucket: bucket.to_string(),
|
||||
scanner_maintenance_change,
|
||||
});
|
||||
|
||||
let response = client.load_bucket_metadata(request).await?.into_inner();
|
||||
@@ -908,6 +934,25 @@ impl PeerRestClient {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn scanner_activity(&self) -> Result<ScannerPeerActivity> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await?
|
||||
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
|
||||
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
|
||||
let response = client
|
||||
.scanner_activity(Request::new(ScannerActivityRequest {}))
|
||||
.await?
|
||||
.into_inner();
|
||||
decode_scanner_activity(response)
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn get_metacache_listing(&self) -> Result<()> {
|
||||
warn!("get_metacache_listing is not implemented in PeerRestClient");
|
||||
Err(Error::NotImplemented)
|
||||
@@ -1157,6 +1202,48 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_requires_restart_safe_peer_identity() {
|
||||
let missing_instance = ScannerActivityResponse {
|
||||
instance_id: String::new(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
};
|
||||
assert!(
|
||||
decode_scanner_activity(missing_instance)
|
||||
.expect_err("an empty instance ID is not restart safe")
|
||||
.to_string()
|
||||
.contains("instance ID")
|
||||
);
|
||||
|
||||
let malformed_instance = ScannerActivityResponse {
|
||||
instance_id: "ABCDEF0123456789ABCDEF0123456789".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
};
|
||||
assert!(
|
||||
decode_scanner_activity(malformed_instance)
|
||||
.expect_err("activity instance IDs must use the canonical lowercase hex form")
|
||||
.to_string()
|
||||
.contains("instance ID")
|
||||
);
|
||||
|
||||
let activity = decode_scanner_activity(ScannerActivityResponse {
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
})
|
||||
.expect("complete activity responses should be accepted");
|
||||
assert_eq!(
|
||||
activity,
|
||||
ScannerPeerActivity {
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_marks_network_like_errors() {
|
||||
assert!(PeerRestClient::is_network_like_error(&Error::other("transport error")));
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::PeerRestClient;
|
||||
use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity};
|
||||
use crate::diagnostics::admin_server_info::get_commit_id;
|
||||
use crate::disk::DiskAPI;
|
||||
use crate::error::{Error, Result};
|
||||
@@ -41,6 +41,7 @@ const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3;
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification";
|
||||
const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation";
|
||||
const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Cached result from the last successful admin call to a peer.
|
||||
struct PeerAdminCache {
|
||||
@@ -91,7 +92,6 @@ pub fn get_global_notification_sys() -> Option<Arc<NotificationSys>> {
|
||||
|
||||
pub struct NotificationSys {
|
||||
pub peer_clients: Vec<Option<PeerRestClient>>,
|
||||
#[allow(dead_code)]
|
||||
pub all_peer_clients: Vec<Option<PeerRestClient>>,
|
||||
peer_admin_caches: Vec<Mutex<PeerAdminCache>>,
|
||||
}
|
||||
@@ -756,6 +756,14 @@ impl NotificationSys {
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str) -> Result<()> {
|
||||
self.load_bucket_metadata_with_scanner_maintenance(bucket, false).await
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata_for_scanner_maintenance(&self, bucket: &str) -> Result<()> {
|
||||
self.load_bucket_metadata_with_scanner_maintenance(bucket, true).await
|
||||
}
|
||||
|
||||
async fn load_bucket_metadata_with_scanner_maintenance(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
let operation = format!("load_bucket_metadata({bucket})");
|
||||
let mut failures = Vec::new();
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
@@ -763,7 +771,12 @@ impl NotificationSys {
|
||||
if let Some(client) = client {
|
||||
let host = client.host.to_string();
|
||||
let b = bucket.to_string();
|
||||
futures.push(async move { client.load_bucket_metadata(&b).await.map_err(|err| (host, err)) });
|
||||
futures.push(async move {
|
||||
client
|
||||
.load_bucket_metadata(&b, scanner_maintenance_change)
|
||||
.await
|
||||
.map_err(|err| (host, err))
|
||||
});
|
||||
} else {
|
||||
failures.push(format!("peer[{idx}] {operation} failed: peer is not reachable"));
|
||||
}
|
||||
@@ -976,6 +989,36 @@ impl NotificationSys {
|
||||
join_all(futures).await
|
||||
}
|
||||
|
||||
pub async fn scanner_activity_snapshots(&self) -> Result<Vec<(String, ScannerPeerActivity)>> {
|
||||
if self.peer_clients.is_empty() {
|
||||
return Err(Error::other("scanner activity probe has no remote peers"));
|
||||
}
|
||||
if self.all_peer_clients.len() != self.peer_clients.len() + 1 {
|
||||
return Err(Error::other(format!(
|
||||
"scanner activity peer topology is incomplete: {} remote peers for {} cluster members",
|
||||
self.peer_clients.len(),
|
||||
self.all_peer_clients.len()
|
||||
)));
|
||||
}
|
||||
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for (idx, client) in self.peer_clients.iter().cloned().enumerate() {
|
||||
futures.push(async move {
|
||||
let client = client.ok_or_else(|| Error::other(format!("scanner activity peer[{idx}] is unreachable")))?;
|
||||
let host = client.grid_host.clone();
|
||||
scanner_activity_with_timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, &host, client.scanner_activity())
|
||||
.await
|
||||
.map(|activity| (host, activity))
|
||||
});
|
||||
}
|
||||
|
||||
let mut generations = Vec::with_capacity(futures.len());
|
||||
for result in join_all(futures).await {
|
||||
generations.push(result?);
|
||||
}
|
||||
Ok(generations)
|
||||
}
|
||||
|
||||
pub async fn reload_site_replication_config(&self) -> Vec<NotificationPeerErr> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for client in self.peer_clients.iter() {
|
||||
@@ -1029,6 +1072,15 @@ impl NotificationSys {
|
||||
}
|
||||
}
|
||||
|
||||
async fn scanner_activity_with_timeout<F>(timeout_duration: Duration, host: &str, activity: F) -> Result<ScannerPeerActivity>
|
||||
where
|
||||
F: Future<Output = Result<ScannerPeerActivity>>,
|
||||
{
|
||||
timeout(timeout_duration, activity)
|
||||
.await
|
||||
.map_err(|_| Error::other(format!("scanner activity peer {host} timed out after {timeout_duration:?}")))?
|
||||
}
|
||||
|
||||
async fn call_peer_with_timeout<F, Fut>(
|
||||
timeout_dur: Duration,
|
||||
host_label: &str,
|
||||
@@ -1568,6 +1620,72 @@ mod tests {
|
||||
assert!(msg.contains("peer[0]"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_activity_probe_reports_unreachable_peers() {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: vec![None, None],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
};
|
||||
|
||||
let err = sys
|
||||
.scanner_activity_snapshots()
|
||||
.await
|
||||
.expect_err("unreachable peers must disable scanner idle backoff");
|
||||
|
||||
assert!(err.to_string().contains("scanner activity peer[0] is unreachable"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_activity_probe_rejects_an_empty_peer_set() {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: Vec::new(),
|
||||
peer_admin_caches: Vec::new(),
|
||||
};
|
||||
|
||||
let err = sys
|
||||
.scanner_activity_snapshots()
|
||||
.await
|
||||
.expect_err("a missing peer set must disable scanner idle backoff");
|
||||
|
||||
assert!(err.to_string().contains("no remote peers"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_activity_probe_rejects_an_incomplete_peer_topology() {
|
||||
let client = PeerRestClient::new(
|
||||
"127.0.0.1:9000".to_string().try_into().expect("peer host should parse"),
|
||||
"http://127.0.0.1:9000".to_string(),
|
||||
);
|
||||
let sys = NotificationSys {
|
||||
peer_clients: vec![Some(client)],
|
||||
all_peer_clients: vec![None],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
};
|
||||
|
||||
let err = sys
|
||||
.scanner_activity_snapshots()
|
||||
.await
|
||||
.expect_err("an incomplete peer topology must disable scanner idle backoff");
|
||||
|
||||
assert!(err.to_string().contains("peer topology is incomplete"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_activity_probe_times_out() {
|
||||
let err = scanner_activity_with_timeout(
|
||||
Duration::from_millis(5),
|
||||
"peer-1",
|
||||
std::future::pending::<Result<ScannerPeerActivity>>(),
|
||||
)
|
||||
.await
|
||||
.expect_err("a stalled peer must not block scanner scheduling");
|
||||
|
||||
assert!(err.to_string().contains("timed out"));
|
||||
assert!(err.to_string().contains("peer-1"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_bucket_metadata_reports_unreachable_peers() {
|
||||
let sys = NotificationSys {
|
||||
|
||||
@@ -392,16 +392,28 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn create_bucket_with_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
|
||||
let generation_before_make = ecstore.scanner_namespace_mutation_generation();
|
||||
ecstore
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_make.saturating_add(1),
|
||||
"successful bucket creation should advance scanner namespace activity"
|
||||
);
|
||||
|
||||
let generation_before_put = ecstore.scanner_namespace_mutation_generation();
|
||||
let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
|
||||
ecstore
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("object should be written");
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_put.saturating_add(1),
|
||||
"successful object creation should advance scanner namespace activity"
|
||||
);
|
||||
ecstore
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
@@ -499,6 +511,7 @@ mod tests {
|
||||
create_bucket_with_object(&ecstore, &bucket, object).await;
|
||||
assert!(metadata_sys::get(&bucket).await.is_ok());
|
||||
|
||||
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
|
||||
ecstore
|
||||
.delete_bucket(
|
||||
&bucket,
|
||||
@@ -509,6 +522,11 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("MarkDelete should not reject non-empty bucket data");
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_delete.saturating_add(1),
|
||||
"successful bucket deletion should advance scanner namespace activity"
|
||||
);
|
||||
|
||||
assert!(
|
||||
any_disk_has_object_metadata(&disk_paths, &bucket).await,
|
||||
@@ -536,6 +554,7 @@ mod tests {
|
||||
write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
|
||||
assert!(any_disk_path_exists(&disk_paths, &metadata_prefix).await);
|
||||
|
||||
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
|
||||
ecstore
|
||||
.delete_bucket(
|
||||
&bucket,
|
||||
@@ -547,6 +566,11 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("Purge should force-delete bucket data");
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_delete.saturating_add(1),
|
||||
"successful bucket purge should advance scanner namespace activity"
|
||||
);
|
||||
|
||||
assert!(!any_disk_path_exists(&disk_paths, &bucket).await, "Purge should remove the bucket volume");
|
||||
assert!(
|
||||
@@ -568,12 +592,18 @@ mod tests {
|
||||
|
||||
create_bucket_with_object(&ecstore, &bucket, object).await;
|
||||
|
||||
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
|
||||
let err = ecstore
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect_err("default S3 DeleteBucket should reject non-empty buckets");
|
||||
|
||||
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_delete,
|
||||
"failed bucket deletion must not advance scanner namespace activity"
|
||||
);
|
||||
assert!(
|
||||
any_disk_has_object_metadata(&disk_paths, &bucket).await,
|
||||
"failed default S3 DeleteBucket must keep object data"
|
||||
|
||||
@@ -541,6 +541,7 @@ impl PersistentListMetadataObject {
|
||||
static PERSISTENT_KEY_ONLY_INDEX_CACHE: OnceCell<RwLock<Option<PersistentKeyOnlyIndexCache>>> = OnceCell::const_new();
|
||||
static LIST_OBJECTS_NAMESPACE_JOURNAL_LOCK: OnceCell<RwLock<()>> = OnceCell::const_new();
|
||||
static LIST_OBJECTS_MUTATION_SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
||||
static SCANNER_NAMESPACE_MUTATION_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
static LIST_OBJECTS_BUCKET_MUTATION_SEQUENCE: OnceCell<RwLock<HashMap<String, u64>>> = OnceCell::const_new();
|
||||
static LIST_OBJECTS_NAMESPACE_JOURNAL_DEGRADED_BUCKETS: OnceCell<RwLock<HashSet<String>>> = OnceCell::const_new();
|
||||
static LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_CONFIG: OnceCell<Option<NamespaceMutationJournalChaosConfig>> = OnceCell::const_new();
|
||||
@@ -607,6 +608,19 @@ async fn advance_list_objects_mutation_sequence(bucket: &str, sequence: u64) ->
|
||||
sequence
|
||||
}
|
||||
|
||||
pub(super) fn scanner_namespace_mutation_generation() -> u64 {
|
||||
SCANNER_NAMESPACE_MUTATION_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(super) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) {
|
||||
if bucket == RUSTFS_META_BUCKET {
|
||||
return;
|
||||
}
|
||||
|
||||
let _ = SCANNER_NAMESPACE_MUTATION_GENERATION
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(delta)));
|
||||
}
|
||||
|
||||
pub(super) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 {
|
||||
observe_list_objects_mutations(store, bucket, 1).await.unwrap_or_default()
|
||||
}
|
||||
@@ -621,6 +635,7 @@ async fn observe_list_objects_mutations_with_store(store: Option<&ECStore>, buck
|
||||
}
|
||||
|
||||
let delta = u64::try_from(count).unwrap_or(u64::MAX);
|
||||
observe_scanner_namespace_mutations(bucket, delta);
|
||||
let next = LIST_OBJECTS_MUTATION_SEQUENCE
|
||||
.fetch_add(delta, Ordering::AcqRel)
|
||||
.saturating_add(delta);
|
||||
@@ -641,6 +656,7 @@ async fn current_list_objects_mutation_sequence(bucket: &str) -> u64 {
|
||||
#[cfg(test)]
|
||||
async fn reset_list_objects_mutation_sequences_for_test() {
|
||||
LIST_OBJECTS_MUTATION_SEQUENCE.store(0, Ordering::Release);
|
||||
SCANNER_NAMESPACE_MUTATION_GENERATION.store(0, Ordering::Release);
|
||||
let sequences = list_objects_bucket_mutation_sequence().await;
|
||||
sequences.write().await.clear();
|
||||
let degraded = list_objects_namespace_journal_degraded_buckets().await;
|
||||
@@ -6646,11 +6662,11 @@ mod test {
|
||||
ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend,
|
||||
NamespaceMutationJournalSnapshot, NamespaceMutationJournalStatus, PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER,
|
||||
PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER,
|
||||
PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex, PersistentListMetadataObject, VerifiedIndexCandidateStats,
|
||||
VersionMarker, current_list_objects_mutation_sequence, encode_persistent_list_metadata_object,
|
||||
enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results,
|
||||
latest_listing_allow_agreed_objects, latest_listing_object_quorum, latest_listing_raw_min_disks,
|
||||
latest_listing_required_object_quorum, list_marker_key, list_metadata_resolution_params,
|
||||
PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex, PersistentListMetadataObject, RUSTFS_META_BUCKET,
|
||||
VerifiedIndexCandidateStats, VersionMarker, current_list_objects_mutation_sequence,
|
||||
encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum,
|
||||
fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects, latest_listing_object_quorum,
|
||||
latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key, list_metadata_resolution_params,
|
||||
list_objects_from_metadata_snapshot_candidates, list_objects_from_verified_index_candidates,
|
||||
list_objects_from_verified_index_candidates_with_optional_stats, list_objects_from_verified_index_candidates_with_stats,
|
||||
list_objects_index_mode_from_env, list_objects_index_provider_from_env, list_objects_index_provider_state_from_env,
|
||||
@@ -6664,8 +6680,9 @@ mod test {
|
||||
parse_version_marker, persist_observed_list_objects_mutation, persistent_key_only_index_has_complete_metadata_snapshot,
|
||||
persistent_key_only_index_health, persistent_key_only_index_matches_provider, record_list_objects_index_opt_in_fallback,
|
||||
reset_list_objects_mutation_sequences_for_test, resolve_agreed_listing_entry, resolve_listing_entries,
|
||||
select_list_index_provider_source_mode, select_list_index_source_mode, send_or_cancel, version_marker_for_entries,
|
||||
walk_result_from_set_errors, write_namespace_mutation_journal_state, write_persistent_key_only_index,
|
||||
scanner_namespace_mutation_generation, select_list_index_provider_source_mode, select_list_index_source_mode,
|
||||
send_or_cancel, version_marker_for_entries, walk_result_from_set_errors, write_namespace_mutation_journal_state,
|
||||
write_persistent_key_only_index,
|
||||
};
|
||||
use crate::cache_value::metacache_set::{FallbackClaimTracker, TestReaderBehavior, list_path_raw};
|
||||
use crate::disk::{DiskAPI, DiskOption, endpoint::Endpoint, error::DiskError, new_disk};
|
||||
@@ -8362,6 +8379,19 @@ mod test {
|
||||
assert_eq!(current_list_objects_mutation_sequence("bucket-b").await, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn scanner_namespace_generation_tracks_only_user_namespace_mutations() {
|
||||
reset_list_objects_mutation_sequences_for_test().await;
|
||||
|
||||
assert_eq!(scanner_namespace_mutation_generation(), 0);
|
||||
assert_eq!(observe_list_objects_mutations_with_store(None, RUSTFS_META_BUCKET, 3).await, Some(3));
|
||||
assert_eq!(scanner_namespace_mutation_generation(), 0);
|
||||
|
||||
assert_eq!(observe_list_objects_mutations_with_store(None, "photos", 2).await, Some(5));
|
||||
assert_eq!(scanner_namespace_mutation_generation(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_objects_index_provider_state_uses_lifecycle_active_generation() {
|
||||
let provider = ListObjectsIndexProviderState::from_kind(ListObjectsIndexProviderKind::WalkerKeyOnly);
|
||||
|
||||
@@ -318,6 +318,10 @@ impl ECStore {
|
||||
pub async fn setup_is_erasure_sd(&self) -> bool {
|
||||
self.ctx.is_erasure_sd().await
|
||||
}
|
||||
|
||||
pub fn scanner_namespace_mutation_generation(&self) -> u64 {
|
||||
list_objects::scanner_namespace_mutation_generation()
|
||||
}
|
||||
}
|
||||
|
||||
// impl Clone for ECStore {
|
||||
@@ -436,7 +440,11 @@ impl BucketOperations for ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
|
||||
self.handle_make_bucket(bucket, opts).await
|
||||
let result = self.handle_make_bucket(bucket, opts).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
@@ -449,7 +457,11 @@ impl BucketOperations for ECStore {
|
||||
}
|
||||
#[instrument(skip(self))]
|
||||
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
|
||||
self.handle_delete_bucket(bucket, opts).await
|
||||
let result = self.handle_delete_bucket(bucket, opts).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -923,6 +923,8 @@ pub struct GetAllBucketStatsResponse {
|
||||
pub struct LoadBucketMetadataRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub bucket: ::prost::alloc::string::String,
|
||||
#[prost(bool, tag = "2")]
|
||||
pub scanner_maintenance_change: bool,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct LoadBucketMetadataResponse {
|
||||
@@ -1067,6 +1069,17 @@ pub struct SignalServiceResponse {
|
||||
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerActivityRequest {}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerActivityResponse {
|
||||
#[prost(string, tag = "1")]
|
||||
pub instance_id: ::prost::alloc::string::String,
|
||||
#[prost(uint64, tag = "2")]
|
||||
pub namespace_generation: u64,
|
||||
#[prost(uint64, tag = "3")]
|
||||
pub maintenance_generation: u64,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct BackgroundHealStatusRequest {}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct BackgroundHealStatusResponse {
|
||||
@@ -2351,6 +2364,21 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "SignalService"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn scanner_activity(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::ScannerActivityRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::ScannerActivityResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ScannerActivity");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "ScannerActivity"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn background_heal_status(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::BackgroundHealStatusRequest>,
|
||||
@@ -2825,6 +2853,10 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::SignalServiceRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SignalServiceResponse>, tonic::Status>;
|
||||
async fn scanner_activity(
|
||||
&self,
|
||||
request: tonic::Request<super::ScannerActivityRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::ScannerActivityResponse>, tonic::Status>;
|
||||
async fn background_heal_status(
|
||||
&self,
|
||||
request: tonic::Request<super::BackgroundHealStatusRequest>,
|
||||
@@ -4933,6 +4965,34 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/ScannerActivity" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct ScannerActivitySvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::ScannerActivityRequest> for ScannerActivitySvc<T> {
|
||||
type Response = super::ScannerActivityResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::ScannerActivityRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::scanner_activity(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = ScannerActivitySvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/BackgroundHealStatus" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct BackgroundHealStatusSvc<T: NodeService>(pub Arc<T>);
|
||||
|
||||
@@ -649,6 +649,7 @@ message GetAllBucketStatsResponse {
|
||||
|
||||
message LoadBucketMetadataRequest {
|
||||
string bucket = 1;
|
||||
bool scanner_maintenance_change = 2;
|
||||
}
|
||||
|
||||
message LoadBucketMetadataResponse {
|
||||
@@ -756,6 +757,14 @@ message SignalServiceResponse {
|
||||
optional string error_info = 2;
|
||||
}
|
||||
|
||||
message ScannerActivityRequest {}
|
||||
|
||||
message ScannerActivityResponse {
|
||||
string instance_id = 1;
|
||||
uint64 namespace_generation = 2;
|
||||
uint64 maintenance_generation = 3;
|
||||
}
|
||||
|
||||
message BackgroundHealStatusRequest {}
|
||||
|
||||
message BackgroundHealStatusResponse {
|
||||
@@ -944,6 +953,7 @@ service NodeService {
|
||||
// rpc VerifyBinary() returns () {};
|
||||
// rpc CommitBinary() returns () {};
|
||||
rpc SignalService(SignalServiceRequest) returns (SignalServiceResponse) {};
|
||||
rpc ScannerActivity(ScannerActivityRequest) returns (ScannerActivityResponse) {};
|
||||
rpc BackgroundHealStatus(BackgroundHealStatusRequest) returns (BackgroundHealStatusResponse) {};
|
||||
rpc GetMetacacheListing(GetMetacacheListingRequest) returns (GetMetacacheListingResponse) {};
|
||||
rpc UpdateMetacacheListing(UpdateMetacacheListingRequest) returns (UpdateMetacacheListingResponse) {};
|
||||
|
||||
@@ -58,8 +58,11 @@ pub use data_usage_define::*;
|
||||
pub use error::ScannerError;
|
||||
pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
|
||||
pub use rustfs_common::last_minute;
|
||||
pub use scanner::init_data_scanner;
|
||||
pub use scanner_io::{clear_dirty_usage_bucket, record_dirty_usage_bucket};
|
||||
pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status};
|
||||
pub use scanner_io::{
|
||||
clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch,
|
||||
scanner_maintenance_generation,
|
||||
};
|
||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
||||
|
||||
@@ -38,11 +38,14 @@ use std::fmt;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{LazyLock, RwLock};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Notify;
|
||||
use tracing::warn;
|
||||
|
||||
const LOG_COMPONENT_SCANNER: &str = "scanner";
|
||||
#[cfg(test)]
|
||||
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
|
||||
const LOG_SUBSYSTEM_RUNTIME_CONFIG: &str = "runtime_config";
|
||||
#[cfg(test)]
|
||||
const EVENT_SCANNER_RUNTIME_CONFIG: &str = "scanner_runtime_config";
|
||||
const EVENT_SCANNER_RUNTIME_CONFIG_PARSE: &str = "scanner_runtime_config_parse";
|
||||
|
||||
@@ -52,6 +55,8 @@ const MAX_SCANNER_DELAY_FACTOR: f64 = 10_000.0;
|
||||
const SCANNER_DELAY_RANGE_REASON: &str = "expected scanner delay between 0 and 10000";
|
||||
|
||||
static SCANNER_DEFAULT_CYCLE_SECS: AtomicU64 = AtomicU64::new(NO_DEFAULT_CYCLE_OVERRIDE);
|
||||
static SCANNER_RUNTIME_CONFIG_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
static SCANNER_RUNTIME_CONFIG_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
|
||||
static SCANNER_RUNTIME_CONFIG: LazyLock<RwLock<ScannerRuntimeConfig>> =
|
||||
LazyLock::new(|| RwLock::new(lookup_scanner_runtime_config(None).unwrap_or_default()));
|
||||
@@ -678,6 +683,7 @@ fn current_server_config() -> Option<ServerConfig> {
|
||||
resolve_scanner_server_config()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn resolve_scanner_runtime_config_from_global() -> ScannerRuntimeConfig {
|
||||
let config = current_server_config();
|
||||
match lookup_scanner_runtime_config(config.as_ref()) {
|
||||
@@ -705,9 +711,19 @@ pub fn apply_scanner_runtime_config(config: &ServerConfig) -> Result<(), Scanner
|
||||
validate_scanner_runtime_config(config)?;
|
||||
let resolved = lookup_scanner_runtime_config(Some(config))?;
|
||||
apply_resolved_runtime_config(resolved);
|
||||
SCANNER_RUNTIME_CONFIG_GENERATION.fetch_add(1, Ordering::AcqRel);
|
||||
SCANNER_RUNTIME_CONFIG_NOTIFY.notify_one();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_runtime_config_generation() -> u64 {
|
||||
SCANNER_RUNTIME_CONFIG_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) async fn scanner_runtime_config_changed() {
|
||||
SCANNER_RUNTIME_CONFIG_NOTIFY.notified().await;
|
||||
}
|
||||
|
||||
pub(crate) fn refresh_scanner_runtime_config_from_global() -> Result<(), ScannerRuntimeConfigError> {
|
||||
let config = current_server_config();
|
||||
let resolved = lookup_scanner_runtime_config(config.as_ref())?;
|
||||
@@ -846,11 +862,11 @@ mod tests {
|
||||
use crate::init_ecstore_config_for_scanner_tests;
|
||||
use rustfs_config::server_config::{Config as ServerConfig, KVS};
|
||||
use rustfs_config::{
|
||||
DEFAULT_DELIMITER, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE,
|
||||
ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE,
|
||||
HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES,
|
||||
SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS,
|
||||
ScannerSpeed,
|
||||
DEFAULT_DELIMITER, DEFAULT_HEAL_BITROT_CYCLE_SECS, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS,
|
||||
ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED,
|
||||
HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE,
|
||||
SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
|
||||
SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::collections::HashMap;
|
||||
@@ -927,6 +943,22 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_normalizes_persisted_default_speed() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "default")]);
|
||||
|
||||
with_var_unset(ENV_SCANNER_SPEED, || {
|
||||
with_var_unset(ENV_SCANNER_CYCLE, || {
|
||||
let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
|
||||
|
||||
assert_eq!(resolved.speed, ScannerSpeed::Default);
|
||||
assert_eq!(resolved.speed_source, ScannerRuntimeConfigSource::Default);
|
||||
assert_eq!(resolved.cycle_interval_source, ScannerRuntimeConfigSource::Default);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_prefers_env_over_persisted_config() {
|
||||
@@ -974,6 +1006,23 @@ mod tests {
|
||||
super::refresh_scanner_runtime_config_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_normalizes_persisted_default_bitrot_cycles() {
|
||||
let default_cycle = DEFAULT_HEAL_BITROT_CYCLE_SECS.to_string();
|
||||
for config in [
|
||||
server_config_with_scanner_and_heal(&[], &[(HEAL_BITROT_CYCLE, default_cycle.as_str())]),
|
||||
server_config_with_scanner(&[(SCANNER_BITROT_CYCLE, default_cycle.as_str())]),
|
||||
] {
|
||||
with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || {
|
||||
let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
|
||||
|
||||
assert_eq!(resolved.bitrot_cycle, Some(Duration::from_secs(DEFAULT_HEAL_BITROT_CYCLE_SECS)));
|
||||
assert_eq!(resolved.bitrot_cycle_source, ScannerRuntimeConfigSource::Default);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_runtime_config_rejects_invalid_persisted_speed() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "warp")]);
|
||||
@@ -1058,6 +1107,23 @@ mod tests {
|
||||
super::refresh_scanner_runtime_config_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn applied_runtime_config_is_the_authoritative_scheduler_state() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_CYCLE, "321")]);
|
||||
|
||||
with_var_unset(ENV_SCANNER_CYCLE, || {
|
||||
let generation = super::scanner_runtime_config_generation();
|
||||
super::apply_scanner_runtime_config(&config).expect("scanner runtime config should apply");
|
||||
|
||||
let applied = super::current_scanner_runtime_config();
|
||||
assert_eq!(applied.cycle_interval, Duration::from_secs(321));
|
||||
assert_eq!(applied.cycle_interval_source, ScannerRuntimeConfigSource::Config);
|
||||
assert!(super::scanner_runtime_config_generation() > generation);
|
||||
});
|
||||
super::refresh_scanner_runtime_config_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_status_reports_persisted_pacing_overrides() {
|
||||
|
||||
+2368
-103
File diff suppressed because it is too large
Load Diff
@@ -31,7 +31,7 @@ use rustfs_utils::path::path_join_buf;
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{LazyLock, Mutex as StdMutex, MutexGuard};
|
||||
use std::time::{Instant, SystemTime};
|
||||
use std::{fmt::Debug, sync::Arc};
|
||||
@@ -72,6 +72,13 @@ const METRIC_SCANNER_DISK_BUCKET_SCANS_QUEUED: &str = "rustfs_scanner_disk_bucke
|
||||
|
||||
pub type DirtyUsageBuckets = HashMap<String, u64>;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct DirtyUsageSnapshot {
|
||||
buckets: Arc<DirtyUsageBuckets>,
|
||||
generation: u64,
|
||||
covers_all_pending: bool,
|
||||
}
|
||||
|
||||
pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool {
|
||||
matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR))
|
||||
}
|
||||
@@ -115,6 +122,7 @@ pub struct ScannerBucketScanPlan {
|
||||
buckets: Vec<BucketInfo>,
|
||||
dirty_usage_buckets: Arc<DirtyUsageBuckets>,
|
||||
failed_dirty_buckets: Arc<Mutex<HashSet<String>>>,
|
||||
pending_maintenance_work: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl ScannerBucketScanPlan {
|
||||
@@ -122,11 +130,13 @@ impl ScannerBucketScanPlan {
|
||||
buckets: Vec<BucketInfo>,
|
||||
dirty_usage_buckets: Arc<DirtyUsageBuckets>,
|
||||
failed_dirty_buckets: Arc<Mutex<HashSet<String>>>,
|
||||
pending_maintenance_work: Arc<AtomicBool>,
|
||||
) -> Self {
|
||||
Self {
|
||||
buckets,
|
||||
dirty_usage_buckets,
|
||||
failed_dirty_buckets,
|
||||
pending_maintenance_work,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -134,6 +144,9 @@ impl ScannerBucketScanPlan {
|
||||
static DIRTY_USAGE_BUCKET_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
static DIRTY_USAGE_BUCKETS: LazyLock<StdMutex<DirtyUsageBuckets>> = LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
static SCANNER_ACTIVITY_EPOCH: LazyLock<String> = LazyLock::new(|| format!("{:032x}", rand::random::<u128>()));
|
||||
static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
static SCANNER_MAINTENANCE_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
|
||||
fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
|
||||
DIRTY_USAGE_BUCKETS.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
@@ -148,9 +161,9 @@ pub fn record_dirty_usage_bucket(bucket: &str) {
|
||||
return;
|
||||
}
|
||||
|
||||
let generation = DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let generation = DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
dirty_buckets.insert(bucket.to_string(), generation);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
@@ -158,6 +171,32 @@ pub fn record_dirty_usage_bucket(bucket: &str) {
|
||||
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
||||
}
|
||||
|
||||
pub fn record_scanner_maintenance_change(bucket: &str) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
SCANNER_MAINTENANCE_GENERATION.fetch_add(1, Ordering::AcqRel);
|
||||
SCANNER_MAINTENANCE_NOTIFY.notify_one();
|
||||
record_dirty_usage_bucket(bucket);
|
||||
}
|
||||
|
||||
pub fn scanner_maintenance_generation() -> u64 {
|
||||
SCANNER_MAINTENANCE_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) async fn scanner_maintenance_changed() {
|
||||
SCANNER_MAINTENANCE_NOTIFY.notified().await;
|
||||
}
|
||||
|
||||
pub fn scanner_activity_epoch() -> &'static str {
|
||||
SCANNER_ACTIVITY_EPOCH.as_str()
|
||||
}
|
||||
|
||||
pub(crate) fn dirty_usage_generation() -> u64 {
|
||||
DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub fn clear_dirty_usage_bucket(bucket: &str) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
@@ -166,25 +205,39 @@ pub fn clear_dirty_usage_bucket(bucket: &str) {
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
dirty_buckets.remove(bucket);
|
||||
DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_clear(usize_to_u64_saturated(pending_buckets));
|
||||
}
|
||||
|
||||
fn snapshot_dirty_usage_buckets(buckets: &[BucketInfo]) -> DirtyUsageBuckets {
|
||||
let snapshot = {
|
||||
fn snapshot_dirty_usage_buckets(buckets: &[BucketInfo], absent_generation_cutoff: u64) -> DirtyUsageSnapshot {
|
||||
let (snapshot, generation, covers_all_pending) = {
|
||||
let dirty_buckets = dirty_usage_buckets();
|
||||
buckets
|
||||
let listed_buckets = dirty_buckets
|
||||
.values()
|
||||
.any(|generation| *generation > absent_generation_cutoff)
|
||||
.then(|| buckets.iter().map(|bucket| bucket.name.as_str()).collect::<HashSet<_>>());
|
||||
let snapshot = dirty_buckets
|
||||
.iter()
|
||||
.filter_map(|bucket| {
|
||||
dirty_buckets
|
||||
.get(&bucket.name)
|
||||
.map(|generation| (bucket.name.clone(), *generation))
|
||||
.filter(|(bucket, generation)| {
|
||||
**generation <= absent_generation_cutoff
|
||||
|| listed_buckets
|
||||
.as_ref()
|
||||
.is_some_and(|listed_buckets| listed_buckets.contains(bucket.as_str()))
|
||||
})
|
||||
.collect::<DirtyUsageBuckets>()
|
||||
.map(|(bucket, generation)| (bucket.clone(), *generation))
|
||||
.collect::<DirtyUsageBuckets>();
|
||||
let generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire);
|
||||
let covers_all_pending = generation == absent_generation_cutoff && snapshot.len() == dirty_buckets.len();
|
||||
(snapshot, generation, covers_all_pending)
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_cycle_snapshot(usize_to_u64_saturated(snapshot.len()));
|
||||
snapshot
|
||||
DirtyUsageSnapshot {
|
||||
buckets: Arc::new(snapshot),
|
||||
generation,
|
||||
covers_all_pending,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn dirty_usage_buckets_pending() -> bool {
|
||||
@@ -205,6 +258,9 @@ fn clear_dirty_usage_buckets(snapshot: &DirtyUsageBuckets) {
|
||||
cleared_buckets += 1;
|
||||
}
|
||||
}
|
||||
if cleared_buckets > 0 {
|
||||
DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
(cleared_buckets, dirty_buckets.len())
|
||||
};
|
||||
global_metrics()
|
||||
@@ -237,13 +293,8 @@ async fn record_failed_dirty_bucket(failed_buckets: &Arc<Mutex<HashSet<String>>>
|
||||
failed_buckets.lock().await.insert(bucket.to_string());
|
||||
}
|
||||
|
||||
fn dirty_usage_snapshot_covers_current(snapshot: &DirtyUsageBuckets) -> bool {
|
||||
let dirty_buckets = dirty_usage_buckets();
|
||||
dirty_buckets.iter().all(|(bucket, generation)| {
|
||||
snapshot
|
||||
.get(bucket)
|
||||
.is_some_and(|snapshot_generation| snapshot_generation == generation)
|
||||
})
|
||||
fn dirty_usage_snapshot_covers_current(snapshot: &DirtyUsageSnapshot) -> bool {
|
||||
snapshot.covers_all_pending && DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire) == snapshot.generation
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -252,10 +303,15 @@ fn dirty_usage_bucket_count() -> usize {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn clear_dirty_usage_buckets_for_tests() {
|
||||
pub(crate) fn clear_dirty_usage_buckets_for_tests() {
|
||||
dirty_usage_buckets().clear();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn dirty_usage_buckets_for_tests() -> DirtyUsageBuckets {
|
||||
dirty_usage_buckets().clone()
|
||||
}
|
||||
|
||||
fn bucket_usage_scan_order(
|
||||
buckets: &[BucketInfo],
|
||||
old_cache: &DataUsageCache,
|
||||
@@ -479,6 +535,34 @@ fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option<Error
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn classify_nsscanner_cycle(
|
||||
completed_all_sets: bool,
|
||||
budget_elapsed: bool,
|
||||
cancelled: bool,
|
||||
has_failed_buckets: bool,
|
||||
dirty_usage_current: bool,
|
||||
) -> ScannerCycleStatus {
|
||||
if completed_all_sets && !budget_elapsed && !cancelled && !has_failed_buckets && dirty_usage_current {
|
||||
ScannerCycleStatus::Complete
|
||||
} else {
|
||||
ScannerCycleStatus::Incomplete
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_results_have_pending_maintenance_work(results: &[DataUsageCache]) -> bool {
|
||||
results.iter().any(|result| !result.info.pending_heals.is_empty())
|
||||
}
|
||||
|
||||
fn pending_maintenance_work_for_cycle(pending: &AtomicBool, results: &[DataUsageCache]) -> bool {
|
||||
pending.load(Ordering::Acquire) || scanner_results_have_pending_maintenance_work(results)
|
||||
}
|
||||
|
||||
fn record_bucket_pending_maintenance_work(cache: &DataUsageCache, pending: &AtomicBool) {
|
||||
if !cache.info.pending_heals.is_empty() {
|
||||
pending.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
fn is_xl_meta_path(path: &str) -> bool {
|
||||
Path::new(path)
|
||||
.file_name()
|
||||
@@ -588,7 +672,9 @@ mod publish_gate_tests {
|
||||
async fn send_cache_root_entry_info(
|
||||
bucket_result_tx: &Arc<Mutex<mpsc::Sender<DataUsageEntryInfo>>>,
|
||||
cache: &DataUsageCache,
|
||||
pending_maintenance_work: &AtomicBool,
|
||||
) -> std::result::Result<(), mpsc::error::SendError<DataUsageEntryInfo>> {
|
||||
record_bucket_pending_maintenance_work(cache, pending_maintenance_work);
|
||||
bucket_result_tx.lock().await.send(cache_root_entry_info(cache)).await
|
||||
}
|
||||
|
||||
@@ -656,6 +742,18 @@ pub trait ScannerIO: Send + Sync + Debug + 'static {
|
||||
) -> Result<()>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait ScannerIOCycle: Send + Sync + Debug + 'static {
|
||||
async fn nsscanner_with_status(
|
||||
&self,
|
||||
ctx: CancellationToken,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
updates: mpsc::Sender<DataUsageInfo>,
|
||||
want_cycle: u64,
|
||||
scan_mode: HealScanMode,
|
||||
) -> Result<ScannerCycleResult>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait ScannerIOCache: Send + Sync + Debug + 'static {
|
||||
async fn nsscanner_cache(
|
||||
@@ -689,9 +787,61 @@ pub enum ScannerDiskScanOutcome {
|
||||
Partial(DataUsageCache),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ScannerCycleStatus {
|
||||
Complete,
|
||||
Incomplete,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ScannerCycleResult {
|
||||
pub(crate) status: ScannerCycleStatus,
|
||||
dirty_usage_clear: Option<DirtyUsageBuckets>,
|
||||
failed_dirty_usage: bool,
|
||||
pending_maintenance_work: bool,
|
||||
}
|
||||
|
||||
impl ScannerCycleResult {
|
||||
pub(crate) fn new(status: ScannerCycleStatus, dirty_usage_clear: Option<DirtyUsageBuckets>) -> Self {
|
||||
Self {
|
||||
status,
|
||||
dirty_usage_clear,
|
||||
failed_dirty_usage: false,
|
||||
pending_maintenance_work: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_failed_dirty_usage(mut self, failed_dirty_usage: bool) -> Self {
|
||||
self.failed_dirty_usage = failed_dirty_usage;
|
||||
self
|
||||
}
|
||||
|
||||
fn with_pending_maintenance_work(mut self, pending_maintenance_work: bool) -> Self {
|
||||
self.pending_maintenance_work = pending_maintenance_work;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn acknowledge_durable_usage(self) {
|
||||
if let Some(snapshot) = self.dirty_usage_clear {
|
||||
clear_dirty_usage_buckets(&snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_dirty_usage_to_acknowledge(&self) -> bool {
|
||||
self.dirty_usage_clear.as_ref().is_some_and(|snapshot| !snapshot.is_empty())
|
||||
}
|
||||
|
||||
pub(crate) fn has_failed_dirty_usage(&self) -> bool {
|
||||
self.failed_dirty_usage
|
||||
}
|
||||
|
||||
pub(crate) fn has_pending_maintenance_work(&self) -> bool {
|
||||
self.pending_maintenance_work
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ScannerIO for ECStore {
|
||||
#[tracing::instrument(skip(self, budget, updates))]
|
||||
async fn nsscanner(
|
||||
&self,
|
||||
ctx: CancellationToken,
|
||||
@@ -700,13 +850,40 @@ impl ScannerIO for ECStore {
|
||||
want_cycle: u64,
|
||||
scan_mode: HealScanMode,
|
||||
) -> Result<()> {
|
||||
// Preserve the public API's pre-existing completion semantics for
|
||||
// embedders. The main scanner uses nsscanner_with_status so it can
|
||||
// delay this acknowledgement until usage persistence succeeds.
|
||||
ScannerIOCycle::nsscanner_with_status(self, ctx, budget, updates, want_cycle, scan_mode)
|
||||
.await?
|
||||
.acknowledge_durable_usage();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ScannerIOCycle for ECStore {
|
||||
#[tracing::instrument(skip(self, budget, updates))]
|
||||
async fn nsscanner_with_status(
|
||||
&self,
|
||||
ctx: CancellationToken,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
updates: mpsc::Sender<DataUsageInfo>,
|
||||
want_cycle: u64,
|
||||
scan_mode: HealScanMode,
|
||||
) -> Result<ScannerCycleResult> {
|
||||
let child_token = ctx.child_token();
|
||||
|
||||
let dirty_generation_before_bucket_list = dirty_usage_generation();
|
||||
let all_buckets = self.list_bucket(&BucketOptions::default()).await?;
|
||||
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
|
||||
|
||||
if all_buckets.is_empty() {
|
||||
reset_set_scan_gauges();
|
||||
if let Err(e) = updates.send(DataUsageInfo::default()).await {
|
||||
let empty_usage = DataUsageInfo {
|
||||
last_update: Some(SystemTime::now()),
|
||||
..Default::default()
|
||||
};
|
||||
if let Err(e) = updates.send(empty_usage).await {
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_SET_STATE,
|
||||
@@ -717,7 +894,14 @@ impl ScannerIO for ECStore {
|
||||
"Scanner set state update failed"
|
||||
);
|
||||
}
|
||||
return Ok(());
|
||||
let status = if dirty_usage_snapshot_covers_current(&dirty_usage_snapshot) {
|
||||
ScannerCycleStatus::Complete
|
||||
} else {
|
||||
ScannerCycleStatus::Incomplete
|
||||
};
|
||||
let dirty_usage_clear =
|
||||
(status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone());
|
||||
return Ok(ScannerCycleResult::new(status, dirty_usage_clear));
|
||||
}
|
||||
|
||||
let mut total_results = 0;
|
||||
@@ -735,12 +919,12 @@ impl ScannerIO for ECStore {
|
||||
"Scanner set state update detected missing disk sets"
|
||||
);
|
||||
reset_set_scan_gauges();
|
||||
return Ok(());
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
|
||||
}
|
||||
|
||||
let set_scan_limit = scanner_max_concurrent_set_scans(total_results);
|
||||
let dirty_usage_buckets = Arc::new(snapshot_dirty_usage_buckets(&all_buckets));
|
||||
let failed_dirty_buckets = Arc::new(Mutex::new(HashSet::<String>::new()));
|
||||
let pending_maintenance_work = Arc::new(AtomicBool::new(false));
|
||||
record_set_scan_concurrency_limit(set_scan_limit);
|
||||
debug!(
|
||||
target: "rustfs::scanner::io",
|
||||
@@ -794,8 +978,12 @@ impl ScannerIO for ECStore {
|
||||
});
|
||||
wait_futs.push(receiver_fut);
|
||||
|
||||
let scan_plan =
|
||||
ScannerBucketScanPlan::new(all_buckets.clone(), dirty_usage_buckets.clone(), failed_dirty_buckets.clone());
|
||||
let scan_plan = ScannerBucketScanPlan::new(
|
||||
all_buckets.clone(),
|
||||
dirty_usage_snapshot.buckets.clone(),
|
||||
failed_dirty_buckets.clone(),
|
||||
pending_maintenance_work.clone(),
|
||||
);
|
||||
// Spawn task to run the scanner
|
||||
let scanner_fut = tokio::spawn(async move {
|
||||
let permit_wait = child_token_clone.clone();
|
||||
@@ -870,7 +1058,7 @@ impl ScannerIO for ECStore {
|
||||
let results_mutex_for_updates = results_mutex.clone();
|
||||
let budget_for_updates = budget.clone();
|
||||
let child_token_for_updates = child_token.clone();
|
||||
let dirty_usage_buckets_for_updates = dirty_usage_buckets.clone();
|
||||
let dirty_usage_snapshot_for_updates = dirty_usage_snapshot.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut last_update = SystemTime::UNIX_EPOCH;
|
||||
let mut has_sent_once = false;
|
||||
@@ -893,7 +1081,7 @@ impl ScannerIO for ECStore {
|
||||
&all_buckets_clone,
|
||||
budget_for_updates.budget_elapsed(),
|
||||
child_token_for_updates.is_cancelled(),
|
||||
dirty_usage_snapshot_covers_current(dirty_usage_buckets_for_updates.as_ref()),
|
||||
dirty_usage_snapshot_covers_current(dirty_usage_snapshot_for_updates.as_ref()),
|
||||
)
|
||||
};
|
||||
|
||||
@@ -912,7 +1100,7 @@ impl ScannerIO for ECStore {
|
||||
&all_buckets_clone,
|
||||
budget_for_updates.budget_elapsed(),
|
||||
child_token_for_updates.is_cancelled(),
|
||||
dirty_usage_snapshot_covers_current(dirty_usage_buckets_for_updates.as_ref()),
|
||||
dirty_usage_snapshot_covers_current(dirty_usage_snapshot_for_updates.as_ref()),
|
||||
)
|
||||
};
|
||||
|
||||
@@ -954,16 +1142,27 @@ impl ScannerIO for ECStore {
|
||||
let completed_all_sets = results.iter().all(|result| result.info.last_update.is_some());
|
||||
let result = finalize_nsscanner_result(&results, first_err);
|
||||
let failed_buckets = failed_dirty_buckets.lock().await.clone();
|
||||
if let Some(clear_snapshot) = should_clear_dirty_usage_snapshot(
|
||||
let pending_maintenance_work = pending_maintenance_work_for_cycle(&pending_maintenance_work, &results);
|
||||
let budget_elapsed = budget.budget_elapsed();
|
||||
let dirty_usage_current = dirty_usage_snapshot_covers_current(&dirty_usage_snapshot);
|
||||
let cycle_status = classify_nsscanner_cycle(
|
||||
completed_all_sets,
|
||||
budget_elapsed,
|
||||
ctx.is_cancelled(),
|
||||
!failed_buckets.is_empty(),
|
||||
dirty_usage_current,
|
||||
);
|
||||
let dirty_usage_clear = should_clear_dirty_usage_snapshot(
|
||||
result.is_ok(),
|
||||
completed_all_sets,
|
||||
budget.budget_elapsed(),
|
||||
&dirty_usage_buckets,
|
||||
budget_elapsed,
|
||||
&dirty_usage_snapshot.buckets,
|
||||
&failed_buckets,
|
||||
) {
|
||||
clear_dirty_usage_buckets(&clear_snapshot);
|
||||
}
|
||||
result
|
||||
);
|
||||
result?;
|
||||
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
|
||||
.with_failed_dirty_usage(!failed_buckets.is_empty())
|
||||
.with_pending_maintenance_work(pending_maintenance_work))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -983,6 +1182,7 @@ impl ScannerIOCache for SetDisks {
|
||||
buckets,
|
||||
dirty_usage_buckets,
|
||||
failed_dirty_buckets,
|
||||
pending_maintenance_work,
|
||||
} = scan_plan;
|
||||
let pool_label = self.pool_index.to_string();
|
||||
let set_label = self.set_index.to_string();
|
||||
@@ -1127,6 +1327,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let pool_label_clone = pool_label.clone();
|
||||
let set_label_clone = set_label.clone();
|
||||
let failed_dirty_buckets_clone = failed_dirty_buckets.clone();
|
||||
let pending_maintenance_work_clone = pending_maintenance_work.clone();
|
||||
futs.push(tokio::spawn(async move {
|
||||
loop {
|
||||
let Some(bucket) = bucket_rx_mutex_clone.lock().await.recv().await else {
|
||||
@@ -1322,7 +1523,6 @@ impl ScannerIOCache for SetDisks {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
debug!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_DISK_BUCKET_STATE,
|
||||
@@ -1349,7 +1549,9 @@ impl ScannerIOCache for SetDisks {
|
||||
"Scanner root entry publish started"
|
||||
);
|
||||
|
||||
if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await {
|
||||
if let Err(e) =
|
||||
send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache, &pending_maintenance_work_clone).await
|
||||
{
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
@@ -1761,10 +1963,10 @@ mod tests {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
let buckets = vec![bucket_info("photos")];
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets);
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation());
|
||||
|
||||
record_dirty_usage_bucket("photos");
|
||||
clear_dirty_usage_buckets(&snapshot);
|
||||
clear_dirty_usage_buckets(&snapshot.buckets);
|
||||
|
||||
assert_eq!(dirty_usage_bucket_count(), 1);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
@@ -1776,7 +1978,7 @@ mod tests {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
let buckets = vec![bucket_info("photos")];
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets);
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation());
|
||||
|
||||
assert!(dirty_usage_snapshot_covers_current(&snapshot));
|
||||
|
||||
@@ -1786,6 +1988,83 @@ mod tests {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
record_dirty_usage_bucket("temporarily-omitted");
|
||||
let generation_before_bucket_list = dirty_usage_generation();
|
||||
|
||||
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], generation_before_bucket_list);
|
||||
|
||||
assert!(snapshot.buckets.contains_key("photos"));
|
||||
assert!(snapshot.buckets.contains_key("temporarily-omitted"));
|
||||
assert!(dirty_usage_buckets().contains_key("temporarily-omitted"));
|
||||
assert!(dirty_usage_snapshot_covers_current(&snapshot));
|
||||
|
||||
ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone()))
|
||||
.acknowledge_durable_usage();
|
||||
assert!(!dirty_usage_buckets().contains_key("temporarily-omitted"));
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_snapshot_preserves_an_absent_bucket_recorded_after_listing_started() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let generation_before_bucket_list = dirty_usage_generation();
|
||||
record_dirty_usage_bucket("new-or-racing-bucket");
|
||||
|
||||
let snapshot = snapshot_dirty_usage_buckets(&[], generation_before_bucket_list);
|
||||
|
||||
assert!(!snapshot.buckets.contains_key("new-or-racing-bucket"));
|
||||
assert!(!dirty_usage_snapshot_covers_current(&snapshot));
|
||||
assert!(dirty_usage_buckets().contains_key("new-or-racing-bucket"));
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation());
|
||||
assert!(dirty_usage_snapshot_covers_current(&snapshot));
|
||||
|
||||
record_dirty_usage_bucket("photos");
|
||||
|
||||
assert!(!dirty_usage_snapshot_covers_current(&snapshot));
|
||||
assert!(dirty_usage_buckets().contains_key("photos"));
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn deleting_a_bucket_during_listing_invalidates_the_resulting_usage_snapshot() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let generation_before_bucket_list = dirty_usage_generation();
|
||||
|
||||
record_dirty_usage_bucket("photos");
|
||||
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], generation_before_bucket_list);
|
||||
|
||||
assert!(!dirty_usage_snapshot_covers_current(&snapshot));
|
||||
assert!(dirty_usage_buckets().contains_key("photos"));
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_maintenance_change_advances_generation_and_marks_usage_dirty() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let generation = scanner_maintenance_generation();
|
||||
|
||||
record_scanner_maintenance_change("photos");
|
||||
|
||||
assert!(scanner_maintenance_generation() > generation);
|
||||
assert!(dirty_usage_buckets().contains_key("photos"));
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_clear_excludes_failed_buckets() {
|
||||
@@ -1793,9 +2072,9 @@ mod tests {
|
||||
record_dirty_usage_bucket("photos");
|
||||
record_dirty_usage_bucket("videos");
|
||||
let buckets = vec![bucket_info("photos"), bucket_info("videos")];
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets);
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation());
|
||||
let failed_buckets = HashSet::from(["videos".to_string()]);
|
||||
let clear_snapshot = dirty_usage_buckets_excluding_failed(&snapshot, &failed_buckets);
|
||||
let clear_snapshot = dirty_usage_buckets_excluding_failed(&snapshot.buckets, &failed_buckets);
|
||||
|
||||
clear_dirty_usage_buckets(&clear_snapshot);
|
||||
|
||||
@@ -1818,6 +2097,23 @@ mod tests {
|
||||
assert!(!clear_snapshot.contains_key("videos"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_is_acknowledged_only_after_durable_usage_confirmation() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation());
|
||||
|
||||
let unconfirmed = ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone()));
|
||||
drop(unconfirmed);
|
||||
assert!(dirty_usage_buckets().contains_key("photos"));
|
||||
|
||||
let confirmed = ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone()));
|
||||
confirmed.acknowledge_durable_usage();
|
||||
assert!(!dirty_usage_buckets().contains_key("photos"));
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn clear_dirty_usage_bucket_removes_deleted_bucket_marker() {
|
||||
@@ -1828,9 +2124,9 @@ mod tests {
|
||||
clear_dirty_usage_bucket("photos");
|
||||
|
||||
let buckets = vec![bucket_info("photos"), bucket_info("videos")];
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets);
|
||||
assert!(!snapshot.contains_key("photos"));
|
||||
assert!(snapshot.contains_key("videos"));
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation());
|
||||
assert!(!snapshot.buckets.contains_key("photos"));
|
||||
assert!(snapshot.buckets.contains_key("videos"));
|
||||
assert_eq!(dirty_usage_bucket_count(), 1);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
@@ -1896,6 +2192,78 @@ mod tests {
|
||||
assert!(err.to_string().contains("set failed"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
|
||||
assert_eq!(classify_nsscanner_cycle(true, false, false, false, true), ScannerCycleStatus::Complete);
|
||||
|
||||
for status in [
|
||||
classify_nsscanner_cycle(false, false, false, false, true),
|
||||
classify_nsscanner_cycle(true, true, false, false, true),
|
||||
classify_nsscanner_cycle(true, false, true, false, true),
|
||||
classify_nsscanner_cycle(true, false, false, true, true),
|
||||
classify_nsscanner_cycle(true, false, false, false, false),
|
||||
] {
|
||||
assert_eq!(status, ScannerCycleStatus::Incomplete);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_cycle_surfaces_persisted_pending_heal_work() {
|
||||
let clean = DataUsageCache::default();
|
||||
assert!(!scanner_results_have_pending_maintenance_work(std::slice::from_ref(&clean)));
|
||||
|
||||
let mut pending = clean;
|
||||
pending.info.pending_heals.push(crate::PendingScannerHeal {
|
||||
kind: crate::PendingScannerHealKind::Object,
|
||||
bucket: "photos".to_string(),
|
||||
object: Some("image.jpg".to_string()),
|
||||
version_id: None,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
first_seen: 1,
|
||||
last_attempt: 1,
|
||||
attempts: 1,
|
||||
last_admission_result: "queue_full".to_string(),
|
||||
last_admission_reason: "capacity".to_string(),
|
||||
});
|
||||
|
||||
assert!(scanner_results_have_pending_maintenance_work(&[pending]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_cache_pending_heal_reaches_cycle_maintenance_state() {
|
||||
let pending_maintenance_work = Arc::new(AtomicBool::new(false));
|
||||
let mut bucket_cache = DataUsageCache::default();
|
||||
bucket_cache.info.pending_heals.push(crate::PendingScannerHeal {
|
||||
kind: crate::PendingScannerHealKind::Object,
|
||||
bucket: "photos".to_string(),
|
||||
object: Some("image.jpg".to_string()),
|
||||
version_id: None,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
first_seen: 1,
|
||||
last_attempt: 1,
|
||||
attempts: 1,
|
||||
last_admission_result: "queue_full".to_string(),
|
||||
last_admission_reason: "capacity".to_string(),
|
||||
});
|
||||
let (sender, mut receiver) = mpsc::channel(1);
|
||||
let sender = Arc::new(Mutex::new(sender));
|
||||
|
||||
send_cache_root_entry_info(&sender, &bucket_cache, &pending_maintenance_work)
|
||||
.await
|
||||
.expect("bucket result should send");
|
||||
|
||||
let cycle_pending = pending_maintenance_work_for_cycle(&pending_maintenance_work, &[]);
|
||||
assert!(cycle_pending);
|
||||
assert_eq!(
|
||||
crate::scanner::scanner_cycle_outcome_with_pending_maintenance(
|
||||
crate::scanner::ScannerCycleOutcome::Completed,
|
||||
cycle_pending,
|
||||
),
|
||||
crate::scanner::ScannerCycleOutcome::CompletedWithPendingMaintenance
|
||||
);
|
||||
assert!(receiver.recv().await.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
|
||||
|
||||
Reference in New Issue
Block a user