mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
fix(scanner): back off clean idle scans across erasure clusters (#4984)
* fix(scanner): back off clean single-disk cycles * fix(scanner): extend idle backoff across erasure clusters --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
@@ -2434,6 +2434,18 @@ impl Metrics {
|
||||
self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn current_scan_cycle_has_unresolved_heal_work(&self) -> bool {
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
|
||||
[ScannerWorkSource::Heal, ScannerWorkSource::Bitrot]
|
||||
.into_iter()
|
||||
.filter_map(|source| source_work.get(source.index()))
|
||||
.any(|work| work.queued > 0 || work.skipped > 0 || work.failed > 0 || work.missed > 0)
|
||||
}
|
||||
|
||||
fn scan_cycle_work_snapshot(&self) -> ScanCycleWorkSnapshot {
|
||||
ScanCycleWorkSnapshot {
|
||||
objects_scanned: self.lifetime(Metric::ScanObject),
|
||||
@@ -3361,6 +3373,40 @@ mod tests {
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unresolved_heal_work_only_reflects_the_active_cycle() {
|
||||
let metrics = Metrics::new();
|
||||
assert!(!metrics.current_scan_cycle_has_unresolved_heal_work());
|
||||
|
||||
let start = metrics.start_scan_cycle_work();
|
||||
metrics.record_scanner_source_missed(ScannerWorkSource::Heal, 1);
|
||||
assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
|
||||
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
assert!(!metrics.current_scan_cycle_has_unresolved_heal_work());
|
||||
|
||||
let start = metrics.start_scan_cycle_work();
|
||||
metrics.record_scanner_source_queued(ScannerWorkSource::Heal, 1);
|
||||
assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
|
||||
let start = metrics.start_scan_cycle_work();
|
||||
metrics.record_scanner_source_work(
|
||||
ScannerWorkSource::Bitrot,
|
||||
ScannerSourceWorkUpdate {
|
||||
skipped: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
|
||||
let start = metrics.start_scan_cycle_work();
|
||||
metrics.record_scanner_source_failed(ScannerWorkSource::Bitrot, 1);
|
||||
assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_marks_transition_failures_as_blocked_lifecycle_control() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -798,6 +798,13 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn scanner_activity(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ScannerActivityRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::ScannerActivityResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn background_heal_status(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::BackgroundHealStatusRequest>,
|
||||
|
||||
@@ -376,7 +376,7 @@ pub mod rio {
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, TonicInterceptor, gen_tonic_signature_interceptor,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, verify_rpc_signature,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
pub use internode_data_transport::build_internode_data_transport_from_env;
|
||||
pub use peer_rest_client::{
|
||||
PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
ScannerPeerActivity,
|
||||
};
|
||||
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
|
||||
pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys};
|
||||
|
||||
@@ -34,9 +34,9 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest, GetSysConfigRequest,
|
||||
GetSysErrorsRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
|
||||
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ServerInfoRequest,
|
||||
SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ScannerActivityRequest,
|
||||
ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest,
|
||||
StopRebalanceRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use rustfs_utils::XHost;
|
||||
use serde::{Deserialize, Serialize as _};
|
||||
@@ -62,6 +62,31 @@ pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
|
||||
pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
|
||||
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
|
||||
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerPeerActivity {
|
||||
pub instance_id: String,
|
||||
pub namespace_generation: u64,
|
||||
pub maintenance_generation: u64,
|
||||
}
|
||||
|
||||
fn decode_scanner_activity(response: ScannerActivityResponse) -> Result<ScannerPeerActivity> {
|
||||
let instance_id = response.instance_id;
|
||||
if instance_id.len() != 32
|
||||
|| !instance_id
|
||||
.as_bytes()
|
||||
.iter()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte))
|
||||
{
|
||||
return Err(Error::other("peer returned an invalid scanner activity instance ID"));
|
||||
}
|
||||
Ok(ScannerPeerActivity {
|
||||
instance_id,
|
||||
namespace_generation: response.namespace_generation,
|
||||
maintenance_generation: response.maintenance_generation,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerLiveEventsBatch {
|
||||
@@ -639,12 +664,13 @@ impl PeerRestClient {
|
||||
Err(Error::NotImplemented)
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str) -> Result<()> {
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(LoadBucketMetadataRequest {
|
||||
bucket: bucket.to_string(),
|
||||
scanner_maintenance_change,
|
||||
});
|
||||
|
||||
let response = client.load_bucket_metadata(request).await?.into_inner();
|
||||
@@ -908,6 +934,25 @@ impl PeerRestClient {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn scanner_activity(&self) -> Result<ScannerPeerActivity> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await?
|
||||
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
|
||||
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
|
||||
let response = client
|
||||
.scanner_activity(Request::new(ScannerActivityRequest {}))
|
||||
.await?
|
||||
.into_inner();
|
||||
decode_scanner_activity(response)
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn get_metacache_listing(&self) -> Result<()> {
|
||||
warn!("get_metacache_listing is not implemented in PeerRestClient");
|
||||
Err(Error::NotImplemented)
|
||||
@@ -1157,6 +1202,48 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_requires_restart_safe_peer_identity() {
|
||||
let missing_instance = ScannerActivityResponse {
|
||||
instance_id: String::new(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
};
|
||||
assert!(
|
||||
decode_scanner_activity(missing_instance)
|
||||
.expect_err("an empty instance ID is not restart safe")
|
||||
.to_string()
|
||||
.contains("instance ID")
|
||||
);
|
||||
|
||||
let malformed_instance = ScannerActivityResponse {
|
||||
instance_id: "ABCDEF0123456789ABCDEF0123456789".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
};
|
||||
assert!(
|
||||
decode_scanner_activity(malformed_instance)
|
||||
.expect_err("activity instance IDs must use the canonical lowercase hex form")
|
||||
.to_string()
|
||||
.contains("instance ID")
|
||||
);
|
||||
|
||||
let activity = decode_scanner_activity(ScannerActivityResponse {
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
})
|
||||
.expect("complete activity responses should be accepted");
|
||||
assert_eq!(
|
||||
activity,
|
||||
ScannerPeerActivity {
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_marks_network_like_errors() {
|
||||
assert!(PeerRestClient::is_network_like_error(&Error::other("transport error")));
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::PeerRestClient;
|
||||
use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity};
|
||||
use crate::diagnostics::admin_server_info::get_commit_id;
|
||||
use crate::disk::DiskAPI;
|
||||
use crate::error::{Error, Result};
|
||||
@@ -41,6 +41,7 @@ const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3;
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification";
|
||||
const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation";
|
||||
const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Cached result from the last successful admin call to a peer.
|
||||
struct PeerAdminCache {
|
||||
@@ -91,7 +92,6 @@ pub fn get_global_notification_sys() -> Option<Arc<NotificationSys>> {
|
||||
|
||||
pub struct NotificationSys {
|
||||
pub peer_clients: Vec<Option<PeerRestClient>>,
|
||||
#[allow(dead_code)]
|
||||
pub all_peer_clients: Vec<Option<PeerRestClient>>,
|
||||
peer_admin_caches: Vec<Mutex<PeerAdminCache>>,
|
||||
}
|
||||
@@ -756,6 +756,14 @@ impl NotificationSys {
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str) -> Result<()> {
|
||||
self.load_bucket_metadata_with_scanner_maintenance(bucket, false).await
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata_for_scanner_maintenance(&self, bucket: &str) -> Result<()> {
|
||||
self.load_bucket_metadata_with_scanner_maintenance(bucket, true).await
|
||||
}
|
||||
|
||||
async fn load_bucket_metadata_with_scanner_maintenance(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
let operation = format!("load_bucket_metadata({bucket})");
|
||||
let mut failures = Vec::new();
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
@@ -763,7 +771,12 @@ impl NotificationSys {
|
||||
if let Some(client) = client {
|
||||
let host = client.host.to_string();
|
||||
let b = bucket.to_string();
|
||||
futures.push(async move { client.load_bucket_metadata(&b).await.map_err(|err| (host, err)) });
|
||||
futures.push(async move {
|
||||
client
|
||||
.load_bucket_metadata(&b, scanner_maintenance_change)
|
||||
.await
|
||||
.map_err(|err| (host, err))
|
||||
});
|
||||
} else {
|
||||
failures.push(format!("peer[{idx}] {operation} failed: peer is not reachable"));
|
||||
}
|
||||
@@ -976,6 +989,36 @@ impl NotificationSys {
|
||||
join_all(futures).await
|
||||
}
|
||||
|
||||
pub async fn scanner_activity_snapshots(&self) -> Result<Vec<(String, ScannerPeerActivity)>> {
|
||||
if self.peer_clients.is_empty() {
|
||||
return Err(Error::other("scanner activity probe has no remote peers"));
|
||||
}
|
||||
if self.all_peer_clients.len() != self.peer_clients.len() + 1 {
|
||||
return Err(Error::other(format!(
|
||||
"scanner activity peer topology is incomplete: {} remote peers for {} cluster members",
|
||||
self.peer_clients.len(),
|
||||
self.all_peer_clients.len()
|
||||
)));
|
||||
}
|
||||
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for (idx, client) in self.peer_clients.iter().cloned().enumerate() {
|
||||
futures.push(async move {
|
||||
let client = client.ok_or_else(|| Error::other(format!("scanner activity peer[{idx}] is unreachable")))?;
|
||||
let host = client.grid_host.clone();
|
||||
scanner_activity_with_timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, &host, client.scanner_activity())
|
||||
.await
|
||||
.map(|activity| (host, activity))
|
||||
});
|
||||
}
|
||||
|
||||
let mut generations = Vec::with_capacity(futures.len());
|
||||
for result in join_all(futures).await {
|
||||
generations.push(result?);
|
||||
}
|
||||
Ok(generations)
|
||||
}
|
||||
|
||||
pub async fn reload_site_replication_config(&self) -> Vec<NotificationPeerErr> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for client in self.peer_clients.iter() {
|
||||
@@ -1029,6 +1072,15 @@ impl NotificationSys {
|
||||
}
|
||||
}
|
||||
|
||||
async fn scanner_activity_with_timeout<F>(timeout_duration: Duration, host: &str, activity: F) -> Result<ScannerPeerActivity>
|
||||
where
|
||||
F: Future<Output = Result<ScannerPeerActivity>>,
|
||||
{
|
||||
timeout(timeout_duration, activity)
|
||||
.await
|
||||
.map_err(|_| Error::other(format!("scanner activity peer {host} timed out after {timeout_duration:?}")))?
|
||||
}
|
||||
|
||||
async fn call_peer_with_timeout<F, Fut>(
|
||||
timeout_dur: Duration,
|
||||
host_label: &str,
|
||||
@@ -1568,6 +1620,72 @@ mod tests {
|
||||
assert!(msg.contains("peer[0]"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_activity_probe_reports_unreachable_peers() {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: vec![None, None],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
};
|
||||
|
||||
let err = sys
|
||||
.scanner_activity_snapshots()
|
||||
.await
|
||||
.expect_err("unreachable peers must disable scanner idle backoff");
|
||||
|
||||
assert!(err.to_string().contains("scanner activity peer[0] is unreachable"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_activity_probe_rejects_an_empty_peer_set() {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: Vec::new(),
|
||||
peer_admin_caches: Vec::new(),
|
||||
};
|
||||
|
||||
let err = sys
|
||||
.scanner_activity_snapshots()
|
||||
.await
|
||||
.expect_err("a missing peer set must disable scanner idle backoff");
|
||||
|
||||
assert!(err.to_string().contains("no remote peers"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_activity_probe_rejects_an_incomplete_peer_topology() {
|
||||
let client = PeerRestClient::new(
|
||||
"127.0.0.1:9000".to_string().try_into().expect("peer host should parse"),
|
||||
"http://127.0.0.1:9000".to_string(),
|
||||
);
|
||||
let sys = NotificationSys {
|
||||
peer_clients: vec![Some(client)],
|
||||
all_peer_clients: vec![None],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
};
|
||||
|
||||
let err = sys
|
||||
.scanner_activity_snapshots()
|
||||
.await
|
||||
.expect_err("an incomplete peer topology must disable scanner idle backoff");
|
||||
|
||||
assert!(err.to_string().contains("peer topology is incomplete"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_activity_probe_times_out() {
|
||||
let err = scanner_activity_with_timeout(
|
||||
Duration::from_millis(5),
|
||||
"peer-1",
|
||||
std::future::pending::<Result<ScannerPeerActivity>>(),
|
||||
)
|
||||
.await
|
||||
.expect_err("a stalled peer must not block scanner scheduling");
|
||||
|
||||
assert!(err.to_string().contains("timed out"));
|
||||
assert!(err.to_string().contains("peer-1"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_bucket_metadata_reports_unreachable_peers() {
|
||||
let sys = NotificationSys {
|
||||
|
||||
@@ -392,16 +392,28 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn create_bucket_with_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
|
||||
let generation_before_make = ecstore.scanner_namespace_mutation_generation();
|
||||
ecstore
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_make.saturating_add(1),
|
||||
"successful bucket creation should advance scanner namespace activity"
|
||||
);
|
||||
|
||||
let generation_before_put = ecstore.scanner_namespace_mutation_generation();
|
||||
let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
|
||||
ecstore
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("object should be written");
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_put.saturating_add(1),
|
||||
"successful object creation should advance scanner namespace activity"
|
||||
);
|
||||
ecstore
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
@@ -499,6 +511,7 @@ mod tests {
|
||||
create_bucket_with_object(&ecstore, &bucket, object).await;
|
||||
assert!(metadata_sys::get(&bucket).await.is_ok());
|
||||
|
||||
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
|
||||
ecstore
|
||||
.delete_bucket(
|
||||
&bucket,
|
||||
@@ -509,6 +522,11 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("MarkDelete should not reject non-empty bucket data");
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_delete.saturating_add(1),
|
||||
"successful bucket deletion should advance scanner namespace activity"
|
||||
);
|
||||
|
||||
assert!(
|
||||
any_disk_has_object_metadata(&disk_paths, &bucket).await,
|
||||
@@ -536,6 +554,7 @@ mod tests {
|
||||
write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
|
||||
assert!(any_disk_path_exists(&disk_paths, &metadata_prefix).await);
|
||||
|
||||
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
|
||||
ecstore
|
||||
.delete_bucket(
|
||||
&bucket,
|
||||
@@ -547,6 +566,11 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("Purge should force-delete bucket data");
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_delete.saturating_add(1),
|
||||
"successful bucket purge should advance scanner namespace activity"
|
||||
);
|
||||
|
||||
assert!(!any_disk_path_exists(&disk_paths, &bucket).await, "Purge should remove the bucket volume");
|
||||
assert!(
|
||||
@@ -568,12 +592,18 @@ mod tests {
|
||||
|
||||
create_bucket_with_object(&ecstore, &bucket, object).await;
|
||||
|
||||
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
|
||||
let err = ecstore
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect_err("default S3 DeleteBucket should reject non-empty buckets");
|
||||
|
||||
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_delete,
|
||||
"failed bucket deletion must not advance scanner namespace activity"
|
||||
);
|
||||
assert!(
|
||||
any_disk_has_object_metadata(&disk_paths, &bucket).await,
|
||||
"failed default S3 DeleteBucket must keep object data"
|
||||
|
||||
@@ -541,6 +541,7 @@ impl PersistentListMetadataObject {
|
||||
static PERSISTENT_KEY_ONLY_INDEX_CACHE: OnceCell<RwLock<Option<PersistentKeyOnlyIndexCache>>> = OnceCell::const_new();
|
||||
static LIST_OBJECTS_NAMESPACE_JOURNAL_LOCK: OnceCell<RwLock<()>> = OnceCell::const_new();
|
||||
static LIST_OBJECTS_MUTATION_SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
||||
static SCANNER_NAMESPACE_MUTATION_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
static LIST_OBJECTS_BUCKET_MUTATION_SEQUENCE: OnceCell<RwLock<HashMap<String, u64>>> = OnceCell::const_new();
|
||||
static LIST_OBJECTS_NAMESPACE_JOURNAL_DEGRADED_BUCKETS: OnceCell<RwLock<HashSet<String>>> = OnceCell::const_new();
|
||||
static LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_CONFIG: OnceCell<Option<NamespaceMutationJournalChaosConfig>> = OnceCell::const_new();
|
||||
@@ -607,6 +608,19 @@ async fn advance_list_objects_mutation_sequence(bucket: &str, sequence: u64) ->
|
||||
sequence
|
||||
}
|
||||
|
||||
pub(super) fn scanner_namespace_mutation_generation() -> u64 {
|
||||
SCANNER_NAMESPACE_MUTATION_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(super) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) {
|
||||
if bucket == RUSTFS_META_BUCKET {
|
||||
return;
|
||||
}
|
||||
|
||||
let _ = SCANNER_NAMESPACE_MUTATION_GENERATION
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(delta)));
|
||||
}
|
||||
|
||||
pub(super) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 {
|
||||
observe_list_objects_mutations(store, bucket, 1).await.unwrap_or_default()
|
||||
}
|
||||
@@ -621,6 +635,7 @@ async fn observe_list_objects_mutations_with_store(store: Option<&ECStore>, buck
|
||||
}
|
||||
|
||||
let delta = u64::try_from(count).unwrap_or(u64::MAX);
|
||||
observe_scanner_namespace_mutations(bucket, delta);
|
||||
let next = LIST_OBJECTS_MUTATION_SEQUENCE
|
||||
.fetch_add(delta, Ordering::AcqRel)
|
||||
.saturating_add(delta);
|
||||
@@ -641,6 +656,7 @@ async fn current_list_objects_mutation_sequence(bucket: &str) -> u64 {
|
||||
#[cfg(test)]
|
||||
async fn reset_list_objects_mutation_sequences_for_test() {
|
||||
LIST_OBJECTS_MUTATION_SEQUENCE.store(0, Ordering::Release);
|
||||
SCANNER_NAMESPACE_MUTATION_GENERATION.store(0, Ordering::Release);
|
||||
let sequences = list_objects_bucket_mutation_sequence().await;
|
||||
sequences.write().await.clear();
|
||||
let degraded = list_objects_namespace_journal_degraded_buckets().await;
|
||||
@@ -6646,11 +6662,11 @@ mod test {
|
||||
ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend,
|
||||
NamespaceMutationJournalSnapshot, NamespaceMutationJournalStatus, PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER,
|
||||
PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER,
|
||||
PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex, PersistentListMetadataObject, VerifiedIndexCandidateStats,
|
||||
VersionMarker, current_list_objects_mutation_sequence, encode_persistent_list_metadata_object,
|
||||
enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results,
|
||||
latest_listing_allow_agreed_objects, latest_listing_object_quorum, latest_listing_raw_min_disks,
|
||||
latest_listing_required_object_quorum, list_marker_key, list_metadata_resolution_params,
|
||||
PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex, PersistentListMetadataObject, RUSTFS_META_BUCKET,
|
||||
VerifiedIndexCandidateStats, VersionMarker, current_list_objects_mutation_sequence,
|
||||
encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum,
|
||||
fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects, latest_listing_object_quorum,
|
||||
latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key, list_metadata_resolution_params,
|
||||
list_objects_from_metadata_snapshot_candidates, list_objects_from_verified_index_candidates,
|
||||
list_objects_from_verified_index_candidates_with_optional_stats, list_objects_from_verified_index_candidates_with_stats,
|
||||
list_objects_index_mode_from_env, list_objects_index_provider_from_env, list_objects_index_provider_state_from_env,
|
||||
@@ -6664,8 +6680,9 @@ mod test {
|
||||
parse_version_marker, persist_observed_list_objects_mutation, persistent_key_only_index_has_complete_metadata_snapshot,
|
||||
persistent_key_only_index_health, persistent_key_only_index_matches_provider, record_list_objects_index_opt_in_fallback,
|
||||
reset_list_objects_mutation_sequences_for_test, resolve_agreed_listing_entry, resolve_listing_entries,
|
||||
select_list_index_provider_source_mode, select_list_index_source_mode, send_or_cancel, version_marker_for_entries,
|
||||
walk_result_from_set_errors, write_namespace_mutation_journal_state, write_persistent_key_only_index,
|
||||
scanner_namespace_mutation_generation, select_list_index_provider_source_mode, select_list_index_source_mode,
|
||||
send_or_cancel, version_marker_for_entries, walk_result_from_set_errors, write_namespace_mutation_journal_state,
|
||||
write_persistent_key_only_index,
|
||||
};
|
||||
use crate::cache_value::metacache_set::{FallbackClaimTracker, TestReaderBehavior, list_path_raw};
|
||||
use crate::disk::{DiskAPI, DiskOption, endpoint::Endpoint, error::DiskError, new_disk};
|
||||
@@ -8362,6 +8379,19 @@ mod test {
|
||||
assert_eq!(current_list_objects_mutation_sequence("bucket-b").await, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn scanner_namespace_generation_tracks_only_user_namespace_mutations() {
|
||||
reset_list_objects_mutation_sequences_for_test().await;
|
||||
|
||||
assert_eq!(scanner_namespace_mutation_generation(), 0);
|
||||
assert_eq!(observe_list_objects_mutations_with_store(None, RUSTFS_META_BUCKET, 3).await, Some(3));
|
||||
assert_eq!(scanner_namespace_mutation_generation(), 0);
|
||||
|
||||
assert_eq!(observe_list_objects_mutations_with_store(None, "photos", 2).await, Some(5));
|
||||
assert_eq!(scanner_namespace_mutation_generation(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_objects_index_provider_state_uses_lifecycle_active_generation() {
|
||||
let provider = ListObjectsIndexProviderState::from_kind(ListObjectsIndexProviderKind::WalkerKeyOnly);
|
||||
|
||||
@@ -318,6 +318,10 @@ impl ECStore {
|
||||
pub async fn setup_is_erasure_sd(&self) -> bool {
|
||||
self.ctx.is_erasure_sd().await
|
||||
}
|
||||
|
||||
pub fn scanner_namespace_mutation_generation(&self) -> u64 {
|
||||
list_objects::scanner_namespace_mutation_generation()
|
||||
}
|
||||
}
|
||||
|
||||
// impl Clone for ECStore {
|
||||
@@ -436,7 +440,11 @@ impl BucketOperations for ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
|
||||
self.handle_make_bucket(bucket, opts).await
|
||||
let result = self.handle_make_bucket(bucket, opts).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
@@ -449,7 +457,11 @@ impl BucketOperations for ECStore {
|
||||
}
|
||||
#[instrument(skip(self))]
|
||||
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
|
||||
self.handle_delete_bucket(bucket, opts).await
|
||||
let result = self.handle_delete_bucket(bucket, opts).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -923,6 +923,8 @@ pub struct GetAllBucketStatsResponse {
|
||||
pub struct LoadBucketMetadataRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub bucket: ::prost::alloc::string::String,
|
||||
#[prost(bool, tag = "2")]
|
||||
pub scanner_maintenance_change: bool,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct LoadBucketMetadataResponse {
|
||||
@@ -1067,6 +1069,17 @@ pub struct SignalServiceResponse {
|
||||
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerActivityRequest {}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerActivityResponse {
|
||||
#[prost(string, tag = "1")]
|
||||
pub instance_id: ::prost::alloc::string::String,
|
||||
#[prost(uint64, tag = "2")]
|
||||
pub namespace_generation: u64,
|
||||
#[prost(uint64, tag = "3")]
|
||||
pub maintenance_generation: u64,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct BackgroundHealStatusRequest {}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct BackgroundHealStatusResponse {
|
||||
@@ -2351,6 +2364,21 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "SignalService"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn scanner_activity(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::ScannerActivityRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::ScannerActivityResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ScannerActivity");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "ScannerActivity"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn background_heal_status(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::BackgroundHealStatusRequest>,
|
||||
@@ -2825,6 +2853,10 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::SignalServiceRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SignalServiceResponse>, tonic::Status>;
|
||||
async fn scanner_activity(
|
||||
&self,
|
||||
request: tonic::Request<super::ScannerActivityRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::ScannerActivityResponse>, tonic::Status>;
|
||||
async fn background_heal_status(
|
||||
&self,
|
||||
request: tonic::Request<super::BackgroundHealStatusRequest>,
|
||||
@@ -4933,6 +4965,34 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/ScannerActivity" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct ScannerActivitySvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::ScannerActivityRequest> for ScannerActivitySvc<T> {
|
||||
type Response = super::ScannerActivityResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::ScannerActivityRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::scanner_activity(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = ScannerActivitySvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/BackgroundHealStatus" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct BackgroundHealStatusSvc<T: NodeService>(pub Arc<T>);
|
||||
|
||||
@@ -649,6 +649,7 @@ message GetAllBucketStatsResponse {
|
||||
|
||||
message LoadBucketMetadataRequest {
|
||||
string bucket = 1;
|
||||
bool scanner_maintenance_change = 2;
|
||||
}
|
||||
|
||||
message LoadBucketMetadataResponse {
|
||||
@@ -756,6 +757,14 @@ message SignalServiceResponse {
|
||||
optional string error_info = 2;
|
||||
}
|
||||
|
||||
message ScannerActivityRequest {}
|
||||
|
||||
message ScannerActivityResponse {
|
||||
string instance_id = 1;
|
||||
uint64 namespace_generation = 2;
|
||||
uint64 maintenance_generation = 3;
|
||||
}
|
||||
|
||||
message BackgroundHealStatusRequest {}
|
||||
|
||||
message BackgroundHealStatusResponse {
|
||||
@@ -944,6 +953,7 @@ service NodeService {
|
||||
// rpc VerifyBinary() returns () {};
|
||||
// rpc CommitBinary() returns () {};
|
||||
rpc SignalService(SignalServiceRequest) returns (SignalServiceResponse) {};
|
||||
rpc ScannerActivity(ScannerActivityRequest) returns (ScannerActivityResponse) {};
|
||||
rpc BackgroundHealStatus(BackgroundHealStatusRequest) returns (BackgroundHealStatusResponse) {};
|
||||
rpc GetMetacacheListing(GetMetacacheListingRequest) returns (GetMetacacheListingResponse) {};
|
||||
rpc UpdateMetacacheListing(UpdateMetacacheListingRequest) returns (UpdateMetacacheListingResponse) {};
|
||||
|
||||
@@ -58,8 +58,11 @@ pub use data_usage_define::*;
|
||||
pub use error::ScannerError;
|
||||
pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
|
||||
pub use rustfs_common::last_minute;
|
||||
pub use scanner::init_data_scanner;
|
||||
pub use scanner_io::{clear_dirty_usage_bucket, record_dirty_usage_bucket};
|
||||
pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status};
|
||||
pub use scanner_io::{
|
||||
clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch,
|
||||
scanner_maintenance_generation,
|
||||
};
|
||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
||||
|
||||
@@ -38,11 +38,14 @@ use std::fmt;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{LazyLock, RwLock};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Notify;
|
||||
use tracing::warn;
|
||||
|
||||
const LOG_COMPONENT_SCANNER: &str = "scanner";
|
||||
#[cfg(test)]
|
||||
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
|
||||
const LOG_SUBSYSTEM_RUNTIME_CONFIG: &str = "runtime_config";
|
||||
#[cfg(test)]
|
||||
const EVENT_SCANNER_RUNTIME_CONFIG: &str = "scanner_runtime_config";
|
||||
const EVENT_SCANNER_RUNTIME_CONFIG_PARSE: &str = "scanner_runtime_config_parse";
|
||||
|
||||
@@ -52,6 +55,8 @@ const MAX_SCANNER_DELAY_FACTOR: f64 = 10_000.0;
|
||||
const SCANNER_DELAY_RANGE_REASON: &str = "expected scanner delay between 0 and 10000";
|
||||
|
||||
static SCANNER_DEFAULT_CYCLE_SECS: AtomicU64 = AtomicU64::new(NO_DEFAULT_CYCLE_OVERRIDE);
|
||||
static SCANNER_RUNTIME_CONFIG_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
static SCANNER_RUNTIME_CONFIG_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
|
||||
static SCANNER_RUNTIME_CONFIG: LazyLock<RwLock<ScannerRuntimeConfig>> =
|
||||
LazyLock::new(|| RwLock::new(lookup_scanner_runtime_config(None).unwrap_or_default()));
|
||||
@@ -678,6 +683,7 @@ fn current_server_config() -> Option<ServerConfig> {
|
||||
resolve_scanner_server_config()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn resolve_scanner_runtime_config_from_global() -> ScannerRuntimeConfig {
|
||||
let config = current_server_config();
|
||||
match lookup_scanner_runtime_config(config.as_ref()) {
|
||||
@@ -705,9 +711,19 @@ pub fn apply_scanner_runtime_config(config: &ServerConfig) -> Result<(), Scanner
|
||||
validate_scanner_runtime_config(config)?;
|
||||
let resolved = lookup_scanner_runtime_config(Some(config))?;
|
||||
apply_resolved_runtime_config(resolved);
|
||||
SCANNER_RUNTIME_CONFIG_GENERATION.fetch_add(1, Ordering::AcqRel);
|
||||
SCANNER_RUNTIME_CONFIG_NOTIFY.notify_one();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_runtime_config_generation() -> u64 {
|
||||
SCANNER_RUNTIME_CONFIG_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) async fn scanner_runtime_config_changed() {
|
||||
SCANNER_RUNTIME_CONFIG_NOTIFY.notified().await;
|
||||
}
|
||||
|
||||
pub(crate) fn refresh_scanner_runtime_config_from_global() -> Result<(), ScannerRuntimeConfigError> {
|
||||
let config = current_server_config();
|
||||
let resolved = lookup_scanner_runtime_config(config.as_ref())?;
|
||||
@@ -846,11 +862,11 @@ mod tests {
|
||||
use crate::init_ecstore_config_for_scanner_tests;
|
||||
use rustfs_config::server_config::{Config as ServerConfig, KVS};
|
||||
use rustfs_config::{
|
||||
DEFAULT_DELIMITER, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE,
|
||||
ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE,
|
||||
HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES,
|
||||
SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS,
|
||||
ScannerSpeed,
|
||||
DEFAULT_DELIMITER, DEFAULT_HEAL_BITROT_CYCLE_SECS, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS,
|
||||
ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED,
|
||||
HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE,
|
||||
SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
|
||||
SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::collections::HashMap;
|
||||
@@ -927,6 +943,22 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_normalizes_persisted_default_speed() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "default")]);
|
||||
|
||||
with_var_unset(ENV_SCANNER_SPEED, || {
|
||||
with_var_unset(ENV_SCANNER_CYCLE, || {
|
||||
let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
|
||||
|
||||
assert_eq!(resolved.speed, ScannerSpeed::Default);
|
||||
assert_eq!(resolved.speed_source, ScannerRuntimeConfigSource::Default);
|
||||
assert_eq!(resolved.cycle_interval_source, ScannerRuntimeConfigSource::Default);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_prefers_env_over_persisted_config() {
|
||||
@@ -974,6 +1006,23 @@ mod tests {
|
||||
super::refresh_scanner_runtime_config_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_normalizes_persisted_default_bitrot_cycles() {
|
||||
let default_cycle = DEFAULT_HEAL_BITROT_CYCLE_SECS.to_string();
|
||||
for config in [
|
||||
server_config_with_scanner_and_heal(&[], &[(HEAL_BITROT_CYCLE, default_cycle.as_str())]),
|
||||
server_config_with_scanner(&[(SCANNER_BITROT_CYCLE, default_cycle.as_str())]),
|
||||
] {
|
||||
with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || {
|
||||
let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
|
||||
|
||||
assert_eq!(resolved.bitrot_cycle, Some(Duration::from_secs(DEFAULT_HEAL_BITROT_CYCLE_SECS)));
|
||||
assert_eq!(resolved.bitrot_cycle_source, ScannerRuntimeConfigSource::Default);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_runtime_config_rejects_invalid_persisted_speed() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "warp")]);
|
||||
@@ -1058,6 +1107,23 @@ mod tests {
|
||||
super::refresh_scanner_runtime_config_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn applied_runtime_config_is_the_authoritative_scheduler_state() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_CYCLE, "321")]);
|
||||
|
||||
with_var_unset(ENV_SCANNER_CYCLE, || {
|
||||
let generation = super::scanner_runtime_config_generation();
|
||||
super::apply_scanner_runtime_config(&config).expect("scanner runtime config should apply");
|
||||
|
||||
let applied = super::current_scanner_runtime_config();
|
||||
assert_eq!(applied.cycle_interval, Duration::from_secs(321));
|
||||
assert_eq!(applied.cycle_interval_source, ScannerRuntimeConfigSource::Config);
|
||||
assert!(super::scanner_runtime_config_generation() > generation);
|
||||
});
|
||||
super::refresh_scanner_runtime_config_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_runtime_config_status_reports_persisted_pacing_overrides() {
|
||||
|
||||
+2368
-103
File diff suppressed because it is too large
Load Diff
@@ -31,7 +31,7 @@ use rustfs_utils::path::path_join_buf;
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{LazyLock, Mutex as StdMutex, MutexGuard};
|
||||
use std::time::{Instant, SystemTime};
|
||||
use std::{fmt::Debug, sync::Arc};
|
||||
@@ -72,6 +72,13 @@ const METRIC_SCANNER_DISK_BUCKET_SCANS_QUEUED: &str = "rustfs_scanner_disk_bucke
|
||||
|
||||
pub type DirtyUsageBuckets = HashMap<String, u64>;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct DirtyUsageSnapshot {
|
||||
buckets: Arc<DirtyUsageBuckets>,
|
||||
generation: u64,
|
||||
covers_all_pending: bool,
|
||||
}
|
||||
|
||||
pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool {
|
||||
matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR))
|
||||
}
|
||||
@@ -115,6 +122,7 @@ pub struct ScannerBucketScanPlan {
|
||||
buckets: Vec<BucketInfo>,
|
||||
dirty_usage_buckets: Arc<DirtyUsageBuckets>,
|
||||
failed_dirty_buckets: Arc<Mutex<HashSet<String>>>,
|
||||
pending_maintenance_work: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl ScannerBucketScanPlan {
|
||||
@@ -122,11 +130,13 @@ impl ScannerBucketScanPlan {
|
||||
buckets: Vec<BucketInfo>,
|
||||
dirty_usage_buckets: Arc<DirtyUsageBuckets>,
|
||||
failed_dirty_buckets: Arc<Mutex<HashSet<String>>>,
|
||||
pending_maintenance_work: Arc<AtomicBool>,
|
||||
) -> Self {
|
||||
Self {
|
||||
buckets,
|
||||
dirty_usage_buckets,
|
||||
failed_dirty_buckets,
|
||||
pending_maintenance_work,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -134,6 +144,9 @@ impl ScannerBucketScanPlan {
|
||||
static DIRTY_USAGE_BUCKET_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
static DIRTY_USAGE_BUCKETS: LazyLock<StdMutex<DirtyUsageBuckets>> = LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
static SCANNER_ACTIVITY_EPOCH: LazyLock<String> = LazyLock::new(|| format!("{:032x}", rand::random::<u128>()));
|
||||
static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
static SCANNER_MAINTENANCE_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
|
||||
fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
|
||||
DIRTY_USAGE_BUCKETS.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
@@ -148,9 +161,9 @@ pub fn record_dirty_usage_bucket(bucket: &str) {
|
||||
return;
|
||||
}
|
||||
|
||||
let generation = DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let generation = DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
dirty_buckets.insert(bucket.to_string(), generation);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
@@ -158,6 +171,32 @@ pub fn record_dirty_usage_bucket(bucket: &str) {
|
||||
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
||||
}
|
||||
|
||||
pub fn record_scanner_maintenance_change(bucket: &str) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
SCANNER_MAINTENANCE_GENERATION.fetch_add(1, Ordering::AcqRel);
|
||||
SCANNER_MAINTENANCE_NOTIFY.notify_one();
|
||||
record_dirty_usage_bucket(bucket);
|
||||
}
|
||||
|
||||
pub fn scanner_maintenance_generation() -> u64 {
|
||||
SCANNER_MAINTENANCE_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) async fn scanner_maintenance_changed() {
|
||||
SCANNER_MAINTENANCE_NOTIFY.notified().await;
|
||||
}
|
||||
|
||||
pub fn scanner_activity_epoch() -> &'static str {
|
||||
SCANNER_ACTIVITY_EPOCH.as_str()
|
||||
}
|
||||
|
||||
pub(crate) fn dirty_usage_generation() -> u64 {
|
||||
DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub fn clear_dirty_usage_bucket(bucket: &str) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
@@ -166,25 +205,39 @@ pub fn clear_dirty_usage_bucket(bucket: &str) {
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
dirty_buckets.remove(bucket);
|
||||
DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_clear(usize_to_u64_saturated(pending_buckets));
|
||||
}
|
||||
|
||||
fn snapshot_dirty_usage_buckets(buckets: &[BucketInfo]) -> DirtyUsageBuckets {
|
||||
let snapshot = {
|
||||
fn snapshot_dirty_usage_buckets(buckets: &[BucketInfo], absent_generation_cutoff: u64) -> DirtyUsageSnapshot {
|
||||
let (snapshot, generation, covers_all_pending) = {
|
||||
let dirty_buckets = dirty_usage_buckets();
|
||||
buckets
|
||||
let listed_buckets = dirty_buckets
|
||||
.values()
|
||||
.any(|generation| *generation > absent_generation_cutoff)
|
||||
.then(|| buckets.iter().map(|bucket| bucket.name.as_str()).collect::<HashSet<_>>());
|
||||
let snapshot = dirty_buckets
|
||||
.iter()
|
||||
.filter_map(|bucket| {
|
||||
dirty_buckets
|
||||
.get(&bucket.name)
|
||||
.map(|generation| (bucket.name.clone(), *generation))
|
||||
.filter(|(bucket, generation)| {
|
||||
**generation <= absent_generation_cutoff
|
||||
|| listed_buckets
|
||||
.as_ref()
|
||||
.is_some_and(|listed_buckets| listed_buckets.contains(bucket.as_str()))
|
||||
})
|
||||
.collect::<DirtyUsageBuckets>()
|
||||
.map(|(bucket, generation)| (bucket.clone(), *generation))
|
||||
.collect::<DirtyUsageBuckets>();
|
||||
let generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire);
|
||||
let covers_all_pending = generation == absent_generation_cutoff && snapshot.len() == dirty_buckets.len();
|
||||
(snapshot, generation, covers_all_pending)
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_cycle_snapshot(usize_to_u64_saturated(snapshot.len()));
|
||||
snapshot
|
||||
DirtyUsageSnapshot {
|
||||
buckets: Arc::new(snapshot),
|
||||
generation,
|
||||
covers_all_pending,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn dirty_usage_buckets_pending() -> bool {
|
||||
@@ -205,6 +258,9 @@ fn clear_dirty_usage_buckets(snapshot: &DirtyUsageBuckets) {
|
||||
cleared_buckets += 1;
|
||||
}
|
||||
}
|
||||
if cleared_buckets > 0 {
|
||||
DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
(cleared_buckets, dirty_buckets.len())
|
||||
};
|
||||
global_metrics()
|
||||
@@ -237,13 +293,8 @@ async fn record_failed_dirty_bucket(failed_buckets: &Arc<Mutex<HashSet<String>>>
|
||||
failed_buckets.lock().await.insert(bucket.to_string());
|
||||
}
|
||||
|
||||
fn dirty_usage_snapshot_covers_current(snapshot: &DirtyUsageBuckets) -> bool {
|
||||
let dirty_buckets = dirty_usage_buckets();
|
||||
dirty_buckets.iter().all(|(bucket, generation)| {
|
||||
snapshot
|
||||
.get(bucket)
|
||||
.is_some_and(|snapshot_generation| snapshot_generation == generation)
|
||||
})
|
||||
fn dirty_usage_snapshot_covers_current(snapshot: &DirtyUsageSnapshot) -> bool {
|
||||
snapshot.covers_all_pending && DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire) == snapshot.generation
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -252,10 +303,15 @@ fn dirty_usage_bucket_count() -> usize {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn clear_dirty_usage_buckets_for_tests() {
|
||||
pub(crate) fn clear_dirty_usage_buckets_for_tests() {
|
||||
dirty_usage_buckets().clear();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn dirty_usage_buckets_for_tests() -> DirtyUsageBuckets {
|
||||
dirty_usage_buckets().clone()
|
||||
}
|
||||
|
||||
fn bucket_usage_scan_order(
|
||||
buckets: &[BucketInfo],
|
||||
old_cache: &DataUsageCache,
|
||||
@@ -479,6 +535,34 @@ fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option<Error
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn classify_nsscanner_cycle(
|
||||
completed_all_sets: bool,
|
||||
budget_elapsed: bool,
|
||||
cancelled: bool,
|
||||
has_failed_buckets: bool,
|
||||
dirty_usage_current: bool,
|
||||
) -> ScannerCycleStatus {
|
||||
if completed_all_sets && !budget_elapsed && !cancelled && !has_failed_buckets && dirty_usage_current {
|
||||
ScannerCycleStatus::Complete
|
||||
} else {
|
||||
ScannerCycleStatus::Incomplete
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_results_have_pending_maintenance_work(results: &[DataUsageCache]) -> bool {
|
||||
results.iter().any(|result| !result.info.pending_heals.is_empty())
|
||||
}
|
||||
|
||||
fn pending_maintenance_work_for_cycle(pending: &AtomicBool, results: &[DataUsageCache]) -> bool {
|
||||
pending.load(Ordering::Acquire) || scanner_results_have_pending_maintenance_work(results)
|
||||
}
|
||||
|
||||
fn record_bucket_pending_maintenance_work(cache: &DataUsageCache, pending: &AtomicBool) {
|
||||
if !cache.info.pending_heals.is_empty() {
|
||||
pending.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
fn is_xl_meta_path(path: &str) -> bool {
|
||||
Path::new(path)
|
||||
.file_name()
|
||||
@@ -588,7 +672,9 @@ mod publish_gate_tests {
|
||||
async fn send_cache_root_entry_info(
|
||||
bucket_result_tx: &Arc<Mutex<mpsc::Sender<DataUsageEntryInfo>>>,
|
||||
cache: &DataUsageCache,
|
||||
pending_maintenance_work: &AtomicBool,
|
||||
) -> std::result::Result<(), mpsc::error::SendError<DataUsageEntryInfo>> {
|
||||
record_bucket_pending_maintenance_work(cache, pending_maintenance_work);
|
||||
bucket_result_tx.lock().await.send(cache_root_entry_info(cache)).await
|
||||
}
|
||||
|
||||
@@ -656,6 +742,18 @@ pub trait ScannerIO: Send + Sync + Debug + 'static {
|
||||
) -> Result<()>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait ScannerIOCycle: Send + Sync + Debug + 'static {
|
||||
async fn nsscanner_with_status(
|
||||
&self,
|
||||
ctx: CancellationToken,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
updates: mpsc::Sender<DataUsageInfo>,
|
||||
want_cycle: u64,
|
||||
scan_mode: HealScanMode,
|
||||
) -> Result<ScannerCycleResult>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait ScannerIOCache: Send + Sync + Debug + 'static {
|
||||
async fn nsscanner_cache(
|
||||
@@ -689,9 +787,61 @@ pub enum ScannerDiskScanOutcome {
|
||||
Partial(DataUsageCache),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ScannerCycleStatus {
|
||||
Complete,
|
||||
Incomplete,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ScannerCycleResult {
|
||||
pub(crate) status: ScannerCycleStatus,
|
||||
dirty_usage_clear: Option<DirtyUsageBuckets>,
|
||||
failed_dirty_usage: bool,
|
||||
pending_maintenance_work: bool,
|
||||
}
|
||||
|
||||
impl ScannerCycleResult {
|
||||
pub(crate) fn new(status: ScannerCycleStatus, dirty_usage_clear: Option<DirtyUsageBuckets>) -> Self {
|
||||
Self {
|
||||
status,
|
||||
dirty_usage_clear,
|
||||
failed_dirty_usage: false,
|
||||
pending_maintenance_work: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_failed_dirty_usage(mut self, failed_dirty_usage: bool) -> Self {
|
||||
self.failed_dirty_usage = failed_dirty_usage;
|
||||
self
|
||||
}
|
||||
|
||||
fn with_pending_maintenance_work(mut self, pending_maintenance_work: bool) -> Self {
|
||||
self.pending_maintenance_work = pending_maintenance_work;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn acknowledge_durable_usage(self) {
|
||||
if let Some(snapshot) = self.dirty_usage_clear {
|
||||
clear_dirty_usage_buckets(&snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_dirty_usage_to_acknowledge(&self) -> bool {
|
||||
self.dirty_usage_clear.as_ref().is_some_and(|snapshot| !snapshot.is_empty())
|
||||
}
|
||||
|
||||
pub(crate) fn has_failed_dirty_usage(&self) -> bool {
|
||||
self.failed_dirty_usage
|
||||
}
|
||||
|
||||
pub(crate) fn has_pending_maintenance_work(&self) -> bool {
|
||||
self.pending_maintenance_work
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ScannerIO for ECStore {
|
||||
#[tracing::instrument(skip(self, budget, updates))]
|
||||
async fn nsscanner(
|
||||
&self,
|
||||
ctx: CancellationToken,
|
||||
@@ -700,13 +850,40 @@ impl ScannerIO for ECStore {
|
||||
want_cycle: u64,
|
||||
scan_mode: HealScanMode,
|
||||
) -> Result<()> {
|
||||
// Preserve the public API's pre-existing completion semantics for
|
||||
// embedders. The main scanner uses nsscanner_with_status so it can
|
||||
// delay this acknowledgement until usage persistence succeeds.
|
||||
ScannerIOCycle::nsscanner_with_status(self, ctx, budget, updates, want_cycle, scan_mode)
|
||||
.await?
|
||||
.acknowledge_durable_usage();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ScannerIOCycle for ECStore {
|
||||
#[tracing::instrument(skip(self, budget, updates))]
|
||||
async fn nsscanner_with_status(
|
||||
&self,
|
||||
ctx: CancellationToken,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
updates: mpsc::Sender<DataUsageInfo>,
|
||||
want_cycle: u64,
|
||||
scan_mode: HealScanMode,
|
||||
) -> Result<ScannerCycleResult> {
|
||||
let child_token = ctx.child_token();
|
||||
|
||||
let dirty_generation_before_bucket_list = dirty_usage_generation();
|
||||
let all_buckets = self.list_bucket(&BucketOptions::default()).await?;
|
||||
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
|
||||
|
||||
if all_buckets.is_empty() {
|
||||
reset_set_scan_gauges();
|
||||
if let Err(e) = updates.send(DataUsageInfo::default()).await {
|
||||
let empty_usage = DataUsageInfo {
|
||||
last_update: Some(SystemTime::now()),
|
||||
..Default::default()
|
||||
};
|
||||
if let Err(e) = updates.send(empty_usage).await {
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_SET_STATE,
|
||||
@@ -717,7 +894,14 @@ impl ScannerIO for ECStore {
|
||||
"Scanner set state update failed"
|
||||
);
|
||||
}
|
||||
return Ok(());
|
||||
let status = if dirty_usage_snapshot_covers_current(&dirty_usage_snapshot) {
|
||||
ScannerCycleStatus::Complete
|
||||
} else {
|
||||
ScannerCycleStatus::Incomplete
|
||||
};
|
||||
let dirty_usage_clear =
|
||||
(status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone());
|
||||
return Ok(ScannerCycleResult::new(status, dirty_usage_clear));
|
||||
}
|
||||
|
||||
let mut total_results = 0;
|
||||
@@ -735,12 +919,12 @@ impl ScannerIO for ECStore {
|
||||
"Scanner set state update detected missing disk sets"
|
||||
);
|
||||
reset_set_scan_gauges();
|
||||
return Ok(());
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
|
||||
}
|
||||
|
||||
let set_scan_limit = scanner_max_concurrent_set_scans(total_results);
|
||||
let dirty_usage_buckets = Arc::new(snapshot_dirty_usage_buckets(&all_buckets));
|
||||
let failed_dirty_buckets = Arc::new(Mutex::new(HashSet::<String>::new()));
|
||||
let pending_maintenance_work = Arc::new(AtomicBool::new(false));
|
||||
record_set_scan_concurrency_limit(set_scan_limit);
|
||||
debug!(
|
||||
target: "rustfs::scanner::io",
|
||||
@@ -794,8 +978,12 @@ impl ScannerIO for ECStore {
|
||||
});
|
||||
wait_futs.push(receiver_fut);
|
||||
|
||||
let scan_plan =
|
||||
ScannerBucketScanPlan::new(all_buckets.clone(), dirty_usage_buckets.clone(), failed_dirty_buckets.clone());
|
||||
let scan_plan = ScannerBucketScanPlan::new(
|
||||
all_buckets.clone(),
|
||||
dirty_usage_snapshot.buckets.clone(),
|
||||
failed_dirty_buckets.clone(),
|
||||
pending_maintenance_work.clone(),
|
||||
);
|
||||
// Spawn task to run the scanner
|
||||
let scanner_fut = tokio::spawn(async move {
|
||||
let permit_wait = child_token_clone.clone();
|
||||
@@ -870,7 +1058,7 @@ impl ScannerIO for ECStore {
|
||||
let results_mutex_for_updates = results_mutex.clone();
|
||||
let budget_for_updates = budget.clone();
|
||||
let child_token_for_updates = child_token.clone();
|
||||
let dirty_usage_buckets_for_updates = dirty_usage_buckets.clone();
|
||||
let dirty_usage_snapshot_for_updates = dirty_usage_snapshot.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut last_update = SystemTime::UNIX_EPOCH;
|
||||
let mut has_sent_once = false;
|
||||
@@ -893,7 +1081,7 @@ impl ScannerIO for ECStore {
|
||||
&all_buckets_clone,
|
||||
budget_for_updates.budget_elapsed(),
|
||||
child_token_for_updates.is_cancelled(),
|
||||
dirty_usage_snapshot_covers_current(dirty_usage_buckets_for_updates.as_ref()),
|
||||
dirty_usage_snapshot_covers_current(dirty_usage_snapshot_for_updates.as_ref()),
|
||||
)
|
||||
};
|
||||
|
||||
@@ -912,7 +1100,7 @@ impl ScannerIO for ECStore {
|
||||
&all_buckets_clone,
|
||||
budget_for_updates.budget_elapsed(),
|
||||
child_token_for_updates.is_cancelled(),
|
||||
dirty_usage_snapshot_covers_current(dirty_usage_buckets_for_updates.as_ref()),
|
||||
dirty_usage_snapshot_covers_current(dirty_usage_snapshot_for_updates.as_ref()),
|
||||
)
|
||||
};
|
||||
|
||||
@@ -954,16 +1142,27 @@ impl ScannerIO for ECStore {
|
||||
let completed_all_sets = results.iter().all(|result| result.info.last_update.is_some());
|
||||
let result = finalize_nsscanner_result(&results, first_err);
|
||||
let failed_buckets = failed_dirty_buckets.lock().await.clone();
|
||||
if let Some(clear_snapshot) = should_clear_dirty_usage_snapshot(
|
||||
let pending_maintenance_work = pending_maintenance_work_for_cycle(&pending_maintenance_work, &results);
|
||||
let budget_elapsed = budget.budget_elapsed();
|
||||
let dirty_usage_current = dirty_usage_snapshot_covers_current(&dirty_usage_snapshot);
|
||||
let cycle_status = classify_nsscanner_cycle(
|
||||
completed_all_sets,
|
||||
budget_elapsed,
|
||||
ctx.is_cancelled(),
|
||||
!failed_buckets.is_empty(),
|
||||
dirty_usage_current,
|
||||
);
|
||||
let dirty_usage_clear = should_clear_dirty_usage_snapshot(
|
||||
result.is_ok(),
|
||||
completed_all_sets,
|
||||
budget.budget_elapsed(),
|
||||
&dirty_usage_buckets,
|
||||
budget_elapsed,
|
||||
&dirty_usage_snapshot.buckets,
|
||||
&failed_buckets,
|
||||
) {
|
||||
clear_dirty_usage_buckets(&clear_snapshot);
|
||||
}
|
||||
result
|
||||
);
|
||||
result?;
|
||||
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
|
||||
.with_failed_dirty_usage(!failed_buckets.is_empty())
|
||||
.with_pending_maintenance_work(pending_maintenance_work))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -983,6 +1182,7 @@ impl ScannerIOCache for SetDisks {
|
||||
buckets,
|
||||
dirty_usage_buckets,
|
||||
failed_dirty_buckets,
|
||||
pending_maintenance_work,
|
||||
} = scan_plan;
|
||||
let pool_label = self.pool_index.to_string();
|
||||
let set_label = self.set_index.to_string();
|
||||
@@ -1127,6 +1327,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let pool_label_clone = pool_label.clone();
|
||||
let set_label_clone = set_label.clone();
|
||||
let failed_dirty_buckets_clone = failed_dirty_buckets.clone();
|
||||
let pending_maintenance_work_clone = pending_maintenance_work.clone();
|
||||
futs.push(tokio::spawn(async move {
|
||||
loop {
|
||||
let Some(bucket) = bucket_rx_mutex_clone.lock().await.recv().await else {
|
||||
@@ -1322,7 +1523,6 @@ impl ScannerIOCache for SetDisks {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
debug!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_DISK_BUCKET_STATE,
|
||||
@@ -1349,7 +1549,9 @@ impl ScannerIOCache for SetDisks {
|
||||
"Scanner root entry publish started"
|
||||
);
|
||||
|
||||
if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await {
|
||||
if let Err(e) =
|
||||
send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache, &pending_maintenance_work_clone).await
|
||||
{
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
@@ -1761,10 +1963,10 @@ mod tests {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
let buckets = vec![bucket_info("photos")];
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets);
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation());
|
||||
|
||||
record_dirty_usage_bucket("photos");
|
||||
clear_dirty_usage_buckets(&snapshot);
|
||||
clear_dirty_usage_buckets(&snapshot.buckets);
|
||||
|
||||
assert_eq!(dirty_usage_bucket_count(), 1);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
@@ -1776,7 +1978,7 @@ mod tests {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
let buckets = vec![bucket_info("photos")];
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets);
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation());
|
||||
|
||||
assert!(dirty_usage_snapshot_covers_current(&snapshot));
|
||||
|
||||
@@ -1786,6 +1988,83 @@ mod tests {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
record_dirty_usage_bucket("temporarily-omitted");
|
||||
let generation_before_bucket_list = dirty_usage_generation();
|
||||
|
||||
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], generation_before_bucket_list);
|
||||
|
||||
assert!(snapshot.buckets.contains_key("photos"));
|
||||
assert!(snapshot.buckets.contains_key("temporarily-omitted"));
|
||||
assert!(dirty_usage_buckets().contains_key("temporarily-omitted"));
|
||||
assert!(dirty_usage_snapshot_covers_current(&snapshot));
|
||||
|
||||
ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone()))
|
||||
.acknowledge_durable_usage();
|
||||
assert!(!dirty_usage_buckets().contains_key("temporarily-omitted"));
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_snapshot_preserves_an_absent_bucket_recorded_after_listing_started() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let generation_before_bucket_list = dirty_usage_generation();
|
||||
record_dirty_usage_bucket("new-or-racing-bucket");
|
||||
|
||||
let snapshot = snapshot_dirty_usage_buckets(&[], generation_before_bucket_list);
|
||||
|
||||
assert!(!snapshot.buckets.contains_key("new-or-racing-bucket"));
|
||||
assert!(!dirty_usage_snapshot_covers_current(&snapshot));
|
||||
assert!(dirty_usage_buckets().contains_key("new-or-racing-bucket"));
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation());
|
||||
assert!(dirty_usage_snapshot_covers_current(&snapshot));
|
||||
|
||||
record_dirty_usage_bucket("photos");
|
||||
|
||||
assert!(!dirty_usage_snapshot_covers_current(&snapshot));
|
||||
assert!(dirty_usage_buckets().contains_key("photos"));
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn deleting_a_bucket_during_listing_invalidates_the_resulting_usage_snapshot() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let generation_before_bucket_list = dirty_usage_generation();
|
||||
|
||||
record_dirty_usage_bucket("photos");
|
||||
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], generation_before_bucket_list);
|
||||
|
||||
assert!(!dirty_usage_snapshot_covers_current(&snapshot));
|
||||
assert!(dirty_usage_buckets().contains_key("photos"));
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_maintenance_change_advances_generation_and_marks_usage_dirty() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let generation = scanner_maintenance_generation();
|
||||
|
||||
record_scanner_maintenance_change("photos");
|
||||
|
||||
assert!(scanner_maintenance_generation() > generation);
|
||||
assert!(dirty_usage_buckets().contains_key("photos"));
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_clear_excludes_failed_buckets() {
|
||||
@@ -1793,9 +2072,9 @@ mod tests {
|
||||
record_dirty_usage_bucket("photos");
|
||||
record_dirty_usage_bucket("videos");
|
||||
let buckets = vec![bucket_info("photos"), bucket_info("videos")];
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets);
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation());
|
||||
let failed_buckets = HashSet::from(["videos".to_string()]);
|
||||
let clear_snapshot = dirty_usage_buckets_excluding_failed(&snapshot, &failed_buckets);
|
||||
let clear_snapshot = dirty_usage_buckets_excluding_failed(&snapshot.buckets, &failed_buckets);
|
||||
|
||||
clear_dirty_usage_buckets(&clear_snapshot);
|
||||
|
||||
@@ -1818,6 +2097,23 @@ mod tests {
|
||||
assert!(!clear_snapshot.contains_key("videos"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_is_acknowledged_only_after_durable_usage_confirmation() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_bucket("photos");
|
||||
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation());
|
||||
|
||||
let unconfirmed = ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone()));
|
||||
drop(unconfirmed);
|
||||
assert!(dirty_usage_buckets().contains_key("photos"));
|
||||
|
||||
let confirmed = ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone()));
|
||||
confirmed.acknowledge_durable_usage();
|
||||
assert!(!dirty_usage_buckets().contains_key("photos"));
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn clear_dirty_usage_bucket_removes_deleted_bucket_marker() {
|
||||
@@ -1828,9 +2124,9 @@ mod tests {
|
||||
clear_dirty_usage_bucket("photos");
|
||||
|
||||
let buckets = vec![bucket_info("photos"), bucket_info("videos")];
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets);
|
||||
assert!(!snapshot.contains_key("photos"));
|
||||
assert!(snapshot.contains_key("videos"));
|
||||
let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation());
|
||||
assert!(!snapshot.buckets.contains_key("photos"));
|
||||
assert!(snapshot.buckets.contains_key("videos"));
|
||||
assert_eq!(dirty_usage_bucket_count(), 1);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
@@ -1896,6 +2192,78 @@ mod tests {
|
||||
assert!(err.to_string().contains("set failed"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
|
||||
assert_eq!(classify_nsscanner_cycle(true, false, false, false, true), ScannerCycleStatus::Complete);
|
||||
|
||||
for status in [
|
||||
classify_nsscanner_cycle(false, false, false, false, true),
|
||||
classify_nsscanner_cycle(true, true, false, false, true),
|
||||
classify_nsscanner_cycle(true, false, true, false, true),
|
||||
classify_nsscanner_cycle(true, false, false, true, true),
|
||||
classify_nsscanner_cycle(true, false, false, false, false),
|
||||
] {
|
||||
assert_eq!(status, ScannerCycleStatus::Incomplete);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_cycle_surfaces_persisted_pending_heal_work() {
|
||||
let clean = DataUsageCache::default();
|
||||
assert!(!scanner_results_have_pending_maintenance_work(std::slice::from_ref(&clean)));
|
||||
|
||||
let mut pending = clean;
|
||||
pending.info.pending_heals.push(crate::PendingScannerHeal {
|
||||
kind: crate::PendingScannerHealKind::Object,
|
||||
bucket: "photos".to_string(),
|
||||
object: Some("image.jpg".to_string()),
|
||||
version_id: None,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
first_seen: 1,
|
||||
last_attempt: 1,
|
||||
attempts: 1,
|
||||
last_admission_result: "queue_full".to_string(),
|
||||
last_admission_reason: "capacity".to_string(),
|
||||
});
|
||||
|
||||
assert!(scanner_results_have_pending_maintenance_work(&[pending]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_cache_pending_heal_reaches_cycle_maintenance_state() {
|
||||
let pending_maintenance_work = Arc::new(AtomicBool::new(false));
|
||||
let mut bucket_cache = DataUsageCache::default();
|
||||
bucket_cache.info.pending_heals.push(crate::PendingScannerHeal {
|
||||
kind: crate::PendingScannerHealKind::Object,
|
||||
bucket: "photos".to_string(),
|
||||
object: Some("image.jpg".to_string()),
|
||||
version_id: None,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
first_seen: 1,
|
||||
last_attempt: 1,
|
||||
attempts: 1,
|
||||
last_admission_result: "queue_full".to_string(),
|
||||
last_admission_reason: "capacity".to_string(),
|
||||
});
|
||||
let (sender, mut receiver) = mpsc::channel(1);
|
||||
let sender = Arc::new(Mutex::new(sender));
|
||||
|
||||
send_cache_root_entry_info(&sender, &bucket_cache, &pending_maintenance_work)
|
||||
.await
|
||||
.expect("bucket result should send");
|
||||
|
||||
let cycle_pending = pending_maintenance_work_for_cycle(&pending_maintenance_work, &[]);
|
||||
assert!(cycle_pending);
|
||||
assert_eq!(
|
||||
crate::scanner::scanner_cycle_outcome_with_pending_maintenance(
|
||||
crate::scanner::ScannerCycleOutcome::Completed,
|
||||
cycle_pending,
|
||||
),
|
||||
crate::scanner::ScannerCycleOutcome::CompletedWithPendingMaintenance
|
||||
);
|
||||
assert!(receiver.recv().await.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
|
||||
|
||||
@@ -70,6 +70,38 @@ sleep multiplier, maximum wait, and cycle interval. Use `scanner.delay`,
|
||||
`scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis
|
||||
needs a precise override.
|
||||
|
||||
## Single-disk clean-idle scheduling
|
||||
|
||||
An erasure single-disk deployment using the built-in cycle and bitrot defaults
|
||||
automatically backs off repeated clean idle scans instead of walking the same
|
||||
unchanged namespace every minute. Each successful timer-driven cycle that
|
||||
finds no dirty usage or unresolved maintenance work doubles the next interval.
|
||||
The status endpoint reports the effective interval and multiplier.
|
||||
|
||||
The backoff is reset to the base interval by object or bucket mutations,
|
||||
lifecycle or replication configuration changes, partial or failed cycles,
|
||||
usage persistence failures, and unresolved scanner-originated heal or bitrot
|
||||
work. Active lifecycle or replication rules keep the base cadence. An explicit
|
||||
cycle, a non-default persisted speed, any environment speed or start-delay
|
||||
override, an environment bitrot override, or a non-default persisted active
|
||||
bitrot cycle also keeps the configured cadence rather than applying the
|
||||
automatic policy. Persisting `scanner.speed=default` or the default bitrot cycle
|
||||
is normalized to the built-in default and therefore keeps automatic scheduling
|
||||
enabled.
|
||||
|
||||
Lifecycle and replication configuration inspection is bounded so a slow
|
||||
metadata read cannot stall scanner startup or scheduling. A failed or timed-out
|
||||
inspection keeps the base cadence and is retried after 5 minutes, doubling up
|
||||
to a maximum of 60 minutes while failures continue. A lifecycle or replication
|
||||
configuration change wakes the scanner and retries inspection immediately.
|
||||
|
||||
With the default 30-day bitrot cycle, the clean-idle interval is capped at the
|
||||
bitrot cycle divided by the object selection window. With the default selection
|
||||
window this is about 42 minutes, which preserves the intended wall-clock bitrot
|
||||
coverage. If periodic bitrot is disabled, the clean-idle policy cap is 24 hours.
|
||||
The effective interval is jittered by up to 10 percent to avoid synchronized
|
||||
scanner starts.
|
||||
|
||||
## Status Endpoint
|
||||
|
||||
The scanner status route is:
|
||||
@@ -79,9 +111,12 @@ GET /v3/scanner/status
|
||||
```
|
||||
|
||||
The request must be authenticated with an admin identity that has
|
||||
`ServerInfoAdminAction`. The JSON response has two top-level objects:
|
||||
`ServerInfoAdminAction`. The JSON response has three scanner-specific top-level
|
||||
objects:
|
||||
|
||||
- `runtime_config`: the effective runtime controls and their value sources.
|
||||
- `cycle_schedule`: the current effective cycle interval and clean-idle
|
||||
backoff state.
|
||||
- `metrics`: scanner work, pressure, checkpoint, lifecycle, replication, heal,
|
||||
bitrot, and alert counters.
|
||||
|
||||
@@ -93,6 +128,9 @@ runtime_config.delay.value
|
||||
runtime_config.max_wait_seconds.value
|
||||
runtime_config.cycle_interval_seconds.value
|
||||
runtime_config.bitrot_cycle_seconds.value
|
||||
cycle_schedule.effective_interval_seconds
|
||||
cycle_schedule.clean_idle_backoff_enabled
|
||||
cycle_schedule.clean_idle_backoff_multiplier
|
||||
metrics.pacing_pressure.primary_pressure
|
||||
metrics.pacing_pressure.last_cycle_budget_limited
|
||||
metrics.lifecycle_transition.current_queued
|
||||
|
||||
@@ -37,6 +37,7 @@ struct ScannerStatusResponse {
|
||||
disabled_reason: Option<String>,
|
||||
freshness: ScannerFreshnessStatus,
|
||||
metrics: ScannerMetricsReport,
|
||||
cycle_schedule: rustfs_scanner::ScannerCycleScheduleStatus,
|
||||
runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus,
|
||||
}
|
||||
|
||||
@@ -55,13 +56,15 @@ fn scanner_disabled_reason(enabled: bool) -> Option<String> {
|
||||
fn scanner_freshness_status(
|
||||
metrics: &ScannerMetricsReport,
|
||||
runtime_config: &rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus,
|
||||
effective_cycle_interval_seconds: u64,
|
||||
) -> ScannerFreshnessStatus {
|
||||
const FRESHNESS_MULTIPLIER: u64 = 2;
|
||||
|
||||
let max_expected_age_seconds = runtime_config
|
||||
let expected_cycle_interval_seconds = runtime_config
|
||||
.cycle_interval_seconds
|
||||
.value
|
||||
.saturating_mul(FRESHNESS_MULTIPLIER);
|
||||
.max(effective_cycle_interval_seconds);
|
||||
let max_expected_age_seconds = expected_cycle_interval_seconds.saturating_mul(FRESHNESS_MULTIPLIER);
|
||||
if metrics.last_cycle_end_unix_secs == 0 {
|
||||
return ScannerFreshnessStatus {
|
||||
state: "unknown",
|
||||
@@ -90,6 +93,23 @@ fn scanner_freshness_status(
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_status_response(
|
||||
enabled: bool,
|
||||
metrics: ScannerMetricsReport,
|
||||
runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus,
|
||||
cycle_schedule: rustfs_scanner::ScannerCycleScheduleStatus,
|
||||
) -> ScannerStatusResponse {
|
||||
let freshness = scanner_freshness_status(&metrics, &runtime_config, cycle_schedule.effective_interval_seconds());
|
||||
ScannerStatusResponse {
|
||||
enabled,
|
||||
disabled_reason: scanner_disabled_reason(enabled),
|
||||
freshness,
|
||||
metrics,
|
||||
cycle_schedule,
|
||||
runtime_config,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register_scanner_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
|
||||
r.insert(
|
||||
Method::GET,
|
||||
@@ -142,14 +162,8 @@ impl Operation for ScannerStatusHandler {
|
||||
let enabled = scanner_enabled_from_env();
|
||||
let metrics = current_scanner_metrics_report().await;
|
||||
let runtime_config = rustfs_scanner::scanner_runtime_config_status();
|
||||
let freshness = scanner_freshness_status(&metrics, &runtime_config);
|
||||
let response = ScannerStatusResponse {
|
||||
enabled,
|
||||
disabled_reason: scanner_disabled_reason(enabled),
|
||||
freshness,
|
||||
metrics,
|
||||
runtime_config,
|
||||
};
|
||||
let cycle_schedule = rustfs_scanner::scanner_cycle_schedule_status();
|
||||
let response = scanner_status_response(enabled, metrics, runtime_config, cycle_schedule);
|
||||
let body = serde_json::to_vec(&response).map_err(|err| {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, format!("failed to encode scanner status: {err}"))
|
||||
})?;
|
||||
@@ -174,7 +188,7 @@ mod tests {
|
||||
let mut runtime_config = rustfs_scanner::scanner_runtime_config_status();
|
||||
runtime_config.cycle_interval_seconds.value = 60;
|
||||
|
||||
let freshness = scanner_freshness_status(&metrics, &runtime_config);
|
||||
let freshness = scanner_freshness_status(&metrics, &runtime_config, 0);
|
||||
|
||||
assert_eq!(freshness.state, "unknown");
|
||||
assert_eq!(freshness.last_cycle_end_unix_secs, 0);
|
||||
@@ -191,10 +205,43 @@ mod tests {
|
||||
let mut runtime_config = rustfs_scanner::scanner_runtime_config_status();
|
||||
runtime_config.cycle_interval_seconds.value = 60;
|
||||
|
||||
let freshness = scanner_freshness_status(&metrics, &runtime_config);
|
||||
let freshness = scanner_freshness_status(&metrics, &runtime_config, 0);
|
||||
|
||||
assert_eq!(freshness.state, "stale");
|
||||
assert_eq!(freshness.max_expected_age_seconds, 120);
|
||||
assert_eq!(freshness.reason, Some("last cycle is older than freshness window"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_freshness_uses_effective_clean_idle_interval() {
|
||||
let metrics = ScannerMetricsReport {
|
||||
last_cycle_end_unix_secs: u64::try_from(Utc::now().timestamp().max(0))
|
||||
.expect("non-negative timestamp should fit in u64")
|
||||
.saturating_sub(300),
|
||||
..Default::default()
|
||||
};
|
||||
let mut runtime_config = rustfs_scanner::scanner_runtime_config_status();
|
||||
runtime_config.cycle_interval_seconds.value = 60;
|
||||
|
||||
let freshness = scanner_freshness_status(&metrics, &runtime_config, 3_600);
|
||||
|
||||
assert_eq!(freshness.state, "fresh");
|
||||
assert_eq!(freshness.max_expected_age_seconds, 7_200);
|
||||
assert_eq!(freshness.reason, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_status_serializes_cycle_schedule_contract() {
|
||||
let response = scanner_status_response(
|
||||
true,
|
||||
ScannerMetricsReport::default(),
|
||||
rustfs_scanner::scanner_runtime_config_status(),
|
||||
rustfs_scanner::ScannerCycleScheduleStatus::default(),
|
||||
);
|
||||
|
||||
let encoded = serde_json::to_value(response).expect("scanner status should serialize");
|
||||
assert_eq!(encoded["cycle_schedule"]["effective_interval_seconds"], 0);
|
||||
assert_eq!(encoded["cycle_schedule"]["clean_idle_backoff_enabled"], false);
|
||||
assert_eq!(encoded["cycle_schedule"]["clean_idle_backoff_multiplier"], 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,11 +245,11 @@ pub(crate) mod metadata_sys {
|
||||
}
|
||||
|
||||
pub(crate) async fn update(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
|
||||
super::ecstore_bucket::metadata_sys::update(bucket, config_file, data).await
|
||||
crate::storage::storage_api::update_bucket_metadata_config(bucket, config_file, data).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete(bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
|
||||
super::ecstore_bucket::metadata_sys::delete(bucket, config_file).await
|
||||
crate::storage::storage_api::delete_bucket_metadata_config(bucket, config_file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
|
||||
|
||||
@@ -485,16 +485,29 @@ fn notify_bucket_metadata_reload(
|
||||
bucket: String,
|
||||
operation: &'static str,
|
||||
request_context: Option<request_context::RequestContext>,
|
||||
scanner_maintenance_change: bool,
|
||||
) {
|
||||
record_local_scanner_maintenance_reload(&bucket, scanner_maintenance_change);
|
||||
spawn_background_with_context(request_context, async move {
|
||||
if let Some(notification_sys) = current_notification_system()
|
||||
&& let Err(err) = notification_sys.load_bucket_metadata(&bucket).await
|
||||
{
|
||||
warn!(bucket = %bucket, error = %err, "failed to notify peers after {operation}");
|
||||
if let Some(notification_sys) = current_notification_system() {
|
||||
let result = if scanner_maintenance_change {
|
||||
notification_sys.load_bucket_metadata_for_scanner_maintenance(&bucket).await
|
||||
} else {
|
||||
notification_sys.load_bucket_metadata(&bucket).await
|
||||
};
|
||||
if let Err(err) = result {
|
||||
warn!(bucket = %bucket, error = %err, "failed to notify peers after {operation}");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn record_local_scanner_maintenance_reload(bucket: &str, scanner_maintenance_change: bool) {
|
||||
if scanner_maintenance_change {
|
||||
rustfs_scanner::record_scanner_maintenance_change(bucket);
|
||||
}
|
||||
}
|
||||
|
||||
/// Notify peers to drop their cached metadata for a bucket that was just deleted,
|
||||
/// so they stop serving stale bucket configuration. Runs in the background to
|
||||
/// avoid blocking the delete response.
|
||||
@@ -1205,7 +1218,9 @@ impl DefaultBucketUsecase {
|
||||
// Invalidate bucket validation cache
|
||||
crate::storage::invalidate_bucket_validation_cache(&input.bucket);
|
||||
|
||||
rustfs_scanner::clear_dirty_usage_bucket(&input.bucket);
|
||||
// Re-evaluate lifecycle/replication after bucket removal and keep an
|
||||
// absent-bucket dirty marker until a complete usage snapshot is durable.
|
||||
rustfs_scanner::record_scanner_maintenance_change(&input.bucket);
|
||||
if let Err(err) = remove_bucket_usage_from_backend(store.clone(), &input.bucket).await {
|
||||
warn!(bucket = %input.bucket, error = ?err, "failed to remove deleted bucket from data usage");
|
||||
}
|
||||
@@ -1338,7 +1353,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket encryption", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket encryption", request_context, false);
|
||||
|
||||
let item = sr_bucket_meta_item(bucket.clone(), "sse-config");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
@@ -1369,7 +1384,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket cors", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket cors", request_context, false);
|
||||
|
||||
let item = sr_bucket_meta_item(bucket.clone(), "cors-config");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
@@ -1400,14 +1415,13 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket lifecycle", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket lifecycle", request_context, true);
|
||||
|
||||
let item = sr_bucket_meta_item(bucket.clone(), "lc-config");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket lifecycle delete hook failed");
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_bucket(&bucket);
|
||||
Ok(S3Response::new(DeleteBucketLifecycleOutput::default()))
|
||||
}
|
||||
|
||||
@@ -1431,7 +1445,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket policy", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket policy", request_context, false);
|
||||
|
||||
let item = sr_bucket_meta_item(bucket.clone(), "policy");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
@@ -1481,7 +1495,7 @@ impl DefaultBucketUsecase {
|
||||
}
|
||||
drop(targets_guard);
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket replication", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket replication", request_context, true);
|
||||
|
||||
let item = sr_bucket_meta_item(bucket.clone(), "replication-config");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
@@ -1490,7 +1504,6 @@ impl DefaultBucketUsecase {
|
||||
|
||||
info!(bucket = %bucket, "deleted bucket replication config");
|
||||
|
||||
rustfs_scanner::record_dirty_usage_bucket(&bucket);
|
||||
Ok(S3Response::new(DeleteBucketReplicationOutput::default()))
|
||||
}
|
||||
|
||||
@@ -1506,7 +1519,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket tagging", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket tagging", request_context, false);
|
||||
|
||||
let item = sr_bucket_meta_item(bucket.clone(), "tags");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
@@ -1538,7 +1551,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete public access block", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete public access block", request_context, false);
|
||||
|
||||
Ok(S3Response::with_status(DeletePublicAccessBlockOutput::default(), StatusCode::NO_CONTENT))
|
||||
}
|
||||
@@ -1990,7 +2003,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket encryption", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket encryption", request_context, false);
|
||||
|
||||
let mut item = sr_bucket_meta_item(bucket.clone(), "sse-config");
|
||||
item.sse_config = Some(
|
||||
@@ -2051,7 +2064,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket lifecycle", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket lifecycle", request_context, true);
|
||||
|
||||
let mut item = sr_bucket_meta_item(bucket.clone(), "lc-config");
|
||||
item.expiry_lc_config =
|
||||
@@ -2096,7 +2109,6 @@ impl DefaultBucketUsecase {
|
||||
});
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_bucket(&bucket);
|
||||
Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default()))
|
||||
}
|
||||
|
||||
@@ -2129,7 +2141,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket notification", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket notification", request_context, false);
|
||||
|
||||
let region = resolve_notification_region(self.global_region(), request_region);
|
||||
let notify = current_notify_interface_for_context(self.context.as_deref());
|
||||
@@ -2232,7 +2244,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket policy", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket policy", request_context, false);
|
||||
|
||||
let mut item = sr_bucket_meta_item(bucket.clone(), "policy");
|
||||
item.policy = Some(serde_json::from_str(&policy).map_err(|e| s3_error!(InvalidArgument, "parse policy failed {:?}", e))?);
|
||||
@@ -2266,7 +2278,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket cors", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket cors", request_context, false);
|
||||
|
||||
let mut item = sr_bucket_meta_item(bucket.clone(), "cors-config");
|
||||
item.cors =
|
||||
@@ -2307,7 +2319,7 @@ impl DefaultBucketUsecase {
|
||||
.map_err(ApiError::from)?;
|
||||
drop(targets_guard);
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket replication", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket replication", request_context, true);
|
||||
|
||||
let mut item = sr_bucket_meta_item(bucket.clone(), "replication-config");
|
||||
item.replication_config = Some(
|
||||
@@ -2317,7 +2329,6 @@ impl DefaultBucketUsecase {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket replication-config hook failed");
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_bucket(&bucket);
|
||||
Ok(S3Response::new(PutBucketReplicationOutput::default()))
|
||||
}
|
||||
|
||||
@@ -2347,7 +2358,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put public access block", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put public access block", request_context, false);
|
||||
|
||||
Ok(S3Response::new(PutPublicAccessBlockOutput::default()))
|
||||
}
|
||||
@@ -2375,7 +2386,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket tagging", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket tagging", request_context, false);
|
||||
|
||||
let mut item = sr_bucket_meta_item(bucket.clone(), "tags");
|
||||
item.tags = Some(serialize_config(&tagging).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?);
|
||||
@@ -2407,7 +2418,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket versioning", request_context);
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket versioning", request_context, false);
|
||||
|
||||
let mut item = sr_bucket_meta_item(bucket.clone(), "version-config");
|
||||
item.versioning = Some(
|
||||
@@ -2654,23 +2665,23 @@ mod tests {
|
||||
#[test]
|
||||
fn bucket_metadata_config_changes_notify_peer_metadata_reload() {
|
||||
let source = include_str!("bucket_usecase.rs");
|
||||
for (method, operation) in [
|
||||
("execute_delete_bucket_policy", "delete bucket policy"),
|
||||
("execute_put_bucket_policy", "put bucket policy"),
|
||||
("execute_delete_public_access_block", "delete public access block"),
|
||||
("execute_put_public_access_block", "put public access block"),
|
||||
("execute_delete_bucket_lifecycle", "delete bucket lifecycle"),
|
||||
("execute_put_bucket_lifecycle_configuration", "put bucket lifecycle"),
|
||||
("execute_put_bucket_versioning", "put bucket versioning"),
|
||||
("execute_delete_bucket_tagging", "delete bucket tagging"),
|
||||
("execute_put_bucket_tagging", "put bucket tagging"),
|
||||
("execute_delete_bucket_replication", "delete bucket replication"),
|
||||
("execute_put_bucket_replication", "put bucket replication"),
|
||||
("execute_delete_bucket_cors", "delete bucket cors"),
|
||||
("execute_put_bucket_cors", "put bucket cors"),
|
||||
("execute_delete_bucket_encryption", "delete bucket encryption"),
|
||||
("execute_put_bucket_encryption", "put bucket encryption"),
|
||||
("execute_put_bucket_notification_configuration", "put bucket notification"),
|
||||
for (method, operation, scanner_maintenance_change) in [
|
||||
("execute_delete_bucket_policy", "delete bucket policy", false),
|
||||
("execute_put_bucket_policy", "put bucket policy", false),
|
||||
("execute_delete_public_access_block", "delete public access block", false),
|
||||
("execute_put_public_access_block", "put public access block", false),
|
||||
("execute_delete_bucket_lifecycle", "delete bucket lifecycle", true),
|
||||
("execute_put_bucket_lifecycle_configuration", "put bucket lifecycle", true),
|
||||
("execute_put_bucket_versioning", "put bucket versioning", false),
|
||||
("execute_delete_bucket_tagging", "delete bucket tagging", false),
|
||||
("execute_put_bucket_tagging", "put bucket tagging", false),
|
||||
("execute_delete_bucket_replication", "delete bucket replication", true),
|
||||
("execute_put_bucket_replication", "put bucket replication", true),
|
||||
("execute_delete_bucket_cors", "delete bucket cors", false),
|
||||
("execute_put_bucket_cors", "put bucket cors", false),
|
||||
("execute_delete_bucket_encryption", "delete bucket encryption", false),
|
||||
("execute_put_bucket_encryption", "put bucket encryption", false),
|
||||
("execute_put_bucket_notification_configuration", "put bucket notification", false),
|
||||
] {
|
||||
let body = usecase_method_source(source, method);
|
||||
assert!(
|
||||
@@ -2681,9 +2692,31 @@ mod tests {
|
||||
body.contains(operation),
|
||||
"{method} should identify the bucket metadata operation in reload logs"
|
||||
);
|
||||
let expected_reload = format!(
|
||||
"notify_bucket_metadata_reload(bucket.clone(), \"{operation}\", request_context, {scanner_maintenance_change});"
|
||||
);
|
||||
assert!(
|
||||
body.contains(&expected_reload),
|
||||
"{method} should propagate scanner_maintenance_change={scanner_maintenance_change}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn scanner_maintenance_reload_marks_only_scanner_owned_config_changes() {
|
||||
const BUCKET: &str = "scanner-maintenance-reload-test";
|
||||
rustfs_scanner::clear_dirty_usage_bucket(BUCKET);
|
||||
let before = rustfs_scanner::scanner_maintenance_generation();
|
||||
|
||||
record_local_scanner_maintenance_reload(BUCKET, false);
|
||||
assert_eq!(rustfs_scanner::scanner_maintenance_generation(), before);
|
||||
|
||||
record_local_scanner_maintenance_reload(BUCKET, true);
|
||||
assert_eq!(rustfs_scanner::scanner_maintenance_generation(), before.saturating_add(1));
|
||||
rustfs_scanner::clear_dirty_usage_bucket(BUCKET);
|
||||
}
|
||||
|
||||
fn replication_rule_for_target(arn: &str) -> ReplicationRule {
|
||||
ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
|
||||
@@ -473,7 +473,7 @@ pub(crate) mod bucket {
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
) -> Result<OffsetDateTime, crate::storage::storage_api::StorageError> {
|
||||
crate::storage::storage_api::ecstore_bucket::metadata_sys::delete(bucket, config_file).await
|
||||
crate::storage::storage_api::delete_bucket_metadata_config(bucket, config_file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_bucket_policy(
|
||||
@@ -551,7 +551,7 @@ pub(crate) mod bucket {
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
) -> Result<OffsetDateTime, crate::storage::storage_api::StorageError> {
|
||||
crate::storage::storage_api::ecstore_bucket::metadata_sys::update(bucket, config_file, data).await
|
||||
crate::storage::storage_api::update_bucket_metadata_config(bucket, config_file, data).await
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -55,6 +55,14 @@ const EVENT_RPC_RESPONSE_EMITTED: &str = "rpc_response_emitted";
|
||||
const EVENT_RPC_BACKGROUND_TASK_SPAWNED: &str = "rpc_background_task_spawned";
|
||||
const EVENT_RPC_BACKGROUND_TASK_FAILED: &str = "rpc_background_task_failed";
|
||||
|
||||
fn scanner_activity_response(namespace_generation: u64) -> ScannerActivityResponse {
|
||||
ScannerActivityResponse {
|
||||
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
|
||||
namespace_generation,
|
||||
maintenance_generation: rustfs_scanner::scanner_maintenance_generation(),
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! log_load_rebalance_meta_rejected {
|
||||
($reason:expr, $start_rebalance:expr) => {
|
||||
warn!(
|
||||
@@ -980,6 +988,16 @@ impl Node for NodeService {
|
||||
}
|
||||
}
|
||||
|
||||
async fn scanner_activity(
|
||||
&self,
|
||||
_request: Request<ScannerActivityRequest>,
|
||||
) -> Result<Response<ScannerActivityResponse>, Status> {
|
||||
let store = self
|
||||
.resolve_object_store()
|
||||
.ok_or_else(|| Status::unavailable("storage layer is not initialized"))?;
|
||||
Ok(Response::new(scanner_activity_response(store.scanner_namespace_mutation_generation())))
|
||||
}
|
||||
|
||||
async fn background_heal_status(
|
||||
&self,
|
||||
_request: Request<BackgroundHealStatusRequest>,
|
||||
@@ -1216,7 +1234,7 @@ mod tests {
|
||||
use super::{
|
||||
CollectMetricsOpts, Error, MetricType, Node as _, NodeService, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS,
|
||||
background_rebalance_start_error_message, make_server, stop_rebalance_response,
|
||||
background_rebalance_start_error_message, make_server, scanner_activity_response, stop_rebalance_response,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use rustfs_protos::models::PingBodyBuilder;
|
||||
@@ -1232,9 +1250,10 @@ mod tests {
|
||||
LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest,
|
||||
MakeVolumesRequest, Mss, PingRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest,
|
||||
ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest,
|
||||
RenamePartRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest, StatVolumeRequest,
|
||||
StopRebalanceRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, WriteRequest, node_service_client::NodeServiceClient, node_service_server::NodeServiceServer,
|
||||
RenamePartRequest, ScannerActivityRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest,
|
||||
StatVolumeRequest, StopRebalanceRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest,
|
||||
WriteAllRequest, WriteMetadataRequest, WriteRequest, node_service_client::NodeServiceClient,
|
||||
node_service_server::NodeServiceServer,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use tokio::net::TcpListener;
|
||||
@@ -2561,8 +2580,12 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn test_load_bucket_metadata_empty_bucket() {
|
||||
let service = create_test_node_service();
|
||||
let maintenance_generation = rustfs_scanner::scanner_maintenance_generation();
|
||||
|
||||
let request = Request::new(LoadBucketMetadataRequest { bucket: "".to_string() });
|
||||
let request = Request::new(LoadBucketMetadataRequest {
|
||||
bucket: "".to_string(),
|
||||
scanner_maintenance_change: true,
|
||||
});
|
||||
|
||||
let response = service.load_bucket_metadata(request).await;
|
||||
assert!(response.is_ok());
|
||||
@@ -2571,6 +2594,11 @@ mod tests {
|
||||
assert!(!load_response.success);
|
||||
assert!(load_response.error_info.is_some());
|
||||
assert!(load_response.error_info.unwrap().contains("bucket name is missing"));
|
||||
assert_eq!(
|
||||
rustfs_scanner::scanner_maintenance_generation(),
|
||||
maintenance_generation,
|
||||
"rejected metadata reloads must not advance scanner maintenance activity"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2580,6 +2608,7 @@ mod tests {
|
||||
|
||||
let request = Request::new(LoadBucketMetadataRequest {
|
||||
bucket: "test-bucket".to_string(),
|
||||
scanner_maintenance_change: false,
|
||||
});
|
||||
|
||||
let response = service.load_bucket_metadata(request).await;
|
||||
@@ -2829,6 +2858,27 @@ mod tests {
|
||||
assert_eq!(signal_response.error_info.as_deref(), Some("unsupported service signal: 99"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scanner_activity_requires_storage_layer() {
|
||||
let service = create_test_node_service();
|
||||
|
||||
let err = service
|
||||
.scanner_activity(Request::new(ScannerActivityRequest {}))
|
||||
.await
|
||||
.expect_err("activity queries must fail closed before storage is initialized");
|
||||
|
||||
assert_eq!(err.code(), tonic::Code::Unavailable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scanner_activity_response_uses_process_epoch_and_generations() {
|
||||
let response = scanner_activity_response(17);
|
||||
|
||||
assert_eq!(response.instance_id, rustfs_scanner::scanner_activity_epoch());
|
||||
assert_eq!(response.namespace_generation, 17);
|
||||
assert_eq!(response.maintenance_generation, rustfs_scanner::scanner_maintenance_generation());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signal_service_rejects_non_dynamic_subsystem() {
|
||||
let service = create_test_node_service();
|
||||
|
||||
@@ -57,6 +57,7 @@ impl NodeService {
|
||||
request: Request<LoadBucketMetadataRequest>,
|
||||
) -> Result<Response<LoadBucketMetadataResponse>, Status> {
|
||||
let request = request.into_inner();
|
||||
let scanner_maintenance_change = request.scanner_maintenance_change;
|
||||
let bucket = request.bucket;
|
||||
if bucket.is_empty() {
|
||||
return Ok(Response::new(LoadBucketMetadataResponse {
|
||||
@@ -74,12 +75,15 @@ impl NodeService {
|
||||
|
||||
match load_bucket_metadata(store, &bucket).await {
|
||||
Ok(meta) => {
|
||||
if let Err(err) = set_bucket_metadata(bucket, meta).await {
|
||||
if let Err(err) = set_bucket_metadata(bucket.clone(), meta).await {
|
||||
return Ok(Response::new(LoadBucketMetadataResponse {
|
||||
success: false,
|
||||
error_info: Some(err.to_string()),
|
||||
}));
|
||||
};
|
||||
if scanner_maintenance_change {
|
||||
rustfs_scanner::record_scanner_maintenance_change(&bucket);
|
||||
}
|
||||
Ok(Response::new(LoadBucketMetadataResponse {
|
||||
success: true,
|
||||
error_info: None,
|
||||
|
||||
@@ -1169,7 +1169,9 @@ pub(crate) fn get_global_bucket_metadata_sys() -> Option<Arc<tokio::sync::RwLock
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_bucket_metadata_config(bucket: &str, config_file: &str) -> Result<time::OffsetDateTime> {
|
||||
ecstore_bucket::metadata_sys::delete(bucket, config_file).await
|
||||
let updated_at = ecstore_bucket::metadata_sys::delete(bucket, config_file).await?;
|
||||
record_scanner_maintenance_config_change(bucket, config_file);
|
||||
Ok(updated_at)
|
||||
}
|
||||
|
||||
pub(crate) async fn get_bucket_metadata(bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
@@ -1241,7 +1243,22 @@ pub(crate) async fn update_bucket_metadata_config(
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
) -> Result<time::OffsetDateTime> {
|
||||
ecstore_bucket::metadata_sys::update(bucket, config_file, data).await
|
||||
let updated_at = ecstore_bucket::metadata_sys::update(bucket, config_file, data).await?;
|
||||
record_scanner_maintenance_config_change(bucket, config_file);
|
||||
Ok(updated_at)
|
||||
}
|
||||
|
||||
fn record_scanner_maintenance_config_change(bucket: &str, config_file: &str) {
|
||||
if scanner_maintenance_config_file(config_file) {
|
||||
rustfs_scanner::record_scanner_maintenance_change(bucket);
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_maintenance_config_file(config_file: &str) -> bool {
|
||||
matches!(
|
||||
config_file,
|
||||
ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG | ecstore_bucket::metadata::BUCKET_REPLICATION_CONFIG
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn add_object_lock_years(dt: time::OffsetDateTime, years: i32) -> time::OffsetDateTime {
|
||||
@@ -1465,7 +1482,9 @@ pub(crate) async fn init_compression_total_memory_from_backend(store: Arc<ECStor
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{bucket_targets_metadata_lock_shard, lock_bucket_targets_metadata};
|
||||
use super::{
|
||||
bucket_targets_metadata_lock_shard, ecstore_bucket, lock_bucket_targets_metadata, scanner_maintenance_config_file,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1494,4 +1513,11 @@ mod tests {
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_maintenance_config_only_includes_scanner_owned_work() {
|
||||
assert!(scanner_maintenance_config_file(ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG));
|
||||
assert!(scanner_maintenance_config_file(ecstore_bucket::metadata::BUCKET_REPLICATION_CONFIG));
|
||||
assert!(!scanner_maintenance_config_file(ecstore_bucket::metadata::BUCKET_POLICY_CONFIG));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user