mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 20:06:37 +00:00
refactor(scanner): split scanner_io.rs into child modules (#6294)
Split the 5369-line scanner_io.rs (39% inline tests) into a canonical scanner_io.rs + scanner_io/ module tree with zero behavior change: - scanner_io.rs (~660): constants, metadata-error constructors, the bucket scan plan, cycle-status classification helpers, the ScannerIO / ScannerIOCache / ScannerIODisk traits, and ScannerCycleResult - scanner_io/dirty_usage.rs (~300): process-wide dirty-usage statics and the acknowledgment protocol - scanner_io/guards.rs (~270): concurrency gauges and RAII guards - scanner_io/cache.rs (~410): scanner cache locks and the snapshot persist/publish path - scanner_io/io_cycle.rs (~390), io_cache.rs (~1160), io_disk.rs (~230): the ECStore / SetDisks / Disk trait implementations - scanner_io/publish_gate_tests.rs (~750) and tests.rs (~1340): the two inline test modules as child modules All crate paths are unchanged: the lib.rs scanner_io re-exports and every crate::scanner_io:: consumer (scanner.rs, remote_scanner, scanner_folder, and cross-crate rustfs users) resolve through root re-exports with their original visibilities (pub stays pub, pub(crate) stays pub(crate)). Cross-module items gain pub(super), whose scope equals the old single-module privacy domain. Code is moved verbatim apart from those markers, per-module import headers, and rustfmt re-wraps. The logging-guardrail nsscanner_disk skip-set_disks rule now points at scanner_io/io_disk.rs where the function moved; the pattern and thresholds are unchanged. Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
+25
-4733
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,411 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// scanner cache locks and the cache snapshot persist/publish path.
|
||||
use super::*;
|
||||
|
||||
pub(crate) fn scanner_cache_lock_resource(cache_name: &str, source: DataUsageCacheSource) -> String {
|
||||
let lock_name = format!("{SCANNER_CACHE_LOCK_SUFFIX}.pool-{}.set-{}", source.pool_index, source.set_index);
|
||||
path_join_buf(&[crate::BUCKET_META_PREFIX, cache_name, &lock_name])
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_cache_lock_timeout() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5))
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ScannerCacheLockGuards {
|
||||
scoped: NamespaceLockGuard,
|
||||
}
|
||||
|
||||
impl ScannerCacheLockGuards {
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
self.scoped.is_lock_lost()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum ScannerCacheLockError {
|
||||
Create { resource: String, source: Error },
|
||||
Acquire { resource: String, source: LockError },
|
||||
}
|
||||
|
||||
impl ScannerCacheLockError {
|
||||
pub(crate) fn state(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Create { .. } => "lock_create_failed",
|
||||
Self::Acquire { .. } => "lock_acquire_failed",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_contention(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::Acquire {
|
||||
source: LockError::Timeout { .. } | LockError::AlreadyLocked { .. },
|
||||
..
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ScannerCacheLockError {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Create { resource, source } => write!(formatter, "create scanner cache lock {resource}: {source}"),
|
||||
Self::Acquire { resource, source } => write!(formatter, "acquire scanner cache lock {resource}: {source}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_scanner_cache_locks(
|
||||
store: &SetDisks,
|
||||
cache_name: &str,
|
||||
source: DataUsageCacheSource,
|
||||
) -> std::result::Result<ScannerCacheLockGuards, ScannerCacheLockError> {
|
||||
let timeout = scanner_cache_lock_timeout();
|
||||
let scoped_resource = scanner_cache_lock_resource(cache_name, source);
|
||||
let scoped_lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &scoped_resource)
|
||||
.await
|
||||
.map_err(|source| ScannerCacheLockError::Create {
|
||||
resource: scoped_resource.clone(),
|
||||
source,
|
||||
})?;
|
||||
let scoped = scoped_lock
|
||||
.get_write_lock_quiet(timeout)
|
||||
.await
|
||||
.map_err(|source| ScannerCacheLockError::Acquire {
|
||||
resource: scoped_resource,
|
||||
source,
|
||||
})?;
|
||||
|
||||
Ok(ScannerCacheLockGuards { scoped })
|
||||
}
|
||||
|
||||
pub(super) async fn await_scanner_disk_shutdown<F>(scan: Pin<&mut F>)
|
||||
where
|
||||
F: Future,
|
||||
{
|
||||
let _ = tokio::time::timeout(SCANNER_CACHE_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan).await;
|
||||
}
|
||||
|
||||
pub(crate) fn current_cache_root_entry(
|
||||
cache: &DataUsageCache,
|
||||
name: &str,
|
||||
source: DataUsageCacheSource,
|
||||
next_cycle: u64,
|
||||
leader_epoch: u64,
|
||||
scan_plan_digest: DataUsageScanPlanDigest,
|
||||
) -> std::result::Result<Option<DataUsageEntryInfo>, ScannerError> {
|
||||
let metadata_is_current = cache.info.name == name
|
||||
&& cache.info.source == Some(source)
|
||||
&& cache.info.snapshot_complete
|
||||
&& cache.info.scan_plan_digest == Some(scan_plan_digest)
|
||||
&& cache.info.last_update.is_some()
|
||||
&& cache.info.next_cycle == next_cycle
|
||||
&& cache.info.leader_epoch == leader_epoch
|
||||
&& cache.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT;
|
||||
if !metadata_is_current {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
cache_root_entry_info(cache).map(Some)
|
||||
}
|
||||
|
||||
pub(crate) enum DataUsageCacheScanState {
|
||||
Current(Box<DataUsageEntryInfo>),
|
||||
Prepared {
|
||||
outcome: DataUsageCachePrepareOutcome,
|
||||
invalid_current: Option<ScannerError>,
|
||||
},
|
||||
}
|
||||
|
||||
pub(crate) fn current_cache_root_or_prepare(
|
||||
cache: &mut DataUsageCache,
|
||||
name: &str,
|
||||
source: DataUsageCacheSource,
|
||||
next_cycle: u64,
|
||||
leader_epoch: u64,
|
||||
scan_plan_digest: DataUsageScanPlanDigest,
|
||||
require_source: bool,
|
||||
) -> DataUsageCacheScanState {
|
||||
match current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest) {
|
||||
Ok(Some(root)) => DataUsageCacheScanState::Current(Box::new(root)),
|
||||
current => DataUsageCacheScanState::Prepared {
|
||||
invalid_current: current.err(),
|
||||
outcome: cache.prepare_for_scan(name, next_cycle, leader_epoch, source, scan_plan_digest, require_source),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn cache_snapshot_is_current(
|
||||
cache: &DataUsageCache,
|
||||
name: &str,
|
||||
source: DataUsageCacheSource,
|
||||
next_cycle: u64,
|
||||
leader_epoch: u64,
|
||||
scan_plan_digest: DataUsageScanPlanDigest,
|
||||
) -> bool {
|
||||
matches!(
|
||||
current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest),
|
||||
Ok(Some(_))
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn completed_data_usage_info(
|
||||
results: &[DataUsageCache],
|
||||
expected_sources: &HashSet<DataUsageCacheSource>,
|
||||
all_buckets: &[String],
|
||||
bucket_plan_complete: bool,
|
||||
budget_elapsed: bool,
|
||||
cancelled: bool,
|
||||
) -> Option<(DataUsageInfo, SystemTime)> {
|
||||
if !bucket_plan_complete {
|
||||
return None;
|
||||
}
|
||||
let completed_set_count = results.iter().filter(|result| result.info.last_update.is_some()).count();
|
||||
if !should_publish_completed_snapshot(completed_set_count, results.len(), budget_elapsed, cancelled) {
|
||||
return None;
|
||||
}
|
||||
if !scanner_results_form_complete_snapshot(results, expected_sources) {
|
||||
return None;
|
||||
}
|
||||
|
||||
if results.iter().any(|result| result.root().is_none()) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut total = DataUsageEntry::default();
|
||||
let mut buckets_usage = HashMap::with_capacity(all_buckets.len());
|
||||
for bucket in all_buckets {
|
||||
let mut merged = DataUsageEntry::default();
|
||||
for result in results {
|
||||
let entry = result.checked_flatten(bucket)?;
|
||||
if !merged.checked_merge(&entry) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
if !total.checked_merge(&merged) {
|
||||
return None;
|
||||
}
|
||||
buckets_usage.insert(bucket.clone(), checked_bucket_usage_info(&merged)?);
|
||||
}
|
||||
|
||||
let merged_last_update = results.iter().filter_map(|result| result.info.last_update).max()?;
|
||||
let bucket_sizes = buckets_usage
|
||||
.iter()
|
||||
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||
.collect();
|
||||
let data_usage_info = DataUsageInfo {
|
||||
last_update: Some(merged_last_update),
|
||||
scanner_cycle: Some(results.first()?.info.next_cycle),
|
||||
objects_total_count: u64::try_from(total.objects).ok()?,
|
||||
versions_total_count: u64::try_from(total.versions).ok()?,
|
||||
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
|
||||
objects_total_size: u64::try_from(total.size).ok()?,
|
||||
tier_stats: total.all_tier_stats.filter(|tiers| !tiers.is_empty()),
|
||||
buckets_count: u64::try_from(all_buckets.len()).ok()?,
|
||||
bucket_sizes,
|
||||
buckets_usage,
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
Some((data_usage_info, merged_last_update))
|
||||
}
|
||||
|
||||
pub(super) async fn send_cache_root_entry_info(
|
||||
bucket_result_tx: &mpsc::Sender<DataUsageEntryInfo>,
|
||||
cache: &DataUsageCache,
|
||||
pending_maintenance_work: &AtomicBool,
|
||||
) -> std::result::Result<(), ScannerError> {
|
||||
let root = cache_root_entry_info(cache)?;
|
||||
send_cache_root_entry(bucket_result_tx, root, cache, pending_maintenance_work).await
|
||||
}
|
||||
|
||||
pub(super) async fn send_cache_root_entry(
|
||||
bucket_result_tx: &mpsc::Sender<DataUsageEntryInfo>,
|
||||
root: DataUsageEntryInfo,
|
||||
cache: &DataUsageCache,
|
||||
pending_maintenance_work: &AtomicBool,
|
||||
) -> std::result::Result<(), ScannerError> {
|
||||
record_bucket_pending_maintenance_work(cache, pending_maintenance_work);
|
||||
bucket_result_tx
|
||||
.send(root)
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("scanner cache root channel closed: {err}")))
|
||||
}
|
||||
|
||||
pub(super) async fn persist_and_publish_cache_snapshot(
|
||||
store: Arc<SetDisks>,
|
||||
updates: &mpsc::Sender<DataUsageCache>,
|
||||
mut cache_snapshot: DataUsageCache,
|
||||
cache_cycle_floor: &AtomicU64,
|
||||
) -> Option<SystemTime> {
|
||||
let source = cache_snapshot.info.source?;
|
||||
let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
cache_name = DATA_USAGE_CACHE_NAME,
|
||||
state = err.state(),
|
||||
error = %err,
|
||||
"Scanner cache snapshot lock acquisition failed"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
let mut persisted = DataUsageCache::default();
|
||||
let revisions = match persisted.load_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME).await {
|
||||
Ok(revisions) => revisions,
|
||||
Err(err) => {
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
cache_name = DATA_USAGE_CACHE_NAME,
|
||||
state = "load_or_revision_lookup_failed",
|
||||
error = %err,
|
||||
"Scanner cache snapshot load or revision lookup failed"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let scan_plan_digest = cache_snapshot.info.scan_plan_digest?;
|
||||
if persisted.info.next_cycle > cache_snapshot.info.next_cycle {
|
||||
cache_cycle_floor.fetch_max(persisted.info.next_cycle, Ordering::AcqRel);
|
||||
warn!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
cache_name = DATA_USAGE_CACHE_NAME,
|
||||
requested_cycle = cache_snapshot.info.next_cycle,
|
||||
persisted_cycle = persisted.info.next_cycle,
|
||||
state = "stale_cycle_rejected",
|
||||
"Scanner rejected a set cache cycle regression"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
if persisted.info.leader_epoch > cache_snapshot.info.leader_epoch {
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
cache_name = DATA_USAGE_CACHE_NAME,
|
||||
requested_epoch = cache_snapshot.info.leader_epoch,
|
||||
persisted_epoch = persisted.info.leader_epoch,
|
||||
state = "stale_leader_rejected",
|
||||
"Scanner rejected a set cache snapshot from an older leader epoch"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
if matches!(
|
||||
current_cache_root_entry(
|
||||
&persisted,
|
||||
DATA_USAGE_ROOT,
|
||||
source,
|
||||
cache_snapshot.info.next_cycle,
|
||||
cache_snapshot.info.leader_epoch,
|
||||
scan_plan_digest,
|
||||
),
|
||||
Ok(Some(_))
|
||||
) {
|
||||
cache_snapshot = persisted;
|
||||
} else {
|
||||
if guard.is_lock_lost() {
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
cache_name = DATA_USAGE_CACHE_NAME,
|
||||
state = "lock_lost",
|
||||
"Scanner cache snapshot save skipped after lock loss"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = cache_snapshot
|
||||
.save_with_revisions(store, DATA_USAGE_CACHE_NAME, &revisions)
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
cache_name = DATA_USAGE_CACHE_NAME,
|
||||
state = "save_failed",
|
||||
error = %e,
|
||||
"Scanner cache snapshot persistence failed"
|
||||
);
|
||||
done_save();
|
||||
return None;
|
||||
}
|
||||
done_save();
|
||||
}
|
||||
if guard.is_lock_lost() {
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
cache_name = DATA_USAGE_CACHE_NAME,
|
||||
state = "lock_lost_after_save",
|
||||
"Scanner cache snapshot publish skipped after lock loss"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
drop(guard);
|
||||
let last_update = cache_snapshot.info.last_update;
|
||||
|
||||
if let Err(e) = updates.send(cache_snapshot).await {
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
cache_name = DATA_USAGE_CACHE_NAME,
|
||||
state = "publish_failed",
|
||||
error = %e,
|
||||
"Scanner cache snapshot publish failed"
|
||||
);
|
||||
}
|
||||
|
||||
last_update
|
||||
}
|
||||
|
||||
pub(super) async fn send_data_usage_update(updates: &mpsc::Sender<DataUsageInfo>, data_usage_info: DataUsageInfo) -> Result<()> {
|
||||
updates.send(data_usage_info).await.map_err(|e| {
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_DATA_USAGE_STREAM,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
state = "send_failed",
|
||||
error = %e,
|
||||
"Scanner data usage publish failed"
|
||||
);
|
||||
StorageError::other("scanner data usage receiver closed before update delivery")
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// process-wide dirty-usage invalidation state, its acknowledgment protocol, and snapshot helpers.
|
||||
use super::*;
|
||||
|
||||
pub(super) static DIRTY_USAGE_BUCKET_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
pub(super) static DIRTY_USAGE_BUCKETS: LazyLock<StdMutex<DirtyUsageBuckets>> = LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
pub(super) static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
pub(super) static SCANNER_ACTIVITY_EPOCH: LazyLock<String> = LazyLock::new(|| format!("{:032x}", rand::random::<u128>()));
|
||||
pub(super) static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
pub(super) static SCANNER_MAINTENANCE_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerDirtyUsageState {
|
||||
pub generation: u64,
|
||||
pub pending: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum ScannerDirtyUsageAckError {
|
||||
#[error("scanner process instance changed before dirty usage acknowledgement")]
|
||||
ProcessChanged,
|
||||
#[error("scanner dirty usage generation cannot be acknowledged")]
|
||||
InvalidGeneration,
|
||||
}
|
||||
|
||||
pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
|
||||
DIRTY_USAGE_BUCKETS.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
pub(super) fn usize_to_u64_saturated(value: usize) -> u64 {
|
||||
u64::try_from(value).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
pub(super) fn advance_generation(generation: &AtomicU64) -> u64 {
|
||||
generation
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1)))
|
||||
.map_or_else(|current| current, |previous| previous.saturating_add(1))
|
||||
}
|
||||
|
||||
pub fn record_dirty_usage_bucket(bucket: &str) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let generation = advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
|
||||
dirty_buckets.insert(bucket.to_string(), generation);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets));
|
||||
// A write invalidates this bucket's prefix-usage answers on the spot so
|
||||
// admin/console consumers never ride the full TTL after a change
|
||||
// (rustfs/backlog#1872).
|
||||
crate::prefix_usage::invalidate_prefix_usage_cache(bucket);
|
||||
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
||||
}
|
||||
|
||||
pub fn record_scanner_maintenance_change(bucket: &str) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
advance_generation(&SCANNER_MAINTENANCE_GENERATION);
|
||||
SCANNER_MAINTENANCE_NOTIFY.notify_one();
|
||||
record_dirty_usage_bucket(bucket);
|
||||
}
|
||||
|
||||
pub fn scanner_maintenance_generation() -> u64 {
|
||||
SCANNER_MAINTENANCE_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) async fn scanner_maintenance_changed() {
|
||||
SCANNER_MAINTENANCE_NOTIFY.notified().await;
|
||||
}
|
||||
|
||||
pub fn scanner_activity_epoch() -> &'static str {
|
||||
SCANNER_ACTIVITY_EPOCH.as_str()
|
||||
}
|
||||
|
||||
pub fn scanner_dirty_usage_state() -> ScannerDirtyUsageState {
|
||||
let dirty_buckets = dirty_usage_buckets();
|
||||
ScannerDirtyUsageState {
|
||||
generation: DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire),
|
||||
pending: !dirty_buckets.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn acknowledge_dirty_usage_generation(
|
||||
instance_id: &str,
|
||||
generation: u64,
|
||||
) -> std::result::Result<(), ScannerDirtyUsageAckError> {
|
||||
if instance_id != scanner_activity_epoch() {
|
||||
return Err(ScannerDirtyUsageAckError::ProcessChanged);
|
||||
}
|
||||
|
||||
let (cleared_buckets, pending_buckets) = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let current_generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire);
|
||||
if generation == 0 || generation == u64::MAX || current_generation == u64::MAX || generation > current_generation {
|
||||
return Err(ScannerDirtyUsageAckError::InvalidGeneration);
|
||||
}
|
||||
|
||||
let before = dirty_buckets.len();
|
||||
dirty_buckets.retain(|_, dirty_generation| *dirty_generation > generation);
|
||||
let cleared_buckets = before.saturating_sub(dirty_buckets.len());
|
||||
if cleared_buckets > 0 {
|
||||
advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
|
||||
}
|
||||
(cleared_buckets, dirty_buckets.len())
|
||||
};
|
||||
global_metrics()
|
||||
.record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared_buckets), usize_to_u64_saturated(pending_buckets));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn dirty_usage_generation() -> u64 {
|
||||
DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub fn clear_dirty_usage_bucket(bucket: &str) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
dirty_buckets.remove(bucket);
|
||||
advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_clear(usize_to_u64_saturated(pending_buckets));
|
||||
}
|
||||
|
||||
pub(super) fn snapshot_dirty_usage_buckets(buckets: &[BucketInfo], absent_generation_cutoff: u64) -> DirtyUsageSnapshot {
|
||||
let (snapshot, generation, covers_all_pending) = {
|
||||
let dirty_buckets = dirty_usage_buckets();
|
||||
let listed_buckets = dirty_buckets
|
||||
.values()
|
||||
.any(|generation| *generation > absent_generation_cutoff)
|
||||
.then(|| buckets.iter().map(|bucket| bucket.name.as_str()).collect::<HashSet<_>>());
|
||||
let snapshot = dirty_buckets
|
||||
.iter()
|
||||
.filter(|(bucket, generation)| {
|
||||
**generation <= absent_generation_cutoff
|
||||
|| listed_buckets
|
||||
.as_ref()
|
||||
.is_some_and(|listed_buckets| listed_buckets.contains(bucket.as_str()))
|
||||
})
|
||||
.map(|(bucket, generation)| (bucket.clone(), *generation))
|
||||
.collect::<DirtyUsageBuckets>();
|
||||
let generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire);
|
||||
let covers_all_pending = generation == absent_generation_cutoff && snapshot.len() == dirty_buckets.len();
|
||||
(snapshot, generation, covers_all_pending)
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_cycle_snapshot(usize_to_u64_saturated(snapshot.len()));
|
||||
DirtyUsageSnapshot {
|
||||
buckets: Arc::new(snapshot),
|
||||
generation,
|
||||
covers_all_pending,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn dirty_usage_buckets_pending() -> bool {
|
||||
!dirty_usage_buckets().is_empty()
|
||||
}
|
||||
|
||||
pub(crate) async fn dirty_usage_bucket_notified() {
|
||||
DIRTY_USAGE_BUCKET_NOTIFY.notified().await;
|
||||
}
|
||||
|
||||
pub(super) fn clear_dirty_usage_buckets(snapshot: &DirtyUsageBuckets) {
|
||||
let (cleared_buckets, pending_buckets) = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let mut cleared_buckets = 0usize;
|
||||
for (bucket, generation) in snapshot {
|
||||
if dirty_buckets.get(bucket).is_some_and(|current| current == generation) {
|
||||
dirty_buckets.remove(bucket);
|
||||
cleared_buckets += 1;
|
||||
}
|
||||
}
|
||||
if cleared_buckets > 0 {
|
||||
advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
|
||||
}
|
||||
(cleared_buckets, dirty_buckets.len())
|
||||
};
|
||||
global_metrics()
|
||||
.record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared_buckets), usize_to_u64_saturated(pending_buckets));
|
||||
}
|
||||
|
||||
pub(super) fn dirty_usage_buckets_excluding_failed(
|
||||
snapshot: &DirtyUsageBuckets,
|
||||
failed_buckets: &HashSet<String>,
|
||||
) -> DirtyUsageBuckets {
|
||||
snapshot
|
||||
.iter()
|
||||
.filter(|(bucket, _)| !failed_buckets.contains(*bucket))
|
||||
.map(|(bucket, generation)| (bucket.clone(), *generation))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn should_clear_dirty_usage_snapshot(
|
||||
result_ok: bool,
|
||||
completed_all_sets: bool,
|
||||
budget_elapsed: bool,
|
||||
activity_and_generation_current: bool,
|
||||
dirty_buckets: &DirtyUsageBuckets,
|
||||
failed_buckets: &HashSet<String>,
|
||||
) -> Option<DirtyUsageBuckets> {
|
||||
if result_ok && completed_all_sets && !budget_elapsed && activity_and_generation_current {
|
||||
return Some(dirty_usage_buckets_excluding_failed(dirty_buckets, failed_buckets));
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub(super) async fn record_failed_dirty_bucket(failed_buckets: &Arc<Mutex<HashSet<String>>>, bucket: &str) {
|
||||
failed_buckets.lock().await.insert(bucket.to_string());
|
||||
}
|
||||
|
||||
pub(super) async fn record_partial_dirty_bucket(partial_buckets: &Arc<Mutex<HashSet<String>>>, bucket: &str) {
|
||||
partial_buckets.lock().await.insert(bucket.to_string());
|
||||
}
|
||||
|
||||
pub(super) async fn requeue_bucket_work(
|
||||
bucket_tx: &mpsc::Sender<BucketInfo>,
|
||||
bucket: &BucketInfo,
|
||||
work_guard: &mut BucketWorkGuard,
|
||||
) -> bool {
|
||||
if bucket_tx.send(bucket.clone()).await.is_err() {
|
||||
return false;
|
||||
}
|
||||
|
||||
work_guard.mark_requeued();
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) async fn mark_unprocessed_bucket_work_failed(
|
||||
bucket_rx: &Mutex<mpsc::Receiver<BucketInfo>>,
|
||||
remaining: &Arc<AtomicUsize>,
|
||||
complete: &CancellationToken,
|
||||
failed_buckets: &Arc<Mutex<HashSet<String>>>,
|
||||
) -> usize {
|
||||
let mut failed_count = 0;
|
||||
let mut receiver = bucket_rx.lock().await;
|
||||
while let Some(bucket) = receiver.recv().await {
|
||||
record_failed_dirty_bucket(failed_buckets, &bucket.name).await;
|
||||
drop(BucketWorkGuard::new(remaining.clone(), complete.clone()));
|
||||
failed_count += 1;
|
||||
}
|
||||
failed_count
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) enum DirtyUsageSnapshotStatus {
|
||||
Current,
|
||||
Changed,
|
||||
Unverified,
|
||||
}
|
||||
|
||||
pub(super) fn dirty_usage_snapshot_status(snapshot: &DirtyUsageSnapshot) -> DirtyUsageSnapshotStatus {
|
||||
let generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire);
|
||||
if generation == u64::MAX {
|
||||
DirtyUsageSnapshotStatus::Unverified
|
||||
} else if snapshot.covers_all_pending && generation == snapshot.generation {
|
||||
DirtyUsageSnapshotStatus::Current
|
||||
} else {
|
||||
DirtyUsageSnapshotStatus::Changed
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn dirty_usage_bucket_count() -> usize {
|
||||
dirty_usage_buckets().len()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn clear_dirty_usage_buckets_for_tests() {
|
||||
dirty_usage_buckets().clear();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn dirty_usage_buckets_for_tests() -> DirtyUsageBuckets {
|
||||
dirty_usage_buckets().clone()
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// scan concurrency accounting: gauge recorders, RAII guards, and worker limits.
|
||||
use super::*;
|
||||
|
||||
pub(super) fn bucket_usage_scan_order(
|
||||
buckets: &[BucketInfo],
|
||||
old_cache: &DataUsageCache,
|
||||
dirty_buckets: &DirtyUsageBuckets,
|
||||
) -> Vec<BucketInfo> {
|
||||
let mut ordered = Vec::with_capacity(buckets.len());
|
||||
|
||||
for bucket in buckets {
|
||||
if dirty_buckets.contains_key(&bucket.name) {
|
||||
ordered.push(bucket.clone());
|
||||
}
|
||||
}
|
||||
|
||||
for bucket in buckets {
|
||||
if !dirty_buckets.contains_key(&bucket.name) && old_cache.find(&bucket.name).is_none() {
|
||||
ordered.push(bucket.clone());
|
||||
}
|
||||
}
|
||||
|
||||
for bucket in buckets {
|
||||
if !dirty_buckets.contains_key(&bucket.name) && old_cache.find(&bucket.name).is_some() {
|
||||
ordered.push(bucket.clone());
|
||||
}
|
||||
}
|
||||
|
||||
ordered
|
||||
}
|
||||
|
||||
pub(super) fn record_set_scan_concurrency_limit(limit: usize) {
|
||||
metrics::gauge!(METRIC_SCANNER_SET_SCAN_CONCURRENCY_LIMIT).set(limit as f64);
|
||||
global_metrics().record_scanner_set_scan_state(Some(limit), None, None);
|
||||
}
|
||||
|
||||
pub(super) fn record_set_scans_queued(count: usize) {
|
||||
metrics::gauge!(METRIC_SCANNER_SET_SCANS_QUEUED).set(count as f64);
|
||||
global_metrics().record_scanner_set_scan_state(None, Some(count), None);
|
||||
}
|
||||
|
||||
pub(super) fn record_set_scans_active(count: usize) {
|
||||
metrics::gauge!(METRIC_SCANNER_SET_SCANS_ACTIVE).set(count as f64);
|
||||
global_metrics().record_scanner_set_scan_state(None, None, Some(count));
|
||||
}
|
||||
|
||||
pub(super) fn record_disk_scan_concurrency_limit(pool: &str, set: &str, limit: usize) {
|
||||
metrics::gauge!(
|
||||
METRIC_SCANNER_DISK_SCAN_CONCURRENCY_LIMIT,
|
||||
"pool" => pool.to_owned(),
|
||||
"set" => set.to_owned()
|
||||
)
|
||||
.set(limit as f64);
|
||||
global_metrics().record_scanner_disk_bucket_scan_state(pool, set, Some(limit), None, None);
|
||||
}
|
||||
|
||||
pub(super) fn record_disk_bucket_scans_active(count: usize, pool: &str, set: &str) {
|
||||
metrics::gauge!(
|
||||
METRIC_SCANNER_DISK_BUCKET_SCANS_ACTIVE,
|
||||
"pool" => pool.to_owned(),
|
||||
"set" => set.to_owned()
|
||||
)
|
||||
.set(count as f64);
|
||||
global_metrics().record_scanner_disk_bucket_scan_state(pool, set, None, None, Some(count));
|
||||
}
|
||||
|
||||
pub(super) struct SetScanActiveGuard {
|
||||
active: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl SetScanActiveGuard {
|
||||
pub(super) fn new(active: Arc<AtomicUsize>) -> Self {
|
||||
let active_count = active.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
record_set_scans_active(active_count);
|
||||
Self { active }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SetScanActiveGuard {
|
||||
fn drop(&mut self) {
|
||||
let active_count = decrement_atomic_usize(&self.active);
|
||||
record_set_scans_active(active_count);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct DiskBucketScanActiveGuard {
|
||||
active: Arc<AtomicUsize>,
|
||||
pool: String,
|
||||
set: String,
|
||||
}
|
||||
|
||||
pub(super) struct BucketWorkGuard {
|
||||
remaining: Arc<AtomicUsize>,
|
||||
complete: CancellationToken,
|
||||
requeued: bool,
|
||||
}
|
||||
|
||||
impl BucketWorkGuard {
|
||||
pub(super) fn new(remaining: Arc<AtomicUsize>, complete: CancellationToken) -> Self {
|
||||
Self {
|
||||
remaining,
|
||||
complete,
|
||||
requeued: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn mark_requeued(&mut self) {
|
||||
self.requeued = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BucketWorkGuard {
|
||||
fn drop(&mut self) {
|
||||
if !self.requeued && self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 {
|
||||
self.complete.cancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DiskBucketScanActiveGuard {
|
||||
pub(super) fn new(active: Arc<AtomicUsize>, pool: String, set: String) -> Self {
|
||||
let active_count = active.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
record_disk_bucket_scans_active(active_count, &pool, &set);
|
||||
Self { active, pool, set }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DiskBucketScanActiveGuard {
|
||||
fn drop(&mut self) {
|
||||
let active_count = decrement_atomic_usize(&self.active);
|
||||
record_disk_bucket_scans_active(active_count, &self.pool, &self.set);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct BucketDriveFailureGuard {
|
||||
failed: bool,
|
||||
}
|
||||
|
||||
impl BucketDriveFailureGuard {
|
||||
pub(super) fn new() -> Self {
|
||||
Self { failed: true }
|
||||
}
|
||||
|
||||
pub(super) fn mark_not_failed(&mut self) {
|
||||
self.failed = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BucketDriveFailureGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.failed {
|
||||
global_metrics().record_scan_bucket_drive_failure();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct DiskBucketScanGaugeReset {
|
||||
pool: String,
|
||||
set: String,
|
||||
}
|
||||
|
||||
impl DiskBucketScanGaugeReset {
|
||||
pub(super) fn new(pool: String, set: String) -> Self {
|
||||
Self { pool, set }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DiskBucketScanGaugeReset {
|
||||
fn drop(&mut self) {
|
||||
reset_disk_bucket_scan_gauges(&self.pool, &self.set);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn decrement_atomic_usize(counter: &AtomicUsize) -> usize {
|
||||
counter
|
||||
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1)))
|
||||
.map(|previous| previous.saturating_sub(1))
|
||||
.unwrap_or_else(|current| current)
|
||||
}
|
||||
|
||||
pub(super) fn increment_atomic_usize(counter: &AtomicUsize) -> usize {
|
||||
counter
|
||||
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(1)))
|
||||
.map(|previous| previous.saturating_add(1))
|
||||
.unwrap_or_else(|current| current)
|
||||
}
|
||||
|
||||
pub(super) fn record_disk_bucket_scans_queued(count: usize, pool: &str, set: &str) {
|
||||
metrics::gauge!(
|
||||
METRIC_SCANNER_DISK_BUCKET_SCANS_QUEUED,
|
||||
"pool" => pool.to_owned(),
|
||||
"set" => set.to_owned()
|
||||
)
|
||||
.set(count as f64);
|
||||
global_metrics().record_scanner_disk_bucket_scan_state(pool, set, None, Some(count), None);
|
||||
}
|
||||
|
||||
pub(super) fn decrement_disk_bucket_scans_queued(counter: &AtomicUsize, pool: &str, set: &str) {
|
||||
let queued_count = decrement_atomic_usize(counter);
|
||||
record_disk_bucket_scans_queued(queued_count, pool, set);
|
||||
}
|
||||
|
||||
pub(super) fn increment_disk_bucket_scans_queued(counter: &AtomicUsize, pool: &str, set: &str) {
|
||||
let queued_count = increment_atomic_usize(counter);
|
||||
record_disk_bucket_scans_queued(queued_count, pool, set);
|
||||
}
|
||||
|
||||
pub(super) fn reset_set_scan_gauges() {
|
||||
record_set_scan_concurrency_limit(0);
|
||||
record_set_scans_queued(0);
|
||||
record_set_scans_active(0);
|
||||
global_metrics().reset_scanner_set_scan_state();
|
||||
}
|
||||
|
||||
pub(super) fn reset_disk_bucket_scan_gauges(pool: &str, set: &str) {
|
||||
record_disk_scan_concurrency_limit(pool, set, 0);
|
||||
record_disk_bucket_scans_queued(0, pool, set);
|
||||
record_disk_bucket_scans_active(0, pool, set);
|
||||
}
|
||||
|
||||
pub(super) fn scanner_concurrency_limit(configured: usize, available: usize) -> usize {
|
||||
if available == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if crate::current_foreground_read_activity() > 0 {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if configured == 0 {
|
||||
available
|
||||
} else {
|
||||
configured.min(available).max(1)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn scanner_max_concurrent_set_scans(available: usize) -> usize {
|
||||
scanner_concurrency_limit(crate::runtime_config::scanner_max_concurrent_set_scans_configured(), available)
|
||||
}
|
||||
|
||||
pub(super) fn scanner_max_concurrent_disk_scans(available: usize) -> usize {
|
||||
scanner_concurrency_limit(crate::runtime_config::scanner_max_concurrent_disk_scans_configured(), available)
|
||||
}
|
||||
|
||||
pub(super) fn scanner_budgeted_concurrency_limit(configured_limit: usize, requires_serial_progress_accounting: bool) -> usize {
|
||||
if requires_serial_progress_accounting {
|
||||
1
|
||||
} else {
|
||||
configured_limit
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_set_scan_failure(first_err: &mut Option<Error>, err: Error) {
|
||||
if first_err.is_none() {
|
||||
*first_err = Some(err);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError) -> Error {
|
||||
Error::other(format!("{stage} task join failed: {err}"))
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,386 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// ScannerIO/ScannerIOCycle implementations for ECStore: bucket planning and per-set fan-out.
|
||||
use super::*;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ScannerIO for ECStore {
|
||||
async fn nsscanner(
|
||||
&self,
|
||||
ctx: CancellationToken,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
updates: mpsc::Sender<DataUsageInfo>,
|
||||
want_cycle: u64,
|
||||
scan_mode: HealScanMode,
|
||||
) -> Result<()> {
|
||||
// This public path can prove delivery to the receiver, but not that
|
||||
// the receiver persisted the update. Keep dirty usage pending unless
|
||||
// the main scanner confirms durability through nsscanner_with_status.
|
||||
let leader_epoch = crate::scanner::current_scanner_leader_epoch()
|
||||
.await
|
||||
.map_err(|err| StorageError::other(format!("failed to resolve scanner leader epoch: {err}")))?;
|
||||
ScannerIOCycle::nsscanner_with_status(self, ctx, budget, updates, want_cycle, leader_epoch, scan_mode).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ScannerIOCycle for ECStore {
|
||||
#[tracing::instrument(skip(self, budget, updates))]
|
||||
async fn nsscanner_with_status(
|
||||
&self,
|
||||
ctx: CancellationToken,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
updates: mpsc::Sender<DataUsageInfo>,
|
||||
want_cycle: u64,
|
||||
leader_epoch: u64,
|
||||
scan_mode: HealScanMode,
|
||||
) -> Result<ScannerCycleResult> {
|
||||
let child_token = ctx.child_token();
|
||||
|
||||
let distributed = self.setup_is_dist_erasure().await;
|
||||
let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) {
|
||||
ScannerActivityPreflight::Ready(snapshot) => snapshot,
|
||||
ScannerActivityPreflight::ActivityBaselineUnavailable(err) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_SET_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
state = "cycle_activity_baseline_failed",
|
||||
error = %err,
|
||||
"Scanner cycle skipped because cluster activity could not be baselined"
|
||||
);
|
||||
return Ok(ScannerCycleResult::new(
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
||||
None,
|
||||
));
|
||||
}
|
||||
ScannerActivityPreflight::DataMovement => {
|
||||
debug!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_SET_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
state = "cycle_data_movement_active",
|
||||
"Scanner cycle deferred while rebalance or decommission data movement is active"
|
||||
);
|
||||
return Ok(ScannerCycleResult::new(
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
|
||||
None,
|
||||
));
|
||||
}
|
||||
};
|
||||
let dirty_generation_before_bucket_list = dirty_usage_generation();
|
||||
let bucket_listing = self.list_bucket_for_scanner(&BucketOptions::default()).await?;
|
||||
let mut bucket_plan_complete = bucket_listing.topology_complete;
|
||||
let all_buckets = Arc::new(bucket_listing.buckets);
|
||||
let expected_sources = Arc::new(
|
||||
self.pools
|
||||
.iter()
|
||||
.flat_map(|pool| {
|
||||
pool.disk_set
|
||||
.iter()
|
||||
.map(|set| DataUsageCacheSource::new(set.pool_index, set.set_index))
|
||||
})
|
||||
.collect::<HashSet<_>>(),
|
||||
);
|
||||
let mut buckets_by_source = HashMap::with_capacity(bucket_listing.set_buckets.len());
|
||||
for scope in bucket_listing.set_buckets {
|
||||
let source = DataUsageCacheSource::new(scope.pool_index, scope.set_index);
|
||||
if buckets_by_source.insert(source, scope.buckets).is_some() {
|
||||
bucket_plan_complete = false;
|
||||
}
|
||||
}
|
||||
bucket_plan_complete &= buckets_by_source.keys().copied().collect::<HashSet<_>>() == *expected_sources;
|
||||
let scan_plan_digest =
|
||||
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_snapshot_digest(&activity_before));
|
||||
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
|
||||
let cache_cycle_floor = Arc::new(AtomicU64::new(want_cycle));
|
||||
|
||||
if all_buckets.is_empty() {
|
||||
reset_set_scan_gauges();
|
||||
if !bucket_plan_complete {
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
|
||||
}
|
||||
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
|
||||
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
|
||||
let status = classify_nsscanner_cycle(
|
||||
true,
|
||||
false,
|
||||
ctx.is_cancelled(),
|
||||
ScannerBucketScanStatus::Complete,
|
||||
dirty_usage_status,
|
||||
activity_status,
|
||||
);
|
||||
if !publish_usage_snapshot(
|
||||
&updates,
|
||||
status,
|
||||
DataUsageInfo {
|
||||
last_update: Some(SystemTime::now()),
|
||||
scanner_cycle: Some(want_cycle),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(ScannerCycleResult::new(status, None));
|
||||
}
|
||||
let dirty_usage_clear =
|
||||
(status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone());
|
||||
let remote_dirty_usage_acknowledgements = if status == ScannerCycleStatus::Complete {
|
||||
crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
|
||||
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
|
||||
}
|
||||
|
||||
let total_results = expected_sources.len();
|
||||
if total_results == 0 {
|
||||
warn!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_SET_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
bucket_count = all_buckets.len(),
|
||||
state = "no_disk_sets",
|
||||
"Scanner set state update detected missing disk sets"
|
||||
);
|
||||
reset_set_scan_gauges();
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
|
||||
}
|
||||
|
||||
let set_scan_limit = scanner_budgeted_concurrency_limit(
|
||||
scanner_max_concurrent_set_scans(total_results),
|
||||
budget.requires_serial_progress_accounting(),
|
||||
);
|
||||
let bucket_failures = ScannerBucketFailureState::default();
|
||||
let pending_maintenance_work = Arc::new(AtomicBool::new(false));
|
||||
record_set_scan_concurrency_limit(set_scan_limit);
|
||||
debug!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_SET_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
total_sets = total_results,
|
||||
concurrency_limit = set_scan_limit,
|
||||
state = "concurrency_budget",
|
||||
"Scanner set concurrency budget resolved"
|
||||
);
|
||||
let set_scan_semaphore = Arc::new(Semaphore::new(set_scan_limit));
|
||||
let queued_set_scans = Arc::new(AtomicUsize::new(total_results));
|
||||
let active_set_scans = Arc::new(AtomicUsize::new(0));
|
||||
record_set_scans_queued(total_results);
|
||||
record_set_scans_active(0);
|
||||
|
||||
let results = vec![DataUsageCache::default(); total_results];
|
||||
let results_mutex: Arc<Mutex<Vec<DataUsageCache>>> = Arc::new(Mutex::new(results));
|
||||
let first_err_mutex: Arc<Mutex<Option<Error>>> = Arc::new(Mutex::new(None));
|
||||
let mut results_index = 0usize;
|
||||
let mut wait_futs = Vec::new();
|
||||
|
||||
for pool in self.pools.iter() {
|
||||
for set in pool.disk_set.iter() {
|
||||
let results_index_clone = results_index;
|
||||
results_index += 1;
|
||||
// Clone the Arc to move it into the spawned task
|
||||
let set_clone: Arc<SetDisks> = Arc::clone(set);
|
||||
let source = DataUsageCacheSource::new(set.pool_index, set.set_index);
|
||||
let set_buckets = buckets_by_source.remove(&source).unwrap_or_default();
|
||||
let pool_label = set.pool_index.to_string();
|
||||
let set_label = set.set_index.to_string();
|
||||
|
||||
let child_token_clone = child_token.clone();
|
||||
let budget_clone = budget.clone();
|
||||
let want_cycle_clone = want_cycle;
|
||||
let scan_mode_clone = scan_mode;
|
||||
let results_mutex_clone = results_mutex.clone();
|
||||
let first_err_mutex_clone = first_err_mutex.clone();
|
||||
let set_scan_semaphore_clone = set_scan_semaphore.clone();
|
||||
let queued_set_scans_clone = queued_set_scans.clone();
|
||||
let active_set_scans_clone = active_set_scans.clone();
|
||||
|
||||
let (tx, mut rx) = mpsc::channel::<DataUsageCache>(1);
|
||||
|
||||
// Spawn task to receive and store results
|
||||
let receiver_fut = tokio::spawn(async move {
|
||||
while let Some(result) = rx.recv().await {
|
||||
let mut results = results_mutex_clone.lock().await;
|
||||
results[results_index_clone] = result;
|
||||
}
|
||||
});
|
||||
wait_futs.push(receiver_fut);
|
||||
|
||||
let scan_plan = ScannerBucketScanPlan {
|
||||
buckets: set_buckets,
|
||||
all_buckets: Arc::clone(&all_buckets),
|
||||
digest: scan_plan_digest,
|
||||
leader_epoch,
|
||||
dirty_usage_buckets: dirty_usage_snapshot.buckets.clone(),
|
||||
bucket_failures: bucket_failures.clone(),
|
||||
pending_maintenance_work: pending_maintenance_work.clone(),
|
||||
cache_cycle_floor: cache_cycle_floor.clone(),
|
||||
};
|
||||
// Spawn task to run the scanner
|
||||
let scanner_fut = tokio::spawn(async move {
|
||||
let permit_wait = child_token_clone.clone();
|
||||
let permit_wait_start = Instant::now();
|
||||
let _permit = tokio::select! {
|
||||
permit = set_scan_semaphore_clone.acquire_owned() => match permit {
|
||||
Ok(permit) => permit,
|
||||
Err(_) => return,
|
||||
},
|
||||
_ = permit_wait.cancelled() => return,
|
||||
};
|
||||
metrics::histogram!(
|
||||
METRIC_SCANNER_SET_SCAN_WAIT_SECONDS,
|
||||
"pool" => pool_label.clone(),
|
||||
"set" => set_label.clone()
|
||||
)
|
||||
.record(permit_wait_start.elapsed().as_secs_f64());
|
||||
let queued_count = decrement_atomic_usize(&queued_set_scans_clone);
|
||||
record_set_scans_queued(queued_count);
|
||||
let _active_guard = SetScanActiveGuard::new(active_set_scans_clone);
|
||||
|
||||
if let Err(e) = set_clone
|
||||
.nsscanner_cache(
|
||||
child_token_clone.clone(),
|
||||
budget_clone,
|
||||
scan_plan,
|
||||
tx,
|
||||
want_cycle_clone,
|
||||
scan_mode_clone,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if child_token_clone.is_cancelled() {
|
||||
debug!(
|
||||
pool = %pool_label,
|
||||
set = %set_label,
|
||||
error = %e,
|
||||
"Scanner set scan stopped after cancellation"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
counter!(
|
||||
"rustfs_scanner_set_failure_total",
|
||||
"pool" => pool_label.clone(),
|
||||
"set" => set_label.clone(),
|
||||
"stage" => "nsscanner_cache".to_string()
|
||||
)
|
||||
.increment(1);
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_SET_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
pool = %pool_label,
|
||||
set = %set_label,
|
||||
error = %e,
|
||||
state = "set_scan_failed",
|
||||
"Scanner set scan failed; continuing cycle"
|
||||
);
|
||||
let mut first_err = first_err_mutex_clone.lock().await;
|
||||
record_set_scan_failure(&mut first_err, e);
|
||||
}
|
||||
});
|
||||
wait_futs.push(scanner_fut);
|
||||
}
|
||||
}
|
||||
|
||||
for join_result in join_all(wait_futs).await {
|
||||
if let Err(err) = join_result {
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_SET_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
state = "set_task_join_failed",
|
||||
error = %err,
|
||||
"Scanner set task join failed"
|
||||
);
|
||||
let mut first_err = first_err_mutex.lock().await;
|
||||
record_set_scan_failure(&mut first_err, scanner_task_join_error("scanner set", err));
|
||||
}
|
||||
}
|
||||
record_set_scan_concurrency_limit(0);
|
||||
record_set_scans_queued(0);
|
||||
record_set_scans_active(0);
|
||||
|
||||
let first_err = first_err_mutex.lock().await.take();
|
||||
let results = results_mutex.lock().await.clone();
|
||||
let completed_all_sets = bucket_plan_complete && scanner_results_form_complete_snapshot(&results, &expected_sources);
|
||||
let result = finalize_nsscanner_result(&results, first_err);
|
||||
let failed_buckets = bucket_failures.hard.lock().await.clone();
|
||||
let partial_buckets = bucket_failures.partial.lock().await.clone();
|
||||
let namespace_not_found_buckets = bucket_failures.namespace_not_found.lock().await.clone();
|
||||
let scan_scope_matches = scanner_results_match_scan_scope(&results, &expected_sources);
|
||||
let bucket_scan_status = scanner_bucket_scan_status(
|
||||
!failed_buckets.is_empty(),
|
||||
scan_scope_matches && !partial_buckets.is_empty(),
|
||||
scan_scope_matches && !namespace_not_found_buckets.is_empty(),
|
||||
);
|
||||
let pending_maintenance_work = pending_maintenance_work_for_cycle(&pending_maintenance_work, &results);
|
||||
let observed_cycle_floor = cache_cycle_floor.load(Ordering::Acquire);
|
||||
let required_cycle_floor = (observed_cycle_floor > want_cycle).then_some(observed_cycle_floor);
|
||||
let budget_elapsed = budget.budget_elapsed();
|
||||
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
|
||||
let dirty_usage_current = dirty_usage_status == DirtyUsageSnapshotStatus::Current;
|
||||
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
|
||||
let all_bucket_names = all_buckets.iter().map(|bucket| bucket.name.clone()).collect::<Vec<_>>();
|
||||
let completed_usage = completed_data_usage_info(
|
||||
&results,
|
||||
&expected_sources,
|
||||
&all_bucket_names,
|
||||
bucket_plan_complete,
|
||||
budget_elapsed,
|
||||
ctx.is_cancelled(),
|
||||
);
|
||||
let structurally_complete_snapshot = result.is_ok() && completed_all_sets && completed_usage.is_some();
|
||||
let cycle_status = classify_nsscanner_cycle(
|
||||
structurally_complete_snapshot,
|
||||
budget_elapsed,
|
||||
ctx.is_cancelled(),
|
||||
bucket_scan_status,
|
||||
dirty_usage_status,
|
||||
activity_status,
|
||||
);
|
||||
if let Some((data_usage_info, _)) = completed_usage {
|
||||
publish_usage_snapshot(&updates, cycle_status, data_usage_info).await?;
|
||||
}
|
||||
let dirty_usage_clear = should_clear_dirty_usage_snapshot(
|
||||
result.is_ok(),
|
||||
structurally_complete_snapshot,
|
||||
budget_elapsed,
|
||||
activity_status == ScannerCycleActivityStatus::Unchanged && dirty_usage_current,
|
||||
&dirty_usage_snapshot.buckets,
|
||||
&failed_buckets,
|
||||
);
|
||||
result?;
|
||||
let remote_dirty_usage_acknowledgements = if cycle_status == ScannerCycleStatus::Complete {
|
||||
crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
|
||||
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)
|
||||
.with_failed_dirty_usage(!failed_buckets.is_empty())
|
||||
.with_pending_maintenance_work(pending_maintenance_work)
|
||||
.with_required_cycle_floor(required_cycle_floor))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// ScannerIODisk implementation for Disk: get_size and the per-disk bucket scan.
|
||||
use super::*;
|
||||
|
||||
///
|
||||
/// Seed [`SizeSummary::tier_stats`] from the cached tier-name list.
|
||||
///
|
||||
/// Preserves the original seeding semantics: with no tiers configured the map
|
||||
/// stays completely empty (STANDARD/RRS are not seeded either); otherwise the
|
||||
/// standard storage classes are seeded alongside every configured tier so
|
||||
/// per-object accounting always finds its tier key.
|
||||
pub(super) fn tier_stats_template(tier_names: &[String]) -> HashMap<String, TierStats> {
|
||||
let mut tier_stats = HashMap::with_capacity(tier_names.len() + 2);
|
||||
for tier_name in tier_names {
|
||||
tier_stats.insert(tier_name.clone(), TierStats::default());
|
||||
}
|
||||
if !tier_stats.is_empty() {
|
||||
tier_stats.insert(storageclass::STANDARD.to_string(), TierStats::default());
|
||||
tier_stats.insert(storageclass::RRS.to_string(), TierStats::default());
|
||||
}
|
||||
tier_stats
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ScannerIODisk for Disk {
|
||||
async fn get_size(&self, mut item: ScannerItem) -> Result<SizeSummary> {
|
||||
let done_object = Metrics::time(Metric::ScanObject);
|
||||
|
||||
if !is_xl_meta_path(&item.path) {
|
||||
return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
|
||||
}
|
||||
|
||||
let data = match self.read_metadata(&item.bucket, &item.object_path()).await {
|
||||
Ok(data) => data,
|
||||
Err(e) if DiskError::is_err_object_not_found(&e) || DiskError::is_err_version_not_found(&e) => {
|
||||
return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(scanner_metadata_transient_error(
|
||||
format!("failed to read metadata: {e}"),
|
||||
&item.bucket,
|
||||
&item.object_path(),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
item.transform_meta_dir();
|
||||
|
||||
let meta = FileMeta::load(&data).map_err(|e| {
|
||||
scanner_metadata_corrupt_error(format!("failed to load metadata: {e}"), &item.bucket, &item.object_path())
|
||||
})?;
|
||||
let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) {
|
||||
Ok(versions) => versions,
|
||||
Err(e) => {
|
||||
return Err(scanner_metadata_corrupt_error(
|
||||
format!("failed to resolve file info versions: {e}"),
|
||||
&item.bucket,
|
||||
&item.object_path(),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Single versioning lookup per object, shared with `apply_actions`
|
||||
// (which used to query it a second time). On failure keep the
|
||||
// historical fallback: default configuration (versioned = false) plus
|
||||
// the warn that `apply_actions` used to emit.
|
||||
let versioning_config = match BucketVersioningSys::get(&item.bucket).await {
|
||||
Ok(versioning_config) => versioning_config,
|
||||
Err(_) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_LIFECYCLE_ACTION,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %item.bucket,
|
||||
state = "versioning_lookup_failed_defaulting",
|
||||
"Scanner lifecycle action falling back to default bucket versioning"
|
||||
);
|
||||
VersioningConfiguration::default()
|
||||
}
|
||||
};
|
||||
let versioned = versioning_config.versioned(&item.object_path());
|
||||
|
||||
let object_infos = fivs
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), item.object_path().as_str(), versioned))
|
||||
.collect::<Vec<ObjectInfo>>();
|
||||
let free_version_infos = fivs
|
||||
.free_versions
|
||||
.iter()
|
||||
.map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), item.object_path().as_str(), versioned))
|
||||
.collect::<Vec<ObjectInfo>>();
|
||||
|
||||
let mut size_summary = SizeSummary::default();
|
||||
|
||||
// Tier names come from the process-wide TTL cache; seeding from them
|
||||
// replaces the per-object clone of every full TierConfig.
|
||||
let tier_names = runtime_tier_names().await;
|
||||
size_summary.tier_stats = tier_stats_template(&tier_names);
|
||||
|
||||
let lock_config = object_lock_config_for_scanner_item(&item).await;
|
||||
|
||||
// Count every version this object contributes to the scan, independent
|
||||
// of any lifecycle configuration, so scan-coverage metrics stay honest
|
||||
// on clusters without ILM rules. Recorded before `apply_actions` moves
|
||||
// `object_infos`.
|
||||
global_metrics().record_scanner_versions_scanned(object_infos.len() as u64);
|
||||
|
||||
item.apply_actions(object_infos, lock_config, versioning_config, &mut size_summary)
|
||||
.await;
|
||||
|
||||
if !free_version_infos.is_empty() {
|
||||
for oi in free_version_infos {
|
||||
enqueue_runtime_free_version(oi).await;
|
||||
}
|
||||
}
|
||||
|
||||
done_object();
|
||||
|
||||
Ok(size_summary)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, budget, updates, cache, set_disks))]
|
||||
async fn nsscanner_disk(
|
||||
self: Arc<Self>,
|
||||
ctx: CancellationToken,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
set_disks: Vec<Arc<Disk>>,
|
||||
cache: DataUsageCache,
|
||||
updates: Option<mpsc::Sender<DataUsageEntry>>,
|
||||
scan_mode: HealScanMode,
|
||||
) -> Result<ScannerDiskScanOutcome> {
|
||||
let done_drive = Metrics::time(Metric::ScanBucketDrive);
|
||||
let drive_start = std::time::Instant::now();
|
||||
let bucket = cache.info.name.clone();
|
||||
let disk_path = self.path().to_string_lossy().to_string();
|
||||
global_metrics().record_scan_bucket_drive_start();
|
||||
let mut failure_guard = BucketDriveFailureGuard::new();
|
||||
let _guard = self.start_scan();
|
||||
|
||||
let mut cache = cache;
|
||||
|
||||
let (lifecycle_config, _) = get_lifecycle_config(&cache.info.name)
|
||||
.await
|
||||
.unwrap_or_else(|_| (BucketLifecycleConfiguration::default(), OffsetDateTime::now_utc()));
|
||||
|
||||
if lifecycle_config.has_active_rules("") {
|
||||
cache.info.lifecycle = Some(Arc::new(lifecycle_config));
|
||||
}
|
||||
|
||||
let (replication_config, _) = get_replication_config(&cache.info.name).await.unwrap_or((
|
||||
ReplicationConfiguration {
|
||||
role: "".to_string(),
|
||||
rules: vec![],
|
||||
},
|
||||
OffsetDateTime::now_utc(),
|
||||
));
|
||||
|
||||
if replication_config.has_active_rules("", true)
|
||||
&& let Ok(targets) = BucketTargetSys::get().list_bucket_targets(&cache.info.name).await
|
||||
{
|
||||
cache.info.replication = Some(Arc::new(ReplicationConfig::new(Some(replication_config), Some(targets))));
|
||||
}
|
||||
|
||||
if let Ok((object_lock_config, _)) = get_object_lock_config(&cache.info.name).await
|
||||
&& object_lock_config_enabled(&object_lock_config)
|
||||
{
|
||||
cache.info.object_lock = Some(Arc::new(object_lock_config));
|
||||
}
|
||||
|
||||
let result = scan_data_folder(
|
||||
ctx.clone(),
|
||||
budget,
|
||||
set_disks,
|
||||
self.clone(),
|
||||
cache,
|
||||
updates,
|
||||
scan_mode,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(mut data_usage_info) => {
|
||||
done_drive();
|
||||
emit_scan_bucket_drive_complete(true, &bucket, &disk_path, drive_start.elapsed());
|
||||
data_usage_info.info.last_update = Some(SystemTime::now());
|
||||
failure_guard.mark_not_failed();
|
||||
Ok(ScannerDiskScanOutcome::Complete(data_usage_info))
|
||||
}
|
||||
Err(ScannerError::PartialCache(mut partial_cache)) => {
|
||||
done_drive();
|
||||
emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
|
||||
partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
|
||||
failure_guard.mark_not_failed();
|
||||
Ok(ScannerDiskScanOutcome::Partial(*partial_cache))
|
||||
}
|
||||
Err(ScannerError::NamespaceNotFoundCache(mut partial_cache)) => {
|
||||
done_drive();
|
||||
emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
|
||||
partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
|
||||
failure_guard.mark_not_failed();
|
||||
Ok(ScannerDiskScanOutcome::NamespaceNotFound(*partial_cache))
|
||||
}
|
||||
Err(e) => {
|
||||
if ctx.is_cancelled() {
|
||||
emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
|
||||
failure_guard.mark_not_failed();
|
||||
} else {
|
||||
done_drive();
|
||||
emit_scan_bucket_drive_complete(false, &bucket, &disk_path, drive_start.elapsed());
|
||||
}
|
||||
Err(StorageError::other(format!("Failed to scan data folder: {e}")))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,751 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use rustfs_data_usage::{ReplicationAllStats, ReplicationStats};
|
||||
|
||||
const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([7; 32]);
|
||||
|
||||
#[test]
|
||||
fn should_publish_completed_snapshot_requires_full_clean_cycle() {
|
||||
assert!(should_publish_completed_snapshot(3, 3, false, false));
|
||||
assert!(!should_publish_completed_snapshot(2, 3, false, false));
|
||||
assert!(!should_publish_completed_snapshot(3, 3, true, false));
|
||||
assert!(!should_publish_completed_snapshot(3, 3, false, true));
|
||||
assert!(
|
||||
should_publish_completed_snapshot(0, 0, false, false),
|
||||
"a completed empty namespace is an authoritative zero snapshot"
|
||||
);
|
||||
}
|
||||
|
||||
fn incomplete_scope_cache(source: DataUsageCacheSource) -> DataUsageCache {
|
||||
DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: DATA_USAGE_ROOT.to_string(),
|
||||
source: Some(source),
|
||||
snapshot_complete: false,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incomplete_scan_scope_requires_every_expected_set_marker() {
|
||||
let first_source = DataUsageCacheSource::new(0, 0);
|
||||
let second_source = DataUsageCacheSource::new(1, 0);
|
||||
let expected_sources = HashSet::from([first_source, second_source]);
|
||||
let first = incomplete_scope_cache(first_source);
|
||||
let second = incomplete_scope_cache(second_source);
|
||||
|
||||
assert!(scanner_results_match_scan_scope(&[first.clone(), second], &expected_sources));
|
||||
assert!(!scanner_results_form_complete_snapshot(
|
||||
&[first.clone(), incomplete_scope_cache(second_source)],
|
||||
&expected_sources
|
||||
));
|
||||
assert!(!scanner_results_match_scan_scope(
|
||||
&[first.clone(), DataUsageCache::default()],
|
||||
&expected_sources
|
||||
));
|
||||
assert!(!scanner_results_match_scan_scope(
|
||||
&[first.clone(), incomplete_scope_cache(first_source)],
|
||||
&expected_sources
|
||||
));
|
||||
|
||||
let mut mismatched_plan = incomplete_scope_cache(second_source);
|
||||
mismatched_plan.info.scan_plan_digest = Some(DataUsageScanPlanDigest([8; 32]));
|
||||
assert!(!scanner_results_match_scan_scope(&[first, mismatched_plan], &expected_sources));
|
||||
}
|
||||
|
||||
fn completed_root_cache(bucket: &str, objects: usize, update_secs: u64, source: DataUsageCacheSource) -> DataUsageCache {
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: DATA_USAGE_ROOT.to_string(),
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(update_secs)),
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.replace(
|
||||
bucket,
|
||||
DATA_USAGE_ROOT,
|
||||
DataUsageEntry {
|
||||
objects,
|
||||
size: objects.saturating_mul(10),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache
|
||||
}
|
||||
|
||||
fn completed_data_usage_info_for_test(
|
||||
results: &[DataUsageCache],
|
||||
all_buckets: &[String],
|
||||
budget_elapsed: bool,
|
||||
cancelled: bool,
|
||||
) -> Option<(DataUsageInfo, SystemTime)> {
|
||||
let expected_sources = results.iter().filter_map(|result| result.info.source).collect::<HashSet<_>>();
|
||||
completed_data_usage_info(results, &expected_sources, all_buckets, true, budget_elapsed, cancelled)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_publishes_tier_stats_across_sets() {
|
||||
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string()];
|
||||
let warm = |total_size, num_versions, num_objects| {
|
||||
HashMap::from([(
|
||||
"WARM".to_string(),
|
||||
TierStats {
|
||||
total_size,
|
||||
num_versions,
|
||||
num_objects,
|
||||
},
|
||||
)])
|
||||
};
|
||||
|
||||
let mut first_set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
|
||||
let mut tiered = DataUsageEntry::default();
|
||||
tiered.add_tier_sizes(&warm(100, 2, 1));
|
||||
first_set.replace("bucket-b", DATA_USAGE_ROOT, tiered);
|
||||
|
||||
let mut second_set = completed_root_cache("bucket-b", 2, 20, DataUsageCacheSource::new(1, 0));
|
||||
let mut tiered = DataUsageEntry::default();
|
||||
tiered.add_tier_sizes(&warm(50, 1, 1));
|
||||
second_set.replace("bucket-a", DATA_USAGE_ROOT, tiered);
|
||||
|
||||
let (data_usage_info, _) = completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false)
|
||||
.expect("completed sets should publish a snapshot");
|
||||
|
||||
assert_eq!(
|
||||
data_usage_info
|
||||
.tier_stats
|
||||
.expect("tier usage should reach the snapshot")
|
||||
.tiers["WARM"],
|
||||
TierStats {
|
||||
total_size: 150,
|
||||
num_versions: 3,
|
||||
num_objects: 2,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_omits_tier_stats_without_tiered_objects() {
|
||||
let all_buckets = vec!["bucket-a".to_string()];
|
||||
let set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
|
||||
|
||||
let (data_usage_info, _) =
|
||||
completed_data_usage_info_for_test(&[set], &all_buckets, false, false).expect("completed set should publish a snapshot");
|
||||
|
||||
assert!(data_usage_info.tier_stats.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_requires_every_set_before_publish() {
|
||||
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string(), "bucket-empty".to_string()];
|
||||
let mut first_set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
|
||||
first_set.replace("bucket-b", DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||
first_set.replace("bucket-empty", DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||
let mut second_set = completed_root_cache("bucket-b", 2, 20, DataUsageCacheSource::new(1, 0));
|
||||
second_set.replace("bucket-a", DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||
second_set.replace("bucket-empty", DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||
|
||||
assert!(
|
||||
completed_data_usage_info_for_test(&[first_set.clone(), DataUsageCache::default()], &all_buckets, false, false).is_none()
|
||||
);
|
||||
assert!(completed_data_usage_info_for_test(&[first_set.clone(), second_set.clone()], &all_buckets, true, false).is_none());
|
||||
assert!(completed_data_usage_info_for_test(&[first_set.clone(), second_set.clone()], &all_buckets, false, true).is_none());
|
||||
|
||||
let (data_usage_info, last_update) = completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false)
|
||||
.expect("all completed sets should produce a publishable data usage snapshot");
|
||||
assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20));
|
||||
assert_eq!(data_usage_info.scanner_cycle, Some(0));
|
||||
assert_eq!(data_usage_info.objects_total_count, 3);
|
||||
assert_eq!(data_usage_info.buckets_usage.len(), 3);
|
||||
assert!(data_usage_info.usage_snapshot_complete);
|
||||
assert_eq!(
|
||||
data_usage_info
|
||||
.buckets_usage
|
||||
.get("bucket-empty")
|
||||
.map(|usage| (usage.objects_count, usage.size)),
|
||||
Some((0, 0))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_publishes_confirmed_empty_namespace() {
|
||||
let mut completed_set = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: DATA_USAGE_ROOT.to_string(),
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
|
||||
source: Some(DataUsageCacheSource::new(0, 0)),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
completed_set.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
|
||||
|
||||
let (data_usage_info, _) = completed_data_usage_info_for_test(&[completed_set], &[], false, false)
|
||||
.expect("a completed empty namespace should produce an authoritative snapshot");
|
||||
|
||||
assert!(data_usage_info.is_complete_bucket_usage_snapshot());
|
||||
assert_eq!(data_usage_info.buckets_count, 0);
|
||||
assert!(data_usage_info.buckets_usage.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_usage_candidate_with_changed_generation_is_superseded() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let completed_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
|
||||
let completed_usage = completed_data_usage_info_for_test(&[completed_set], &all_buckets, false, false);
|
||||
|
||||
assert!(completed_usage.is_some());
|
||||
assert_eq!(
|
||||
classify_nsscanner_cycle(
|
||||
completed_usage.is_some(),
|
||||
false,
|
||||
false,
|
||||
ScannerBucketScanStatus::Complete,
|
||||
DirtyUsageSnapshotStatus::Changed,
|
||||
ScannerCycleActivityStatus::Unchanged,
|
||||
),
|
||||
ScannerCycleStatus::Superseded
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_adds_same_bucket_across_unique_sets() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
|
||||
let mut second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0));
|
||||
let second_entry = second_set.find("bucket").cloned().expect("second set bucket entry");
|
||||
second_set.replace(
|
||||
"bucket",
|
||||
DATA_USAGE_ROOT,
|
||||
DataUsageEntry {
|
||||
versions: 4,
|
||||
delete_markers: 1,
|
||||
..second_entry
|
||||
},
|
||||
);
|
||||
|
||||
let (data_usage_info, last_update) = completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false)
|
||||
.expect("unique completed set snapshots should be aggregated");
|
||||
let bucket = data_usage_info.buckets_usage.get("bucket").expect("merged bucket usage");
|
||||
|
||||
assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20));
|
||||
assert_eq!(data_usage_info.objects_total_count, 5);
|
||||
assert_eq!(data_usage_info.objects_total_size, 50);
|
||||
assert_eq!(bucket.objects_count, 5);
|
||||
assert_eq!(bucket.size, 50);
|
||||
assert_eq!(bucket.versions_count, 4);
|
||||
assert_eq!(bucket.delete_markers_count, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_flattens_nested_bucket_entries() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let mut first_set = completed_root_cache("bucket", 1, 10, DataUsageCacheSource::new(0, 0));
|
||||
let mut nested = DataUsageEntry {
|
||||
objects: 2,
|
||||
versions: 3,
|
||||
size: 2048,
|
||||
replication_stats: Some(ReplicationAllStats {
|
||||
targets: HashMap::from([(
|
||||
"arn:target".to_string(),
|
||||
ReplicationStats {
|
||||
replicated_size: 2048,
|
||||
replicated_count: 2,
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
replica_size: 2048,
|
||||
replica_count: 2,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
nested.obj_sizes.add(2048);
|
||||
nested.obj_versions.add(3);
|
||||
first_set.replace("bucket/prefix", "bucket", nested);
|
||||
let second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0));
|
||||
|
||||
let (data_usage_info, _) = completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false)
|
||||
.expect("nested bucket entries should be flattened before aggregation");
|
||||
let bucket = data_usage_info.buckets_usage.get("bucket").expect("merged bucket usage");
|
||||
|
||||
assert_eq!(data_usage_info.objects_total_count, 6);
|
||||
assert_eq!(data_usage_info.objects_total_size, 2088);
|
||||
assert_eq!(bucket.objects_count, 6);
|
||||
assert_eq!(bucket.versions_count, 3);
|
||||
assert_eq!(bucket.object_size_histogram["BETWEEN_1024_B_AND_64_KB"], 1);
|
||||
assert_eq!(bucket.object_versions_histogram["BETWEEN_2_AND_10"], 1);
|
||||
assert_eq!(bucket.replica_size, 2048);
|
||||
assert_eq!(bucket.replica_count, 2);
|
||||
assert_eq!(bucket.replication_info["arn:target"].replicated_size, 2048);
|
||||
assert_eq!(bucket.replication_info["arn:target"].replicated_count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_rejects_cyclic_bucket_entries() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let mut cache = completed_root_cache("bucket", 1, 10, DataUsageCacheSource::new(0, 0));
|
||||
cache.replace(
|
||||
"bucket/prefix",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 10,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache
|
||||
.cache
|
||||
.get_mut(&crate::hash_path("bucket/prefix").key())
|
||||
.expect("nested entry")
|
||||
.add_child(&crate::hash_path("bucket"));
|
||||
|
||||
assert!(completed_data_usage_info_for_test(&[cache], &all_buckets, false, false).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_rejects_duplicate_or_incomplete_set_sources() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let first_set = completed_root_cache("bucket", 2, 10, source);
|
||||
let duplicate_set = completed_root_cache("bucket", 3, 20, source);
|
||||
|
||||
assert!(completed_data_usage_info_for_test(&[first_set.clone(), duplicate_set], &all_buckets, false, false).is_none());
|
||||
|
||||
let mut incomplete_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0));
|
||||
incomplete_set.info.snapshot_complete = false;
|
||||
assert!(completed_data_usage_info_for_test(&[first_set, incomplete_set], &all_buckets, false, false).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_requires_exact_topology_sources() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
|
||||
let unexpected_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(99, 99));
|
||||
let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0), DataUsageCacheSource::new(1, 0)]);
|
||||
|
||||
assert!(
|
||||
completed_data_usage_info(&[first_set, unexpected_set], &expected_sources, &all_buckets, true, false, false).is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_rejects_incomplete_bucket_plan() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
|
||||
let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0)]);
|
||||
|
||||
assert!(completed_data_usage_info(&[set], &expected_sources, &all_buckets, false, false, false).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_rejects_missing_bucket_from_any_set() {
|
||||
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string()];
|
||||
let mut complete_set = completed_root_cache("bucket-a", 2, 10, DataUsageCacheSource::new(0, 0));
|
||||
complete_set.replace("bucket-b", DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||
let missing_bucket_set = completed_root_cache("bucket-a", 3, 20, DataUsageCacheSource::new(1, 0));
|
||||
|
||||
assert!(completed_data_usage_info_for_test(&[complete_set, missing_bucket_set], &all_buckets, false, false).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_rejects_mixed_scan_plans() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
|
||||
let mut second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0));
|
||||
second_set.info.scan_plan_digest = Some(DataUsageScanPlanDigest([8; 32]));
|
||||
|
||||
assert!(completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_rejects_mixed_scanner_cycles() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let mut first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
|
||||
let mut second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0));
|
||||
first_set.info.next_cycle = 12;
|
||||
second_set.info.next_cycle = 13;
|
||||
|
||||
assert!(completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_rejects_mixed_leader_epochs() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let mut first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
|
||||
let mut second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0));
|
||||
first_set.info.leader_epoch = 11;
|
||||
second_set.info.leader_epoch = 12;
|
||||
|
||||
assert!(completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_rejects_counter_overflow() {
|
||||
let all_buckets = vec!["bucket".to_string()];
|
||||
let first_set = completed_root_cache("bucket", usize::MAX, 10, DataUsageCacheSource::new(0, 0));
|
||||
let second_set = completed_root_cache("bucket", 1, 20, DataUsageCacheSource::new(1, 0));
|
||||
|
||||
assert!(completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_cache_snapshot_requires_matching_complete_source_and_cycle() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let mut cache = completed_root_cache("bucket", 1, 10, source);
|
||||
cache.info.next_cycle = 10;
|
||||
|
||||
assert!(cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 0, TEST_PLAN_DIGEST));
|
||||
assert!(!cache_snapshot_is_current(&cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST));
|
||||
assert!(!cache_snapshot_is_current(
|
||||
&cache,
|
||||
DATA_USAGE_ROOT,
|
||||
DataUsageCacheSource::new(2, 1),
|
||||
10,
|
||||
0,
|
||||
TEST_PLAN_DIGEST
|
||||
));
|
||||
assert!(!cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 11, 0, TEST_PLAN_DIGEST));
|
||||
assert!(!cache_snapshot_is_current(
|
||||
&cache,
|
||||
DATA_USAGE_ROOT,
|
||||
source,
|
||||
10,
|
||||
0,
|
||||
DataUsageScanPlanDigest([8; 32])
|
||||
));
|
||||
cache.info.leader_epoch = 2;
|
||||
assert!(!cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 1, TEST_PLAN_DIGEST));
|
||||
assert!(cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 2, TEST_PLAN_DIGEST));
|
||||
cache.info.next_cycle = 11;
|
||||
assert!(!cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 2, TEST_PLAN_DIGEST));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_cache_snapshot_rejects_persisted_windows_key_mismatch() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 10,
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.cache.insert(
|
||||
"bucket".to_string(),
|
||||
DataUsageEntry {
|
||||
children: HashSet::from(["bucket/prefix".to_string()]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache.cache.insert(
|
||||
"bucket\\prefix".to_string(),
|
||||
DataUsageEntry {
|
||||
objects: 3,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
assert!(!cache_snapshot_is_current(&cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST));
|
||||
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
|
||||
DataUsageCacheScanState::Prepared {
|
||||
outcome: DataUsageCachePrepareOutcome::Reset,
|
||||
invalid_current: Some(_),
|
||||
} => {}
|
||||
_ => panic!("an invalid current cache must enter the rebuild path"),
|
||||
}
|
||||
assert!(cache.cache.is_empty());
|
||||
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_cache_snapshot_rejects_structurally_valid_legacy_key_format() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 10,
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.cache.insert(
|
||||
"bucket".to_string(),
|
||||
DataUsageEntry {
|
||||
children: HashSet::from(["bucket\\prefix".to_string()]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache.cache.insert(
|
||||
"bucket\\prefix".to_string(),
|
||||
DataUsageEntry {
|
||||
objects: 3,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(3));
|
||||
|
||||
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
|
||||
DataUsageCacheScanState::Prepared {
|
||||
outcome: DataUsageCachePrepareOutcome::Reset,
|
||||
invalid_current: None,
|
||||
} => {}
|
||||
_ => panic!("a legacy key format must enter the rebuild path"),
|
||||
}
|
||||
assert!(cache.cache.is_empty());
|
||||
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_cache_snapshot_rejects_current_bucket_cache_with_detached_entry() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 10,
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.cache.insert(
|
||||
"bucket".to_string(),
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache.cache.insert(
|
||||
"bucket/detached".to_string(),
|
||||
DataUsageEntry {
|
||||
objects: 2,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(1));
|
||||
|
||||
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
|
||||
DataUsageCacheScanState::Prepared {
|
||||
outcome: DataUsageCachePrepareOutcome::Reset,
|
||||
invalid_current: Some(_),
|
||||
} => {}
|
||||
_ => panic!("a detached complete bucket cache must enter the rebuild path"),
|
||||
}
|
||||
assert!(cache.cache.is_empty());
|
||||
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_worker_selection_keeps_coordinator_fallback_disks() {
|
||||
let server_epoch = uuid::Uuid::new_v4();
|
||||
let workers = namespace_scanner_workers(vec!["local", "legacy-remote"], vec![("v4", server_epoch)]);
|
||||
|
||||
assert_eq!(
|
||||
workers,
|
||||
vec![
|
||||
("local", NamespaceScannerWorkerMode::Coordinator),
|
||||
("legacy-remote", NamespaceScannerWorkerMode::Coordinator),
|
||||
("v4", NamespaceScannerWorkerMode::RemoteV4(server_epoch)),
|
||||
]
|
||||
);
|
||||
assert!(namespace_scanner_workers::<()>(Vec::new(), Vec::new()).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_scanner_capability_probes_are_grouped_by_peer() {
|
||||
let mut groups = group_remote_disks_by_peer(vec![("node-a", 0), ("node-b", 0), ("node-a", 1), ("node-b", 1)], |disk| {
|
||||
disk.0.to_string()
|
||||
});
|
||||
groups.sort_by_key(|group| group[0].0);
|
||||
|
||||
assert_eq!(groups.len(), 2);
|
||||
assert_eq!(groups[0], vec![("node-a", 0), ("node-a", 1)]);
|
||||
assert_eq!(groups[1], vec![("node-b", 0), ("node-b", 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_bucket_plan_digest_is_order_independent_and_membership_sensitive() {
|
||||
let activity_digest = [4; 32];
|
||||
let buckets = vec![
|
||||
BucketInfo {
|
||||
name: "photos".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
BucketInfo {
|
||||
name: "videos".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let reversed = vec![buckets[1].clone(), buckets[0].clone()];
|
||||
let changed = vec![
|
||||
buckets[0].clone(),
|
||||
BucketInfo {
|
||||
name: "archives".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let mut regenerated = buckets.clone();
|
||||
regenerated[0].created = Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1));
|
||||
|
||||
assert_eq!(
|
||||
scanner_bucket_plan_digest(&buckets, activity_digest),
|
||||
scanner_bucket_plan_digest(&reversed, activity_digest)
|
||||
);
|
||||
assert_ne!(
|
||||
scanner_bucket_plan_digest(&buckets, activity_digest),
|
||||
scanner_bucket_plan_digest(&changed, activity_digest)
|
||||
);
|
||||
assert_ne!(
|
||||
scanner_bucket_plan_digest(&buckets, activity_digest),
|
||||
scanner_bucket_plan_digest(®enerated, activity_digest)
|
||||
);
|
||||
assert_ne!(
|
||||
scanner_bucket_plan_digest(&buckets, activity_digest),
|
||||
scanner_bucket_plan_digest(&buckets, [5; 32])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dirty_bucket_cache_digest_changes_with_generation() {
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let plan = DataUsageScanPlanDigest([9; 32]);
|
||||
let first = scanner_bucket_cache_digest(plan, Some(7));
|
||||
let second = scanner_bucket_cache_digest(plan, Some(8));
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "photos".to_string(),
|
||||
next_cycle: 11,
|
||||
last_update: Some(SystemTime::now()),
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(first),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.replace("photos", "", DataUsageEntry::default());
|
||||
|
||||
assert_eq!(scanner_bucket_cache_digest(plan, None), plan);
|
||||
assert!(cache_snapshot_is_current(&cache, "photos", source, 11, 0, first));
|
||||
assert!(!cache_snapshot_is_current(&cache, "photos", source, 11, 0, second));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_cache_lock_resource_is_scoped_to_cache_source() {
|
||||
let cache_name = "photos/.usage-cache.bin";
|
||||
let first_source = DataUsageCacheSource::new(0, 1);
|
||||
let same_source = DataUsageCacheSource::new(0, 1);
|
||||
let other_source = DataUsageCacheSource::new(1, 0);
|
||||
|
||||
let first = scanner_cache_lock_resource(cache_name, first_source);
|
||||
assert_eq!(first, scanner_cache_lock_resource(cache_name, same_source));
|
||||
assert_ne!(first, scanner_cache_lock_resource(cache_name, other_source));
|
||||
assert!(first.ends_with(".scanner-cycle.lock.pool-0.set-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_budget_serializes_set_and_disk_work() {
|
||||
assert_eq!(scanner_budgeted_concurrency_limit(8, true), 1);
|
||||
assert_eq!(scanner_budgeted_concurrency_limit(8, false), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn requeued_bucket_work_is_only_completed_after_retry() {
|
||||
let remaining = Arc::new(AtomicUsize::new(1));
|
||||
let complete = CancellationToken::new();
|
||||
let mut first = BucketWorkGuard::new(remaining.clone(), complete.clone());
|
||||
first.mark_requeued();
|
||||
drop(first);
|
||||
assert_eq!(remaining.load(Ordering::Acquire), 1);
|
||||
assert!(!complete.is_cancelled());
|
||||
|
||||
drop(BucketWorkGuard::new(remaining.clone(), complete.clone()));
|
||||
assert_eq!(remaining.load(Ordering::Acquire), 0);
|
||||
assert!(complete.is_cancelled());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exhausted_workers_mark_all_queued_bucket_work_failed() {
|
||||
let (tx, rx) = mpsc::channel(2);
|
||||
tx.send(BucketInfo {
|
||||
name: "photos".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("queue photos");
|
||||
tx.send(BucketInfo {
|
||||
name: "videos".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("queue videos");
|
||||
drop(tx);
|
||||
|
||||
let receiver = Mutex::new(rx);
|
||||
let remaining = Arc::new(AtomicUsize::new(2));
|
||||
let complete = CancellationToken::new();
|
||||
let failed = Arc::new(Mutex::new(HashSet::new()));
|
||||
|
||||
let failed_count = mark_unprocessed_bucket_work_failed(&receiver, &remaining, &complete, &failed).await;
|
||||
|
||||
assert_eq!(failed_count, 2);
|
||||
assert_eq!(remaining.load(Ordering::Acquire), 0);
|
||||
assert!(complete.is_cancelled());
|
||||
assert_eq!(*failed.lock().await, HashSet::from(["photos".to_string(), "videos".to_string()]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn requeued_bucket_remains_pending_for_another_worker() {
|
||||
let (tx, mut rx) = mpsc::channel(1);
|
||||
let remaining = Arc::new(AtomicUsize::new(1));
|
||||
let complete = CancellationToken::new();
|
||||
let mut guard = BucketWorkGuard::new(remaining.clone(), complete.clone());
|
||||
let bucket = BucketInfo {
|
||||
name: "photos".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(requeue_bucket_work(&tx, &bucket, &mut guard).await);
|
||||
drop(guard);
|
||||
|
||||
assert_eq!(remaining.load(Ordering::Acquire), 1);
|
||||
assert!(!complete.is_cancelled());
|
||||
assert_eq!(rx.recv().await.expect("requeued bucket").name, "photos");
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user